##// END OF EJS Templates
schain
RSS
merge con digital_rf y pull
José Chávez -
r1067:7abdb8296c68 merge
parent child
Show More
Side by Side Unified
Context file:
r1067:7abdb8296c68 Loading diff...:
Show file before | Show file after | Hide comments
@@ -9,56 +9,53 import datetime
9 import traceback
9 import traceback
10 import math
10 import math
11 import time
11 import time
12 from multiprocessing import Process, Queue, cpu_count
12 from multiprocessing import Process, cpu_count
13
14 import schainpy
15 import schainpy.admin
16 from schainpy.utils.log import logToFile
17
13
18 from xml.etree.ElementTree import ElementTree, Element, SubElement, tostring
14 from xml.etree.ElementTree import ElementTree, Element, SubElement, tostring
19 from xml.dom import minidom
15 from xml.dom import minidom
20
16
17 import schainpy
18 import schainpy.admin
21 from schainpy.model import *
19 from schainpy.model import *
22 from time import sleep
20 from schainpy.utils import log
23
21
24
22 DTYPES = {
25
23 'Voltage': '.r',
26 def prettify(elem):
24 'Spectra': '.pdata'
27 """Return a pretty-printed XML string for the Element.
25 }
28 """
26
29 rough_string = tostring(elem, 'utf-8')
27 def MPProject(project, n=cpu_count()):
30 reparsed = minidom.parseString(rough_string)
28 '''
31 return reparsed.toprettyxml(indent=" ")
29 Project wrapper to run schain in n processes
32
30 '''
33 def multiSchain(child, nProcess=cpu_count(), startDate=None, endDate=None, by_day=False):
31
34 skip = 0
32 rconf = project.getReadUnitObj()
35 cursor = 0
33 op = rconf.getOperationObj('run')
36 nFiles = None
34 dt1 = op.getParameterValue('startDate')
37 processes = []
35 dt2 = op.getParameterValue('endDate')
38 dt1 = datetime.datetime.strptime(startDate, '%Y/%m/%d')
39 dt2 = datetime.datetime.strptime(endDate, '%Y/%m/%d')
40 days = (dt2 - dt1).days
36 days = (dt2 - dt1).days
41
37
42 for day in range(days+1):
38 for day in range(days+1):
43 skip = 0
39 skip = 0
44 cursor = 0
40 cursor = 0
45 q = Queue()
46 processes = []
41 processes = []
47 dt = (dt1 + datetime.timedelta(day)).strftime('%Y/%m/%d')
42 dt = dt1 + datetime.timedelta(day)
48 firstProcess = Process(target=child, args=(cursor, skip, q, dt))
43 dt_str = dt.strftime('%Y/%m/%d')
49 firstProcess.start()
44 reader = JRODataReader()
50 if by_day:
45 paths, files = reader.searchFilesOffLine(path=rconf.path,
51 continue
46 startDate=dt,
52 nFiles = q.get()
47 endDate=dt,
53 if nFiles==0:
48 ext=DTYPES[rconf.datatype])
49 nFiles = len(files)
50 if nFiles == 0:
54 continue
51 continue
55 firstProcess.terminate()
52 skip = int(math.ceil(nFiles/n))
56 skip = int(math.ceil(nFiles/nProcess))
53 while nFiles > cursor*skip:
57 while True:
54 rconf.update(startDate=dt_str, endDate=dt_str, cursor=cursor,
58 processes.append(Process(target=child, args=(cursor, skip, q, dt)))
55 skip=skip)
59 processes[cursor].start()
56 p = project.clone()
60 if nFiles < cursor*skip:
57 p.start()
61 break
58 processes.append(p)
62 cursor += 1
59 cursor += 1
63
60
64 def beforeExit(exctype, value, trace):
61 def beforeExit(exctype, value, trace):
@@ -75,7 +72,6 def multiSchain(child, nProcess=cpu_count(), startDate=None, endDate=None, by_da
75
72
76 time.sleep(3)
73 time.sleep(3)
77
74
78
79 class ParameterConf():
75 class ParameterConf():
80
76
81 id = None
77 id = None
@@ -112,7 +108,7 class ParameterConf():
112 return self.__formated_value
108 return self.__formated_value
113
109
114 if value == '':
110 if value == '':
115 raise ValueError, "%s: This parameter value is empty" %self.name
111 raise ValueError, '%s: This parameter value is empty' %self.name
116
112
117 if format == 'list':
113 if format == 'list':
118 strList = value.split(',')
114 strList = value.split(',')
@@ -122,10 +118,10 class ParameterConf():
122 return self.__formated_value
118 return self.__formated_value
123
119
124 if format == 'intlist':
120 if format == 'intlist':
125 """
121 '''
126 Example:
122 Example:
127 value = (0,1,2)
123 value = (0,1,2)
128 """
124 '''
129
125
130 new_value = ast.literal_eval(value)
126 new_value = ast.literal_eval(value)
131
127
@@ -137,10 +133,10 class ParameterConf():
137 return self.__formated_value
133 return self.__formated_value
138
134
139 if format == 'floatlist':
135 if format == 'floatlist':
140 """
136 '''
141 Example:
137 Example:
142 value = (0.5, 1.4, 2.7)
138 value = (0.5, 1.4, 2.7)
143 """
139 '''
144
140
145 new_value = ast.literal_eval(value)
141 new_value = ast.literal_eval(value)
146
142
@@ -170,38 +166,38 class ParameterConf():
170 return self.__formated_value
166 return self.__formated_value
171
167
172 if format == 'pairslist':
168 if format == 'pairslist':
173 """
169 '''
174 Example:
170 Example:
175 value = (0,1),(1,2)
171 value = (0,1),(1,2)
176 """
172 '''
177
173
178 new_value = ast.literal_eval(value)
174 new_value = ast.literal_eval(value)
179
175
180 if type(new_value) not in (tuple, list):
176 if type(new_value) not in (tuple, list):
181 raise ValueError, "%s has to be a tuple or list of pairs" %value
177 raise ValueError, '%s has to be a tuple or list of pairs' %value
182
178
183 if type(new_value[0]) not in (tuple, list):
179 if type(new_value[0]) not in (tuple, list):
184 if len(new_value) != 2:
180 if len(new_value) != 2:
185 raise ValueError, "%s has to be a tuple or list of pairs" %value
181 raise ValueError, '%s has to be a tuple or list of pairs' %value
186 new_value = [new_value]
182 new_value = [new_value]
187
183
188 for thisPair in new_value:
184 for thisPair in new_value:
189 if len(thisPair) != 2:
185 if len(thisPair) != 2:
190 raise ValueError, "%s has to be a tuple or list of pairs" %value
186 raise ValueError, '%s has to be a tuple or list of pairs' %value
191
187
192 self.__formated_value = new_value
188 self.__formated_value = new_value
193
189
194 return self.__formated_value
190 return self.__formated_value
195
191
196 if format == 'multilist':
192 if format == 'multilist':
197 """
193 '''
198 Example:
194 Example:
199 value = (0,1,2),(3,4,5)
195 value = (0,1,2),(3,4,5)
200 """
196 '''
201 multiList = ast.literal_eval(value)
197 multiList = ast.literal_eval(value)
202
198
203 if type(multiList[0]) == int:
199 if type(multiList[0]) == int:
204 multiList = ast.literal_eval("(" + value + ")")
200 multiList = ast.literal_eval('(' + value + ')')
205
201
206 self.__formated_value = multiList
202 self.__formated_value = multiList
207
203
@@ -263,9 +259,9 class ParameterConf():
263
259
264 def printattr(self):
260 def printattr(self):
265
261
266 print "Parameter[%s]: name = %s, value = %s, format = %s" %(self.id, self.name, self.value, self.format)
262 print 'Parameter[%s]: name = %s, value = %s, format = %s' %(self.id, self.name, self.value, self.format)
267
263
268 class OperationConf():
264 class OperationConf():
269
265
270 id = None
266 id = None
271 name = None
267 name = None
@@ -371,7 +367,9 class OperationConf():
371 self.parmConfObjList = []
367 self.parmConfObjList = []
372
368
373 def addParameter(self, name, value, format='str'):
369 def addParameter(self, name, value, format='str'):
374
370
371 if value is None:
372 return None
375 id = self.__getNewId()
373 id = self.__getNewId()
376
374
377 parmConfObj = ParameterConf()
375 parmConfObj = ParameterConf()
@@ -431,7 +429,7 class OperationConf():
431
429
432 def printattr(self):
430 def printattr(self):
433
431
434 print "%s[%s]: name = %s, type = %s, priority = %s" %(self.ELEMENTNAME,
432 print '%s[%s]: name = %s, type = %s, priority = %s' %(self.ELEMENTNAME,
435 self.id,
433 self.id,
436 self.name,
434 self.name,
437 self.type,
435 self.type,
@@ -444,12 +442,11 class OperationConf():
444
442
445
443
446 if self.type == 'self':
444 if self.type == 'self':
447 raise ValueError, "This operation type cannot be created"
445 raise ValueError, 'This operation type cannot be created'
448
446
449 if self.type == 'plotter':
447 if self.type == 'plotter':
450 #Plotter(plotter_name)
451 if not plotter_queue:
448 if not plotter_queue:
452 raise ValueError, "plotter_queue is not defined. Use:\nmyProject = Project()\nmyProject.setPlotterQueue(plotter_queue)"
449 raise ValueError, 'plotter_queue is not defined. Use:\nmyProject = Project()\nmyProject.setPlotterQueue(plotter_queue)'
453
450
454 opObj = Plotter(self.name, plotter_queue)
451 opObj = Plotter(self.name, plotter_queue)
455
452
@@ -564,7 +561,7 class ProcUnitConf():
564
561
565 #Compatible with old signal chain version
562 #Compatible with old signal chain version
566 if datatype==None and name==None:
563 if datatype==None and name==None:
567 raise ValueError, "datatype or name should be defined"
564 raise ValueError, 'datatype or name should be defined'
568
565
569 if name==None:
566 if name==None:
570 if 'Proc' in datatype:
567 if 'Proc' in datatype:
@@ -595,7 +592,7 class ProcUnitConf():
595
592
596 def addParameter(self, **kwargs):
593 def addParameter(self, **kwargs):
597 '''
594 '''
598 Add parameters to "run" operation
595 Add parameters to 'run' operation
599 '''
596 '''
600 opObj = self.opConfObjList[0]
597 opObj = self.opConfObjList[0]
601
598
@@ -633,11 +630,11 class ProcUnitConf():
633 self.datatype = upElement.get('datatype')
630 self.datatype = upElement.get('datatype')
634 self.inputId = upElement.get('inputId')
631 self.inputId = upElement.get('inputId')
635
632
636 if self.ELEMENTNAME == "ReadUnit":
633 if self.ELEMENTNAME == 'ReadUnit':
637 self.datatype = self.datatype.replace("Reader", "")
634 self.datatype = self.datatype.replace('Reader', '')
638
635
639 if self.ELEMENTNAME == "ProcUnit":
636 if self.ELEMENTNAME == 'ProcUnit':
640 self.datatype = self.datatype.replace("Proc", "")
637 self.datatype = self.datatype.replace('Proc', '')
641
638
642 if self.inputId == 'None':
639 if self.inputId == 'None':
643 self.inputId = '0'
640 self.inputId = '0'
@@ -653,7 +650,7 class ProcUnitConf():
653
650
654 def printattr(self):
651 def printattr(self):
655
652
656 print "%s[%s]: name = %s, datatype = %s, inputId = %s" %(self.ELEMENTNAME,
653 print '%s[%s]: name = %s, datatype = %s, inputId = %s' %(self.ELEMENTNAME,
657 self.id,
654 self.id,
658 self.name,
655 self.name,
659 self.datatype,
656 self.datatype,
@@ -707,17 +704,9 class ProcUnitConf():
707
704
708 kwargs[parmConfObj.name] = parmConfObj.getValue()
705 kwargs[parmConfObj.name] = parmConfObj.getValue()
709
706
710 #ini = time.time()
711
712 #print "\tRunning the '%s' operation with %s" %(opConfObj.name, opConfObj.id)
713 sts = self.procUnitObj.call(opType = opConfObj.type,
707 sts = self.procUnitObj.call(opType = opConfObj.type,
714 opName = opConfObj.name,
708 opName = opConfObj.name,
715 opId = opConfObj.id)
709 opId = opConfObj.id)
716
717 # total_time = time.time() - ini
718 #
719 # if total_time > 0.002:
720 # print "%s::%s took %f seconds" %(self.name, opConfObj.name, total_time)
721
710
722 is_ok = is_ok or sts
711 is_ok = is_ok or sts
723
712
@@ -762,11 +751,12 class ReadUnitConf(ProcUnitConf):
762
751
763 return self.ELEMENTNAME
752 return self.ELEMENTNAME
764
753
765 def setup(self, id, name, datatype, path='', startDate="", endDate="", startTime="",
754 def setup(self, id, name, datatype, path='', startDate='', endDate='',
766 endTime="", parentId=None, queue=None, server=None, **kwargs):
755 startTime='', endTime='', parentId=None, server=None, **kwargs):
756
767 #Compatible with old signal chain version
757 #Compatible with old signal chain version
768 if datatype==None and name==None:
758 if datatype==None and name==None:
769 raise ValueError, "datatype or name should be defined"
759 raise ValueError, 'datatype or name should be defined'
770
760
771 if name==None:
761 if name==None:
772 if 'Reader' in datatype:
762 if 'Reader' in datatype:
@@ -785,39 +775,28 class ReadUnitConf(ProcUnitConf):
785 self.endDate = endDate
775 self.endDate = endDate
786 self.startTime = startTime
776 self.startTime = startTime
787 self.endTime = endTime
777 self.endTime = endTime
788
789 self.inputId = '0'
778 self.inputId = '0'
790 self.parentId = parentId
779 self.parentId = parentId
791 self.queue = queue
792 self.server = server
780 self.server = server
793 self.addRunOperation(**kwargs)
781 self.addRunOperation(**kwargs)
794
782
795 def update(self, datatype, path, startDate, endDate, startTime, endTime, parentId=None, name=None, **kwargs):
783 def update(self, **kwargs):
796
784
797 #Compatible with old signal chain version
785 if 'datatype' in kwargs:
798 if datatype==None and name==None:
786 datatype = kwargs.pop('datatype')
799 raise ValueError, "datatype or name should be defined"
800
801 if name==None:
802 if 'Reader' in datatype:
787 if 'Reader' in datatype:
803 name = datatype
788 self.name = datatype
804 else:
789 else:
805 name = '%sReader' %(datatype)
790 self.name = '%sReader' %(datatype)
806
791 self.datatype = self.name.replace('Reader', '')
807 if datatype==None:
808 datatype = name.replace('Reader','')
809
810 self.datatype = datatype
811 self.name = name
812 self.path = path
813 self.startDate = startDate
814 self.endDate = endDate
815 self.startTime = startTime
816 self.endTime = endTime
817
792
793 attrs = ('path', 'startDate', 'endDate', 'startTime', 'endTime', 'parentId')
794
795 for attr in attrs:
796 if attr in kwargs:
797 setattr(self, attr, kwargs.pop(attr))
798
818 self.inputId = '0'
799 self.inputId = '0'
819 self.parentId = parentId
820
821 self.updateRunOperation(**kwargs)
800 self.updateRunOperation(**kwargs)
822
801
823 def removeOperations(self):
802 def removeOperations(self):
@@ -832,13 +811,13 class ReadUnitConf(ProcUnitConf):
832 opObj = self.addOperation(name = 'run', optype = 'self')
811 opObj = self.addOperation(name = 'run', optype = 'self')
833
812
834 if self.server is None:
813 if self.server is None:
835 opObj.addParameter(name='datatype' , value=self.datatype, format='str')
814 opObj.addParameter(name='datatype', value=self.datatype, format='str')
836 opObj.addParameter(name='path' , value=self.path, format='str')
815 opObj.addParameter(name='path', value=self.path, format='str')
837 opObj.addParameter(name='startDate' , value=self.startDate, format='date')
816 opObj.addParameter(name='startDate', value=self.startDate, format='date')
838 opObj.addParameter(name='endDate' , value=self.endDate, format='date')
817 opObj.addParameter(name='endDate', value=self.endDate, format='date')
839 opObj.addParameter(name='startTime' , value=self.startTime, format='time')
818 opObj.addParameter(name='startTime', value=self.startTime, format='time')
840 opObj.addParameter(name='endTime' , value=self.endTime, format='time')
819 opObj.addParameter(name='endTime', value=self.endTime, format='time')
841 opObj.addParameter(name='queue' , value=self.queue, format='obj')
820
842 for key, value in kwargs.items():
821 for key, value in kwargs.items():
843 opObj.addParameter(name=key, value=value, format=type(value).__name__)
822 opObj.addParameter(name=key, value=value, format=type(value).__name__)
844 else:
823 else:
@@ -849,32 +828,21 class ReadUnitConf(ProcUnitConf):
849
828
850 def updateRunOperation(self, **kwargs):
829 def updateRunOperation(self, **kwargs):
851
830
852 opObj = self.getOperationObj(name = 'run')
831 opObj = self.getOperationObj(name='run')
853 opObj.removeParameters()
832 opObj.removeParameters()
854
833
855 opObj.addParameter(name='datatype' , value=self.datatype, format='str')
834 opObj.addParameter(name='datatype', value=self.datatype, format='str')
856 opObj.addParameter(name='path' , value=self.path, format='str')
835 opObj.addParameter(name='path', value=self.path, format='str')
857 opObj.addParameter(name='startDate' , value=self.startDate, format='date')
836 opObj.addParameter(name='startDate', value=self.startDate, format='date')
858 opObj.addParameter(name='endDate' , value=self.endDate, format='date')
837 opObj.addParameter(name='endDate', value=self.endDate, format='date')
859 opObj.addParameter(name='startTime' , value=self.startTime, format='time')
838 opObj.addParameter(name='startTime', value=self.startTime, format='time')
860 opObj.addParameter(name='endTime' , value=self.endTime, format='time')
839 opObj.addParameter(name='endTime', value=self.endTime, format='time')
861
840
862 for key, value in kwargs.items():
841 for key, value in kwargs.items():
863 opObj.addParameter(name=key, value=value, format=type(value).__name__)
842 opObj.addParameter(name=key, value=value, format=type(value).__name__)
864
843
865 return opObj
844 return opObj
866
845
867 # def makeXml(self, projectElement):
868 #
869 # procUnitElement = SubElement(projectElement, self.ELEMENTNAME)
870 # procUnitElement.set('id', str(self.id))
871 # procUnitElement.set('name', self.name)
872 # procUnitElement.set('datatype', self.datatype)
873 # procUnitElement.set('inputId', str(self.inputId))
874 #
875 # for opConfObj in self.opConfObjList:
876 # opConfObj.makeXml(procUnitElement)
877
878 def readXml(self, upElement):
846 def readXml(self, upElement):
879
847
880 self.id = upElement.get('id')
848 self.id = upElement.get('id')
@@ -882,8 +850,8 class ReadUnitConf(ProcUnitConf):
882 self.datatype = upElement.get('datatype')
850 self.datatype = upElement.get('datatype')
883 self.inputId = upElement.get('inputId')
851 self.inputId = upElement.get('inputId')
884
852
885 if self.ELEMENTNAME == "ReadUnit":
853 if self.ELEMENTNAME == 'ReadUnit':
886 self.datatype = self.datatype.replace("Reader", "")
854 self.datatype = self.datatype.replace('Reader', '')
887
855
888 if self.inputId == 'None':
856 if self.inputId == 'None':
889 self.inputId = '0'
857 self.inputId = '0'
@@ -905,8 +873,9 class ReadUnitConf(ProcUnitConf):
905 self.endTime = opConfObj.getParameterValue('endTime')
873 self.endTime = opConfObj.getParameterValue('endTime')
906
874
907 class Project(Process):
875 class Project(Process):
876
908 id = None
877 id = None
909 name = None
878 # name = None
910 description = None
879 description = None
911 filename = None
880 filename = None
912
881
@@ -916,17 +885,17 class Project(Process):
916
885
917 plotterQueue = None
886 plotterQueue = None
918
887
919 def __init__(self, plotter_queue=None, logfile=None):
888 def __init__(self, plotter_queue=None):
889
920 Process.__init__(self)
890 Process.__init__(self)
921 self.id = None
891 self.id = None
922 self.name = None
892 # self.name = None
923 self.description = None
893 self.description = None
924 if logfile is not None:
894
925 logToFile(logfile)
926 self.plotterQueue = plotter_queue
895 self.plotterQueue = plotter_queue
927
896
928 self.procUnitConfObjDict = {}
897 self.procUnitConfObjDict = {}
929
898
930 def __getNewId(self):
899 def __getNewId(self):
931
900
932 idList = self.procUnitConfObjDict.keys()
901 idList = self.procUnitConfObjDict.keys()
@@ -972,18 +941,28 class Project(Process):
972
941
973 self.procUnitConfObjDict = newProcUnitConfObjDict
942 self.procUnitConfObjDict = newProcUnitConfObjDict
974
943
975 def setup(self, id, name, description):
944 def setup(self, id, name='', description=''):
976
945
946 print
947 print '*'*60
948 print ' Starting SIGNAL CHAIN PROCESSING v%s ' % schainpy.__version__
949 print '*'*60
950 print
977 self.id = str(id)
951 self.id = str(id)
978 self.name = name
979 self.description = description
952 self.description = description
980
953
981 def update(self, name, description):
954 def update(self, name, description):
982
955
983 self.name = name
984 self.description = description
956 self.description = description
985
957
958 def clone(self):
959
960 p = Project()
961 p.procUnitConfObjDict = self.procUnitConfObjDict
962 return p
963
986 def addReadUnit(self, id=None, datatype=None, name=None, **kwargs):
964 def addReadUnit(self, id=None, datatype=None, name=None, **kwargs):
965
987 if id is None:
966 if id is None:
988 idReadUnit = self.__getNewId()
967 idReadUnit = self.__getNewId()
989 else:
968 else:
@@ -1021,7 +1000,7 class Project(Process):
1021 def getReadUnitObj(self):
1000 def getReadUnitObj(self):
1022
1001
1023 for obj in self.procUnitConfObjDict.values():
1002 for obj in self.procUnitConfObjDict.values():
1024 if obj.getElementName() == "ReadUnit":
1003 if obj.getElementName() == 'ReadUnit':
1025 return obj
1004 return obj
1026
1005
1027 return None
1006 return None
@@ -1066,20 +1045,20 class Project(Process):
1066 if self.filename:
1045 if self.filename:
1067 filename = self.filename
1046 filename = self.filename
1068 else:
1047 else:
1069 filename = "schain.xml"
1048 filename = 'schain.xml'
1070
1049
1071 if not filename:
1050 if not filename:
1072 print "filename has not been defined. Use setFilename(filename) for do it."
1051 print 'filename has not been defined. Use setFilename(filename) for do it.'
1073 return 0
1052 return 0
1074
1053
1075 abs_file = os.path.abspath(filename)
1054 abs_file = os.path.abspath(filename)
1076
1055
1077 if not os.access(os.path.dirname(abs_file), os.W_OK):
1056 if not os.access(os.path.dirname(abs_file), os.W_OK):
1078 print "No write permission on %s" %os.path.dirname(abs_file)
1057 print 'No write permission on %s' %os.path.dirname(abs_file)
1079 return 0
1058 return 0
1080
1059
1081 if os.path.isfile(abs_file) and not(os.access(abs_file, os.W_OK)):
1060 if os.path.isfile(abs_file) and not(os.access(abs_file, os.W_OK)):
1082 print "File %s already exists and it could not be overwriten" %abs_file
1061 print 'File %s already exists and it could not be overwriten' %abs_file
1083 return 0
1062 return 0
1084
1063
1085 self.makeXml()
1064 self.makeXml()
@@ -1093,13 +1072,13 class Project(Process):
1093 def readXml(self, filename = None):
1072 def readXml(self, filename = None):
1094
1073
1095 if not filename:
1074 if not filename:
1096 print "filename is not defined"
1075 print 'filename is not defined'
1097 return 0
1076 return 0
1098
1077
1099 abs_file = os.path.abspath(filename)
1078 abs_file = os.path.abspath(filename)
1100
1079
1101 if not os.path.isfile(abs_file):
1080 if not os.path.isfile(abs_file):
1102 print "%s file does not exist" %abs_file
1081 print '%s file does not exist' %abs_file
1103 return 0
1082 return 0
1104
1083
1105 self.projectElement = None
1084 self.projectElement = None
@@ -1108,7 +1087,7 class Project(Process):
1108 try:
1087 try:
1109 self.projectElement = ElementTree().parse(abs_file)
1088 self.projectElement = ElementTree().parse(abs_file)
1110 except:
1089 except:
1111 print "Error reading %s, verify file format" %filename
1090 print 'Error reading %s, verify file format' %filename
1112 return 0
1091 return 0
1113
1092
1114 self.project = self.projectElement.tag
1093 self.project = self.projectElement.tag
@@ -1145,10 +1124,10 class Project(Process):
1145
1124
1146 def printattr(self):
1125 def printattr(self):
1147
1126
1148 print "Project[%s]: name = %s, description = %s" %(self.id,
1127 print 'Project[%s]: name = %s, description = %s' %(self.id,
1149 self.name,
1128 self.name,
1150 self.description)
1129 self.description)
1151
1130
1152 for procUnitConfObj in self.procUnitConfObjDict.values():
1131 for procUnitConfObj in self.procUnitConfObjDict.values():
1153 procUnitConfObj.printattr()
1132 procUnitConfObj.printattr()
1154
1133
@@ -1179,7 +1158,7 class Project(Process):
1179
1158
1180 self.__connect(puObjIN, thisPUObj)
1159 self.__connect(puObjIN, thisPUObj)
1181
1160
1182 def __handleError(self, procUnitConfObj, send_email=True):
1161 def __handleError(self, procUnitConfObj, send_email=False):
1183
1162
1184 import socket
1163 import socket
1185
1164
@@ -1187,33 +1166,33 class Project(Process):
1187 sys.exc_info()[1],
1166 sys.exc_info()[1],
1188 sys.exc_info()[2])
1167 sys.exc_info()[2])
1189
1168
1190 print "***** Error occurred in %s *****" %(procUnitConfObj.name)
1169 print '***** Error occurred in %s *****' %(procUnitConfObj.name)
1191 print "***** %s" %err[-1]
1170 print '***** %s' %err[-1]
1192
1171
1193 message = "".join(err)
1172 message = ''.join(err)
1194
1173
1195 sys.stderr.write(message)
1174 sys.stderr.write(message)
1196
1175
1197 if not send_email:
1176 if not send_email:
1198 return
1177 return
1199
1178
1200 subject = "SChain v%s: Error running %s\n" %(schainpy.__version__, procUnitConfObj.name)
1179 subject = 'SChain v%s: Error running %s\n' %(schainpy.__version__, procUnitConfObj.name)
1201
1180
1202 subtitle = "%s: %s\n" %(procUnitConfObj.getElementName() ,procUnitConfObj.name)
1181 subtitle = '%s: %s\n' %(procUnitConfObj.getElementName() ,procUnitConfObj.name)
1203 subtitle += "Hostname: %s\n" %socket.gethostbyname(socket.gethostname())
1182 subtitle += 'Hostname: %s\n' %socket.gethostbyname(socket.gethostname())
1204 subtitle += "Working directory: %s\n" %os.path.abspath("./")
1183 subtitle += 'Working directory: %s\n' %os.path.abspath('./')
1205 subtitle += "Configuration file: %s\n" %self.filename
1184 subtitle += 'Configuration file: %s\n' %self.filename
1206 subtitle += "Time: %s\n" %str(datetime.datetime.now())
1185 subtitle += 'Time: %s\n' %str(datetime.datetime.now())
1207
1186
1208 readUnitConfObj = self.getReadUnitObj()
1187 readUnitConfObj = self.getReadUnitObj()
1209 if readUnitConfObj:
1188 if readUnitConfObj:
1210 subtitle += "\nInput parameters:\n"
1189 subtitle += '\nInput parameters:\n'
1211 subtitle += "[Data path = %s]\n" %readUnitConfObj.path
1190 subtitle += '[Data path = %s]\n' %readUnitConfObj.path
1212 subtitle += "[Data type = %s]\n" %readUnitConfObj.datatype
1191 subtitle += '[Data type = %s]\n' %readUnitConfObj.datatype
1213 subtitle += "[Start date = %s]\n" %readUnitConfObj.startDate
1192 subtitle += '[Start date = %s]\n' %readUnitConfObj.startDate
1214 subtitle += "[End date = %s]\n" %readUnitConfObj.endDate
1193 subtitle += '[End date = %s]\n' %readUnitConfObj.endDate
1215 subtitle += "[Start time = %s]\n" %readUnitConfObj.startTime
1194 subtitle += '[Start time = %s]\n' %readUnitConfObj.startTime
1216 subtitle += "[End time = %s]\n" %readUnitConfObj.endTime
1195 subtitle += '[End time = %s]\n' %readUnitConfObj.endTime
1217
1196
1218 adminObj = schainpy.admin.SchainNotify()
1197 adminObj = schainpy.admin.SchainNotify()
1219 adminObj.sendAlert(message=message,
1198 adminObj.sendAlert(message=message,
@@ -1228,15 +1207,15 class Project(Process):
1228 return 0
1207 return 0
1229
1208
1230 def runController(self):
1209 def runController(self):
1231 """
1210 '''
1232 returns 0 when this process has been stopped, 1 otherwise
1211 returns 0 when this process has been stopped, 1 otherwise
1233 """
1212 '''
1234
1213
1235 if self.isPaused():
1214 if self.isPaused():
1236 print "Process suspended"
1215 print 'Process suspended'
1237
1216
1238 while True:
1217 while True:
1239 sleep(0.1)
1218 time.sleep(0.1)
1240
1219
1241 if not self.isPaused():
1220 if not self.isPaused():
1242 break
1221 break
@@ -1244,10 +1223,10 class Project(Process):
1244 if self.isStopped():
1223 if self.isStopped():
1245 break
1224 break
1246
1225
1247 print "Process reinitialized"
1226 print 'Process reinitialized'
1248
1227
1249 if self.isStopped():
1228 if self.isStopped():
1250 print "Process stopped"
1229 print 'Process stopped'
1251 return 0
1230 return 0
1252
1231
1253 return 1
1232 return 1
@@ -1258,29 +1237,23 class Project(Process):
1258
1237
1259 def setPlotterQueue(self, plotter_queue):
1238 def setPlotterQueue(self, plotter_queue):
1260
1239
1261 raise NotImplementedError, "Use schainpy.controller_api.ControllerThread instead Project class"
1240 raise NotImplementedError, 'Use schainpy.controller_api.ControllerThread instead Project class'
1262
1241
1263 def getPlotterQueue(self):
1242 def getPlotterQueue(self):
1264
1243
1265 raise NotImplementedError, "Use schainpy.controller_api.ControllerThread instead Project class"
1244 raise NotImplementedError, 'Use schainpy.controller_api.ControllerThread instead Project class'
1266
1245
1267 def useExternalPlotter(self):
1246 def useExternalPlotter(self):
1268
1247
1269 raise NotImplementedError, "Use schainpy.controller_api.ControllerThread instead Project class"
1248 raise NotImplementedError, 'Use schainpy.controller_api.ControllerThread instead Project class'
1270
1249
1250 def run(self):
1271
1251
1272 def run(self, filename=None):
1252 log.success('Starting {}'.format(self.name))
1273
1253
1274 # self.writeXml(filename)
1275 self.createObjects()
1254 self.createObjects()
1276 self.connectObjects()
1255 self.connectObjects()
1277
1256
1278 print
1279 print "*"*60
1280 print " Starting SIGNAL CHAIN PROCESSING v%s " %schainpy.__version__
1281 print "*"*60
1282 print
1283
1284 keyList = self.procUnitConfObjDict.keys()
1257 keyList = self.procUnitConfObjDict.keys()
1285 keyList.sort()
1258 keyList.sort()
1286
1259
@@ -1289,7 +1262,6 class Project(Process):
1289 is_ok = False
1262 is_ok = False
1290
1263
1291 for procKey in keyList:
1264 for procKey in keyList:
1292 # print "Running the '%s' process with %s" %(procUnitConfObj.name, procUnitConfObj.id)
1293
1265
1294 procUnitConfObj = self.procUnitConfObjDict[procKey]
1266 procUnitConfObj = self.procUnitConfObjDict[procKey]
1295
1267
@@ -1300,19 +1272,18 class Project(Process):
1300 is_ok = False
1272 is_ok = False
1301 break
1273 break
1302 except ValueError, e:
1274 except ValueError, e:
1303 sleep(0.5)
1275 time.sleep(0.5)
1304 self.__handleError(procUnitConfObj, send_email=True)
1276 self.__handleError(procUnitConfObj, send_email=True)
1305 is_ok = False
1277 is_ok = False
1306 break
1278 break
1307 except:
1279 except:
1308 sleep(0.5)
1280 time.sleep(0.5)
1309 self.__handleError(procUnitConfObj)
1281 self.__handleError(procUnitConfObj)
1310 is_ok = False
1282 is_ok = False
1311 break
1283 break
1312
1284
1313 #If every process unit finished so end process
1285 #If every process unit finished so end process
1314 if not(is_ok):
1286 if not(is_ok):
1315 # print "Every process unit have finished"
1316 break
1287 break
1317
1288
1318 if not self.runController():
1289 if not self.runController():
@@ -1322,3 +1293,5 class Project(Process):
1322 for procKey in keyList:
1293 for procKey in keyList:
1323 procUnitConfObj = self.procUnitConfObjDict[procKey]
1294 procUnitConfObj = self.procUnitConfObjDict[procKey]
1324 procUnitConfObj.close()
1295 procUnitConfObj.close()
1296
1297 log.success('{} finished'.format(self.name))
Show file before | Show file after | Hide comments
@@ -6218,3 +6218,6 class ShowMeConsole(QtCore.QObject):
6218 text = text[:-1]
6218 text = text[:-1]
6219
6219
6220 self.textWritten.emit(str(text))
6220 self.textWritten.emit(str(text))
6221
6222 def flush(self):
6223 pass
Show file before | Show file after | Hide comments
@@ -698,7 +698,7 class Spectra(JROData):
698 for pair in pairsList:
698 for pair in pairsList:
699 if pair not in self.pairsList:
699 if pair not in self.pairsList:
700 raise ValueError, "Pair %s is not in dataOut.pairsList" %(pair)
700 raise ValueError, "Pair %s is not in dataOut.pairsList" %(pair)
701 pairsIndexList.append(self.pairsList.index(pair))
701 pairsIndexList.append(self.pairsList.index(pair))
702 for i in range(len(pairsIndexList)):
702 for i in range(len(pairsIndexList)):
703 pair = self.pairsList[pairsIndexList[i]]
703 pair = self.pairsList[pairsIndexList[i]]
704 ccf = numpy.average(self.data_cspc[pairsIndexList[i], :, :], axis=0)
704 ccf = numpy.average(self.data_cspc[pairsIndexList[i], :, :], axis=0)
Show file before | Show file after | Hide comments
This diff has been collapsed as it changes many lines, (1208 lines changed) Show them Hide them
@@ -1,32 +1,33
1
1
2 import os
2 import os
3 import zmq
4 import time
3 import time
5 import numpy
4 import glob
6 import datetime
5 import datetime
7 import numpy as np
6 from multiprocessing import Process
7
8 import zmq
9 import numpy
8 import matplotlib
10 import matplotlib
9 import glob
10 matplotlib.use('TkAgg')
11 import matplotlib.pyplot as plt
11 import matplotlib.pyplot as plt
12 from mpl_toolkits.axes_grid1 import make_axes_locatable
12 from mpl_toolkits.axes_grid1 import make_axes_locatable
13 from matplotlib.ticker import FuncFormatter, LinearLocator
13 from matplotlib.ticker import FuncFormatter, LinearLocator, MultipleLocator
14 from multiprocessing import Process
15
14
16 from schainpy.model.proc.jroproc_base import Operation
15 from schainpy.model.proc.jroproc_base import Operation
17
16 from schainpy.utils import log
18 plt.ion()
19
17
20 func = lambda x, pos: ('%s') %(datetime.datetime.fromtimestamp(x).strftime('%H:%M'))
18 func = lambda x, pos: ('%s') %(datetime.datetime.fromtimestamp(x).strftime('%H:%M'))
21 fromtimestamp = lambda x, mintime : (datetime.datetime.utcfromtimestamp(mintime).replace(hour=(x + 5), minute=0) - d1970).total_seconds()
22
19
20 d1970 = datetime.datetime(1970, 1, 1)
23
21
24 d1970 = datetime.datetime(1970,1,1)
25
22
26 class PlotData(Operation, Process):
23 class PlotData(Operation, Process):
24 '''
25 Base class for Schain plotting operations
26 '''
27
27
28 CODE = 'Figure'
28 CODE = 'Figure'
29 colormap = 'jro'
29 colormap = 'jro'
30 bgcolor = 'white'
30 CONFLATE = False
31 CONFLATE = False
31 __MAXNUMX = 80
32 __MAXNUMX = 80
32 __missing = 1E30
33 __missing = 1E30
@@ -37,54 +38,143 class PlotData(Operation, Process):
37 Process.__init__(self)
38 Process.__init__(self)
38 self.kwargs['code'] = self.CODE
39 self.kwargs['code'] = self.CODE
39 self.mp = False
40 self.mp = False
40 self.dataOut = None
41 self.data = None
41 self.isConfig = False
42 self.isConfig = False
42 self.figure = None
43 self.figures = []
43 self.axes = []
44 self.axes = []
45 self.cb_axes = []
44 self.localtime = kwargs.pop('localtime', True)
46 self.localtime = kwargs.pop('localtime', True)
45 self.show = kwargs.get('show', True)
47 self.show = kwargs.get('show', True)
46 self.save = kwargs.get('save', False)
48 self.save = kwargs.get('save', False)
47 self.colormap = kwargs.get('colormap', self.colormap)
49 self.colormap = kwargs.get('colormap', self.colormap)
48 self.colormap_coh = kwargs.get('colormap_coh', 'jet')
50 self.colormap_coh = kwargs.get('colormap_coh', 'jet')
49 self.colormap_phase = kwargs.get('colormap_phase', 'RdBu_r')
51 self.colormap_phase = kwargs.get('colormap_phase', 'RdBu_r')
50 self.showprofile = kwargs.get('showprofile', True)
52 self.colormaps = kwargs.get('colormaps', None)
51 self.title = kwargs.get('wintitle', '')
53 self.bgcolor = kwargs.get('bgcolor', self.bgcolor)
54 self.showprofile = kwargs.get('showprofile', False)
55 self.title = kwargs.get('wintitle', self.CODE.upper())
56 self.cb_label = kwargs.get('cb_label', None)
57 self.cb_labels = kwargs.get('cb_labels', None)
52 self.xaxis = kwargs.get('xaxis', 'frequency')
58 self.xaxis = kwargs.get('xaxis', 'frequency')
53 self.zmin = kwargs.get('zmin', None)
59 self.zmin = kwargs.get('zmin', None)
54 self.zmax = kwargs.get('zmax', None)
60 self.zmax = kwargs.get('zmax', None)
61 self.zlimits = kwargs.get('zlimits', None)
55 self.xmin = kwargs.get('xmin', None)
62 self.xmin = kwargs.get('xmin', None)
63 if self.xmin is not None:
64 self.xmin += 5
56 self.xmax = kwargs.get('xmax', None)
65 self.xmax = kwargs.get('xmax', None)
57 self.xrange = kwargs.get('xrange', 24)
66 self.xrange = kwargs.get('xrange', 24)
58 self.ymin = kwargs.get('ymin', None)
67 self.ymin = kwargs.get('ymin', None)
59 self.ymax = kwargs.get('ymax', None)
68 self.ymax = kwargs.get('ymax', None)
60 self.__MAXNUMY = kwargs.get('decimation', 5000)
69 self.xlabel = kwargs.get('xlabel', None)
61 self.throttle_value = 5
70 self.__MAXNUMY = kwargs.get('decimation', 100)
62 self.times = []
71 self.showSNR = kwargs.get('showSNR', False)
63 #self.interactive = self.kwargs['parent']
72 self.oneFigure = kwargs.get('oneFigure', True)
73 self.width = kwargs.get('width', None)
74 self.height = kwargs.get('height', None)
75 self.colorbar = kwargs.get('colorbar', True)
76 self.factors = kwargs.get('factors', [1, 1, 1, 1, 1, 1, 1, 1])
77 self.titles = ['' for __ in range(16)]
78
79 def __setup(self):
80 '''
81 Common setup for all figures, here figures and axes are created
82 '''
83
84 self.setup()
85
86 if self.width is None:
87 self.width = 8
64
88
89 self.figures = []
90 self.axes = []
91 self.cb_axes = []
92 self.pf_axes = []
93 self.cmaps = []
94
95 size = '15%' if self.ncols==1 else '30%'
96 pad = '4%' if self.ncols==1 else '8%'
97
98 if self.oneFigure:
99 if self.height is None:
100 self.height = 1.4*self.nrows + 1
101 fig = plt.figure(figsize=(self.width, self.height),
102 edgecolor='k',
103 facecolor='w')
104 self.figures.append(fig)
105 for n in range(self.nplots):
106 ax = fig.add_subplot(self.nrows, self.ncols, n+1)
107 ax.tick_params(labelsize=8)
108 ax.firsttime = True
109 self.axes.append(ax)
110 if self.showprofile:
111 cax = self.__add_axes(ax, size=size, pad=pad)
112 cax.tick_params(labelsize=8)
113 self.pf_axes.append(cax)
114 else:
115 if self.height is None:
116 self.height = 3
117 for n in range(self.nplots):
118 fig = plt.figure(figsize=(self.width, self.height),
119 edgecolor='k',
120 facecolor='w')
121 ax = fig.add_subplot(1, 1, 1)
122 ax.tick_params(labelsize=8)
123 ax.firsttime = True
124 self.figures.append(fig)
125 self.axes.append(ax)
126 if self.showprofile:
127 cax = self.__add_axes(ax, size=size, pad=pad)
128 cax.tick_params(labelsize=8)
129 self.pf_axes.append(cax)
130
131 for n in range(self.nrows):
132 if self.colormaps is not None:
133 cmap = plt.get_cmap(self.colormaps[n])
134 else:
135 cmap = plt.get_cmap(self.colormap)
136 cmap.set_bad(self.bgcolor, 1.)
137 self.cmaps.append(cmap)
138
139 def __add_axes(self, ax, size='30%', pad='8%'):
65 '''
140 '''
66 this new parameter is created to plot data from varius channels at different figures
141 Add new axes to the given figure
67 1. crear una lista de figuras donde se puedan plotear las figuras,
68 2. dar las opciones de configuracion a cada figura, estas opciones son iguales para ambas figuras
69 3. probar?
70 '''
142 '''
71 self.ind_plt_ch = kwargs.get('ind_plt_ch', False)
143 divider = make_axes_locatable(ax)
72 self.figurelist = None
144 nax = divider.new_horizontal(size=size, pad=pad)
145 ax.figure.add_axes(nax)
146 return nax
73
147
74
148
75 def fill_gaps(self, x_buffer, y_buffer, z_buffer):
149 def setup(self):
150 '''
151 This method should be implemented in the child class, the following
152 attributes should be set:
153
154 self.nrows: number of rows
155 self.ncols: number of cols
156 self.nplots: number of plots (channels or pairs)
157 self.ylabel: label for Y axes
158 self.titles: list of axes title
159
160 '''
161 raise(NotImplementedError, 'Implement this method in child class')
76
162
163 def fill_gaps(self, x_buffer, y_buffer, z_buffer):
164 '''
165 Create a masked array for missing data
166 '''
77 if x_buffer.shape[0] < 2:
167 if x_buffer.shape[0] < 2:
78 return x_buffer, y_buffer, z_buffer
168 return x_buffer, y_buffer, z_buffer
79
169
80 deltas = x_buffer[1:] - x_buffer[0:-1]
170 deltas = x_buffer[1:] - x_buffer[0:-1]
81 x_median = np.median(deltas)
171 x_median = numpy.median(deltas)
82
172
83 index = np.where(deltas > 5*x_median)
173 index = numpy.where(deltas > 5*x_median)
84
174
85 if len(index[0]) != 0:
175 if len(index[0]) != 0:
86 z_buffer[::, index[0], ::] = self.__missing
176 z_buffer[::, index[0], ::] = self.__missing
87 z_buffer = np.ma.masked_inside(z_buffer,
177 z_buffer = numpy.ma.masked_inside(z_buffer,
88 0.99*self.__missing,
178 0.99*self.__missing,
89 1.01*self.__missing)
179 1.01*self.__missing)
90
180
@@ -99,110 +189,117 class PlotData(Operation, Process):
99 x = self.x
189 x = self.x
100 y = self.y[::dy]
190 y = self.y[::dy]
101 z = self.z[::, ::, ::dy]
191 z = self.z[::, ::, ::dy]
102
192
103 return x, y, z
193 return x, y, z
104
194
105 '''
195 def format(self):
106 JM:
196 '''
107 elimana las otras imagenes generadas debido a que lso workers no llegan en orden y le pueden
197 Set min and max values, labels, ticks and titles
108 poner otro tiempo a la figura q no necesariamente es el ultimo.
198 '''
109 Solo se realiza cuando termina la imagen.
110 Problemas:
111
199
112 File "/home/ci-81/workspace/schainv2.3/schainpy/model/graphics/jroplot_data.py", line 145, in __plot
200 if self.xmin is None:
113 for n, eachfigure in enumerate(self.figurelist):
201 xmin = self.min_time
114 TypeError: 'NoneType' object is not iterable
202 else:
203 if self.xaxis is 'time':
204 dt = datetime.datetime.fromtimestamp(self.min_time)
205 xmin = (datetime.datetime.combine(dt.date(),
206 datetime.time(int(self.xmin), 0, 0))-d1970).total_seconds()
207 else:
208 xmin = self.xmin
115
209
116 '''
210 if self.xmax is None:
117 def deleteanotherfiles(self):
211 xmax = xmin+self.xrange*60*60
118 figurenames=[]
212 else:
119 if self.figurelist != None:
213 if self.xaxis is 'time':
120 for n, eachfigure in enumerate(self.figurelist):
214 dt = datetime.datetime.fromtimestamp(self.min_time)
121 #add specific name for each channel in channelList
215 xmax = (datetime.datetime.combine(dt.date(),
122 ghostfigname = os.path.join(self.save, '{}_{}_{}'.format(self.titles[n].replace(' ',''),self.CODE,
216 datetime.time(int(self.xmax), 0, 0))-d1970).total_seconds()
123 datetime.datetime.fromtimestamp(self.saveTime).strftime('%y%m%d')))
217 else:
124 figname = os.path.join(self.save, '{}_{}_{}.png'.format(self.titles[n].replace(' ',''),self.CODE,
218 xmax = self.xmax
125 datetime.datetime.fromtimestamp(self.saveTime).strftime('%y%m%d_%H%M%S')))
219
126
220 ymin = self.ymin if self.ymin else numpy.nanmin(self.y)
127 for ghostfigure in glob.glob(ghostfigname+'*'): #ghostfigure will adopt all posible names of figures
221 ymax = self.ymax if self.ymax else numpy.nanmax(self.y)
128 if ghostfigure != figname:
222
129 os.remove(ghostfigure)
223 ystep = 200 if ymax>= 800 else 100 if ymax>=400 else 50 if ymax>=200 else 20
130 print 'Removing GhostFigures:' , figname
224
131 else :
225 for n, ax in enumerate(self.axes):
132 '''Erasing ghost images for just on******************'''
226 if ax.firsttime:
133 ghostfigname = os.path.join(self.save, '{}_{}'.format(self.CODE,datetime.datetime.fromtimestamp(self.saveTime).strftime('%y%m%d')))
227 ax.set_facecolor(self.bgcolor)
134 figname = os.path.join(self.save, '{}_{}.png'.format(self.CODE,datetime.datetime.fromtimestamp(self.saveTime).strftime('%y%m%d_%H%M%S')))
228 ax.yaxis.set_major_locator(MultipleLocator(ystep))
135 for ghostfigure in glob.glob(ghostfigname+'*'): #ghostfigure will adopt all posible names of figures
229 if self.xaxis is 'time':
136 if ghostfigure != figname:
230 ax.xaxis.set_major_formatter(FuncFormatter(func))
137 os.remove(ghostfigure)
231 ax.xaxis.set_major_locator(LinearLocator(9))
138 print 'Removing GhostFigures:' , figname
232 if self.xlabel is not None:
233 ax.set_xlabel(self.xlabel)
234 ax.set_ylabel(self.ylabel)
235 ax.firsttime = False
236 if self.showprofile:
237 self.pf_axes[n].set_ylim(ymin, ymax)
238 self.pf_axes[n].set_xlim(self.zmin, self.zmax)
239 self.pf_axes[n].set_xlabel('dB')
240 self.pf_axes[n].grid(b=True, axis='x')
241 [tick.set_visible(False) for tick in self.pf_axes[n].get_yticklabels()]
242 if self.colorbar:
243 cb = plt.colorbar(ax.plt, ax=ax, pad=0.02)
244 cb.ax.tick_params(labelsize=8)
245 if self.cb_label:
246 cb.set_label(self.cb_label, size=8)
247 elif self.cb_labels:
248 cb.set_label(self.cb_labels[n], size=8)
249
250 ax.set_title('{} - {} UTC'.format(
251 self.titles[n],
252 datetime.datetime.fromtimestamp(self.max_time).strftime('%H:%M:%S')),
253 size=8)
254 ax.set_xlim(xmin, xmax)
255 ax.set_ylim(ymin, ymax)
256
139
257
140 def __plot(self):
258 def __plot(self):
141
259 '''
142 print 'plotting...{}'.format(self.CODE)
260 '''
143 if self.ind_plt_ch is False : #standard
261 log.success('Plotting', self.name)
262
263 self.plot()
264 self.format()
265
266 for n, fig in enumerate(self.figures):
267 if self.nrows == 0 or self.nplots == 0:
268 log.warning('No data', self.name)
269 continue
144 if self.show:
270 if self.show:
145 self.figure.show()
271 fig.show()
146 self.plot()
272
147 plt.tight_layout()
273 fig.tight_layout()
148 self.figure.canvas.manager.set_window_title('{} {} - {}'.format(self.title, self.CODE.upper(),
274 fig.canvas.manager.set_window_title('{} - {}'.format(self.title,
149 datetime.datetime.fromtimestamp(self.max_time).strftime('%Y/%m/%d')))
275 datetime.datetime.fromtimestamp(self.max_time).strftime('%Y/%m/%d')))
150 else :
276 # fig.canvas.draw()
151 print 'len(self.figurelist): ',len(self.figurelist)
277
152 for n, eachfigure in enumerate(self.figurelist):
278 if self.save and self.data.ended:
153 if self.show:
279 channels = range(self.nrows)
154 eachfigure.show()
280 if self.oneFigure:
155
281 label = ''
156 self.plot()
282 else:
157 eachfigure.tight_layout() # ajuste de cada subplot
283 label = '_{}'.format(channels[n])
158 eachfigure.canvas.manager.set_window_title('{} {} - {}'.format(self.title[n], self.CODE.upper(),
284 figname = os.path.join(
159 datetime.datetime.fromtimestamp(self.max_time).strftime('%Y/%m/%d')))
285 self.save,
160
286 '{}{}_{}.png'.format(
161 # if self.save:
287 self.CODE,
162 # if self.ind_plt_ch is False : #standard
288 label,
163 # figname = os.path.join(self.save, '{}_{}.png'.format(self.CODE,
289 datetime.datetime.fromtimestamp(self.saveTime).strftime('%y%m%d_%H%M%S')
164 # datetime.datetime.fromtimestamp(self.saveTime).strftime('%y%m%d_%H%M%S')))
290 )
165 # print 'Saving figure: {}'.format(figname)
291 )
166 # self.figure.savefig(figname)
167 # else :
168 # for n, eachfigure in enumerate(self.figurelist):
169 # #add specific name for each channel in channelList
170 # figname = os.path.join(self.save, '{}_{}_{}.png'.format(self.titles[n],self.CODE,
171 # datetime.datetime.fromtimestamp(self.saveTime).strftime('%y%m%d_%H%M%S')))
172 #
173 # print 'Saving figure: {}'.format(figname)
174 # eachfigure.savefig(figname)
175
176 if self.ind_plt_ch is False :
177 self.figure.canvas.draw()
178 else :
179 for eachfigure in self.figurelist:
180 eachfigure.canvas.draw()
181
182 if self.save:
183 if self.ind_plt_ch is False : #standard
184 figname = os.path.join(self.save, '{}_{}.png'.format(self.CODE,
185 datetime.datetime.fromtimestamp(self.saveTime).strftime('%y%m%d_%H%M%S')))
186 print 'Saving figure: {}'.format(figname)
292 print 'Saving figure: {}'.format(figname)
187 self.figure.savefig(figname)
293 fig.savefig(figname)
188 else :
189 for n, eachfigure in enumerate(self.figurelist):
190 #add specific name for each channel in channelList
191 figname = os.path.join(self.save, '{}_{}_{}.png'.format(self.titles[n].replace(' ',''),self.CODE,
192 datetime.datetime.fromtimestamp(self.saveTime).strftime('%y%m%d_%H%M%S')))
193
194 print 'Saving figure: {}'.format(figname)
195 eachfigure.savefig(figname)
196
197
294
198 def plot(self):
295 def plot(self):
199
296 '''
200 print 'plotting...{}'.format(self.CODE.upper())
297 '''
201 return
298 raise(NotImplementedError, 'Implement this method in child class')
202
299
203 def run(self):
300 def run(self):
204
301
205 print '[Starting] {}'.format(self.name)
302 log.success('Starting', self.name)
206
303
207 context = zmq.Context()
304 context = zmq.Context()
208 receiver = context.socket(zmq.SUB)
305 receiver = context.socket(zmq.SUB)
@@ -212,152 +309,104 class PlotData(Operation, Process):
212 if 'server' in self.kwargs['parent']:
309 if 'server' in self.kwargs['parent']:
213 receiver.connect('ipc:///tmp/{}.plots'.format(self.kwargs['parent']['server']))
310 receiver.connect('ipc:///tmp/{}.plots'.format(self.kwargs['parent']['server']))
214 else:
311 else:
215 receiver.connect("ipc:///tmp/zmq.plots")
312 receiver.connect("ipc:///tmp/zmq.plots")
216
217 seconds_passed = 0
218
313
219 while True:
314 while True:
220 try:
315 try:
221 self.data = receiver.recv_pyobj(flags=zmq.NOBLOCK)#flags=zmq.NOBLOCK
316 self.data = receiver.recv_pyobj(flags=zmq.NOBLOCK)
222 self.started = self.data['STARTED']
317
223 self.dataOut = self.data['dataOut']
318 self.min_time = self.data.times[0]
224
319 self.max_time = self.data.times[-1]
225 if (len(self.times) < len(self.data['times']) and not self.started and self.data['ENDED']):
226 continue
227
228 self.times = self.data['times']
229 self.times.sort()
230 self.throttle_value = self.data['throttle']
231 self.min_time = self.times[0]
232 self.max_time = self.times[-1]
233
320
234 if self.isConfig is False:
321 if self.isConfig is False:
235 print 'setting up'
322 self.__setup()
236 self.setup()
237 self.isConfig = True
323 self.isConfig = True
238 self.__plot()
324
239
325 self.__plot()
240 if self.data['ENDED'] is True:
241 print '********GRAPHIC ENDED********'
242 self.ended = True
243 self.isConfig = False
244 self.__plot()
245 self.deleteanotherfiles() #CLPDG
246 elif seconds_passed >= self.data['throttle']:
247 print 'passed', seconds_passed
248 self.__plot()
249 seconds_passed = 0
250
326
251 except zmq.Again as e:
327 except zmq.Again as e:
252 print 'Waiting for data...'
328 log.log('Waiting for data...')
253 plt.pause(2)
329 if self.data:
254 seconds_passed += 2
330 plt.pause(self.data.throttle)
331 else:
332 time.sleep(2)
255
333
256 def close(self):
334 def close(self):
257 if self.dataOut:
335 if self.data:
258 self.__plot()
336 self.__plot()
259
337
260
338
261 class PlotSpectraData(PlotData):
339 class PlotSpectraData(PlotData):
340 '''
341 Plot for Spectra data
342 '''
262
343
263 CODE = 'spc'
344 CODE = 'spc'
264 colormap = 'jro'
345 colormap = 'jro'
265 CONFLATE = False
266
346
267 def setup(self):
347 def setup(self):
268
348 self.nplots = len(self.data.channels)
269 ncolspan = 1
349 self.ncols = int(numpy.sqrt(self.nplots)+ 0.9)
270 colspan = 1
350 self.nrows = int((1.0*self.nplots/self.ncols) + 0.9)
271 self.ncols = int(numpy.sqrt(self.dataOut.nChannels)+0.9)
351 self.width = 3.4*self.ncols
272 self.nrows = int(self.dataOut.nChannels*1./self.ncols + 0.9)
352 self.height = 3*self.nrows
273 self.width = 3.6*self.ncols
353 self.cb_label = 'dB'
274 self.height = 3.2*self.nrows
354 if self.showprofile:
275 if self.showprofile:
355 self.width += 0.8*self.ncols
276 ncolspan = 3
277 colspan = 2
278 self.width += 1.2*self.ncols
279
356
280 self.ylabel = 'Range [Km]'
357 self.ylabel = 'Range [Km]'
281 self.titles = ['Channel {}'.format(x) for x in self.dataOut.channelList]
282
283 if self.figure is None:
284 self.figure = plt.figure(figsize=(self.width, self.height),
285 edgecolor='k',
286 facecolor='w')
287 else:
288 self.figure.clf()
289
290 n = 0
291 for y in range(self.nrows):
292 for x in range(self.ncols):
293 if n >= self.dataOut.nChannels:
294 break
295 ax = plt.subplot2grid((self.nrows, self.ncols*ncolspan), (y, x*ncolspan), 1, colspan)
296 if self.showprofile:
297 ax.ax_profile = plt.subplot2grid((self.nrows, self.ncols*ncolspan), (y, x*ncolspan+colspan), 1, 1)
298
299 ax.firsttime = True
300 self.axes.append(ax)
301 n += 1
302
358
303 def plot(self):
359 def plot(self):
304
305 if self.xaxis == "frequency":
360 if self.xaxis == "frequency":
306 x = self.dataOut.getFreqRange(1)/1000.
361 x = self.data.xrange[0]
307 xlabel = "Frequency (kHz)"
362 self.xlabel = "Frequency (kHz)"
308 elif self.xaxis == "time":
363 elif self.xaxis == "time":
309 x = self.dataOut.getAcfRange(1)
364 x = self.data.xrange[1]
310 xlabel = "Time (ms)"
365 self.xlabel = "Time (ms)"
311 else:
366 else:
312 x = self.dataOut.getVelRange(1)
367 x = self.data.xrange[2]
313 xlabel = "Velocity (m/s)"
368 self.xlabel = "Velocity (m/s)"
369
370 if self.CODE == 'spc_mean':
371 x = self.data.xrange[2]
372 self.xlabel = "Velocity (m/s)"
314
373
315 y = self.dataOut.getHeiRange()
374 self.titles = []
316 z = self.data[self.CODE]
317
375
376 y = self.data.heights
377 self.y = y
378 z = self.data['spc']
379
318 for n, ax in enumerate(self.axes):
380 for n, ax in enumerate(self.axes):
381 noise = self.data['noise'][n][-1]
382 if self.CODE == 'spc_mean':
383 mean = self.data['mean'][n][-1]
319 if ax.firsttime:
384 if ax.firsttime:
320 self.xmax = self.xmax if self.xmax else np.nanmax(x)
385 self.xmax = self.xmax if self.xmax else numpy.nanmax(x)
321 self.xmin = self.xmin if self.xmin else -self.xmax
386 self.xmin = self.xmin if self.xmin else -self.xmax
322 self.ymin = self.ymin if self.ymin else np.nanmin(y)
387 self.zmin = self.zmin if self.zmin else numpy.nanmin(z)
323 self.ymax = self.ymax if self.ymax else np.nanmax(y)
388 self.zmax = self.zmax if self.zmax else numpy.nanmax(z)
324 self.zmin = self.zmin if self.zmin else np.nanmin(z)
389 ax.plt = ax.pcolormesh(x, y, z[n].T,
325 self.zmax = self.zmax if self.zmax else np.nanmax(z)
390 vmin=self.zmin,
326 ax.plot = ax.pcolormesh(x, y, z[n].T,
391 vmax=self.zmax,
327 vmin=self.zmin,
392 cmap=plt.get_cmap(self.colormap)
328 vmax=self.zmax,
393 )
329 cmap=plt.get_cmap(self.colormap)
330 )
331 divider = make_axes_locatable(ax)
332 cax = divider.new_horizontal(size='3%', pad=0.05)
333 self.figure.add_axes(cax)
334 plt.colorbar(ax.plot, cax)
335
336 ax.set_xlim(self.xmin, self.xmax)
337 ax.set_ylim(self.ymin, self.ymax)
338
339 ax.set_ylabel(self.ylabel)
340 ax.set_xlabel(xlabel)
341
342 ax.firsttime = False
343
394
344 if self.showprofile:
395 if self.showprofile:
345 ax.plot_profile= ax.ax_profile.plot(self.data['rti'][self.max_time][n], y)[0]
396 ax.plt_profile= self.pf_axes[n].plot(self.data['rti'][n][-1], y)[0]
346 ax.ax_profile.set_xlim(self.zmin, self.zmax)
397 ax.plt_noise = self.pf_axes[n].plot(numpy.repeat(noise, len(y)), y,
347 ax.ax_profile.set_ylim(self.ymin, self.ymax)
398 color="k", linestyle="dashed", lw=1)[0]
348 ax.ax_profile.set_xlabel('dB')
399 if self.CODE == 'spc_mean':
349 ax.ax_profile.grid(b=True, axis='x')
400 ax.plt_mean = ax.plot(mean, y, color='k')[0]
350 ax.plot_noise = ax.ax_profile.plot(numpy.repeat(self.data['noise'][self.max_time][n], len(y)), y,
351 color="k", linestyle="dashed", lw=2)[0]
352 [tick.set_visible(False) for tick in ax.ax_profile.get_yticklabels()]
353 else:
401 else:
354 ax.plot.set_array(z[n].T.ravel())
402 ax.plt.set_array(z[n].T.ravel())
355 if self.showprofile:
403 if self.showprofile:
356 ax.plot_profile.set_data(self.data['rti'][self.max_time][n], y)
404 ax.plt_profile.set_data(self.data['rti'][n][-1], y)
357 ax.plot_noise.set_data(numpy.repeat(self.data['noise'][self.max_time][n], len(y)), y)
405 ax.plt_noise.set_data(numpy.repeat(noise, len(y)), y)
406 if self.CODE == 'spc_mean':
407 ax.plt_mean.set_data(mean, y)
358
408
359 ax.set_title('{} - Noise: {:.2f} dB'.format(self.titles[n], self.data['noise'][self.max_time][n]),
409 self.titles.append('CH {}: {:3.2f}dB'.format(n, noise))
360 size=8)
361 self.saveTime = self.max_time
410 self.saveTime = self.max_time
362
411
363
412
@@ -367,545 +416,245 class PlotCrossSpectraData(PlotData):
367 zmin_coh = None
416 zmin_coh = None
368 zmax_coh = None
417 zmax_coh = None
369 zmin_phase = None
418 zmin_phase = None
370 zmax_phase = None
419 zmax_phase = None
371 CONFLATE = False
372
420
373 def setup(self):
421 def setup(self):
374
422
375 ncolspan = 1
423 self.ncols = 4
376 colspan = 1
424 self.nrows = len(self.data.pairs)
377 self.ncols = 2
425 self.nplots = self.nrows*4
378 self.nrows = self.dataOut.nPairs
426 self.width = 3.4*self.ncols
379 self.width = 3.6*self.ncols
427 self.height = 3*self.nrows
380 self.height = 3.2*self.nrows
381
382 self.ylabel = 'Range [Km]'
428 self.ylabel = 'Range [Km]'
383 self.titles = ['Channel {}'.format(x) for x in self.dataOut.channelList]
429 self.showprofile = False
384
385 if self.figure is None:
386 self.figure = plt.figure(figsize=(self.width, self.height),
387 edgecolor='k',
388 facecolor='w')
389 else:
390 self.figure.clf()
391
392 for y in range(self.nrows):
393 for x in range(self.ncols):
394 ax = plt.subplot2grid((self.nrows, self.ncols), (y, x), 1, 1)
395 ax.firsttime = True
396 self.axes.append(ax)
397
430
398 def plot(self):
431 def plot(self):
399
432
400 if self.xaxis == "frequency":
433 if self.xaxis == "frequency":
401 x = self.dataOut.getFreqRange(1)/1000.
434 x = self.data.xrange[0]
402 xlabel = "Frequency (kHz)"
435 self.xlabel = "Frequency (kHz)"
403 elif self.xaxis == "time":
436 elif self.xaxis == "time":
404 x = self.dataOut.getAcfRange(1)
437 x = self.data.xrange[1]
405 xlabel = "Time (ms)"
438 self.xlabel = "Time (ms)"
406 else:
439 else:
407 x = self.dataOut.getVelRange(1)
440 x = self.data.xrange[2]
408 xlabel = "Velocity (m/s)"
441 self.xlabel = "Velocity (m/s)"
442
443 self.titles = []
409
444
410 y = self.dataOut.getHeiRange()
445 y = self.data.heights
411 z_coh = self.data['cspc_coh']
446 self.y = y
412 z_phase = self.data['cspc_phase']
447 spc = self.data['spc']
448 cspc = self.data['cspc']
413
449
414 for n in range(self.nrows):
450 for n in range(self.nrows):
415 ax = self.axes[2*n]
451 noise = self.data['noise'][n][-1]
416 ax1 = self.axes[2*n+1]
452 pair = self.data.pairs[n]
453 ax = self.axes[4*n]
454 ax3 = self.axes[4*n+3]
417 if ax.firsttime:
455 if ax.firsttime:
418 self.xmax = self.xmax if self.xmax else np.nanmax(x)
456 self.xmax = self.xmax if self.xmax else numpy.nanmax(x)
419 self.xmin = self.xmin if self.xmin else -self.xmax
457 self.xmin = self.xmin if self.xmin else -self.xmax
420 self.ymin = self.ymin if self.ymin else np.nanmin(y)
458 self.zmin = self.zmin if self.zmin else numpy.nanmin(spc)
421 self.ymax = self.ymax if self.ymax else np.nanmax(y)
459 self.zmax = self.zmax if self.zmax else numpy.nanmax(spc)
422 self.zmin_coh = self.zmin_coh if self.zmin_coh else 0.0
460 ax.plt = ax.pcolormesh(x, y, spc[pair[0]].T,
423 self.zmax_coh = self.zmax_coh if self.zmax_coh else 1.0
461 vmin=self.zmin,
424 self.zmin_phase = self.zmin_phase if self.zmin_phase else -180
462 vmax=self.zmax,
425 self.zmax_phase = self.zmax_phase if self.zmax_phase else 180
463 cmap=plt.get_cmap(self.colormap)
426
464 )
427 ax.plot = ax.pcolormesh(x, y, z_coh[n].T,
428 vmin=self.zmin_coh,
429 vmax=self.zmax_coh,
430 cmap=plt.get_cmap(self.colormap_coh)
431 )
432 divider = make_axes_locatable(ax)
433 cax = divider.new_horizontal(size='3%', pad=0.05)
434 self.figure.add_axes(cax)
435 plt.colorbar(ax.plot, cax)
436
437 ax.set_xlim(self.xmin, self.xmax)
438 ax.set_ylim(self.ymin, self.ymax)
439
440 ax.set_ylabel(self.ylabel)
441 ax.set_xlabel(xlabel)
442 ax.firsttime = False
443
444 ax1.plot = ax1.pcolormesh(x, y, z_phase[n].T,
445 vmin=self.zmin_phase,
446 vmax=self.zmax_phase,
447 cmap=plt.get_cmap(self.colormap_phase)
448 )
449 divider = make_axes_locatable(ax1)
450 cax = divider.new_horizontal(size='3%', pad=0.05)
451 self.figure.add_axes(cax)
452 plt.colorbar(ax1.plot, cax)
453
454 ax1.set_xlim(self.xmin, self.xmax)
455 ax1.set_ylim(self.ymin, self.ymax)
456
457 ax1.set_ylabel(self.ylabel)
458 ax1.set_xlabel(xlabel)
459 ax1.firsttime = False
460 else:
465 else:
461 ax.plot.set_array(z_coh[n].T.ravel())
466 ax.plt.set_array(spc[pair[0]].T.ravel())
462 ax1.plot.set_array(z_phase[n].T.ravel())
467 self.titles.append('CH {}: {:3.2f}dB'.format(n, noise))
463
464 ax.set_title('Coherence Ch{} * Ch{}'.format(self.dataOut.pairsList[n][0], self.dataOut.pairsList[n][1]), size=8)
465 ax1.set_title('Phase Ch{} * Ch{}'.format(self.dataOut.pairsList[n][0], self.dataOut.pairsList[n][1]), size=8)
466 self.saveTime = self.max_time
467
468
468
469 class PlotSpectraMeanData(PlotSpectraData):
469 ax = self.axes[4*n+1]
470
470 if ax.firsttime:
471 CODE = 'spc_mean'
471 ax.plt = ax.pcolormesh(x, y, spc[pair[1]].T,
472 colormap = 'jet'
473
474 def plot(self):
475
476 if self.xaxis == "frequency":
477 x = self.dataOut.getFreqRange(1)/1000.
478 xlabel = "Frequency (kHz)"
479 elif self.xaxis == "time":
480 x = self.dataOut.getAcfRange(1)
481 xlabel = "Time (ms)"
482 else:
483 x = self.dataOut.getVelRange(1)
484 xlabel = "Velocity (m/s)"
485
486 y = self.dataOut.getHeiRange()
487 z = self.data['spc']
488 mean = self.data['mean'][self.max_time]
489
490 for n, ax in enumerate(self.axes):
491
492 if ax.firsttime:
493 self.xmax = self.xmax if self.xmax else np.nanmax(x)
494 self.xmin = self.xmin if self.xmin else -self.xmax
495 self.ymin = self.ymin if self.ymin else np.nanmin(y)
496 self.ymax = self.ymax if self.ymax else np.nanmax(y)
497 self.zmin = self.zmin if self.zmin else np.nanmin(z)
498 self.zmax = self.zmax if self.zmax else np.nanmax(z)
499 ax.plt = ax.pcolormesh(x, y, z[n].T,
500 vmin=self.zmin,
472 vmin=self.zmin,
501 vmax=self.zmax,
473 vmax=self.zmax,
502 cmap=plt.get_cmap(self.colormap)
474 cmap=plt.get_cmap(self.colormap)
503 )
475 )
504 ax.plt_dop = ax.plot(mean[n], y,
505 color='k')[0]
506
507 divider = make_axes_locatable(ax)
508 cax = divider.new_horizontal(size='3%', pad=0.05)
509 self.figure.add_axes(cax)
510 plt.colorbar(ax.plt, cax)
511
512 ax.set_xlim(self.xmin, self.xmax)
513 ax.set_ylim(self.ymin, self.ymax)
514
515 ax.set_ylabel(self.ylabel)
516 ax.set_xlabel(xlabel)
517
518 ax.firsttime = False
519
520 if self.showprofile:
521 ax.plt_profile= ax.ax_profile.plot(self.data['rti'][self.max_time][n], y)[0]
522 ax.ax_profile.set_xlim(self.zmin, self.zmax)
523 ax.ax_profile.set_ylim(self.ymin, self.ymax)
524 ax.ax_profile.set_xlabel('dB')
525 ax.ax_profile.grid(b=True, axis='x')
526 ax.plt_noise = ax.ax_profile.plot(numpy.repeat(self.data['noise'][self.max_time][n], len(y)), y,
527 color="k", linestyle="dashed", lw=2)[0]
528 [tick.set_visible(False) for tick in ax.ax_profile.get_yticklabels()]
529 else:
476 else:
530 ax.plt.set_array(z[n].T.ravel())
477 ax.plt.set_array(spc[pair[1]].T.ravel())
531 ax.plt_dop.set_data(mean[n], y)
478 self.titles.append('CH {}: {:3.2f}dB'.format(n, noise))
532 if self.showprofile:
479
533 ax.plt_profile.set_data(self.data['rti'][self.max_time][n], y)
480 out = cspc[n]/numpy.sqrt(spc[pair[0]]*spc[pair[1]])
534 ax.plt_noise.set_data(numpy.repeat(self.data['noise'][self.max_time][n], len(y)), y)
481 coh = numpy.abs(out)
482 phase = numpy.arctan2(out.imag, out.real)*180/numpy.pi
483
484 ax = self.axes[4*n+2]
485 if ax.firsttime:
486 ax.plt = ax.pcolormesh(x, y, coh.T,
487 vmin=0,
488 vmax=1,
489 cmap=plt.get_cmap(self.colormap_coh)
490 )
491 else:
492 ax.plt.set_array(coh.T.ravel())
493 self.titles.append('Coherence Ch{} * Ch{}'.format(pair[0], pair[1]))
535
494
536 ax.set_title('{} - Noise: {:.2f} dB'.format(self.titles[n], self.data['noise'][self.max_time][n]),
495 ax = self.axes[4*n+3]
537 size=8)
496 if ax.firsttime:
497 ax.plt = ax.pcolormesh(x, y, phase.T,
498 vmin=-180,
499 vmax=180,
500 cmap=plt.get_cmap(self.colormap_phase)
501 )
502 else:
503 ax.plt.set_array(phase.T.ravel())
504 self.titles.append('Phase CH{} * CH{}'.format(pair[0], pair[1]))
505
538 self.saveTime = self.max_time
506 self.saveTime = self.max_time
539
507
540
508
509 class PlotSpectraMeanData(PlotSpectraData):
510 '''
511 Plot for Spectra and Mean
512 '''
513 CODE = 'spc_mean'
514 colormap = 'jro'
515
516
541 class PlotRTIData(PlotData):
517 class PlotRTIData(PlotData):
518 '''
519 Plot for RTI data
520 '''
542
521
543 CODE = 'rti'
522 CODE = 'rti'
544 colormap = 'jro'
523 colormap = 'jro'
545
524
546 def setup(self):
525 def setup(self):
547 self.ncols = 1
526 self.xaxis = 'time'
548 self.nrows = self.dataOut.nChannels
527 self.ncols = 1
549 self.width = 10
528 self.nrows = len(self.data.channels)
550 #TODO : arreglar la altura de la figura, esta hardcodeada.
529 self.nplots = len(self.data.channels)
551 #Se arreglo, testear!
552 if self.ind_plt_ch:
553 self.height = 3.2#*self.nrows if self.nrows<6 else 12
554 else:
555 self.height = 2.2*self.nrows if self.nrows<6 else 12
556
557 '''
558 if self.nrows==1:
559 self.height += 1
560 '''
561 self.ylabel = 'Range [Km]'
530 self.ylabel = 'Range [Km]'
562 self.titles = ['Channel {}'.format(x) for x in self.dataOut.channelList]
531 self.cb_label = 'dB'
563
532 self.titles = ['{} Channel {}'.format(self.CODE.upper(), x) for x in range(self.nrows)]
564 '''
565 Logica:
566 1) Si la variable ind_plt_ch es True, va a crear mas de 1 figura
567 2) guardamos "Figures" en una lista y "axes" en otra, quizas se deberia guardar el
568 axis dentro de "Figures" como un diccionario.
569 '''
570 if self.ind_plt_ch is False: #standard mode
571
572 if self.figure is None: #solo para la priemra vez
573 self.figure = plt.figure(figsize=(self.width, self.height),
574 edgecolor='k',
575 facecolor='w')
576 else:
577 self.figure.clf()
578 self.axes = []
579
580
581 for n in range(self.nrows):
582 ax = self.figure.add_subplot(self.nrows, self.ncols, n+1)
583 #ax = self.figure(n+1)
584 ax.firsttime = True
585 self.axes.append(ax)
586
587 else : #append one figure foreach channel in channelList
588 if self.figurelist == None:
589 self.figurelist = []
590 for n in range(self.nrows):
591 self.figure = plt.figure(figsize=(self.width, self.height),
592 edgecolor='k',
593 facecolor='w')
594 #add always one subplot
595 self.figurelist.append(self.figure)
596
597 else : # cada dia nuevo limpia el axes, pero mantiene el figure
598 for eachfigure in self.figurelist:
599 eachfigure.clf() # eliminaria todas las figuras de la lista?
600 self.axes = []
601
602 for eachfigure in self.figurelist:
603 ax = eachfigure.add_subplot(1,1,1) #solo 1 axis por figura
604 #ax = self.figure(n+1)
605 ax.firsttime = True
606 #Cada figura tiene un distinto puntero
607 self.axes.append(ax)
608 #plt.close(eachfigure)
609
610
533
611 def plot(self):
534 def plot(self):
535 self.x = self.data.times
536 self.y = self.data.heights
537 self.z = self.data[self.CODE]
538 self.z = numpy.ma.masked_invalid(self.z)
612
539
613 if self.ind_plt_ch is False: #standard mode
540 for n, ax in enumerate(self.axes):
614 self.x = np.array(self.times)
541 x, y, z = self.fill_gaps(*self.decimate())
615 self.y = self.dataOut.getHeiRange()
542 self.zmin = self.zmin if self.zmin else numpy.min(self.z)
616 self.z = []
543 self.zmax = self.zmax if self.zmax else numpy.max(self.z)
617
544 if ax.firsttime:
618 for ch in range(self.nrows):
545 ax.plt = ax.pcolormesh(x, y, z[n].T,
619 self.z.append([self.data[self.CODE][t][ch] for t in self.times])
546 vmin=self.zmin,
620
547 vmax=self.zmax,
621 self.z = np.array(self.z)
548 cmap=plt.get_cmap(self.colormap)
622 for n, ax in enumerate(self.axes):
549 )
623 x, y, z = self.fill_gaps(*self.decimate())
550 if self.showprofile:
624 if self.xmin is None:
551 ax.plot_profile= self.pf_axes[n].plot(self.data['rti'][n][-1], self.y)[0]
625 xmin = self.min_time
552 ax.plot_noise = self.pf_axes[n].plot(numpy.repeat(self.data['noise'][n][-1], len(self.y)), self.y,
626 else:
553 color="k", linestyle="dashed", lw=1)[0]
627 xmin = fromtimestamp(int(self.xmin), self.min_time)
554 else:
628 if self.xmax is None:
555 ax.collections.remove(ax.collections[0])
629 xmax = xmin + self.xrange*60*60
556 ax.plt = ax.pcolormesh(x, y, z[n].T,
630 else:
557 vmin=self.zmin,
631 xmax = xmin + (self.xmax - self.xmin) * 60 * 60
558 vmax=self.zmax,
632 self.zmin = self.zmin if self.zmin else np.min(self.z)
559 cmap=plt.get_cmap(self.colormap)
633 self.zmax = self.zmax if self.zmax else np.max(self.z)
560 )
634 if ax.firsttime:
561 if self.showprofile:
635 self.ymin = self.ymin if self.ymin else np.nanmin(self.y)
562 ax.plot_profile.set_data(self.data['rti'][n][-1], self.y)
636 self.ymax = self.ymax if self.ymax else np.nanmax(self.y)
563 ax.plot_noise.set_data(numpy.repeat(self.data['noise'][n][-1], len(self.y)), self.y)
637 plot = ax.pcolormesh(x, y, z[n].T,
638 vmin=self.zmin,
639 vmax=self.zmax,
640 cmap=plt.get_cmap(self.colormap)
641 )
642 divider = make_axes_locatable(ax)
643 cax = divider.new_horizontal(size='2%', pad=0.05)
644 self.figure.add_axes(cax)
645 plt.colorbar(plot, cax)
646 ax.set_ylim(self.ymin, self.ymax)
647 ax.xaxis.set_major_formatter(FuncFormatter(func))
648 ax.xaxis.set_major_locator(LinearLocator(6))
649 ax.set_ylabel(self.ylabel)
650 # if self.xmin is None:
651 # xmin = self.min_time
652 # else:
653 # xmin = (datetime.datetime.combine(self.dataOut.datatime.date(),
654 # datetime.time(self.xmin, 0, 0))-d1970).total_seconds()
655
656 ax.set_xlim(xmin, xmax)
657 ax.firsttime = False
658 else:
659 ax.collections.remove(ax.collections[0])
660 ax.set_xlim(xmin, xmax)
661 plot = ax.pcolormesh(x, y, z[n].T,
662 vmin=self.zmin,
663 vmax=self.zmax,
664 cmap=plt.get_cmap(self.colormap)
665 )
666 ax.set_title('{} {}'.format(self.titles[n],
667 datetime.datetime.fromtimestamp(self.max_time).strftime('%y/%m/%d %H:%M:%S')),
668 size=8)
669
670 self.saveTime = self.min_time
671 else :
672 self.x = np.array(self.times)
673 self.y = self.dataOut.getHeiRange()
674 self.z = []
675
676 for ch in range(self.nrows):
677 self.z.append([self.data[self.CODE][t][ch] for t in self.times])
678
679 self.z = np.array(self.z)
680 for n, eachfigure in enumerate(self.figurelist): #estaba ax in axes
681
682 x, y, z = self.fill_gaps(*self.decimate())
683 xmin = self.min_time
684 xmax = xmin+self.xrange*60*60
685 self.zmin = self.zmin if self.zmin else np.min(self.z)
686 self.zmax = self.zmax if self.zmax else np.max(self.z)
687 if self.axes[n].firsttime:
688 self.ymin = self.ymin if self.ymin else np.nanmin(self.y)
689 self.ymax = self.ymax if self.ymax else np.nanmax(self.y)
690 plot = self.axes[n].pcolormesh(x, y, z[n].T,
691 vmin=self.zmin,
692 vmax=self.zmax,
693 cmap=plt.get_cmap(self.colormap)
694 )
695 divider = make_axes_locatable(self.axes[n])
696 cax = divider.new_horizontal(size='2%', pad=0.05)
697 eachfigure.add_axes(cax)
698 #self.figure2.add_axes(cax)
699 plt.colorbar(plot, cax)
700 self.axes[n].set_ylim(self.ymin, self.ymax)
701
702 self.axes[n].xaxis.set_major_formatter(FuncFormatter(func))
703 self.axes[n].xaxis.set_major_locator(LinearLocator(6))
704
705 self.axes[n].set_ylabel(self.ylabel)
706
707 if self.xmin is None:
708 xmin = self.min_time
709 else:
710 xmin = (datetime.datetime.combine(self.dataOut.datatime.date(),
711 datetime.time(self.xmin, 0, 0))-d1970).total_seconds()
712
713 self.axes[n].set_xlim(xmin, xmax)
714 self.axes[n].firsttime = False
715 else:
716 self.axes[n].collections.remove(self.axes[n].collections[0])
717 self.axes[n].set_xlim(xmin, xmax)
718 plot = self.axes[n].pcolormesh(x, y, z[n].T,
719 vmin=self.zmin,
720 vmax=self.zmax,
721 cmap=plt.get_cmap(self.colormap)
722 )
723 self.axes[n].set_title('{} {}'.format(self.titles[n],
724 datetime.datetime.fromtimestamp(self.max_time).strftime('%y/%m/%d %H:%M:%S')),
725 size=8)
726
564
727 self.saveTime = self.min_time
565 self.saveTime = self.min_time
728
566
729
567
730 class PlotCOHData(PlotRTIData):
568 class PlotCOHData(PlotRTIData):
569 '''
570 Plot for Coherence data
571 '''
731
572
732 CODE = 'coh'
573 CODE = 'coh'
733
574
734 def setup(self):
575 def setup(self):
735
576 self.xaxis = 'time'
736 self.ncols = 1
577 self.ncols = 1
737 self.nrows = self.dataOut.nPairs
578 self.nrows = len(self.data.pairs)
738 self.width = 10
579 self.nplots = len(self.data.pairs)
739 self.height = 2.2*self.nrows if self.nrows<6 else 12
580 self.ylabel = 'Range [Km]'
740 self.ind_plt_ch = False #just for coherence and phase
581 if self.CODE == 'coh':
741 if self.nrows==1:
582 self.cb_label = ''
742 self.height += 1
583 self.titles = ['Coherence Map Ch{} * Ch{}'.format(x[0], x[1]) for x in self.data.pairs]
743 self.ylabel = 'Range [Km]'
744 self.titles = ['{} Ch{} * Ch{}'.format(self.CODE.upper(), x[0], x[1]) for x in self.dataOut.pairsList]
745
746 if self.figure is None:
747 self.figure = plt.figure(figsize=(self.width, self.height),
748 edgecolor='k',
749 facecolor='w')
750 else:
584 else:
751 self.figure.clf()
585 self.cb_label = 'Degrees'
752 self.axes = []
586 self.titles = ['Phase Map Ch{} * Ch{}'.format(x[0], x[1]) for x in self.data.pairs]
753
587
754 for n in range(self.nrows):
588
755 ax = self.figure.add_subplot(self.nrows, self.ncols, n+1)
589 class PlotPHASEData(PlotCOHData):
756 ax.firsttime = True
590 '''
757 self.axes.append(ax)
591 Plot for Phase map data
592 '''
593
594 CODE = 'phase'
595 colormap = 'seismic'
758
596
759
597
760 class PlotNoiseData(PlotData):
598 class PlotNoiseData(PlotData):
599 '''
600 Plot for noise
601 '''
602
761 CODE = 'noise'
603 CODE = 'noise'
762
604
763 def setup(self):
605 def setup(self):
764
606 self.xaxis = 'time'
765 self.ncols = 1
607 self.ncols = 1
766 self.nrows = 1
608 self.nrows = 1
767 self.width = 10
609 self.nplots = 1
768 self.height = 3.2
769 self.ylabel = 'Intensity [dB]'
610 self.ylabel = 'Intensity [dB]'
770 self.titles = ['Noise']
611 self.titles = ['Noise']
771
612 self.colorbar = False
772 if self.figure is None:
773 self.figure = plt.figure(figsize=(self.width, self.height),
774 edgecolor='k',
775 facecolor='w')
776 else:
777 self.figure.clf()
778 self.axes = []
779
780 self.ax = self.figure.add_subplot(self.nrows, self.ncols, 1)
781 self.ax.firsttime = True
782
613
783 def plot(self):
614 def plot(self):
784
615
785 x = self.times
616 x = self.data.times
786 xmin = self.min_time
617 xmin = self.min_time
787 xmax = xmin+self.xrange*60*60
618 xmax = xmin+self.xrange*60*60
788 if self.ax.firsttime:
619 Y = self.data[self.CODE]
789 for ch in self.dataOut.channelList:
620
790 y = [self.data[self.CODE][t][ch] for t in self.times]
621 if self.axes[0].firsttime:
791 self.ax.plot(x, y, lw=1, label='Ch{}'.format(ch))
622 for ch in self.data.channels:
792 self.ax.firsttime = False
623 y = Y[ch]
793 self.ax.xaxis.set_major_formatter(FuncFormatter(func))
624 self.axes[0].plot(x, y, lw=1, label='Ch{}'.format(ch))
794 self.ax.xaxis.set_major_locator(LinearLocator(6))
795 self.ax.set_ylabel(self.ylabel)
796 plt.legend()
625 plt.legend()
797 else:
626 else:
798 for ch in self.dataOut.channelList:
627 for ch in self.data.channels:
799 y = [self.data[self.CODE][t][ch] for t in self.times]
628 y = Y[ch]
800 self.ax.lines[ch].set_data(x, y)
629 self.axes[0].lines[ch].set_data(x, y)
801
630
802 self.ax.set_xlim(xmin, xmax)
631 self.ymin = numpy.nanmin(Y) - 5
803 self.ax.set_ylim(min(y)-5, max(y)+5)
632 self.ymax = numpy.nanmax(Y) + 5
804 self.saveTime = self.min_time
633 self.saveTime = self.min_time
805
634
806
635
807 class PlotWindProfilerData(PlotRTIData):
808
809 CODE = 'wind'
810 colormap = 'seismic'
811
812 def setup(self):
813 self.ncols = 1
814 self.nrows = self.dataOut.data_output.shape[0]
815 self.width = 10
816 self.height = 2.2*self.nrows
817 self.ylabel = 'Height [Km]'
818 self.titles = ['Zonal Wind' ,'Meridional Wind', 'Vertical Wind']
819 self.clabels = ['Velocity (m/s)','Velocity (m/s)','Velocity (cm/s)']
820 self.windFactor = [1, 1, 100]
821
822 if self.figure is None:
823 self.figure = plt.figure(figsize=(self.width, self.height),
824 edgecolor='k',
825 facecolor='w')
826 else:
827 self.figure.clf()
828 self.axes = []
829
830 for n in range(self.nrows):
831 ax = self.figure.add_subplot(self.nrows, self.ncols, n+1)
832 ax.firsttime = True
833 self.axes.append(ax)
834
835 def plot(self):
836
837 self.x = np.array(self.times)
838 self.y = self.dataOut.heightList
839 self.z = []
840
841 for ch in range(self.nrows):
842 self.z.append([self.data['output'][t][ch] for t in self.times])
843
844 self.z = np.array(self.z)
845 self.z = numpy.ma.masked_invalid(self.z)
846
847 cmap=plt.get_cmap(self.colormap)
848 cmap.set_bad('black', 1.)
849
850 for n, ax in enumerate(self.axes):
851 x, y, z = self.fill_gaps(*self.decimate())
852 xmin = self.min_time
853 xmax = xmin+self.xrange*60*60
854 if ax.firsttime:
855 self.ymin = self.ymin if self.ymin else np.nanmin(self.y)
856 self.ymax = self.ymax if self.ymax else np.nanmax(self.y)
857 self.zmax = self.zmax if self.zmax else numpy.nanmax(abs(self.z[:-1, :]))
858 self.zmin = self.zmin if self.zmin else -self.zmax
859
860 plot = ax.pcolormesh(x, y, z[n].T*self.windFactor[n],
861 vmin=self.zmin,
862 vmax=self.zmax,
863 cmap=cmap
864 )
865 divider = make_axes_locatable(ax)
866 cax = divider.new_horizontal(size='2%', pad=0.05)
867 self.figure.add_axes(cax)
868 cb = plt.colorbar(plot, cax)
869 cb.set_label(self.clabels[n])
870 ax.set_ylim(self.ymin, self.ymax)
871
872 ax.xaxis.set_major_formatter(FuncFormatter(func))
873 ax.xaxis.set_major_locator(LinearLocator(6))
874
875 ax.set_ylabel(self.ylabel)
876
877 ax.set_xlim(xmin, xmax)
878 ax.firsttime = False
879 else:
880 ax.collections.remove(ax.collections[0])
881 ax.set_xlim(xmin, xmax)
882 plot = ax.pcolormesh(x, y, z[n].T*self.windFactor[n],
883 vmin=self.zmin,
884 vmax=self.zmax,
885 cmap=plt.get_cmap(self.colormap)
886 )
887 ax.set_title('{} {}'.format(self.titles[n],
888 datetime.datetime.fromtimestamp(self.max_time).strftime('%y/%m/%d %H:%M:%S')),
889 size=8)
890
891 self.saveTime = self.min_time
892
893
894 class PlotSNRData(PlotRTIData):
636 class PlotSNRData(PlotRTIData):
637 '''
638 Plot for SNR Data
639 '''
640
895 CODE = 'snr'
641 CODE = 'snr'
896 colormap = 'jet'
642 colormap = 'jet'
897
643
644
898 class PlotDOPData(PlotRTIData):
645 class PlotDOPData(PlotRTIData):
646 '''
647 Plot for DOPPLER Data
648 '''
649
899 CODE = 'dop'
650 CODE = 'dop'
900 colormap = 'jet'
651 colormap = 'jet'
901
652
902
653
903 class PlotPHASEData(PlotCOHData):
904 CODE = 'phase'
905 colormap = 'seismic'
906
907
908 class PlotSkyMapData(PlotData):
654 class PlotSkyMapData(PlotData):
655 '''
656 Plot for meteors detection data
657 '''
909
658
910 CODE = 'met'
659 CODE = 'met'
911
660
@@ -932,7 +681,7 class PlotSkyMapData(PlotData):
932
681
933 def plot(self):
682 def plot(self):
934
683
935 arrayParameters = np.concatenate([self.data['param'][t] for t in self.times])
684 arrayParameters = numpy.concatenate([self.data['param'][t] for t in self.data.times])
936 error = arrayParameters[:,-1]
685 error = arrayParameters[:,-1]
937 indValid = numpy.where(error == 0)[0]
686 indValid = numpy.where(error == 0)[0]
938 finalMeteor = arrayParameters[indValid,:]
687 finalMeteor = arrayParameters[indValid,:]
@@ -962,3 +711,72 class PlotSkyMapData(PlotData):
962 self.ax.set_title(title, size=8)
711 self.ax.set_title(title, size=8)
963
712
964 self.saveTime = self.max_time
713 self.saveTime = self.max_time
714
715 class PlotParamData(PlotRTIData):
716 '''
717 Plot for data_param object
718 '''
719
720 CODE = 'param'
721 colormap = 'seismic'
722
723 def setup(self):
724 self.xaxis = 'time'
725 self.ncols = 1
726 self.nrows = self.data.shape(self.CODE)[0]
727 self.nplots = self.nrows
728 if self.showSNR:
729 self.nrows += 1
730
731 self.ylabel = 'Height [Km]'
732 self.titles = self.data.parameters \
733 if self.data.parameters else ['Param {}'.format(x) for x in xrange(self.nrows)]
734 if self.showSNR:
735 self.titles.append('SNR')
736
737 def plot(self):
738 self.data.normalize_heights()
739 self.x = self.data.times
740 self.y = self.data.heights
741 if self.showSNR:
742 self.z = numpy.concatenate(
743 (self.data[self.CODE], self.data['snr'])
744 )
745 else:
746 self.z = self.data[self.CODE]
747
748 self.z = numpy.ma.masked_invalid(self.z)
749
750 for n, ax in enumerate(self.axes):
751
752 x, y, z = self.fill_gaps(*self.decimate())
753
754 if ax.firsttime:
755 if self.zlimits is not None:
756 self.zmin, self.zmax = self.zlimits[n]
757 self.zmax = self.zmax if self.zmax is not None else numpy.nanmax(abs(self.z[:-1, :]))
758 self.zmin = self.zmin if self.zmin is not None else -self.zmax
759 ax.plt = ax.pcolormesh(x, y, z[n, :, :].T*self.factors[n],
760 vmin=self.zmin,
761 vmax=self.zmax,
762 cmap=self.cmaps[n]
763 )
764 else:
765 if self.zlimits is not None:
766 self.zmin, self.zmax = self.zlimits[n]
767 ax.collections.remove(ax.collections[0])
768 ax.plt = ax.pcolormesh(x, y, z[n, :, :].T*self.factors[n],
769 vmin=self.zmin,
770 vmax=self.zmax,
771 cmap=self.cmaps[n]
772 )
773
774 self.saveTime = self.min_time
775
776 class PlotOuputData(PlotParamData):
777 '''
778 Plot data_output object
779 '''
780
781 CODE = 'output'
782 colormap = 'seismic' No newline at end of file
Show file before | Show file after | Hide comments
@@ -267,7 +267,7 class AMISRReader(ProcessingUnit):
267 self.dirnameList = new_dirnameList
267 self.dirnameList = new_dirnameList
268 return 1
268 return 1
269
269
270 def __searchFilesOnline(self,
270 def searchFilesOnLine(self,
271 path,
271 path,
272 walk=True):
272 walk=True):
273
273
@@ -287,7 +287,7 class AMISRReader(ProcessingUnit):
287 return
287 return
288
288
289
289
290 def __searchFilesOffline(self,
290 def searchFilesOffLine(self,
291 path,
291 path,
292 startDate,
292 startDate,
293 endDate,
293 endDate,
@@ -494,9 +494,9 class AMISRReader(ProcessingUnit):
494 self.online = online
494 self.online = online
495 if not(online):
495 if not(online):
496 #Busqueda de archivos offline
496 #Busqueda de archivos offline
497 self.__searchFilesOffline(path, startDate, endDate, startTime, endTime, walk)
497 self.searchFilesOffLine(path, startDate, endDate, startTime, endTime, walk)
498 else:
498 else:
499 self.__searchFilesOnline(path, walk)
499 self.searchFilesOnLine(path, walk)
500
500
501 if not(self.filenameList):
501 if not(self.filenameList):
502 print "There is no files into the folder: %s"%(path)
502 print "There is no files into the folder: %s"%(path)
Show file before | Show file after | Hide comments
This diff has been collapsed as it changes many lines, (503 lines changed) Show them Hide them
@@ -541,8 +541,7 class JRODataIO:
541 return inspect.getargspec(self.run).args
541 return inspect.getargspec(self.run).args
542
542
543 class JRODataReader(JRODataIO):
543 class JRODataReader(JRODataIO):
544
544
545 firstTime = True
546 online = 0
545 online = 0
547
546
548 realtime = 0
547 realtime = 0
@@ -578,8 +577,7 class JRODataReader(JRODataIO):
578 selBlocksize = None
577 selBlocksize = None
579
578
580 selBlocktime = None
579 selBlocktime = None
581
580
582 onlineWithDate = False
583 def __init__(self):
581 def __init__(self):
584
582
585 """
583 """
@@ -603,19 +601,19 class JRODataReader(JRODataIO):
603
601
604 raise NotImplementedError
602 raise NotImplementedError
605
603
606 def __searchFilesOffLine(self,
604 def searchFilesOffLine(self,
607 path,
605 path,
608 startDate=None,
606 startDate=None,
609 endDate=None,
607 endDate=None,
610 startTime=datetime.time(0,0,0),
608 startTime=datetime.time(0,0,0),
611 endTime=datetime.time(23,59,59),
609 endTime=datetime.time(23,59,59),
612 set=None,
610 set=None,
613 expLabel='',
611 expLabel='',
614 ext='.r',
612 ext='.r',
615 queue=None,
613 cursor=None,
616 cursor=None,
614 skip=None,
617 skip=None,
615 walk=True):
618 walk=True):
616
619 self.filenameList = []
617 self.filenameList = []
620 self.datetimeList = []
618 self.datetimeList = []
621
619
@@ -624,8 +622,7 class JRODataReader(JRODataIO):
624 dateList, pathList = self.findDatafiles(path, startDate, endDate, expLabel, ext, walk, include_path=True)
622 dateList, pathList = self.findDatafiles(path, startDate, endDate, expLabel, ext, walk, include_path=True)
625
623
626 if dateList == []:
624 if dateList == []:
627 # print "[Reading] Date range selected invalid [%s - %s]: No *%s files in %s)" %(startDate, endDate, ext, path)
625 return [], []
628 return None, None
629
626
630 if len(dateList) > 1:
627 if len(dateList) > 1:
631 print "[Reading] Data found for date range [%s - %s]: total days = %d" %(startDate, endDate, len(dateList))
628 print "[Reading] Data found for date range [%s - %s]: total days = %d" %(startDate, endDate, len(dateList))
@@ -636,18 +633,15 class JRODataReader(JRODataIO):
636 datetimeList = []
633 datetimeList = []
637
634
638 for thisPath in pathList:
635 for thisPath in pathList:
639 # thisPath = pathList[pathDict[file]]
636
640
641 fileList = glob.glob1(thisPath, "*%s" %ext)
637 fileList = glob.glob1(thisPath, "*%s" %ext)
642 fileList.sort()
638 fileList.sort()
643
639
644 skippedFileList = []
640 skippedFileList = []
645
641
646 if cursor is not None and skip is not None:
642 if cursor is not None andk skip is not None:
647 # if cursor*skip > len(fileList):
643
648 if skip == 0:
644 if skip == 0:
649 if queue is not None:
650 queue.put(len(fileList))
651 skippedFileList = []
645 skippedFileList = []
652 else:
646 else:
653 skippedFileList = fileList[cursor*skip: cursor*skip + skip]
647 skippedFileList = fileList[cursor*skip: cursor*skip + skip]
@@ -672,44 +666,43 class JRODataReader(JRODataIO):
672
666
673 if not(filenameList):
667 if not(filenameList):
674 print "[Reading] Time range selected invalid [%s - %s]: No *%s files in %s)" %(startTime, endTime, ext, path)
668 print "[Reading] Time range selected invalid [%s - %s]: No *%s files in %s)" %(startTime, endTime, ext, path)
675 return None, None
669 return [], []
676
670
677 print "[Reading] %d file(s) was(were) found in time range: %s - %s" %(len(filenameList), startTime, endTime)
671 print "[Reading] %d file(s) was(were) found in time range: %s - %s" %(len(filenameList), startTime, endTime)
678 print
672 print
679
673
680 for i in range(len(filenameList)):
674 # for i in range(len(filenameList)):
681 print "[Reading] %s -> [%s]" %(filenameList[i], datetimeList[i].ctime())
675 # print "[Reading] %s -> [%s]" %(filenameList[i], datetimeList[i].ctime())
682
676
683 self.filenameList = filenameList
677 self.filenameList = filenameList
684 self.datetimeList = datetimeList
678 self.datetimeList = datetimeList
679
685 return pathList, filenameList
680 return pathList, filenameList
686
681
687 def __searchFilesOnLine(self, path, expLabel="", ext=None, walk=True, set=None, startDate=None, startTime=None):
682 def __searchFilesOnLine(self, path, expLabel = "", ext = None, walk=True, set=None):
688
683
689 """
684 """
690 Busca el ultimo archivo de la ultima carpeta (determinada o no por startDateTime) y
685 Busca el ultimo archivo de la ultima carpeta (determinada o no por startDateTime) y
691 devuelve el archivo encontrado ademas de otros datos.
686 devuelve el archivo encontrado ademas de otros datos.
692
687
693 Input:
688 Input:
694 path : carpeta donde estan contenidos los files que contiene data
689 path : carpeta donde estan contenidos los files que contiene data
695
690
696 expLabel : Nombre del subexperimento (subfolder)
691 expLabel : Nombre del subexperimento (subfolder)
697
692
698 ext : extension de los files
693 ext : extension de los files
699
694
700 walk : Si es habilitado no realiza busquedas dentro de los subdirectorios (doypath)
695 walk : Si es habilitado no realiza busquedas dentro de los ubdirectorios (doypath)
701
696
702 Return:
697 Return:
703 directory : eL directorio donde esta el file encontrado
698 directory : eL directorio donde esta el file encontrado
704 filename : el ultimo file de una determinada carpeta
699 filename : el ultimo file de una determinada carpeta
705 year : el anho
700 year : el anho
706 doy : el numero de dia del anho
701 doy : el numero de dia del anho
707 set : el set del archivo
702 set : el set del archivo
708
703
709
704
710 """
705 """
711 pathList = None
712 filenameList = None
713 if not os.path.isdir(path):
706 if not os.path.isdir(path):
714 return None, None, None, None, None, None
707 return None, None, None, None, None, None
715
708
@@ -719,7 +712,7 class JRODataReader(JRODataIO):
719 fullpath = path
712 fullpath = path
720 foldercounter = 0
713 foldercounter = 0
721 else:
714 else:
722 # Filtra solo los directorios
715 #Filtra solo los directorios
723 for thisPath in os.listdir(path):
716 for thisPath in os.listdir(path):
724 if not os.path.isdir(os.path.join(path,thisPath)):
717 if not os.path.isdir(os.path.join(path,thisPath)):
725 continue
718 continue
@@ -755,7 +748,7 class JRODataReader(JRODataIO):
755
748
756 year = int( filename[1:5] )
749 year = int( filename[1:5] )
757 doy = int( filename[5:8] )
750 doy = int( filename[5:8] )
758 set = int( filename[8:11] )
751 set = int( filename[8:11] )
759
752
760 return fullpath, foldercounter, filename, year, doy, set
753 return fullpath, foldercounter, filename, year, doy, set
761
754
@@ -767,7 +760,7 class JRODataReader(JRODataIO):
767 idFile += 1
760 idFile += 1
768 if not(idFile < len(self.filenameList)):
761 if not(idFile < len(self.filenameList)):
769 self.flagNoMoreFiles = 1
762 self.flagNoMoreFiles = 1
770 # print "[Reading] No more Files"
763 # print "[Reading] No more Files"
771 return 0
764 return 0
772
765
773 filename = self.filenameList[idFile]
766 filename = self.filenameList[idFile]
@@ -785,32 +778,31 class JRODataReader(JRODataIO):
785 self.fileSize = fileSize
778 self.fileSize = fileSize
786 self.fp = fp
779 self.fp = fp
787
780
788 #print "[Reading] Setting the file: %s"%self.filename
781 # print "[Reading] Setting the file: %s"%self.filename
789
782
790 return 1
783 return 1
791
784
792 def __setNextFileOnline(self):
785 def __setNextFileOnline(self):
793 """
786 """
794 Busca el siguiente file que tenga suficiente data para ser leida, dentro de un folder especifico, si
787 Busca el siguiente file que tenga suficiente data para ser leida, dentro de un folder especifico, si
795 no encuentra un file valido espera un tiempo determinado y luego busca en los posibles n files
788 no encuentra un file valido espera un tiempo determinado y luego busca en los posibles n files
796 siguientes.
789 siguientes.
797
790
798 Affected:
791 Affected:
799 self.flagIsNewFile
792 self.flagIsNewFile
800 self.filename
793 self.filename
801 self.fileSize
794 self.fileSize
802 self.fp
795 self.fp
803 self.set
796 self.set
804 self.flagNoMoreFiles
797 self.flagNoMoreFiles
805
798
806 Return:
799 Return:
807 0 : si luego de una busqueda del siguiente file valido este no pudo ser encontrado
800 0 : si luego de una busqueda del siguiente file valido este no pudo ser encontrado
808 1 : si el file fue abierto con exito y esta listo a ser leido
801 1 : si el file fue abierto con exito y esta listo a ser leido
809
802
810 Excepciones:
803 Excepciones:
811 Si un determinado file no puede ser abierto
804 Si un determinado file no puede ser abierto
812 """
805 """
813
814 nFiles = 0
806 nFiles = 0
815 fileOk_flag = False
807 fileOk_flag = False
816 firstTime_flag = True
808 firstTime_flag = True
@@ -869,48 +861,27 class JRODataReader(JRODataIO):
869 if self.fp != None: self.fp.close()
861 if self.fp != None: self.fp.close()
870 self.fp = open(fullfilename, 'rb')
862 self.fp = open(fullfilename, 'rb')
871 self.flagNoMoreFiles = 0
863 self.flagNoMoreFiles = 0
872 # print '[Reading] Setting the file: %s' % fullfilename
864 # print '[Reading] Setting the file: %s' % fullfilename
873 else:
865 else:
874 self.fileSize = 0
866 self.fileSize = 0
875 self.filename = None
867 self.filename = None
876 self.flagIsNewFile = 0
868 self.flagIsNewFile = 0
877 self.fp = None
869 self.fp = None
878 self.flagNoMoreFiles = 1
870 self.flagNoMoreFiles = 1
879 # print '[Reading] No more files to read'
871 # print '[Reading] No more files to read'
880
872
881 return fileOk_flag
873 return fileOk_flag
882
874
883 def setNextFile(self):
875 def setNextFile(self):
884 if self.fp != None:
876 if self.fp != None:
885 self.fp.close()
877 self.fp.close()
878
886 if self.online:
879 if self.online:
887 newFile = self.__setNextFileOnline()
880 newFile = self.__setNextFileOnline()
888 else:
881 else:
889 newFile = self.__setNextFileOffline()
882 newFile = self.__setNextFileOffline()
883
890 if not(newFile):
884 if not(newFile):
891 if self.onlineWithDate is True:
892 self.onlineWithDate=False
893 self.online = True
894 self.firstTime = False
895 self.setup(
896 path=self.path,
897 startDate=self.startDate,
898 endDate=self.endDate,
899 startTime=self.startTime ,
900 endTime=self.endTime,
901 set=self.set,
902 expLabel=self.expLabel,
903 ext=self.ext,
904 online=self.online,
905 delay=self.delay,
906 walk=self.walk,
907 getblock=self.getblock,
908 nTxs=self.nTxs,
909 realtime=self.realtime,
910 blocksize=self.blocksize,
911 blocktime=self.blocktime
912 )
913 return 1
914 print '[Reading] No more files to read'
885 print '[Reading] No more files to read'
915 return 0
886 return 0
916
887
@@ -951,7 +922,7 class JRODataReader(JRODataIO):
951 return 1
922 return 1
952
923
953 if self.fileSize == self.fileSizeByHeader:
924 if self.fileSize == self.fileSizeByHeader:
954 # self.flagEoF = True
925 # self.flagEoF = True
955 return 0
926 return 0
956
927
957 print "[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)
928 print "[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)
@@ -1005,13 +976,13 class JRODataReader(JRODataIO):
1005 else:
976 else:
1006 self.fp.seek(self.fp.tell() - neededsize)
977 self.fp.seek(self.fp.tell() - neededsize)
1007 break
978 break
1008
979
1009 # csize = self.fileSize - self.fp.tell()
980 # csize = self.fileSize - self.fp.tell()
1010 # neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
981 # neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
1011 # factor = int(csize/neededsize)
982 # factor = int(csize/neededsize)
1012 # if factor > 0:
983 # if factor > 0:
1013 # self.fp.seek(self.fp.tell() + factor*neededsize)
984 # self.fp.seek(self.fp.tell() + factor*neededsize)
1014
985
1015 self.flagIsNewFile = 0
986 self.flagIsNewFile = 0
1016 self.__isFirstTimeOnline = 0
987 self.__isFirstTimeOnline = 0
1017
988
@@ -1019,10 +990,10 class JRODataReader(JRODataIO):
1019 #if self.server is None:
990 #if self.server is None:
1020 if self.fp == None:
991 if self.fp == None:
1021 return 0
992 return 0
1022
993
1023 # if self.online:
994 # if self.online:
1024 # self.__jumpToLastBlock()
995 # self.__jumpToLastBlock()
1025
996
1026 if self.flagIsNewFile:
997 if self.flagIsNewFile:
1027 self.lastUTTime = self.basicHeaderObj.utc
998 self.lastUTTime = self.basicHeaderObj.utc
1028 return 1
999 return 1
@@ -1065,12 +1036,14 class JRODataReader(JRODataIO):
1065
1036
1066 #Skip block out of startTime and endTime
1037 #Skip block out of startTime and endTime
1067 while True:
1038 while True:
1068 if not(self.__setNewBlock()):
1039 if not(self.__setNewBlock()):
1069 print 'returning'
1070 return 0
1040 return 0
1041
1071 if not(self.readBlock()):
1042 if not(self.readBlock()):
1072 return 0
1043 return 0
1044
1073 self.getBasicHeader()
1045 self.getBasicHeader()
1046
1074 if not isTimeInRange(self.dataOut.datatime.time(), self.startTime, self.endTime):
1047 if not isTimeInRange(self.dataOut.datatime.time(), self.startTime, self.endTime):
1075
1048
1076 print "[Reading] Block No. %d/%d -> %s [Skipping]" %(self.nReadBlocks,
1049 print "[Reading] Block No. %d/%d -> %s [Skipping]" %(self.nReadBlocks,
@@ -1082,8 +1055,8 class JRODataReader(JRODataIO):
1082
1055
1083 if self.verbose:
1056 if self.verbose:
1084 print "[Reading] Block No. %d/%d -> %s" %(self.nReadBlocks,
1057 print "[Reading] Block No. %d/%d -> %s" %(self.nReadBlocks,
1085 self.processingHeaderObj.dataBlocksPerFile,
1058 self.processingHeaderObj.dataBlocksPerFile,
1086 self.dataOut.datatime.ctime())
1059 self.dataOut.datatime.ctime())
1087 return 1
1060 return 1
1088
1061
1089 def __readFirstHeader(self):
1062 def __readFirstHeader(self):
@@ -1114,8 +1087,8 class JRODataReader(JRODataIO):
1114 self.dtype = datatype_str
1087 self.dtype = datatype_str
1115 #self.ippSeconds = 2 * 1000 * self.radarControllerHeaderObj.ipp / self.c
1088 #self.ippSeconds = 2 * 1000 * self.radarControllerHeaderObj.ipp / self.c
1116 self.fileSizeByHeader = self.processingHeaderObj.dataBlocksPerFile * self.processingHeaderObj.blockSize + self.firstHeaderSize + self.basicHeaderSize*(self.processingHeaderObj.dataBlocksPerFile - 1)
1089 self.fileSizeByHeader = self.processingHeaderObj.dataBlocksPerFile * self.processingHeaderObj.blockSize + self.firstHeaderSize + self.basicHeaderSize*(self.processingHeaderObj.dataBlocksPerFile - 1)
1117 # self.dataOut.channelList = numpy.arange(self.systemHeaderObj.numChannels)
1090 # self.dataOut.channelList = numpy.arange(self.systemHeaderObj.numChannels)
1118 # self.dataOut.channelIndexList = numpy.arange(self.systemHeaderObj.numChannels)
1091 # self.dataOut.channelIndexList = numpy.arange(self.systemHeaderObj.numChannels)
1119 self.getBlockDimension()
1092 self.getBlockDimension()
1120
1093
1121 def __verifyFile(self, filename, msgFlag=True):
1094 def __verifyFile(self, filename, msgFlag=True):
@@ -1277,13 +1250,13 class JRODataReader(JRODataIO):
1277
1250
1278 def setup(self,
1251 def setup(self,
1279 path=None,
1252 path=None,
1280 startDate=None,
1253 startDate=None,
1281 endDate=None,
1254 endDate=None,
1282 startTime=datetime.time(0,0,0),
1255 startTime=datetime.time(0,0,0),
1283 endTime=datetime.time(23,59,59),
1256 endTime=datetime.time(23,59,59),
1284 set=None,
1257 set=None,
1285 expLabel = "",
1258 expLabel = "",
1286 ext = None,
1259 ext = None,
1287 online = False,
1260 online = False,
1288 delay = 60,
1261 delay = 60,
1289 walk = True,
1262 walk = True,
@@ -1292,139 +1265,109 class JRODataReader(JRODataIO):
1292 realtime=False,
1265 realtime=False,
1293 blocksize=None,
1266 blocksize=None,
1294 blocktime=None,
1267 blocktime=None,
1268 skip=None,
1269 cursor=None,
1270 warnings=True,
1295 verbose=True,
1271 verbose=True,
1272 server=None,
1296 **kwargs):
1273 **kwargs):
1297
1274 if server is not None:
1298 if path == None:
1275 if 'tcp://' in server:
1299 raise ValueError, "[Reading] The path is not valid"
1276 address = server
1300
1277 else:
1278 address = 'ipc:///tmp/%s' % server
1279 self.server = address
1280 self.context = zmq.Context()
1281 self.receiver = self.context.socket(zmq.PULL)
1282 self.receiver.connect(self.server)
1283 time.sleep(0.5)
1284 print '[Starting] ReceiverData from {}'.format(self.server)
1285 else:
1286 self.server = None
1287 if path == None:
1288 raise ValueError, "[Reading] The path is not valid"
1301
1289
1302 if ext == None:
1290 if ext == None:
1303 ext = self.ext
1291 ext = self.ext
1304
1292
1305 self.verbose=verbose
1293 if online:
1306 self.path = path
1294 print "[Reading] Searching files in online mode..."
1307 self.startDate = startDate
1295
1308 self.endDate = endDate
1296 for nTries in range( self.nTries ):
1309 self.startTime = startTime
1297 fullpath, foldercounter, file, year, doy, set = self.__searchFilesOnLine(path=path, expLabel=expLabel, ext=ext, walk=walk, set=set)
1310 self.endTime = endTime
1298
1311 self.set = set
1299 if fullpath:
1312 self.expLabel = expLabel
1300 break
1313 self.ext = ext
1301
1314 self.online = online
1302 print '[Reading] Waiting %0.2f sec for an valid file in %s: try %02d ...' % (self.delay, path, nTries+1)
1315 self.delay = delay
1303 sleep( self.delay )
1316 self.walk = walk
1304
1317 self.getblock = getblock
1305 if not(fullpath):
1318 self.nTxs = nTxs
1306 print "[Reading] There 'isn't any valid file in %s" % path
1319 self.realtime = realtime
1307 return
1320 self.blocksize = blocksize
1308
1321 self.blocktime = blocktime
1309 self.year = year
1322
1310 self.doy = doy
1323
1311 self.set = set - 1
1324 if self.firstTime is True:
1312 self.path = path
1325 pathList, filenameList = self.__searchFilesOffLine(path, startDate=startDate, endDate=endDate,
1313 self.foldercounter = foldercounter
1314 last_set = None
1315 else:
1316 print "[Reading] Searching files in offline mode ..."
1317 pathList, filenameList = self.searchFilesOffLine(path, startDate=startDate, endDate=endDate,
1326 startTime=startTime, endTime=endTime,
1318 startTime=startTime, endTime=endTime,
1327 set=set, expLabel=expLabel, ext=ext,
1319 set=set, expLabel=expLabel, ext=ext,
1328 walk=walk)
1320 walk=walk, cursor=cursor,
1329 if filenameList is not None: filenameList = filenameList[:-1]
1321 skip=skip)
1330
1322
1331 if pathList is not None and filenameList is not None and online:
1323 if not(pathList):
1332 self.onlineWithDate = True
1333 online = False
1334 self.fileIndex = -1
1324 self.fileIndex = -1
1335 self.pathList = pathList
1325 self.pathList = []
1336 self.filenameList = filenameList
1326 self.filenameList = []
1337 file_name = os.path.basename(filenameList[-1])
1327 return
1338 basename, ext = os.path.splitext(file_name)
1328
1339 last_set = int(basename[-3:])
1340
1341 if online:
1342 print "[Reading] Searching files in online mode..."
1343
1344 for nTries in range(self.nTries):
1345 fullpath, foldercounter, file, year, doy, set = self.__searchFilesOnLine(path=path,
1346 expLabel=expLabel,
1347 ext=ext,
1348 walk=walk,
1349 startDate=startDate,
1350 startTime=startTime,
1351 set=set)
1352
1353 if fullpath:
1354 break
1355 print '[Reading] Waiting %0.2f sec for an valid file in %s: try %02d ...' % (self.delay, path, nTries+1)
1356 sleep( self.delay )
1357
1358 if not(fullpath):
1359 print "[Reading] There 'isn't any valid file in %s" % path
1360 return
1361
1362 self.year = year
1363 self.doy = doy
1364 self.set = set - 1
1365 self.path = path
1366 self.foldercounter = foldercounter
1367 last_set = None
1368 else:
1369 print "[Reading] Searching files in offline mode ..."
1370 pathList, filenameList = self.__searchFilesOffLine(path, startDate=startDate, endDate=endDate,
1371 startTime=startTime, endTime=endTime,
1372 set=set, expLabel=expLabel, ext=ext,
1373 walk=walk)
1374
1375 if not(pathList):
1376 # print "[Reading] No *%s files in %s (%s - %s)"%(ext, path,
1377 # datetime.datetime.combine(startDate,startTime).ctime(),
1378 # datetime.datetime.combine(endDate,endTime).ctime())
1379
1380 # sys.exit(-1)
1381
1382 self.fileIndex = -1
1383 self.pathList = []
1384 self.filenameList = []
1385 return
1386
1387 self.fileIndex = -1
1388 self.pathList = pathList
1389 self.filenameList = filenameList
1390 file_name = os.path.basename(filenameList[-1])
1391 basename, ext = os.path.splitext(file_name)
1392 last_set = int(basename[-3:])
1393
1394
1395 self.online = online
1396 self.realtime = realtime
1397 self.delay = delay
1398 ext = ext.lower()
1399 self.ext = ext
1400 self.getByBlock = getblock
1401 self.nTxs = nTxs
1402 self.startTime = startTime
1403 self.endTime = endTime
1404
1405
1406 #Added-----------------
1407 self.selBlocksize = blocksize
1408 self.selBlocktime = blocktime
1409
1410
1411 if not(self.setNextFile()):
1412 if (startDate!=None) and (endDate!=None):
1413 print "[Reading] No files in range: %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime())
1414 elif startDate != None:
1415 print "[Reading] No files in range: %s" %(datetime.datetime.combine(startDate,startTime).ctime())
1416 else:
1417 print "[Reading] No files"
1418
1419 self.fileIndex = -1
1329 self.fileIndex = -1
1420 self.pathList = []
1330 self.pathList = pathList
1421 self.filenameList = []
1331 self.filenameList = filenameList
1422 return
1332 file_name = os.path.basename(filenameList[-1])
1333 basename, ext = os.path.splitext(file_name)
1334 last_set = int(basename[-3:])
1335
1336 self.online = online
1337 self.realtime = realtime
1338 self.delay = delay
1339 ext = ext.lower()
1340 self.ext = ext
1341 self.getByBlock = getblock
1342 self.nTxs = nTxs
1343 self.startTime = startTime
1344 self.endTime = endTime
1345
1346 #Added-----------------
1347 self.selBlocksize = blocksize
1348 self.selBlocktime = blocktime
1349
1350 # Verbose-----------
1351 self.verbose = verbose
1352 self.warnings = warnings
1353
1354 if not(self.setNextFile()):
1355 if (startDate!=None) and (endDate!=None):
1356 print "[Reading] No files in range: %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime())
1357 elif startDate != None:
1358 print "[Reading] No files in range: %s" %(datetime.datetime.combine(startDate,startTime).ctime())
1359 else:
1360 print "[Reading] No files"
1423
1361
1424 # self.getBasicHeader()
1362 self.fileIndex = -1
1363 self.pathList = []
1364 self.filenameList = []
1365 return
1425
1366
1426 if last_set != None:
1367 # self.getBasicHeader()
1427 self.dataOut.last_block = last_set * self.processingHeaderObj.dataBlocksPerFile + self.basicHeaderObj.dataBlock
1368
1369 if last_set != None:
1370 self.dataOut.last_block = last_set * self.processingHeaderObj.dataBlocksPerFile + self.basicHeaderObj.dataBlock
1428 return
1371 return
1429
1372
1430 def getBasicHeader(self):
1373 def getBasicHeader(self):
@@ -1442,10 +1385,10 class JRODataReader(JRODataIO):
1442 self.dataOut.useLocalTime = self.basicHeaderObj.useLocalTime
1385 self.dataOut.useLocalTime = self.basicHeaderObj.useLocalTime
1443
1386
1444 self.dataOut.ippSeconds = self.radarControllerHeaderObj.ippSeconds/self.nTxs
1387 self.dataOut.ippSeconds = self.radarControllerHeaderObj.ippSeconds/self.nTxs
1445
1388
1446 # self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock*self.nTxs
1389 # self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock*self.nTxs
1447
1390
1448
1391
1449 def getFirstHeader(self):
1392 def getFirstHeader(self):
1450
1393
1451 raise NotImplementedError
1394 raise NotImplementedError
@@ -1517,29 +1460,27 class JRODataReader(JRODataIO):
1517 server=None,
1460 server=None,
1518 verbose=True, **kwargs):
1461 verbose=True, **kwargs):
1519 if not(self.isConfig):
1462 if not(self.isConfig):
1520 # self.dataOut = dataOut
1463 self.setup(path=path,
1521 self.setup( path=path,
1464 startDate=startDate,
1522 startDate=startDate,
1465 endDate=endDate,
1523 endDate=endDate,
1466 startTime=startTime,
1524 startTime=startTime,
1467 endTime=endTime,
1525 endTime=endTime,
1468 set=set,
1526 set=set,
1469 expLabel=expLabel,
1527 expLabel=expLabel,
1470 ext=ext,
1528 ext=ext,
1471 online=online,
1529 online=online,
1472 delay=delay,
1530 delay=delay,
1473 walk=walk,
1531 walk=walk,
1474 getblock=getblock,
1532 getblock=getblock,
1475 nTxs=nTxs,
1533 nTxs=nTxs,
1476 realtime=realtime,
1534 realtime=realtime,
1477 blocksize=blocksize,
1535 blocksize=blocksize,
1478 blocktime=blocktime,
1536 blocktime=blocktime,
1479 skip=skip,
1537 queue=queue,
1480 cursor=cursor,
1538 skip=skip,
1481 warnings=warnings,
1539 cursor=cursor,
1482 server=server,
1540 warnings=warnings,
1483 verbose=verbose)
1541 server=server,
1542 verbose=verbose, **kwargs)
1543 self.isConfig = True
1484 self.isConfig = True
1544 if server is None:
1485 if server is None:
1545 self.getData()
1486 self.getData()
@@ -1662,9 +1603,9 class JRODataWriter(JRODataIO):
1662 Return:
1603 Return:
1663 None
1604 None
1664 """
1605 """
1665
1606
1666 # CALCULAR PARAMETROS
1607 # CALCULAR PARAMETROS
1667
1608
1668 sizeLongHeader = self.systemHeaderObj.size + self.radarControllerHeaderObj.size + self.processingHeaderObj.size
1609 sizeLongHeader = self.systemHeaderObj.size + self.radarControllerHeaderObj.size + self.processingHeaderObj.size
1669 self.basicHeaderObj.size = self.basicHeaderSize + sizeLongHeader
1610 self.basicHeaderObj.size = self.basicHeaderSize + sizeLongHeader
1670
1611
@@ -1791,7 +1732,7 class JRODataWriter(JRODataIO):
1791
1732
1792 return 1
1733 return 1
1793
1734
1794 def setup(self, dataOut, path, blocksPerFile, profilesPerBlock=64, set=None, ext=None, datatype=4, verbose=True):
1735 def setup(self, dataOut, path, blocksPerFile, profilesPerBlock=64, set=None, ext=None, datatype=4):
1795 """
1736 """
1796 Setea el tipo de formato en la cual sera guardada la data y escribe el First Header
1737 Setea el tipo de formato en la cual sera guardada la data y escribe el First Header
1797
1738
Show file before | Show file after | Hide comments
@@ -453,7 +453,7 class FitsReader(ProcessingUnit):
453 # self.blockIndex = 1
453 # self.blockIndex = 1
454 return 1
454 return 1
455
455
456 def __searchFilesOffLine(self,
456 def searchFilesOffLine(self,
457 path,
457 path,
458 startDate,
458 startDate,
459 endDate,
459 endDate,
@@ -559,7 +559,7 class FitsReader(ProcessingUnit):
559
559
560 if not(online):
560 if not(online):
561 print "Searching files in offline mode ..."
561 print "Searching files in offline mode ..."
562 pathList, filenameList = self.__searchFilesOffLine(path, startDate=startDate, endDate=endDate,
562 pathList, filenameList = self.searchFilesOffLine(path, startDate=startDate, endDate=endDate,
563 startTime=startTime, endTime=endTime,
563 startTime=startTime, endTime=endTime,
564 set=set, expLabel=expLabel, ext=ext,
564 set=set, expLabel=expLabel, ext=ext,
565 walk=walk)
565 walk=walk)
Show file before | Show file after | Hide comments
@@ -415,7 +415,7 class HFReader(ProcessingUnit):
415
415
416
416
417
417
418 def __searchFilesOffline(self,
418 def searchFilesOffLine(self,
419 path,
419 path,
420 startDate,
420 startDate,
421 endDate,
421 endDate,
@@ -438,7 +438,7 class HFReader(ProcessingUnit):
438
438
439 return
439 return
440
440
441 def __searchFilesOnline(self,
441 def searchFilesOnLine(self,
442 path,
442 path,
443 expLabel= "",
443 expLabel= "",
444 ext=None,
444 ext=None,
@@ -636,10 +636,10 class HFReader(ProcessingUnit):
636 if not(online):
636 if not(online):
637 print "Searching files in offline mode..."
637 print "Searching files in offline mode..."
638
638
639 self.__searchFilesOffline(path, startDate, endDate, ext, startTime, endTime, walk)
639 self.searchFilesOffLine(path, startDate, endDate, ext, startTime, endTime, walk)
640 else:
640 else:
641 print "Searching files in online mode..."
641 print "Searching files in online mode..."
642 self.__searchFilesOnline(path, walk,ext,set=set)
642 self.searchFilesOnLine(path, walk,ext,set=set)
643 if set==None:
643 if set==None:
644 pass
644 pass
645 else:
645 else:
@@ -647,7 +647,7 class HFReader(ProcessingUnit):
647
647
648 # for nTries in range(self.nTries):
648 # for nTries in range(self.nTries):
649 #
649 #
650 # fullpath,file,year,month,day,set = self.__searchFilesOnline(path=path,expLabel=expLabel,ext=ext, walk=walk,set=set)
650 # fullpath,file,year,month,day,set = self.searchFilesOnLine(path=path,expLabel=expLabel,ext=ext, walk=walk,set=set)
651 #
651 #
652 # if fullpath:
652 # if fullpath:
653 # break
653 # break
Show file before | Show file after | Hide comments
@@ -106,9 +106,9 class AMISRReader(ProcessingUnit):
106 #self.findFiles()
106 #self.findFiles()
107 if not(online):
107 if not(online):
108 #Busqueda de archivos offline
108 #Busqueda de archivos offline
109 self.__searchFilesOffline(path, startDate, endDate, startTime, endTime, walk)
109 self.searchFilesOffLine(path, startDate, endDate, startTime, endTime, walk)
110 else:
110 else:
111 self.__searchFilesOnline(path, startDate, endDate, startTime,endTime,walk)
111 self.searchFilesOnLine(path, startDate, endDate, startTime,endTime,walk)
112
112
113 if not(self.filenameList):
113 if not(self.filenameList):
114 print "There is no files into the folder: %s"%(path)
114 print "There is no files into the folder: %s"%(path)
@@ -329,7 +329,7 class AMISRReader(ProcessingUnit):
329 self.dirnameList = new_dirnameList
329 self.dirnameList = new_dirnameList
330 return 1
330 return 1
331
331
332 def __searchFilesOnline(self, path, startDate, endDate, startTime=datetime.time(0,0,0),
332 def searchFilesOnLine(self, path, startDate, endDate, startTime=datetime.time(0,0,0),
333 endTime=datetime.time(23,59,59),walk=True):
333 endTime=datetime.time(23,59,59),walk=True):
334
334
335 if endDate ==None:
335 if endDate ==None:
@@ -349,7 +349,7 class AMISRReader(ProcessingUnit):
349 return
349 return
350
350
351
351
352 def __searchFilesOffline(self,
352 def searchFilesOffLine(self,
353 path,
353 path,
354 startDate,
354 startDate,
355 endDate,
355 endDate,
Show file before | Show file after | Hide comments
@@ -97,7 +97,7 class ParamReader(ProcessingUnit):
97 self.timezone = 'lt'
97 self.timezone = 'lt'
98
98
99 print "[Reading] Searching files in offline mode ..."
99 print "[Reading] Searching files in offline mode ..."
100 pathList, filenameList = self.__searchFilesOffLine(path, startDate=startDate, endDate=endDate,
100 pathList, filenameList = self.searchFilesOffLine(path, startDate=startDate, endDate=endDate,
101 startTime=startTime, endTime=endTime,
101 startTime=startTime, endTime=endTime,
102 ext=ext, walk=walk)
102 ext=ext, walk=walk)
103
103
@@ -115,7 +115,7 class ParamReader(ProcessingUnit):
115
115
116 return
116 return
117
117
118 def __searchFilesOffLine(self,
118 def searchFilesOffLine(self,
119 path,
119 path,
120 startDate=None,
120 startDate=None,
121 endDate=None,
121 endDate=None,
Show file before | Show file after | Hide comments
This diff has been collapsed as it changes many lines, (2660 lines changed) Show them Hide them
@@ -1,39 +1,79
1 import numpy
1 import numpy
2 import math
2 import math
3 from scipy import optimize, interpolate, signal, stats, ndimage
3 from scipy import optimize, interpolate, signal, stats, ndimage
4 import scipy
4 import re
5 import re
5 import datetime
6 import datetime
6 import copy
7 import copy
7 import sys
8 import sys
8 import importlib
9 import importlib
9 import itertools
10 import itertools
10
11 from multiprocessing import Pool, TimeoutError
12 from multiprocessing.pool import ThreadPool
13 import copy_reg
14 import cPickle
15 import types
16 from functools import partial
17 import time
18 #from sklearn.cluster import KMeans
19
20 import matplotlib.pyplot as plt
21
22 from scipy.optimize import fmin_l_bfgs_b #optimize with bounds on state papameters
11 from jroproc_base import ProcessingUnit, Operation
23 from jroproc_base import ProcessingUnit, Operation
12 from schainpy.model.data.jrodata import Parameters, hildebrand_sekhon
24 from schainpy.model.data.jrodata import Parameters, hildebrand_sekhon
25 from scipy import asarray as ar,exp
26 from scipy.optimize import curve_fit
13
27
28 import warnings
29 from numpy import NaN
30 from scipy.optimize.optimize import OptimizeWarning
31 warnings.filterwarnings('ignore')
14
32
15 class ParametersProc(ProcessingUnit):
16
33
34 SPEED_OF_LIGHT = 299792458
35
36
37 '''solving pickling issue'''
38
39 def _pickle_method(method):
40 func_name = method.im_func.__name__
41 obj = method.im_self
42 cls = method.im_class
43 return _unpickle_method, (func_name, obj, cls)
44
45 def _unpickle_method(func_name, obj, cls):
46 for cls in cls.mro():
47 try:
48 func = cls.__dict__[func_name]
49 except KeyError:
50 pass
51 else:
52 break
53 return func.__get__(obj, cls)
54
55 class ParametersProc(ProcessingUnit):
56
17 nSeconds = None
57 nSeconds = None
18
58
19 def __init__(self):
59 def __init__(self):
20 ProcessingUnit.__init__(self)
60 ProcessingUnit.__init__(self)
21
61
22 # self.objectDict = {}
62 # self.objectDict = {}
23 self.buffer = None
63 self.buffer = None
24 self.firstdatatime = None
64 self.firstdatatime = None
25 self.profIndex = 0
65 self.profIndex = 0
26 self.dataOut = Parameters()
66 self.dataOut = Parameters()
27
67
28 def __updateObjFromInput(self):
68 def __updateObjFromInput(self):
29
69
30 self.dataOut.inputUnit = self.dataIn.type
70 self.dataOut.inputUnit = self.dataIn.type
31
71
32 self.dataOut.timeZone = self.dataIn.timeZone
72 self.dataOut.timeZone = self.dataIn.timeZone
33 self.dataOut.dstFlag = self.dataIn.dstFlag
73 self.dataOut.dstFlag = self.dataIn.dstFlag
34 self.dataOut.errorCount = self.dataIn.errorCount
74 self.dataOut.errorCount = self.dataIn.errorCount
35 self.dataOut.useLocalTime = self.dataIn.useLocalTime
75 self.dataOut.useLocalTime = self.dataIn.useLocalTime
36
76
37 self.dataOut.radarControllerHeaderObj = self.dataIn.radarControllerHeaderObj.copy()
77 self.dataOut.radarControllerHeaderObj = self.dataIn.radarControllerHeaderObj.copy()
38 self.dataOut.systemHeaderObj = self.dataIn.systemHeaderObj.copy()
78 self.dataOut.systemHeaderObj = self.dataIn.systemHeaderObj.copy()
39 self.dataOut.channelList = self.dataIn.channelList
79 self.dataOut.channelList = self.dataIn.channelList
@@ -55,25 +95,25 class ParametersProc(ProcessingUnit):
55 self.dataOut.ippSeconds = self.dataIn.ippSeconds
95 self.dataOut.ippSeconds = self.dataIn.ippSeconds
56 # self.dataOut.windowOfFilter = self.dataIn.windowOfFilter
96 # self.dataOut.windowOfFilter = self.dataIn.windowOfFilter
57 self.dataOut.timeInterval1 = self.dataIn.timeInterval
97 self.dataOut.timeInterval1 = self.dataIn.timeInterval
58 self.dataOut.heightList = self.dataIn.getHeiRange()
98 self.dataOut.heightList = self.dataIn.getHeiRange()
59 self.dataOut.frequency = self.dataIn.frequency
99 self.dataOut.frequency = self.dataIn.frequency
60 #self.dataOut.noise = self.dataIn.noise
100 # self.dataOut.noise = self.dataIn.noise
61
101
62 def run(self):
102 def run(self):
63
103
64 #---------------------- Voltage Data ---------------------------
104 #---------------------- Voltage Data ---------------------------
65
105
66 if self.dataIn.type == "Voltage":
106 if self.dataIn.type == "Voltage":
67
107
68 self.__updateObjFromInput()
108 self.__updateObjFromInput()
69 self.dataOut.data_pre = self.dataIn.data.copy()
109 self.dataOut.data_pre = self.dataIn.data.copy()
70 self.dataOut.flagNoData = False
110 self.dataOut.flagNoData = False
71 self.dataOut.utctimeInit = self.dataIn.utctime
111 self.dataOut.utctimeInit = self.dataIn.utctime
72 self.dataOut.paramInterval = self.dataIn.nProfiles*self.dataIn.nCohInt*self.dataIn.ippSeconds
112 self.dataOut.paramInterval = self.dataIn.nProfiles*self.dataIn.nCohInt*self.dataIn.ippSeconds
73 return
113 return
74
114
75 #---------------------- Spectra Data ---------------------------
115 #---------------------- Spectra Data ---------------------------
76
116
77 if self.dataIn.type == "Spectra":
117 if self.dataIn.type == "Spectra":
78
118
79 self.dataOut.data_pre = (self.dataIn.data_spc, self.dataIn.data_cspc)
119 self.dataOut.data_pre = (self.dataIn.data_spc, self.dataIn.data_cspc)
@@ -83,107 +123,1307 class ParametersProc(ProcessingUnit):
83 self.dataOut.nIncohInt = self.dataIn.nIncohInt
123 self.dataOut.nIncohInt = self.dataIn.nIncohInt
84 self.dataOut.nFFTPoints = self.dataIn.nFFTPoints
124 self.dataOut.nFFTPoints = self.dataIn.nFFTPoints
85 self.dataOut.ippFactor = self.dataIn.ippFactor
125 self.dataOut.ippFactor = self.dataIn.ippFactor
86 #self.dataOut.normFactor = self.dataIn.getNormFactor()
87 self.dataOut.pairsList = self.dataIn.pairsList
88 self.dataOut.groupList = self.dataIn.pairsList
89 self.dataOut.abscissaList = self.dataIn.getVelRange(1)
126 self.dataOut.abscissaList = self.dataIn.getVelRange(1)
127 self.dataOut.spc_noise = self.dataIn.getNoise()
128 self.dataOut.spc_range = (self.dataIn.getFreqRange(1)/1000. , self.dataIn.getAcfRange(1) , self.dataIn.getVelRange(1))
129 self.dataOut.pairsList = self.dataIn.pairsList
130 self.dataOut.groupList = self.dataIn.pairsList
90 self.dataOut.flagNoData = False
131 self.dataOut.flagNoData = False
91
132
133 if hasattr(self.dataIn, 'ChanDist'): #Distances of receiver channels
134 self.dataOut.ChanDist = self.dataIn.ChanDist
135 else: self.dataOut.ChanDist = None
136
137 if hasattr(self.dataIn, 'VelRange'): #Velocities range
138 self.dataOut.VelRange = self.dataIn.VelRange
139 else: self.dataOut.VelRange = None
140
141 if hasattr(self.dataIn, 'RadarConst'): #Radar Constant
142 self.dataOut.RadarConst = self.dataIn.RadarConst
143
144 if hasattr(self.dataIn, 'NPW'): #NPW
145 self.dataOut.NPW = self.dataIn.NPW
146
147 if hasattr(self.dataIn, 'COFA'): #COFA
148 self.dataOut.COFA = self.dataIn.COFA
149
150
151
92 #---------------------- Correlation Data ---------------------------
152 #---------------------- Correlation Data ---------------------------
93
153
94 if self.dataIn.type == "Correlation":
154 if self.dataIn.type == "Correlation":
95 acf_ind, ccf_ind, acf_pairs, ccf_pairs, data_acf, data_ccf = self.dataIn.splitFunctions()
155 acf_ind, ccf_ind, acf_pairs, ccf_pairs, data_acf, data_ccf = self.dataIn.splitFunctions()
96
156
97 self.dataOut.data_pre = (self.dataIn.data_cf[acf_ind,:], self.dataIn.data_cf[ccf_ind,:,:])
157 self.dataOut.data_pre = (self.dataIn.data_cf[acf_ind,:], self.dataIn.data_cf[ccf_ind,:,:])
98 self.dataOut.normFactor = (self.dataIn.normFactor[acf_ind,:], self.dataIn.normFactor[ccf_ind,:])
158 self.dataOut.normFactor = (self.dataIn.normFactor[acf_ind,:], self.dataIn.normFactor[ccf_ind,:])
99 self.dataOut.groupList = (acf_pairs, ccf_pairs)
159 self.dataOut.groupList = (acf_pairs, ccf_pairs)
100
160
101 self.dataOut.abscissaList = self.dataIn.lagRange
161 self.dataOut.abscissaList = self.dataIn.lagRange
102 self.dataOut.noise = self.dataIn.noise
162 self.dataOut.noise = self.dataIn.noise
103 self.dataOut.data_SNR = self.dataIn.SNR
163 self.dataOut.data_SNR = self.dataIn.SNR
104 self.dataOut.flagNoData = False
164 self.dataOut.flagNoData = False
105 self.dataOut.nAvg = self.dataIn.nAvg
165 self.dataOut.nAvg = self.dataIn.nAvg
106
166
107 #---------------------- Parameters Data ---------------------------
167 #---------------------- Parameters Data ---------------------------
108
168
109 if self.dataIn.type == "Parameters":
169 if self.dataIn.type == "Parameters":
110 self.dataOut.copy(self.dataIn)
170 self.dataOut.copy(self.dataIn)
111 self.dataOut.utctimeInit = self.dataIn.utctime
112 self.dataOut.flagNoData = False
171 self.dataOut.flagNoData = False
113
172
114 return True
173 return True
115
174
116 self.__updateObjFromInput()
175 self.__updateObjFromInput()
117 self.dataOut.utctimeInit = self.dataIn.utctime
176 self.dataOut.utctimeInit = self.dataIn.utctime
118 self.dataOut.paramInterval = self.dataIn.timeInterval
177 self.dataOut.paramInterval = self.dataIn.timeInterval
119
178
120 return
179 return
121
180
122 class SpectralMoments(Operation):
123
181
182 def target(tups):
183
184 obj, args = tups
185 #print 'TARGETTT', obj, args
186 return obj.FitGau(args)
187
188 class GaussianFit(Operation):
189
124 '''
190 '''
125 Function SpectralMoments()
191 Function that fit of one and two generalized gaussians (gg) based
192 on the PSD shape across an "power band" identified from a cumsum of
193 the measured spectrum - noise.
194
195 Input:
196 self.dataOut.data_pre : SelfSpectra
197
198 Output:
199 self.dataOut.GauSPC : SPC_ch1, SPC_ch2
200
201 '''
202 def __init__(self, **kwargs):
203 Operation.__init__(self, **kwargs)
204 self.i=0
205
206
207 def run(self, dataOut, num_intg=7, pnoise=1., vel_arr=None, SNRlimit=-9): #num_intg: Incoherent integrations, pnoise: Noise, vel_arr: range of velocities, similar to the ftt points
208 """This routine will find a couple of generalized Gaussians to a power spectrum
209 input: spc
210 output:
211 Amplitude0,shift0,width0,p0,Amplitude1,shift1,width1,p1,noise
212 """
213
214 self.spc = dataOut.data_pre[0].copy()
215
216
217 print 'SelfSpectra Shape', numpy.asarray(self.spc).shape
218
219
220 #plt.figure(50)
221 #plt.subplot(121)
222 #plt.plot(self.spc,'k',label='spc(66)')
223 #plt.plot(xFrec,ySamples[1],'g',label='Ch1')
224 #plt.plot(xFrec,ySamples[2],'r',label='Ch2')
225 #plt.plot(xFrec,FitGauss,'yo:',label='fit')
226 #plt.legend()
227 #plt.title('DATOS A ALTURA DE 7500 METROS')
228 #plt.show()
229
230 self.Num_Hei = self.spc.shape[2]
231 #self.Num_Bin = len(self.spc)
232 self.Num_Bin = self.spc.shape[1]
233 self.Num_Chn = self.spc.shape[0]
234
235 Vrange = dataOut.abscissaList
236
237 #print 'self.spc2', numpy.asarray(self.spc).shape
238
239 GauSPC = numpy.empty([2,self.Num_Bin,self.Num_Hei])
240 SPC_ch1 = numpy.empty([self.Num_Bin,self.Num_Hei])
241 SPC_ch2 = numpy.empty([self.Num_Bin,self.Num_Hei])
242 SPC_ch1[:] = numpy.NaN
243 SPC_ch2[:] = numpy.NaN
126
244
127 Calculates moments (power, mean, standard deviation) and SNR of the signal
245
246 start_time = time.time()
247
248 noise_ = dataOut.spc_noise[0].copy()
249
250
251
252 pool = Pool(processes=self.Num_Chn)
253 args = [(Vrange, Ch, pnoise, noise_, num_intg, SNRlimit) for Ch in range(self.Num_Chn)]
254 objs = [self for __ in range(self.Num_Chn)]
255 attrs = zip(objs, args)
256 gauSPC = pool.map(target, attrs)
257 dataOut.GauSPC = numpy.asarray(gauSPC)
258 # ret = []
259 # for n in range(self.Num_Chn):
260 # self.FitGau(args[n])
261 # dataOut.GauSPC = ret
262
263
264
265 # for ch in range(self.Num_Chn):
266 #
267 # for ht in range(self.Num_Hei):
268 # #print (numpy.asarray(self.spc).shape)
269 # spc = numpy.asarray(self.spc)[ch,:,ht]
270 #
271 # #############################################
272 # # normalizing spc and noise
273 # # This part differs from gg1
274 # spc_norm_max = max(spc)
275 # spc = spc / spc_norm_max
276 # pnoise = pnoise / spc_norm_max
277 # #############################################
278 #
279 # if abs(vel_arr[0])<15.0: # this switch is for spectra collected with different length IPP's
280 # fatspectra=1.0
281 # else:
282 # fatspectra=0.5
283 #
284 # wnoise = noise_ / spc_norm_max
285 # #print 'wnoise', noise_, dataOut.spc_noise[0], wnoise
286 # #wnoise,stdv,i_max,index =enoise(spc,num_intg) #noise estimate using Hildebrand Sekhon, only wnoise is used
287 # #if wnoise>1.1*pnoise: # to be tested later
288 # # wnoise=pnoise
289 # noisebl=wnoise*0.9; noisebh=wnoise*1.1
290 # spc=spc-wnoise
291 #
292 # minx=numpy.argmin(spc)
293 # spcs=numpy.roll(spc,-minx)
294 # cum=numpy.cumsum(spcs)
295 # tot_noise=wnoise * self.Num_Bin #64;
296 # #tot_signal=sum(cum[-5:])/5.; ''' How does this line work? '''
297 # #snr=tot_signal/tot_noise
298 # #snr=cum[-1]/tot_noise
299 #
300 # #print 'spc' , spcs[5:8] , 'tot_noise', tot_noise
301 #
302 # snr = sum(spcs)/tot_noise
303 # snrdB=10.*numpy.log10(snr)
304 #
305 # #if snrdB < -9 :
306 # # snrdB = numpy.NaN
307 # # continue
308 #
309 # #print 'snr',snrdB # , sum(spcs) , tot_noise
310 #
311 #
312 # #if snrdB<-18 or numpy.isnan(snrdB) or num_intg<4:
313 # # return [None,]*4,[None,]*4,None,snrdB,None,None,[None,]*5,[None,]*9,None
314 #
315 # cummax=max(cum); epsi=0.08*fatspectra # cumsum to narrow down the energy region
316 # cumlo=cummax*epsi;
317 # cumhi=cummax*(1-epsi)
318 # powerindex=numpy.array(numpy.where(numpy.logical_and(cum>cumlo, cum<cumhi))[0])
319 #
320 # #if len(powerindex)==1:
321 # ##return [numpy.mod(powerindex[0]+minx,64),None,None,None,],[None,]*4,None,snrdB,None,None,[None,]*5,[None,]*9,None
322 # #return [numpy.mod(powerindex[0]+minx, self.Num_Bin ),None,None,None,],[None,]*4,None,snrdB,None,None,[None,]*5,[None,]*9,None
323 # #elif len(powerindex)<4*fatspectra:
324 # #return [None,]*4,[None,]*4,None,snrdB,None,None,[None,]*5,[None,]*9,None
325 #
326 # if len(powerindex) < 1:# case for powerindex 0
327 # continue
328 # powerlo=powerindex[0]
329 # powerhi=powerindex[-1]
330 # powerwidth=powerhi-powerlo
331 #
332 # firstpeak=powerlo+powerwidth/10.# first gaussian energy location
333 # secondpeak=powerhi-powerwidth/10.#second gaussian energy location
334 # midpeak=(firstpeak+secondpeak)/2.
335 # firstamp=spcs[int(firstpeak)]
336 # secondamp=spcs[int(secondpeak)]
337 # midamp=spcs[int(midpeak)]
338 # #x=numpy.spc.shape[1]
339 #
340 # #x=numpy.arange(64)
341 # x=numpy.arange( self.Num_Bin )
342 # y_data=spc+wnoise
343 #
344 # # single gaussian
345 # #shift0=numpy.mod(midpeak+minx,64)
346 # shift0=numpy.mod(midpeak+minx, self.Num_Bin )
347 # width0=powerwidth/4.#Initialization entire power of spectrum divided by 4
348 # power0=2.
349 # amplitude0=midamp
350 # state0=[shift0,width0,amplitude0,power0,wnoise]
351 # #bnds=((0,63),(1,powerwidth),(0,None),(0.5,3.),(noisebl,noisebh))
352 # bnds=(( 0,(self.Num_Bin-1) ),(1,powerwidth),(0,None),(0.5,3.),(noisebl,noisebh))
353 # #bnds=(( 0,(self.Num_Bin-1) ),(1,powerwidth),(0,None),(0.5,3.),(0.1,0.5))
354 # # bnds = range of fft, power width, amplitude, power, noise
355 # lsq1=fmin_l_bfgs_b(self.misfit1,state0,args=(y_data,x,num_intg),bounds=bnds,approx_grad=True)
356 #
357 # chiSq1=lsq1[1];
358 # jack1= self.y_jacobian1(x,lsq1[0])
359 #
360 #
361 # try:
362 # sigmas1=numpy.sqrt(chiSq1*numpy.diag(numpy.linalg.inv(numpy.dot(jack1.T,jack1))))
363 # except:
364 # std1=32.; sigmas1=numpy.ones(5)
365 # else:
366 # std1=sigmas1[0]
367 #
368 #
369 # if fatspectra<1.0 and powerwidth<4:
370 # choice=0
371 # Amplitude0=lsq1[0][2]
372 # shift0=lsq1[0][0]
373 # width0=lsq1[0][1]
374 # p0=lsq1[0][3]
375 # Amplitude1=0.
376 # shift1=0.
377 # width1=0.
378 # p1=0.
379 # noise=lsq1[0][4]
380 # #return (numpy.array([shift0,width0,Amplitude0,p0]),
381 # # numpy.array([shift1,width1,Amplitude1,p1]),noise,snrdB,chiSq1,6.,sigmas1,[None,]*9,choice)
382 #
383 # # two gaussians
384 # #shift0=numpy.mod(firstpeak+minx,64); shift1=numpy.mod(secondpeak+minx,64)
385 # shift0=numpy.mod(firstpeak+minx, self.Num_Bin );
386 # shift1=numpy.mod(secondpeak+minx, self.Num_Bin )
387 # width0=powerwidth/6.;
388 # width1=width0
389 # power0=2.;
390 # power1=power0
391 # amplitude0=firstamp;
392 # amplitude1=secondamp
393 # state0=[shift0,width0,amplitude0,power0,shift1,width1,amplitude1,power1,wnoise]
394 # #bnds=((0,63),(1,powerwidth/2.),(0,None),(0.5,3.),(0,63),(1,powerwidth/2.),(0,None),(0.5,3.),(noisebl,noisebh))
395 # bnds=(( 0,(self.Num_Bin-1) ),(1,powerwidth/2.),(0,None),(0.5,3.),( 0,(self.Num_Bin-1)),(1,powerwidth/2.),(0,None),(0.5,3.),(noisebl,noisebh))
396 # #bnds=(( 0,(self.Num_Bin-1) ),(1,powerwidth/2.),(0,None),(0.5,3.),( 0,(self.Num_Bin-1)),(1,powerwidth/2.),(0,None),(0.5,3.),(0.1,0.5))
397 #
398 # lsq2=fmin_l_bfgs_b(self.misfit2,state0,args=(y_data,x,num_intg),bounds=bnds,approx_grad=True)
399 #
400 #
401 # chiSq2=lsq2[1]; jack2=self.y_jacobian2(x,lsq2[0])
402 #
403 #
404 # try:
405 # sigmas2=numpy.sqrt(chiSq2*numpy.diag(numpy.linalg.inv(numpy.dot(jack2.T,jack2))))
406 # except:
407 # std2a=32.; std2b=32.; sigmas2=numpy.ones(9)
408 # else:
409 # std2a=sigmas2[0]; std2b=sigmas2[4]
410 #
411 #
412 #
413 # oneG=(chiSq1<5 and chiSq1/chiSq2<2.0) and (abs(lsq2[0][0]-lsq2[0][4])<(lsq2[0][1]+lsq2[0][5])/3. or abs(lsq2[0][0]-lsq2[0][4])<10)
414 #
415 # if snrdB>-9: # when SNR is strong pick the peak with least shift (LOS velocity) error
416 # if oneG:
417 # choice=0
418 # else:
419 # w1=lsq2[0][1]; w2=lsq2[0][5]
420 # a1=lsq2[0][2]; a2=lsq2[0][6]
421 # p1=lsq2[0][3]; p2=lsq2[0][7]
422 # s1=(2**(1+1./p1))*scipy.special.gamma(1./p1)/p1; s2=(2**(1+1./p2))*scipy.special.gamma(1./p2)/p2;
423 # gp1=a1*w1*s1; gp2=a2*w2*s2 # power content of each ggaussian with proper p scaling
424 #
425 # if gp1>gp2:
426 # if a1>0.7*a2:
427 # choice=1
428 # else:
429 # choice=2
430 # elif gp2>gp1:
431 # if a2>0.7*a1:
432 # choice=2
433 # else:
434 # choice=1
435 # else:
436 # choice=numpy.argmax([a1,a2])+1
437 # #else:
438 # #choice=argmin([std2a,std2b])+1
439 #
440 # else: # with low SNR go to the most energetic peak
441 # choice=numpy.argmax([lsq1[0][2]*lsq1[0][1],lsq2[0][2]*lsq2[0][1],lsq2[0][6]*lsq2[0][5]])
442 #
443 # #print 'choice',choice
444 #
445 # if choice==0: # pick the single gaussian fit
446 # Amplitude0=lsq1[0][2]
447 # shift0=lsq1[0][0]
448 # width0=lsq1[0][1]
449 # p0=lsq1[0][3]
450 # Amplitude1=0.
451 # shift1=0.
452 # width1=0.
453 # p1=0.
454 # noise=lsq1[0][4]
455 # elif choice==1: # take the first one of the 2 gaussians fitted
456 # Amplitude0 = lsq2[0][2]
457 # shift0 = lsq2[0][0]
458 # width0 = lsq2[0][1]
459 # p0 = lsq2[0][3]
460 # Amplitude1 = lsq2[0][6] # This is 0 in gg1
461 # shift1 = lsq2[0][4] # This is 0 in gg1
462 # width1 = lsq2[0][5] # This is 0 in gg1
463 # p1 = lsq2[0][7] # This is 0 in gg1
464 # noise = lsq2[0][8]
465 # else: # the second one
466 # Amplitude0 = lsq2[0][6]
467 # shift0 = lsq2[0][4]
468 # width0 = lsq2[0][5]
469 # p0 = lsq2[0][7]
470 # Amplitude1 = lsq2[0][2] # This is 0 in gg1
471 # shift1 = lsq2[0][0] # This is 0 in gg1
472 # width1 = lsq2[0][1] # This is 0 in gg1
473 # p1 = lsq2[0][3] # This is 0 in gg1
474 # noise = lsq2[0][8]
475 #
476 # #print len(noise + Amplitude0*numpy.exp(-0.5*(abs(x-shift0))/width0)**p0)
477 # SPC_ch1[:,ht] = noise + Amplitude0*numpy.exp(-0.5*(abs(x-shift0))/width0)**p0
478 # SPC_ch2[:,ht] = noise + Amplitude1*numpy.exp(-0.5*(abs(x-shift1))/width1)**p1
479 # #print 'SPC_ch1.shape',SPC_ch1.shape
480 # #print 'SPC_ch2.shape',SPC_ch2.shape
481 # #dataOut.data_param = SPC_ch1
482 # GauSPC[0] = SPC_ch1
483 # GauSPC[1] = SPC_ch2
484
485 # #plt.gcf().clear()
486 # plt.figure(50+self.i)
487 # self.i=self.i+1
488 # #plt.subplot(121)
489 # plt.plot(self.spc,'k')#,label='spc(66)')
490 # plt.plot(SPC_ch1[ch,ht],'b')#,label='gg1')
491 # #plt.plot(SPC_ch2,'r')#,label='gg2')
492 # #plt.plot(xFrec,ySamples[1],'g',label='Ch1')
493 # #plt.plot(xFrec,ySamples[2],'r',label='Ch2')
494 # #plt.plot(xFrec,FitGauss,'yo:',label='fit')
495 # plt.legend()
496 # plt.title('DATOS A ALTURA DE 7500 METROS')
497 # plt.show()
498 # print 'shift0', shift0
499 # print 'Amplitude0', Amplitude0
500 # print 'width0', width0
501 # print 'p0', p0
502 # print '========================'
503 # print 'shift1', shift1
504 # print 'Amplitude1', Amplitude1
505 # print 'width1', width1
506 # print 'p1', p1
507 # print 'noise', noise
508 # print 's_noise', wnoise
509
510 print '========================================================'
511 print 'total_time: ', time.time()-start_time
512
513 # re-normalizing spc and noise
514 # This part differs from gg1
515
516
517
518 ''' Parameters:
519 1. Amplitude
520 2. Shift
521 3. Width
522 4. Power
523 '''
524
525
526 ###############################################################################
527 def FitGau(self, X):
528
529 Vrange, ch, pnoise, noise_, num_intg, SNRlimit = X
530 #print 'VARSSSS', ch, pnoise, noise, num_intg
531
532 #print 'HEIGHTS', self.Num_Hei
533
534 GauSPC = []
535 SPC_ch1 = numpy.empty([self.Num_Bin,self.Num_Hei])
536 SPC_ch2 = numpy.empty([self.Num_Bin,self.Num_Hei])
537 SPC_ch1[:] = 0#numpy.NaN
538 SPC_ch2[:] = 0#numpy.NaN
539
540
541
542 for ht in range(self.Num_Hei):
543 #print (numpy.asarray(self.spc).shape)
544
545 #print 'TTTTT', ch , ht
546 #print self.spc.shape
547
548
549 spc = numpy.asarray(self.spc)[ch,:,ht]
550
551 #############################################
552 # normalizing spc and noise
553 # This part differs from gg1
554 spc_norm_max = max(spc)
555 spc = spc / spc_norm_max
556 pnoise = pnoise / spc_norm_max
557 #############################################
558
559 fatspectra=1.0
560
561 wnoise = noise_ / spc_norm_max
562 #wnoise,stdv,i_max,index =enoise(spc,num_intg) #noise estimate using Hildebrand Sekhon, only wnoise is used
563 #if wnoise>1.1*pnoise: # to be tested later
564 # wnoise=pnoise
565 noisebl=wnoise*0.9; noisebh=wnoise*1.1
566 spc=spc-wnoise
567 # print 'wnoise', noise_[0], spc_norm_max, wnoise
568 minx=numpy.argmin(spc)
569 spcs=numpy.roll(spc,-minx)
570 cum=numpy.cumsum(spcs)
571 tot_noise=wnoise * self.Num_Bin #64;
572 #print 'spc' , spcs[5:8] , 'tot_noise', tot_noise
573 #tot_signal=sum(cum[-5:])/5.; ''' How does this line work? '''
574 #snr=tot_signal/tot_noise
575 #snr=cum[-1]/tot_noise
576 snr = sum(spcs)/tot_noise
577 snrdB=10.*numpy.log10(snr)
578
579 if snrdB < SNRlimit :
580 snr = numpy.NaN
581 SPC_ch1[:,ht] = 0#numpy.NaN
582 SPC_ch1[:,ht] = 0#numpy.NaN
583 GauSPC = (SPC_ch1,SPC_ch2)
584 continue
585 #print 'snr',snrdB #, sum(spcs) , tot_noise
586
587
588
589 #if snrdB<-18 or numpy.isnan(snrdB) or num_intg<4:
590 # return [None,]*4,[None,]*4,None,snrdB,None,None,[None,]*5,[None,]*9,None
591
592 cummax=max(cum); epsi=0.08*fatspectra # cumsum to narrow down the energy region
593 cumlo=cummax*epsi;
594 cumhi=cummax*(1-epsi)
595 powerindex=numpy.array(numpy.where(numpy.logical_and(cum>cumlo, cum<cumhi))[0])
596
597
598 if len(powerindex) < 1:# case for powerindex 0
599 continue
600 powerlo=powerindex[0]
601 powerhi=powerindex[-1]
602 powerwidth=powerhi-powerlo
603
604 firstpeak=powerlo+powerwidth/10.# first gaussian energy location
605 secondpeak=powerhi-powerwidth/10.#second gaussian energy location
606 midpeak=(firstpeak+secondpeak)/2.
607 firstamp=spcs[int(firstpeak)]
608 secondamp=spcs[int(secondpeak)]
609 midamp=spcs[int(midpeak)]
610
611 x=numpy.arange( self.Num_Bin )
612 y_data=spc+wnoise
613
614 # single gaussian
615 shift0=numpy.mod(midpeak+minx, self.Num_Bin )
616 width0=powerwidth/4.#Initialization entire power of spectrum divided by 4
617 power0=2.
618 amplitude0=midamp
619 state0=[shift0,width0,amplitude0,power0,wnoise]
620 bnds=(( 0,(self.Num_Bin-1) ),(1,powerwidth),(0,None),(0.5,3.),(noisebl,noisebh))
621 lsq1=fmin_l_bfgs_b(self.misfit1,state0,args=(y_data,x,num_intg),bounds=bnds,approx_grad=True)
622
623 chiSq1=lsq1[1];
624 jack1= self.y_jacobian1(x,lsq1[0])
625
626
627 try:
628 sigmas1=numpy.sqrt(chiSq1*numpy.diag(numpy.linalg.inv(numpy.dot(jack1.T,jack1))))
629 except:
630 std1=32.; sigmas1=numpy.ones(5)
631 else:
632 std1=sigmas1[0]
633
634
635 if fatspectra<1.0 and powerwidth<4:
636 choice=0
637 Amplitude0=lsq1[0][2]
638 shift0=lsq1[0][0]
639 width0=lsq1[0][1]
640 p0=lsq1[0][3]
641 Amplitude1=0.
642 shift1=0.
643 width1=0.
644 p1=0.
645 noise=lsq1[0][4]
646 #return (numpy.array([shift0,width0,Amplitude0,p0]),
647 # numpy.array([shift1,width1,Amplitude1,p1]),noise,snrdB,chiSq1,6.,sigmas1,[None,]*9,choice)
648
649 # two gaussians
650 #shift0=numpy.mod(firstpeak+minx,64); shift1=numpy.mod(secondpeak+minx,64)
651 shift0=numpy.mod(firstpeak+minx, self.Num_Bin );
652 shift1=numpy.mod(secondpeak+minx, self.Num_Bin )
653 width0=powerwidth/6.;
654 width1=width0
655 power0=2.;
656 power1=power0
657 amplitude0=firstamp;
658 amplitude1=secondamp
659 state0=[shift0,width0,amplitude0,power0,shift1,width1,amplitude1,power1,wnoise]
660 #bnds=((0,63),(1,powerwidth/2.),(0,None),(0.5,3.),(0,63),(1,powerwidth/2.),(0,None),(0.5,3.),(noisebl,noisebh))
661 bnds=(( 0,(self.Num_Bin-1) ),(1,powerwidth/2.),(0,None),(0.5,3.),( 0,(self.Num_Bin-1)),(1,powerwidth/2.),(0,None),(0.5,3.),(noisebl,noisebh))
662 #bnds=(( 0,(self.Num_Bin-1) ),(1,powerwidth/2.),(0,None),(0.5,3.),( 0,(self.Num_Bin-1)),(1,powerwidth/2.),(0,None),(0.5,3.),(0.1,0.5))
663
664 lsq2=fmin_l_bfgs_b(self.misfit2,state0,args=(y_data,x,num_intg),bounds=bnds,approx_grad=True)
665
666
667 chiSq2=lsq2[1]; jack2=self.y_jacobian2(x,lsq2[0])
668
669
670 try:
671 sigmas2=numpy.sqrt(chiSq2*numpy.diag(numpy.linalg.inv(numpy.dot(jack2.T,jack2))))
672 except:
673 std2a=32.; std2b=32.; sigmas2=numpy.ones(9)
674 else:
675 std2a=sigmas2[0]; std2b=sigmas2[4]
676
677
678
679 oneG=(chiSq1<5 and chiSq1/chiSq2<2.0) and (abs(lsq2[0][0]-lsq2[0][4])<(lsq2[0][1]+lsq2[0][5])/3. or abs(lsq2[0][0]-lsq2[0][4])<10)
680
681 if snrdB>-6: # when SNR is strong pick the peak with least shift (LOS velocity) error
682 if oneG:
683 choice=0
684 else:
685 w1=lsq2[0][1]; w2=lsq2[0][5]
686 a1=lsq2[0][2]; a2=lsq2[0][6]
687 p1=lsq2[0][3]; p2=lsq2[0][7]
688 s1=(2**(1+1./p1))*scipy.special.gamma(1./p1)/p1;
689 s2=(2**(1+1./p2))*scipy.special.gamma(1./p2)/p2;
690 gp1=a1*w1*s1; gp2=a2*w2*s2 # power content of each ggaussian with proper p scaling
691
692 if gp1>gp2:
693 if a1>0.7*a2:
694 choice=1
695 else:
696 choice=2
697 elif gp2>gp1:
698 if a2>0.7*a1:
699 choice=2
700 else:
701 choice=1
702 else:
703 choice=numpy.argmax([a1,a2])+1
704 #else:
705 #choice=argmin([std2a,std2b])+1
706
707 else: # with low SNR go to the most energetic peak
708 choice=numpy.argmax([lsq1[0][2]*lsq1[0][1],lsq2[0][2]*lsq2[0][1],lsq2[0][6]*lsq2[0][5]])
709
710
711 shift0=lsq2[0][0]; vel0=Vrange[0] + shift0*(Vrange[1]-Vrange[0])
712 shift1=lsq2[0][4]; vel1=Vrange[0] + shift1*(Vrange[1]-Vrange[0])
713
714 max_vel = 20
715
716 #first peak will be 0, second peak will be 1
717 if vel0 > 0 and vel0 < max_vel : #first peak is in the correct range
718 shift0=lsq2[0][0]
719 width0=lsq2[0][1]
720 Amplitude0=lsq2[0][2]
721 p0=lsq2[0][3]
722
723 shift1=lsq2[0][4]
724 width1=lsq2[0][5]
725 Amplitude1=lsq2[0][6]
726 p1=lsq2[0][7]
727 noise=lsq2[0][8]
728 else:
729 shift1=lsq2[0][0]
730 width1=lsq2[0][1]
731 Amplitude1=lsq2[0][2]
732 p1=lsq2[0][3]
733
734 shift0=lsq2[0][4]
735 width0=lsq2[0][5]
736 Amplitude0=lsq2[0][6]
737 p0=lsq2[0][7]
738 noise=lsq2[0][8]
739
740 if Amplitude0<0.1: # in case the peak is noise
741 shift0,width0,Amplitude0,p0 = 4*[numpy.NaN]
742 if Amplitude1<0.1:
743 shift1,width1,Amplitude1,p1 = 4*[numpy.NaN]
744
745
746 # if choice==0: # pick the single gaussian fit
747 # Amplitude0=lsq1[0][2]
748 # shift0=lsq1[0][0]
749 # width0=lsq1[0][1]
750 # p0=lsq1[0][3]
751 # Amplitude1=0.
752 # shift1=0.
753 # width1=0.
754 # p1=0.
755 # noise=lsq1[0][4]
756 # elif choice==1: # take the first one of the 2 gaussians fitted
757 # Amplitude0 = lsq2[0][2]
758 # shift0 = lsq2[0][0]
759 # width0 = lsq2[0][1]
760 # p0 = lsq2[0][3]
761 # Amplitude1 = lsq2[0][6] # This is 0 in gg1
762 # shift1 = lsq2[0][4] # This is 0 in gg1
763 # width1 = lsq2[0][5] # This is 0 in gg1
764 # p1 = lsq2[0][7] # This is 0 in gg1
765 # noise = lsq2[0][8]
766 # else: # the second one
767 # Amplitude0 = lsq2[0][6]
768 # shift0 = lsq2[0][4]
769 # width0 = lsq2[0][5]
770 # p0 = lsq2[0][7]
771 # Amplitude1 = lsq2[0][2] # This is 0 in gg1
772 # shift1 = lsq2[0][0] # This is 0 in gg1
773 # width1 = lsq2[0][1] # This is 0 in gg1
774 # p1 = lsq2[0][3] # This is 0 in gg1
775 # noise = lsq2[0][8]
776
777 #print len(noise + Amplitude0*numpy.exp(-0.5*(abs(x-shift0))/width0)**p0)
778 SPC_ch1[:,ht] = noise + Amplitude0*numpy.exp(-0.5*(abs(x-shift0))/width0)**p0
779 SPC_ch2[:,ht] = noise + Amplitude1*numpy.exp(-0.5*(abs(x-shift1))/width1)**p1
780 #print 'SPC_ch1.shape',SPC_ch1.shape
781 #print 'SPC_ch2.shape',SPC_ch2.shape
782 #dataOut.data_param = SPC_ch1
783 GauSPC = (SPC_ch1,SPC_ch2)
784 #GauSPC[1] = SPC_ch2
785
786 # print 'shift0', shift0
787 # print 'Amplitude0', Amplitude0
788 # print 'width0', width0
789 # print 'p0', p0
790 # print '========================'
791 # print 'shift1', shift1
792 # print 'Amplitude1', Amplitude1
793 # print 'width1', width1
794 # print 'p1', p1
795 # print 'noise', noise
796 # print 's_noise', wnoise
797
798 return GauSPC
799
800
801 def y_jacobian1(self,x,state): # This function is for further analysis of generalized Gaussians, it is not too importan for the signal discrimination.
802 y_model=self.y_model1(x,state)
803 s0,w0,a0,p0,n=state
804 e0=((x-s0)/w0)**2;
805
806 e0u=((x-s0-self.Num_Bin)/w0)**2;
807
808 e0d=((x-s0+self.Num_Bin)/w0)**2
809 m0=numpy.exp(-0.5*e0**(p0/2.));
810 m0u=numpy.exp(-0.5*e0u**(p0/2.));
811 m0d=numpy.exp(-0.5*e0d**(p0/2.))
812 JA=m0+m0u+m0d
813 JP=(-1/4.)*a0*m0*e0**(p0/2.)*numpy.log(e0)+(-1/4.)*a0*m0u*e0u**(p0/2.)*numpy.log(e0u)+(-1/4.)*a0*m0d*e0d**(p0/2.)*numpy.log(e0d)
814
815 JS=(p0/w0/2.)*a0*m0*e0**(p0/2.-1)*((x-s0)/w0)+(p0/w0/2.)*a0*m0u*e0u**(p0/2.-1)*((x-s0- self.Num_Bin )/w0)+(p0/w0/2.)*a0*m0d*e0d**(p0/2.-1)*((x-s0+ self.Num_Bin )/w0)
816
817 JW=(p0/w0/2.)*a0*m0*e0**(p0/2.-1)*((x-s0)/w0)**2+(p0/w0/2.)*a0*m0u*e0u**(p0/2.-1)*((x-s0- self.Num_Bin )/w0)**2+(p0/w0/2.)*a0*m0d*e0d**(p0/2.-1)*((x-s0+ self.Num_Bin )/w0)**2
818 jack1=numpy.sqrt(7)*numpy.array([JS/y_model,JW/y_model,JA/y_model,JP/y_model,1./y_model])
819 return jack1.T
820
821 def y_jacobian2(self,x,state):
822 y_model=self.y_model2(x,state)
823 s0,w0,a0,p0,s1,w1,a1,p1,n=state
824 e0=((x-s0)/w0)**2;
825
826 e0u=((x-s0- self.Num_Bin )/w0)**2;
827
828 e0d=((x-s0+ self.Num_Bin )/w0)**2
829 e1=((x-s1)/w1)**2;
830
831 e1u=((x-s1- self.Num_Bin )/w1)**2;
832
833 e1d=((x-s1+ self.Num_Bin )/w1)**2
834 m0=numpy.exp(-0.5*e0**(p0/2.));
835 m0u=numpy.exp(-0.5*e0u**(p0/2.));
836 m0d=numpy.exp(-0.5*e0d**(p0/2.))
837 m1=numpy.exp(-0.5*e1**(p1/2.));
838 m1u=numpy.exp(-0.5*e1u**(p1/2.));
839 m1d=numpy.exp(-0.5*e1d**(p1/2.))
840 JA=m0+m0u+m0d
841 JA1=m1+m1u+m1d
842 JP=(-1/4.)*a0*m0*e0**(p0/2.)*numpy.log(e0)+(-1/4.)*a0*m0u*e0u**(p0/2.)*numpy.log(e0u)+(-1/4.)*a0*m0d*e0d**(p0/2.)*numpy.log(e0d)
843 JP1=(-1/4.)*a1*m1*e1**(p1/2.)*numpy.log(e1)+(-1/4.)*a1*m1u*e1u**(p1/2.)*numpy.log(e1u)+(-1/4.)*a1*m1d*e1d**(p1/2.)*numpy.log(e1d)
844
845 JS=(p0/w0/2.)*a0*m0*e0**(p0/2.-1)*((x-s0)/w0)+(p0/w0/2.)*a0*m0u*e0u**(p0/2.-1)*((x-s0- self.Num_Bin )/w0)+(p0/w0/2.)*a0*m0d*e0d**(p0/2.-1)*((x-s0+ self.Num_Bin )/w0)
846
847 JS1=(p1/w1/2.)*a1*m1*e1**(p1/2.-1)*((x-s1)/w1)+(p1/w1/2.)*a1*m1u*e1u**(p1/2.-1)*((x-s1- self.Num_Bin )/w1)+(p1/w1/2.)*a1*m1d*e1d**(p1/2.-1)*((x-s1+ self.Num_Bin )/w1)
848
849 JW=(p0/w0/2.)*a0*m0*e0**(p0/2.-1)*((x-s0)/w0)**2+(p0/w0/2.)*a0*m0u*e0u**(p0/2.-1)*((x-s0- self.Num_Bin )/w0)**2+(p0/w0/2.)*a0*m0d*e0d**(p0/2.-1)*((x-s0+ self.Num_Bin )/w0)**2
850
851 JW1=(p1/w1/2.)*a1*m1*e1**(p1/2.-1)*((x-s1)/w1)**2+(p1/w1/2.)*a1*m1u*e1u**(p1/2.-1)*((x-s1- self.Num_Bin )/w1)**2+(p1/w1/2.)*a1*m1d*e1d**(p1/2.-1)*((x-s1+ self.Num_Bin )/w1)**2
852 jack2=numpy.sqrt(7)*numpy.array([JS/y_model,JW/y_model,JA/y_model,JP/y_model,JS1/y_model,JW1/y_model,JA1/y_model,JP1/y_model,1./y_model])
853 return jack2.T
854
855 def y_model1(self,x,state):
856 shift0,width0,amplitude0,power0,noise=state
857 model0=amplitude0*numpy.exp(-0.5*abs((x-shift0)/width0)**power0)
858
859 model0u=amplitude0*numpy.exp(-0.5*abs((x-shift0- self.Num_Bin )/width0)**power0)
860
861 model0d=amplitude0*numpy.exp(-0.5*abs((x-shift0+ self.Num_Bin )/width0)**power0)
862 return model0+model0u+model0d+noise
863
864 def y_model2(self,x,state): #Equation for two generalized Gaussians with Nyquist
865 shift0,width0,amplitude0,power0,shift1,width1,amplitude1,power1,noise=state
866 model0=amplitude0*numpy.exp(-0.5*abs((x-shift0)/width0)**power0)
867
868 model0u=amplitude0*numpy.exp(-0.5*abs((x-shift0- self.Num_Bin )/width0)**power0)
869
870 model0d=amplitude0*numpy.exp(-0.5*abs((x-shift0+ self.Num_Bin )/width0)**power0)
871 model1=amplitude1*numpy.exp(-0.5*abs((x-shift1)/width1)**power1)
872
873 model1u=amplitude1*numpy.exp(-0.5*abs((x-shift1- self.Num_Bin )/width1)**power1)
874
875 model1d=amplitude1*numpy.exp(-0.5*abs((x-shift1+ self.Num_Bin )/width1)**power1)
876 return model0+model0u+model0d+model1+model1u+model1d+noise
877
878 def misfit1(self,state,y_data,x,num_intg): # This function compares how close real data is with the model data, the close it is, the better it is.
128
879
129 Type of dataIn: Spectra
880 return num_intg*sum((numpy.log(y_data)-numpy.log(self.y_model1(x,state)))**2)#/(64-5.) # /(64-5.) can be commented
881
882 def misfit2(self,state,y_data,x,num_intg):
883 return num_intg*sum((numpy.log(y_data)-numpy.log(self.y_model2(x,state)))**2)#/(64-9.)
884
130
885
131 Configuration Parameters:
886 class PrecipitationProc(Operation):
887
888 '''
889 Operator that estimates Reflectivity factor (Z), and estimates rainfall Rate (R)
890
891 Input:
892 self.dataOut.data_pre : SelfSpectra
893
894 Output:
895
896 self.dataOut.data_output : Reflectivity factor, rainfall Rate
897
898
899 Parameters affected:
900 '''
901
902
903 def run(self, dataOut, radar=None, Pt=None, Gt=None, Gr=None, Lambda=None, aL=None,
904 tauW=None, ThetaT=None, ThetaR=None, Km = 0.93, Altitude=None):
905
906 self.spc = dataOut.data_pre[0].copy()
907 self.Num_Hei = self.spc.shape[2]
908 self.Num_Bin = self.spc.shape[1]
909 self.Num_Chn = self.spc.shape[0]
910
911 Velrange = dataOut.abscissaList
912
913 if radar == "MIRA35C" :
914
915 Ze = self.dBZeMODE2(dataOut)
916
917 else:
918
919 self.Pt = Pt
920 self.Gt = Gt
921 self.Gr = Gr
922 self.Lambda = Lambda
923 self.aL = aL
924 self.tauW = tauW
925 self.ThetaT = ThetaT
926 self.ThetaR = ThetaR
927
928 RadarConstant = GetRadarConstant()
929 SPCmean = numpy.mean(self.spc,0)
930 ETA = numpy.zeros(self.Num_Hei)
931 Pr = numpy.sum(SPCmean,0)
932
933 #for R in range(self.Num_Hei):
934 # ETA[R] = RadarConstant * Pr[R] * R**2 #Reflectivity (ETA)
935
936 D_range = numpy.zeros(self.Num_Hei)
937 EqSec = numpy.zeros(self.Num_Hei)
938 del_V = numpy.zeros(self.Num_Hei)
939
940 for R in range(self.Num_Hei):
941 ETA[R] = RadarConstant * Pr[R] * R**2 #Reflectivity (ETA)
942
943 h = R + Altitude #Range from ground to radar pulse altitude
944 del_V[R] = 1 + 3.68 * 10**-5 * h + 1.71 * 10**-9 * h**2 #Density change correction for velocity
945
946 D_range[R] = numpy.log( (9.65 - (Velrange[R]/del_V[R])) / 10.3 ) / -0.6 #Range of Diameter of drops related to velocity
947 SIGMA[R] = numpy.pi**5 / Lambda**4 * Km * D_range[R]**6 #Equivalent Section of drops (sigma)
948
949 N_dist[R] = ETA[R] / SIGMA[R]
950
951 Ze = (ETA * Lambda**4) / (numpy.pi * Km)
952 Z = numpy.sum( N_dist * D_range**6 )
953 RR = 6*10**-4*numpy.pi * numpy.sum( D_range**3 * N_dist * Velrange ) #Rainfall rate
954
955
956 RR = (Ze/200)**(1/1.6)
957 dBRR = 10*numpy.log10(RR)
958
959 dBZe = 10*numpy.log10(Ze)
960 dataOut.data_output = Ze
961 dataOut.data_param = numpy.ones([2,self.Num_Hei])
962 dataOut.channelList = [0,1]
963 print 'channelList', dataOut.channelList
964 dataOut.data_param[0]=dBZe
965 dataOut.data_param[1]=dBRR
966 print 'RR SHAPE', dBRR.shape
967 print 'Ze SHAPE', dBZe.shape
968 print 'dataOut.data_param SHAPE', dataOut.data_param.shape
969
970
971 def dBZeMODE2(self, dataOut): # Processing for MIRA35C
972
973 NPW = dataOut.NPW
974 COFA = dataOut.COFA
975
976 SNR = numpy.array([self.spc[0,:,:] / NPW[0]]) #, self.spc[1,:,:] / NPW[1]])
977 RadarConst = dataOut.RadarConst
978 #frequency = 34.85*10**9
979
980 ETA = numpy.zeros(([self.Num_Chn ,self.Num_Hei]))
981 data_output = numpy.ones([self.Num_Chn , self.Num_Hei])*numpy.NaN
982
983 ETA = numpy.sum(SNR,1)
984 print 'ETA' , ETA
985 ETA = numpy.where(ETA is not 0. , ETA, numpy.NaN)
986
987 Ze = numpy.ones([self.Num_Chn, self.Num_Hei] )
988
989 for r in range(self.Num_Hei):
990
991 Ze[0,r] = ( ETA[0,r] ) * COFA[0,r][0] * RadarConst * ((r/5000.)**2)
992 #Ze[1,r] = ( ETA[1,r] ) * COFA[1,r][0] * RadarConst * ((r/5000.)**2)
993
994 return Ze
995
996 def GetRadarConstant(self):
997
998 """
999 Constants:
1000
1001 Pt: Transmission Power dB
1002 Gt: Transmission Gain dB
1003 Gr: Reception Gain dB
1004 Lambda: Wavelenght m
1005 aL: Attenuation loses dB
1006 tauW: Width of transmission pulse s
1007 ThetaT: Transmission antenna bean angle rad
1008 ThetaR: Reception antenna beam angle rad
1009
1010 """
1011 Numerator = ( (4*numpy.pi)**3 * aL**2 * 16 * numpy.log(2) )
1012 Denominator = ( Pt * Gt * Gr * Lambda**2 * SPEED_OF_LIGHT * TauW * numpy.pi * ThetaT * TheraR)
1013 RadarConstant = Numerator / Denominator
1014
1015 return RadarConstant
1016
1017
1018
1019 class FullSpectralAnalysis(Operation):
1020
1021 """
1022 Function that implements Full Spectral Analisys technique.
1023
1024 Input:
1025 self.dataOut.data_pre : SelfSpectra and CrossSPectra data
1026 self.dataOut.groupList : Pairlist of channels
1027 self.dataOut.ChanDist : Physical distance between receivers
1028
1029
1030 Output:
1031
1032 self.dataOut.data_output : Zonal wind, Meridional wind and Vertical wind
1033
1034
1035 Parameters affected: Winds, height range, SNR
1036
1037 """
1038 def run(self, dataOut, E01=None, E02=None, E12=None, N01=None, N02=None, N12=None, SNRlimit=7):
1039
1040 spc = dataOut.data_pre[0].copy()
1041 cspc = dataOut.data_pre[1].copy()
1042
1043 nChannel = spc.shape[0]
1044 nProfiles = spc.shape[1]
1045 nHeights = spc.shape[2]
1046
1047 pairsList = dataOut.groupList
1048 if dataOut.ChanDist is not None :
1049 ChanDist = dataOut.ChanDist
1050 else:
1051 ChanDist = numpy.array([[E01, N01],[E02,N02],[E12,N12]])
1052
1053 #print 'ChanDist', ChanDist
1054
1055 if dataOut.VelRange is not None:
1056 VelRange= dataOut.VelRange
1057 else:
1058 VelRange= dataOut.abscissaList
1059
1060 ySamples=numpy.ones([nChannel,nProfiles])
1061 phase=numpy.ones([nChannel,nProfiles])
1062 CSPCSamples=numpy.ones([nChannel,nProfiles],dtype=numpy.complex_)
1063 coherence=numpy.ones([nChannel,nProfiles])
1064 PhaseSlope=numpy.ones(nChannel)
1065 PhaseInter=numpy.ones(nChannel)
1066 dataSNR = dataOut.data_SNR
1067
1068
1069
1070 data = dataOut.data_pre
1071 noise = dataOut.noise
1072 print 'noise',noise
1073 #SNRdB = 10*numpy.log10(dataOut.data_SNR)
1074
1075 FirstMoment = numpy.average(dataOut.data_param[:,1,:],0)
1076 #SNRdBMean = []
132
1077
1078
1079 #for j in range(nHeights):
1080 # FirstMoment = numpy.append(FirstMoment,numpy.mean([dataOut.data_param[0,1,j],dataOut.data_param[1,1,j],dataOut.data_param[2,1,j]]))
1081 # SNRdBMean = numpy.append(SNRdBMean,numpy.mean([SNRdB[0,j],SNRdB[1,j],SNRdB[2,j]]))
1082
1083 data_output=numpy.ones([3,spc.shape[2]])*numpy.NaN
1084
1085 velocityX=[]
1086 velocityY=[]
1087 velocityV=[]
1088
1089 dbSNR = 10*numpy.log10(dataSNR)
1090 dbSNR = numpy.average(dbSNR,0)
1091 for Height in range(nHeights):
1092
1093 [Vzon,Vmer,Vver, GaussCenter]= self.WindEstimation(spc, cspc, pairsList, ChanDist, Height, noise, VelRange, dbSNR[Height], SNRlimit)
1094
1095 if abs(Vzon)<100. and abs(Vzon)> 0.:
1096 velocityX=numpy.append(velocityX, Vzon)#Vmag
1097
1098 else:
1099 print 'Vzon',Vzon
1100 velocityX=numpy.append(velocityX, numpy.NaN)
1101
1102 if abs(Vmer)<100. and abs(Vmer) > 0.:
1103 velocityY=numpy.append(velocityY, Vmer)#Vang
1104
1105 else:
1106 print 'Vmer',Vmer
1107 velocityY=numpy.append(velocityY, numpy.NaN)
1108
1109 if dbSNR[Height] > SNRlimit:
1110 velocityV=numpy.append(velocityV, FirstMoment[Height])
1111 else:
1112 velocityV=numpy.append(velocityV, numpy.NaN)
1113 #FirstMoment[Height]= numpy.NaN
1114 # if SNRdBMean[Height] <12:
1115 # FirstMoment[Height] = numpy.NaN
1116 # velocityX[Height] = numpy.NaN
1117 # velocityY[Height] = numpy.NaN
1118
1119
1120 data_output[0]=numpy.array(velocityX)
1121 data_output[1]=numpy.array(velocityY)
1122 data_output[2]=-velocityV#FirstMoment
1123
1124 print ' '
1125 #print 'FirstMoment'
1126 #print FirstMoment
1127 print 'velocityX',data_output[0]
1128 print ' '
1129 print 'velocityY',data_output[1]
1130 #print numpy.array(velocityY)
1131 print ' '
1132 #print 'SNR'
1133 #print 10*numpy.log10(dataOut.data_SNR)
1134 #print numpy.shape(10*numpy.log10(dataOut.data_SNR))
1135 print ' '
1136
1137
1138 dataOut.data_output=data_output
1139 return
1140
1141
1142 def moving_average(self,x, N=2):
1143 return numpy.convolve(x, numpy.ones((N,))/N)[(N-1):]
1144
1145 def gaus(self,xSamples,a,x0,sigma):
1146 return a*numpy.exp(-(xSamples-x0)**2/(2*sigma**2))
1147
1148 def Find(self,x,value):
1149 for index in range(len(x)):
1150 if x[index]==value:
1151 return index
1152
1153 def WindEstimation(self, spc, cspc, pairsList, ChanDist, Height, noise, VelRange, dbSNR, SNRlimit):
1154
1155 ySamples=numpy.ones([spc.shape[0],spc.shape[1]])
1156 phase=numpy.ones([spc.shape[0],spc.shape[1]])
1157 CSPCSamples=numpy.ones([spc.shape[0],spc.shape[1]],dtype=numpy.complex_)
1158 coherence=numpy.ones([spc.shape[0],spc.shape[1]])
1159 PhaseSlope=numpy.ones(spc.shape[0])
1160 PhaseInter=numpy.ones(spc.shape[0])
1161 xFrec=VelRange
1162
1163 '''Getting Eij and Nij'''
1164
1165 E01=ChanDist[0][0]
1166 N01=ChanDist[0][1]
1167
1168 E02=ChanDist[1][0]
1169 N02=ChanDist[1][1]
1170
1171 E12=ChanDist[2][0]
1172 N12=ChanDist[2][1]
1173
1174 z = spc.copy()
1175 z = numpy.where(numpy.isfinite(z), z, numpy.NAN)
1176
1177 for i in range(spc.shape[0]):
1178
1179 '''****** Line of Data SPC ******'''
1180 zline=z[i,:,Height]
1181
1182 '''****** SPC is normalized ******'''
1183 FactNorm= (zline.copy()-noise[i]) / numpy.sum(zline.copy())
1184 FactNorm= FactNorm/numpy.sum(FactNorm)
1185
1186 SmoothSPC=self.moving_average(FactNorm,N=3)
1187
1188 xSamples = ar(range(len(SmoothSPC)))
1189 ySamples[i] = SmoothSPC
1190
1191 #dbSNR=10*numpy.log10(dataSNR)
1192 print ' '
1193 print ' '
1194 print ' '
1195
1196 #print 'dataSNR', dbSNR.shape, dbSNR[0,40:120]
1197 print 'SmoothSPC', SmoothSPC.shape, SmoothSPC[0:20]
1198 print 'noise',noise
1199 print 'zline',zline.shape, zline[0:20]
1200 print 'FactNorm',FactNorm.shape, FactNorm[0:20]
1201 print 'FactNorm suma', numpy.sum(FactNorm)
1202
1203 for i in range(spc.shape[0]):
1204
1205 '''****** Line of Data CSPC ******'''
1206 cspcLine=cspc[i,:,Height].copy()
1207
1208 '''****** CSPC is normalized ******'''
1209 chan_index0 = pairsList[i][0]
1210 chan_index1 = pairsList[i][1]
1211 CSPCFactor= abs(numpy.sum(ySamples[chan_index0]) * numpy.sum(ySamples[chan_index1])) #
1212
1213 CSPCNorm = (cspcLine.copy() -noise[i]) / numpy.sqrt(CSPCFactor)
1214
1215 CSPCSamples[i] = CSPCNorm
1216 coherence[i] = numpy.abs(CSPCSamples[i]) / numpy.sqrt(CSPCFactor)
1217
1218 coherence[i]= self.moving_average(coherence[i],N=2)
1219
1220 phase[i] = self.moving_average( numpy.arctan2(CSPCSamples[i].imag, CSPCSamples[i].real),N=1)#*180/numpy.pi
1221
1222 print 'cspcLine', cspcLine.shape, cspcLine[0:20]
1223 print 'CSPCFactor', CSPCFactor#, CSPCFactor[0:20]
1224 print numpy.sum(ySamples[chan_index0]), numpy.sum(ySamples[chan_index1]), -noise[i]
1225 print 'CSPCNorm', CSPCNorm.shape, CSPCNorm[0:20]
1226 print 'CSPCNorm suma', numpy.sum(CSPCNorm)
1227 print 'CSPCSamples', CSPCSamples.shape, CSPCSamples[0,0:20]
1228
1229 '''****** Getting fij width ******'''
1230
1231 yMean=[]
1232 yMean2=[]
1233
1234 for j in range(len(ySamples[1])):
1235 yMean=numpy.append(yMean,numpy.mean([ySamples[0,j],ySamples[1,j],ySamples[2,j]]))
1236
1237 '''******* Getting fitting Gaussian ******'''
1238 meanGauss=sum(xSamples*yMean) / len(xSamples)
1239 sigma=sum(yMean*(xSamples-meanGauss)**2) / len(xSamples)
1240
1241 print '****************************'
1242 print 'len(xSamples): ',len(xSamples)
1243 print 'yMean: ', yMean.shape, yMean[0:20]
1244 print 'ySamples', ySamples.shape, ySamples[0,0:20]
1245 print 'xSamples: ',xSamples.shape, xSamples[0:20]
1246
1247 print 'meanGauss',meanGauss
1248 print 'sigma',sigma
1249
1250 #if (abs(meanGauss/sigma**2) > 0.0001) : #0.000000001):
1251 if dbSNR > SNRlimit :
1252 try:
1253 popt,pcov = curve_fit(self.gaus,xSamples,yMean,p0=[1,meanGauss,sigma])
1254
1255 if numpy.amax(popt)>numpy.amax(yMean)*0.3:
1256 FitGauss=self.gaus(xSamples,*popt)
1257
1258 else:
1259 FitGauss=numpy.ones(len(xSamples))*numpy.mean(yMean)
1260 print 'Verificador: Dentro', Height
1261 except :#RuntimeError:
1262 FitGauss=numpy.ones(len(xSamples))*numpy.mean(yMean)
1263
1264
1265 else:
1266 FitGauss=numpy.ones(len(xSamples))*numpy.mean(yMean)
1267
1268 Maximun=numpy.amax(yMean)
1269 eMinus1=Maximun*numpy.exp(-1)#*0.8
1270
1271 HWpos=self.Find(FitGauss,min(FitGauss, key=lambda value:abs(value-eMinus1)))
1272 HalfWidth= xFrec[HWpos]
1273 GCpos=self.Find(FitGauss, numpy.amax(FitGauss))
1274 Vpos=self.Find(FactNorm, numpy.amax(FactNorm))
1275
1276 #Vpos=FirstMoment[]
1277
1278 '''****** Getting Fij ******'''
1279
1280 GaussCenter=xFrec[GCpos]
1281 if (GaussCenter<0 and HalfWidth>0) or (GaussCenter>0 and HalfWidth<0):
1282 Fij=abs(GaussCenter)+abs(HalfWidth)+0.0000001
1283 else:
1284 Fij=abs(GaussCenter-HalfWidth)+0.0000001
1285
1286 '''****** Getting Frecuency range of significant data ******'''
1287
1288 Rangpos=self.Find(FitGauss,min(FitGauss, key=lambda value:abs(value-Maximun*0.10)))
1289
1290 if Rangpos<GCpos:
1291 Range=numpy.array([Rangpos,2*GCpos-Rangpos])
1292 elif Rangpos< ( len(xFrec)- len(xFrec)*0.1):
1293 Range=numpy.array([2*GCpos-Rangpos,Rangpos])
1294 else:
1295 Range = numpy.array([0,0])
1296
1297 print ' '
1298 print 'GCpos',GCpos, ( len(xFrec)- len(xFrec)*0.1)
1299 print 'Rangpos',Rangpos
1300 print 'RANGE: ', Range
1301 FrecRange=xFrec[Range[0]:Range[1]]
1302
1303 '''****** Getting SCPC Slope ******'''
1304
1305 for i in range(spc.shape[0]):
1306
1307 if len(FrecRange)>5 and len(FrecRange)<spc.shape[1]*0.5:
1308 PhaseRange=self.moving_average(phase[i,Range[0]:Range[1]],N=3)
1309
1310 print 'FrecRange', len(FrecRange) , FrecRange
1311 print 'PhaseRange', len(PhaseRange), PhaseRange
1312 print ' '
1313 if len(FrecRange) == len(PhaseRange):
1314 slope, intercept, r_value, p_value, std_err = stats.linregress(FrecRange,PhaseRange)
1315 PhaseSlope[i]=slope
1316 PhaseInter[i]=intercept
1317 else:
1318 PhaseSlope[i]=0
1319 PhaseInter[i]=0
1320 else:
1321 PhaseSlope[i]=0
1322 PhaseInter[i]=0
1323
1324 '''Getting constant C'''
1325 cC=(Fij*numpy.pi)**2
1326
1327 '''****** Getting constants F and G ******'''
1328 MijEijNij=numpy.array([[E02,N02], [E12,N12]])
1329 MijResult0=(-PhaseSlope[1]*cC) / (2*numpy.pi)
1330 MijResult1=(-PhaseSlope[2]*cC) / (2*numpy.pi)
1331 MijResults=numpy.array([MijResult0,MijResult1])
1332 (cF,cG) = numpy.linalg.solve(MijEijNij, MijResults)
1333
1334 '''****** Getting constants A, B and H ******'''
1335 W01=numpy.amax(coherence[0])
1336 W02=numpy.amax(coherence[1])
1337 W12=numpy.amax(coherence[2])
1338
1339 WijResult0=((cF*E01+cG*N01)**2)/cC - numpy.log(W01 / numpy.sqrt(numpy.pi/cC))
1340 WijResult1=((cF*E02+cG*N02)**2)/cC - numpy.log(W02 / numpy.sqrt(numpy.pi/cC))
1341 WijResult2=((cF*E12+cG*N12)**2)/cC - numpy.log(W12 / numpy.sqrt(numpy.pi/cC))
1342
1343 WijResults=numpy.array([WijResult0, WijResult1, WijResult2])
1344
1345 WijEijNij=numpy.array([ [E01**2, N01**2, 2*E01*N01] , [E02**2, N02**2, 2*E02*N02] , [E12**2, N12**2, 2*E12*N12] ])
1346 (cA,cB,cH) = numpy.linalg.solve(WijEijNij, WijResults)
1347
1348 VxVy=numpy.array([[cA,cH],[cH,cB]])
1349
1350 VxVyResults=numpy.array([-cF,-cG])
1351 (Vx,Vy) = numpy.linalg.solve(VxVy, VxVyResults)
1352
1353 Vzon = Vy
1354 Vmer = Vx
1355 Vmag=numpy.sqrt(Vzon**2+Vmer**2)
1356 Vang=numpy.arctan2(Vmer,Vzon)
1357 Vver=xFrec[Vpos]
1358 print 'vzon y vmer', Vzon, Vmer
1359 return Vzon, Vmer, Vver, GaussCenter
1360
1361 class SpectralMoments(Operation):
1362
1363 '''
1364 Function SpectralMoments()
1365
1366 Calculates moments (power, mean, standard deviation) and SNR of the signal
1367
1368 Type of dataIn: Spectra
1369
1370 Configuration Parameters:
1371
133 dirCosx : Cosine director in X axis
1372 dirCosx : Cosine director in X axis
134 dirCosy : Cosine director in Y axis
1373 dirCosy : Cosine director in Y axis
135
1374
136 elevation :
1375 elevation :
137 azimuth :
1376 azimuth :
138
1377
139 Input:
1378 Input:
140 channelList : simple channel list to select e.g. [2,3,7]
1379 channelList : simple channel list to select e.g. [2,3,7]
141 self.dataOut.data_pre : Spectral data
1380 self.dataOut.data_pre : Spectral data
142 self.dataOut.abscissaList : List of frequencies
1381 self.dataOut.abscissaList : List of frequencies
143 self.dataOut.noise : Noise level per channel
1382 self.dataOut.noise : Noise level per channel
144
1383
145 Affected:
1384 Affected:
146 self.dataOut.data_param : Parameters per channel
1385 self.dataOut.data_param : Parameters per channel
147 self.dataOut.data_SNR : SNR per channel
1386 self.dataOut.data_SNR : SNR per channel
148
1387
149 '''
1388 '''
150
1389
151 def run(self, dataOut):
1390 def run(self, dataOut):
152
1391
153 #dataOut.data_pre = dataOut.data_pre[0]
1392 #dataOut.data_pre = dataOut.data_pre[0]
154 data = dataOut.data_pre[0]
1393 data = dataOut.data_pre[0]
155 absc = dataOut.abscissaList[:-1]
1394 absc = dataOut.abscissaList[:-1]
156 noise = dataOut.noise
1395 noise = dataOut.noise
157 nChannel = data.shape[0]
1396 nChannel = data.shape[0]
158 data_param = numpy.zeros((nChannel, 4, data.shape[2]))
1397 data_param = numpy.zeros((nChannel, 4, data.shape[2]))
159
1398
160 for ind in range(nChannel):
1399 for ind in range(nChannel):
161 data_param[ind,:,:] = self.__calculateMoments(data[ind,:,:], absc, noise[ind])
1400 data_param[ind,:,:] = self.__calculateMoments( data[ind,:,:] , absc , noise[ind] )
162
1401
163 dataOut.data_param = data_param[:,1:,:]
1402 dataOut.data_param = data_param[:,1:,:]
164 dataOut.data_SNR = data_param[:,0]
1403 dataOut.data_SNR = data_param[:,0]
165 dataOut.data_DOP = data_param[:,1]
1404 dataOut.data_DOP = data_param[:,1]
166 dataOut.data_MEAN = data_param[:,2]
1405 dataOut.data_MEAN = data_param[:,2]
167 dataOut.data_STD = data_param[:,3]
1406 dataOut.data_STD = data_param[:,3]
168 return
1407 return
169
1408
170 def __calculateMoments(self, oldspec, oldfreq, n0, nicoh = None, graph = None, smooth = None, type1 = None, fwindow = None, snrth = None, dc = None, aliasing = None, oldfd = None, wwauto = None):
1409 def __calculateMoments(self, oldspec, oldfreq, n0,
171
1410 nicoh = None, graph = None, smooth = None, type1 = None, fwindow = None, snrth = None, dc = None, aliasing = None, oldfd = None, wwauto = None):
172 if (nicoh is None): nicoh = 1
1411
173 if (graph is None): graph = 0
1412 if (nicoh == None): nicoh = 1
174 if (smooth is None): smooth = 0
1413 if (graph == None): graph = 0
1414 if (smooth == None): smooth = 0
175 elif (self.smooth < 3): smooth = 0
1415 elif (self.smooth < 3): smooth = 0
176
1416
177 if (type1 is None): type1 = 0
1417 if (type1 == None): type1 = 0
178 if (fwindow is None): fwindow = numpy.zeros(oldfreq.size) + 1
1418 if (fwindow == None): fwindow = numpy.zeros(oldfreq.size) + 1
179 if (snrth is None): snrth = -3
1419 if (snrth == None): snrth = -3
180 if (dc is None): dc = 0
1420 if (dc == None): dc = 0
181 if (aliasing is None): aliasing = 0
1421 if (aliasing == None): aliasing = 0
182 if (oldfd is None): oldfd = 0
1422 if (oldfd == None): oldfd = 0
183 if (wwauto is None): wwauto = 0
1423 if (wwauto == None): wwauto = 0
184
1424
185 if (n0 < 1.e-20): n0 = 1.e-20
1425 if (n0 < 1.e-20): n0 = 1.e-20
186
1426
187 freq = oldfreq
1427 freq = oldfreq
188 vec_power = numpy.zeros(oldspec.shape[1])
1428 vec_power = numpy.zeros(oldspec.shape[1])
189 vec_fd = numpy.zeros(oldspec.shape[1])
1429 vec_fd = numpy.zeros(oldspec.shape[1])
@@ -191,86 +1431,86 class SpectralMoments(Operation):
191 vec_snr = numpy.zeros(oldspec.shape[1])
1431 vec_snr = numpy.zeros(oldspec.shape[1])
192
1432
193 for ind in range(oldspec.shape[1]):
1433 for ind in range(oldspec.shape[1]):
194
1434
195 spec = oldspec[:,ind]
1435 spec = oldspec[:,ind]
196 aux = spec*fwindow
1436 aux = spec*fwindow
197 max_spec = aux.max()
1437 max_spec = aux.max()
198 m = list(aux).index(max_spec)
1438 m = list(aux).index(max_spec)
199
1439
200 #Smooth
1440 #Smooth
201 if (smooth == 0): spec2 = spec
1441 if (smooth == 0): spec2 = spec
202 else: spec2 = scipy.ndimage.filters.uniform_filter1d(spec,size=smooth)
1442 else: spec2 = scipy.ndimage.filters.uniform_filter1d(spec,size=smooth)
203
1443
204 # Calculo de Momentos
1444 # Calculo de Momentos
205 bb = spec2[range(m,spec2.size)]
1445 bb = spec2[range(m,spec2.size)]
206 bb = (bb<n0).nonzero()
1446 bb = (bb<n0).nonzero()
207 bb = bb[0]
1447 bb = bb[0]
208
1448
209 ss = spec2[range(0,m + 1)]
1449 ss = spec2[range(0,m + 1)]
210 ss = (ss<n0).nonzero()
1450 ss = (ss<n0).nonzero()
211 ss = ss[0]
1451 ss = ss[0]
212
1452
213 if (bb.size == 0):
1453 if (bb.size == 0):
214 bb0 = spec.size - 1 - m
1454 bb0 = spec.size - 1 - m
215 else:
1455 else:
216 bb0 = bb[0] - 1
1456 bb0 = bb[0] - 1
217 if (bb0 < 0):
1457 if (bb0 < 0):
218 bb0 = 0
1458 bb0 = 0
219
1459
220 if (ss.size == 0): ss1 = 1
1460 if (ss.size == 0): ss1 = 1
221 else: ss1 = max(ss) + 1
1461 else: ss1 = max(ss) + 1
222
1462
223 if (ss1 > m): ss1 = m
1463 if (ss1 > m): ss1 = m
224
1464
225 valid = numpy.asarray(range(int(m + bb0 - ss1 + 1))) + ss1
1465 valid = numpy.asarray(range(int(m + bb0 - ss1 + 1))) + ss1
226 power = ((spec2[valid] - n0)*fwindow[valid]).sum()
1466 power = ((spec2[valid] - n0)*fwindow[valid]).sum()
227 fd = ((spec2[valid]- n0)*freq[valid]*fwindow[valid]).sum()/power
1467 fd = ((spec2[valid]- n0)*freq[valid]*fwindow[valid]).sum()/power
228 w = math.sqrt(((spec2[valid] - n0)*fwindow[valid]*(freq[valid]- fd)**2).sum()/power)
1468 w = math.sqrt(((spec2[valid] - n0)*fwindow[valid]*(freq[valid]- fd)**2).sum()/power)
229 snr = (spec2.mean()-n0)/n0
1469 snr = (spec2.mean()-n0)/n0
230
1470
231 if (snr < 1.e-20) :
1471 if (snr < 1.e-20) :
232 snr = 1.e-20
1472 snr = 1.e-20
233
1473
234 vec_power[ind] = power
1474 vec_power[ind] = power
235 vec_fd[ind] = fd
1475 vec_fd[ind] = fd
236 vec_w[ind] = w
1476 vec_w[ind] = w
237 vec_snr[ind] = snr
1477 vec_snr[ind] = snr
238
1478
239 moments = numpy.vstack((vec_snr, vec_power, vec_fd, vec_w))
1479 moments = numpy.vstack((vec_snr, vec_power, vec_fd, vec_w))
240 return moments
1480 return moments
241
1481
242 #------------------ Get SA Parameters --------------------------
1482 #------------------ Get SA Parameters --------------------------
243
1483
244 def GetSAParameters(self):
1484 def GetSAParameters(self):
245 #SA en frecuencia
1485 #SA en frecuencia
246 pairslist = self.dataOut.groupList
1486 pairslist = self.dataOut.groupList
247 num_pairs = len(pairslist)
1487 num_pairs = len(pairslist)
248
1488
249 vel = self.dataOut.abscissaList
1489 vel = self.dataOut.abscissaList
250 spectra = self.dataOut.data_pre[0]
1490 spectra = self.dataOut.data_pre
251 cspectra = self.dataOut.data_pre[1]
1491 cspectra = self.dataIn.data_cspc
252 delta_v = vel[1] - vel[0]
1492 delta_v = vel[1] - vel[0]
253
1493
254 #Calculating the power spectrum
1494 #Calculating the power spectrum
255 spc_pow = numpy.sum(spectra, 3)*delta_v
1495 spc_pow = numpy.sum(spectra, 3)*delta_v
256 #Normalizing Spectra
1496 #Normalizing Spectra
257 norm_spectra = spectra/spc_pow
1497 norm_spectra = spectra/spc_pow
258 #Calculating the norm_spectra at peak
1498 #Calculating the norm_spectra at peak
259 max_spectra = numpy.max(norm_spectra, 3)
1499 max_spectra = numpy.max(norm_spectra, 3)
260
1500
261 #Normalizing Cross Spectra
1501 #Normalizing Cross Spectra
262 norm_cspectra = numpy.zeros(cspectra.shape)
1502 norm_cspectra = numpy.zeros(cspectra.shape)
263
1503
264 for i in range(num_chan):
1504 for i in range(num_chan):
265 norm_cspectra[i,:,:] = cspectra[i,:,:]/numpy.sqrt(spc_pow[pairslist[i][0],:]*spc_pow[pairslist[i][1],:])
1505 norm_cspectra[i,:,:] = cspectra[i,:,:]/numpy.sqrt(spc_pow[pairslist[i][0],:]*spc_pow[pairslist[i][1],:])
266
1506
267 max_cspectra = numpy.max(norm_cspectra,2)
1507 max_cspectra = numpy.max(norm_cspectra,2)
268 max_cspectra_index = numpy.argmax(norm_cspectra, 2)
1508 max_cspectra_index = numpy.argmax(norm_cspectra, 2)
269
1509
270 for i in range(num_pairs):
1510 for i in range(num_pairs):
271 cspc_par[i,:,:] = __calculateMoments(norm_cspectra)
1511 cspc_par[i,:,:] = __calculateMoments(norm_cspectra)
272 #------------------- Get Lags ----------------------------------
1512 #------------------- Get Lags ----------------------------------
273
1513
274 class SALags(Operation):
1514 class SALags(Operation):
275 '''
1515 '''
276 Function GetMoments()
1516 Function GetMoments()
@@ -283,19 +1523,19 class SALags(Operation):
283 self.dataOut.data_SNR
1523 self.dataOut.data_SNR
284 self.dataOut.groupList
1524 self.dataOut.groupList
285 self.dataOut.nChannels
1525 self.dataOut.nChannels
286
1526
287 Affected:
1527 Affected:
288 self.dataOut.data_param
1528 self.dataOut.data_param
289
1529
290 '''
1530 '''
291 def run(self, dataOut):
1531 def run(self, dataOut):
292 data_acf = dataOut.data_pre[0]
1532 data_acf = dataOut.data_pre[0]
293 data_ccf = dataOut.data_pre[1]
1533 data_ccf = dataOut.data_pre[1]
294 normFactor_acf = dataOut.normFactor[0]
1534 normFactor_acf = dataOut.normFactor[0]
295 normFactor_ccf = dataOut.normFactor[1]
1535 normFactor_ccf = dataOut.normFactor[1]
296 pairs_acf = dataOut.groupList[0]
1536 pairs_acf = dataOut.groupList[0]
297 pairs_ccf = dataOut.groupList[1]
1537 pairs_ccf = dataOut.groupList[1]
298
1538
299 nHeights = dataOut.nHeights
1539 nHeights = dataOut.nHeights
300 absc = dataOut.abscissaList
1540 absc = dataOut.abscissaList
301 noise = dataOut.noise
1541 noise = dataOut.noise
@@ -306,97 +1546,97 class SALags(Operation):
306
1546
307 for l in range(len(pairs_acf)):
1547 for l in range(len(pairs_acf)):
308 data_acf[l,:,:] = data_acf[l,:,:]/normFactor_acf[l,:]
1548 data_acf[l,:,:] = data_acf[l,:,:]/normFactor_acf[l,:]
309
1549
310 for l in range(len(pairs_ccf)):
1550 for l in range(len(pairs_ccf)):
311 data_ccf[l,:,:] = data_ccf[l,:,:]/normFactor_ccf[l,:]
1551 data_ccf[l,:,:] = data_ccf[l,:,:]/normFactor_ccf[l,:]
312
1552
313 dataOut.data_param = numpy.zeros((len(pairs_ccf)*2 + 1, nHeights))
1553 dataOut.data_param = numpy.zeros((len(pairs_ccf)*2 + 1, nHeights))
314 dataOut.data_param[:-1,:] = self.__calculateTaus(data_acf, data_ccf, absc)
1554 dataOut.data_param[:-1,:] = self.__calculateTaus(data_acf, data_ccf, absc)
315 dataOut.data_param[-1,:] = self.__calculateLag1Phase(data_acf, absc)
1555 dataOut.data_param[-1,:] = self.__calculateLag1Phase(data_acf, absc)
316 return
1556 return
317
1557
318 # def __getPairsAutoCorr(self, pairsList, nChannels):
1558 # def __getPairsAutoCorr(self, pairsList, nChannels):
319 #
1559 #
320 # pairsAutoCorr = numpy.zeros(nChannels, dtype = 'int')*numpy.nan
1560 # pairsAutoCorr = numpy.zeros(nChannels, dtype = 'int')*numpy.nan
321 #
1561 #
322 # for l in range(len(pairsList)):
1562 # for l in range(len(pairsList)):
323 # firstChannel = pairsList[l][0]
1563 # firstChannel = pairsList[l][0]
324 # secondChannel = pairsList[l][1]
1564 # secondChannel = pairsList[l][1]
325 #
1565 #
326 # #Obteniendo pares de Autocorrelacion
1566 # #Obteniendo pares de Autocorrelacion
327 # if firstChannel == secondChannel:
1567 # if firstChannel == secondChannel:
328 # pairsAutoCorr[firstChannel] = int(l)
1568 # pairsAutoCorr[firstChannel] = int(l)
329 #
1569 #
330 # pairsAutoCorr = pairsAutoCorr.astype(int)
1570 # pairsAutoCorr = pairsAutoCorr.astype(int)
331 #
1571 #
332 # pairsCrossCorr = range(len(pairsList))
1572 # pairsCrossCorr = range(len(pairsList))
333 # pairsCrossCorr = numpy.delete(pairsCrossCorr,pairsAutoCorr)
1573 # pairsCrossCorr = numpy.delete(pairsCrossCorr,pairsAutoCorr)
334 #
1574 #
335 # return pairsAutoCorr, pairsCrossCorr
1575 # return pairsAutoCorr, pairsCrossCorr
336
1576
337 def __calculateTaus(self, data_acf, data_ccf, lagRange):
1577 def __calculateTaus(self, data_acf, data_ccf, lagRange):
338
1578
339 lag0 = data_acf.shape[1]/2
1579 lag0 = data_acf.shape[1]/2
340 #Funcion de Autocorrelacion
1580 #Funcion de Autocorrelacion
341 mean_acf = stats.nanmean(data_acf, axis = 0)
1581 mean_acf = stats.nanmean(data_acf, axis = 0)
342
1582
343 #Obtencion Indice de TauCross
1583 #Obtencion Indice de TauCross
344 ind_ccf = data_ccf.argmax(axis = 1)
1584 ind_ccf = data_ccf.argmax(axis = 1)
345 #Obtencion Indice de TauAuto
1585 #Obtencion Indice de TauAuto
346 ind_acf = numpy.zeros(ind_ccf.shape,dtype = 'int')
1586 ind_acf = numpy.zeros(ind_ccf.shape,dtype = 'int')
347 ccf_lag0 = data_ccf[:,lag0,:]
1587 ccf_lag0 = data_ccf[:,lag0,:]
348
1588
349 for i in range(ccf_lag0.shape[0]):
1589 for i in range(ccf_lag0.shape[0]):
350 ind_acf[i,:] = numpy.abs(mean_acf - ccf_lag0[i,:]).argmin(axis = 0)
1590 ind_acf[i,:] = numpy.abs(mean_acf - ccf_lag0[i,:]).argmin(axis = 0)
351
1591
352 #Obtencion de TauCross y TauAuto
1592 #Obtencion de TauCross y TauAuto
353 tau_ccf = lagRange[ind_ccf]
1593 tau_ccf = lagRange[ind_ccf]
354 tau_acf = lagRange[ind_acf]
1594 tau_acf = lagRange[ind_acf]
355
1595
356 Nan1, Nan2 = numpy.where(tau_ccf == lagRange[0])
1596 Nan1, Nan2 = numpy.where(tau_ccf == lagRange[0])
357
1597
358 tau_ccf[Nan1,Nan2] = numpy.nan
1598 tau_ccf[Nan1,Nan2] = numpy.nan
359 tau_acf[Nan1,Nan2] = numpy.nan
1599 tau_acf[Nan1,Nan2] = numpy.nan
360 tau = numpy.vstack((tau_ccf,tau_acf))
1600 tau = numpy.vstack((tau_ccf,tau_acf))
361
1601
362 return tau
1602 return tau
363
1603
364 def __calculateLag1Phase(self, data, lagTRange):
1604 def __calculateLag1Phase(self, data, lagTRange):
365 data1 = stats.nanmean(data, axis = 0)
1605 data1 = stats.nanmean(data, axis = 0)
366 lag1 = numpy.where(lagTRange == 0)[0][0] + 1
1606 lag1 = numpy.where(lagTRange == 0)[0][0] + 1
367
1607
368 phase = numpy.angle(data1[lag1,:])
1608 phase = numpy.angle(data1[lag1,:])
369
1609
370 return phase
1610 return phase
371
1611
372 class SpectralFitting(Operation):
1612 class SpectralFitting(Operation):
373 '''
1613 '''
374 Function GetMoments()
1614 Function GetMoments()
375
1615
376 Input:
1616 Input:
377 Output:
1617 Output:
378 Variables modified:
1618 Variables modified:
379 '''
1619 '''
380
1620
381 def run(self, dataOut, getSNR = True, path=None, file=None, groupList=None):
1621 def run(self, dataOut, getSNR = True, path=None, file=None, groupList=None):
382
1622
383
1623
384 if path != None:
1624 if path != None:
385 sys.path.append(path)
1625 sys.path.append(path)
386 self.dataOut.library = importlib.import_module(file)
1626 self.dataOut.library = importlib.import_module(file)
387
1627
388 #To be inserted as a parameter
1628 #To be inserted as a parameter
389 groupArray = numpy.array(groupList)
1629 groupArray = numpy.array(groupList)
390 # groupArray = numpy.array([[0,1],[2,3]])
1630 # groupArray = numpy.array([[0,1],[2,3]])
391 self.dataOut.groupList = groupArray
1631 self.dataOut.groupList = groupArray
392
1632
393 nGroups = groupArray.shape[0]
1633 nGroups = groupArray.shape[0]
394 nChannels = self.dataIn.nChannels
1634 nChannels = self.dataIn.nChannels
395 nHeights=self.dataIn.heightList.size
1635 nHeights=self.dataIn.heightList.size
396
1636
397 #Parameters Array
1637 #Parameters Array
398 self.dataOut.data_param = None
1638 self.dataOut.data_param = None
399
1639
400 #Set constants
1640 #Set constants
401 constants = self.dataOut.library.setConstants(self.dataIn)
1641 constants = self.dataOut.library.setConstants(self.dataIn)
402 self.dataOut.constants = constants
1642 self.dataOut.constants = constants
@@ -405,24 +1645,24 class SpectralFitting(Operation):
405 ippSeconds = self.dataIn.ippSeconds
1645 ippSeconds = self.dataIn.ippSeconds
406 K = self.dataIn.nIncohInt
1646 K = self.dataIn.nIncohInt
407 pairsArray = numpy.array(self.dataIn.pairsList)
1647 pairsArray = numpy.array(self.dataIn.pairsList)
408
1648
409 #List of possible combinations
1649 #List of possible combinations
410 listComb = itertools.combinations(numpy.arange(groupArray.shape[1]),2)
1650 listComb = itertools.combinations(numpy.arange(groupArray.shape[1]),2)
411 indCross = numpy.zeros(len(list(listComb)), dtype = 'int')
1651 indCross = numpy.zeros(len(list(listComb)), dtype = 'int')
412
1652
413 if getSNR:
1653 if getSNR:
414 listChannels = groupArray.reshape((groupArray.size))
1654 listChannels = groupArray.reshape((groupArray.size))
415 listChannels.sort()
1655 listChannels.sort()
416 noise = self.dataIn.getNoise()
1656 noise = self.dataIn.getNoise()
417 self.dataOut.data_SNR = self.__getSNR(self.dataIn.data_spc[listChannels,:,:], noise[listChannels])
1657 self.dataOut.data_SNR = self.__getSNR(self.dataIn.data_spc[listChannels,:,:], noise[listChannels])
418
1658
419 for i in range(nGroups):
1659 for i in range(nGroups):
420 coord = groupArray[i,:]
1660 coord = groupArray[i,:]
421
1661
422 #Input data array
1662 #Input data array
423 data = self.dataIn.data_spc[coord,:,:]/(M*N)
1663 data = self.dataIn.data_spc[coord,:,:]/(M*N)
424 data = data.reshape((data.shape[0]*data.shape[1],data.shape[2]))
1664 data = data.reshape((data.shape[0]*data.shape[1],data.shape[2]))
425
1665
426 #Cross Spectra data array for Covariance Matrixes
1666 #Cross Spectra data array for Covariance Matrixes
427 ind = 0
1667 ind = 0
428 for pairs in listComb:
1668 for pairs in listComb:
@@ -431,10 +1671,10 class SpectralFitting(Operation):
431 ind += 1
1671 ind += 1
432 dataCross = self.dataIn.data_cspc[indCross,:,:]/(M*N)
1672 dataCross = self.dataIn.data_cspc[indCross,:,:]/(M*N)
433 dataCross = dataCross**2/K
1673 dataCross = dataCross**2/K
434
1674
435 for h in range(nHeights):
1675 for h in range(nHeights):
436 # print self.dataOut.heightList[h]
1676 # print self.dataOut.heightList[h]
437
1677
438 #Input
1678 #Input
439 d = data[:,h]
1679 d = data[:,h]
440
1680
@@ -443,7 +1683,7 class SpectralFitting(Operation):
443 ind = 0
1683 ind = 0
444 for pairs in listComb:
1684 for pairs in listComb:
445 #Coordinates in Covariance Matrix
1685 #Coordinates in Covariance Matrix
446 x = pairs[0]
1686 x = pairs[0]
447 y = pairs[1]
1687 y = pairs[1]
448 #Channel Index
1688 #Channel Index
449 S12 = dataCross[ind,:,h]
1689 S12 = dataCross[ind,:,h]
@@ -457,15 +1697,15 class SpectralFitting(Operation):
457 LT=L.T
1697 LT=L.T
458
1698
459 dp = numpy.dot(LT,d)
1699 dp = numpy.dot(LT,d)
460
1700
461 #Initial values
1701 #Initial values
462 data_spc = self.dataIn.data_spc[coord,:,h]
1702 data_spc = self.dataIn.data_spc[coord,:,h]
463
1703
464 if (h>0)and(error1[3]<5):
1704 if (h>0)and(error1[3]<5):
465 p0 = self.dataOut.data_param[i,:,h-1]
1705 p0 = self.dataOut.data_param[i,:,h-1]
466 else:
1706 else:
467 p0 = numpy.array(self.dataOut.library.initialValuesFunction(data_spc, constants, i))
1707 p0 = numpy.array(self.dataOut.library.initialValuesFunction(data_spc, constants, i))
468
1708
469 try:
1709 try:
470 #Least Squares
1710 #Least Squares
471 minp,covp,infodict,mesg,ier = optimize.leastsq(self.__residFunction,p0,args=(dp,LT,constants),full_output=True)
1711 minp,covp,infodict,mesg,ier = optimize.leastsq(self.__residFunction,p0,args=(dp,LT,constants),full_output=True)
@@ -478,30 +1718,30 class SpectralFitting(Operation):
478 minp = p0*numpy.nan
1718 minp = p0*numpy.nan
479 error0 = numpy.nan
1719 error0 = numpy.nan
480 error1 = p0*numpy.nan
1720 error1 = p0*numpy.nan
481
1721
482 #Save
1722 #Save
483 if self.dataOut.data_param is None:
1723 if self.dataOut.data_param == None:
484 self.dataOut.data_param = numpy.zeros((nGroups, p0.size, nHeights))*numpy.nan
1724 self.dataOut.data_param = numpy.zeros((nGroups, p0.size, nHeights))*numpy.nan
485 self.dataOut.data_error = numpy.zeros((nGroups, p0.size + 1, nHeights))*numpy.nan
1725 self.dataOut.data_error = numpy.zeros((nGroups, p0.size + 1, nHeights))*numpy.nan
486
1726
487 self.dataOut.data_error[i,:,h] = numpy.hstack((error0,error1))
1727 self.dataOut.data_error[i,:,h] = numpy.hstack((error0,error1))
488 self.dataOut.data_param[i,:,h] = minp
1728 self.dataOut.data_param[i,:,h] = minp
489 return
1729 return
490
1730
491 def __residFunction(self, p, dp, LT, constants):
1731 def __residFunction(self, p, dp, LT, constants):
492
1732
493 fm = self.dataOut.library.modelFunction(p, constants)
1733 fm = self.dataOut.library.modelFunction(p, constants)
494 fmp=numpy.dot(LT,fm)
1734 fmp=numpy.dot(LT,fm)
495
1735
496 return dp-fmp
1736 return dp-fmp
497
1737
498 def __getSNR(self, z, noise):
1738 def __getSNR(self, z, noise):
499
1739
500 avg = numpy.average(z, axis=1)
1740 avg = numpy.average(z, axis=1)
501 SNR = (avg.T-noise)/noise
1741 SNR = (avg.T-noise)/noise
502 SNR = SNR.T
1742 SNR = SNR.T
503 return SNR
1743 return SNR
504
1744
505 def __chisq(p,chindex,hindex):
1745 def __chisq(p,chindex,hindex):
506 #similar to Resid but calculates CHI**2
1746 #similar to Resid but calculates CHI**2
507 [LT,d,fm]=setupLTdfm(p,chindex,hindex)
1747 [LT,d,fm]=setupLTdfm(p,chindex,hindex)
@@ -509,50 +1749,53 class SpectralFitting(Operation):
509 fmp=numpy.dot(LT,fm)
1749 fmp=numpy.dot(LT,fm)
510 chisq=numpy.dot((dp-fmp).T,(dp-fmp))
1750 chisq=numpy.dot((dp-fmp).T,(dp-fmp))
511 return chisq
1751 return chisq
512
1752
513 class WindProfiler(Operation):
1753 class WindProfiler(Operation):
514
1754
515 __isConfig = False
1755 __isConfig = False
516
1756
517 __initime = None
1757 __initime = None
518 __lastdatatime = None
1758 __lastdatatime = None
519 __integrationtime = None
1759 __integrationtime = None
520
1760
521 __buffer = None
1761 __buffer = None
522
1762
523 __dataReady = False
1763 __dataReady = False
524
1764
525 __firstdata = None
1765 __firstdata = None
526
1766
527 n = None
1767 n = None
528
1768
1769 def __init__(self):
1770 Operation.__init__(self)
1771
529 def __calculateCosDir(self, elev, azim):
1772 def __calculateCosDir(self, elev, azim):
530 zen = (90 - elev)*numpy.pi/180
1773 zen = (90 - elev)*numpy.pi/180
531 azim = azim*numpy.pi/180
1774 azim = azim*numpy.pi/180
532 cosDirX = numpy.sqrt((1-numpy.cos(zen)**2)/((1+numpy.tan(azim)**2)))
1775 cosDirX = numpy.sqrt((1-numpy.cos(zen)**2)/((1+numpy.tan(azim)**2)))
533 cosDirY = numpy.sqrt(1-numpy.cos(zen)**2-cosDirX**2)
1776 cosDirY = numpy.sqrt(1-numpy.cos(zen)**2-cosDirX**2)
534
1777
535 signX = numpy.sign(numpy.cos(azim))
1778 signX = numpy.sign(numpy.cos(azim))
536 signY = numpy.sign(numpy.sin(azim))
1779 signY = numpy.sign(numpy.sin(azim))
537
1780
538 cosDirX = numpy.copysign(cosDirX, signX)
1781 cosDirX = numpy.copysign(cosDirX, signX)
539 cosDirY = numpy.copysign(cosDirY, signY)
1782 cosDirY = numpy.copysign(cosDirY, signY)
540 return cosDirX, cosDirY
1783 return cosDirX, cosDirY
541
1784
542 def __calculateAngles(self, theta_x, theta_y, azimuth):
1785 def __calculateAngles(self, theta_x, theta_y, azimuth):
543
1786
544 dir_cosw = numpy.sqrt(1-theta_x**2-theta_y**2)
1787 dir_cosw = numpy.sqrt(1-theta_x**2-theta_y**2)
545 zenith_arr = numpy.arccos(dir_cosw)
1788 zenith_arr = numpy.arccos(dir_cosw)
546 azimuth_arr = numpy.arctan2(theta_x,theta_y) + azimuth*math.pi/180
1789 azimuth_arr = numpy.arctan2(theta_x,theta_y) + azimuth*math.pi/180
547
1790
548 dir_cosu = numpy.sin(azimuth_arr)*numpy.sin(zenith_arr)
1791 dir_cosu = numpy.sin(azimuth_arr)*numpy.sin(zenith_arr)
549 dir_cosv = numpy.cos(azimuth_arr)*numpy.sin(zenith_arr)
1792 dir_cosv = numpy.cos(azimuth_arr)*numpy.sin(zenith_arr)
550
1793
551 return azimuth_arr, zenith_arr, dir_cosu, dir_cosv, dir_cosw
1794 return azimuth_arr, zenith_arr, dir_cosu, dir_cosv, dir_cosw
552
1795
553 def __calculateMatA(self, dir_cosu, dir_cosv, dir_cosw, horOnly):
1796 def __calculateMatA(self, dir_cosu, dir_cosv, dir_cosw, horOnly):
554
1797
555 #
1798 #
556 if horOnly:
1799 if horOnly:
557 A = numpy.c_[dir_cosu,dir_cosv]
1800 A = numpy.c_[dir_cosu,dir_cosv]
558 else:
1801 else:
@@ -566,37 +1809,37 class WindProfiler(Operation):
566 listPhi = phi.tolist()
1809 listPhi = phi.tolist()
567 maxid = listPhi.index(max(listPhi))
1810 maxid = listPhi.index(max(listPhi))
568 minid = listPhi.index(min(listPhi))
1811 minid = listPhi.index(min(listPhi))
569
1812
570 rango = range(len(phi))
1813 rango = range(len(phi))
571 # rango = numpy.delete(rango,maxid)
1814 # rango = numpy.delete(rango,maxid)
572
1815
573 heiRang1 = heiRang*math.cos(phi[maxid])
1816 heiRang1 = heiRang*math.cos(phi[maxid])
574 heiRangAux = heiRang*math.cos(phi[minid])
1817 heiRangAux = heiRang*math.cos(phi[minid])
575 indOut = (heiRang1 < heiRangAux[0]).nonzero()
1818 indOut = (heiRang1 < heiRangAux[0]).nonzero()
576 heiRang1 = numpy.delete(heiRang1,indOut)
1819 heiRang1 = numpy.delete(heiRang1,indOut)
577
1820
578 velRadial1 = numpy.zeros([len(phi),len(heiRang1)])
1821 velRadial1 = numpy.zeros([len(phi),len(heiRang1)])
579 SNR1 = numpy.zeros([len(phi),len(heiRang1)])
1822 SNR1 = numpy.zeros([len(phi),len(heiRang1)])
580
1823
581 for i in rango:
1824 for i in rango:
582 x = heiRang*math.cos(phi[i])
1825 x = heiRang*math.cos(phi[i])
583 y1 = velRadial[i,:]
1826 y1 = velRadial[i,:]
584 f1 = interpolate.interp1d(x,y1,kind = 'cubic')
1827 f1 = interpolate.interp1d(x,y1,kind = 'cubic')
585
1828
586 x1 = heiRang1
1829 x1 = heiRang1
587 y11 = f1(x1)
1830 y11 = f1(x1)
588
1831
589 y2 = SNR[i,:]
1832 y2 = SNR[i,:]
590 f2 = interpolate.interp1d(x,y2,kind = 'cubic')
1833 f2 = interpolate.interp1d(x,y2,kind = 'cubic')
591 y21 = f2(x1)
1834 y21 = f2(x1)
592
1835
593 velRadial1[i,:] = y11
1836 velRadial1[i,:] = y11
594 SNR1[i,:] = y21
1837 SNR1[i,:] = y21
595
1838
596 return heiRang1, velRadial1, SNR1
1839 return heiRang1, velRadial1, SNR1
597
1840
598 def __calculateVelUVW(self, A, velRadial):
1841 def __calculateVelUVW(self, A, velRadial):
599
1842
600 #Operacion Matricial
1843 #Operacion Matricial
601 # velUVW = numpy.zeros((velRadial.shape[1],3))
1844 # velUVW = numpy.zeros((velRadial.shape[1],3))
602 # for ind in range(velRadial.shape[1]):
1845 # for ind in range(velRadial.shape[1]):
@@ -604,27 +1847,27 class WindProfiler(Operation):
604 # velUVW = velUVW.transpose()
1847 # velUVW = velUVW.transpose()
605 velUVW = numpy.zeros((A.shape[0],velRadial.shape[1]))
1848 velUVW = numpy.zeros((A.shape[0],velRadial.shape[1]))
606 velUVW[:,:] = numpy.dot(A,velRadial)
1849 velUVW[:,:] = numpy.dot(A,velRadial)
607
1850
608
1851
609 return velUVW
1852 return velUVW
610
1853
611 # def techniqueDBS(self, velRadial0, dirCosx, disrCosy, azimuth, correct, horizontalOnly, heiRang, SNR0):
1854 # def techniqueDBS(self, velRadial0, dirCosx, disrCosy, azimuth, correct, horizontalOnly, heiRang, SNR0):
612
1855
613 def techniqueDBS(self, kwargs):
1856 def techniqueDBS(self, kwargs):
614 """
1857 """
615 Function that implements Doppler Beam Swinging (DBS) technique.
1858 Function that implements Doppler Beam Swinging (DBS) technique.
616
1859
617 Input: Radial velocities, Direction cosines (x and y) of the Beam, Antenna azimuth,
1860 Input: Radial velocities, Direction cosines (x and y) of the Beam, Antenna azimuth,
618 Direction correction (if necessary), Ranges and SNR
1861 Direction correction (if necessary), Ranges and SNR
619
1862
620 Output: Winds estimation (Zonal, Meridional and Vertical)
1863 Output: Winds estimation (Zonal, Meridional and Vertical)
621
1864
622 Parameters affected: Winds, height range, SNR
1865 Parameters affected: Winds, height range, SNR
623 """
1866 """
624 velRadial0 = kwargs['velRadial']
1867 velRadial0 = kwargs['velRadial']
625 heiRang = kwargs['heightList']
1868 heiRang = kwargs['heightList']
626 SNR0 = kwargs['SNR']
1869 SNR0 = kwargs['SNR']
627
1870
628 if kwargs.has_key('dirCosx') and kwargs.has_key('dirCosy'):
1871 if kwargs.has_key('dirCosx') and kwargs.has_key('dirCosy'):
629 theta_x = numpy.array(kwargs['dirCosx'])
1872 theta_x = numpy.array(kwargs['dirCosx'])
630 theta_y = numpy.array(kwargs['dirCosy'])
1873 theta_y = numpy.array(kwargs['dirCosy'])
@@ -632,7 +1875,7 class WindProfiler(Operation):
632 elev = numpy.array(kwargs['elevation'])
1875 elev = numpy.array(kwargs['elevation'])
633 azim = numpy.array(kwargs['azimuth'])
1876 azim = numpy.array(kwargs['azimuth'])
634 theta_x, theta_y = self.__calculateCosDir(elev, azim)
1877 theta_x, theta_y = self.__calculateCosDir(elev, azim)
635 azimuth = kwargs['correctAzimuth']
1878 azimuth = kwargs['correctAzimuth']
636 if kwargs.has_key('horizontalOnly'):
1879 if kwargs.has_key('horizontalOnly'):
637 horizontalOnly = kwargs['horizontalOnly']
1880 horizontalOnly = kwargs['horizontalOnly']
638 else: horizontalOnly = False
1881 else: horizontalOnly = False
@@ -647,22 +1890,22 class WindProfiler(Operation):
647 param = param[arrayChannel,:,:]
1890 param = param[arrayChannel,:,:]
648 theta_x = theta_x[arrayChannel]
1891 theta_x = theta_x[arrayChannel]
649 theta_y = theta_y[arrayChannel]
1892 theta_y = theta_y[arrayChannel]
650
1893
651 azimuth_arr, zenith_arr, dir_cosu, dir_cosv, dir_cosw = self.__calculateAngles(theta_x, theta_y, azimuth)
1894 azimuth_arr, zenith_arr, dir_cosu, dir_cosv, dir_cosw = self.__calculateAngles(theta_x, theta_y, azimuth)
652 heiRang1, velRadial1, SNR1 = self.__correctValues(heiRang, zenith_arr, correctFactor*velRadial0, SNR0)
1895 heiRang1, velRadial1, SNR1 = self.__correctValues(heiRang, zenith_arr, correctFactor*velRadial0, SNR0)
653 A = self.__calculateMatA(dir_cosu, dir_cosv, dir_cosw, horizontalOnly)
1896 A = self.__calculateMatA(dir_cosu, dir_cosv, dir_cosw, horizontalOnly)
654
1897
655 #Calculo de Componentes de la velocidad con DBS
1898 #Calculo de Componentes de la velocidad con DBS
656 winds = self.__calculateVelUVW(A,velRadial1)
1899 winds = self.__calculateVelUVW(A,velRadial1)
657
1900
658 return winds, heiRang1, SNR1
1901 return winds, heiRang1, SNR1
659
1902
660 def __calculateDistance(self, posx, posy, pairs_ccf, azimuth = None):
1903 def __calculateDistance(self, posx, posy, pairs_ccf, azimuth = None):
661
1904
662 nPairs = len(pairs_ccf)
1905 nPairs = len(pairs_ccf)
663 posx = numpy.asarray(posx)
1906 posx = numpy.asarray(posx)
664 posy = numpy.asarray(posy)
1907 posy = numpy.asarray(posy)
665
1908
666 #Rotacion Inversa para alinear con el azimuth
1909 #Rotacion Inversa para alinear con el azimuth
667 if azimuth!= None:
1910 if azimuth!= None:
668 azimuth = azimuth*math.pi/180
1911 azimuth = azimuth*math.pi/180
@@ -671,126 +1914,126 class WindProfiler(Operation):
671 else:
1914 else:
672 posx1 = posx
1915 posx1 = posx
673 posy1 = posy
1916 posy1 = posy
674
1917
675 #Calculo de Distancias
1918 #Calculo de Distancias
676 distx = numpy.zeros(nPairs)
1919 distx = numpy.zeros(nPairs)
677 disty = numpy.zeros(nPairs)
1920 disty = numpy.zeros(nPairs)
678 dist = numpy.zeros(nPairs)
1921 dist = numpy.zeros(nPairs)
679 ang = numpy.zeros(nPairs)
1922 ang = numpy.zeros(nPairs)
680
1923
681 for i in range(nPairs):
1924 for i in range(nPairs):
682 distx[i] = posx1[pairs_ccf[i][1]] - posx1[pairs_ccf[i][0]]
1925 distx[i] = posx1[pairs_ccf[i][1]] - posx1[pairs_ccf[i][0]]
683 disty[i] = posy1[pairs_ccf[i][1]] - posy1[pairs_ccf[i][0]]
1926 disty[i] = posy1[pairs_ccf[i][1]] - posy1[pairs_ccf[i][0]]
684 dist[i] = numpy.sqrt(distx[i]**2 + disty[i]**2)
1927 dist[i] = numpy.sqrt(distx[i]**2 + disty[i]**2)
685 ang[i] = numpy.arctan2(disty[i],distx[i])
1928 ang[i] = numpy.arctan2(disty[i],distx[i])
686
1929
687 return distx, disty, dist, ang
1930 return distx, disty, dist, ang
688 #Calculo de Matrices
1931 #Calculo de Matrices
689 # nPairs = len(pairs)
1932 # nPairs = len(pairs)
690 # ang1 = numpy.zeros((nPairs, 2, 1))
1933 # ang1 = numpy.zeros((nPairs, 2, 1))
691 # dist1 = numpy.zeros((nPairs, 2, 1))
1934 # dist1 = numpy.zeros((nPairs, 2, 1))
692 #
1935 #
693 # for j in range(nPairs):
1936 # for j in range(nPairs):
694 # dist1[j,0,0] = dist[pairs[j][0]]
1937 # dist1[j,0,0] = dist[pairs[j][0]]
695 # dist1[j,1,0] = dist[pairs[j][1]]
1938 # dist1[j,1,0] = dist[pairs[j][1]]
696 # ang1[j,0,0] = ang[pairs[j][0]]
1939 # ang1[j,0,0] = ang[pairs[j][0]]
697 # ang1[j,1,0] = ang[pairs[j][1]]
1940 # ang1[j,1,0] = ang[pairs[j][1]]
698 #
1941 #
699 # return distx,disty, dist1,ang1
1942 # return distx,disty, dist1,ang1
700
1943
701
1944
702 def __calculateVelVer(self, phase, lagTRange, _lambda):
1945 def __calculateVelVer(self, phase, lagTRange, _lambda):
703
1946
704 Ts = lagTRange[1] - lagTRange[0]
1947 Ts = lagTRange[1] - lagTRange[0]
705 velW = -_lambda*phase/(4*math.pi*Ts)
1948 velW = -_lambda*phase/(4*math.pi*Ts)
706
1949
707 return velW
1950 return velW
708
1951
709 def __calculateVelHorDir(self, dist, tau1, tau2, ang):
1952 def __calculateVelHorDir(self, dist, tau1, tau2, ang):
710 nPairs = tau1.shape[0]
1953 nPairs = tau1.shape[0]
711 nHeights = tau1.shape[1]
1954 nHeights = tau1.shape[1]
712 vel = numpy.zeros((nPairs,3,nHeights))
1955 vel = numpy.zeros((nPairs,3,nHeights))
713 dist1 = numpy.reshape(dist, (dist.size,1))
1956 dist1 = numpy.reshape(dist, (dist.size,1))
714
1957
715 angCos = numpy.cos(ang)
1958 angCos = numpy.cos(ang)
716 angSin = numpy.sin(ang)
1959 angSin = numpy.sin(ang)
717
1960
718 vel0 = dist1*tau1/(2*tau2**2)
1961 vel0 = dist1*tau1/(2*tau2**2)
719 vel[:,0,:] = (vel0*angCos).sum(axis = 1)
1962 vel[:,0,:] = (vel0*angCos).sum(axis = 1)
720 vel[:,1,:] = (vel0*angSin).sum(axis = 1)
1963 vel[:,1,:] = (vel0*angSin).sum(axis = 1)
721
1964
722 ind = numpy.where(numpy.isinf(vel))
1965 ind = numpy.where(numpy.isinf(vel))
723 vel[ind] = numpy.nan
1966 vel[ind] = numpy.nan
724
1967
725 return vel
1968 return vel
726
1969
727 # def __getPairsAutoCorr(self, pairsList, nChannels):
1970 # def __getPairsAutoCorr(self, pairsList, nChannels):
728 #
1971 #
729 # pairsAutoCorr = numpy.zeros(nChannels, dtype = 'int')*numpy.nan
1972 # pairsAutoCorr = numpy.zeros(nChannels, dtype = 'int')*numpy.nan
730 #
1973 #
731 # for l in range(len(pairsList)):
1974 # for l in range(len(pairsList)):
732 # firstChannel = pairsList[l][0]
1975 # firstChannel = pairsList[l][0]
733 # secondChannel = pairsList[l][1]
1976 # secondChannel = pairsList[l][1]
734 #
1977 #
735 # #Obteniendo pares de Autocorrelacion
1978 # #Obteniendo pares de Autocorrelacion
736 # if firstChannel == secondChannel:
1979 # if firstChannel == secondChannel:
737 # pairsAutoCorr[firstChannel] = int(l)
1980 # pairsAutoCorr[firstChannel] = int(l)
738 #
1981 #
739 # pairsAutoCorr = pairsAutoCorr.astype(int)
1982 # pairsAutoCorr = pairsAutoCorr.astype(int)
740 #
1983 #
741 # pairsCrossCorr = range(len(pairsList))
1984 # pairsCrossCorr = range(len(pairsList))
742 # pairsCrossCorr = numpy.delete(pairsCrossCorr,pairsAutoCorr)
1985 # pairsCrossCorr = numpy.delete(pairsCrossCorr,pairsAutoCorr)
743 #
1986 #
744 # return pairsAutoCorr, pairsCrossCorr
1987 # return pairsAutoCorr, pairsCrossCorr
745
1988
746 # def techniqueSA(self, pairsSelected, pairsList, nChannels, tau, azimuth, _lambda, position_x, position_y, lagTRange, correctFactor):
1989 # def techniqueSA(self, pairsSelected, pairsList, nChannels, tau, azimuth, _lambda, position_x, position_y, lagTRange, correctFactor):
747 def techniqueSA(self, kwargs):
1990 def techniqueSA(self, kwargs):
748
1991
749 """
1992 """
750 Function that implements Spaced Antenna (SA) technique.
1993 Function that implements Spaced Antenna (SA) technique.
751
1994
752 Input: Radial velocities, Direction cosines (x and y) of the Beam, Antenna azimuth,
1995 Input: Radial velocities, Direction cosines (x and y) of the Beam, Antenna azimuth,
753 Direction correction (if necessary), Ranges and SNR
1996 Direction correction (if necessary), Ranges and SNR
754
1997
755 Output: Winds estimation (Zonal, Meridional and Vertical)
1998 Output: Winds estimation (Zonal, Meridional and Vertical)
756
1999
757 Parameters affected: Winds
2000 Parameters affected: Winds
758 """
2001 """
759 position_x = kwargs['positionX']
2002 position_x = kwargs['positionX']
760 position_y = kwargs['positionY']
2003 position_y = kwargs['positionY']
761 azimuth = kwargs['azimuth']
2004 azimuth = kwargs['azimuth']
762
2005
763 if kwargs.has_key('correctFactor'):
2006 if kwargs.has_key('correctFactor'):
764 correctFactor = kwargs['correctFactor']
2007 correctFactor = kwargs['correctFactor']
765 else:
2008 else:
766 correctFactor = 1
2009 correctFactor = 1
767
2010
768 groupList = kwargs['groupList']
2011 groupList = kwargs['groupList']
769 pairs_ccf = groupList[1]
2012 pairs_ccf = groupList[1]
770 tau = kwargs['tau']
2013 tau = kwargs['tau']
771 _lambda = kwargs['_lambda']
2014 _lambda = kwargs['_lambda']
772
2015
773 #Cross Correlation pairs obtained
2016 #Cross Correlation pairs obtained
774 # pairsAutoCorr, pairsCrossCorr = self.__getPairsAutoCorr(pairssList, nChannels)
2017 # pairsAutoCorr, pairsCrossCorr = self.__getPairsAutoCorr(pairssList, nChannels)
775 # pairsArray = numpy.array(pairsList)[pairsCrossCorr]
2018 # pairsArray = numpy.array(pairsList)[pairsCrossCorr]
776 # pairsSelArray = numpy.array(pairsSelected)
2019 # pairsSelArray = numpy.array(pairsSelected)
777 # pairs = []
2020 # pairs = []
778 #
2021 #
779 # #Wind estimation pairs obtained
2022 # #Wind estimation pairs obtained
780 # for i in range(pairsSelArray.shape[0]/2):
2023 # for i in range(pairsSelArray.shape[0]/2):
781 # ind1 = numpy.where(numpy.all(pairsArray == pairsSelArray[2*i], axis = 1))[0][0]
2024 # ind1 = numpy.where(numpy.all(pairsArray == pairsSelArray[2*i], axis = 1))[0][0]
782 # ind2 = numpy.where(numpy.all(pairsArray == pairsSelArray[2*i + 1], axis = 1))[0][0]
2025 # ind2 = numpy.where(numpy.all(pairsArray == pairsSelArray[2*i + 1], axis = 1))[0][0]
783 # pairs.append((ind1,ind2))
2026 # pairs.append((ind1,ind2))
784
2027
785 indtau = tau.shape[0]/2
2028 indtau = tau.shape[0]/2
786 tau1 = tau[:indtau,:]
2029 tau1 = tau[:indtau,:]
787 tau2 = tau[indtau:-1,:]
2030 tau2 = tau[indtau:-1,:]
788 # tau1 = tau1[pairs,:]
2031 # tau1 = tau1[pairs,:]
789 # tau2 = tau2[pairs,:]
2032 # tau2 = tau2[pairs,:]
790 phase1 = tau[-1,:]
2033 phase1 = tau[-1,:]
791
2034
792 #---------------------------------------------------------------------
2035 #---------------------------------------------------------------------
793 #Metodo Directo
2036 #Metodo Directo
794 distx, disty, dist, ang = self.__calculateDistance(position_x, position_y, pairs_ccf,azimuth)
2037 distx, disty, dist, ang = self.__calculateDistance(position_x, position_y, pairs_ccf,azimuth)
795 winds = self.__calculateVelHorDir(dist, tau1, tau2, ang)
2038 winds = self.__calculateVelHorDir(dist, tau1, tau2, ang)
796 winds = stats.nanmean(winds, axis=0)
2039 winds = stats.nanmean(winds, axis=0)
@@ -806,100 +2049,100 class WindProfiler(Operation):
806 winds[2,:] = self.__calculateVelVer(phase1, lagTRange, _lambda)
2049 winds[2,:] = self.__calculateVelVer(phase1, lagTRange, _lambda)
807 winds = correctFactor*winds
2050 winds = correctFactor*winds
808 return winds
2051 return winds
809
2052
810 def __checkTime(self, currentTime, paramInterval, outputInterval):
2053 def __checkTime(self, currentTime, paramInterval, outputInterval):
811
2054
812 dataTime = currentTime + paramInterval
2055 dataTime = currentTime + paramInterval
813 deltaTime = dataTime - self.__initime
2056 deltaTime = dataTime - self.__initime
814
2057
815 if deltaTime >= outputInterval or deltaTime < 0:
2058 if deltaTime >= outputInterval or deltaTime < 0:
816 self.__dataReady = True
2059 self.__dataReady = True
817 return
2060 return
818
2061
819 def techniqueMeteors(self, arrayMeteor, meteorThresh, heightMin, heightMax, binkm=2):
2062 def techniqueMeteors(self, arrayMeteor, meteorThresh, heightMin, heightMax):
820 '''
2063 '''
821 Function that implements winds estimation technique with detected meteors.
2064 Function that implements winds estimation technique with detected meteors.
822
2065
823 Input: Detected meteors, Minimum meteor quantity to wind estimation
2066 Input: Detected meteors, Minimum meteor quantity to wind estimation
824
2067
825 Output: Winds estimation (Zonal and Meridional)
2068 Output: Winds estimation (Zonal and Meridional)
826
2069
827 Parameters affected: Winds
2070 Parameters affected: Winds
828 '''
2071 '''
829 # print arrayMeteor.shape
2072 # print arrayMeteor.shape
830 #Settings
2073 #Settings
831 nInt = (heightMax - heightMin)/binkm
2074 nInt = (heightMax - heightMin)/2
832 # print nInt
2075 # print nInt
833 nInt = int(nInt)
2076 nInt = int(nInt)
834 # print nInt
2077 # print nInt
835 winds = numpy.zeros((2,nInt))*numpy.nan
2078 winds = numpy.zeros((2,nInt))*numpy.nan
836
2079
837 #Filter errors
2080 #Filter errors
838 error = numpy.where(arrayMeteor[:,-1] == 0)[0]
2081 error = numpy.where(arrayMeteor[:,-1] == 0)[0]
839 finalMeteor = arrayMeteor[error,:]
2082 finalMeteor = arrayMeteor[error,:]
840
2083
841 #Meteor Histogram
2084 #Meteor Histogram
842 finalHeights = finalMeteor[:,2]
2085 finalHeights = finalMeteor[:,2]
843 hist = numpy.histogram(finalHeights, bins = nInt, range = (heightMin,heightMax))
2086 hist = numpy.histogram(finalHeights, bins = nInt, range = (heightMin,heightMax))
844 nMeteorsPerI = hist[0]
2087 nMeteorsPerI = hist[0]
845 heightPerI = hist[1]
2088 heightPerI = hist[1]
846
2089
847 #Sort of meteors
2090 #Sort of meteors
848 indSort = finalHeights.argsort()
2091 indSort = finalHeights.argsort()
849 finalMeteor2 = finalMeteor[indSort,:]
2092 finalMeteor2 = finalMeteor[indSort,:]
850
2093
851 # Calculating winds
2094 # Calculating winds
852 ind1 = 0
2095 ind1 = 0
853 ind2 = 0
2096 ind2 = 0
854
2097
855 for i in range(nInt):
2098 for i in range(nInt):
856 nMet = nMeteorsPerI[i]
2099 nMet = nMeteorsPerI[i]
857 ind1 = ind2
2100 ind1 = ind2
858 ind2 = ind1 + nMet
2101 ind2 = ind1 + nMet
859
2102
860 meteorAux = finalMeteor2[ind1:ind2,:]
2103 meteorAux = finalMeteor2[ind1:ind2,:]
861
2104
862 if meteorAux.shape[0] >= meteorThresh:
2105 if meteorAux.shape[0] >= meteorThresh:
863 vel = meteorAux[:, 6]
2106 vel = meteorAux[:, 6]
864 zen = meteorAux[:, 4]*numpy.pi/180
2107 zen = meteorAux[:, 4]*numpy.pi/180
865 azim = meteorAux[:, 3]*numpy.pi/180
2108 azim = meteorAux[:, 3]*numpy.pi/180
866
2109
867 n = numpy.cos(zen)
2110 n = numpy.cos(zen)
868 # m = (1 - n**2)/(1 - numpy.tan(azim)**2)
2111 # m = (1 - n**2)/(1 - numpy.tan(azim)**2)
869 # l = m*numpy.tan(azim)
2112 # l = m*numpy.tan(azim)
870 l = numpy.sin(zen)*numpy.sin(azim)
2113 l = numpy.sin(zen)*numpy.sin(azim)
871 m = numpy.sin(zen)*numpy.cos(azim)
2114 m = numpy.sin(zen)*numpy.cos(azim)
872
2115
873 A = numpy.vstack((l, m)).transpose()
2116 A = numpy.vstack((l, m)).transpose()
874 A1 = numpy.dot(numpy.linalg.inv( numpy.dot(A.transpose(),A) ),A.transpose())
2117 A1 = numpy.dot(numpy.linalg.inv( numpy.dot(A.transpose(),A) ),A.transpose())
875 windsAux = numpy.dot(A1, vel)
2118 windsAux = numpy.dot(A1, vel)
876
2119
877 winds[0,i] = windsAux[0]
2120 winds[0,i] = windsAux[0]
878 winds[1,i] = windsAux[1]
2121 winds[1,i] = windsAux[1]
879
2122
880 return winds, heightPerI[:-1]
2123 return winds, heightPerI[:-1]
881
2124
882 def techniqueNSM_SA(self, **kwargs):
2125 def techniqueNSM_SA(self, **kwargs):
883 metArray = kwargs['metArray']
2126 metArray = kwargs['metArray']
884 heightList = kwargs['heightList']
2127 heightList = kwargs['heightList']
885 timeList = kwargs['timeList']
2128 timeList = kwargs['timeList']
886
2129
887 rx_location = kwargs['rx_location']
2130 rx_location = kwargs['rx_location']
888 groupList = kwargs['groupList']
2131 groupList = kwargs['groupList']
889 azimuth = kwargs['azimuth']
2132 azimuth = kwargs['azimuth']
890 dfactor = kwargs['dfactor']
2133 dfactor = kwargs['dfactor']
891 k = kwargs['k']
2134 k = kwargs['k']
892
2135
893 azimuth1, dist = self.__calculateAzimuth1(rx_location, groupList, azimuth)
2136 azimuth1, dist = self.__calculateAzimuth1(rx_location, groupList, azimuth)
894 d = dist*dfactor
2137 d = dist*dfactor
895 #Phase calculation
2138 #Phase calculation
896 metArray1 = self.__getPhaseSlope(metArray, heightList, timeList)
2139 metArray1 = self.__getPhaseSlope(metArray, heightList, timeList)
897
2140
898 metArray1[:,-2] = metArray1[:,-2]*metArray1[:,2]*1000/(k*d[metArray1[:,1].astype(int)]) #angles into velocities
2141 metArray1[:,-2] = metArray1[:,-2]*metArray1[:,2]*1000/(k*d[metArray1[:,1].astype(int)]) #angles into velocities
899
2142
900 velEst = numpy.zeros((heightList.size,2))*numpy.nan
2143 velEst = numpy.zeros((heightList.size,2))*numpy.nan
901 azimuth1 = azimuth1*numpy.pi/180
2144 azimuth1 = azimuth1*numpy.pi/180
902
2145
903 for i in range(heightList.size):
2146 for i in range(heightList.size):
904 h = heightList[i]
2147 h = heightList[i]
905 indH = numpy.where((metArray1[:,2] == h)&(numpy.abs(metArray1[:,-2]) < 100))[0]
2148 indH = numpy.where((metArray1[:,2] == h)&(numpy.abs(metArray1[:,-2]) < 100))[0]
@@ -912,71 +2155,71 class WindProfiler(Operation):
912 A = numpy.asmatrix(A)
2155 A = numpy.asmatrix(A)
913 A1 = numpy.linalg.pinv(A.transpose()*A)*A.transpose()
2156 A1 = numpy.linalg.pinv(A.transpose()*A)*A.transpose()
914 velHor = numpy.dot(A1,velAux)
2157 velHor = numpy.dot(A1,velAux)
915
2158
916 velEst[i,:] = numpy.squeeze(velHor)
2159 velEst[i,:] = numpy.squeeze(velHor)
917 return velEst
2160 return velEst
918
2161
919 def __getPhaseSlope(self, metArray, heightList, timeList):
2162 def __getPhaseSlope(self, metArray, heightList, timeList):
920 meteorList = []
2163 meteorList = []
921 #utctime sec1 height SNR velRad ph0 ph1 ph2 coh0 coh1 coh2
2164 #utctime sec1 height SNR velRad ph0 ph1 ph2 coh0 coh1 coh2
922 #Putting back together the meteor matrix
2165 #Putting back together the meteor matrix
923 utctime = metArray[:,0]
2166 utctime = metArray[:,0]
924 uniqueTime = numpy.unique(utctime)
2167 uniqueTime = numpy.unique(utctime)
925
2168
926 phaseDerThresh = 0.5
2169 phaseDerThresh = 0.5
927 ippSeconds = timeList[1] - timeList[0]
2170 ippSeconds = timeList[1] - timeList[0]
928 sec = numpy.where(timeList>1)[0][0]
2171 sec = numpy.where(timeList>1)[0][0]
929 nPairs = metArray.shape[1] - 6
2172 nPairs = metArray.shape[1] - 6
930 nHeights = len(heightList)
2173 nHeights = len(heightList)
931
2174
932 for t in uniqueTime:
2175 for t in uniqueTime:
933 metArray1 = metArray[utctime==t,:]
2176 metArray1 = metArray[utctime==t,:]
934 # phaseDerThresh = numpy.pi/4 #reducir Phase thresh
2177 # phaseDerThresh = numpy.pi/4 #reducir Phase thresh
935 tmet = metArray1[:,1].astype(int)
2178 tmet = metArray1[:,1].astype(int)
936 hmet = metArray1[:,2].astype(int)
2179 hmet = metArray1[:,2].astype(int)
937
2180
938 metPhase = numpy.zeros((nPairs, heightList.size, timeList.size - 1))
2181 metPhase = numpy.zeros((nPairs, heightList.size, timeList.size - 1))
939 metPhase[:,:] = numpy.nan
2182 metPhase[:,:] = numpy.nan
940 metPhase[:,hmet,tmet] = metArray1[:,6:].T
2183 metPhase[:,hmet,tmet] = metArray1[:,6:].T
941
2184
942 #Delete short trails
2185 #Delete short trails
943 metBool = ~numpy.isnan(metPhase[0,:,:])
2186 metBool = ~numpy.isnan(metPhase[0,:,:])
944 heightVect = numpy.sum(metBool, axis = 1)
2187 heightVect = numpy.sum(metBool, axis = 1)
945 metBool[heightVect<sec,:] = False
2188 metBool[heightVect<sec,:] = False
946 metPhase[:,heightVect<sec,:] = numpy.nan
2189 metPhase[:,heightVect<sec,:] = numpy.nan
947
2190
948 #Derivative
2191 #Derivative
949 metDer = numpy.abs(metPhase[:,:,1:] - metPhase[:,:,:-1])
2192 metDer = numpy.abs(metPhase[:,:,1:] - metPhase[:,:,:-1])
950 phDerAux = numpy.dstack((numpy.full((nPairs,nHeights,1), False, dtype=bool),metDer > phaseDerThresh))
2193 phDerAux = numpy.dstack((numpy.full((nPairs,nHeights,1), False, dtype=bool),metDer > phaseDerThresh))
951 metPhase[phDerAux] = numpy.nan
2194 metPhase[phDerAux] = numpy.nan
952
2195
953 #--------------------------METEOR DETECTION -----------------------------------------
2196 #--------------------------METEOR DETECTION -----------------------------------------
954 indMet = numpy.where(numpy.any(metBool,axis=1))[0]
2197 indMet = numpy.where(numpy.any(metBool,axis=1))[0]
955
2198
956 for p in numpy.arange(nPairs):
2199 for p in numpy.arange(nPairs):
957 phase = metPhase[p,:,:]
2200 phase = metPhase[p,:,:]
958 phDer = metDer[p,:,:]
2201 phDer = metDer[p,:,:]
959
2202
960 for h in indMet:
2203 for h in indMet:
961 height = heightList[h]
2204 height = heightList[h]
962 phase1 = phase[h,:] #82
2205 phase1 = phase[h,:] #82
963 phDer1 = phDer[h,:]
2206 phDer1 = phDer[h,:]
964
2207
965 phase1[~numpy.isnan(phase1)] = numpy.unwrap(phase1[~numpy.isnan(phase1)]) #Unwrap
2208 phase1[~numpy.isnan(phase1)] = numpy.unwrap(phase1[~numpy.isnan(phase1)]) #Unwrap
966
2209
967 indValid = numpy.where(~numpy.isnan(phase1))[0]
2210 indValid = numpy.where(~numpy.isnan(phase1))[0]
968 initMet = indValid[0]
2211 initMet = indValid[0]
969 endMet = 0
2212 endMet = 0
970
2213
971 for i in range(len(indValid)-1):
2214 for i in range(len(indValid)-1):
972
2215
973 #Time difference
2216 #Time difference
974 inow = indValid[i]
2217 inow = indValid[i]
975 inext = indValid[i+1]
2218 inext = indValid[i+1]
976 idiff = inext - inow
2219 idiff = inext - inow
977 #Phase difference
2220 #Phase difference
978 phDiff = numpy.abs(phase1[inext] - phase1[inow])
2221 phDiff = numpy.abs(phase1[inext] - phase1[inow])
979
2222
980 if idiff>sec or phDiff>numpy.pi/4 or inext==indValid[-1]: #End of Meteor
2223 if idiff>sec or phDiff>numpy.pi/4 or inext==indValid[-1]: #End of Meteor
981 sizeTrail = inow - initMet + 1
2224 sizeTrail = inow - initMet + 1
982 if sizeTrail>3*sec: #Too short meteors
2225 if sizeTrail>3*sec: #Too short meteors
@@ -992,28 +2235,28 class WindProfiler(Operation):
992 vel = slope#*height*1000/(k*d)
2235 vel = slope#*height*1000/(k*d)
993 estAux = numpy.array([utctime,p,height, vel, rsq])
2236 estAux = numpy.array([utctime,p,height, vel, rsq])
994 meteorList.append(estAux)
2237 meteorList.append(estAux)
995 initMet = inext
2238 initMet = inext
996 metArray2 = numpy.array(meteorList)
2239 metArray2 = numpy.array(meteorList)
997
2240
998 return metArray2
2241 return metArray2
999
2242
1000 def __calculateAzimuth1(self, rx_location, pairslist, azimuth0):
2243 def __calculateAzimuth1(self, rx_location, pairslist, azimuth0):
1001
2244
1002 azimuth1 = numpy.zeros(len(pairslist))
2245 azimuth1 = numpy.zeros(len(pairslist))
1003 dist = numpy.zeros(len(pairslist))
2246 dist = numpy.zeros(len(pairslist))
1004
2247
1005 for i in range(len(rx_location)):
2248 for i in range(len(rx_location)):
1006 ch0 = pairslist[i][0]
2249 ch0 = pairslist[i][0]
1007 ch1 = pairslist[i][1]
2250 ch1 = pairslist[i][1]
1008
2251
1009 diffX = rx_location[ch0][0] - rx_location[ch1][0]
2252 diffX = rx_location[ch0][0] - rx_location[ch1][0]
1010 diffY = rx_location[ch0][1] - rx_location[ch1][1]
2253 diffY = rx_location[ch0][1] - rx_location[ch1][1]
1011 azimuth1[i] = numpy.arctan2(diffY,diffX)*180/numpy.pi
2254 azimuth1[i] = numpy.arctan2(diffY,diffX)*180/numpy.pi
1012 dist[i] = numpy.sqrt(diffX**2 + diffY**2)
2255 dist[i] = numpy.sqrt(diffX**2 + diffY**2)
1013
2256
1014 azimuth1 -= azimuth0
2257 azimuth1 -= azimuth0
1015 return azimuth1, dist
2258 return azimuth1, dist
1016
2259
1017 def techniqueNSM_DBS(self, **kwargs):
2260 def techniqueNSM_DBS(self, **kwargs):
1018 metArray = kwargs['metArray']
2261 metArray = kwargs['metArray']
1019 heightList = kwargs['heightList']
2262 heightList = kwargs['heightList']
@@ -1068,39 +2311,39 class WindProfiler(Operation):
1068 # noise = dataOut.noise
2311 # noise = dataOut.noise
1069 heightList = dataOut.heightList
2312 heightList = dataOut.heightList
1070 SNR = dataOut.data_SNR
2313 SNR = dataOut.data_SNR
1071
2314
1072 if technique == 'DBS':
2315 if technique == 'DBS':
1073
2316
1074 kwargs['velRadial'] = param[:,1,:] #Radial velocity
2317 kwargs['velRadial'] = param[:,1,:] #Radial velocity
1075 kwargs['heightList'] = heightList
2318 kwargs['heightList'] = heightList
1076 kwargs['SNR'] = SNR
2319 kwargs['SNR'] = SNR
1077
2320
1078 dataOut.data_output, dataOut.heightList, dataOut.data_SNR = self.techniqueDBS(kwargs) #DBS Function
2321 dataOut.data_output, dataOut.heightList, dataOut.data_SNR = self.techniqueDBS(kwargs) #DBS Function
1079 dataOut.utctimeInit = dataOut.utctime
2322 dataOut.utctimeInit = dataOut.utctime
1080 dataOut.outputInterval = dataOut.paramInterval
2323 dataOut.outputInterval = dataOut.paramInterval
1081
2324
1082 elif technique == 'SA':
2325 elif technique == 'SA':
1083
2326
1084 #Parameters
2327 #Parameters
1085 # position_x = kwargs['positionX']
2328 # position_x = kwargs['positionX']
1086 # position_y = kwargs['positionY']
2329 # position_y = kwargs['positionY']
1087 # azimuth = kwargs['azimuth']
2330 # azimuth = kwargs['azimuth']
1088 #
2331 #
1089 # if kwargs.has_key('crosspairsList'):
2332 # if kwargs.has_key('crosspairsList'):
1090 # pairs = kwargs['crosspairsList']
2333 # pairs = kwargs['crosspairsList']
1091 # else:
2334 # else:
1092 # pairs = None
2335 # pairs = None
1093 #
2336 #
1094 # if kwargs.has_key('correctFactor'):
2337 # if kwargs.has_key('correctFactor'):
1095 # correctFactor = kwargs['correctFactor']
2338 # correctFactor = kwargs['correctFactor']
1096 # else:
2339 # else:
1097 # correctFactor = 1
2340 # correctFactor = 1
1098
2341
1099 # tau = dataOut.data_param
2342 # tau = dataOut.data_param
1100 # _lambda = dataOut.C/dataOut.frequency
2343 # _lambda = dataOut.C/dataOut.frequency
1101 # pairsList = dataOut.groupList
2344 # pairsList = dataOut.groupList
1102 # nChannels = dataOut.nChannels
2345 # nChannels = dataOut.nChannels
1103
2346
1104 kwargs['groupList'] = dataOut.groupList
2347 kwargs['groupList'] = dataOut.groupList
1105 kwargs['tau'] = dataOut.data_param
2348 kwargs['tau'] = dataOut.data_param
1106 kwargs['_lambda'] = dataOut.C/dataOut.frequency
2349 kwargs['_lambda'] = dataOut.C/dataOut.frequency
@@ -1108,35 +2351,30 class WindProfiler(Operation):
1108 dataOut.data_output = self.techniqueSA(kwargs)
2351 dataOut.data_output = self.techniqueSA(kwargs)
1109 dataOut.utctimeInit = dataOut.utctime
2352 dataOut.utctimeInit = dataOut.utctime
1110 dataOut.outputInterval = dataOut.timeInterval
2353 dataOut.outputInterval = dataOut.timeInterval
1111
2354
1112 elif technique == 'Meteors':
2355 elif technique == 'Meteors':
1113 dataOut.flagNoData = True
2356 dataOut.flagNoData = True
1114 self.__dataReady = False
2357 self.__dataReady = False
1115
2358
1116 if kwargs.has_key('nHours'):
2359 if kwargs.has_key('nHours'):
1117 nHours = kwargs['nHours']
2360 nHours = kwargs['nHours']
1118 else:
2361 else:
1119 nHours = 1
2362 nHours = 1
1120
2363
1121 if kwargs.has_key('meteorsPerBin'):
2364 if kwargs.has_key('meteorsPerBin'):
1122 meteorThresh = kwargs['meteorsPerBin']
2365 meteorThresh = kwargs['meteorsPerBin']
1123 else:
2366 else:
1124 meteorThresh = 6
2367 meteorThresh = 6
1125
2368
1126 if kwargs.has_key('hmin'):
2369 if kwargs.has_key('hmin'):
1127 hmin = kwargs['hmin']
2370 hmin = kwargs['hmin']
1128 else: hmin = 70
2371 else: hmin = 70
1129 if kwargs.has_key('hmax'):
2372 if kwargs.has_key('hmax'):
1130 hmax = kwargs['hmax']
2373 hmax = kwargs['hmax']
1131 else: hmax = 110
2374 else: hmax = 110
1132
2375
1133 if kwargs.has_key('BinKm'):
1134 binkm = kwargs['BinKm']
1135 else:
1136 binkm = 2
1137
1138 dataOut.outputInterval = nHours*3600
2376 dataOut.outputInterval = nHours*3600
1139
2377
1140 if self.__isConfig == False:
2378 if self.__isConfig == False:
1141 # self.__initime = dataOut.datatime.replace(minute = 0, second = 0, microsecond = 03)
2379 # self.__initime = dataOut.datatime.replace(minute = 0, second = 0, microsecond = 03)
1142 #Get Initial LTC time
2380 #Get Initial LTC time
@@ -1144,29 +2382,29 class WindProfiler(Operation):
1144 self.__initime = (self.__initime.replace(minute = 0, second = 0, microsecond = 0) - datetime.datetime(1970, 1, 1)).total_seconds()
2382 self.__initime = (self.__initime.replace(minute = 0, second = 0, microsecond = 0) - datetime.datetime(1970, 1, 1)).total_seconds()
1145
2383
1146 self.__isConfig = True
2384 self.__isConfig = True
1147
2385
1148 if self.__buffer is None:
2386 if self.__buffer == None:
1149 self.__buffer = dataOut.data_param
2387 self.__buffer = dataOut.data_param
1150 self.__firstdata = copy.copy(dataOut)
2388 self.__firstdata = copy.copy(dataOut)
1151
2389
1152 else:
2390 else:
1153 self.__buffer = numpy.vstack((self.__buffer, dataOut.data_param))
2391 self.__buffer = numpy.vstack((self.__buffer, dataOut.data_param))
1154
2392
1155 self.__checkTime(dataOut.utctime, dataOut.paramInterval, dataOut.outputInterval) #Check if the buffer is ready
2393 self.__checkTime(dataOut.utctime, dataOut.paramInterval, dataOut.outputInterval) #Check if the buffer is ready
1156
2394
1157 if self.__dataReady:
2395 if self.__dataReady:
1158 dataOut.utctimeInit = self.__initime
2396 dataOut.utctimeInit = self.__initime
1159
2397
1160 self.__initime += dataOut.outputInterval #to erase time offset
2398 self.__initime += dataOut.outputInterval #to erase time offset
1161
2399
1162 dataOut.data_output, dataOut.heightList = self.techniqueMeteors(self.__buffer, meteorThresh, hmin, hmax, binkm)
2400 dataOut.data_output, dataOut.heightList = self.techniqueMeteors(self.__buffer, meteorThresh, hmin, hmax)
1163 dataOut.flagNoData = False
2401 dataOut.flagNoData = False
1164 self.__buffer = None
2402 self.__buffer = None
1165
2403
1166 elif technique == 'Meteors1':
2404 elif technique == 'Meteors1':
1167 dataOut.flagNoData = True
2405 dataOut.flagNoData = True
1168 self.__dataReady = False
2406 self.__dataReady = False
1169
2407
1170 if kwargs.has_key('nMins'):
2408 if kwargs.has_key('nMins'):
1171 nMins = kwargs['nMins']
2409 nMins = kwargs['nMins']
1172 else: nMins = 20
2410 else: nMins = 20
@@ -1187,17 +2425,17 class WindProfiler(Operation):
1187 else: mode = 'SA'
2425 else: mode = 'SA'
1188
2426
1189 #Borrar luego esto
2427 #Borrar luego esto
1190 if dataOut.groupList is None:
2428 if dataOut.groupList == None:
1191 dataOut.groupList = [(0,1),(0,2),(1,2)]
2429 dataOut.groupList = [(0,1),(0,2),(1,2)]
1192 groupList = dataOut.groupList
2430 groupList = dataOut.groupList
1193 C = 3e8
2431 C = 3e8
1194 freq = 50e6
2432 freq = 50e6
1195 lamb = C/freq
2433 lamb = C/freq
1196 k = 2*numpy.pi/lamb
2434 k = 2*numpy.pi/lamb
1197
2435
1198 timeList = dataOut.abscissaList
2436 timeList = dataOut.abscissaList
1199 heightList = dataOut.heightList
2437 heightList = dataOut.heightList
1200
2438
1201 if self.__isConfig == False:
2439 if self.__isConfig == False:
1202 dataOut.outputInterval = nMins*60
2440 dataOut.outputInterval = nMins*60
1203 # self.__initime = dataOut.datatime.replace(minute = 0, second = 0, microsecond = 03)
2441 # self.__initime = dataOut.datatime.replace(minute = 0, second = 0, microsecond = 03)
@@ -1208,20 +2446,20 class WindProfiler(Operation):
1208 self.__initime = (initime.replace(minute = minuteNew, second = 0, microsecond = 0) - datetime.datetime(1970, 1, 1)).total_seconds()
2446 self.__initime = (initime.replace(minute = minuteNew, second = 0, microsecond = 0) - datetime.datetime(1970, 1, 1)).total_seconds()
1209
2447
1210 self.__isConfig = True
2448 self.__isConfig = True
1211
2449
1212 if self.__buffer is None:
2450 if self.__buffer == None:
1213 self.__buffer = dataOut.data_param
2451 self.__buffer = dataOut.data_param
1214 self.__firstdata = copy.copy(dataOut)
2452 self.__firstdata = copy.copy(dataOut)
1215
2453
1216 else:
2454 else:
1217 self.__buffer = numpy.vstack((self.__buffer, dataOut.data_param))
2455 self.__buffer = numpy.vstack((self.__buffer, dataOut.data_param))
1218
2456
1219 self.__checkTime(dataOut.utctime, dataOut.paramInterval, dataOut.outputInterval) #Check if the buffer is ready
2457 self.__checkTime(dataOut.utctime, dataOut.paramInterval, dataOut.outputInterval) #Check if the buffer is ready
1220
2458
1221 if self.__dataReady:
2459 if self.__dataReady:
1222 dataOut.utctimeInit = self.__initime
2460 dataOut.utctimeInit = self.__initime
1223 self.__initime += dataOut.outputInterval #to erase time offset
2461 self.__initime += dataOut.outputInterval #to erase time offset
1224
2462
1225 metArray = self.__buffer
2463 metArray = self.__buffer
1226 if mode == 'SA':
2464 if mode == 'SA':
1227 dataOut.data_output = self.techniqueNSM_SA(rx_location=rx_location, groupList=groupList, azimuth=azimuth, dfactor=dfactor, k=k,metArray=metArray, heightList=heightList,timeList=timeList)
2465 dataOut.data_output = self.techniqueNSM_SA(rx_location=rx_location, groupList=groupList, azimuth=azimuth, dfactor=dfactor, k=k,metArray=metArray, heightList=heightList,timeList=timeList)
@@ -1232,69 +2470,71 class WindProfiler(Operation):
1232 self.__buffer = None
2470 self.__buffer = None
1233
2471
1234 return
2472 return
1235
2473
1236 class EWDriftsEstimation(Operation):
2474 class EWDriftsEstimation(Operation):
1237
2475
1238
2476 def __init__(self):
2477 Operation.__init__(self)
2478
1239 def __correctValues(self, heiRang, phi, velRadial, SNR):
2479 def __correctValues(self, heiRang, phi, velRadial, SNR):
1240 listPhi = phi.tolist()
2480 listPhi = phi.tolist()
1241 maxid = listPhi.index(max(listPhi))
2481 maxid = listPhi.index(max(listPhi))
1242 minid = listPhi.index(min(listPhi))
2482 minid = listPhi.index(min(listPhi))
1243
2483
1244 rango = range(len(phi))
2484 rango = range(len(phi))
1245 # rango = numpy.delete(rango,maxid)
2485 # rango = numpy.delete(rango,maxid)
1246
2486
1247 heiRang1 = heiRang*math.cos(phi[maxid])
2487 heiRang1 = heiRang*math.cos(phi[maxid])
1248 heiRangAux = heiRang*math.cos(phi[minid])
2488 heiRangAux = heiRang*math.cos(phi[minid])
1249 indOut = (heiRang1 < heiRangAux[0]).nonzero()
2489 indOut = (heiRang1 < heiRangAux[0]).nonzero()
1250 heiRang1 = numpy.delete(heiRang1,indOut)
2490 heiRang1 = numpy.delete(heiRang1,indOut)
1251
2491
1252 velRadial1 = numpy.zeros([len(phi),len(heiRang1)])
2492 velRadial1 = numpy.zeros([len(phi),len(heiRang1)])
1253 SNR1 = numpy.zeros([len(phi),len(heiRang1)])
2493 SNR1 = numpy.zeros([len(phi),len(heiRang1)])
1254
2494
1255 for i in rango:
2495 for i in rango:
1256 x = heiRang*math.cos(phi[i])
2496 x = heiRang*math.cos(phi[i])
1257 y1 = velRadial[i,:]
2497 y1 = velRadial[i,:]
1258 f1 = interpolate.interp1d(x,y1,kind = 'cubic')
2498 f1 = interpolate.interp1d(x,y1,kind = 'cubic')
1259
2499
1260 x1 = heiRang1
2500 x1 = heiRang1
1261 y11 = f1(x1)
2501 y11 = f1(x1)
1262
2502
1263 y2 = SNR[i,:]
2503 y2 = SNR[i,:]
1264 f2 = interpolate.interp1d(x,y2,kind = 'cubic')
2504 f2 = interpolate.interp1d(x,y2,kind = 'cubic')
1265 y21 = f2(x1)
2505 y21 = f2(x1)
1266
2506
1267 velRadial1[i,:] = y11
2507 velRadial1[i,:] = y11
1268 SNR1[i,:] = y21
2508 SNR1[i,:] = y21
1269
2509
1270 return heiRang1, velRadial1, SNR1
2510 return heiRang1, velRadial1, SNR1
1271
2511
1272 def run(self, dataOut, zenith, zenithCorrection):
2512 def run(self, dataOut, zenith, zenithCorrection):
1273 heiRang = dataOut.heightList
2513 heiRang = dataOut.heightList
1274 velRadial = dataOut.data_param[:,3,:]
2514 velRadial = dataOut.data_param[:,3,:]
1275 SNR = dataOut.data_SNR
2515 SNR = dataOut.data_SNR
1276
2516
1277 zenith = numpy.array(zenith)
2517 zenith = numpy.array(zenith)
1278 zenith -= zenithCorrection
2518 zenith -= zenithCorrection
1279 zenith *= numpy.pi/180
2519 zenith *= numpy.pi/180
1280
2520
1281 heiRang1, velRadial1, SNR1 = self.__correctValues(heiRang, numpy.abs(zenith), velRadial, SNR)
2521 heiRang1, velRadial1, SNR1 = self.__correctValues(heiRang, numpy.abs(zenith), velRadial, SNR)
1282
2522
1283 alp = zenith[0]
2523 alp = zenith[0]
1284 bet = zenith[1]
2524 bet = zenith[1]
1285
2525
1286 w_w = velRadial1[0,:]
2526 w_w = velRadial1[0,:]
1287 w_e = velRadial1[1,:]
2527 w_e = velRadial1[1,:]
1288
2528
1289 w = (w_w*numpy.sin(bet) - w_e*numpy.sin(alp))/(numpy.cos(alp)*numpy.sin(bet) - numpy.cos(bet)*numpy.sin(alp))
2529 w = (w_w*numpy.sin(bet) - w_e*numpy.sin(alp))/(numpy.cos(alp)*numpy.sin(bet) - numpy.cos(bet)*numpy.sin(alp))
1290 u = (w_w*numpy.cos(bet) - w_e*numpy.cos(alp))/(numpy.sin(alp)*numpy.cos(bet) - numpy.sin(bet)*numpy.cos(alp))
2530 u = (w_w*numpy.cos(bet) - w_e*numpy.cos(alp))/(numpy.sin(alp)*numpy.cos(bet) - numpy.sin(bet)*numpy.cos(alp))
1291
2531
1292 winds = numpy.vstack((u,w))
2532 winds = numpy.vstack((u,w))
1293
2533
1294 dataOut.heightList = heiRang1
2534 dataOut.heightList = heiRang1
1295 dataOut.data_output = winds
2535 dataOut.data_output = winds
1296 dataOut.data_SNR = SNR1
2536 dataOut.data_SNR = SNR1
1297
2537
1298 dataOut.utctimeInit = dataOut.utctime
2538 dataOut.utctimeInit = dataOut.utctime
1299 dataOut.outputInterval = dataOut.timeInterval
2539 dataOut.outputInterval = dataOut.timeInterval
1300 return
2540 return
@@ -1333,7 +2573,7 class NonSpecularMeteorDetection(Operation):
1333 SNR[i] = (power[i]-noise[i])/noise[i]
2573 SNR[i] = (power[i]-noise[i])/noise[i]
1334 SNRm = numpy.nanmean(SNR, axis = 0)
2574 SNRm = numpy.nanmean(SNR, axis = 0)
1335 SNRdB = 10*numpy.log10(SNR)
2575 SNRdB = 10*numpy.log10(SNR)
1336
2576
1337 if mode == 'SA':
2577 if mode == 'SA':
1338 dataOut.groupList = dataOut.groupList[1]
2578 dataOut.groupList = dataOut.groupList[1]
1339 nPairs = data_ccf.shape[0]
2579 nPairs = data_ccf.shape[0]
@@ -1341,22 +2581,22 class NonSpecularMeteorDetection(Operation):
1341 phase = numpy.zeros(data_ccf[:,0,:,:].shape)
2581 phase = numpy.zeros(data_ccf[:,0,:,:].shape)
1342 # phase1 = numpy.copy(phase)
2582 # phase1 = numpy.copy(phase)
1343 coh1 = numpy.zeros(data_ccf[:,0,:,:].shape)
2583 coh1 = numpy.zeros(data_ccf[:,0,:,:].shape)
1344
2584
1345 for p in range(nPairs):
2585 for p in range(nPairs):
1346 ch0 = pairsList[p][0]
2586 ch0 = pairsList[p][0]
1347 ch1 = pairsList[p][1]
2587 ch1 = pairsList[p][1]
1348 ccf = data_ccf[p,0,:,:]/numpy.sqrt(data_acf[ch0,0,:,:]*data_acf[ch1,0,:,:])
2588 ccf = data_ccf[p,0,:,:]/numpy.sqrt(data_acf[ch0,0,:,:]*data_acf[ch1,0,:,:])
1349 phase[p,:,:] = ndimage.median_filter(numpy.angle(ccf), size = (5,1)) #median filter
2589 phase[p,:,:] = ndimage.median_filter(numpy.angle(ccf), size = (5,1)) #median filter
1350 # phase1[p,:,:] = numpy.angle(ccf) #median filter
2590 # phase1[p,:,:] = numpy.angle(ccf) #median filter
1351 coh1[p,:,:] = ndimage.median_filter(numpy.abs(ccf), 5) #median filter
2591 coh1[p,:,:] = ndimage.median_filter(numpy.abs(ccf), 5) #median filter
1352 # coh1[p,:,:] = numpy.abs(ccf) #median filter
2592 # coh1[p,:,:] = numpy.abs(ccf) #median filter
1353 coh = numpy.nanmax(coh1, axis = 0)
2593 coh = numpy.nanmax(coh1, axis = 0)
1354 # struc = numpy.ones((5,1))
2594 # struc = numpy.ones((5,1))
1355 # coh = ndimage.morphology.grey_dilation(coh, size=(10,1))
2595 # coh = ndimage.morphology.grey_dilation(coh, size=(10,1))
1356 #---------------------- Radial Velocity ----------------------------
2596 #---------------------- Radial Velocity ----------------------------
1357 phaseAux = numpy.mean(numpy.angle(data_acf[:,1,:,:]), axis = 0)
2597 phaseAux = numpy.mean(numpy.angle(data_acf[:,1,:,:]), axis = 0)
1358 velRad = phaseAux*lamb/(4*numpy.pi*tSamp)
2598 velRad = phaseAux*lamb/(4*numpy.pi*tSamp)
1359
2599
1360 if allData:
2600 if allData:
1361 boolMetFin = ~numpy.isnan(SNRm)
2601 boolMetFin = ~numpy.isnan(SNRm)
1362 # coh[:-1,:] = numpy.nanmean(numpy.abs(phase[:,1:,:] - phase[:,:-1,:]),axis=0)
2602 # coh[:-1,:] = numpy.nanmean(numpy.abs(phase[:,1:,:] - phase[:,:-1,:]),axis=0)
@@ -1364,31 +2604,31 class NonSpecularMeteorDetection(Operation):
1364 #------------------------ Meteor mask ---------------------------------
2604 #------------------------ Meteor mask ---------------------------------
1365 # #SNR mask
2605 # #SNR mask
1366 # boolMet = (SNRdB>SNRthresh)#|(~numpy.isnan(SNRdB))
2606 # boolMet = (SNRdB>SNRthresh)#|(~numpy.isnan(SNRdB))
1367 #
2607 #
1368 # #Erase small objects
2608 # #Erase small objects
1369 # boolMet1 = self.__erase_small(boolMet, 2*sec, 5)
2609 # boolMet1 = self.__erase_small(boolMet, 2*sec, 5)
1370 #
2610 #
1371 # auxEEJ = numpy.sum(boolMet1,axis=0)
2611 # auxEEJ = numpy.sum(boolMet1,axis=0)
1372 # indOver = auxEEJ>nProfiles*0.8 #Use this later
2612 # indOver = auxEEJ>nProfiles*0.8 #Use this later
1373 # indEEJ = numpy.where(indOver)[0]
2613 # indEEJ = numpy.where(indOver)[0]
1374 # indNEEJ = numpy.where(~indOver)[0]
2614 # indNEEJ = numpy.where(~indOver)[0]
1375 #
2615 #
1376 # boolMetFin = boolMet1
2616 # boolMetFin = boolMet1
1377 #
2617 #
1378 # if indEEJ.size > 0:
2618 # if indEEJ.size > 0:
1379 # boolMet1[:,indEEJ] = False #Erase heights with EEJ
2619 # boolMet1[:,indEEJ] = False #Erase heights with EEJ
1380 #
2620 #
1381 # boolMet2 = coh > cohThresh
2621 # boolMet2 = coh > cohThresh
1382 # boolMet2 = self.__erase_small(boolMet2, 2*sec,5)
2622 # boolMet2 = self.__erase_small(boolMet2, 2*sec,5)
1383 #
2623 #
1384 # #Final Meteor mask
2624 # #Final Meteor mask
1385 # boolMetFin = boolMet1|boolMet2
2625 # boolMetFin = boolMet1|boolMet2
1386
2626
1387 #Coherence mask
2627 #Coherence mask
1388 boolMet1 = coh > 0.75
2628 boolMet1 = coh > 0.75
1389 struc = numpy.ones((30,1))
2629 struc = numpy.ones((30,1))
1390 boolMet1 = ndimage.morphology.binary_dilation(boolMet1, structure=struc)
2630 boolMet1 = ndimage.morphology.binary_dilation(boolMet1, structure=struc)
1391
2631
1392 #Derivative mask
2632 #Derivative mask
1393 derPhase = numpy.nanmean(numpy.abs(phase[:,1:,:] - phase[:,:-1,:]),axis=0)
2633 derPhase = numpy.nanmean(numpy.abs(phase[:,1:,:] - phase[:,:-1,:]),axis=0)
1394 boolMet2 = derPhase < 0.2
2634 boolMet2 = derPhase < 0.2
@@ -1405,7 +2645,7 class NonSpecularMeteorDetection(Operation):
1405
2645
1406 tmet = coordMet[0]
2646 tmet = coordMet[0]
1407 hmet = coordMet[1]
2647 hmet = coordMet[1]
1408
2648
1409 data_param = numpy.zeros((tmet.size, 6 + nPairs))
2649 data_param = numpy.zeros((tmet.size, 6 + nPairs))
1410 data_param[:,0] = utctime
2650 data_param[:,0] = utctime
1411 data_param[:,1] = tmet
2651 data_param[:,1] = tmet
@@ -1414,7 +2654,7 class NonSpecularMeteorDetection(Operation):
1414 data_param[:,4] = velRad[tmet,hmet]
2654 data_param[:,4] = velRad[tmet,hmet]
1415 data_param[:,5] = coh[tmet,hmet]
2655 data_param[:,5] = coh[tmet,hmet]
1416 data_param[:,6:] = phase[:,tmet,hmet].T
2656 data_param[:,6:] = phase[:,tmet,hmet].T
1417
2657
1418 elif mode == 'DBS':
2658 elif mode == 'DBS':
1419 dataOut.groupList = numpy.arange(nChannels)
2659 dataOut.groupList = numpy.arange(nChannels)
1420
2660
@@ -1422,7 +2662,7 class NonSpecularMeteorDetection(Operation):
1422 phase = numpy.angle(data_acf[:,1,:,:])
2662 phase = numpy.angle(data_acf[:,1,:,:])
1423 # phase = ndimage.median_filter(numpy.angle(data_acf[:,1,:,:]), size = (1,5,1))
2663 # phase = ndimage.median_filter(numpy.angle(data_acf[:,1,:,:]), size = (1,5,1))
1424 velRad = phase*lamb/(4*numpy.pi*tSamp)
2664 velRad = phase*lamb/(4*numpy.pi*tSamp)
1425
2665
1426 #Spectral width
2666 #Spectral width
1427 # acf1 = ndimage.median_filter(numpy.abs(data_acf[:,1,:,:]), size = (1,5,1))
2667 # acf1 = ndimage.median_filter(numpy.abs(data_acf[:,1,:,:]), size = (1,5,1))
1428 # acf2 = ndimage.median_filter(numpy.abs(data_acf[:,2,:,:]), size = (1,5,1))
2668 # acf2 = ndimage.median_filter(numpy.abs(data_acf[:,2,:,:]), size = (1,5,1))
@@ -1437,24 +2677,24 class NonSpecularMeteorDetection(Operation):
1437 #SNR
2677 #SNR
1438 boolMet1 = (SNRdB>SNRthresh) #SNR mask
2678 boolMet1 = (SNRdB>SNRthresh) #SNR mask
1439 boolMet1 = ndimage.median_filter(boolMet1, size=(1,5,5))
2679 boolMet1 = ndimage.median_filter(boolMet1, size=(1,5,5))
1440
2680
1441 #Radial velocity
2681 #Radial velocity
1442 boolMet2 = numpy.abs(velRad) < 20
2682 boolMet2 = numpy.abs(velRad) < 20
1443 boolMet2 = ndimage.median_filter(boolMet2, (1,5,5))
2683 boolMet2 = ndimage.median_filter(boolMet2, (1,5,5))
1444
2684
1445 #Spectral Width
2685 #Spectral Width
1446 boolMet3 = spcWidth < 30
2686 boolMet3 = spcWidth < 30
1447 boolMet3 = ndimage.median_filter(boolMet3, (1,5,5))
2687 boolMet3 = ndimage.median_filter(boolMet3, (1,5,5))
1448 # boolMetFin = self.__erase_small(boolMet1, 10,5)
2688 # boolMetFin = self.__erase_small(boolMet1, 10,5)
1449 boolMetFin = boolMet1&boolMet2&boolMet3
2689 boolMetFin = boolMet1&boolMet2&boolMet3
1450
2690
1451 #Creating data_param
2691 #Creating data_param
1452 coordMet = numpy.where(boolMetFin)
2692 coordMet = numpy.where(boolMetFin)
1453
2693
1454 cmet = coordMet[0]
2694 cmet = coordMet[0]
1455 tmet = coordMet[1]
2695 tmet = coordMet[1]
1456 hmet = coordMet[2]
2696 hmet = coordMet[2]
1457
2697
1458 data_param = numpy.zeros((tmet.size, 7))
2698 data_param = numpy.zeros((tmet.size, 7))
1459 data_param[:,0] = utctime
2699 data_param[:,0] = utctime
1460 data_param[:,1] = cmet
2700 data_param[:,1] = cmet
@@ -1463,7 +2703,7 class NonSpecularMeteorDetection(Operation):
1463 data_param[:,4] = SNR[cmet,tmet,hmet].T
2703 data_param[:,4] = SNR[cmet,tmet,hmet].T
1464 data_param[:,5] = velRad[cmet,tmet,hmet].T
2704 data_param[:,5] = velRad[cmet,tmet,hmet].T
1465 data_param[:,6] = spcWidth[cmet,tmet,hmet].T
2705 data_param[:,6] = spcWidth[cmet,tmet,hmet].T
1466
2706
1467 # self.dataOut.data_param = data_int
2707 # self.dataOut.data_param = data_int
1468 if len(data_param) == 0:
2708 if len(data_param) == 0:
1469 dataOut.flagNoData = True
2709 dataOut.flagNoData = True
@@ -1473,21 +2713,21 class NonSpecularMeteorDetection(Operation):
1473 def __erase_small(self, binArray, threshX, threshY):
2713 def __erase_small(self, binArray, threshX, threshY):
1474 labarray, numfeat = ndimage.measurements.label(binArray)
2714 labarray, numfeat = ndimage.measurements.label(binArray)
1475 binArray1 = numpy.copy(binArray)
2715 binArray1 = numpy.copy(binArray)
1476
2716
1477 for i in range(1,numfeat + 1):
2717 for i in range(1,numfeat + 1):
1478 auxBin = (labarray==i)
2718 auxBin = (labarray==i)
1479 auxSize = auxBin.sum()
2719 auxSize = auxBin.sum()
1480
2720
1481 x,y = numpy.where(auxBin)
2721 x,y = numpy.where(auxBin)
1482 widthX = x.max() - x.min()
2722 widthX = x.max() - x.min()
1483 widthY = y.max() - y.min()
2723 widthY = y.max() - y.min()
1484
2724
1485 #width X: 3 seg -> 12.5*3
2725 #width X: 3 seg -> 12.5*3
1486 #width Y:
2726 #width Y:
1487
2727
1488 if (auxSize < 50) or (widthX < threshX) or (widthY < threshY):
2728 if (auxSize < 50) or (widthX < threshX) or (widthY < threshY):
1489 binArray1[auxBin] = False
2729 binArray1[auxBin] = False
1490
2730
1491 return binArray1
2731 return binArray1
1492
2732
1493 #--------------- Specular Meteor ----------------
2733 #--------------- Specular Meteor ----------------
@@ -1497,36 +2737,36 class SMDetection(Operation):
1497 Function DetectMeteors()
2737 Function DetectMeteors()
1498 Project developed with paper:
2738 Project developed with paper:
1499 HOLDSWORTH ET AL. 2004
2739 HOLDSWORTH ET AL. 2004
1500
2740
1501 Input:
2741 Input:
1502 self.dataOut.data_pre
2742 self.dataOut.data_pre
1503
2743
1504 centerReceiverIndex: From the channels, which is the center receiver
2744 centerReceiverIndex: From the channels, which is the center receiver
1505
2745
1506 hei_ref: Height reference for the Beacon signal extraction
2746 hei_ref: Height reference for the Beacon signal extraction
1507 tauindex:
2747 tauindex:
1508 predefinedPhaseShifts: Predefined phase offset for the voltge signals
2748 predefinedPhaseShifts: Predefined phase offset for the voltge signals
1509
2749
1510 cohDetection: Whether to user Coherent detection or not
2750 cohDetection: Whether to user Coherent detection or not
1511 cohDet_timeStep: Coherent Detection calculation time step
2751 cohDet_timeStep: Coherent Detection calculation time step
1512 cohDet_thresh: Coherent Detection phase threshold to correct phases
2752 cohDet_thresh: Coherent Detection phase threshold to correct phases
1513
2753
1514 noise_timeStep: Noise calculation time step
2754 noise_timeStep: Noise calculation time step
1515 noise_multiple: Noise multiple to define signal threshold
2755 noise_multiple: Noise multiple to define signal threshold
1516
2756
1517 multDet_timeLimit: Multiple Detection Removal time limit in seconds
2757 multDet_timeLimit: Multiple Detection Removal time limit in seconds
1518 multDet_rangeLimit: Multiple Detection Removal range limit in km
2758 multDet_rangeLimit: Multiple Detection Removal range limit in km
1519
2759
1520 phaseThresh: Maximum phase difference between receiver to be consider a meteor
2760 phaseThresh: Maximum phase difference between receiver to be consider a meteor
1521 SNRThresh: Minimum SNR threshold of the meteor signal to be consider a meteor
2761 SNRThresh: Minimum SNR threshold of the meteor signal to be consider a meteor
1522
2762
1523 hmin: Minimum Height of the meteor to use it in the further wind estimations
2763 hmin: Minimum Height of the meteor to use it in the further wind estimations
1524 hmax: Maximum Height of the meteor to use it in the further wind estimations
2764 hmax: Maximum Height of the meteor to use it in the further wind estimations
1525 azimuth: Azimuth angle correction
2765 azimuth: Azimuth angle correction
1526
2766
1527 Affected:
2767 Affected:
1528 self.dataOut.data_param
2768 self.dataOut.data_param
1529
2769
1530 Rejection Criteria (Errors):
2770 Rejection Criteria (Errors):
1531 0: No error; analysis OK
2771 0: No error; analysis OK
1532 1: SNR < SNR threshold
2772 1: SNR < SNR threshold
@@ -1545,9 +2785,9 class SMDetection(Operation):
1545 14: height ambiguous echo: more then one possible height within 70 to 110 km
2785 14: height ambiguous echo: more then one possible height within 70 to 110 km
1546 15: radial drift velocity or projected horizontal velocity exceeds 200 m/s
2786 15: radial drift velocity or projected horizontal velocity exceeds 200 m/s
1547 16: oscilatory echo, indicating event most likely not an underdense echo
2787 16: oscilatory echo, indicating event most likely not an underdense echo
1548
2788
1549 17: phase difference in meteor Reestimation
2789 17: phase difference in meteor Reestimation
1550
2790
1551 Data Storage:
2791 Data Storage:
1552 Meteors for Wind Estimation (8):
2792 Meteors for Wind Estimation (8):
1553 Utc Time | Range Height
2793 Utc Time | Range Height
@@ -1555,21 +2795,21 class SMDetection(Operation):
1555 VelRad errorVelRad
2795 VelRad errorVelRad
1556 Phase0 Phase1 Phase2 Phase3
2796 Phase0 Phase1 Phase2 Phase3
1557 TypeError
2797 TypeError
1558
2798
1559 '''
2799 '''
1560
2800
1561 def run(self, dataOut, hei_ref = None, tauindex = 0,
2801 def run(self, dataOut, hei_ref = None, tauindex = 0,
1562 phaseOffsets = None,
2802 phaseOffsets = None,
1563 cohDetection = False, cohDet_timeStep = 1, cohDet_thresh = 25,
2803 cohDetection = False, cohDet_timeStep = 1, cohDet_thresh = 25,
1564 noise_timeStep = 4, noise_multiple = 4,
2804 noise_timeStep = 4, noise_multiple = 4,
1565 multDet_timeLimit = 1, multDet_rangeLimit = 3,
2805 multDet_timeLimit = 1, multDet_rangeLimit = 3,
1566 phaseThresh = 20, SNRThresh = 5,
2806 phaseThresh = 20, SNRThresh = 5,
1567 hmin = 50, hmax=150, azimuth = 0,
2807 hmin = 50, hmax=150, azimuth = 0,
1568 channelPositions = None) :
2808 channelPositions = None) :
1569
2809
1570
2810
1571 #Getting Pairslist
2811 #Getting Pairslist
1572 if channelPositions is None:
2812 if channelPositions == None:
1573 # channelPositions = [(2.5,0), (0,2.5), (0,0), (0,4.5), (-2,0)] #T
2813 # channelPositions = [(2.5,0), (0,2.5), (0,0), (0,4.5), (-2,0)] #T
1574 channelPositions = [(4.5,2), (2,4.5), (2,2), (2,0), (0,2)] #Estrella
2814 channelPositions = [(4.5,2), (2,4.5), (2,2), (2,0), (0,2)] #Estrella
1575 meteorOps = SMOperations()
2815 meteorOps = SMOperations()
@@ -1577,53 +2817,53 class SMDetection(Operation):
1577 heiRang = dataOut.getHeiRange()
2817 heiRang = dataOut.getHeiRange()
1578 #Get Beacon signal - No Beacon signal anymore
2818 #Get Beacon signal - No Beacon signal anymore
1579 # newheis = numpy.where(self.dataOut.heightList>self.dataOut.radarControllerHeaderObj.Taus[tauindex])
2819 # newheis = numpy.where(self.dataOut.heightList>self.dataOut.radarControllerHeaderObj.Taus[tauindex])
1580 #
2820 #
1581 # if hei_ref != None:
2821 # if hei_ref != None:
1582 # newheis = numpy.where(self.dataOut.heightList>hei_ref)
2822 # newheis = numpy.where(self.dataOut.heightList>hei_ref)
1583 #
2823 #
1584
2824
1585
2825
1586 #****************REMOVING HARDWARE PHASE DIFFERENCES***************
2826 #****************REMOVING HARDWARE PHASE DIFFERENCES***************
1587 # see if the user put in pre defined phase shifts
2827 # see if the user put in pre defined phase shifts
1588 voltsPShift = dataOut.data_pre.copy()
2828 voltsPShift = dataOut.data_pre.copy()
1589
2829
1590 # if predefinedPhaseShifts != None:
2830 # if predefinedPhaseShifts != None:
1591 # hardwarePhaseShifts = numpy.array(predefinedPhaseShifts)*numpy.pi/180
2831 # hardwarePhaseShifts = numpy.array(predefinedPhaseShifts)*numpy.pi/180
1592 #
2832 #
1593 # # elif beaconPhaseShifts:
2833 # # elif beaconPhaseShifts:
1594 # # #get hardware phase shifts using beacon signal
2834 # # #get hardware phase shifts using beacon signal
1595 # # hardwarePhaseShifts = self.__getHardwarePhaseDiff(self.dataOut.data_pre, pairslist, newheis, 10)
2835 # # hardwarePhaseShifts = self.__getHardwarePhaseDiff(self.dataOut.data_pre, pairslist, newheis, 10)
1596 # # hardwarePhaseShifts = numpy.insert(hardwarePhaseShifts,centerReceiverIndex,0)
2836 # # hardwarePhaseShifts = numpy.insert(hardwarePhaseShifts,centerReceiverIndex,0)
1597 #
2837 #
1598 # else:
2838 # else:
1599 # hardwarePhaseShifts = numpy.zeros(5)
2839 # hardwarePhaseShifts = numpy.zeros(5)
1600 #
2840 #
1601 # voltsPShift = numpy.zeros((self.dataOut.data_pre.shape[0],self.dataOut.data_pre.shape[1],self.dataOut.data_pre.shape[2]), dtype = 'complex')
2841 # voltsPShift = numpy.zeros((self.dataOut.data_pre.shape[0],self.dataOut.data_pre.shape[1],self.dataOut.data_pre.shape[2]), dtype = 'complex')
1602 # for i in range(self.dataOut.data_pre.shape[0]):
2842 # for i in range(self.dataOut.data_pre.shape[0]):
1603 # voltsPShift[i,:,:] = self.__shiftPhase(self.dataOut.data_pre[i,:,:], hardwarePhaseShifts[i])
2843 # voltsPShift[i,:,:] = self.__shiftPhase(self.dataOut.data_pre[i,:,:], hardwarePhaseShifts[i])
1604
2844
1605 #******************END OF REMOVING HARDWARE PHASE DIFFERENCES*********
2845 #******************END OF REMOVING HARDWARE PHASE DIFFERENCES*********
1606
2846
1607 #Remove DC
2847 #Remove DC
1608 voltsDC = numpy.mean(voltsPShift,1)
2848 voltsDC = numpy.mean(voltsPShift,1)
1609 voltsDC = numpy.mean(voltsDC,1)
2849 voltsDC = numpy.mean(voltsDC,1)
1610 for i in range(voltsDC.shape[0]):
2850 for i in range(voltsDC.shape[0]):
1611 voltsPShift[i] = voltsPShift[i] - voltsDC[i]
2851 voltsPShift[i] = voltsPShift[i] - voltsDC[i]
1612
2852
1613 #Don't considerate last heights, theyre used to calculate Hardware Phase Shift
2853 #Don't considerate last heights, theyre used to calculate Hardware Phase Shift
1614 # voltsPShift = voltsPShift[:,:,:newheis[0][0]]
2854 # voltsPShift = voltsPShift[:,:,:newheis[0][0]]
1615
2855
1616 #************ FIND POWER OF DATA W/COH OR NON COH DETECTION (3.4) **********
2856 #************ FIND POWER OF DATA W/COH OR NON COH DETECTION (3.4) **********
1617 #Coherent Detection
2857 #Coherent Detection
1618 if cohDetection:
2858 if cohDetection:
1619 #use coherent detection to get the net power
2859 #use coherent detection to get the net power
1620 cohDet_thresh = cohDet_thresh*numpy.pi/180
2860 cohDet_thresh = cohDet_thresh*numpy.pi/180
1621 voltsPShift = self.__coherentDetection(voltsPShift, cohDet_timeStep, dataOut.timeInterval, pairslist0, cohDet_thresh)
2861 voltsPShift = self.__coherentDetection(voltsPShift, cohDet_timeStep, dataOut.timeInterval, pairslist0, cohDet_thresh)
1622
2862
1623 #Non-coherent detection!
2863 #Non-coherent detection!
1624 powerNet = numpy.nansum(numpy.abs(voltsPShift[:,:,:])**2,0)
2864 powerNet = numpy.nansum(numpy.abs(voltsPShift[:,:,:])**2,0)
1625 #********** END OF COH/NON-COH POWER CALCULATION**********************
2865 #********** END OF COH/NON-COH POWER CALCULATION**********************
1626
2866
1627 #********** FIND THE NOISE LEVEL AND POSSIBLE METEORS ****************
2867 #********** FIND THE NOISE LEVEL AND POSSIBLE METEORS ****************
1628 #Get noise
2868 #Get noise
1629 noise, noise1 = self.__getNoise(powerNet, noise_timeStep, dataOut.timeInterval)
2869 noise, noise1 = self.__getNoise(powerNet, noise_timeStep, dataOut.timeInterval)
@@ -1633,7 +2873,7 class SMDetection(Operation):
1633 #Meteor echoes detection
2873 #Meteor echoes detection
1634 listMeteors = self.__findMeteors(powerNet, signalThresh)
2874 listMeteors = self.__findMeteors(powerNet, signalThresh)
1635 #******* END OF NOISE LEVEL AND POSSIBLE METEORS CACULATION **********
2875 #******* END OF NOISE LEVEL AND POSSIBLE METEORS CACULATION **********
1636
2876
1637 #************** REMOVE MULTIPLE DETECTIONS (3.5) ***************************
2877 #************** REMOVE MULTIPLE DETECTIONS (3.5) ***************************
1638 #Parameters
2878 #Parameters
1639 heiRange = dataOut.getHeiRange()
2879 heiRange = dataOut.getHeiRange()
@@ -1643,7 +2883,7 class SMDetection(Operation):
1643 #Multiple detection removals
2883 #Multiple detection removals
1644 listMeteors1 = self.__removeMultipleDetections(listMeteors, rangeLimit, timeLimit)
2884 listMeteors1 = self.__removeMultipleDetections(listMeteors, rangeLimit, timeLimit)
1645 #************ END OF REMOVE MULTIPLE DETECTIONS **********************
2885 #************ END OF REMOVE MULTIPLE DETECTIONS **********************
1646
2886
1647 #********************* METEOR REESTIMATION (3.7, 3.8, 3.9, 3.10) ********************
2887 #********************* METEOR REESTIMATION (3.7, 3.8, 3.9, 3.10) ********************
1648 #Parameters
2888 #Parameters
1649 phaseThresh = phaseThresh*numpy.pi/180
2889 phaseThresh = phaseThresh*numpy.pi/180
@@ -1654,40 +2894,40 class SMDetection(Operation):
1654 #Estimation of decay times (Errors N 7, 8, 11)
2894 #Estimation of decay times (Errors N 7, 8, 11)
1655 listMeteors3 = self.__estimateDecayTime(listMeteors2, listMeteorsPower, dataOut.timeInterval, dataOut.frequency)
2895 listMeteors3 = self.__estimateDecayTime(listMeteors2, listMeteorsPower, dataOut.timeInterval, dataOut.frequency)
1656 #******************* END OF METEOR REESTIMATION *******************
2896 #******************* END OF METEOR REESTIMATION *******************
1657
2897
1658 #********************* METEOR PARAMETERS CALCULATION (3.11, 3.12, 3.13) **************************
2898 #********************* METEOR PARAMETERS CALCULATION (3.11, 3.12, 3.13) **************************
1659 #Calculating Radial Velocity (Error N 15)
2899 #Calculating Radial Velocity (Error N 15)
1660 radialStdThresh = 10
2900 radialStdThresh = 10
1661 listMeteors4 = self.__getRadialVelocity(listMeteors3, listMeteorsVolts, radialStdThresh, pairslist0, dataOut.timeInterval)
2901 listMeteors4 = self.__getRadialVelocity(listMeteors3, listMeteorsVolts, radialStdThresh, pairslist0, dataOut.timeInterval)
1662
2902
1663 if len(listMeteors4) > 0:
2903 if len(listMeteors4) > 0:
1664 #Setting New Array
2904 #Setting New Array
1665 date = dataOut.utctime
2905 date = dataOut.utctime
1666 arrayParameters = self.__setNewArrays(listMeteors4, date, heiRang)
2906 arrayParameters = self.__setNewArrays(listMeteors4, date, heiRang)
1667
2907
1668 #Correcting phase offset
2908 #Correcting phase offset
1669 if phaseOffsets != None:
2909 if phaseOffsets != None:
1670 phaseOffsets = numpy.array(phaseOffsets)*numpy.pi/180
2910 phaseOffsets = numpy.array(phaseOffsets)*numpy.pi/180
1671 arrayParameters[:,8:12] = numpy.unwrap(arrayParameters[:,8:12] + phaseOffsets)
2911 arrayParameters[:,8:12] = numpy.unwrap(arrayParameters[:,8:12] + phaseOffsets)
1672
2912
1673 #Second Pairslist
2913 #Second Pairslist
1674 pairsList = []
2914 pairsList = []
1675 pairx = (0,1)
2915 pairx = (0,1)
1676 pairy = (2,3)
2916 pairy = (2,3)
1677 pairsList.append(pairx)
2917 pairsList.append(pairx)
1678 pairsList.append(pairy)
2918 pairsList.append(pairy)
1679
2919
1680 jph = numpy.array([0,0,0,0])
2920 jph = numpy.array([0,0,0,0])
1681 h = (hmin,hmax)
2921 h = (hmin,hmax)
1682 arrayParameters = meteorOps.getMeteorParams(arrayParameters, azimuth, h, pairsList, distances, jph)
2922 arrayParameters = meteorOps.getMeteorParams(arrayParameters, azimuth, h, pairsList, distances, jph)
1683
2923
1684 # #Calculate AOA (Error N 3, 4)
2924 # #Calculate AOA (Error N 3, 4)
1685 # #JONES ET AL. 1998
2925 # #JONES ET AL. 1998
1686 # error = arrayParameters[:,-1]
2926 # error = arrayParameters[:,-1]
1687 # AOAthresh = numpy.pi/8
2927 # AOAthresh = numpy.pi/8
1688 # phases = -arrayParameters[:,9:13]
2928 # phases = -arrayParameters[:,9:13]
1689 # arrayParameters[:,4:7], arrayParameters[:,-1] = meteorOps.getAOA(phases, pairsList, error, AOAthresh, azimuth)
2929 # arrayParameters[:,4:7], arrayParameters[:,-1] = meteorOps.getAOA(phases, pairsList, error, AOAthresh, azimuth)
1690 #
2930 #
1691 # #Calculate Heights (Error N 13 and 14)
2931 # #Calculate Heights (Error N 13 and 14)
1692 # error = arrayParameters[:,-1]
2932 # error = arrayParameters[:,-1]
1693 # Ranges = arrayParameters[:,2]
2933 # Ranges = arrayParameters[:,2]
@@ -1695,73 +2935,73 class SMDetection(Operation):
1695 # arrayParameters[:,3], arrayParameters[:,-1] = meteorOps.getHeights(Ranges, zenith, error, hmin, hmax)
2935 # arrayParameters[:,3], arrayParameters[:,-1] = meteorOps.getHeights(Ranges, zenith, error, hmin, hmax)
1696 # error = arrayParameters[:,-1]
2936 # error = arrayParameters[:,-1]
1697 #********************* END OF PARAMETERS CALCULATION **************************
2937 #********************* END OF PARAMETERS CALCULATION **************************
1698
2938
1699 #***************************+ PASS DATA TO NEXT STEP **********************
2939 #***************************+ PASS DATA TO NEXT STEP **********************
1700 # arrayFinal = arrayParameters.reshape((1,arrayParameters.shape[0],arrayParameters.shape[1]))
2940 # arrayFinal = arrayParameters.reshape((1,arrayParameters.shape[0],arrayParameters.shape[1]))
1701 dataOut.data_param = arrayParameters
2941 dataOut.data_param = arrayParameters
1702
2942
1703 if arrayParameters is None:
2943 if arrayParameters == None:
1704 dataOut.flagNoData = True
2944 dataOut.flagNoData = True
1705 else:
2945 else:
1706 dataOut.flagNoData = True
2946 dataOut.flagNoData = True
1707
2947
1708 return
2948 return
1709
2949
1710 def __getHardwarePhaseDiff(self, voltage0, pairslist, newheis, n):
2950 def __getHardwarePhaseDiff(self, voltage0, pairslist, newheis, n):
1711
2951
1712 minIndex = min(newheis[0])
2952 minIndex = min(newheis[0])
1713 maxIndex = max(newheis[0])
2953 maxIndex = max(newheis[0])
1714
2954
1715 voltage = voltage0[:,:,minIndex:maxIndex+1]
2955 voltage = voltage0[:,:,minIndex:maxIndex+1]
1716 nLength = voltage.shape[1]/n
2956 nLength = voltage.shape[1]/n
1717 nMin = 0
2957 nMin = 0
1718 nMax = 0
2958 nMax = 0
1719 phaseOffset = numpy.zeros((len(pairslist),n))
2959 phaseOffset = numpy.zeros((len(pairslist),n))
1720
2960
1721 for i in range(n):
2961 for i in range(n):
1722 nMax += nLength
2962 nMax += nLength
1723 phaseCCF = -numpy.angle(self.__calculateCCF(voltage[:,nMin:nMax,:], pairslist, [0]))
2963 phaseCCF = -numpy.angle(self.__calculateCCF(voltage[:,nMin:nMax,:], pairslist, [0]))
1724 phaseCCF = numpy.mean(phaseCCF, axis = 2)
2964 phaseCCF = numpy.mean(phaseCCF, axis = 2)
1725 phaseOffset[:,i] = phaseCCF.transpose()
2965 phaseOffset[:,i] = phaseCCF.transpose()
1726 nMin = nMax
2966 nMin = nMax
1727 # phaseDiff, phaseArrival = self.estimatePhaseDifference(voltage, pairslist)
2967 # phaseDiff, phaseArrival = self.estimatePhaseDifference(voltage, pairslist)
1728
2968
1729 #Remove Outliers
2969 #Remove Outliers
1730 factor = 2
2970 factor = 2
1731 wt = phaseOffset - signal.medfilt(phaseOffset,(1,5))
2971 wt = phaseOffset - signal.medfilt(phaseOffset,(1,5))
1732 dw = numpy.std(wt,axis = 1)
2972 dw = numpy.std(wt,axis = 1)
1733 dw = dw.reshape((dw.size,1))
2973 dw = dw.reshape((dw.size,1))
1734 ind = numpy.where(numpy.logical_or(wt>dw*factor,wt<-dw*factor))
2974 ind = numpy.where(numpy.logical_or(wt>dw*factor,wt<-dw*factor))
1735 phaseOffset[ind] = numpy.nan
2975 phaseOffset[ind] = numpy.nan
1736 phaseOffset = stats.nanmean(phaseOffset, axis=1)
2976 phaseOffset = stats.nanmean(phaseOffset, axis=1)
1737
2977
1738 return phaseOffset
2978 return phaseOffset
1739
2979
1740 def __shiftPhase(self, data, phaseShift):
2980 def __shiftPhase(self, data, phaseShift):
1741 #this will shift the phase of a complex number
2981 #this will shift the phase of a complex number
1742 dataShifted = numpy.abs(data) * numpy.exp((numpy.angle(data)+phaseShift)*1j)
2982 dataShifted = numpy.abs(data) * numpy.exp((numpy.angle(data)+phaseShift)*1j)
1743 return dataShifted
2983 return dataShifted
1744
2984
1745 def __estimatePhaseDifference(self, array, pairslist):
2985 def __estimatePhaseDifference(self, array, pairslist):
1746 nChannel = array.shape[0]
2986 nChannel = array.shape[0]
1747 nHeights = array.shape[2]
2987 nHeights = array.shape[2]
1748 numPairs = len(pairslist)
2988 numPairs = len(pairslist)
1749 # phaseCCF = numpy.zeros((nChannel, 5, nHeights))
2989 # phaseCCF = numpy.zeros((nChannel, 5, nHeights))
1750 phaseCCF = numpy.angle(self.__calculateCCF(array, pairslist, [-2,-1,0,1,2]))
2990 phaseCCF = numpy.angle(self.__calculateCCF(array, pairslist, [-2,-1,0,1,2]))
1751
2991
1752 #Correct phases
2992 #Correct phases
1753 derPhaseCCF = phaseCCF[:,1:,:] - phaseCCF[:,0:-1,:]
2993 derPhaseCCF = phaseCCF[:,1:,:] - phaseCCF[:,0:-1,:]
1754 indDer = numpy.where(numpy.abs(derPhaseCCF) > numpy.pi)
2994 indDer = numpy.where(numpy.abs(derPhaseCCF) > numpy.pi)
1755
2995
1756 if indDer[0].shape[0] > 0:
2996 if indDer[0].shape[0] > 0:
1757 for i in range(indDer[0].shape[0]):
2997 for i in range(indDer[0].shape[0]):
1758 signo = -numpy.sign(derPhaseCCF[indDer[0][i],indDer[1][i],indDer[2][i]])
2998 signo = -numpy.sign(derPhaseCCF[indDer[0][i],indDer[1][i],indDer[2][i]])
1759 phaseCCF[indDer[0][i],indDer[1][i]+1:,:] += signo*2*numpy.pi
2999 phaseCCF[indDer[0][i],indDer[1][i]+1:,:] += signo*2*numpy.pi
1760
3000
1761 # for j in range(numSides):
3001 # for j in range(numSides):
1762 # phaseCCFAux = self.calculateCCF(arrayCenter, arraySides[j,:,:], [-2,1,0,1,2])
3002 # phaseCCFAux = self.calculateCCF(arrayCenter, arraySides[j,:,:], [-2,1,0,1,2])
1763 # phaseCCF[j,:,:] = numpy.angle(phaseCCFAux)
3003 # phaseCCF[j,:,:] = numpy.angle(phaseCCFAux)
1764 #
3004 #
1765 #Linear
3005 #Linear
1766 phaseInt = numpy.zeros((numPairs,1))
3006 phaseInt = numpy.zeros((numPairs,1))
1767 angAllCCF = phaseCCF[:,[0,1,3,4],0]
3007 angAllCCF = phaseCCF[:,[0,1,3,4],0]
@@ -1771,16 +3011,16 class SMDetection(Operation):
1771 #Phase Differences
3011 #Phase Differences
1772 phaseDiff = phaseInt - phaseCCF[:,2,:]
3012 phaseDiff = phaseInt - phaseCCF[:,2,:]
1773 phaseArrival = phaseInt.reshape(phaseInt.size)
3013 phaseArrival = phaseInt.reshape(phaseInt.size)
1774
3014
1775 #Dealias
3015 #Dealias
1776 phaseArrival = numpy.angle(numpy.exp(1j*phaseArrival))
3016 phaseArrival = numpy.angle(numpy.exp(1j*phaseArrival))
1777 # indAlias = numpy.where(phaseArrival > numpy.pi)
3017 # indAlias = numpy.where(phaseArrival > numpy.pi)
1778 # phaseArrival[indAlias] -= 2*numpy.pi
3018 # phaseArrival[indAlias] -= 2*numpy.pi
1779 # indAlias = numpy.where(phaseArrival < -numpy.pi)
3019 # indAlias = numpy.where(phaseArrival < -numpy.pi)
1780 # phaseArrival[indAlias] += 2*numpy.pi
3020 # phaseArrival[indAlias] += 2*numpy.pi
1781
3021
1782 return phaseDiff, phaseArrival
3022 return phaseDiff, phaseArrival
1783
3023
1784 def __coherentDetection(self, volts, timeSegment, timeInterval, pairslist, thresh):
3024 def __coherentDetection(self, volts, timeSegment, timeInterval, pairslist, thresh):
1785 #this function will run the coherent detection used in Holdworth et al. 2004 and return the net power
3025 #this function will run the coherent detection used in Holdworth et al. 2004 and return the net power
1786 #find the phase shifts of each channel over 1 second intervals
3026 #find the phase shifts of each channel over 1 second intervals
@@ -1790,25 +3030,25 class SMDetection(Operation):
1790 numHeights = volts.shape[2]
3030 numHeights = volts.shape[2]
1791 nChannel = volts.shape[0]
3031 nChannel = volts.shape[0]
1792 voltsCohDet = volts.copy()
3032 voltsCohDet = volts.copy()
1793
3033
1794 pairsarray = numpy.array(pairslist)
3034 pairsarray = numpy.array(pairslist)
1795 indSides = pairsarray[:,1]
3035 indSides = pairsarray[:,1]
1796 # indSides = numpy.array(range(nChannel))
3036 # indSides = numpy.array(range(nChannel))
1797 # indSides = numpy.delete(indSides, indCenter)
3037 # indSides = numpy.delete(indSides, indCenter)
1798 #
3038 #
1799 # listCenter = numpy.array_split(volts[indCenter,:,:], numBlocks, 0)
3039 # listCenter = numpy.array_split(volts[indCenter,:,:], numBlocks, 0)
1800 listBlocks = numpy.array_split(volts, numBlocks, 1)
3040 listBlocks = numpy.array_split(volts, numBlocks, 1)
1801
3041
1802 startInd = 0
3042 startInd = 0
1803 endInd = 0
3043 endInd = 0
1804
3044
1805 for i in range(numBlocks):
3045 for i in range(numBlocks):
1806 startInd = endInd
3046 startInd = endInd
1807 endInd = endInd + listBlocks[i].shape[1]
3047 endInd = endInd + listBlocks[i].shape[1]
1808
3048
1809 arrayBlock = listBlocks[i]
3049 arrayBlock = listBlocks[i]
1810 # arrayBlockCenter = listCenter[i]
3050 # arrayBlockCenter = listCenter[i]
1811
3051
1812 #Estimate the Phase Difference
3052 #Estimate the Phase Difference
1813 phaseDiff, aux = self.__estimatePhaseDifference(arrayBlock, pairslist)
3053 phaseDiff, aux = self.__estimatePhaseDifference(arrayBlock, pairslist)
1814 #Phase Difference RMS
3054 #Phase Difference RMS
@@ -1820,21 +3060,21 class SMDetection(Operation):
1820 for j in range(indSides.size):
3060 for j in range(indSides.size):
1821 arrayBlock[indSides[j],:,indPhase] = self.__shiftPhase(arrayBlock[indSides[j],:,indPhase], phaseDiff[j,indPhase].transpose())
3061 arrayBlock[indSides[j],:,indPhase] = self.__shiftPhase(arrayBlock[indSides[j],:,indPhase], phaseDiff[j,indPhase].transpose())
1822 voltsCohDet[:,startInd:endInd,:] = arrayBlock
3062 voltsCohDet[:,startInd:endInd,:] = arrayBlock
1823
3063
1824 return voltsCohDet
3064 return voltsCohDet
1825
3065
1826 def __calculateCCF(self, volts, pairslist ,laglist):
3066 def __calculateCCF(self, volts, pairslist ,laglist):
1827
3067
1828 nHeights = volts.shape[2]
3068 nHeights = volts.shape[2]
1829 nPoints = volts.shape[1]
3069 nPoints = volts.shape[1]
1830 voltsCCF = numpy.zeros((len(pairslist), len(laglist), nHeights),dtype = 'complex')
3070 voltsCCF = numpy.zeros((len(pairslist), len(laglist), nHeights),dtype = 'complex')
1831
3071
1832 for i in range(len(pairslist)):
3072 for i in range(len(pairslist)):
1833 volts1 = volts[pairslist[i][0]]
3073 volts1 = volts[pairslist[i][0]]
1834 volts2 = volts[pairslist[i][1]]
3074 volts2 = volts[pairslist[i][1]]
1835
3075
1836 for t in range(len(laglist)):
3076 for t in range(len(laglist)):
1837 idxT = laglist[t]
3077 idxT = laglist[t]
1838 if idxT >= 0:
3078 if idxT >= 0:
1839 vStacked = numpy.vstack((volts2[idxT:,:],
3079 vStacked = numpy.vstack((volts2[idxT:,:],
1840 numpy.zeros((idxT, nHeights),dtype='complex')))
3080 numpy.zeros((idxT, nHeights),dtype='complex')))
@@ -1842,10 +3082,10 class SMDetection(Operation):
1842 vStacked = numpy.vstack((numpy.zeros((-idxT, nHeights),dtype='complex'),
3082 vStacked = numpy.vstack((numpy.zeros((-idxT, nHeights),dtype='complex'),
1843 volts2[:(nPoints + idxT),:]))
3083 volts2[:(nPoints + idxT),:]))
1844 voltsCCF[i,t,:] = numpy.sum((numpy.conjugate(volts1)*vStacked),axis=0)
3084 voltsCCF[i,t,:] = numpy.sum((numpy.conjugate(volts1)*vStacked),axis=0)
1845
3085
1846 vStacked = None
3086 vStacked = None
1847 return voltsCCF
3087 return voltsCCF
1848
3088
1849 def __getNoise(self, power, timeSegment, timeInterval):
3089 def __getNoise(self, power, timeSegment, timeInterval):
1850 numProfPerBlock = numpy.ceil(timeSegment/timeInterval)
3090 numProfPerBlock = numpy.ceil(timeSegment/timeInterval)
1851 numBlocks = int(power.shape[0]/numProfPerBlock)
3091 numBlocks = int(power.shape[0]/numProfPerBlock)
@@ -1854,133 +3094,133 class SMDetection(Operation):
1854 listPower = numpy.array_split(power, numBlocks, 0)
3094 listPower = numpy.array_split(power, numBlocks, 0)
1855 noise = numpy.zeros((power.shape[0], power.shape[1]))
3095 noise = numpy.zeros((power.shape[0], power.shape[1]))
1856 noise1 = numpy.zeros((power.shape[0], power.shape[1]))
3096 noise1 = numpy.zeros((power.shape[0], power.shape[1]))
1857
3097
1858 startInd = 0
3098 startInd = 0
1859 endInd = 0
3099 endInd = 0
1860
3100
1861 for i in range(numBlocks): #split por canal
3101 for i in range(numBlocks): #split por canal
1862 startInd = endInd
3102 startInd = endInd
1863 endInd = endInd + listPower[i].shape[0]
3103 endInd = endInd + listPower[i].shape[0]
1864
3104
1865 arrayBlock = listPower[i]
3105 arrayBlock = listPower[i]
1866 noiseAux = numpy.mean(arrayBlock, 0)
3106 noiseAux = numpy.mean(arrayBlock, 0)
1867 # noiseAux = numpy.median(noiseAux)
3107 # noiseAux = numpy.median(noiseAux)
1868 # noiseAux = numpy.mean(arrayBlock)
3108 # noiseAux = numpy.mean(arrayBlock)
1869 noise[startInd:endInd,:] = noise[startInd:endInd,:] + noiseAux
3109 noise[startInd:endInd,:] = noise[startInd:endInd,:] + noiseAux
1870
3110
1871 noiseAux1 = numpy.mean(arrayBlock)
3111 noiseAux1 = numpy.mean(arrayBlock)
1872 noise1[startInd:endInd,:] = noise1[startInd:endInd,:] + noiseAux1
3112 noise1[startInd:endInd,:] = noise1[startInd:endInd,:] + noiseAux1
1873
3113
1874 return noise, noise1
3114 return noise, noise1
1875
3115
1876 def __findMeteors(self, power, thresh):
3116 def __findMeteors(self, power, thresh):
1877 nProf = power.shape[0]
3117 nProf = power.shape[0]
1878 nHeights = power.shape[1]
3118 nHeights = power.shape[1]
1879 listMeteors = []
3119 listMeteors = []
1880
3120
1881 for i in range(nHeights):
3121 for i in range(nHeights):
1882 powerAux = power[:,i]
3122 powerAux = power[:,i]
1883 threshAux = thresh[:,i]
3123 threshAux = thresh[:,i]
1884
3124
1885 indUPthresh = numpy.where(powerAux > threshAux)[0]
3125 indUPthresh = numpy.where(powerAux > threshAux)[0]
1886 indDNthresh = numpy.where(powerAux <= threshAux)[0]
3126 indDNthresh = numpy.where(powerAux <= threshAux)[0]
1887
3127
1888 j = 0
3128 j = 0
1889
3129
1890 while (j < indUPthresh.size - 2):
3130 while (j < indUPthresh.size - 2):
1891 if (indUPthresh[j + 2] == indUPthresh[j] + 2):
3131 if (indUPthresh[j + 2] == indUPthresh[j] + 2):
1892 indDNAux = numpy.where(indDNthresh > indUPthresh[j])
3132 indDNAux = numpy.where(indDNthresh > indUPthresh[j])
1893 indDNthresh = indDNthresh[indDNAux]
3133 indDNthresh = indDNthresh[indDNAux]
1894
3134
1895 if (indDNthresh.size > 0):
3135 if (indDNthresh.size > 0):
1896 indEnd = indDNthresh[0] - 1
3136 indEnd = indDNthresh[0] - 1
1897 indInit = indUPthresh[j] if isinstance(indUPthresh[j], (int, float)) else indUPthresh[j][0] ##CHECK!!!!
3137 indInit = indUPthresh[j]
1898
3138
1899 meteor = powerAux[indInit:indEnd + 1]
3139 meteor = powerAux[indInit:indEnd + 1]
1900 indPeak = meteor.argmax() + indInit
3140 indPeak = meteor.argmax() + indInit
1901 FLA = sum(numpy.conj(meteor)*numpy.hstack((meteor[1:],0)))
3141 FLA = sum(numpy.conj(meteor)*numpy.hstack((meteor[1:],0)))
1902
3142
1903 listMeteors.append(numpy.array([i,indInit,indPeak,indEnd,FLA])) #CHEQUEAR!!!!!
3143 listMeteors.append(numpy.array([i,indInit,indPeak,indEnd,FLA])) #CHEQUEAR!!!!!
1904 j = numpy.where(indUPthresh == indEnd)[0] + 1
3144 j = numpy.where(indUPthresh == indEnd)[0] + 1
1905 else: j+=1
3145 else: j+=1
1906 else: j+=1
3146 else: j+=1
1907
3147
1908 return listMeteors
3148 return listMeteors
1909
3149
1910 def __removeMultipleDetections(self,listMeteors, rangeLimit, timeLimit):
3150 def __removeMultipleDetections(self,listMeteors, rangeLimit, timeLimit):
1911
3151
1912 arrayMeteors = numpy.asarray(listMeteors)
3152 arrayMeteors = numpy.asarray(listMeteors)
1913 listMeteors1 = []
3153 listMeteors1 = []
1914
3154
1915 while arrayMeteors.shape[0] > 0:
3155 while arrayMeteors.shape[0] > 0:
1916 FLAs = arrayMeteors[:,4]
3156 FLAs = arrayMeteors[:,4]
1917 maxFLA = FLAs.argmax()
3157 maxFLA = FLAs.argmax()
1918 listMeteors1.append(arrayMeteors[maxFLA,:])
3158 listMeteors1.append(arrayMeteors[maxFLA,:])
1919
3159
1920 MeteorInitTime = arrayMeteors[maxFLA,1]
3160 MeteorInitTime = arrayMeteors[maxFLA,1]
1921 MeteorEndTime = arrayMeteors[maxFLA,3]
3161 MeteorEndTime = arrayMeteors[maxFLA,3]
1922 MeteorHeight = arrayMeteors[maxFLA,0]
3162 MeteorHeight = arrayMeteors[maxFLA,0]
1923
3163
1924 #Check neighborhood
3164 #Check neighborhood
1925 maxHeightIndex = MeteorHeight + rangeLimit
3165 maxHeightIndex = MeteorHeight + rangeLimit
1926 minHeightIndex = MeteorHeight - rangeLimit
3166 minHeightIndex = MeteorHeight - rangeLimit
1927 minTimeIndex = MeteorInitTime - timeLimit
3167 minTimeIndex = MeteorInitTime - timeLimit
1928 maxTimeIndex = MeteorEndTime + timeLimit
3168 maxTimeIndex = MeteorEndTime + timeLimit
1929
3169
1930 #Check Heights
3170 #Check Heights
1931 indHeight = numpy.logical_and(arrayMeteors[:,0] >= minHeightIndex, arrayMeteors[:,0] <= maxHeightIndex)
3171 indHeight = numpy.logical_and(arrayMeteors[:,0] >= minHeightIndex, arrayMeteors[:,0] <= maxHeightIndex)
1932 indTime = numpy.logical_and(arrayMeteors[:,3] >= minTimeIndex, arrayMeteors[:,1] <= maxTimeIndex)
3172 indTime = numpy.logical_and(arrayMeteors[:,3] >= minTimeIndex, arrayMeteors[:,1] <= maxTimeIndex)
1933 indBoth = numpy.where(numpy.logical_and(indTime,indHeight))
3173 indBoth = numpy.where(numpy.logical_and(indTime,indHeight))
1934
3174
1935 arrayMeteors = numpy.delete(arrayMeteors, indBoth, axis = 0)
3175 arrayMeteors = numpy.delete(arrayMeteors, indBoth, axis = 0)
1936
3176
1937 return listMeteors1
3177 return listMeteors1
1938
3178
1939 def __meteorReestimation(self, listMeteors, volts, pairslist, thresh, noise, timeInterval,frequency):
3179 def __meteorReestimation(self, listMeteors, volts, pairslist, thresh, noise, timeInterval,frequency):
1940 numHeights = volts.shape[2]
3180 numHeights = volts.shape[2]
1941 nChannel = volts.shape[0]
3181 nChannel = volts.shape[0]
1942
3182
1943 thresholdPhase = thresh[0]
3183 thresholdPhase = thresh[0]
1944 thresholdNoise = thresh[1]
3184 thresholdNoise = thresh[1]
1945 thresholdDB = float(thresh[2])
3185 thresholdDB = float(thresh[2])
1946
3186
1947 thresholdDB1 = 10**(thresholdDB/10)
3187 thresholdDB1 = 10**(thresholdDB/10)
1948 pairsarray = numpy.array(pairslist)
3188 pairsarray = numpy.array(pairslist)
1949 indSides = pairsarray[:,1]
3189 indSides = pairsarray[:,1]
1950
3190
1951 pairslist1 = list(pairslist)
3191 pairslist1 = list(pairslist)
1952 pairslist1.append((0,4))
3192 pairslist1.append((0,1))
1953 pairslist1.append((1,3))
3193 pairslist1.append((3,4))
1954
3194
1955 listMeteors1 = []
3195 listMeteors1 = []
1956 listPowerSeries = []
3196 listPowerSeries = []
1957 listVoltageSeries = []
3197 listVoltageSeries = []
1958 #volts has the war data
3198 #volts has the war data
1959
3199
1960 if frequency == 30.175e6:
3200 if frequency == 30e6:
1961 timeLag = 45*10**-3
3201 timeLag = 45*10**-3
1962 else:
3202 else:
1963 timeLag = 15*10**-3
3203 timeLag = 15*10**-3
1964 lag = int(numpy.ceil(timeLag/timeInterval))
3204 lag = numpy.ceil(timeLag/timeInterval)
1965
3205
1966 for i in range(len(listMeteors)):
3206 for i in range(len(listMeteors)):
1967
3207
1968 ###################### 3.6 - 3.7 PARAMETERS REESTIMATION #########################
3208 ###################### 3.6 - 3.7 PARAMETERS REESTIMATION #########################
1969 meteorAux = numpy.zeros(16)
3209 meteorAux = numpy.zeros(16)
1970
3210
1971 #Loading meteor Data (mHeight, mStart, mPeak, mEnd)
3211 #Loading meteor Data (mHeight, mStart, mPeak, mEnd)
1972 mHeight = int(listMeteors[i][0])
3212 mHeight = listMeteors[i][0]
1973 mStart = int(listMeteors[i][1])
3213 mStart = listMeteors[i][1]
1974 mPeak = int(listMeteors[i][2])
3214 mPeak = listMeteors[i][2]
1975 mEnd = int(listMeteors[i][3])
3215 mEnd = listMeteors[i][3]
1976
3216
1977 #get the volt data between the start and end times of the meteor
3217 #get the volt data between the start and end times of the meteor
1978 meteorVolts = volts[:,mStart:mEnd+1,mHeight]
3218 meteorVolts = volts[:,mStart:mEnd+1,mHeight]
1979 meteorVolts = meteorVolts.reshape(meteorVolts.shape[0], meteorVolts.shape[1], 1)
3219 meteorVolts = meteorVolts.reshape(meteorVolts.shape[0], meteorVolts.shape[1], 1)
1980
3220
1981 #3.6. Phase Difference estimation
3221 #3.6. Phase Difference estimation
1982 phaseDiff, aux = self.__estimatePhaseDifference(meteorVolts, pairslist)
3222 phaseDiff, aux = self.__estimatePhaseDifference(meteorVolts, pairslist)
1983
3223
1984 #3.7. Phase difference removal & meteor start, peak and end times reestimated
3224 #3.7. Phase difference removal & meteor start, peak and end times reestimated
1985 #meteorVolts0.- all Channels, all Profiles
3225 #meteorVolts0.- all Channels, all Profiles
1986 meteorVolts0 = volts[:,:,mHeight]
3226 meteorVolts0 = volts[:,:,mHeight]
@@ -1988,15 +3228,15 class SMDetection(Operation):
1988 meteorNoise = noise[:,mHeight]
3228 meteorNoise = noise[:,mHeight]
1989 meteorVolts0[indSides,:] = self.__shiftPhase(meteorVolts0[indSides,:], phaseDiff) #Phase Shifting
3229 meteorVolts0[indSides,:] = self.__shiftPhase(meteorVolts0[indSides,:], phaseDiff) #Phase Shifting
1990 powerNet0 = numpy.nansum(numpy.abs(meteorVolts0)**2, axis = 0) #Power
3230 powerNet0 = numpy.nansum(numpy.abs(meteorVolts0)**2, axis = 0) #Power
1991
3231
1992 #Times reestimation
3232 #Times reestimation
1993 mStart1 = numpy.where(powerNet0[:mPeak] < meteorThresh[:mPeak])[0]
3233 mStart1 = numpy.where(powerNet0[:mPeak] < meteorThresh[:mPeak])[0]
1994 if mStart1.size > 0:
3234 if mStart1.size > 0:
1995 mStart1 = mStart1[-1] + 1
3235 mStart1 = mStart1[-1] + 1
1996
3236
1997 else:
3237 else:
1998 mStart1 = mPeak
3238 mStart1 = mPeak
1999
3239
2000 mEnd1 = numpy.where(powerNet0[mPeak:] < meteorThresh[mPeak:])[0][0] + mPeak - 1
3240 mEnd1 = numpy.where(powerNet0[mPeak:] < meteorThresh[mPeak:])[0][0] + mPeak - 1
2001 mEndDecayTime1 = numpy.where(powerNet0[mPeak:] < meteorNoise[mPeak:])[0]
3241 mEndDecayTime1 = numpy.where(powerNet0[mPeak:] < meteorNoise[mPeak:])[0]
2002 if mEndDecayTime1.size == 0:
3242 if mEndDecayTime1.size == 0:
@@ -2004,7 +3244,7 class SMDetection(Operation):
2004 else:
3244 else:
2005 mEndDecayTime1 = mEndDecayTime1[0] + mPeak - 1
3245 mEndDecayTime1 = mEndDecayTime1[0] + mPeak - 1
2006 # mPeak1 = meteorVolts0[mStart1:mEnd1 + 1].argmax()
3246 # mPeak1 = meteorVolts0[mStart1:mEnd1 + 1].argmax()
2007
3247
2008 #meteorVolts1.- all Channels, from start to end
3248 #meteorVolts1.- all Channels, from start to end
2009 meteorVolts1 = meteorVolts0[:,mStart1:mEnd1 + 1]
3249 meteorVolts1 = meteorVolts0[:,mStart1:mEnd1 + 1]
2010 meteorVolts2 = meteorVolts0[:,mPeak + lag:mEnd1 + 1]
3250 meteorVolts2 = meteorVolts0[:,mPeak + lag:mEnd1 + 1]
@@ -2013,17 +3253,17 class SMDetection(Operation):
2013 meteorVolts1 = meteorVolts1.reshape(meteorVolts1.shape[0], meteorVolts1.shape[1], 1)
3253 meteorVolts1 = meteorVolts1.reshape(meteorVolts1.shape[0], meteorVolts1.shape[1], 1)
2014 meteorVolts2 = meteorVolts2.reshape(meteorVolts2.shape[0], meteorVolts2.shape[1], 1)
3254 meteorVolts2 = meteorVolts2.reshape(meteorVolts2.shape[0], meteorVolts2.shape[1], 1)
2015 ##################### END PARAMETERS REESTIMATION #########################
3255 ##################### END PARAMETERS REESTIMATION #########################
2016
3256
2017 ##################### 3.8 PHASE DIFFERENCE REESTIMATION ########################
3257 ##################### 3.8 PHASE DIFFERENCE REESTIMATION ########################
2018 # if mEnd1 - mStart1 > 4: #Error Number 6: echo less than 5 samples long; too short for analysis
3258 # if mEnd1 - mStart1 > 4: #Error Number 6: echo less than 5 samples long; too short for analysis
2019 if meteorVolts2.shape[1] > 0:
3259 if meteorVolts2.shape[1] > 0:
2020 #Phase Difference re-estimation
3260 #Phase Difference re-estimation
2021 phaseDiff1, phaseDiffint = self.__estimatePhaseDifference(meteorVolts2, pairslist1) #Phase Difference Estimation
3261 phaseDiff1, phaseDiffint = self.__estimatePhaseDifference(meteorVolts2, pairslist1) #Phase Difference Estimation
2022 # phaseDiff1, phaseDiffint = self.estimatePhaseDifference(meteorVolts2, pairslist)
3262 # phaseDiff1, phaseDiffint = self.estimatePhaseDifference(meteorVolts2, pairslist)
2023 meteorVolts2 = meteorVolts2.reshape(meteorVolts2.shape[0], meteorVolts2.shape[1])
3263 meteorVolts2 = meteorVolts2.reshape(meteorVolts2.shape[0], meteorVolts2.shape[1])
2024 phaseDiff11 = numpy.reshape(phaseDiff1, (phaseDiff1.shape[0],1))
3264 phaseDiff11 = numpy.reshape(phaseDiff1, (phaseDiff1.shape[0],1))
2025 meteorVolts2[indSides,:] = self.__shiftPhase(meteorVolts2[indSides,:], phaseDiff11[0:4]) #Phase Shifting
3265 meteorVolts2[indSides,:] = self.__shiftPhase(meteorVolts2[indSides,:], phaseDiff11[0:4]) #Phase Shifting
2026
3266
2027 #Phase Difference RMS
3267 #Phase Difference RMS
2028 phaseRMS1 = numpy.sqrt(numpy.mean(numpy.square(phaseDiff1)))
3268 phaseRMS1 = numpy.sqrt(numpy.mean(numpy.square(phaseDiff1)))
2029 powerNet1 = numpy.nansum(numpy.abs(meteorVolts1[:,:])**2,0)
3269 powerNet1 = numpy.nansum(numpy.abs(meteorVolts1[:,:])**2,0)
@@ -2038,50 +3278,50 class SMDetection(Operation):
2038 #Vectorize
3278 #Vectorize
2039 meteorAux[0:7] = [mHeight, mStart1, mPeak1, mEnd1, mPeakPower1, mSNR1, phaseRMS1]
3279 meteorAux[0:7] = [mHeight, mStart1, mPeak1, mEnd1, mPeakPower1, mSNR1, phaseRMS1]
2040 meteorAux[7:11] = phaseDiffint[0:4]
3280 meteorAux[7:11] = phaseDiffint[0:4]
2041
3281
2042 #Rejection Criterions
3282 #Rejection Criterions
2043 if phaseRMS1 > thresholdPhase: #Error Number 17: Phase variation
3283 if phaseRMS1 > thresholdPhase: #Error Number 17: Phase variation
2044 meteorAux[-1] = 17
3284 meteorAux[-1] = 17
2045 elif mSNR1 < thresholdDB1: #Error Number 1: SNR < threshold dB
3285 elif mSNR1 < thresholdDB1: #Error Number 1: SNR < threshold dB
2046 meteorAux[-1] = 1
3286 meteorAux[-1] = 1
2047
3287
2048
3288
2049 else:
3289 else:
2050 meteorAux[0:4] = [mHeight, mStart, mPeak, mEnd]
3290 meteorAux[0:4] = [mHeight, mStart, mPeak, mEnd]
2051 meteorAux[-1] = 6 #Error Number 6: echo less than 5 samples long; too short for analysis
3291 meteorAux[-1] = 6 #Error Number 6: echo less than 5 samples long; too short for analysis
2052 PowerSeries = 0
3292 PowerSeries = 0
2053
3293
2054 listMeteors1.append(meteorAux)
3294 listMeteors1.append(meteorAux)
2055 listPowerSeries.append(PowerSeries)
3295 listPowerSeries.append(PowerSeries)
2056 listVoltageSeries.append(meteorVolts1)
3296 listVoltageSeries.append(meteorVolts1)
2057
3297
2058 return listMeteors1, listPowerSeries, listVoltageSeries
3298 return listMeteors1, listPowerSeries, listVoltageSeries
2059
3299
2060 def __estimateDecayTime(self, listMeteors, listPower, timeInterval, frequency):
3300 def __estimateDecayTime(self, listMeteors, listPower, timeInterval, frequency):
2061
3301
2062 threshError = 10
3302 threshError = 10
2063 #Depending if it is 30 or 50 MHz
3303 #Depending if it is 30 or 50 MHz
2064 if frequency == 30.175e6:
3304 if frequency == 30e6:
2065 timeLag = 45*10**-3
3305 timeLag = 45*10**-3
2066 else:
3306 else:
2067 timeLag = 15*10**-3
3307 timeLag = 15*10**-3
2068 lag = int(numpy.ceil(timeLag/timeInterval))
3308 lag = numpy.ceil(timeLag/timeInterval)
2069
3309
2070 listMeteors1 = []
3310 listMeteors1 = []
2071
3311
2072 for i in range(len(listMeteors)):
3312 for i in range(len(listMeteors)):
2073 meteorPower = listPower[i]
3313 meteorPower = listPower[i]
2074 meteorAux = listMeteors[i]
3314 meteorAux = listMeteors[i]
2075
3315
2076 if meteorAux[-1] == 0:
3316 if meteorAux[-1] == 0:
2077
3317
2078 try:
3318 try:
2079 indmax = meteorPower.argmax()
3319 indmax = meteorPower.argmax()
2080 indlag = indmax + lag
3320 indlag = indmax + lag
2081
3321
2082 y = meteorPower[indlag:]
3322 y = meteorPower[indlag:]
2083 x = numpy.arange(0, y.size)*timeLag
3323 x = numpy.arange(0, y.size)*timeLag
2084
3324
2085 #first guess
3325 #first guess
2086 a = y[0]
3326 a = y[0]
2087 tau = timeLag
3327 tau = timeLag
@@ -2090,26 +3330,26 class SMDetection(Operation):
2090 y1 = self.__exponential_function(x, *popt)
3330 y1 = self.__exponential_function(x, *popt)
2091 #error estimation
3331 #error estimation
2092 error = sum((y - y1)**2)/(numpy.var(y)*(y.size - popt.size))
3332 error = sum((y - y1)**2)/(numpy.var(y)*(y.size - popt.size))
2093
3333
2094 decayTime = popt[1]
3334 decayTime = popt[1]
2095 riseTime = indmax*timeInterval
3335 riseTime = indmax*timeInterval
2096 meteorAux[11:13] = [decayTime, error]
3336 meteorAux[11:13] = [decayTime, error]
2097
3337
2098 #Table items 7, 8 and 11
3338 #Table items 7, 8 and 11
2099 if (riseTime > 0.3): #Number 7: Echo rise exceeds 0.3s
3339 if (riseTime > 0.3): #Number 7: Echo rise exceeds 0.3s
2100 meteorAux[-1] = 7
3340 meteorAux[-1] = 7
2101 elif (decayTime < 2*riseTime) : #Number 8: Echo decay time less than than twice rise time
3341 elif (decayTime < 2*riseTime) : #Number 8: Echo decay time less than than twice rise time
2102 meteorAux[-1] = 8
3342 meteorAux[-1] = 8
2103 if (error > threshError): #Number 11: Poor fit to amplitude for estimation of decay time
3343 if (error > threshError): #Number 11: Poor fit to amplitude for estimation of decay time
2104 meteorAux[-1] = 11
3344 meteorAux[-1] = 11
2105
3345
2106
3346
2107 except:
3347 except:
2108 meteorAux[-1] = 11
3348 meteorAux[-1] = 11
2109
3349
2110
3350
2111 listMeteors1.append(meteorAux)
3351 listMeteors1.append(meteorAux)
2112
3352
2113 return listMeteors1
3353 return listMeteors1
2114
3354
2115 #Exponential Function
3355 #Exponential Function
@@ -2117,45 +3357,45 class SMDetection(Operation):
2117 def __exponential_function(self, x, a, tau):
3357 def __exponential_function(self, x, a, tau):
2118 y = a*numpy.exp(-x/tau)
3358 y = a*numpy.exp(-x/tau)
2119 return y
3359 return y
2120
3360
2121 def __getRadialVelocity(self, listMeteors, listVolts, radialStdThresh, pairslist, timeInterval):
3361 def __getRadialVelocity(self, listMeteors, listVolts, radialStdThresh, pairslist, timeInterval):
2122
3362
2123 pairslist1 = list(pairslist)
3363 pairslist1 = list(pairslist)
2124 pairslist1.append((0,4))
3364 pairslist1.append((0,1))
2125 pairslist1.append((1,3))
3365 pairslist1.append((3,4))
2126 numPairs = len(pairslist1)
3366 numPairs = len(pairslist1)
2127 #Time Lag
3367 #Time Lag
2128 timeLag = 45*10**-3
3368 timeLag = 45*10**-3
2129 c = 3e8
3369 c = 3e8
2130 lag = numpy.ceil(timeLag/timeInterval)
3370 lag = numpy.ceil(timeLag/timeInterval)
2131 freq = 30.175e6
3371 freq = 30e6
2132
3372
2133 listMeteors1 = []
3373 listMeteors1 = []
2134
3374
2135 for i in range(len(listMeteors)):
3375 for i in range(len(listMeteors)):
2136 meteorAux = listMeteors[i]
3376 meteorAux = listMeteors[i]
2137 if meteorAux[-1] == 0:
3377 if meteorAux[-1] == 0:
2138 mStart = listMeteors[i][1]
3378 mStart = listMeteors[i][1]
2139 mPeak = listMeteors[i][2]
3379 mPeak = listMeteors[i][2]
2140 mLag = mPeak - mStart + lag
3380 mLag = mPeak - mStart + lag
2141
3381
2142 #get the volt data between the start and end times of the meteor
3382 #get the volt data between the start and end times of the meteor
2143 meteorVolts = listVolts[i]
3383 meteorVolts = listVolts[i]
2144 meteorVolts = meteorVolts.reshape(meteorVolts.shape[0], meteorVolts.shape[1], 1)
3384 meteorVolts = meteorVolts.reshape(meteorVolts.shape[0], meteorVolts.shape[1], 1)
2145
3385
2146 #Get CCF
3386 #Get CCF
2147 allCCFs = self.__calculateCCF(meteorVolts, pairslist1, [-2,-1,0,1,2])
3387 allCCFs = self.__calculateCCF(meteorVolts, pairslist1, [-2,-1,0,1,2])
2148
3388
2149 #Method 2
3389 #Method 2
2150 slopes = numpy.zeros(numPairs)
3390 slopes = numpy.zeros(numPairs)
2151 time = numpy.array([-2,-1,1,2])*timeInterval
3391 time = numpy.array([-2,-1,1,2])*timeInterval
2152 angAllCCF = numpy.angle(allCCFs[:,[0,4,2,3],0])
3392 angAllCCF = numpy.angle(allCCFs[:,[0,1,3,4],0])
2153
3393
2154 #Correct phases
3394 #Correct phases
2155 derPhaseCCF = angAllCCF[:,1:] - angAllCCF[:,0:-1]
3395 derPhaseCCF = angAllCCF[:,1:] - angAllCCF[:,0:-1]
2156 indDer = numpy.where(numpy.abs(derPhaseCCF) > numpy.pi)
3396 indDer = numpy.where(numpy.abs(derPhaseCCF) > numpy.pi)
2157
3397
2158 if indDer[0].shape[0] > 0:
3398 if indDer[0].shape[0] > 0:
2159 for i in range(indDer[0].shape[0]):
3399 for i in range(indDer[0].shape[0]):
2160 signo = -numpy.sign(derPhaseCCF[indDer[0][i],indDer[1][i]])
3400 signo = -numpy.sign(derPhaseCCF[indDer[0][i],indDer[1][i]])
2161 angAllCCF[indDer[0][i],indDer[1][i]+1:] += signo*2*numpy.pi
3401 angAllCCF[indDer[0][i],indDer[1][i]+1:] += signo*2*numpy.pi
@@ -2164,51 +3404,51 class SMDetection(Operation):
2164 for j in range(numPairs):
3404 for j in range(numPairs):
2165 fit = stats.linregress(time, angAllCCF[j,:])
3405 fit = stats.linregress(time, angAllCCF[j,:])
2166 slopes[j] = fit[0]
3406 slopes[j] = fit[0]
2167
3407
2168 #Remove Outlier
3408 #Remove Outlier
2169 # indOut = numpy.argmax(numpy.abs(slopes - numpy.mean(slopes)))
3409 # indOut = numpy.argmax(numpy.abs(slopes - numpy.mean(slopes)))
2170 # slopes = numpy.delete(slopes,indOut)
3410 # slopes = numpy.delete(slopes,indOut)
2171 # indOut = numpy.argmax(numpy.abs(slopes - numpy.mean(slopes)))
3411 # indOut = numpy.argmax(numpy.abs(slopes - numpy.mean(slopes)))
2172 # slopes = numpy.delete(slopes,indOut)
3412 # slopes = numpy.delete(slopes,indOut)
2173
3413
2174 radialVelocity = -numpy.mean(slopes)*(0.25/numpy.pi)*(c/freq)
3414 radialVelocity = -numpy.mean(slopes)*(0.25/numpy.pi)*(c/freq)
2175 radialError = numpy.std(slopes)*(0.25/numpy.pi)*(c/freq)
3415 radialError = numpy.std(slopes)*(0.25/numpy.pi)*(c/freq)
2176 meteorAux[-2] = radialError
3416 meteorAux[-2] = radialError
2177 meteorAux[-3] = radialVelocity
3417 meteorAux[-3] = radialVelocity
2178
3418
2179 #Setting Error
3419 #Setting Error
2180 #Number 15: Radial Drift velocity or projected horizontal velocity exceeds 200 m/s
3420 #Number 15: Radial Drift velocity or projected horizontal velocity exceeds 200 m/s
2181 if numpy.abs(radialVelocity) > 200:
3421 if numpy.abs(radialVelocity) > 200:
2182 meteorAux[-1] = 15
3422 meteorAux[-1] = 15
2183 #Number 12: Poor fit to CCF variation for estimation of radial drift velocity
3423 #Number 12: Poor fit to CCF variation for estimation of radial drift velocity
2184 elif radialError > radialStdThresh:
3424 elif radialError > radialStdThresh:
2185 meteorAux[-1] = 12
3425 meteorAux[-1] = 12
2186
3426
2187 listMeteors1.append(meteorAux)
3427 listMeteors1.append(meteorAux)
2188 return listMeteors1
3428 return listMeteors1
2189
3429
2190 def __setNewArrays(self, listMeteors, date, heiRang):
3430 def __setNewArrays(self, listMeteors, date, heiRang):
2191
3431
2192 #New arrays
3432 #New arrays
2193 arrayMeteors = numpy.array(listMeteors)
3433 arrayMeteors = numpy.array(listMeteors)
2194 arrayParameters = numpy.zeros((len(listMeteors), 13))
3434 arrayParameters = numpy.zeros((len(listMeteors), 13))
2195
3435
2196 #Date inclusion
3436 #Date inclusion
2197 # date = re.findall(r'\((.*?)\)', date)
3437 # date = re.findall(r'\((.*?)\)', date)
2198 # date = date[0].split(',')
3438 # date = date[0].split(',')
2199 # date = map(int, date)
3439 # date = map(int, date)
2200 #
3440 #
2201 # if len(date)<6:
3441 # if len(date)<6:
2202 # date.append(0)
3442 # date.append(0)
2203 #
3443 #
2204 # date = [date[0]*10000 + date[1]*100 + date[2], date[3]*10000 + date[4]*100 + date[5]]
3444 # date = [date[0]*10000 + date[1]*100 + date[2], date[3]*10000 + date[4]*100 + date[5]]
2205 # arrayDate = numpy.tile(date, (len(listMeteors), 1))
3445 # arrayDate = numpy.tile(date, (len(listMeteors), 1))
2206 arrayDate = numpy.tile(date, (len(listMeteors)))
3446 arrayDate = numpy.tile(date, (len(listMeteors)))
2207
3447
2208 #Meteor array
3448 #Meteor array
2209 # arrayMeteors[:,0] = heiRang[arrayMeteors[:,0].astype(int)]
3449 # arrayMeteors[:,0] = heiRang[arrayMeteors[:,0].astype(int)]
2210 # arrayMeteors = numpy.hstack((arrayDate, arrayMeteors))
3450 # arrayMeteors = numpy.hstack((arrayDate, arrayMeteors))
2211
3451
2212 #Parameters Array
3452 #Parameters Array
2213 arrayParameters[:,0] = arrayDate #Date
3453 arrayParameters[:,0] = arrayDate #Date
2214 arrayParameters[:,1] = heiRang[arrayMeteors[:,0].astype(int)] #Range
3454 arrayParameters[:,1] = heiRang[arrayMeteors[:,0].astype(int)] #Range
@@ -2216,13 +3456,13 class SMDetection(Operation):
2216 arrayParameters[:,8:12] = arrayMeteors[:,7:11] #Phases
3456 arrayParameters[:,8:12] = arrayMeteors[:,7:11] #Phases
2217 arrayParameters[:,-1] = arrayMeteors[:,-1] #Error
3457 arrayParameters[:,-1] = arrayMeteors[:,-1] #Error
2218
3458
2219
3459
2220 return arrayParameters
3460 return arrayParameters
2221
3461
2222 class CorrectSMPhases(Operation):
3462 class CorrectSMPhases(Operation):
2223
3463
2224 def run(self, dataOut, phaseOffsets, hmin = 50, hmax = 150, azimuth = 45, channelPositions = None):
3464 def run(self, dataOut, phaseOffsets, hmin = 50, hmax = 150, azimuth = 45, channelPositions = None):
2225
3465
2226 arrayParameters = dataOut.data_param
3466 arrayParameters = dataOut.data_param
2227 pairsList = []
3467 pairsList = []
2228 pairx = (0,1)
3468 pairx = (0,1)
@@ -2230,49 +3470,49 class CorrectSMPhases(Operation):
2230 pairsList.append(pairx)
3470 pairsList.append(pairx)
2231 pairsList.append(pairy)
3471 pairsList.append(pairy)
2232 jph = numpy.zeros(4)
3472 jph = numpy.zeros(4)
2233
3473
2234 phaseOffsets = numpy.array(phaseOffsets)*numpy.pi/180
3474 phaseOffsets = numpy.array(phaseOffsets)*numpy.pi/180
2235 # arrayParameters[:,8:12] = numpy.unwrap(arrayParameters[:,8:12] + phaseOffsets)
3475 # arrayParameters[:,8:12] = numpy.unwrap(arrayParameters[:,8:12] + phaseOffsets)
2236 arrayParameters[:,8:12] = numpy.angle(numpy.exp(1j*(arrayParameters[:,8:12] + phaseOffsets)))
3476 arrayParameters[:,8:12] = numpy.angle(numpy.exp(1j*(arrayParameters[:,8:12] + phaseOffsets)))
2237
3477
2238 meteorOps = SMOperations()
3478 meteorOps = SMOperations()
2239 if channelPositions is None:
3479 if channelPositions == None:
2240 # channelPositions = [(2.5,0), (0,2.5), (0,0), (0,4.5), (-2,0)] #T
3480 # channelPositions = [(2.5,0), (0,2.5), (0,0), (0,4.5), (-2,0)] #T
2241 channelPositions = [(4.5,2), (2,4.5), (2,2), (2,0), (0,2)] #Estrella
3481 channelPositions = [(4.5,2), (2,4.5), (2,2), (2,0), (0,2)] #Estrella
2242
3482
2243 pairslist0, distances = meteorOps.getPhasePairs(channelPositions)
3483 pairslist0, distances = meteorOps.getPhasePairs(channelPositions)
2244 h = (hmin,hmax)
3484 h = (hmin,hmax)
2245
3485
2246 arrayParameters = meteorOps.getMeteorParams(arrayParameters, azimuth, h, pairsList, distances, jph)
3486 arrayParameters = meteorOps.getMeteorParams(arrayParameters, azimuth, h, pairsList, distances, jph)
2247
3487
2248 dataOut.data_param = arrayParameters
3488 dataOut.data_param = arrayParameters
2249 return
3489 return
2250
3490
2251 class SMPhaseCalibration(Operation):
3491 class SMPhaseCalibration(Operation):
2252
3492
2253 __buffer = None
3493 __buffer = None
2254
3494
2255 __initime = None
3495 __initime = None
2256
3496
2257 __dataReady = False
3497 __dataReady = False
2258
3498
2259 __isConfig = False
3499 __isConfig = False
2260
3500
2261 def __checkTime(self, currentTime, initTime, paramInterval, outputInterval):
3501 def __checkTime(self, currentTime, initTime, paramInterval, outputInterval):
2262
3502
2263 dataTime = currentTime + paramInterval
3503 dataTime = currentTime + paramInterval
2264 deltaTime = dataTime - initTime
3504 deltaTime = dataTime - initTime
2265
3505
2266 if deltaTime >= outputInterval or deltaTime < 0:
3506 if deltaTime >= outputInterval or deltaTime < 0:
2267 return True
3507 return True
2268
3508
2269 return False
3509 return False
2270
3510
2271 def __getGammas(self, pairs, d, phases):
3511 def __getGammas(self, pairs, d, phases):
2272 gammas = numpy.zeros(2)
3512 gammas = numpy.zeros(2)
2273
3513
2274 for i in range(len(pairs)):
3514 for i in range(len(pairs)):
2275
3515
2276 pairi = pairs[i]
3516 pairi = pairs[i]
2277
3517
2278 phip3 = phases[:,pairi[0]]
3518 phip3 = phases[:,pairi[0]]
@@ -2286,7 +3526,7 class SMPhaseCalibration(Operation):
2286 jgamma = numpy.angle(numpy.exp(1j*jgamma))
3526 jgamma = numpy.angle(numpy.exp(1j*jgamma))
2287 # jgamma[jgamma>numpy.pi] -= 2*numpy.pi
3527 # jgamma[jgamma>numpy.pi] -= 2*numpy.pi
2288 # jgamma[jgamma<-numpy.pi] += 2*numpy.pi
3528 # jgamma[jgamma<-numpy.pi] += 2*numpy.pi
2289
3529
2290 #Revised distribution
3530 #Revised distribution
2291 jgammaArray = numpy.hstack((jgamma,jgamma+0.5*numpy.pi,jgamma-0.5*numpy.pi))
3531 jgammaArray = numpy.hstack((jgamma,jgamma+0.5*numpy.pi,jgamma-0.5*numpy.pi))
2292
3532
@@ -2295,39 +3535,39 class SMPhaseCalibration(Operation):
2295 rmin = -0.5*numpy.pi
3535 rmin = -0.5*numpy.pi
2296 rmax = 0.5*numpy.pi
3536 rmax = 0.5*numpy.pi
2297 phaseHisto = numpy.histogram(jgammaArray, bins=nBins, range=(rmin,rmax))
3537 phaseHisto = numpy.histogram(jgammaArray, bins=nBins, range=(rmin,rmax))
2298
3538
2299 meteorsY = phaseHisto[0]
3539 meteorsY = phaseHisto[0]
2300 phasesX = phaseHisto[1][:-1]
3540 phasesX = phaseHisto[1][:-1]
2301 width = phasesX[1] - phasesX[0]
3541 width = phasesX[1] - phasesX[0]
2302 phasesX += width/2
3542 phasesX += width/2
2303
3543
2304 #Gaussian aproximation
3544 #Gaussian aproximation
2305 bpeak = meteorsY.argmax()
3545 bpeak = meteorsY.argmax()
2306 peak = meteorsY.max()
3546 peak = meteorsY.max()
2307 jmin = bpeak - 5
3547 jmin = bpeak - 5
2308 jmax = bpeak + 5 + 1
3548 jmax = bpeak + 5 + 1
2309
3549
2310 if jmin<0:
3550 if jmin<0:
2311 jmin = 0
3551 jmin = 0
2312 jmax = 6
3552 jmax = 6
2313 elif jmax > meteorsY.size:
3553 elif jmax > meteorsY.size:
2314 jmin = meteorsY.size - 6
3554 jmin = meteorsY.size - 6
2315 jmax = meteorsY.size
3555 jmax = meteorsY.size
2316
3556
2317 x0 = numpy.array([peak,bpeak,50])
3557 x0 = numpy.array([peak,bpeak,50])
2318 coeff = optimize.leastsq(self.__residualFunction, x0, args=(meteorsY[jmin:jmax], phasesX[jmin:jmax]))
3558 coeff = optimize.leastsq(self.__residualFunction, x0, args=(meteorsY[jmin:jmax], phasesX[jmin:jmax]))
2319
3559
2320 #Gammas
3560 #Gammas
2321 gammas[i] = coeff[0][1]
3561 gammas[i] = coeff[0][1]
2322
3562
2323 return gammas
3563 return gammas
2324
3564
2325 def __residualFunction(self, coeffs, y, t):
3565 def __residualFunction(self, coeffs, y, t):
2326
3566
2327 return y - self.__gauss_function(t, coeffs)
3567 return y - self.__gauss_function(t, coeffs)
2328
3568
2329 def __gauss_function(self, t, coeffs):
3569 def __gauss_function(self, t, coeffs):
2330
3570
2331 return coeffs[0]*numpy.exp(-0.5*((t - coeffs[1]) / coeffs[2])**2)
3571 return coeffs[0]*numpy.exp(-0.5*((t - coeffs[1]) / coeffs[2])**2)
2332
3572
2333 def __getPhases(self, azimuth, h, pairsList, d, gammas, meteorsArray):
3573 def __getPhases(self, azimuth, h, pairsList, d, gammas, meteorsArray):
@@ -2348,16 +3588,16 class SMPhaseCalibration(Operation):
2348 max_xangle = range_angle[iz]/2 + center_xangle
3588 max_xangle = range_angle[iz]/2 + center_xangle
2349 min_yangle = -range_angle[iz]/2 + center_yangle
3589 min_yangle = -range_angle[iz]/2 + center_yangle
2350 max_yangle = range_angle[iz]/2 + center_yangle
3590 max_yangle = range_angle[iz]/2 + center_yangle
2351
3591
2352 inc_x = (max_xangle-min_xangle)/nstepsx
3592 inc_x = (max_xangle-min_xangle)/nstepsx
2353 inc_y = (max_yangle-min_yangle)/nstepsy
3593 inc_y = (max_yangle-min_yangle)/nstepsy
2354
3594
2355 alpha_y = numpy.arange(nstepsy)*inc_y + min_yangle
3595 alpha_y = numpy.arange(nstepsy)*inc_y + min_yangle
2356 alpha_x = numpy.arange(nstepsx)*inc_x + min_xangle
3596 alpha_x = numpy.arange(nstepsx)*inc_x + min_xangle
2357 penalty = numpy.zeros((nstepsx,nstepsy))
3597 penalty = numpy.zeros((nstepsx,nstepsy))
2358 jph_array = numpy.zeros((nchan,nstepsx,nstepsy))
3598 jph_array = numpy.zeros((nchan,nstepsx,nstepsy))
2359 jph = numpy.zeros(nchan)
3599 jph = numpy.zeros(nchan)
2360
3600
2361 # Iterations looking for the offset
3601 # Iterations looking for the offset
2362 for iy in range(int(nstepsy)):
3602 for iy in range(int(nstepsy)):
2363 for ix in range(int(nstepsx)):
3603 for ix in range(int(nstepsx)):
@@ -2387,24 +3627,24 class SMPhaseCalibration(Operation):
2387 error = meteorsArray1[:,-1]
3627 error = meteorsArray1[:,-1]
2388 ind1 = numpy.where(error==0)[0]
3628 ind1 = numpy.where(error==0)[0]
2389 penalty[ix,iy] = ind1.size
3629 penalty[ix,iy] = ind1.size
2390
3630
2391 i,j = numpy.unravel_index(penalty.argmax(), penalty.shape)
3631 i,j = numpy.unravel_index(penalty.argmax(), penalty.shape)
2392 phOffset = jph_array[:,i,j]
3632 phOffset = jph_array[:,i,j]
2393
3633
2394 center_xangle = phOffset[pairx[1]]
3634 center_xangle = phOffset[pairx[1]]
2395 center_yangle = phOffset[pairy[1]]
3635 center_yangle = phOffset[pairy[1]]
2396
3636
2397 phOffset = numpy.angle(numpy.exp(1j*jph_array[:,i,j]))
3637 phOffset = numpy.angle(numpy.exp(1j*jph_array[:,i,j]))
2398 phOffset = phOffset*180/numpy.pi
3638 phOffset = phOffset*180/numpy.pi
2399 return phOffset
3639 return phOffset
2400
3640
2401
3641
2402 def run(self, dataOut, hmin, hmax, channelPositions=None, nHours = 1):
3642 def run(self, dataOut, hmin, hmax, channelPositions=None, nHours = 1):
2403
3643
2404 dataOut.flagNoData = True
3644 dataOut.flagNoData = True
2405 self.__dataReady = False
3645 self.__dataReady = False
2406 dataOut.outputInterval = nHours*3600
3646 dataOut.outputInterval = nHours*3600
2407
3647
2408 if self.__isConfig == False:
3648 if self.__isConfig == False:
2409 # self.__initime = dataOut.datatime.replace(minute = 0, second = 0, microsecond = 03)
3649 # self.__initime = dataOut.datatime.replace(minute = 0, second = 0, microsecond = 03)
2410 #Get Initial LTC time
3650 #Get Initial LTC time
@@ -2412,19 +3652,19 class SMPhaseCalibration(Operation):
2412 self.__initime = (self.__initime.replace(minute = 0, second = 0, microsecond = 0) - datetime.datetime(1970, 1, 1)).total_seconds()
3652 self.__initime = (self.__initime.replace(minute = 0, second = 0, microsecond = 0) - datetime.datetime(1970, 1, 1)).total_seconds()
2413
3653
2414 self.__isConfig = True
3654 self.__isConfig = True
2415
3655
2416 if self.__buffer is None:
3656 if self.__buffer == None:
2417 self.__buffer = dataOut.data_param.copy()
3657 self.__buffer = dataOut.data_param.copy()
2418
3658
2419 else:
3659 else:
2420 self.__buffer = numpy.vstack((self.__buffer, dataOut.data_param))
3660 self.__buffer = numpy.vstack((self.__buffer, dataOut.data_param))
2421
3661
2422 self.__dataReady = self.__checkTime(dataOut.utctime, self.__initime, dataOut.paramInterval, dataOut.outputInterval) #Check if the buffer is ready
3662 self.__dataReady = self.__checkTime(dataOut.utctime, self.__initime, dataOut.paramInterval, dataOut.outputInterval) #Check if the buffer is ready
2423
3663
2424 if self.__dataReady:
3664 if self.__dataReady:
2425 dataOut.utctimeInit = self.__initime
3665 dataOut.utctimeInit = self.__initime
2426 self.__initime += dataOut.outputInterval #to erase time offset
3666 self.__initime += dataOut.outputInterval #to erase time offset
2427
3667
2428 freq = dataOut.frequency
3668 freq = dataOut.frequency
2429 c = dataOut.C #m/s
3669 c = dataOut.C #m/s
2430 lamb = c/freq
3670 lamb = c/freq
@@ -2452,7 +3692,7 class SMPhaseCalibration(Operation):
2452 else:
3692 else:
2453 pairs.append((2,3))
3693 pairs.append((2,3))
2454 # distances1 = [-distances[0]*lamb, distances[1]*lamb, -distances[2]*lamb, distances[3]*lamb]
3694 # distances1 = [-distances[0]*lamb, distances[1]*lamb, -distances[2]*lamb, distances[3]*lamb]
2455
3695
2456 meteorsArray = self.__buffer
3696 meteorsArray = self.__buffer
2457 error = meteorsArray[:,-1]
3697 error = meteorsArray[:,-1]
2458 boolError = (error==0)|(error==3)|(error==4)|(error==13)|(error==14)
3698 boolError = (error==0)|(error==3)|(error==4)|(error==13)|(error==14)
@@ -2460,7 +3700,7 class SMPhaseCalibration(Operation):
2460 meteorsArray = meteorsArray[ind1,:]
3700 meteorsArray = meteorsArray[ind1,:]
2461 meteorsArray[:,-1] = 0
3701 meteorsArray[:,-1] = 0
2462 phases = meteorsArray[:,8:12]
3702 phases = meteorsArray[:,8:12]
2463
3703
2464 #Calculate Gammas
3704 #Calculate Gammas
2465 gammas = self.__getGammas(pairs, distances, phases)
3705 gammas = self.__getGammas(pairs, distances, phases)
2466 # gammas = numpy.array([-21.70409463,45.76935864])*numpy.pi/180
3706 # gammas = numpy.array([-21.70409463,45.76935864])*numpy.pi/180
@@ -2469,24 +3709,23 class SMPhaseCalibration(Operation):
2469 phasesOff = phasesOff.reshape((1,phasesOff.size))
3709 phasesOff = phasesOff.reshape((1,phasesOff.size))
2470 dataOut.data_output = -phasesOff
3710 dataOut.data_output = -phasesOff
2471 dataOut.flagNoData = False
3711 dataOut.flagNoData = False
2472 dataOut.channelList = pairslist0
2473 self.__buffer = None
3712 self.__buffer = None
2474
3713
2475
3714
2476 return
3715 return
2477
3716
2478 class SMOperations():
3717 class SMOperations():
2479
3718
2480 def __init__(self):
3719 def __init__(self):
2481
3720
2482 return
3721 return
2483
3722
2484 def getMeteorParams(self, arrayParameters0, azimuth, h, pairsList, distances, jph):
3723 def getMeteorParams(self, arrayParameters0, azimuth, h, pairsList, distances, jph):
2485
3724
2486 arrayParameters = arrayParameters0.copy()
3725 arrayParameters = arrayParameters0.copy()
2487 hmin = h[0]
3726 hmin = h[0]
2488 hmax = h[1]
3727 hmax = h[1]
2489
3728
2490 #Calculate AOA (Error N 3, 4)
3729 #Calculate AOA (Error N 3, 4)
2491 #JONES ET AL. 1998
3730 #JONES ET AL. 1998
2492 AOAthresh = numpy.pi/8
3731 AOAthresh = numpy.pi/8
@@ -2494,72 +3733,72 class SMOperations():
2494 phases = -arrayParameters[:,8:12] + jph
3733 phases = -arrayParameters[:,8:12] + jph
2495 # phases = numpy.unwrap(phases)
3734 # phases = numpy.unwrap(phases)
2496 arrayParameters[:,3:6], arrayParameters[:,-1] = self.__getAOA(phases, pairsList, distances, error, AOAthresh, azimuth)
3735 arrayParameters[:,3:6], arrayParameters[:,-1] = self.__getAOA(phases, pairsList, distances, error, AOAthresh, azimuth)
2497
3736
2498 #Calculate Heights (Error N 13 and 14)
3737 #Calculate Heights (Error N 13 and 14)
2499 error = arrayParameters[:,-1]
3738 error = arrayParameters[:,-1]
2500 Ranges = arrayParameters[:,1]
3739 Ranges = arrayParameters[:,1]
2501 zenith = arrayParameters[:,4]
3740 zenith = arrayParameters[:,4]
2502 arrayParameters[:,2], arrayParameters[:,-1] = self.__getHeights(Ranges, zenith, error, hmin, hmax)
3741 arrayParameters[:,2], arrayParameters[:,-1] = self.__getHeights(Ranges, zenith, error, hmin, hmax)
2503
3742
2504 #----------------------- Get Final data ------------------------------------
3743 #----------------------- Get Final data ------------------------------------
2505 # error = arrayParameters[:,-1]
3744 # error = arrayParameters[:,-1]
2506 # ind1 = numpy.where(error==0)[0]
3745 # ind1 = numpy.where(error==0)[0]
2507 # arrayParameters = arrayParameters[ind1,:]
3746 # arrayParameters = arrayParameters[ind1,:]
2508
3747
2509 return arrayParameters
3748 return arrayParameters
2510
3749
2511 def __getAOA(self, phases, pairsList, directions, error, AOAthresh, azimuth):
3750 def __getAOA(self, phases, pairsList, directions, error, AOAthresh, azimuth):
2512
3751
2513 arrayAOA = numpy.zeros((phases.shape[0],3))
3752 arrayAOA = numpy.zeros((phases.shape[0],3))
2514 cosdir0, cosdir = self.__getDirectionCosines(phases, pairsList,directions)
3753 cosdir0, cosdir = self.__getDirectionCosines(phases, pairsList,directions)
2515
3754
2516 arrayAOA[:,:2] = self.__calculateAOA(cosdir, azimuth)
3755 arrayAOA[:,:2] = self.__calculateAOA(cosdir, azimuth)
2517 cosDirError = numpy.sum(numpy.abs(cosdir0 - cosdir), axis = 1)
3756 cosDirError = numpy.sum(numpy.abs(cosdir0 - cosdir), axis = 1)
2518 arrayAOA[:,2] = cosDirError
3757 arrayAOA[:,2] = cosDirError
2519
3758
2520 azimuthAngle = arrayAOA[:,0]
3759 azimuthAngle = arrayAOA[:,0]
2521 zenithAngle = arrayAOA[:,1]
3760 zenithAngle = arrayAOA[:,1]
2522
3761
2523 #Setting Error
3762 #Setting Error
2524 indError = numpy.where(numpy.logical_or(error == 3, error == 4))[0]
3763 indError = numpy.where(numpy.logical_or(error == 3, error == 4))[0]
2525 error[indError] = 0
3764 error[indError] = 0
2526 #Number 3: AOA not fesible
3765 #Number 3: AOA not fesible
2527 indInvalid = numpy.where(numpy.logical_and((numpy.logical_or(numpy.isnan(zenithAngle), numpy.isnan(azimuthAngle))),error == 0))[0]
3766 indInvalid = numpy.where(numpy.logical_and((numpy.logical_or(numpy.isnan(zenithAngle), numpy.isnan(azimuthAngle))),error == 0))[0]
2528 error[indInvalid] = 3
3767 error[indInvalid] = 3
2529 #Number 4: Large difference in AOAs obtained from different antenna baselines
3768 #Number 4: Large difference in AOAs obtained from different antenna baselines
2530 indInvalid = numpy.where(numpy.logical_and(cosDirError > AOAthresh,error == 0))[0]
3769 indInvalid = numpy.where(numpy.logical_and(cosDirError > AOAthresh,error == 0))[0]
2531 error[indInvalid] = 4
3770 error[indInvalid] = 4
2532 return arrayAOA, error
3771 return arrayAOA, error
2533
3772
2534 def __getDirectionCosines(self, arrayPhase, pairsList, distances):
3773 def __getDirectionCosines(self, arrayPhase, pairsList, distances):
2535
3774
2536 #Initializing some variables
3775 #Initializing some variables
2537 ang_aux = numpy.array([-8,-7,-6,-5,-4,-3,-2,-1,0,1,2,3,4,5,6,7,8])*2*numpy.pi
3776 ang_aux = numpy.array([-8,-7,-6,-5,-4,-3,-2,-1,0,1,2,3,4,5,6,7,8])*2*numpy.pi
2538 ang_aux = ang_aux.reshape(1,ang_aux.size)
3777 ang_aux = ang_aux.reshape(1,ang_aux.size)
2539
3778
2540 cosdir = numpy.zeros((arrayPhase.shape[0],2))
3779 cosdir = numpy.zeros((arrayPhase.shape[0],2))
2541 cosdir0 = numpy.zeros((arrayPhase.shape[0],2))
3780 cosdir0 = numpy.zeros((arrayPhase.shape[0],2))
2542
3781
2543
3782
2544 for i in range(2):
3783 for i in range(2):
2545 ph0 = arrayPhase[:,pairsList[i][0]]
3784 ph0 = arrayPhase[:,pairsList[i][0]]
2546 ph1 = arrayPhase[:,pairsList[i][1]]
3785 ph1 = arrayPhase[:,pairsList[i][1]]
2547 d0 = distances[pairsList[i][0]]
3786 d0 = distances[pairsList[i][0]]
2548 d1 = distances[pairsList[i][1]]
3787 d1 = distances[pairsList[i][1]]
2549
3788
2550 ph0_aux = ph0 + ph1
3789 ph0_aux = ph0 + ph1
2551 ph0_aux = numpy.angle(numpy.exp(1j*ph0_aux))
3790 ph0_aux = numpy.angle(numpy.exp(1j*ph0_aux))
2552 # ph0_aux[ph0_aux > numpy.pi] -= 2*numpy.pi
3791 # ph0_aux[ph0_aux > numpy.pi] -= 2*numpy.pi
2553 # ph0_aux[ph0_aux < -numpy.pi] += 2*numpy.pi
3792 # ph0_aux[ph0_aux < -numpy.pi] += 2*numpy.pi
2554 #First Estimation
3793 #First Estimation
2555 cosdir0[:,i] = (ph0_aux)/(2*numpy.pi*(d0 - d1))
3794 cosdir0[:,i] = (ph0_aux)/(2*numpy.pi*(d0 - d1))
2556
3795
2557 #Most-Accurate Second Estimation
3796 #Most-Accurate Second Estimation
2558 phi1_aux = ph0 - ph1
3797 phi1_aux = ph0 - ph1
2559 phi1_aux = phi1_aux.reshape(phi1_aux.size,1)
3798 phi1_aux = phi1_aux.reshape(phi1_aux.size,1)
2560 #Direction Cosine 1
3799 #Direction Cosine 1
2561 cosdir1 = (phi1_aux + ang_aux)/(2*numpy.pi*(d0 + d1))
3800 cosdir1 = (phi1_aux + ang_aux)/(2*numpy.pi*(d0 + d1))
2562
3801
2563 #Searching the correct Direction Cosine
3802 #Searching the correct Direction Cosine
2564 cosdir0_aux = cosdir0[:,i]
3803 cosdir0_aux = cosdir0[:,i]
2565 cosdir0_aux = cosdir0_aux.reshape(cosdir0_aux.size,1)
3804 cosdir0_aux = cosdir0_aux.reshape(cosdir0_aux.size,1)
@@ -2568,59 +3807,59 class SMOperations():
2568 indcos = cosDiff.argmin(axis = 1)
3807 indcos = cosDiff.argmin(axis = 1)
2569 #Saving Value obtained
3808 #Saving Value obtained
2570 cosdir[:,i] = cosdir1[numpy.arange(len(indcos)),indcos]
3809 cosdir[:,i] = cosdir1[numpy.arange(len(indcos)),indcos]
2571
3810
2572 return cosdir0, cosdir
3811 return cosdir0, cosdir
2573
3812
2574 def __calculateAOA(self, cosdir, azimuth):
3813 def __calculateAOA(self, cosdir, azimuth):
2575 cosdirX = cosdir[:,0]
3814 cosdirX = cosdir[:,0]
2576 cosdirY = cosdir[:,1]
3815 cosdirY = cosdir[:,1]
2577
3816
2578 zenithAngle = numpy.arccos(numpy.sqrt(1 - cosdirX**2 - cosdirY**2))*180/numpy.pi
3817 zenithAngle = numpy.arccos(numpy.sqrt(1 - cosdirX**2 - cosdirY**2))*180/numpy.pi
2579 azimuthAngle = numpy.arctan2(cosdirX,cosdirY)*180/numpy.pi + azimuth#0 deg north, 90 deg east
3818 azimuthAngle = numpy.arctan2(cosdirX,cosdirY)*180/numpy.pi + azimuth#0 deg north, 90 deg east
2580 angles = numpy.vstack((azimuthAngle, zenithAngle)).transpose()
3819 angles = numpy.vstack((azimuthAngle, zenithAngle)).transpose()
2581
3820
2582 return angles
3821 return angles
2583
3822
2584 def __getHeights(self, Ranges, zenith, error, minHeight, maxHeight):
3823 def __getHeights(self, Ranges, zenith, error, minHeight, maxHeight):
2585
3824
2586 Ramb = 375 #Ramb = c/(2*PRF)
3825 Ramb = 375 #Ramb = c/(2*PRF)
2587 Re = 6371 #Earth Radius
3826 Re = 6371 #Earth Radius
2588 heights = numpy.zeros(Ranges.shape)
3827 heights = numpy.zeros(Ranges.shape)
2589
3828
2590 R_aux = numpy.array([0,1,2])*Ramb
3829 R_aux = numpy.array([0,1,2])*Ramb
2591 R_aux = R_aux.reshape(1,R_aux.size)
3830 R_aux = R_aux.reshape(1,R_aux.size)
2592
3831
2593 Ranges = Ranges.reshape(Ranges.size,1)
3832 Ranges = Ranges.reshape(Ranges.size,1)
2594
3833
2595 Ri = Ranges + R_aux
3834 Ri = Ranges + R_aux
2596 hi = numpy.sqrt(Re**2 + Ri**2 + (2*Re*numpy.cos(zenith*numpy.pi/180)*Ri.transpose()).transpose()) - Re
3835 hi = numpy.sqrt(Re**2 + Ri**2 + (2*Re*numpy.cos(zenith*numpy.pi/180)*Ri.transpose()).transpose()) - Re
2597
3836
2598 #Check if there is a height between 70 and 110 km
3837 #Check if there is a height between 70 and 110 km
2599 h_bool = numpy.sum(numpy.logical_and(hi > minHeight, hi < maxHeight), axis = 1)
3838 h_bool = numpy.sum(numpy.logical_and(hi > minHeight, hi < maxHeight), axis = 1)
2600 ind_h = numpy.where(h_bool == 1)[0]
3839 ind_h = numpy.where(h_bool == 1)[0]
2601
3840
2602 hCorr = hi[ind_h, :]
3841 hCorr = hi[ind_h, :]
2603 ind_hCorr = numpy.where(numpy.logical_and(hi > minHeight, hi < maxHeight))
3842 ind_hCorr = numpy.where(numpy.logical_and(hi > minHeight, hi < maxHeight))
2604
3843
2605 hCorr = hi[ind_hCorr][:len(ind_h)]
3844 hCorr = hi[ind_hCorr][:len(ind_h)]
2606 heights[ind_h] = hCorr
3845 heights[ind_h] = hCorr
2607
3846
2608 #Setting Error
3847 #Setting Error
2609 #Number 13: Height unresolvable echo: not valid height within 70 to 110 km
3848 #Number 13: Height unresolvable echo: not valid height within 70 to 110 km
2610 #Number 14: Height ambiguous echo: more than one possible height within 70 to 110 km
3849 #Number 14: Height ambiguous echo: more than one possible height within 70 to 110 km
2611 indError = numpy.where(numpy.logical_or(error == 13, error == 14))[0]
3850 indError = numpy.where(numpy.logical_or(error == 13, error == 14))[0]
2612 error[indError] = 0
3851 error[indError] = 0
2613 indInvalid2 = numpy.where(numpy.logical_and(h_bool > 1, error == 0))[0]
3852 indInvalid2 = numpy.where(numpy.logical_and(h_bool > 1, error == 0))[0]
2614 error[indInvalid2] = 14
3853 error[indInvalid2] = 14
2615 indInvalid1 = numpy.where(numpy.logical_and(h_bool == 0, error == 0))[0]
3854 indInvalid1 = numpy.where(numpy.logical_and(h_bool == 0, error == 0))[0]
2616 error[indInvalid1] = 13
3855 error[indInvalid1] = 13
2617
3856
2618 return heights, error
3857 return heights, error
2619
3858
2620 def getPhasePairs(self, channelPositions):
3859 def getPhasePairs(self, channelPositions):
2621 chanPos = numpy.array(channelPositions)
3860 chanPos = numpy.array(channelPositions)
2622 listOper = list(itertools.combinations(range(5),2))
3861 listOper = list(itertools.combinations(range(5),2))
2623
3862
2624 distances = numpy.zeros(4)
3863 distances = numpy.zeros(4)
2625 axisX = []
3864 axisX = []
2626 axisY = []
3865 axisY = []
@@ -2628,15 +3867,15 class SMOperations():
2628 distY = numpy.zeros(3)
3867 distY = numpy.zeros(3)
2629 ix = 0
3868 ix = 0
2630 iy = 0
3869 iy = 0
2631
3870
2632 pairX = numpy.zeros((2,2))
3871 pairX = numpy.zeros((2,2))
2633 pairY = numpy.zeros((2,2))
3872 pairY = numpy.zeros((2,2))
2634
3873
2635 for i in range(len(listOper)):
3874 for i in range(len(listOper)):
2636 pairi = listOper[i]
3875 pairi = listOper[i]
2637
3876
2638 posDif = numpy.abs(chanPos[pairi[0],:] - chanPos[pairi[1],:])
3877 posDif = numpy.abs(chanPos[pairi[0],:] - chanPos[pairi[1],:])
2639
3878
2640 if posDif[0] == 0:
3879 if posDif[0] == 0:
2641 axisY.append(pairi)
3880 axisY.append(pairi)
2642 distY[iy] = posDif[1]
3881 distY[iy] = posDif[1]
@@ -2645,7 +3884,7 class SMOperations():
2645 axisX.append(pairi)
3884 axisX.append(pairi)
2646 distX[ix] = posDif[0]
3885 distX[ix] = posDif[0]
2647 ix += 1
3886 ix += 1
2648
3887
2649 for i in range(2):
3888 for i in range(2):
2650 if i==0:
3889 if i==0:
2651 dist0 = distX
3890 dist0 = distX
@@ -2653,7 +3892,7 class SMOperations():
2653 else:
3892 else:
2654 dist0 = distY
3893 dist0 = distY
2655 axis0 = axisY
3894 axis0 = axisY
2656
3895
2657 side = numpy.argsort(dist0)[:-1]
3896 side = numpy.argsort(dist0)[:-1]
2658 axis0 = numpy.array(axis0)[side,:]
3897 axis0 = numpy.array(axis0)[side,:]
2659 chanC = int(numpy.intersect1d(axis0[0,:], axis0[1,:])[0])
3898 chanC = int(numpy.intersect1d(axis0[0,:], axis0[1,:])[0])
@@ -2661,7 +3900,7 class SMOperations():
2661 side = axis1[axis1 != chanC]
3900 side = axis1[axis1 != chanC]
2662 diff1 = chanPos[chanC,i] - chanPos[side[0],i]
3901 diff1 = chanPos[chanC,i] - chanPos[side[0],i]
2663 diff2 = chanPos[chanC,i] - chanPos[side[1],i]
3902 diff2 = chanPos[chanC,i] - chanPos[side[1],i]
2664 if diff1<0:
3903 if diff1<0:
2665 chan2 = side[0]
3904 chan2 = side[0]
2666 d2 = numpy.abs(diff1)
3905 d2 = numpy.abs(diff1)
2667 chan1 = side[1]
3906 chan1 = side[1]
@@ -2671,7 +3910,7 class SMOperations():
2671 d2 = numpy.abs(diff2)
3910 d2 = numpy.abs(diff2)
2672 chan1 = side[0]
3911 chan1 = side[0]
2673 d1 = numpy.abs(diff1)
3912 d1 = numpy.abs(diff1)
2674
3913
2675 if i==0:
3914 if i==0:
2676 chanCX = chanC
3915 chanCX = chanC
2677 chan1X = chan1
3916 chan1X = chan1
@@ -2683,10 +3922,10 class SMOperations():
2683 chan2Y = chan2
3922 chan2Y = chan2
2684 distances[2:4] = numpy.array([d1,d2])
3923 distances[2:4] = numpy.array([d1,d2])
2685 # axisXsides = numpy.reshape(axisX[ix,:],4)
3924 # axisXsides = numpy.reshape(axisX[ix,:],4)
2686 #
3925 #
2687 # channelCentX = int(numpy.intersect1d(pairX[0,:], pairX[1,:])[0])
3926 # channelCentX = int(numpy.intersect1d(pairX[0,:], pairX[1,:])[0])
2688 # channelCentY = int(numpy.intersect1d(pairY[0,:], pairY[1,:])[0])
3927 # channelCentY = int(numpy.intersect1d(pairY[0,:], pairY[1,:])[0])
2689 #
3928 #
2690 # ind25X = numpy.where(pairX[0,:] != channelCentX)[0][0]
3929 # ind25X = numpy.where(pairX[0,:] != channelCentX)[0][0]
2691 # ind20X = numpy.where(pairX[1,:] != channelCentX)[0][0]
3930 # ind20X = numpy.where(pairX[1,:] != channelCentX)[0][0]
2692 # channel25X = int(pairX[0,ind25X])
3931 # channel25X = int(pairX[0,ind25X])
@@ -2695,59 +3934,59 class SMOperations():
2695 # ind20Y = numpy.where(pairY[1,:] != channelCentY)[0][0]
3934 # ind20Y = numpy.where(pairY[1,:] != channelCentY)[0][0]
2696 # channel25Y = int(pairY[0,ind25Y])
3935 # channel25Y = int(pairY[0,ind25Y])
2697 # channel20Y = int(pairY[1,ind20Y])
3936 # channel20Y = int(pairY[1,ind20Y])
2698
3937
2699 # pairslist = [(channelCentX, channel25X),(channelCentX, channel20X),(channelCentY,channel25Y),(channelCentY, channel20Y)]
3938 # pairslist = [(channelCentX, channel25X),(channelCentX, channel20X),(channelCentY,channel25Y),(channelCentY, channel20Y)]
2700 pairslist = [(chanCX, chan1X),(chanCX, chan2X),(chanCY,chan1Y),(chanCY, chan2Y)]
3939 pairslist = [(chanCX, chan1X),(chanCX, chan2X),(chanCY,chan1Y),(chanCY, chan2Y)]
2701
3940
2702 return pairslist, distances
3941 return pairslist, distances
2703 # def __getAOA(self, phases, pairsList, error, AOAthresh, azimuth):
3942 # def __getAOA(self, phases, pairsList, error, AOAthresh, azimuth):
2704 #
3943 #
2705 # arrayAOA = numpy.zeros((phases.shape[0],3))
3944 # arrayAOA = numpy.zeros((phases.shape[0],3))
2706 # cosdir0, cosdir = self.__getDirectionCosines(phases, pairsList)
3945 # cosdir0, cosdir = self.__getDirectionCosines(phases, pairsList)
2707 #
3946 #
2708 # arrayAOA[:,:2] = self.__calculateAOA(cosdir, azimuth)
3947 # arrayAOA[:,:2] = self.__calculateAOA(cosdir, azimuth)
2709 # cosDirError = numpy.sum(numpy.abs(cosdir0 - cosdir), axis = 1)
3948 # cosDirError = numpy.sum(numpy.abs(cosdir0 - cosdir), axis = 1)
2710 # arrayAOA[:,2] = cosDirError
3949 # arrayAOA[:,2] = cosDirError
2711 #
3950 #
2712 # azimuthAngle = arrayAOA[:,0]
3951 # azimuthAngle = arrayAOA[:,0]
2713 # zenithAngle = arrayAOA[:,1]
3952 # zenithAngle = arrayAOA[:,1]
2714 #
3953 #
2715 # #Setting Error
3954 # #Setting Error
2716 # #Number 3: AOA not fesible
3955 # #Number 3: AOA not fesible
2717 # indInvalid = numpy.where(numpy.logical_and((numpy.logical_or(numpy.isnan(zenithAngle), numpy.isnan(azimuthAngle))),error == 0))[0]
3956 # indInvalid = numpy.where(numpy.logical_and((numpy.logical_or(numpy.isnan(zenithAngle), numpy.isnan(azimuthAngle))),error == 0))[0]
2718 # error[indInvalid] = 3
3957 # error[indInvalid] = 3
2719 # #Number 4: Large difference in AOAs obtained from different antenna baselines
3958 # #Number 4: Large difference in AOAs obtained from different antenna baselines
2720 # indInvalid = numpy.where(numpy.logical_and(cosDirError > AOAthresh,error == 0))[0]
3959 # indInvalid = numpy.where(numpy.logical_and(cosDirError > AOAthresh,error == 0))[0]
2721 # error[indInvalid] = 4
3960 # error[indInvalid] = 4
2722 # return arrayAOA, error
3961 # return arrayAOA, error
2723 #
3962 #
2724 # def __getDirectionCosines(self, arrayPhase, pairsList):
3963 # def __getDirectionCosines(self, arrayPhase, pairsList):
2725 #
3964 #
2726 # #Initializing some variables
3965 # #Initializing some variables
2727 # ang_aux = numpy.array([-8,-7,-6,-5,-4,-3,-2,-1,0,1,2,3,4,5,6,7,8])*2*numpy.pi
3966 # ang_aux = numpy.array([-8,-7,-6,-5,-4,-3,-2,-1,0,1,2,3,4,5,6,7,8])*2*numpy.pi
2728 # ang_aux = ang_aux.reshape(1,ang_aux.size)
3967 # ang_aux = ang_aux.reshape(1,ang_aux.size)
2729 #
3968 #
2730 # cosdir = numpy.zeros((arrayPhase.shape[0],2))
3969 # cosdir = numpy.zeros((arrayPhase.shape[0],2))
2731 # cosdir0 = numpy.zeros((arrayPhase.shape[0],2))
3970 # cosdir0 = numpy.zeros((arrayPhase.shape[0],2))
2732 #
3971 #
2733 #
3972 #
2734 # for i in range(2):
3973 # for i in range(2):
2735 # #First Estimation
3974 # #First Estimation
2736 # phi0_aux = arrayPhase[:,pairsList[i][0]] + arrayPhase[:,pairsList[i][1]]
3975 # phi0_aux = arrayPhase[:,pairsList[i][0]] + arrayPhase[:,pairsList[i][1]]
2737 # #Dealias
3976 # #Dealias
2738 # indcsi = numpy.where(phi0_aux > numpy.pi)
3977 # indcsi = numpy.where(phi0_aux > numpy.pi)
2739 # phi0_aux[indcsi] -= 2*numpy.pi
3978 # phi0_aux[indcsi] -= 2*numpy.pi
2740 # indcsi = numpy.where(phi0_aux < -numpy.pi)
3979 # indcsi = numpy.where(phi0_aux < -numpy.pi)
2741 # phi0_aux[indcsi] += 2*numpy.pi
3980 # phi0_aux[indcsi] += 2*numpy.pi
2742 # #Direction Cosine 0
3981 # #Direction Cosine 0
2743 # cosdir0[:,i] = -(phi0_aux)/(2*numpy.pi*0.5)
3982 # cosdir0[:,i] = -(phi0_aux)/(2*numpy.pi*0.5)
2744 #
3983 #
2745 # #Most-Accurate Second Estimation
3984 # #Most-Accurate Second Estimation
2746 # phi1_aux = arrayPhase[:,pairsList[i][0]] - arrayPhase[:,pairsList[i][1]]
3985 # phi1_aux = arrayPhase[:,pairsList[i][0]] - arrayPhase[:,pairsList[i][1]]
2747 # phi1_aux = phi1_aux.reshape(phi1_aux.size,1)
3986 # phi1_aux = phi1_aux.reshape(phi1_aux.size,1)
2748 # #Direction Cosine 1
3987 # #Direction Cosine 1
2749 # cosdir1 = -(phi1_aux + ang_aux)/(2*numpy.pi*4.5)
3988 # cosdir1 = -(phi1_aux + ang_aux)/(2*numpy.pi*4.5)
2750 #
3989 #
2751 # #Searching the correct Direction Cosine
3990 # #Searching the correct Direction Cosine
2752 # cosdir0_aux = cosdir0[:,i]
3991 # cosdir0_aux = cosdir0[:,i]
2753 # cosdir0_aux = cosdir0_aux.reshape(cosdir0_aux.size,1)
3992 # cosdir0_aux = cosdir0_aux.reshape(cosdir0_aux.size,1)
@@ -2756,50 +3995,51 class SMOperations():
2756 # indcos = cosDiff.argmin(axis = 1)
3995 # indcos = cosDiff.argmin(axis = 1)
2757 # #Saving Value obtained
3996 # #Saving Value obtained
2758 # cosdir[:,i] = cosdir1[numpy.arange(len(indcos)),indcos]
3997 # cosdir[:,i] = cosdir1[numpy.arange(len(indcos)),indcos]
2759 #
3998 #
2760 # return cosdir0, cosdir
3999 # return cosdir0, cosdir
2761 #
4000 #
2762 # def __calculateAOA(self, cosdir, azimuth):
4001 # def __calculateAOA(self, cosdir, azimuth):
2763 # cosdirX = cosdir[:,0]
4002 # cosdirX = cosdir[:,0]
2764 # cosdirY = cosdir[:,1]
4003 # cosdirY = cosdir[:,1]
2765 #
4004 #
2766 # zenithAngle = numpy.arccos(numpy.sqrt(1 - cosdirX**2 - cosdirY**2))*180/numpy.pi
4005 # zenithAngle = numpy.arccos(numpy.sqrt(1 - cosdirX**2 - cosdirY**2))*180/numpy.pi
2767 # azimuthAngle = numpy.arctan2(cosdirX,cosdirY)*180/numpy.pi + azimuth #0 deg north, 90 deg east
4006 # azimuthAngle = numpy.arctan2(cosdirX,cosdirY)*180/numpy.pi + azimuth #0 deg north, 90 deg east
2768 # angles = numpy.vstack((azimuthAngle, zenithAngle)).transpose()
4007 # angles = numpy.vstack((azimuthAngle, zenithAngle)).transpose()
2769 #
4008 #
2770 # return angles
4009 # return angles
2771 #
4010 #
2772 # def __getHeights(self, Ranges, zenith, error, minHeight, maxHeight):
4011 # def __getHeights(self, Ranges, zenith, error, minHeight, maxHeight):
2773 #
4012 #
2774 # Ramb = 375 #Ramb = c/(2*PRF)
4013 # Ramb = 375 #Ramb = c/(2*PRF)
2775 # Re = 6371 #Earth Radius
4014 # Re = 6371 #Earth Radius
2776 # heights = numpy.zeros(Ranges.shape)
4015 # heights = numpy.zeros(Ranges.shape)
2777 #
4016 #
2778 # R_aux = numpy.array([0,1,2])*Ramb
4017 # R_aux = numpy.array([0,1,2])*Ramb
2779 # R_aux = R_aux.reshape(1,R_aux.size)
4018 # R_aux = R_aux.reshape(1,R_aux.size)
2780 #
4019 #
2781 # Ranges = Ranges.reshape(Ranges.size,1)
4020 # Ranges = Ranges.reshape(Ranges.size,1)
2782 #
4021 #
2783 # Ri = Ranges + R_aux
4022 # Ri = Ranges + R_aux
2784 # hi = numpy.sqrt(Re**2 + Ri**2 + (2*Re*numpy.cos(zenith*numpy.pi/180)*Ri.transpose()).transpose()) - Re
4023 # hi = numpy.sqrt(Re**2 + Ri**2 + (2*Re*numpy.cos(zenith*numpy.pi/180)*Ri.transpose()).transpose()) - Re
2785 #
4024 #
2786 # #Check if there is a height between 70 and 110 km
4025 # #Check if there is a height between 70 and 110 km
2787 # h_bool = numpy.sum(numpy.logical_and(hi > minHeight, hi < maxHeight), axis = 1)
4026 # h_bool = numpy.sum(numpy.logical_and(hi > minHeight, hi < maxHeight), axis = 1)
2788 # ind_h = numpy.where(h_bool == 1)[0]
4027 # ind_h = numpy.where(h_bool == 1)[0]
2789 #
4028 #
2790 # hCorr = hi[ind_h, :]
4029 # hCorr = hi[ind_h, :]
2791 # ind_hCorr = numpy.where(numpy.logical_and(hi > minHeight, hi < maxHeight))
4030 # ind_hCorr = numpy.where(numpy.logical_and(hi > minHeight, hi < maxHeight))
2792 #
4031 #
2793 # hCorr = hi[ind_hCorr]
4032 # hCorr = hi[ind_hCorr]
2794 # heights[ind_h] = hCorr
4033 # heights[ind_h] = hCorr
2795 #
4034 #
2796 # #Setting Error
4035 # #Setting Error
2797 # #Number 13: Height unresolvable echo: not valid height within 70 to 110 km
4036 # #Number 13: Height unresolvable echo: not valid height within 70 to 110 km
2798 # #Number 14: Height ambiguous echo: more than one possible height within 70 to 110 km
4037 # #Number 14: Height ambiguous echo: more than one possible height within 70 to 110 km
2799 #
4038 #
2800 # indInvalid2 = numpy.where(numpy.logical_and(h_bool > 1, error == 0))[0]
4039 # indInvalid2 = numpy.where(numpy.logical_and(h_bool > 1, error == 0))[0]
2801 # error[indInvalid2] = 14
4040 # error[indInvalid2] = 14
2802 # indInvalid1 = numpy.where(numpy.logical_and(h_bool == 0, error == 0))[0]
4041 # indInvalid1 = numpy.where(numpy.logical_and(h_bool == 0, error == 0))[0]
2803 # error[indInvalid1] = 13
4042 # error[indInvalid1] = 13
2804 #
4043 #
2805 # return heights, error
4044 # return heights, error
4045 No newline at end of file
Show file before | Show file after | Hide comments
@@ -1,3 +1,5
1 import itertools
2
1 import numpy
3 import numpy
2
4
3 from jroproc_base import ProcessingUnit, Operation
5 from jroproc_base import ProcessingUnit, Operation
@@ -109,7 +111,14 class SpectraProc(ProcessingUnit):
109
111
110 if self.dataIn.type == "Spectra":
112 if self.dataIn.type == "Spectra":
111 self.dataOut.copy(self.dataIn)
113 self.dataOut.copy(self.dataIn)
114 <<<<<<< HEAD
112 # self.__selectPairs(pairsList)
115 # self.__selectPairs(pairsList)
116 =======
117 if not pairsList:
118 pairsList = itertools.combinations(self.dataOut.channelList, 2)
119 if self.dataOut.data_cspc is not None:
120 self.__selectPairs(pairsList)
121 >>>>>>> 8048843f4edfb980d0968f42f82054783b84e1cc
113 return True
122 return True
114
123
115 if self.dataIn.type == "Voltage":
124 if self.dataIn.type == "Voltage":
@@ -178,27 +187,21 class SpectraProc(ProcessingUnit):
178
187
179 def __selectPairs(self, pairsList):
188 def __selectPairs(self, pairsList):
180
189
181 if channelList == None:
190 if not pairsList:
182 return
191 return
183
192
184 pairsIndexListSelected = []
193 pairs = []
185
194 pairsIndex = []
186 for thisPair in pairsList:
187
195
188 if thisPair not in self.dataOut.pairsList:
196 for pair in pairsList:
197 if pair[0] not in self.dataOut.channelList or pair[1] not in self.dataOut.channelList:
189 continue
198 continue
190
199 pairs.append(pair)
191 pairIndex = self.dataOut.pairsList.index(thisPair)
200 pairsIndex.append(pairs.index(pair))
192
201
193 pairsIndexListSelected.append(pairIndex)
202 self.dataOut.data_cspc = self.dataOut.data_cspc[pairsIndex]
194
203 self.dataOut.pairsList = pairs
195 if not pairsIndexListSelected:
204 self.dataOut.pairsIndexList = pairsIndex
196 self.dataOut.data_cspc = None
197 self.dataOut.pairsList = []
198 return
199
200 self.dataOut.data_cspc = self.dataOut.data_cspc[pairsIndexListSelected]
201 self.dataOut.pairsList = [self.dataOut.pairsList[i] for i in pairsIndexListSelected]
202
205
203 return
206 return
204
207
Show file before | Show file after | Hide comments
@@ -1,6 +1,5
1 import sys
1 import sys
2 import numpy
2 import numpy
3 from profilehooks import profile
4 from scipy import interpolate
3 from scipy import interpolate
5 from schainpy import cSchain
4 from schainpy import cSchain
6 from jroproc_base import ProcessingUnit, Operation
5 from jroproc_base import ProcessingUnit, Operation
@@ -623,13 +622,6 class Decoder(Operation):
623
622
624 return self.datadecTime
623 return self.datadecTime
625
624
626 #@profile
627 def oldCorrelate(self, i, data, code_block):
628 profilesList = xrange(self.__nProfiles)
629 for j in profilesList:
630 self.datadecTime[i,j,:] = numpy.correlate(data[i,j,:], code_block[j,:], mode='full')[self.nBaud-1:]
631
632 #@profile
633 def __convolutionByBlockInTime(self, data):
625 def __convolutionByBlockInTime(self, data):
634
626
635 repetitions = self.__nProfiles / self.nCode
627 repetitions = self.__nProfiles / self.nCode
@@ -639,25 +631,10 class Decoder(Operation):
639 code_block = numpy.reshape(junk, (self.nCode*repetitions, self.nBaud))
631 code_block = numpy.reshape(junk, (self.nCode*repetitions, self.nBaud))
640 profilesList = xrange(self.__nProfiles)
632 profilesList = xrange(self.__nProfiles)
641
633
642 # def toVectorize(a,b):
634 for i in range(self.__nChannels):
643 # return numpy.correlate(a,b, mode='full')
635 for j in profilesList:
644 # vectorized = numpy.vectorize(toVectorize, signature='(n),(m)->(k)')
636 self.datadecTime[i,j,:] = numpy.correlate(data[i,j,:], code_block[j,:], mode='full')[self.nBaud-1:]
645 for i in range(self.__nChannels):
637 return self.datadecTime
646 # self.datadecTime[i,:,:] = numpy.array([numpy.correlate(data[i,j,:], code_block[j,:], mode='full')[self.nBaud-1:] for j in profilesList ])
647 # def func(i, j):
648 # self.datadecTime[i,j,:] = numpy.correlate(data[i,j,:], code_block[j,:], mode='full')[self.nBaud-1:]
649 # map(lambda j: func(i, j), range(self.__nProfiles))
650 #print data[i,:,:].shape
651 # self.datadecTime[i,:,:] = vectorized(data[i,:,:], code_block[:,:])[:,self.nBaud-1:]
652 for j in profilesList:
653 self.datadecTime[i,j,:] = numpy.correlate(data[i,j,:], code_block[j,:], mode='full')[self.nBaud-1:]
654 # print data[i,:,:]
655 # print cSchain.correlateByBlock(data[i,:,:], code_block, 2)
656 # self.datadecTime[i,:,:] = cSchain.correlateByBlock(data[i,:,:], code_block, 2)
657 # print self.datadecTime[i,:,:]
658 #print self.datadecTime[i,:,:].shape
659 return self.datadecTime
660
661
638
662 def __convolutionByBlockInFreq(self, data):
639 def __convolutionByBlockInFreq(self, data):
663
640
Show file before | Show file after | Hide comments
@@ -7,7 +7,6 import json
7 import numpy
7 import numpy
8 import paho.mqtt.client as mqtt
8 import paho.mqtt.client as mqtt
9 import zmq
9 import zmq
10 from profilehooks import profile
11 import datetime
10 import datetime
12 from zmq.utils.monitor import recv_monitor_message
11 from zmq.utils.monitor import recv_monitor_message
13 from functools import wraps
12 from functools import wraps
@@ -16,6 +15,7 from multiprocessing import Process
16
15
17 from schainpy.model.proc.jroproc_base import Operation, ProcessingUnit
16 from schainpy.model.proc.jroproc_base import Operation, ProcessingUnit
18 from schainpy.model.data.jrodata import JROData
17 from schainpy.model.data.jrodata import JROData
18 from schainpy.utils import log
19
19
20 MAXNUMX = 100
20 MAXNUMX = 100
21 MAXNUMY = 100
21 MAXNUMY = 100
@@ -31,14 +31,13 def roundFloats(obj):
31 return round(obj, 2)
31 return round(obj, 2)
32
32
33 def decimate(z, MAXNUMY):
33 def decimate(z, MAXNUMY):
34 # dx = int(len(self.x)/self.__MAXNUMX) + 1
35
36 dy = int(len(z[0])/MAXNUMY) + 1
34 dy = int(len(z[0])/MAXNUMY) + 1
37
35
38 return z[::, ::dy]
36 return z[::, ::dy]
39
37
40 class throttle(object):
38 class throttle(object):
41 """Decorator that prevents a function from being called more than once every
39 '''
40 Decorator that prevents a function from being called more than once every
42 time period.
41 time period.
43 To create a function that cannot be called more than once a minute, but
42 To create a function that cannot be called more than once a minute, but
44 will sleep until it can be called:
43 will sleep until it can be called:
@@ -49,7 +48,7 class throttle(object):
49 for i in range(10):
48 for i in range(10):
50 foo()
49 foo()
51 print "This function has run %s times." % i
50 print "This function has run %s times." % i
52 """
51 '''
53
52
54 def __init__(self, seconds=0, minutes=0, hours=0):
53 def __init__(self, seconds=0, minutes=0, hours=0):
55 self.throttle_period = datetime.timedelta(
54 self.throttle_period = datetime.timedelta(
@@ -73,9 +72,169 class throttle(object):
73
72
74 return wrapper
73 return wrapper
75
74
75 class Data(object):
76 '''
77 Object to hold data to be plotted
78 '''
79
80 def __init__(self, plottypes, throttle_value):
81 self.plottypes = plottypes
82 self.throttle = throttle_value
83 self.ended = False
84 self.__times = []
85
86 def __str__(self):
87 dum = ['{}{}'.format(key, self.shape(key)) for key in self.data]
88 return 'Data[{}][{}]'.format(';'.join(dum), len(self.__times))
89
90 def __len__(self):
91 return len(self.__times)
92
93 def __getitem__(self, key):
94 if key not in self.data:
95 raise KeyError(log.error('Missing key: {}'.format(key)))
96
97 if 'spc' in key:
98 ret = self.data[key]
99 else:
100 ret = numpy.array([self.data[key][x] for x in self.times])
101 if ret.ndim > 1:
102 ret = numpy.swapaxes(ret, 0, 1)
103 return ret
104
105 def setup(self):
106 '''
107 Configure object
108 '''
109
110 self.ended = False
111 self.data = {}
112 self.__times = []
113 self.__heights = []
114 self.__all_heights = set()
115 for plot in self.plottypes:
116 self.data[plot] = {}
117
118 def shape(self, key):
119 '''
120 Get the shape of the one-element data for the given key
121 '''
122
123 if len(self.data[key]):
124 if 'spc' in key:
125 return self.data[key].shape
126 return self.data[key][self.__times[0]].shape
127 return (0,)
128
129 def update(self, dataOut):
130 '''
131 Update data object with new dataOut
132 '''
133
134 tm = dataOut.utctime
135 if tm in self.__times:
136 return
137
138 self.parameters = getattr(dataOut, 'parameters', [])
139 self.pairs = dataOut.pairsList
140 self.channels = dataOut.channelList
141 self.xrange = (dataOut.getFreqRange(1)/1000. , dataOut.getAcfRange(1) , dataOut.getVelRange(1))
142 self.interval = dataOut.getTimeInterval()
143 self.__heights.append(dataOut.heightList)
144 self.__all_heights.update(dataOut.heightList)
145 self.__times.append(tm)
146
147 for plot in self.plottypes:
148 if plot == 'spc':
149 z = dataOut.data_spc/dataOut.normFactor
150 self.data[plot] = 10*numpy.log10(z)
151 if plot == 'cspc':
152 self.data[plot] = dataOut.data_cspc
153 if plot == 'noise':
154 self.data[plot][tm] = 10*numpy.log10(dataOut.getNoise()/dataOut.normFactor)
155 if plot == 'rti':
156 self.data[plot][tm] = dataOut.getPower()
157 if plot == 'snr_db':
158 self.data['snr'][tm] = dataOut.data_SNR
159 if plot == 'snr':
160 self.data[plot][tm] = 10*numpy.log10(dataOut.data_SNR)
161 if plot == 'dop':
162 self.data[plot][tm] = 10*numpy.log10(dataOut.data_DOP)
163 if plot == 'mean':
164 self.data[plot][tm] = dataOut.data_MEAN
165 if plot == 'std':
166 self.data[plot][tm] = dataOut.data_STD
167 if plot == 'coh':
168 self.data[plot][tm] = dataOut.getCoherence()
169 if plot == 'phase':
170 self.data[plot][tm] = dataOut.getCoherence(phase=True)
171 if plot == 'output':
172 self.data[plot][tm] = dataOut.data_output
173 if plot == 'param':
174 self.data[plot][tm] = dataOut.data_param
175
176 def normalize_heights(self):
177 '''
178 Ensure same-dimension of the data for different heighList
179 '''
180
181 H = numpy.array(list(self.__all_heights))
182 H.sort()
183 for key in self.data:
184 shape = self.shape(key)[:-1] + H.shape
185 for tm, obj in self.data[key].items():
186 h = self.__heights[self.__times.index(tm)]
187 if H.size == h.size:
188 continue
189 index = numpy.where(numpy.in1d(H, h))[0]
190 dummy = numpy.zeros(shape) + numpy.nan
191 if len(shape) == 2:
192 dummy[:, index] = obj
193 else:
194 dummy[index] = obj
195 self.data[key][tm] = dummy
196
197 self.__heights = [H for tm in self.__times]
198
199 def jsonify(self, decimate=False):
200 '''
201 Convert data to json
202 '''
203
204 ret = {}
205 tm = self.times[-1]
206
207 for key, value in self.data:
208 if key in ('spc', 'cspc'):
209 ret[key] = roundFloats(self.data[key].to_list())
210 else:
211 ret[key] = roundFloats(self.data[key][tm].to_list())
212
213 ret['timestamp'] = tm
214 ret['interval'] = self.interval
215
216 @property
217 def times(self):
218 '''
219 Return the list of times of the current data
220 '''
221
222 ret = numpy.array(self.__times)
223 ret.sort()
224 return ret
225
226 @property
227 def heights(self):
228 '''
229 Return the list of heights of the current data
230 '''
231
232 return numpy.array(self.__heights[-1])
76
233
77 class PublishData(Operation):
234 class PublishData(Operation):
78 """Clase publish."""
235 '''
236 Operation to send data over zmq.
237 '''
79
238
80 def __init__(self, **kwargs):
239 def __init__(self, **kwargs):
81 """Inicio."""
240 """Inicio."""
@@ -87,11 +246,11 class PublishData(Operation):
87
246
88 def on_disconnect(self, client, userdata, rc):
247 def on_disconnect(self, client, userdata, rc):
89 if rc != 0:
248 if rc != 0:
90 print("Unexpected disconnection.")
249 log.warning('Unexpected disconnection.')
91 self.connect()
250 self.connect()
92
251
93 def connect(self):
252 def connect(self):
94 print 'trying to connect'
253 log.warning('trying to connect')
95 try:
254 try:
96 self.client.connect(
255 self.client.connect(
97 host=self.host,
256 host=self.host,
@@ -105,7 +264,7 class PublishData(Operation):
105 # retain=True
264 # retain=True
106 # )
265 # )
107 except:
266 except:
108 print "MQTT Conection error."
267 log.error('MQTT Conection error.')
109 self.client = False
268 self.client = False
110
269
111 def setup(self, port=1883, username=None, password=None, clientId="user", zeromq=1, verbose=True, **kwargs):
270 def setup(self, port=1883, username=None, password=None, clientId="user", zeromq=1, verbose=True, **kwargs):
@@ -120,8 +279,7 class PublishData(Operation):
120 self.zeromq = zeromq
279 self.zeromq = zeromq
121 self.mqtt = kwargs.get('plottype', 0)
280 self.mqtt = kwargs.get('plottype', 0)
122 self.client = None
281 self.client = None
123 self.verbose = verbose
282 self.verbose = verbose
124 self.dataOut.firstdata = True
125 setup = []
283 setup = []
126 if mqtt is 1:
284 if mqtt is 1:
127 self.client = mqtt.Client(
285 self.client = mqtt.Client(
@@ -176,7 +334,6 class PublishData(Operation):
176 'type': self.plottype,
334 'type': self.plottype,
177 'yData': yData
335 'yData': yData
178 }
336 }
179 # print payload
180
337
181 elif self.plottype in ('rti', 'power'):
338 elif self.plottype in ('rti', 'power'):
182 data = getattr(self.dataOut, 'data_spc')
339 data = getattr(self.dataOut, 'data_spc')
@@ -230,15 +387,16 class PublishData(Operation):
230 'timestamp': 'None',
387 'timestamp': 'None',
231 'type': None
388 'type': None
232 }
389 }
233 # print 'Publishing data to {}'.format(self.host)
390
234 self.client.publish(self.topic + self.plottype, json.dumps(payload), qos=0)
391 self.client.publish(self.topic + self.plottype, json.dumps(payload), qos=0)
235
392
236 if self.zeromq is 1:
393 if self.zeromq is 1:
237 if self.verbose:
394 if self.verbose:
238 print '[Sending] {} - {}'.format(self.dataOut.type, self.dataOut.datatime)
395 log.log(
396 '{} - {}'.format(self.dataOut.type, self.dataOut.datatime),
397 'Sending'
398 )
239 self.zmq_socket.send_pyobj(self.dataOut)
399 self.zmq_socket.send_pyobj(self.dataOut)
240 self.dataOut.firstdata = False
241
242
400
243 def run(self, dataOut, **kwargs):
401 def run(self, dataOut, **kwargs):
244 self.dataOut = dataOut
402 self.dataOut = dataOut
@@ -253,6 +411,7 class PublishData(Operation):
253 if self.zeromq is 1:
411 if self.zeromq is 1:
254 self.dataOut.finished = True
412 self.dataOut.finished = True
255 self.zmq_socket.send_pyobj(self.dataOut)
413 self.zmq_socket.send_pyobj(self.dataOut)
414 time.sleep(0.1)
256 self.zmq_socket.close()
415 self.zmq_socket.close()
257 if self.client:
416 if self.client:
258 self.client.loop_stop()
417 self.client.loop_stop()
@@ -281,7 +440,7 class ReceiverData(ProcessingUnit):
281 self.receiver = self.context.socket(zmq.PULL)
440 self.receiver = self.context.socket(zmq.PULL)
282 self.receiver.bind(self.address)
441 self.receiver.bind(self.address)
283 time.sleep(0.5)
442 time.sleep(0.5)
284 print '[Starting] ReceiverData from {}'.format(self.address)
443 log.success('ReceiverData from {}'.format(self.address))
285
444
286
445
287 def run(self):
446 def run(self):
@@ -291,8 +450,9 class ReceiverData(ProcessingUnit):
291 self.isConfig = True
450 self.isConfig = True
292
451
293 self.dataOut = self.receiver.recv_pyobj()
452 self.dataOut = self.receiver.recv_pyobj()
294 print '[Receiving] {} - {}'.format(self.dataOut.type,
453 log.log('{} - {}'.format(self.dataOut.type,
295 self.dataOut.datatime.ctime())
454 self.dataOut.datatime.ctime(),),
455 'Receiving')
296
456
297
457
298 class PlotterReceiver(ProcessingUnit, Process):
458 class PlotterReceiver(ProcessingUnit, Process):
@@ -306,7 +466,6 class PlotterReceiver(ProcessingUnit, Process):
306 self.mp = False
466 self.mp = False
307 self.isConfig = False
467 self.isConfig = False
308 self.isWebConfig = False
468 self.isWebConfig = False
309 self.plottypes = []
310 self.connections = 0
469 self.connections = 0
311 server = kwargs.get('server', 'zmq.pipe')
470 server = kwargs.get('server', 'zmq.pipe')
312 plot_server = kwargs.get('plot_server', 'zmq.web')
471 plot_server = kwargs.get('plot_server', 'zmq.web')
@@ -326,19 +485,13 class PlotterReceiver(ProcessingUnit, Process):
326 self.realtime = kwargs.get('realtime', False)
485 self.realtime = kwargs.get('realtime', False)
327 self.throttle_value = kwargs.get('throttle', 5)
486 self.throttle_value = kwargs.get('throttle', 5)
328 self.sendData = self.initThrottle(self.throttle_value)
487 self.sendData = self.initThrottle(self.throttle_value)
488 self.dates = []
329 self.setup()
489 self.setup()
330
490
331 def setup(self):
491 def setup(self):
332
492
333 self.data = {}
493 self.data = Data(self.plottypes, self.throttle_value)
334 self.data['times'] = []
494 self.isConfig = True
335 for plottype in self.plottypes:
336 self.data[plottype] = {}
337 self.data['noise'] = {}
338 self.data['throttle'] = self.throttle_value
339 self.data['ENDED'] = False
340 self.isConfig = True
341 self.data_web = {}
342
495
343 def event_monitor(self, monitor):
496 def event_monitor(self, monitor):
344
497
@@ -355,15 +508,13 class PlotterReceiver(ProcessingUnit, Process):
355 self.connections += 1
508 self.connections += 1
356 if evt['event'] == 512:
509 if evt['event'] == 512:
357 pass
510 pass
358 if self.connections == 0 and self.started is True:
359 self.ended = True
360
511
361 evt.update({'description': events[evt['event']]})
512 evt.update({'description': events[evt['event']]})
362
513
363 if evt['event'] == zmq.EVENT_MONITOR_STOPPED:
514 if evt['event'] == zmq.EVENT_MONITOR_STOPPED:
364 break
515 break
365 monitor.close()
516 monitor.close()
366 print("event monitor thread done!")
517 print('event monitor thread done!')
367
518
368 def initThrottle(self, throttle_value):
519 def initThrottle(self, throttle_value):
369
520
@@ -373,65 +524,16 class PlotterReceiver(ProcessingUnit, Process):
373
524
374 return sendDataThrottled
525 return sendDataThrottled
375
526
376
377 def send(self, data):
527 def send(self, data):
378 # print '[sending] data=%s size=%s' % (data.keys(), len(data['times']))
528 log.success('Sending {}'.format(data), self.name)
379 self.sender.send_pyobj(data)
529 self.sender.send_pyobj(data)
380
530
381
382 def update(self):
383 t = self.dataOut.utctime
384
385 if t in self.data['times']:
386 return
387
388 self.data['times'].append(t)
389 self.data['dataOut'] = self.dataOut
390
391 for plottype in self.plottypes:
392 if plottype == 'spc':
393 z = self.dataOut.data_spc/self.dataOut.normFactor
394 self.data[plottype] = 10*numpy.log10(z)
395 self.data['noise'][t] = 10*numpy.log10(self.dataOut.getNoise()/self.dataOut.normFactor)
396 if plottype == 'cspc':
397 jcoherence = self.dataOut.data_cspc/numpy.sqrt(self.dataOut.data_spc*self.dataOut.data_spc)
398 self.data['cspc_coh'] = numpy.abs(jcoherence)
399 self.data['cspc_phase'] = numpy.arctan2(jcoherence.imag, jcoherence.real)*180/numpy.pi
400 if plottype == 'rti':
401 self.data[plottype][t] = self.dataOut.getPower()
402 if plottype == 'snr':
403 self.data[plottype][t] = 10*numpy.log10(self.dataOut.data_SNR)
404 if plottype == 'dop':
405 self.data[plottype][t] = 10*numpy.log10(self.dataOut.data_DOP)
406 if plottype == 'mean':
407 self.data[plottype][t] = self.dataOut.data_MEAN
408 if plottype == 'std':
409 self.data[plottype][t] = self.dataOut.data_STD
410 if plottype == 'coh':
411 self.data[plottype][t] = self.dataOut.getCoherence()
412 if plottype == 'phase':
413 self.data[plottype][t] = self.dataOut.getCoherence(phase=True)
414 if plottype == 'output':
415 self.data[plottype][t] = self.dataOut.data_output
416 if plottype == 'param':
417 self.data[plottype][t] = self.dataOut.data_param
418 if self.realtime:
419 self.data_web['timestamp'] = t
420 if plottype == 'spc':
421 self.data_web[plottype] = roundFloats(decimate(self.data[plottype]).tolist())
422 elif plottype == 'cspc':
423 self.data_web['cspc_coh'] = roundFloats(decimate(self.data['cspc_coh']).tolist())
424 self.data_web['cspc_phase'] = roundFloats(decimate(self.data['cspc_phase']).tolist())
425 elif plottype == 'noise':
426 self.data_web['noise'] = roundFloats(self.data['noise'][t].tolist())
427 else:
428 self.data_web[plottype] = roundFloats(decimate(self.data[plottype][t]).tolist())
429 self.data_web['interval'] = self.dataOut.getTimeInterval()
430 self.data_web['type'] = plottype
431
432 def run(self):
531 def run(self):
433
532
434 print '[Starting] {} from {}'.format(self.name, self.address)
533 log.success(
534 'Starting from {}'.format(self.address),
535 self.name
536 )
435
537
436 self.context = zmq.Context()
538 self.context = zmq.Context()
437 self.receiver = self.context.socket(zmq.PULL)
539 self.receiver = self.context.socket(zmq.PULL)
@@ -448,39 +550,39 class PlotterReceiver(ProcessingUnit, Process):
448 else:
550 else:
449 self.sender.bind("ipc:///tmp/zmq.plots")
551 self.sender.bind("ipc:///tmp/zmq.plots")
450
552
451 time.sleep(3)
553 time.sleep(2)
452
554
453 t = Thread(target=self.event_monitor, args=(monitor,))
555 t = Thread(target=self.event_monitor, args=(monitor,))
454 t.start()
556 t.start()
455
557
456 while True:
558 while True:
457 self.dataOut = self.receiver.recv_pyobj()
559 dataOut = self.receiver.recv_pyobj()
458 # print '[Receiving] {} - {}'.format(self.dataOut.type,
560 dt = datetime.datetime.fromtimestamp(dataOut.utctime).date()
459 # self.dataOut.datatime.ctime())
561 sended = False
460
562 if dt not in self.dates:
461 self.update()
563 if self.data:
564 self.data.ended = True
565 self.send(self.data)
566 sended = True
567 self.data.setup()
568 self.dates.append(dt)
462
569
463 if self.dataOut.firstdata is True:
570 self.data.update(dataOut)
464 self.data['STARTED'] = True
465
571
466 if self.dataOut.finished is True:
572 if dataOut.finished is True:
467 self.send(self.data)
468 self.connections -= 1
573 self.connections -= 1
469 if self.connections == 0 and self.started:
574 if self.connections == 0 and dt in self.dates:
470 self.ended = True
575 self.data.ended = True
471 self.data['ENDED'] = True
472 self.send(self.data)
576 self.send(self.data)
473 self.setup()
577 self.data.setup()
474 self.started = False
475 else:
578 else:
476 if self.realtime:
579 if self.realtime:
477 self.send(self.data)
580 self.send(self.data)
478 self.sender_web.send_string(json.dumps(self.data_web))
581 # self.sender_web.send_string(self.data.jsonify())
479 else:
582 else:
480 self.sendData(self.send, self.data)
583 if not sended:
481 self.started = True
584 self.sendData(self.send, self.data)
482
585
483 self.data['STARTED'] = False
484 return
586 return
485
587
486 def sendToWeb(self):
588 def sendToWeb(self):
@@ -497,6 +599,6 class PlotterReceiver(ProcessingUnit, Process):
497 time.sleep(1)
599 time.sleep(1)
498 for kwargs in self.operationKwargs.values():
600 for kwargs in self.operationKwargs.values():
499 if 'plot' in kwargs:
601 if 'plot' in kwargs:
500 print '[Sending] Config data to web for {}'.format(kwargs['code'].upper())
602 log.success('[Sending] Config data to web for {}'.format(kwargs['code'].upper()))
501 sender_web_config.send_string(json.dumps(kwargs))
603 sender_web_config.send_string(json.dumps(kwargs))
502 self.isWebConfig = True
604 self.isWebConfig = True No newline at end of file
Show file before | Show file after | Hide comments
@@ -1,4 +1,4
1 """
1 '''
2 SCHAINPY - LOG
2 SCHAINPY - LOG
3 Simple helper for log standarization
3 Simple helper for log standarization
4 Usage:
4 Usage:
@@ -13,47 +13,32 SCHAINPY - LOG
13 which will look like this:
13 which will look like this:
14 [NEVER GONNA] - give you up
14 [NEVER GONNA] - give you up
15 with color red as background and white as foreground.
15 with color red as background and white as foreground.
16 """
16 '''
17 import os
17
18 import sys
19 import click
18 import click
20
19
21 def warning(message):
20 def warning(message, tag='Warning'):
22 click.echo(click.style('[WARNING] - ' + message, fg='yellow'))
21 click.echo(click.style('[{}] {}'.format(tag, message), fg='yellow'))
22 pass
23
23
24
25 def error(message, tag='Error'):
26 click.echo(click.style('[{}] {}'.format(tag, message), fg='red'))
27 pass
24
28
25 def error(message):
26 click.echo(click.style('[ERROR] - ' + message, fg='red', bg='black'))
27
29
30 def success(message, tag='Info'):
31 click.echo(click.style('[{}] {}'.format(tag, message), fg='green'))
32 pass
28
33
29 def success(message):
30 click.echo(click.style(message, fg='green'))
31
34
35 def log(message, tag='Info'):
36 click.echo('[{}] {}'.format(tag, message))
37 pass
32
38
33 def log(message, topic='LOG'):
34 click.echo('[{}] - {}'.format(topic, message))
35
39
36 def makelogger(topic, bg='reset', fg='reset'):
40 def makelogger(tag, bg='reset', fg='reset'):
37 def func(message):
41 def func(message):
38 click.echo(click.style('[{}] - '.format(topic.upper()) + message,
42 click.echo(click.style('[{}] {}'.format(tag.upper(), message),
39 bg=bg, fg=fg))
43 bg=bg, fg=fg))
40 return func
44 return func
41
42 class LoggerForFile():
43 def __init__(self, filename):
44 self.old_stdout=sys.stdout
45 cwd = os.getcwd()
46 self.log_file = open(os.path.join(cwd, filename), 'w+')
47 def write(self, text):
48 text = text.rstrip()
49 if not text:
50 return
51 self.log_file.write(text + '\n')
52 self.old_stdout.write(text + '\n')
53 def flush(self):
54 self.old_stdout.flush()
55
56 def logToFile(filename='log.log'):
57 logger = LoggerForFile(filename)
58 sys.stdout = logger
59
Show file before | Show file after | Hide comments
@@ -1,6 +0,0
1 [Desktop Entry]
2 Encoding=UTF-8
3 Name=Link to
4 Type=Link
5 URL=file:///home/nanosat/schain/schainpy/utils/parameters.txt
6 Icon=text-plain
General Comments 0
You need to be logged in to leave comments. Login now