##// END OF EJS Templates
schain-jc
RSS
merge con digital_rf y pull
José Chávez -
r1067:7abdb8296c68 merge
parent child
Show More
Side by Side Unified
Context file:
r1067:7abdb8296c68 Loading diff...:

The requested changes are too big and content was truncated. Show full diff

Show file before | Show file after | Hide comments
@@ -9,56 +9,53 import datetime
9 9 import traceback
10 10 import math
11 11 import time
12 from multiprocessing import Process, Queue, cpu_count
13
14 import schainpy
15 import schainpy.admin
16 from schainpy.utils.log import logToFile
12 from multiprocessing import Process, cpu_count
17 13
18 14 from xml.etree.ElementTree import ElementTree, Element, SubElement, tostring
19 15 from xml.dom import minidom
20 16
17 import schainpy
18 import schainpy.admin
21 19 from schainpy.model import *
22 from time import sleep
23
24
25
26 def prettify(elem):
27 """Return a pretty-printed XML string for the Element.
28 """
29 rough_string = tostring(elem, 'utf-8')
30 reparsed = minidom.parseString(rough_string)
31 return reparsed.toprettyxml(indent=" ")
32
33 def multiSchain(child, nProcess=cpu_count(), startDate=None, endDate=None, by_day=False):
34 skip = 0
35 cursor = 0
36 nFiles = None
37 processes = []
38 dt1 = datetime.datetime.strptime(startDate, '%Y/%m/%d')
39 dt2 = datetime.datetime.strptime(endDate, '%Y/%m/%d')
20 from schainpy.utils import log
21
22 DTYPES = {
23 'Voltage': '.r',
24 'Spectra': '.pdata'
25 }
26
27 def MPProject(project, n=cpu_count()):
28 '''
29 Project wrapper to run schain in n processes
30 '''
31
32 rconf = project.getReadUnitObj()
33 op = rconf.getOperationObj('run')
34 dt1 = op.getParameterValue('startDate')
35 dt2 = op.getParameterValue('endDate')
40 36 days = (dt2 - dt1).days
41
37
42 38 for day in range(days+1):
43 39 skip = 0
44 40 cursor = 0
45 q = Queue()
46 41 processes = []
47 dt = (dt1 + datetime.timedelta(day)).strftime('%Y/%m/%d')
48 firstProcess = Process(target=child, args=(cursor, skip, q, dt))
49 firstProcess.start()
50 if by_day:
51 continue
52 nFiles = q.get()
53 if nFiles==0:
42 dt = dt1 + datetime.timedelta(day)
43 dt_str = dt.strftime('%Y/%m/%d')
44 reader = JRODataReader()
45 paths, files = reader.searchFilesOffLine(path=rconf.path,
46 startDate=dt,
47 endDate=dt,
48 ext=DTYPES[rconf.datatype])
49 nFiles = len(files)
50 if nFiles == 0:
54 51 continue
55 firstProcess.terminate()
56 skip = int(math.ceil(nFiles/nProcess))
57 while True:
58 processes.append(Process(target=child, args=(cursor, skip, q, dt)))
59 processes[cursor].start()
60 if nFiles < cursor*skip:
61 break
52 skip = int(math.ceil(nFiles/n))
53 while nFiles > cursor*skip:
54 rconf.update(startDate=dt_str, endDate=dt_str, cursor=cursor,
55 skip=skip)
56 p = project.clone()
57 p.start()
58 processes.append(p)
62 59 cursor += 1
63 60
64 61 def beforeExit(exctype, value, trace):
@@ -75,7 +72,6 def multiSchain(child, nProcess=cpu_count(), startDate=None, endDate=None, by_da
75 72
76 73 time.sleep(3)
77 74
78
79 75 class ParameterConf():
80 76
81 77 id = None
@@ -112,7 +108,7 class ParameterConf():
112 108 return self.__formated_value
113 109
114 110 if value == '':
115 raise ValueError, "%s: This parameter value is empty" %self.name
111 raise ValueError, '%s: This parameter value is empty' %self.name
116 112
117 113 if format == 'list':
118 114 strList = value.split(',')
@@ -122,10 +118,10 class ParameterConf():
122 118 return self.__formated_value
123 119
124 120 if format == 'intlist':
125 """
121 '''
126 122 Example:
127 123 value = (0,1,2)
128 """
124 '''
129 125
130 126 new_value = ast.literal_eval(value)
131 127
@@ -137,10 +133,10 class ParameterConf():
137 133 return self.__formated_value
138 134
139 135 if format == 'floatlist':
140 """
136 '''
141 137 Example:
142 138 value = (0.5, 1.4, 2.7)
143 """
139 '''
144 140
145 141 new_value = ast.literal_eval(value)
146 142
@@ -170,38 +166,38 class ParameterConf():
170 166 return self.__formated_value
171 167
172 168 if format == 'pairslist':
173 """
169 '''
174 170 Example:
175 171 value = (0,1),(1,2)
176 """
172 '''
177 173
178 174 new_value = ast.literal_eval(value)
179 175
180 176 if type(new_value) not in (tuple, list):
181 raise ValueError, "%s has to be a tuple or list of pairs" %value
177 raise ValueError, '%s has to be a tuple or list of pairs' %value
182 178
183 179 if type(new_value[0]) not in (tuple, list):
184 180 if len(new_value) != 2:
185 raise ValueError, "%s has to be a tuple or list of pairs" %value
181 raise ValueError, '%s has to be a tuple or list of pairs' %value
186 182 new_value = [new_value]
187 183
188 184 for thisPair in new_value:
189 185 if len(thisPair) != 2:
190 raise ValueError, "%s has to be a tuple or list of pairs" %value
186 raise ValueError, '%s has to be a tuple or list of pairs' %value
191 187
192 188 self.__formated_value = new_value
193 189
194 190 return self.__formated_value
195 191
196 192 if format == 'multilist':
197 """
193 '''
198 194 Example:
199 195 value = (0,1,2),(3,4,5)
200 """
196 '''
201 197 multiList = ast.literal_eval(value)
202 198
203 199 if type(multiList[0]) == int:
204 multiList = ast.literal_eval("(" + value + ")")
200 multiList = ast.literal_eval('(' + value + ')')
205 201
206 202 self.__formated_value = multiList
207 203
@@ -263,9 +259,9 class ParameterConf():
263 259
264 260 def printattr(self):
265 261
266 print "Parameter[%s]: name = %s, value = %s, format = %s" %(self.id, self.name, self.value, self.format)
262 print 'Parameter[%s]: name = %s, value = %s, format = %s' %(self.id, self.name, self.value, self.format)
267 263
268 class OperationConf():
264 class OperationConf():
269 265
270 266 id = None
271 267 name = None
@@ -371,7 +367,9 class OperationConf():
371 367 self.parmConfObjList = []
372 368
373 369 def addParameter(self, name, value, format='str'):
374
370
371 if value is None:
372 return None
375 373 id = self.__getNewId()
376 374
377 375 parmConfObj = ParameterConf()
@@ -431,7 +429,7 class OperationConf():
431 429
432 430 def printattr(self):
433 431
434 print "%s[%s]: name = %s, type = %s, priority = %s" %(self.ELEMENTNAME,
432 print '%s[%s]: name = %s, type = %s, priority = %s' %(self.ELEMENTNAME,
435 433 self.id,
436 434 self.name,
437 435 self.type,
@@ -444,12 +442,11 class OperationConf():
444 442
445 443
446 444 if self.type == 'self':
447 raise ValueError, "This operation type cannot be created"
445 raise ValueError, 'This operation type cannot be created'
448 446
449 if self.type == 'plotter':
450 #Plotter(plotter_name)
447 if self.type == 'plotter':
451 448 if not plotter_queue:
452 raise ValueError, "plotter_queue is not defined. Use:\nmyProject = Project()\nmyProject.setPlotterQueue(plotter_queue)"
449 raise ValueError, 'plotter_queue is not defined. Use:\nmyProject = Project()\nmyProject.setPlotterQueue(plotter_queue)'
453 450
454 451 opObj = Plotter(self.name, plotter_queue)
455 452
@@ -564,7 +561,7 class ProcUnitConf():
564 561
565 562 #Compatible with old signal chain version
566 563 if datatype==None and name==None:
567 raise ValueError, "datatype or name should be defined"
564 raise ValueError, 'datatype or name should be defined'
568 565
569 566 if name==None:
570 567 if 'Proc' in datatype:
@@ -595,7 +592,7 class ProcUnitConf():
595 592
596 593 def addParameter(self, **kwargs):
597 594 '''
598 Add parameters to "run" operation
595 Add parameters to 'run' operation
599 596 '''
600 597 opObj = self.opConfObjList[0]
601 598
@@ -633,11 +630,11 class ProcUnitConf():
633 630 self.datatype = upElement.get('datatype')
634 631 self.inputId = upElement.get('inputId')
635 632
636 if self.ELEMENTNAME == "ReadUnit":
637 self.datatype = self.datatype.replace("Reader", "")
633 if self.ELEMENTNAME == 'ReadUnit':
634 self.datatype = self.datatype.replace('Reader', '')
638 635
639 if self.ELEMENTNAME == "ProcUnit":
640 self.datatype = self.datatype.replace("Proc", "")
636 if self.ELEMENTNAME == 'ProcUnit':
637 self.datatype = self.datatype.replace('Proc', '')
641 638
642 639 if self.inputId == 'None':
643 640 self.inputId = '0'
@@ -653,7 +650,7 class ProcUnitConf():
653 650
654 651 def printattr(self):
655 652
656 print "%s[%s]: name = %s, datatype = %s, inputId = %s" %(self.ELEMENTNAME,
653 print '%s[%s]: name = %s, datatype = %s, inputId = %s' %(self.ELEMENTNAME,
657 654 self.id,
658 655 self.name,
659 656 self.datatype,
@@ -707,17 +704,9 class ProcUnitConf():
707 704
708 705 kwargs[parmConfObj.name] = parmConfObj.getValue()
709 706
710 #ini = time.time()
711
712 #print "\tRunning the '%s' operation with %s" %(opConfObj.name, opConfObj.id)
713 707 sts = self.procUnitObj.call(opType = opConfObj.type,
714 708 opName = opConfObj.name,
715 709 opId = opConfObj.id)
716
717 # total_time = time.time() - ini
718 #
719 # if total_time > 0.002:
720 # print "%s::%s took %f seconds" %(self.name, opConfObj.name, total_time)
721 710
722 711 is_ok = is_ok or sts
723 712
@@ -762,11 +751,12 class ReadUnitConf(ProcUnitConf):
762 751
763 752 return self.ELEMENTNAME
764 753
765 def setup(self, id, name, datatype, path='', startDate="", endDate="", startTime="",
766 endTime="", parentId=None, queue=None, server=None, **kwargs):
754 def setup(self, id, name, datatype, path='', startDate='', endDate='',
755 startTime='', endTime='', parentId=None, server=None, **kwargs):
756
767 757 #Compatible with old signal chain version
768 758 if datatype==None and name==None:
769 raise ValueError, "datatype or name should be defined"
759 raise ValueError, 'datatype or name should be defined'
770 760
771 761 if name==None:
772 762 if 'Reader' in datatype:
@@ -785,39 +775,28 class ReadUnitConf(ProcUnitConf):
785 775 self.endDate = endDate
786 776 self.startTime = startTime
787 777 self.endTime = endTime
788
789 778 self.inputId = '0'
790 779 self.parentId = parentId
791 self.queue = queue
792 780 self.server = server
793 781 self.addRunOperation(**kwargs)
794 782
795 def update(self, datatype, path, startDate, endDate, startTime, endTime, parentId=None, name=None, **kwargs):
783 def update(self, **kwargs):
796 784
797 #Compatible with old signal chain version
798 if datatype==None and name==None:
799 raise ValueError, "datatype or name should be defined"
800
801 if name==None:
785 if 'datatype' in kwargs:
786 datatype = kwargs.pop('datatype')
802 787 if 'Reader' in datatype:
803 name = datatype
788 self.name = datatype
804 789 else:
805 name = '%sReader' %(datatype)
806
807 if datatype==None:
808 datatype = name.replace('Reader','')
809
810 self.datatype = datatype
811 self.name = name
812 self.path = path
813 self.startDate = startDate
814 self.endDate = endDate
815 self.startTime = startTime
816 self.endTime = endTime
790 self.name = '%sReader' %(datatype)
791 self.datatype = self.name.replace('Reader', '')
817 792
793 attrs = ('path', 'startDate', 'endDate', 'startTime', 'endTime', 'parentId')
794
795 for attr in attrs:
796 if attr in kwargs:
797 setattr(self, attr, kwargs.pop(attr))
798
818 799 self.inputId = '0'
819 self.parentId = parentId
820
821 800 self.updateRunOperation(**kwargs)
822 801
823 802 def removeOperations(self):
@@ -832,13 +811,13 class ReadUnitConf(ProcUnitConf):
832 811 opObj = self.addOperation(name = 'run', optype = 'self')
833 812
834 813 if self.server is None:
835 opObj.addParameter(name='datatype' , value=self.datatype, format='str')
836 opObj.addParameter(name='path' , value=self.path, format='str')
837 opObj.addParameter(name='startDate' , value=self.startDate, format='date')
838 opObj.addParameter(name='endDate' , value=self.endDate, format='date')
839 opObj.addParameter(name='startTime' , value=self.startTime, format='time')
840 opObj.addParameter(name='endTime' , value=self.endTime, format='time')
841 opObj.addParameter(name='queue' , value=self.queue, format='obj')
814 opObj.addParameter(name='datatype', value=self.datatype, format='str')
815 opObj.addParameter(name='path', value=self.path, format='str')
816 opObj.addParameter(name='startDate', value=self.startDate, format='date')
817 opObj.addParameter(name='endDate', value=self.endDate, format='date')
818 opObj.addParameter(name='startTime', value=self.startTime, format='time')
819 opObj.addParameter(name='endTime', value=self.endTime, format='time')
820
842 821 for key, value in kwargs.items():
843 822 opObj.addParameter(name=key, value=value, format=type(value).__name__)
844 823 else:
@@ -849,32 +828,21 class ReadUnitConf(ProcUnitConf):
849 828
850 829 def updateRunOperation(self, **kwargs):
851 830
852 opObj = self.getOperationObj(name = 'run')
831 opObj = self.getOperationObj(name='run')
853 832 opObj.removeParameters()
854 833
855 opObj.addParameter(name='datatype' , value=self.datatype, format='str')
856 opObj.addParameter(name='path' , value=self.path, format='str')
857 opObj.addParameter(name='startDate' , value=self.startDate, format='date')
858 opObj.addParameter(name='endDate' , value=self.endDate, format='date')
859 opObj.addParameter(name='startTime' , value=self.startTime, format='time')
860 opObj.addParameter(name='endTime' , value=self.endTime, format='time')
861
834 opObj.addParameter(name='datatype', value=self.datatype, format='str')
835 opObj.addParameter(name='path', value=self.path, format='str')
836 opObj.addParameter(name='startDate', value=self.startDate, format='date')
837 opObj.addParameter(name='endDate', value=self.endDate, format='date')
838 opObj.addParameter(name='startTime', value=self.startTime, format='time')
839 opObj.addParameter(name='endTime', value=self.endTime, format='time')
840
862 841 for key, value in kwargs.items():
863 842 opObj.addParameter(name=key, value=value, format=type(value).__name__)
864 843
865 844 return opObj
866 845
867 # def makeXml(self, projectElement):
868 #
869 # procUnitElement = SubElement(projectElement, self.ELEMENTNAME)
870 # procUnitElement.set('id', str(self.id))
871 # procUnitElement.set('name', self.name)
872 # procUnitElement.set('datatype', self.datatype)
873 # procUnitElement.set('inputId', str(self.inputId))
874 #
875 # for opConfObj in self.opConfObjList:
876 # opConfObj.makeXml(procUnitElement)
877
878 846 def readXml(self, upElement):
879 847
880 848 self.id = upElement.get('id')
@@ -882,8 +850,8 class ReadUnitConf(ProcUnitConf):
882 850 self.datatype = upElement.get('datatype')
883 851 self.inputId = upElement.get('inputId')
884 852
885 if self.ELEMENTNAME == "ReadUnit":
886 self.datatype = self.datatype.replace("Reader", "")
853 if self.ELEMENTNAME == 'ReadUnit':
854 self.datatype = self.datatype.replace('Reader', '')
887 855
888 856 if self.inputId == 'None':
889 857 self.inputId = '0'
@@ -905,8 +873,9 class ReadUnitConf(ProcUnitConf):
905 873 self.endTime = opConfObj.getParameterValue('endTime')
906 874
907 875 class Project(Process):
876
908 877 id = None
909 name = None
878 # name = None
910 879 description = None
911 880 filename = None
912 881
@@ -916,17 +885,17 class Project(Process):
916 885
917 886 plotterQueue = None
918 887
919 def __init__(self, plotter_queue=None, logfile=None):
888 def __init__(self, plotter_queue=None):
889
920 890 Process.__init__(self)
921 891 self.id = None
922 self.name = None
892 # self.name = None
923 893 self.description = None
924 if logfile is not None:
925 logToFile(logfile)
894
926 895 self.plotterQueue = plotter_queue
927 896
928 897 self.procUnitConfObjDict = {}
929
898
930 899 def __getNewId(self):
931 900
932 901 idList = self.procUnitConfObjDict.keys()
@@ -972,18 +941,28 class Project(Process):
972 941
973 942 self.procUnitConfObjDict = newProcUnitConfObjDict
974 943
975 def setup(self, id, name, description):
944 def setup(self, id, name='', description=''):
976 945
946 print
947 print '*'*60
948 print ' Starting SIGNAL CHAIN PROCESSING v%s ' % schainpy.__version__
949 print '*'*60
950 print
977 951 self.id = str(id)
978 self.name = name
979 952 self.description = description
980 953
981 954 def update(self, name, description):
982 955
983 self.name = name
984 956 self.description = description
985 957
958 def clone(self):
959
960 p = Project()
961 p.procUnitConfObjDict = self.procUnitConfObjDict
962 return p
963
986 964 def addReadUnit(self, id=None, datatype=None, name=None, **kwargs):
965
987 966 if id is None:
988 967 idReadUnit = self.__getNewId()
989 968 else:
@@ -1021,7 +1000,7 class Project(Process):
1021 1000 def getReadUnitObj(self):
1022 1001
1023 1002 for obj in self.procUnitConfObjDict.values():
1024 if obj.getElementName() == "ReadUnit":
1003 if obj.getElementName() == 'ReadUnit':
1025 1004 return obj
1026 1005
1027 1006 return None
@@ -1066,20 +1045,20 class Project(Process):
1066 1045 if self.filename:
1067 1046 filename = self.filename
1068 1047 else:
1069 filename = "schain.xml"
1048 filename = 'schain.xml'
1070 1049
1071 1050 if not filename:
1072 print "filename has not been defined. Use setFilename(filename) for do it."
1051 print 'filename has not been defined. Use setFilename(filename) for do it.'
1073 1052 return 0
1074 1053
1075 1054 abs_file = os.path.abspath(filename)
1076 1055
1077 1056 if not os.access(os.path.dirname(abs_file), os.W_OK):
1078 print "No write permission on %s" %os.path.dirname(abs_file)
1057 print 'No write permission on %s' %os.path.dirname(abs_file)
1079 1058 return 0
1080 1059
1081 1060 if os.path.isfile(abs_file) and not(os.access(abs_file, os.W_OK)):
1082 print "File %s already exists and it could not be overwriten" %abs_file
1061 print 'File %s already exists and it could not be overwriten' %abs_file
1083 1062 return 0
1084 1063
1085 1064 self.makeXml()
@@ -1093,13 +1072,13 class Project(Process):
1093 1072 def readXml(self, filename = None):
1094 1073
1095 1074 if not filename:
1096 print "filename is not defined"
1075 print 'filename is not defined'
1097 1076 return 0
1098 1077
1099 1078 abs_file = os.path.abspath(filename)
1100 1079
1101 1080 if not os.path.isfile(abs_file):
1102 print "%s file does not exist" %abs_file
1081 print '%s file does not exist' %abs_file
1103 1082 return 0
1104 1083
1105 1084 self.projectElement = None
@@ -1108,7 +1087,7 class Project(Process):
1108 1087 try:
1109 1088 self.projectElement = ElementTree().parse(abs_file)
1110 1089 except:
1111 print "Error reading %s, verify file format" %filename
1090 print 'Error reading %s, verify file format' %filename
1112 1091 return 0
1113 1092
1114 1093 self.project = self.projectElement.tag
@@ -1145,10 +1124,10 class Project(Process):
1145 1124
1146 1125 def printattr(self):
1147 1126
1148 print "Project[%s]: name = %s, description = %s" %(self.id,
1149 self.name,
1150 self.description)
1151
1127 print 'Project[%s]: name = %s, description = %s' %(self.id,
1128 self.name,
1129 self.description)
1130
1152 1131 for procUnitConfObj in self.procUnitConfObjDict.values():
1153 1132 procUnitConfObj.printattr()
1154 1133
@@ -1179,7 +1158,7 class Project(Process):
1179 1158
1180 1159 self.__connect(puObjIN, thisPUObj)
1181 1160
1182 def __handleError(self, procUnitConfObj, send_email=True):
1161 def __handleError(self, procUnitConfObj, send_email=False):
1183 1162
1184 1163 import socket
1185 1164
@@ -1187,33 +1166,33 class Project(Process):
1187 1166 sys.exc_info()[1],
1188 1167 sys.exc_info()[2])
1189 1168
1190 print "***** Error occurred in %s *****" %(procUnitConfObj.name)
1191 print "***** %s" %err[-1]
1169 print '***** Error occurred in %s *****' %(procUnitConfObj.name)
1170 print '***** %s' %err[-1]
1192 1171
1193 message = "".join(err)
1172 message = ''.join(err)
1194 1173
1195 1174 sys.stderr.write(message)
1196 1175
1197 1176 if not send_email:
1198 1177 return
1199 1178
1200 subject = "SChain v%s: Error running %s\n" %(schainpy.__version__, procUnitConfObj.name)
1179 subject = 'SChain v%s: Error running %s\n' %(schainpy.__version__, procUnitConfObj.name)
1201 1180
1202 subtitle = "%s: %s\n" %(procUnitConfObj.getElementName() ,procUnitConfObj.name)
1203 subtitle += "Hostname: %s\n" %socket.gethostbyname(socket.gethostname())
1204 subtitle += "Working directory: %s\n" %os.path.abspath("./")
1205 subtitle += "Configuration file: %s\n" %self.filename
1206 subtitle += "Time: %s\n" %str(datetime.datetime.now())
1181 subtitle = '%s: %s\n' %(procUnitConfObj.getElementName() ,procUnitConfObj.name)
1182 subtitle += 'Hostname: %s\n' %socket.gethostbyname(socket.gethostname())
1183 subtitle += 'Working directory: %s\n' %os.path.abspath('./')
1184 subtitle += 'Configuration file: %s\n' %self.filename
1185 subtitle += 'Time: %s\n' %str(datetime.datetime.now())
1207 1186
1208 1187 readUnitConfObj = self.getReadUnitObj()
1209 1188 if readUnitConfObj:
1210 subtitle += "\nInput parameters:\n"
1211 subtitle += "[Data path = %s]\n" %readUnitConfObj.path
1212 subtitle += "[Data type = %s]\n" %readUnitConfObj.datatype
1213 subtitle += "[Start date = %s]\n" %readUnitConfObj.startDate
1214 subtitle += "[End date = %s]\n" %readUnitConfObj.endDate
1215 subtitle += "[Start time = %s]\n" %readUnitConfObj.startTime
1216 subtitle += "[End time = %s]\n" %readUnitConfObj.endTime
1189 subtitle += '\nInput parameters:\n'
1190 subtitle += '[Data path = %s]\n' %readUnitConfObj.path
1191 subtitle += '[Data type = %s]\n' %readUnitConfObj.datatype
1192 subtitle += '[Start date = %s]\n' %readUnitConfObj.startDate
1193 subtitle += '[End date = %s]\n' %readUnitConfObj.endDate
1194 subtitle += '[Start time = %s]\n' %readUnitConfObj.startTime
1195 subtitle += '[End time = %s]\n' %readUnitConfObj.endTime
1217 1196
1218 1197 adminObj = schainpy.admin.SchainNotify()
1219 1198 adminObj.sendAlert(message=message,
@@ -1228,15 +1207,15 class Project(Process):
1228 1207 return 0
1229 1208
1230 1209 def runController(self):
1231 """
1210 '''
1232 1211 returns 0 when this process has been stopped, 1 otherwise
1233 """
1212 '''
1234 1213
1235 1214 if self.isPaused():
1236 print "Process suspended"
1215 print 'Process suspended'
1237 1216
1238 1217 while True:
1239 sleep(0.1)
1218 time.sleep(0.1)
1240 1219
1241 1220 if not self.isPaused():
1242 1221 break
@@ -1244,10 +1223,10 class Project(Process):
1244 1223 if self.isStopped():
1245 1224 break
1246 1225
1247 print "Process reinitialized"
1226 print 'Process reinitialized'
1248 1227
1249 1228 if self.isStopped():
1250 print "Process stopped"
1229 print 'Process stopped'
1251 1230 return 0
1252 1231
1253 1232 return 1
@@ -1258,29 +1237,23 class Project(Process):
1258 1237
1259 1238 def setPlotterQueue(self, plotter_queue):
1260 1239
1261 raise NotImplementedError, "Use schainpy.controller_api.ControllerThread instead Project class"
1240 raise NotImplementedError, 'Use schainpy.controller_api.ControllerThread instead Project class'
1262 1241
1263 1242 def getPlotterQueue(self):
1264 1243
1265 raise NotImplementedError, "Use schainpy.controller_api.ControllerThread instead Project class"
1244 raise NotImplementedError, 'Use schainpy.controller_api.ControllerThread instead Project class'
1266 1245
1267 1246 def useExternalPlotter(self):
1268 1247
1269 raise NotImplementedError, "Use schainpy.controller_api.ControllerThread instead Project class"
1248 raise NotImplementedError, 'Use schainpy.controller_api.ControllerThread instead Project class'
1270 1249
1250 def run(self):
1271 1251
1272 def run(self, filename=None):
1252 log.success('Starting {}'.format(self.name))
1273 1253
1274 # self.writeXml(filename)
1275 1254 self.createObjects()
1276 1255 self.connectObjects()
1277 1256
1278 print
1279 print "*"*60
1280 print " Starting SIGNAL CHAIN PROCESSING v%s " %schainpy.__version__
1281 print "*"*60
1282 print
1283
1284 1257 keyList = self.procUnitConfObjDict.keys()
1285 1258 keyList.sort()
1286 1259
@@ -1289,7 +1262,6 class Project(Process):
1289 1262 is_ok = False
1290 1263
1291 1264 for procKey in keyList:
1292 # print "Running the '%s' process with %s" %(procUnitConfObj.name, procUnitConfObj.id)
1293 1265
1294 1266 procUnitConfObj = self.procUnitConfObjDict[procKey]
1295 1267
@@ -1300,19 +1272,18 class Project(Process):
1300 1272 is_ok = False
1301 1273 break
1302 1274 except ValueError, e:
1303 sleep(0.5)
1275 time.sleep(0.5)
1304 1276 self.__handleError(procUnitConfObj, send_email=True)
1305 1277 is_ok = False
1306 1278 break
1307 1279 except:
1308 sleep(0.5)
1280 time.sleep(0.5)
1309 1281 self.__handleError(procUnitConfObj)
1310 1282 is_ok = False
1311 1283 break
1312 1284
1313 1285 #If every process unit finished so end process
1314 1286 if not(is_ok):
1315 # print "Every process unit have finished"
1316 1287 break
1317 1288
1318 1289 if not self.runController():
@@ -1322,3 +1293,5 class Project(Process):
1322 1293 for procKey in keyList:
1323 1294 procUnitConfObj = self.procUnitConfObjDict[procKey]
1324 1295 procUnitConfObj.close()
1296
1297 log.success('{} finished'.format(self.name))
Show file before | Show file after | Hide comments
@@ -6218,3 +6218,6 class ShowMeConsole(QtCore.QObject):
6218 6218 text = text[:-1]
6219 6219
6220 6220 self.textWritten.emit(str(text))
6221
6222 def flush(self):
6223 pass
Show file before | Show file after | Hide comments
@@ -698,7 +698,7 class Spectra(JROData):
698 698 for pair in pairsList:
699 699 if pair not in self.pairsList:
700 700 raise ValueError, "Pair %s is not in dataOut.pairsList" %(pair)
701 pairsIndexList.append(self.pairsList.index(pair))
701 pairsIndexList.append(self.pairsList.index(pair))
702 702 for i in range(len(pairsIndexList)):
703 703 pair = self.pairsList[pairsIndexList[i]]
704 704 ccf = numpy.average(self.data_cspc[pairsIndexList[i], :, :], axis=0)
Show file before | Show file after | Hide comments
This diff has been collapsed as it changes many lines, (1208 lines changed) Show them Hide them
@@ -1,32 +1,33
1 1
2 2 import os
3 import zmq
4 3 import time
5 import numpy
4 import glob
6 5 import datetime
7 import numpy as np
6 from multiprocessing import Process
7
8 import zmq
9 import numpy
8 10 import matplotlib
9 import glob
10 matplotlib.use('TkAgg')
11 11 import matplotlib.pyplot as plt
12 12 from mpl_toolkits.axes_grid1 import make_axes_locatable
13 from matplotlib.ticker import FuncFormatter, LinearLocator
14 from multiprocessing import Process
13 from matplotlib.ticker import FuncFormatter, LinearLocator, MultipleLocator
15 14
16 15 from schainpy.model.proc.jroproc_base import Operation
17
18 plt.ion()
16 from schainpy.utils import log
19 17
20 18 func = lambda x, pos: ('%s') %(datetime.datetime.fromtimestamp(x).strftime('%H:%M'))
21 fromtimestamp = lambda x, mintime : (datetime.datetime.utcfromtimestamp(mintime).replace(hour=(x + 5), minute=0) - d1970).total_seconds()
22 19
20 d1970 = datetime.datetime(1970, 1, 1)
23 21
24 d1970 = datetime.datetime(1970,1,1)
25 22
26 23 class PlotData(Operation, Process):
24 '''
25 Base class for Schain plotting operations
26 '''
27 27
28 28 CODE = 'Figure'
29 29 colormap = 'jro'
30 bgcolor = 'white'
30 31 CONFLATE = False
31 32 __MAXNUMX = 80
32 33 __missing = 1E30
@@ -37,54 +38,143 class PlotData(Operation, Process):
37 38 Process.__init__(self)
38 39 self.kwargs['code'] = self.CODE
39 40 self.mp = False
40 self.dataOut = None
41 self.isConfig = False
42 self.figure = None
41 self.data = None
42 self.isConfig = False
43 self.figures = []
43 44 self.axes = []
45 self.cb_axes = []
44 46 self.localtime = kwargs.pop('localtime', True)
45 47 self.show = kwargs.get('show', True)
46 48 self.save = kwargs.get('save', False)
47 49 self.colormap = kwargs.get('colormap', self.colormap)
48 50 self.colormap_coh = kwargs.get('colormap_coh', 'jet')
49 51 self.colormap_phase = kwargs.get('colormap_phase', 'RdBu_r')
50 self.showprofile = kwargs.get('showprofile', True)
51 self.title = kwargs.get('wintitle', '')
52 self.colormaps = kwargs.get('colormaps', None)
53 self.bgcolor = kwargs.get('bgcolor', self.bgcolor)
54 self.showprofile = kwargs.get('showprofile', False)
55 self.title = kwargs.get('wintitle', self.CODE.upper())
56 self.cb_label = kwargs.get('cb_label', None)
57 self.cb_labels = kwargs.get('cb_labels', None)
52 58 self.xaxis = kwargs.get('xaxis', 'frequency')
53 59 self.zmin = kwargs.get('zmin', None)
54 60 self.zmax = kwargs.get('zmax', None)
61 self.zlimits = kwargs.get('zlimits', None)
55 62 self.xmin = kwargs.get('xmin', None)
63 if self.xmin is not None:
64 self.xmin += 5
56 65 self.xmax = kwargs.get('xmax', None)
57 66 self.xrange = kwargs.get('xrange', 24)
58 67 self.ymin = kwargs.get('ymin', None)
59 68 self.ymax = kwargs.get('ymax', None)
60 self.__MAXNUMY = kwargs.get('decimation', 5000)
61 self.throttle_value = 5
62 self.times = []
63 #self.interactive = self.kwargs['parent']
69 self.xlabel = kwargs.get('xlabel', None)
70 self.__MAXNUMY = kwargs.get('decimation', 100)
71 self.showSNR = kwargs.get('showSNR', False)
72 self.oneFigure = kwargs.get('oneFigure', True)
73 self.width = kwargs.get('width', None)
74 self.height = kwargs.get('height', None)
75 self.colorbar = kwargs.get('colorbar', True)
76 self.factors = kwargs.get('factors', [1, 1, 1, 1, 1, 1, 1, 1])
77 self.titles = ['' for __ in range(16)]
78
79 def __setup(self):
80 '''
81 Common setup for all figures, here figures and axes are created
82 '''
83
84 self.setup()
85
86 if self.width is None:
87 self.width = 8
64 88
89 self.figures = []
90 self.axes = []
91 self.cb_axes = []
92 self.pf_axes = []
93 self.cmaps = []
94
95 size = '15%' if self.ncols==1 else '30%'
96 pad = '4%' if self.ncols==1 else '8%'
97
98 if self.oneFigure:
99 if self.height is None:
100 self.height = 1.4*self.nrows + 1
101 fig = plt.figure(figsize=(self.width, self.height),
102 edgecolor='k',
103 facecolor='w')
104 self.figures.append(fig)
105 for n in range(self.nplots):
106 ax = fig.add_subplot(self.nrows, self.ncols, n+1)
107 ax.tick_params(labelsize=8)
108 ax.firsttime = True
109 self.axes.append(ax)
110 if self.showprofile:
111 cax = self.__add_axes(ax, size=size, pad=pad)
112 cax.tick_params(labelsize=8)
113 self.pf_axes.append(cax)
114 else:
115 if self.height is None:
116 self.height = 3
117 for n in range(self.nplots):
118 fig = plt.figure(figsize=(self.width, self.height),
119 edgecolor='k',
120 facecolor='w')
121 ax = fig.add_subplot(1, 1, 1)
122 ax.tick_params(labelsize=8)
123 ax.firsttime = True
124 self.figures.append(fig)
125 self.axes.append(ax)
126 if self.showprofile:
127 cax = self.__add_axes(ax, size=size, pad=pad)
128 cax.tick_params(labelsize=8)
129 self.pf_axes.append(cax)
130
131 for n in range(self.nrows):
132 if self.colormaps is not None:
133 cmap = plt.get_cmap(self.colormaps[n])
134 else:
135 cmap = plt.get_cmap(self.colormap)
136 cmap.set_bad(self.bgcolor, 1.)
137 self.cmaps.append(cmap)
138
139 def __add_axes(self, ax, size='30%', pad='8%'):
65 140 '''
66 this new parameter is created to plot data from varius channels at different figures
67 1. crear una lista de figuras donde se puedan plotear las figuras,
68 2. dar las opciones de configuracion a cada figura, estas opciones son iguales para ambas figuras
69 3. probar?
141 Add new axes to the given figure
70 142 '''
71 self.ind_plt_ch = kwargs.get('ind_plt_ch', False)
72 self.figurelist = None
143 divider = make_axes_locatable(ax)
144 nax = divider.new_horizontal(size=size, pad=pad)
145 ax.figure.add_axes(nax)
146 return nax
73 147
74 148
75 def fill_gaps(self, x_buffer, y_buffer, z_buffer):
149 def setup(self):
150 '''
151 This method should be implemented in the child class, the following
152 attributes should be set:
153
154 self.nrows: number of rows
155 self.ncols: number of cols
156 self.nplots: number of plots (channels or pairs)
157 self.ylabel: label for Y axes
158 self.titles: list of axes title
159
160 '''
161 raise(NotImplementedError, 'Implement this method in child class')
76 162
163 def fill_gaps(self, x_buffer, y_buffer, z_buffer):
164 '''
165 Create a masked array for missing data
166 '''
77 167 if x_buffer.shape[0] < 2:
78 168 return x_buffer, y_buffer, z_buffer
79 169
80 170 deltas = x_buffer[1:] - x_buffer[0:-1]
81 x_median = np.median(deltas)
171 x_median = numpy.median(deltas)
82 172
83 index = np.where(deltas > 5*x_median)
173 index = numpy.where(deltas > 5*x_median)
84 174
85 175 if len(index[0]) != 0:
86 176 z_buffer[::, index[0], ::] = self.__missing
87 z_buffer = np.ma.masked_inside(z_buffer,
177 z_buffer = numpy.ma.masked_inside(z_buffer,
88 178 0.99*self.__missing,
89 179 1.01*self.__missing)
90 180
@@ -99,110 +189,117 class PlotData(Operation, Process):
99 189 x = self.x
100 190 y = self.y[::dy]
101 191 z = self.z[::, ::, ::dy]
102
192
103 193 return x, y, z
104 194
105 '''
106 JM:
107 elimana las otras imagenes generadas debido a que lso workers no llegan en orden y le pueden
108 poner otro tiempo a la figura q no necesariamente es el ultimo.
109 Solo se realiza cuando termina la imagen.
110 Problemas:
195 def format(self):
196 '''
197 Set min and max values, labels, ticks and titles
198 '''
111 199
112 File "/home/ci-81/workspace/schainv2.3/schainpy/model/graphics/jroplot_data.py", line 145, in __plot
113 for n, eachfigure in enumerate(self.figurelist):
114 TypeError: 'NoneType' object is not iterable
200 if self.xmin is None:
201 xmin = self.min_time
202 else:
203 if self.xaxis is 'time':
204 dt = datetime.datetime.fromtimestamp(self.min_time)
205 xmin = (datetime.datetime.combine(dt.date(),
206 datetime.time(int(self.xmin), 0, 0))-d1970).total_seconds()
207 else:
208 xmin = self.xmin
115 209
116 '''
117 def deleteanotherfiles(self):
118 figurenames=[]
119 if self.figurelist != None:
120 for n, eachfigure in enumerate(self.figurelist):
121 #add specific name for each channel in channelList
122 ghostfigname = os.path.join(self.save, '{}_{}_{}'.format(self.titles[n].replace(' ',''),self.CODE,
123 datetime.datetime.fromtimestamp(self.saveTime).strftime('%y%m%d')))
124 figname = os.path.join(self.save, '{}_{}_{}.png'.format(self.titles[n].replace(' ',''),self.CODE,
125 datetime.datetime.fromtimestamp(self.saveTime).strftime('%y%m%d_%H%M%S')))
126
127 for ghostfigure in glob.glob(ghostfigname+'*'): #ghostfigure will adopt all posible names of figures
128 if ghostfigure != figname:
129 os.remove(ghostfigure)
130 print 'Removing GhostFigures:' , figname
131 else :
132 '''Erasing ghost images for just on******************'''
133 ghostfigname = os.path.join(self.save, '{}_{}'.format(self.CODE,datetime.datetime.fromtimestamp(self.saveTime).strftime('%y%m%d')))
134 figname = os.path.join(self.save, '{}_{}.png'.format(self.CODE,datetime.datetime.fromtimestamp(self.saveTime).strftime('%y%m%d_%H%M%S')))
135 for ghostfigure in glob.glob(ghostfigname+'*'): #ghostfigure will adopt all posible names of figures
136 if ghostfigure != figname:
137 os.remove(ghostfigure)
138 print 'Removing GhostFigures:' , figname
210 if self.xmax is None:
211 xmax = xmin+self.xrange*60*60
212 else:
213 if self.xaxis is 'time':
214 dt = datetime.datetime.fromtimestamp(self.min_time)
215 xmax = (datetime.datetime.combine(dt.date(),
216 datetime.time(int(self.xmax), 0, 0))-d1970).total_seconds()
217 else:
218 xmax = self.xmax
219
220 ymin = self.ymin if self.ymin else numpy.nanmin(self.y)
221 ymax = self.ymax if self.ymax else numpy.nanmax(self.y)
222
223 ystep = 200 if ymax>= 800 else 100 if ymax>=400 else 50 if ymax>=200 else 20
224
225 for n, ax in enumerate(self.axes):
226 if ax.firsttime:
227 ax.set_facecolor(self.bgcolor)
228 ax.yaxis.set_major_locator(MultipleLocator(ystep))
229 if self.xaxis is 'time':
230 ax.xaxis.set_major_formatter(FuncFormatter(func))
231 ax.xaxis.set_major_locator(LinearLocator(9))
232 if self.xlabel is not None:
233 ax.set_xlabel(self.xlabel)
234 ax.set_ylabel(self.ylabel)
235 ax.firsttime = False
236 if self.showprofile:
237 self.pf_axes[n].set_ylim(ymin, ymax)
238 self.pf_axes[n].set_xlim(self.zmin, self.zmax)
239 self.pf_axes[n].set_xlabel('dB')
240 self.pf_axes[n].grid(b=True, axis='x')
241 [tick.set_visible(False) for tick in self.pf_axes[n].get_yticklabels()]
242 if self.colorbar:
243 cb = plt.colorbar(ax.plt, ax=ax, pad=0.02)
244 cb.ax.tick_params(labelsize=8)
245 if self.cb_label:
246 cb.set_label(self.cb_label, size=8)
247 elif self.cb_labels:
248 cb.set_label(self.cb_labels[n], size=8)
249
250 ax.set_title('{} - {} UTC'.format(
251 self.titles[n],
252 datetime.datetime.fromtimestamp(self.max_time).strftime('%H:%M:%S')),
253 size=8)
254 ax.set_xlim(xmin, xmax)
255 ax.set_ylim(ymin, ymax)
256
139 257
140 258 def __plot(self):
141
142 print 'plotting...{}'.format(self.CODE)
143 if self.ind_plt_ch is False : #standard
259 '''
260 '''
261 log.success('Plotting', self.name)
262
263 self.plot()
264 self.format()
265
266 for n, fig in enumerate(self.figures):
267 if self.nrows == 0 or self.nplots == 0:
268 log.warning('No data', self.name)
269 continue
144 270 if self.show:
145 self.figure.show()
146 self.plot()
147 plt.tight_layout()
148 self.figure.canvas.manager.set_window_title('{} {} - {}'.format(self.title, self.CODE.upper(),
149 datetime.datetime.fromtimestamp(self.max_time).strftime('%Y/%m/%d')))
150 else :
151 print 'len(self.figurelist): ',len(self.figurelist)
152 for n, eachfigure in enumerate(self.figurelist):
153 if self.show:
154 eachfigure.show()
155
156 self.plot()
157 eachfigure.tight_layout() # ajuste de cada subplot
158 eachfigure.canvas.manager.set_window_title('{} {} - {}'.format(self.title[n], self.CODE.upper(),
159 datetime.datetime.fromtimestamp(self.max_time).strftime('%Y/%m/%d')))
160
161 # if self.save:
162 # if self.ind_plt_ch is False : #standard
163 # figname = os.path.join(self.save, '{}_{}.png'.format(self.CODE,
164 # datetime.datetime.fromtimestamp(self.saveTime).strftime('%y%m%d_%H%M%S')))
165 # print 'Saving figure: {}'.format(figname)
166 # self.figure.savefig(figname)
167 # else :
168 # for n, eachfigure in enumerate(self.figurelist):
169 # #add specific name for each channel in channelList
170 # figname = os.path.join(self.save, '{}_{}_{}.png'.format(self.titles[n],self.CODE,
171 # datetime.datetime.fromtimestamp(self.saveTime).strftime('%y%m%d_%H%M%S')))
172 #
173 # print 'Saving figure: {}'.format(figname)
174 # eachfigure.savefig(figname)
175
176 if self.ind_plt_ch is False :
177 self.figure.canvas.draw()
178 else :
179 for eachfigure in self.figurelist:
180 eachfigure.canvas.draw()
181
182 if self.save:
183 if self.ind_plt_ch is False : #standard
184 figname = os.path.join(self.save, '{}_{}.png'.format(self.CODE,
185 datetime.datetime.fromtimestamp(self.saveTime).strftime('%y%m%d_%H%M%S')))
271 fig.show()
272
273 fig.tight_layout()
274 fig.canvas.manager.set_window_title('{} - {}'.format(self.title,
275 datetime.datetime.fromtimestamp(self.max_time).strftime('%Y/%m/%d')))
276 # fig.canvas.draw()
277
278 if self.save and self.data.ended:
279 channels = range(self.nrows)
280 if self.oneFigure:
281 label = ''
282 else:
283 label = '_{}'.format(channels[n])
284 figname = os.path.join(
285 self.save,
286 '{}{}_{}.png'.format(
287 self.CODE,
288 label,
289 datetime.datetime.fromtimestamp(self.saveTime).strftime('%y%m%d_%H%M%S')
290 )
291 )
186 292 print 'Saving figure: {}'.format(figname)
187 self.figure.savefig(figname)
188 else :
189 for n, eachfigure in enumerate(self.figurelist):
190 #add specific name for each channel in channelList
191 figname = os.path.join(self.save, '{}_{}_{}.png'.format(self.titles[n].replace(' ',''),self.CODE,
192 datetime.datetime.fromtimestamp(self.saveTime).strftime('%y%m%d_%H%M%S')))
193
194 print 'Saving figure: {}'.format(figname)
195 eachfigure.savefig(figname)
196
293 fig.savefig(figname)
197 294
198 295 def plot(self):
199
200 print 'plotting...{}'.format(self.CODE.upper())
201 return
296 '''
297 '''
298 raise(NotImplementedError, 'Implement this method in child class')
202 299
203 300 def run(self):
204 301
205 print '[Starting] {}'.format(self.name)
302 log.success('Starting', self.name)
206 303
207 304 context = zmq.Context()
208 305 receiver = context.socket(zmq.SUB)
@@ -212,152 +309,104 class PlotData(Operation, Process):
212 309 if 'server' in self.kwargs['parent']:
213 310 receiver.connect('ipc:///tmp/{}.plots'.format(self.kwargs['parent']['server']))
214 311 else:
215 receiver.connect("ipc:///tmp/zmq.plots")
216
217 seconds_passed = 0
312 receiver.connect("ipc:///tmp/zmq.plots")
218 313
219 314 while True:
220 315 try:
221 self.data = receiver.recv_pyobj(flags=zmq.NOBLOCK)#flags=zmq.NOBLOCK
222 self.started = self.data['STARTED']
223 self.dataOut = self.data['dataOut']
224
225 if (len(self.times) < len(self.data['times']) and not self.started and self.data['ENDED']):
226 continue
227
228 self.times = self.data['times']
229 self.times.sort()
230 self.throttle_value = self.data['throttle']
231 self.min_time = self.times[0]
232 self.max_time = self.times[-1]
316 self.data = receiver.recv_pyobj(flags=zmq.NOBLOCK)
317
318 self.min_time = self.data.times[0]
319 self.max_time = self.data.times[-1]
233 320
234 321 if self.isConfig is False:
235 print 'setting up'
236 self.setup()
322 self.__setup()
237 323 self.isConfig = True
238 self.__plot()
239
240 if self.data['ENDED'] is True:
241 print '********GRAPHIC ENDED********'
242 self.ended = True
243 self.isConfig = False
244 self.__plot()
245 self.deleteanotherfiles() #CLPDG
246 elif seconds_passed >= self.data['throttle']:
247 print 'passed', seconds_passed
248 self.__plot()
249 seconds_passed = 0
324
325 self.__plot()
250 326
251 327 except zmq.Again as e:
252 print 'Waiting for data...'
253 plt.pause(2)
254 seconds_passed += 2
328 log.log('Waiting for data...')
329 if self.data:
330 plt.pause(self.data.throttle)
331 else:
332 time.sleep(2)
255 333
256 334 def close(self):
257 if self.dataOut:
335 if self.data:
258 336 self.__plot()
259 337
260 338
261 339 class PlotSpectraData(PlotData):
340 '''
341 Plot for Spectra data
342 '''
262 343
263 344 CODE = 'spc'
264 colormap = 'jro'
265 CONFLATE = False
345 colormap = 'jro'
266 346
267 347 def setup(self):
268
269 ncolspan = 1
270 colspan = 1
271 self.ncols = int(numpy.sqrt(self.dataOut.nChannels)+0.9)
272 self.nrows = int(self.dataOut.nChannels*1./self.ncols + 0.9)
273 self.width = 3.6*self.ncols
274 self.height = 3.2*self.nrows
275 if self.showprofile:
276 ncolspan = 3
277 colspan = 2
278 self.width += 1.2*self.ncols
348 self.nplots = len(self.data.channels)
349 self.ncols = int(numpy.sqrt(self.nplots)+ 0.9)
350 self.nrows = int((1.0*self.nplots/self.ncols) + 0.9)
351 self.width = 3.4*self.ncols
352 self.height = 3*self.nrows
353 self.cb_label = 'dB'
354 if self.showprofile:
355 self.width += 0.8*self.ncols
279 356
280 357 self.ylabel = 'Range [Km]'
281 self.titles = ['Channel {}'.format(x) for x in self.dataOut.channelList]
282
283 if self.figure is None:
284 self.figure = plt.figure(figsize=(self.width, self.height),
285 edgecolor='k',
286 facecolor='w')
287 else:
288 self.figure.clf()
289
290 n = 0
291 for y in range(self.nrows):
292 for x in range(self.ncols):
293 if n >= self.dataOut.nChannels:
294 break
295 ax = plt.subplot2grid((self.nrows, self.ncols*ncolspan), (y, x*ncolspan), 1, colspan)
296 if self.showprofile:
297 ax.ax_profile = plt.subplot2grid((self.nrows, self.ncols*ncolspan), (y, x*ncolspan+colspan), 1, 1)
298
299 ax.firsttime = True
300 self.axes.append(ax)
301 n += 1
302 358
303 359 def plot(self):
304
305 360 if self.xaxis == "frequency":
306 x = self.dataOut.getFreqRange(1)/1000.
307 xlabel = "Frequency (kHz)"
361 x = self.data.xrange[0]
362 self.xlabel = "Frequency (kHz)"
308 363 elif self.xaxis == "time":
309 x = self.dataOut.getAcfRange(1)
310 xlabel = "Time (ms)"
364 x = self.data.xrange[1]
365 self.xlabel = "Time (ms)"
311 366 else:
312 x = self.dataOut.getVelRange(1)
313 xlabel = "Velocity (m/s)"
367 x = self.data.xrange[2]
368 self.xlabel = "Velocity (m/s)"
369
370 if self.CODE == 'spc_mean':
371 x = self.data.xrange[2]
372 self.xlabel = "Velocity (m/s)"
314 373
315 y = self.dataOut.getHeiRange()
316 z = self.data[self.CODE]
374 self.titles = []
317 375
376 y = self.data.heights
377 self.y = y
378 z = self.data['spc']
379
318 380 for n, ax in enumerate(self.axes):
381 noise = self.data['noise'][n][-1]
382 if self.CODE == 'spc_mean':
383 mean = self.data['mean'][n][-1]
319 384 if ax.firsttime:
320 self.xmax = self.xmax if self.xmax else np.nanmax(x)
385 self.xmax = self.xmax if self.xmax else numpy.nanmax(x)
321 386 self.xmin = self.xmin if self.xmin else -self.xmax
322 self.ymin = self.ymin if self.ymin else np.nanmin(y)
323 self.ymax = self.ymax if self.ymax else np.nanmax(y)
324 self.zmin = self.zmin if self.zmin else np.nanmin(z)
325 self.zmax = self.zmax if self.zmax else np.nanmax(z)
326 ax.plot = ax.pcolormesh(x, y, z[n].T,
327 vmin=self.zmin,
328 vmax=self.zmax,
329 cmap=plt.get_cmap(self.colormap)
330 )
331 divider = make_axes_locatable(ax)
332 cax = divider.new_horizontal(size='3%', pad=0.05)
333 self.figure.add_axes(cax)
334 plt.colorbar(ax.plot, cax)
335
336 ax.set_xlim(self.xmin, self.xmax)
337 ax.set_ylim(self.ymin, self.ymax)
338
339 ax.set_ylabel(self.ylabel)
340 ax.set_xlabel(xlabel)
341
342 ax.firsttime = False
387 self.zmin = self.zmin if self.zmin else numpy.nanmin(z)
388 self.zmax = self.zmax if self.zmax else numpy.nanmax(z)
389 ax.plt = ax.pcolormesh(x, y, z[n].T,
390 vmin=self.zmin,
391 vmax=self.zmax,
392 cmap=plt.get_cmap(self.colormap)
393 )
343 394
344 395 if self.showprofile:
345 ax.plot_profile= ax.ax_profile.plot(self.data['rti'][self.max_time][n], y)[0]
346 ax.ax_profile.set_xlim(self.zmin, self.zmax)
347 ax.ax_profile.set_ylim(self.ymin, self.ymax)
348 ax.ax_profile.set_xlabel('dB')
349 ax.ax_profile.grid(b=True, axis='x')
350 ax.plot_noise = ax.ax_profile.plot(numpy.repeat(self.data['noise'][self.max_time][n], len(y)), y,
351 color="k", linestyle="dashed", lw=2)[0]
352 [tick.set_visible(False) for tick in ax.ax_profile.get_yticklabels()]
396 ax.plt_profile= self.pf_axes[n].plot(self.data['rti'][n][-1], y)[0]
397 ax.plt_noise = self.pf_axes[n].plot(numpy.repeat(noise, len(y)), y,
398 color="k", linestyle="dashed", lw=1)[0]
399 if self.CODE == 'spc_mean':
400 ax.plt_mean = ax.plot(mean, y, color='k')[0]
353 401 else:
354 ax.plot.set_array(z[n].T.ravel())
402 ax.plt.set_array(z[n].T.ravel())
355 403 if self.showprofile:
356 ax.plot_profile.set_data(self.data['rti'][self.max_time][n], y)
357 ax.plot_noise.set_data(numpy.repeat(self.data['noise'][self.max_time][n], len(y)), y)
404 ax.plt_profile.set_data(self.data['rti'][n][-1], y)
405 ax.plt_noise.set_data(numpy.repeat(noise, len(y)), y)
406 if self.CODE == 'spc_mean':
407 ax.plt_mean.set_data(mean, y)
358 408
359 ax.set_title('{} - Noise: {:.2f} dB'.format(self.titles[n], self.data['noise'][self.max_time][n]),
360 size=8)
409 self.titles.append('CH {}: {:3.2f}dB'.format(n, noise))
361 410 self.saveTime = self.max_time
362 411
363 412
@@ -367,545 +416,245 class PlotCrossSpectraData(PlotData):
367 416 zmin_coh = None
368 417 zmax_coh = None
369 418 zmin_phase = None
370 zmax_phase = None
371 CONFLATE = False
419 zmax_phase = None
372 420
373 421 def setup(self):
374 422
375 ncolspan = 1
376 colspan = 1
377 self.ncols = 2
378 self.nrows = self.dataOut.nPairs
379 self.width = 3.6*self.ncols
380 self.height = 3.2*self.nrows
381
423 self.ncols = 4
424 self.nrows = len(self.data.pairs)
425 self.nplots = self.nrows*4
426 self.width = 3.4*self.ncols
427 self.height = 3*self.nrows
382 428 self.ylabel = 'Range [Km]'
383 self.titles = ['Channel {}'.format(x) for x in self.dataOut.channelList]
384
385 if self.figure is None:
386 self.figure = plt.figure(figsize=(self.width, self.height),
387 edgecolor='k',
388 facecolor='w')
389 else:
390 self.figure.clf()
391
392 for y in range(self.nrows):
393 for x in range(self.ncols):
394 ax = plt.subplot2grid((self.nrows, self.ncols), (y, x), 1, 1)
395 ax.firsttime = True
396 self.axes.append(ax)
429 self.showprofile = False
397 430
398 431 def plot(self):
399 432
400 433 if self.xaxis == "frequency":
401 x = self.dataOut.getFreqRange(1)/1000.
402 xlabel = "Frequency (kHz)"
434 x = self.data.xrange[0]
435 self.xlabel = "Frequency (kHz)"
403 436 elif self.xaxis == "time":
404 x = self.dataOut.getAcfRange(1)
405 xlabel = "Time (ms)"
437 x = self.data.xrange[1]
438 self.xlabel = "Time (ms)"
406 439 else:
407 x = self.dataOut.getVelRange(1)
408 xlabel = "Velocity (m/s)"
440 x = self.data.xrange[2]
441 self.xlabel = "Velocity (m/s)"
442
443 self.titles = []
409 444
410 y = self.dataOut.getHeiRange()
411 z_coh = self.data['cspc_coh']
412 z_phase = self.data['cspc_phase']
445 y = self.data.heights
446 self.y = y
447 spc = self.data['spc']
448 cspc = self.data['cspc']
413 449
414 450 for n in range(self.nrows):
415 ax = self.axes[2*n]
416 ax1 = self.axes[2*n+1]
451 noise = self.data['noise'][n][-1]
452 pair = self.data.pairs[n]
453 ax = self.axes[4*n]
454 ax3 = self.axes[4*n+3]
417 455 if ax.firsttime:
418 self.xmax = self.xmax if self.xmax else np.nanmax(x)
456 self.xmax = self.xmax if self.xmax else numpy.nanmax(x)
419 457 self.xmin = self.xmin if self.xmin else -self.xmax
420 self.ymin = self.ymin if self.ymin else np.nanmin(y)
421 self.ymax = self.ymax if self.ymax else np.nanmax(y)
422 self.zmin_coh = self.zmin_coh if self.zmin_coh else 0.0
423 self.zmax_coh = self.zmax_coh if self.zmax_coh else 1.0
424 self.zmin_phase = self.zmin_phase if self.zmin_phase else -180
425 self.zmax_phase = self.zmax_phase if self.zmax_phase else 180
426
427 ax.plot = ax.pcolormesh(x, y, z_coh[n].T,
428 vmin=self.zmin_coh,
429 vmax=self.zmax_coh,
430 cmap=plt.get_cmap(self.colormap_coh)
431 )
432 divider = make_axes_locatable(ax)
433 cax = divider.new_horizontal(size='3%', pad=0.05)
434 self.figure.add_axes(cax)
435 plt.colorbar(ax.plot, cax)
436
437 ax.set_xlim(self.xmin, self.xmax)
438 ax.set_ylim(self.ymin, self.ymax)
439
440 ax.set_ylabel(self.ylabel)
441 ax.set_xlabel(xlabel)
442 ax.firsttime = False
443
444 ax1.plot = ax1.pcolormesh(x, y, z_phase[n].T,
445 vmin=self.zmin_phase,
446 vmax=self.zmax_phase,
447 cmap=plt.get_cmap(self.colormap_phase)
448 )
449 divider = make_axes_locatable(ax1)
450 cax = divider.new_horizontal(size='3%', pad=0.05)
451 self.figure.add_axes(cax)
452 plt.colorbar(ax1.plot, cax)
453
454 ax1.set_xlim(self.xmin, self.xmax)
455 ax1.set_ylim(self.ymin, self.ymax)
456
457 ax1.set_ylabel(self.ylabel)
458 ax1.set_xlabel(xlabel)
459 ax1.firsttime = False
458 self.zmin = self.zmin if self.zmin else numpy.nanmin(spc)
459 self.zmax = self.zmax if self.zmax else numpy.nanmax(spc)
460 ax.plt = ax.pcolormesh(x, y, spc[pair[0]].T,
461 vmin=self.zmin,
462 vmax=self.zmax,
463 cmap=plt.get_cmap(self.colormap)
464 )
460 465 else:
461 ax.plot.set_array(z_coh[n].T.ravel())
462 ax1.plot.set_array(z_phase[n].T.ravel())
463
464 ax.set_title('Coherence Ch{} * Ch{}'.format(self.dataOut.pairsList[n][0], self.dataOut.pairsList[n][1]), size=8)
465 ax1.set_title('Phase Ch{} * Ch{}'.format(self.dataOut.pairsList[n][0], self.dataOut.pairsList[n][1]), size=8)
466 self.saveTime = self.max_time
467
466 ax.plt.set_array(spc[pair[0]].T.ravel())
467 self.titles.append('CH {}: {:3.2f}dB'.format(n, noise))
468 468
469 class PlotSpectraMeanData(PlotSpectraData):
470
471 CODE = 'spc_mean'
472 colormap = 'jet'
473
474 def plot(self):
475
476 if self.xaxis == "frequency":
477 x = self.dataOut.getFreqRange(1)/1000.
478 xlabel = "Frequency (kHz)"
479 elif self.xaxis == "time":
480 x = self.dataOut.getAcfRange(1)
481 xlabel = "Time (ms)"
482 else:
483 x = self.dataOut.getVelRange(1)
484 xlabel = "Velocity (m/s)"
485
486 y = self.dataOut.getHeiRange()
487 z = self.data['spc']
488 mean = self.data['mean'][self.max_time]
489
490 for n, ax in enumerate(self.axes):
491
492 if ax.firsttime:
493 self.xmax = self.xmax if self.xmax else np.nanmax(x)
494 self.xmin = self.xmin if self.xmin else -self.xmax
495 self.ymin = self.ymin if self.ymin else np.nanmin(y)
496 self.ymax = self.ymax if self.ymax else np.nanmax(y)
497 self.zmin = self.zmin if self.zmin else np.nanmin(z)
498 self.zmax = self.zmax if self.zmax else np.nanmax(z)
499 ax.plt = ax.pcolormesh(x, y, z[n].T,
469 ax = self.axes[4*n+1]
470 if ax.firsttime:
471 ax.plt = ax.pcolormesh(x, y, spc[pair[1]].T,
500 472 vmin=self.zmin,
501 473 vmax=self.zmax,
502 474 cmap=plt.get_cmap(self.colormap)
503 475 )
504 ax.plt_dop = ax.plot(mean[n], y,
505 color='k')[0]
506
507 divider = make_axes_locatable(ax)
508 cax = divider.new_horizontal(size='3%', pad=0.05)
509 self.figure.add_axes(cax)
510 plt.colorbar(ax.plt, cax)
511
512 ax.set_xlim(self.xmin, self.xmax)
513 ax.set_ylim(self.ymin, self.ymax)
514
515 ax.set_ylabel(self.ylabel)
516 ax.set_xlabel(xlabel)
517
518 ax.firsttime = False
519
520 if self.showprofile:
521 ax.plt_profile= ax.ax_profile.plot(self.data['rti'][self.max_time][n], y)[0]
522 ax.ax_profile.set_xlim(self.zmin, self.zmax)
523 ax.ax_profile.set_ylim(self.ymin, self.ymax)
524 ax.ax_profile.set_xlabel('dB')
525 ax.ax_profile.grid(b=True, axis='x')
526 ax.plt_noise = ax.ax_profile.plot(numpy.repeat(self.data['noise'][self.max_time][n], len(y)), y,
527 color="k", linestyle="dashed", lw=2)[0]
528 [tick.set_visible(False) for tick in ax.ax_profile.get_yticklabels()]
529 476 else:
530 ax.plt.set_array(z[n].T.ravel())
531 ax.plt_dop.set_data(mean[n], y)
532 if self.showprofile:
533 ax.plt_profile.set_data(self.data['rti'][self.max_time][n], y)
534 ax.plt_noise.set_data(numpy.repeat(self.data['noise'][self.max_time][n], len(y)), y)
477 ax.plt.set_array(spc[pair[1]].T.ravel())
478 self.titles.append('CH {}: {:3.2f}dB'.format(n, noise))
479
480 out = cspc[n]/numpy.sqrt(spc[pair[0]]*spc[pair[1]])
481 coh = numpy.abs(out)
482 phase = numpy.arctan2(out.imag, out.real)*180/numpy.pi
483
484 ax = self.axes[4*n+2]
485 if ax.firsttime:
486 ax.plt = ax.pcolormesh(x, y, coh.T,
487 vmin=0,
488 vmax=1,
489 cmap=plt.get_cmap(self.colormap_coh)
490 )
491 else:
492 ax.plt.set_array(coh.T.ravel())
493 self.titles.append('Coherence Ch{} * Ch{}'.format(pair[0], pair[1]))
535 494
536 ax.set_title('{} - Noise: {:.2f} dB'.format(self.titles[n], self.data['noise'][self.max_time][n]),
537 size=8)
495 ax = self.axes[4*n+3]
496 if ax.firsttime:
497 ax.plt = ax.pcolormesh(x, y, phase.T,
498 vmin=-180,
499 vmax=180,
500 cmap=plt.get_cmap(self.colormap_phase)
501 )
502 else:
503 ax.plt.set_array(phase.T.ravel())
504 self.titles.append('Phase CH{} * CH{}'.format(pair[0], pair[1]))
505
538 506 self.saveTime = self.max_time
539 507
540 508
509 class PlotSpectraMeanData(PlotSpectraData):
510 '''
511 Plot for Spectra and Mean
512 '''
513 CODE = 'spc_mean'
514 colormap = 'jro'
515
516
541 517 class PlotRTIData(PlotData):
518 '''
519 Plot for RTI data
520 '''
542 521
543 522 CODE = 'rti'
544 523 colormap = 'jro'
545 524
546 525 def setup(self):
547 self.ncols = 1
548 self.nrows = self.dataOut.nChannels
549 self.width = 10
550 #TODO : arreglar la altura de la figura, esta hardcodeada.
551 #Se arreglo, testear!
552 if self.ind_plt_ch:
553 self.height = 3.2#*self.nrows if self.nrows<6 else 12
554 else:
555 self.height = 2.2*self.nrows if self.nrows<6 else 12
556
557 '''
558 if self.nrows==1:
559 self.height += 1
560 '''
526 self.xaxis = 'time'
527 self.ncols = 1
528 self.nrows = len(self.data.channels)
529 self.nplots = len(self.data.channels)
561 530 self.ylabel = 'Range [Km]'
562 self.titles = ['Channel {}'.format(x) for x in self.dataOut.channelList]
563
564 '''
565 Logica:
566 1) Si la variable ind_plt_ch es True, va a crear mas de 1 figura
567 2) guardamos "Figures" en una lista y "axes" en otra, quizas se deberia guardar el
568 axis dentro de "Figures" como un diccionario.
569 '''
570 if self.ind_plt_ch is False: #standard mode
571
572 if self.figure is None: #solo para la priemra vez
573 self.figure = plt.figure(figsize=(self.width, self.height),
574 edgecolor='k',
575 facecolor='w')
576 else:
577 self.figure.clf()
578 self.axes = []
579
580
581 for n in range(self.nrows):
582 ax = self.figure.add_subplot(self.nrows, self.ncols, n+1)
583 #ax = self.figure(n+1)
584 ax.firsttime = True
585 self.axes.append(ax)
586
587 else : #append one figure foreach channel in channelList
588 if self.figurelist == None:
589 self.figurelist = []
590 for n in range(self.nrows):
591 self.figure = plt.figure(figsize=(self.width, self.height),
592 edgecolor='k',
593 facecolor='w')
594 #add always one subplot
595 self.figurelist.append(self.figure)
596
597 else : # cada dia nuevo limpia el axes, pero mantiene el figure
598 for eachfigure in self.figurelist:
599 eachfigure.clf() # eliminaria todas las figuras de la lista?
600 self.axes = []
601
602 for eachfigure in self.figurelist:
603 ax = eachfigure.add_subplot(1,1,1) #solo 1 axis por figura
604 #ax = self.figure(n+1)
605 ax.firsttime = True
606 #Cada figura tiene un distinto puntero
607 self.axes.append(ax)
608 #plt.close(eachfigure)
609
531 self.cb_label = 'dB'
532 self.titles = ['{} Channel {}'.format(self.CODE.upper(), x) for x in range(self.nrows)]
610 533
611 534 def plot(self):
535 self.x = self.data.times
536 self.y = self.data.heights
537 self.z = self.data[self.CODE]
538 self.z = numpy.ma.masked_invalid(self.z)
612 539
613 if self.ind_plt_ch is False: #standard mode
614 self.x = np.array(self.times)
615 self.y = self.dataOut.getHeiRange()
616 self.z = []
617
618 for ch in range(self.nrows):
619 self.z.append([self.data[self.CODE][t][ch] for t in self.times])
620
621 self.z = np.array(self.z)
622 for n, ax in enumerate(self.axes):
623 x, y, z = self.fill_gaps(*self.decimate())
624 if self.xmin is None:
625 xmin = self.min_time
626 else:
627 xmin = fromtimestamp(int(self.xmin), self.min_time)
628 if self.xmax is None:
629 xmax = xmin + self.xrange*60*60
630 else:
631 xmax = xmin + (self.xmax - self.xmin) * 60 * 60
632 self.zmin = self.zmin if self.zmin else np.min(self.z)
633 self.zmax = self.zmax if self.zmax else np.max(self.z)
634 if ax.firsttime:
635 self.ymin = self.ymin if self.ymin else np.nanmin(self.y)
636 self.ymax = self.ymax if self.ymax else np.nanmax(self.y)
637 plot = ax.pcolormesh(x, y, z[n].T,
638 vmin=self.zmin,
639 vmax=self.zmax,
640 cmap=plt.get_cmap(self.colormap)
641 )
642 divider = make_axes_locatable(ax)
643 cax = divider.new_horizontal(size='2%', pad=0.05)
644 self.figure.add_axes(cax)
645 plt.colorbar(plot, cax)
646 ax.set_ylim(self.ymin, self.ymax)
647 ax.xaxis.set_major_formatter(FuncFormatter(func))
648 ax.xaxis.set_major_locator(LinearLocator(6))
649 ax.set_ylabel(self.ylabel)
650 # if self.xmin is None:
651 # xmin = self.min_time
652 # else:
653 # xmin = (datetime.datetime.combine(self.dataOut.datatime.date(),
654 # datetime.time(self.xmin, 0, 0))-d1970).total_seconds()
655
656 ax.set_xlim(xmin, xmax)
657 ax.firsttime = False
658 else:
659 ax.collections.remove(ax.collections[0])
660 ax.set_xlim(xmin, xmax)
661 plot = ax.pcolormesh(x, y, z[n].T,
662 vmin=self.zmin,
663 vmax=self.zmax,
664 cmap=plt.get_cmap(self.colormap)
665 )
666 ax.set_title('{} {}'.format(self.titles[n],
667 datetime.datetime.fromtimestamp(self.max_time).strftime('%y/%m/%d %H:%M:%S')),
668 size=8)
669
670 self.saveTime = self.min_time
671 else :
672 self.x = np.array(self.times)
673 self.y = self.dataOut.getHeiRange()
674 self.z = []
675
676 for ch in range(self.nrows):
677 self.z.append([self.data[self.CODE][t][ch] for t in self.times])
678
679 self.z = np.array(self.z)
680 for n, eachfigure in enumerate(self.figurelist): #estaba ax in axes
681
682 x, y, z = self.fill_gaps(*self.decimate())
683 xmin = self.min_time
684 xmax = xmin+self.xrange*60*60
685 self.zmin = self.zmin if self.zmin else np.min(self.z)
686 self.zmax = self.zmax if self.zmax else np.max(self.z)
687 if self.axes[n].firsttime:
688 self.ymin = self.ymin if self.ymin else np.nanmin(self.y)
689 self.ymax = self.ymax if self.ymax else np.nanmax(self.y)
690 plot = self.axes[n].pcolormesh(x, y, z[n].T,
691 vmin=self.zmin,
692 vmax=self.zmax,
693 cmap=plt.get_cmap(self.colormap)
694 )
695 divider = make_axes_locatable(self.axes[n])
696 cax = divider.new_horizontal(size='2%', pad=0.05)
697 eachfigure.add_axes(cax)
698 #self.figure2.add_axes(cax)
699 plt.colorbar(plot, cax)
700 self.axes[n].set_ylim(self.ymin, self.ymax)
701
702 self.axes[n].xaxis.set_major_formatter(FuncFormatter(func))
703 self.axes[n].xaxis.set_major_locator(LinearLocator(6))
704
705 self.axes[n].set_ylabel(self.ylabel)
706
707 if self.xmin is None:
708 xmin = self.min_time
709 else:
710 xmin = (datetime.datetime.combine(self.dataOut.datatime.date(),
711 datetime.time(self.xmin, 0, 0))-d1970).total_seconds()
712
713 self.axes[n].set_xlim(xmin, xmax)
714 self.axes[n].firsttime = False
715 else:
716 self.axes[n].collections.remove(self.axes[n].collections[0])
717 self.axes[n].set_xlim(xmin, xmax)
718 plot = self.axes[n].pcolormesh(x, y, z[n].T,
719 vmin=self.zmin,
720 vmax=self.zmax,
721 cmap=plt.get_cmap(self.colormap)
722 )
723 self.axes[n].set_title('{} {}'.format(self.titles[n],
724 datetime.datetime.fromtimestamp(self.max_time).strftime('%y/%m/%d %H:%M:%S')),
725 size=8)
540 for n, ax in enumerate(self.axes):
541 x, y, z = self.fill_gaps(*self.decimate())
542 self.zmin = self.zmin if self.zmin else numpy.min(self.z)
543 self.zmax = self.zmax if self.zmax else numpy.max(self.z)
544 if ax.firsttime:
545 ax.plt = ax.pcolormesh(x, y, z[n].T,
546 vmin=self.zmin,
547 vmax=self.zmax,
548 cmap=plt.get_cmap(self.colormap)
549 )
550 if self.showprofile:
551 ax.plot_profile= self.pf_axes[n].plot(self.data['rti'][n][-1], self.y)[0]
552 ax.plot_noise = self.pf_axes[n].plot(numpy.repeat(self.data['noise'][n][-1], len(self.y)), self.y,
553 color="k", linestyle="dashed", lw=1)[0]
554 else:
555 ax.collections.remove(ax.collections[0])
556 ax.plt = ax.pcolormesh(x, y, z[n].T,
557 vmin=self.zmin,
558 vmax=self.zmax,
559 cmap=plt.get_cmap(self.colormap)
560 )
561 if self.showprofile:
562 ax.plot_profile.set_data(self.data['rti'][n][-1], self.y)
563 ax.plot_noise.set_data(numpy.repeat(self.data['noise'][n][-1], len(self.y)), self.y)
726 564
727 self.saveTime = self.min_time
565 self.saveTime = self.min_time
728 566
729 567
730 568 class PlotCOHData(PlotRTIData):
569 '''
570 Plot for Coherence data
571 '''
731 572
732 573 CODE = 'coh'
733 574
734 575 def setup(self):
735
576 self.xaxis = 'time'
736 577 self.ncols = 1
737 self.nrows = self.dataOut.nPairs
738 self.width = 10
739 self.height = 2.2*self.nrows if self.nrows<6 else 12
740 self.ind_plt_ch = False #just for coherence and phase
741 if self.nrows==1:
742 self.height += 1
743 self.ylabel = 'Range [Km]'
744 self.titles = ['{} Ch{} * Ch{}'.format(self.CODE.upper(), x[0], x[1]) for x in self.dataOut.pairsList]
745
746 if self.figure is None:
747 self.figure = plt.figure(figsize=(self.width, self.height),
748 edgecolor='k',
749 facecolor='w')
578 self.nrows = len(self.data.pairs)
579 self.nplots = len(self.data.pairs)
580 self.ylabel = 'Range [Km]'
581 if self.CODE == 'coh':
582 self.cb_label = ''
583 self.titles = ['Coherence Map Ch{} * Ch{}'.format(x[0], x[1]) for x in self.data.pairs]
750 584 else:
751 self.figure.clf()
752 self.axes = []
585 self.cb_label = 'Degrees'
586 self.titles = ['Phase Map Ch{} * Ch{}'.format(x[0], x[1]) for x in self.data.pairs]
753 587
754 for n in range(self.nrows):
755 ax = self.figure.add_subplot(self.nrows, self.ncols, n+1)
756 ax.firsttime = True
757 self.axes.append(ax)
588
589 class PlotPHASEData(PlotCOHData):
590 '''
591 Plot for Phase map data
592 '''
593
594 CODE = 'phase'
595 colormap = 'seismic'
758 596
759 597
760 598 class PlotNoiseData(PlotData):
599 '''
600 Plot for noise
601 '''
602
761 603 CODE = 'noise'
762 604
763 605 def setup(self):
764
606 self.xaxis = 'time'
765 607 self.ncols = 1
766 608 self.nrows = 1
767 self.width = 10
768 self.height = 3.2
609 self.nplots = 1
769 610 self.ylabel = 'Intensity [dB]'
770 611 self.titles = ['Noise']
771
772 if self.figure is None:
773 self.figure = plt.figure(figsize=(self.width, self.height),
774 edgecolor='k',
775 facecolor='w')
776 else:
777 self.figure.clf()
778 self.axes = []
779
780 self.ax = self.figure.add_subplot(self.nrows, self.ncols, 1)
781 self.ax.firsttime = True
612 self.colorbar = False
782 613
783 614 def plot(self):
784 615
785 x = self.times
616 x = self.data.times
786 617 xmin = self.min_time
787 618 xmax = xmin+self.xrange*60*60
788 if self.ax.firsttime:
789 for ch in self.dataOut.channelList:
790 y = [self.data[self.CODE][t][ch] for t in self.times]
791 self.ax.plot(x, y, lw=1, label='Ch{}'.format(ch))
792 self.ax.firsttime = False
793 self.ax.xaxis.set_major_formatter(FuncFormatter(func))
794 self.ax.xaxis.set_major_locator(LinearLocator(6))
795 self.ax.set_ylabel(self.ylabel)
619 Y = self.data[self.CODE]
620
621 if self.axes[0].firsttime:
622 for ch in self.data.channels:
623 y = Y[ch]
624 self.axes[0].plot(x, y, lw=1, label='Ch{}'.format(ch))
796 625 plt.legend()
797 626 else:
798 for ch in self.dataOut.channelList:
799 y = [self.data[self.CODE][t][ch] for t in self.times]
800 self.ax.lines[ch].set_data(x, y)
801
802 self.ax.set_xlim(xmin, xmax)
803 self.ax.set_ylim(min(y)-5, max(y)+5)
627 for ch in self.data.channels:
628 y = Y[ch]
629 self.axes[0].lines[ch].set_data(x, y)
630
631 self.ymin = numpy.nanmin(Y) - 5
632 self.ymax = numpy.nanmax(Y) + 5
804 633 self.saveTime = self.min_time
805 634
806 635
807 class PlotWindProfilerData(PlotRTIData):
808
809 CODE = 'wind'
810 colormap = 'seismic'
811
812 def setup(self):
813 self.ncols = 1
814 self.nrows = self.dataOut.data_output.shape[0]
815 self.width = 10
816 self.height = 2.2*self.nrows
817 self.ylabel = 'Height [Km]'
818 self.titles = ['Zonal Wind' ,'Meridional Wind', 'Vertical Wind']
819 self.clabels = ['Velocity (m/s)','Velocity (m/s)','Velocity (cm/s)']
820 self.windFactor = [1, 1, 100]
821
822 if self.figure is None:
823 self.figure = plt.figure(figsize=(self.width, self.height),
824 edgecolor='k',
825 facecolor='w')
826 else:
827 self.figure.clf()
828 self.axes = []
829
830 for n in range(self.nrows):
831 ax = self.figure.add_subplot(self.nrows, self.ncols, n+1)
832 ax.firsttime = True
833 self.axes.append(ax)
834
835 def plot(self):
836
837 self.x = np.array(self.times)
838 self.y = self.dataOut.heightList
839 self.z = []
840
841 for ch in range(self.nrows):
842 self.z.append([self.data['output'][t][ch] for t in self.times])
843
844 self.z = np.array(self.z)
845 self.z = numpy.ma.masked_invalid(self.z)
846
847 cmap=plt.get_cmap(self.colormap)
848 cmap.set_bad('black', 1.)
849
850 for n, ax in enumerate(self.axes):
851 x, y, z = self.fill_gaps(*self.decimate())
852 xmin = self.min_time
853 xmax = xmin+self.xrange*60*60
854 if ax.firsttime:
855 self.ymin = self.ymin if self.ymin else np.nanmin(self.y)
856 self.ymax = self.ymax if self.ymax else np.nanmax(self.y)
857 self.zmax = self.zmax if self.zmax else numpy.nanmax(abs(self.z[:-1, :]))
858 self.zmin = self.zmin if self.zmin else -self.zmax
859
860 plot = ax.pcolormesh(x, y, z[n].T*self.windFactor[n],
861 vmin=self.zmin,
862 vmax=self.zmax,
863 cmap=cmap
864 )
865 divider = make_axes_locatable(ax)
866 cax = divider.new_horizontal(size='2%', pad=0.05)
867 self.figure.add_axes(cax)
868 cb = plt.colorbar(plot, cax)
869 cb.set_label(self.clabels[n])
870 ax.set_ylim(self.ymin, self.ymax)
871
872 ax.xaxis.set_major_formatter(FuncFormatter(func))
873 ax.xaxis.set_major_locator(LinearLocator(6))
874
875 ax.set_ylabel(self.ylabel)
876
877 ax.set_xlim(xmin, xmax)
878 ax.firsttime = False
879 else:
880 ax.collections.remove(ax.collections[0])
881 ax.set_xlim(xmin, xmax)
882 plot = ax.pcolormesh(x, y, z[n].T*self.windFactor[n],
883 vmin=self.zmin,
884 vmax=self.zmax,
885 cmap=plt.get_cmap(self.colormap)
886 )
887 ax.set_title('{} {}'.format(self.titles[n],
888 datetime.datetime.fromtimestamp(self.max_time).strftime('%y/%m/%d %H:%M:%S')),
889 size=8)
890
891 self.saveTime = self.min_time
892
893
894 636 class PlotSNRData(PlotRTIData):
637 '''
638 Plot for SNR Data
639 '''
640
895 641 CODE = 'snr'
896 642 colormap = 'jet'
897 643
644
898 645 class PlotDOPData(PlotRTIData):
646 '''
647 Plot for DOPPLER Data
648 '''
649
899 650 CODE = 'dop'
900 651 colormap = 'jet'
901 652
902 653
903 class PlotPHASEData(PlotCOHData):
904 CODE = 'phase'
905 colormap = 'seismic'
906
907
908 654 class PlotSkyMapData(PlotData):
655 '''
656 Plot for meteors detection data
657 '''
909 658
910 659 CODE = 'met'
911 660
@@ -932,7 +681,7 class PlotSkyMapData(PlotData):
932 681
933 682 def plot(self):
934 683
935 arrayParameters = np.concatenate([self.data['param'][t] for t in self.times])
684 arrayParameters = numpy.concatenate([self.data['param'][t] for t in self.data.times])
936 685 error = arrayParameters[:,-1]
937 686 indValid = numpy.where(error == 0)[0]
938 687 finalMeteor = arrayParameters[indValid,:]
@@ -962,3 +711,72 class PlotSkyMapData(PlotData):
962 711 self.ax.set_title(title, size=8)
963 712
964 713 self.saveTime = self.max_time
714
715 class PlotParamData(PlotRTIData):
716 '''
717 Plot for data_param object
718 '''
719
720 CODE = 'param'
721 colormap = 'seismic'
722
723 def setup(self):
724 self.xaxis = 'time'
725 self.ncols = 1
726 self.nrows = self.data.shape(self.CODE)[0]
727 self.nplots = self.nrows
728 if self.showSNR:
729 self.nrows += 1
730
731 self.ylabel = 'Height [Km]'
732 self.titles = self.data.parameters \
733 if self.data.parameters else ['Param {}'.format(x) for x in xrange(self.nrows)]
734 if self.showSNR:
735 self.titles.append('SNR')
736
737 def plot(self):
738 self.data.normalize_heights()
739 self.x = self.data.times
740 self.y = self.data.heights
741 if self.showSNR:
742 self.z = numpy.concatenate(
743 (self.data[self.CODE], self.data['snr'])
744 )
745 else:
746 self.z = self.data[self.CODE]
747
748 self.z = numpy.ma.masked_invalid(self.z)
749
750 for n, ax in enumerate(self.axes):
751
752 x, y, z = self.fill_gaps(*self.decimate())
753
754 if ax.firsttime:
755 if self.zlimits is not None:
756 self.zmin, self.zmax = self.zlimits[n]
757 self.zmax = self.zmax if self.zmax is not None else numpy.nanmax(abs(self.z[:-1, :]))
758 self.zmin = self.zmin if self.zmin is not None else -self.zmax
759 ax.plt = ax.pcolormesh(x, y, z[n, :, :].T*self.factors[n],
760 vmin=self.zmin,
761 vmax=self.zmax,
762 cmap=self.cmaps[n]
763 )
764 else:
765 if self.zlimits is not None:
766 self.zmin, self.zmax = self.zlimits[n]
767 ax.collections.remove(ax.collections[0])
768 ax.plt = ax.pcolormesh(x, y, z[n, :, :].T*self.factors[n],
769 vmin=self.zmin,
770 vmax=self.zmax,
771 cmap=self.cmaps[n]
772 )
773
774 self.saveTime = self.min_time
775
776 class PlotOuputData(PlotParamData):
777 '''
778 Plot data_output object
779 '''
780
781 CODE = 'output'
782 colormap = 'seismic' No newline at end of file
Show file before | Show file after | Hide comments
@@ -267,7 +267,7 class AMISRReader(ProcessingUnit):
267 267 self.dirnameList = new_dirnameList
268 268 return 1
269 269
270 def __searchFilesOnline(self,
270 def searchFilesOnLine(self,
271 271 path,
272 272 walk=True):
273 273
@@ -287,7 +287,7 class AMISRReader(ProcessingUnit):
287 287 return
288 288
289 289
290 def __searchFilesOffline(self,
290 def searchFilesOffLine(self,
291 291 path,
292 292 startDate,
293 293 endDate,
@@ -494,9 +494,9 class AMISRReader(ProcessingUnit):
494 494 self.online = online
495 495 if not(online):
496 496 #Busqueda de archivos offline
497 self.__searchFilesOffline(path, startDate, endDate, startTime, endTime, walk)
497 self.searchFilesOffLine(path, startDate, endDate, startTime, endTime, walk)
498 498 else:
499 self.__searchFilesOnline(path, walk)
499 self.searchFilesOnLine(path, walk)
500 500
501 501 if not(self.filenameList):
502 502 print "There is no files into the folder: %s"%(path)
Show file before | Show file after | Hide comments
This diff has been collapsed as it changes many lines, (503 lines changed) Show them Hide them
@@ -541,8 +541,7 class JRODataIO:
541 541 return inspect.getargspec(self.run).args
542 542
543 543 class JRODataReader(JRODataIO):
544
545 firstTime = True
544
546 545 online = 0
547 546
548 547 realtime = 0
@@ -578,8 +577,7 class JRODataReader(JRODataIO):
578 577 selBlocksize = None
579 578
580 579 selBlocktime = None
581
582 onlineWithDate = False
580
583 581 def __init__(self):
584 582
585 583 """
@@ -603,19 +601,19 class JRODataReader(JRODataIO):
603 601
604 602 raise NotImplementedError
605 603
606 def __searchFilesOffLine(self,
607 path,
608 startDate=None,
609 endDate=None,
610 startTime=datetime.time(0,0,0),
611 endTime=datetime.time(23,59,59),
612 set=None,
613 expLabel='',
614 ext='.r',
615 queue=None,
616 cursor=None,
617 skip=None,
618 walk=True):
604 def searchFilesOffLine(self,
605 path,
606 startDate=None,
607 endDate=None,
608 startTime=datetime.time(0,0,0),
609 endTime=datetime.time(23,59,59),
610 set=None,
611 expLabel='',
612 ext='.r',
613 cursor=None,
614 skip=None,
615 walk=True):
616
619 617 self.filenameList = []
620 618 self.datetimeList = []
621 619
@@ -624,8 +622,7 class JRODataReader(JRODataIO):
624 622 dateList, pathList = self.findDatafiles(path, startDate, endDate, expLabel, ext, walk, include_path=True)
625 623
626 624 if dateList == []:
627 # print "[Reading] Date range selected invalid [%s - %s]: No *%s files in %s)" %(startDate, endDate, ext, path)
628 return None, None
625 return [], []
629 626
630 627 if len(dateList) > 1:
631 628 print "[Reading] Data found for date range [%s - %s]: total days = %d" %(startDate, endDate, len(dateList))
@@ -636,18 +633,15 class JRODataReader(JRODataIO):
636 633 datetimeList = []
637 634
638 635 for thisPath in pathList:
639 # thisPath = pathList[pathDict[file]]
640
636
641 637 fileList = glob.glob1(thisPath, "*%s" %ext)
642 638 fileList.sort()
643 639
644 640 skippedFileList = []
645 641
646 if cursor is not None and skip is not None:
647 # if cursor*skip > len(fileList):
642 if cursor is not None andk skip is not None:
643
648 644 if skip == 0:
649 if queue is not None:
650 queue.put(len(fileList))
651 645 skippedFileList = []
652 646 else:
653 647 skippedFileList = fileList[cursor*skip: cursor*skip + skip]
@@ -672,44 +666,43 class JRODataReader(JRODataIO):
672 666
673 667 if not(filenameList):
674 668 print "[Reading] Time range selected invalid [%s - %s]: No *%s files in %s)" %(startTime, endTime, ext, path)
675 return None, None
669 return [], []
676 670
677 671 print "[Reading] %d file(s) was(were) found in time range: %s - %s" %(len(filenameList), startTime, endTime)
678 672 print
679 673
680 for i in range(len(filenameList)):
681 print "[Reading] %s -> [%s]" %(filenameList[i], datetimeList[i].ctime())
674 # for i in range(len(filenameList)):
675 # print "[Reading] %s -> [%s]" %(filenameList[i], datetimeList[i].ctime())
682 676
683 677 self.filenameList = filenameList
684 self.datetimeList = datetimeList
678 self.datetimeList = datetimeList
679
685 680 return pathList, filenameList
686 681
687 def __searchFilesOnLine(self, path, expLabel="", ext=None, walk=True, set=None, startDate=None, startTime=None):
688
682 def __searchFilesOnLine(self, path, expLabel = "", ext = None, walk=True, set=None):
683
689 684 """
690 Busca el ultimo archivo de la ultima carpeta (determinada o no por startDateTime) y
691 devuelve el archivo encontrado ademas de otros datos.
692
693 Input:
694 path : carpeta donde estan contenidos los files que contiene data
695
696 expLabel : Nombre del subexperimento (subfolder)
697
698 ext : extension de los files
699
700 walk : Si es habilitado no realiza busquedas dentro de los subdirectorios (doypath)
701
702 Return:
703 directory : eL directorio donde esta el file encontrado
704 filename : el ultimo file de una determinada carpeta
705 year : el anho
706 doy : el numero de dia del anho
707 set : el set del archivo
708
709
685 Busca el ultimo archivo de la ultima carpeta (determinada o no por startDateTime) y
686 devuelve el archivo encontrado ademas de otros datos.
687
688 Input:
689 path : carpeta donde estan contenidos los files que contiene data
690
691 expLabel : Nombre del subexperimento (subfolder)
692
693 ext : extension de los files
694
695 walk : Si es habilitado no realiza busquedas dentro de los ubdirectorios (doypath)
696
697 Return:
698 directory : eL directorio donde esta el file encontrado
699 filename : el ultimo file de una determinada carpeta
700 year : el anho
701 doy : el numero de dia del anho
702 set : el set del archivo
703
704
710 705 """
711 pathList = None
712 filenameList = None
713 706 if not os.path.isdir(path):
714 707 return None, None, None, None, None, None
715 708
@@ -719,7 +712,7 class JRODataReader(JRODataIO):
719 712 fullpath = path
720 713 foldercounter = 0
721 714 else:
722 # Filtra solo los directorios
715 #Filtra solo los directorios
723 716 for thisPath in os.listdir(path):
724 717 if not os.path.isdir(os.path.join(path,thisPath)):
725 718 continue
@@ -755,7 +748,7 class JRODataReader(JRODataIO):
755 748
756 749 year = int( filename[1:5] )
757 750 doy = int( filename[5:8] )
758 set = int( filename[8:11] )
751 set = int( filename[8:11] )
759 752
760 753 return fullpath, foldercounter, filename, year, doy, set
761 754
@@ -767,7 +760,7 class JRODataReader(JRODataIO):
767 760 idFile += 1
768 761 if not(idFile < len(self.filenameList)):
769 762 self.flagNoMoreFiles = 1
770 # print "[Reading] No more Files"
763 # print "[Reading] No more Files"
771 764 return 0
772 765
773 766 filename = self.filenameList[idFile]
@@ -785,32 +778,31 class JRODataReader(JRODataIO):
785 778 self.fileSize = fileSize
786 779 self.fp = fp
787 780
788 #print "[Reading] Setting the file: %s"%self.filename
781 # print "[Reading] Setting the file: %s"%self.filename
789 782
790 783 return 1
791 784
792 785 def __setNextFileOnline(self):
793 786 """
794 Busca el siguiente file que tenga suficiente data para ser leida, dentro de un folder especifico, si
795 no encuentra un file valido espera un tiempo determinado y luego busca en los posibles n files
796 siguientes.
797
798 Affected:
799 self.flagIsNewFile
800 self.filename
801 self.fileSize
802 self.fp
803 self.set
804 self.flagNoMoreFiles
805
806 Return:
807 0 : si luego de una busqueda del siguiente file valido este no pudo ser encontrado
808 1 : si el file fue abierto con exito y esta listo a ser leido
809
810 Excepciones:
811 Si un determinado file no puede ser abierto
812 """
813
787 Busca el siguiente file que tenga suficiente data para ser leida, dentro de un folder especifico, si
788 no encuentra un file valido espera un tiempo determinado y luego busca en los posibles n files
789 siguientes.
790
791 Affected:
792 self.flagIsNewFile
793 self.filename
794 self.fileSize
795 self.fp
796 self.set
797 self.flagNoMoreFiles
798
799 Return:
800 0 : si luego de una busqueda del siguiente file valido este no pudo ser encontrado
801 1 : si el file fue abierto con exito y esta listo a ser leido
802
803 Excepciones:
804 Si un determinado file no puede ser abierto
805 """
814 806 nFiles = 0
815 807 fileOk_flag = False
816 808 firstTime_flag = True
@@ -869,48 +861,27 class JRODataReader(JRODataIO):
869 861 if self.fp != None: self.fp.close()
870 862 self.fp = open(fullfilename, 'rb')
871 863 self.flagNoMoreFiles = 0
872 # print '[Reading] Setting the file: %s' % fullfilename
864 # print '[Reading] Setting the file: %s' % fullfilename
873 865 else:
874 866 self.fileSize = 0
875 867 self.filename = None
876 868 self.flagIsNewFile = 0
877 869 self.fp = None
878 870 self.flagNoMoreFiles = 1
879 # print '[Reading] No more files to read'
871 # print '[Reading] No more files to read'
880 872
881 873 return fileOk_flag
882 874
883 875 def setNextFile(self):
884 876 if self.fp != None:
885 877 self.fp.close()
878
886 879 if self.online:
887 880 newFile = self.__setNextFileOnline()
888 881 else:
889 882 newFile = self.__setNextFileOffline()
883
890 884 if not(newFile):
891 if self.onlineWithDate is True:
892 self.onlineWithDate=False
893 self.online = True
894 self.firstTime = False
895 self.setup(
896 path=self.path,
897 startDate=self.startDate,
898 endDate=self.endDate,
899 startTime=self.startTime ,
900 endTime=self.endTime,
901 set=self.set,
902 expLabel=self.expLabel,
903 ext=self.ext,
904 online=self.online,
905 delay=self.delay,
906 walk=self.walk,
907 getblock=self.getblock,
908 nTxs=self.nTxs,
909 realtime=self.realtime,
910 blocksize=self.blocksize,
911 blocktime=self.blocktime
912 )
913 return 1
914 885 print '[Reading] No more files to read'
915 886 return 0
916 887
@@ -951,7 +922,7 class JRODataReader(JRODataIO):
951 922 return 1
952 923
953 924 if self.fileSize == self.fileSizeByHeader:
954 # self.flagEoF = True
925 # self.flagEoF = True
955 926 return 0
956 927
957 928 print "[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)
@@ -1005,13 +976,13 class JRODataReader(JRODataIO):
1005 976 else:
1006 977 self.fp.seek(self.fp.tell() - neededsize)
1007 978 break
1008
1009 # csize = self.fileSize - self.fp.tell()
1010 # neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
1011 # factor = int(csize/neededsize)
1012 # if factor > 0:
1013 # self.fp.seek(self.fp.tell() + factor*neededsize)
1014
979
980 # csize = self.fileSize - self.fp.tell()
981 # neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
982 # factor = int(csize/neededsize)
983 # if factor > 0:
984 # self.fp.seek(self.fp.tell() + factor*neededsize)
985
1015 986 self.flagIsNewFile = 0
1016 987 self.__isFirstTimeOnline = 0
1017 988
@@ -1019,10 +990,10 class JRODataReader(JRODataIO):
1019 990 #if self.server is None:
1020 991 if self.fp == None:
1021 992 return 0
1022
1023 # if self.online:
1024 # self.__jumpToLastBlock()
1025
993
994 # if self.online:
995 # self.__jumpToLastBlock()
996
1026 997 if self.flagIsNewFile:
1027 998 self.lastUTTime = self.basicHeaderObj.utc
1028 999 return 1
@@ -1065,12 +1036,14 class JRODataReader(JRODataIO):
1065 1036
1066 1037 #Skip block out of startTime and endTime
1067 1038 while True:
1068 if not(self.__setNewBlock()):
1069 print 'returning'
1039 if not(self.__setNewBlock()):
1070 1040 return 0
1041
1071 1042 if not(self.readBlock()):
1072 1043 return 0
1044
1073 1045 self.getBasicHeader()
1046
1074 1047 if not isTimeInRange(self.dataOut.datatime.time(), self.startTime, self.endTime):
1075 1048
1076 1049 print "[Reading] Block No. %d/%d -> %s [Skipping]" %(self.nReadBlocks,
@@ -1082,8 +1055,8 class JRODataReader(JRODataIO):
1082 1055
1083 1056 if self.verbose:
1084 1057 print "[Reading] Block No. %d/%d -> %s" %(self.nReadBlocks,
1085 self.processingHeaderObj.dataBlocksPerFile,
1086 self.dataOut.datatime.ctime())
1058 self.processingHeaderObj.dataBlocksPerFile,
1059 self.dataOut.datatime.ctime())
1087 1060 return 1
1088 1061
1089 1062 def __readFirstHeader(self):
@@ -1114,8 +1087,8 class JRODataReader(JRODataIO):
1114 1087 self.dtype = datatype_str
1115 1088 #self.ippSeconds = 2 * 1000 * self.radarControllerHeaderObj.ipp / self.c
1116 1089 self.fileSizeByHeader = self.processingHeaderObj.dataBlocksPerFile * self.processingHeaderObj.blockSize + self.firstHeaderSize + self.basicHeaderSize*(self.processingHeaderObj.dataBlocksPerFile - 1)
1117 # self.dataOut.channelList = numpy.arange(self.systemHeaderObj.numChannels)
1118 # self.dataOut.channelIndexList = numpy.arange(self.systemHeaderObj.numChannels)
1090 # self.dataOut.channelList = numpy.arange(self.systemHeaderObj.numChannels)
1091 # self.dataOut.channelIndexList = numpy.arange(self.systemHeaderObj.numChannels)
1119 1092 self.getBlockDimension()
1120 1093
1121 1094 def __verifyFile(self, filename, msgFlag=True):
@@ -1277,13 +1250,13 class JRODataReader(JRODataIO):
1277 1250
1278 1251 def setup(self,
1279 1252 path=None,
1280 startDate=None,
1281 endDate=None,
1282 startTime=datetime.time(0,0,0),
1283 endTime=datetime.time(23,59,59),
1284 set=None,
1285 expLabel = "",
1286 ext = None,
1253 startDate=None,
1254 endDate=None,
1255 startTime=datetime.time(0,0,0),
1256 endTime=datetime.time(23,59,59),
1257 set=None,
1258 expLabel = "",
1259 ext = None,
1287 1260 online = False,
1288 1261 delay = 60,
1289 1262 walk = True,
@@ -1292,139 +1265,109 class JRODataReader(JRODataIO):
1292 1265 realtime=False,
1293 1266 blocksize=None,
1294 1267 blocktime=None,
1268 skip=None,
1269 cursor=None,
1270 warnings=True,
1295 1271 verbose=True,
1272 server=None,
1296 1273 **kwargs):
1297
1298 if path == None:
1299 raise ValueError, "[Reading] The path is not valid"
1300
1274 if server is not None:
1275 if 'tcp://' in server:
1276 address = server
1277 else:
1278 address = 'ipc:///tmp/%s' % server
1279 self.server = address
1280 self.context = zmq.Context()
1281 self.receiver = self.context.socket(zmq.PULL)
1282 self.receiver.connect(self.server)
1283 time.sleep(0.5)
1284 print '[Starting] ReceiverData from {}'.format(self.server)
1285 else:
1286 self.server = None
1287 if path == None:
1288 raise ValueError, "[Reading] The path is not valid"
1301 1289
1302 if ext == None:
1303 ext = self.ext
1304
1305 self.verbose=verbose
1306 self.path = path
1307 self.startDate = startDate
1308 self.endDate = endDate
1309 self.startTime = startTime
1310 self.endTime = endTime
1311 self.set = set
1312 self.expLabel = expLabel
1313 self.ext = ext
1314 self.online = online
1315 self.delay = delay
1316 self.walk = walk
1317 self.getblock = getblock
1318 self.nTxs = nTxs
1319 self.realtime = realtime
1320 self.blocksize = blocksize
1321 self.blocktime = blocktime
1322
1323
1324 if self.firstTime is True:
1325 pathList, filenameList = self.__searchFilesOffLine(path, startDate=startDate, endDate=endDate,
1290 if ext == None:
1291 ext = self.ext
1292
1293 if online:
1294 print "[Reading] Searching files in online mode..."
1295
1296 for nTries in range( self.nTries ):
1297 fullpath, foldercounter, file, year, doy, set = self.__searchFilesOnLine(path=path, expLabel=expLabel, ext=ext, walk=walk, set=set)
1298
1299 if fullpath:
1300 break
1301
1302 print '[Reading] Waiting %0.2f sec for an valid file in %s: try %02d ...' % (self.delay, path, nTries+1)
1303 sleep( self.delay )
1304
1305 if not(fullpath):
1306 print "[Reading] There 'isn't any valid file in %s" % path
1307 return
1308
1309 self.year = year
1310 self.doy = doy
1311 self.set = set - 1
1312 self.path = path
1313 self.foldercounter = foldercounter
1314 last_set = None
1315 else:
1316 print "[Reading] Searching files in offline mode ..."
1317 pathList, filenameList = self.searchFilesOffLine(path, startDate=startDate, endDate=endDate,
1326 1318 startTime=startTime, endTime=endTime,
1327 1319 set=set, expLabel=expLabel, ext=ext,
1328 walk=walk)
1329 if filenameList is not None: filenameList = filenameList[:-1]
1320 walk=walk, cursor=cursor,
1321 skip=skip)
1330 1322
1331 if pathList is not None and filenameList is not None and online:
1332 self.onlineWithDate = True
1333 online = False
1323 if not(pathList):
1334 1324 self.fileIndex = -1
1335 self.pathList = pathList
1336 self.filenameList = filenameList
1337 file_name = os.path.basename(filenameList[-1])
1338 basename, ext = os.path.splitext(file_name)
1339 last_set = int(basename[-3:])
1340
1341 if online:
1342 print "[Reading] Searching files in online mode..."
1343
1344 for nTries in range(self.nTries):
1345 fullpath, foldercounter, file, year, doy, set = self.__searchFilesOnLine(path=path,
1346 expLabel=expLabel,
1347 ext=ext,
1348 walk=walk,
1349 startDate=startDate,
1350 startTime=startTime,
1351 set=set)
1352
1353 if fullpath:
1354 break
1355 print '[Reading] Waiting %0.2f sec for an valid file in %s: try %02d ...' % (self.delay, path, nTries+1)
1356 sleep( self.delay )
1357
1358 if not(fullpath):
1359 print "[Reading] There 'isn't any valid file in %s" % path
1360 return
1361
1362 self.year = year
1363 self.doy = doy
1364 self.set = set - 1
1365 self.path = path
1366 self.foldercounter = foldercounter
1367 last_set = None
1368 else:
1369 print "[Reading] Searching files in offline mode ..."
1370 pathList, filenameList = self.__searchFilesOffLine(path, startDate=startDate, endDate=endDate,
1371 startTime=startTime, endTime=endTime,
1372 set=set, expLabel=expLabel, ext=ext,
1373 walk=walk)
1374
1375 if not(pathList):
1376 # print "[Reading] No *%s files in %s (%s - %s)"%(ext, path,
1377 # datetime.datetime.combine(startDate,startTime).ctime(),
1378 # datetime.datetime.combine(endDate,endTime).ctime())
1379
1380 # sys.exit(-1)
1381
1382 self.fileIndex = -1
1383 self.pathList = []
1384 self.filenameList = []
1385 return
1386
1387 self.fileIndex = -1
1388 self.pathList = pathList
1389 self.filenameList = filenameList
1390 file_name = os.path.basename(filenameList[-1])
1391 basename, ext = os.path.splitext(file_name)
1392 last_set = int(basename[-3:])
1393
1394
1395 self.online = online
1396 self.realtime = realtime
1397 self.delay = delay
1398 ext = ext.lower()
1399 self.ext = ext
1400 self.getByBlock = getblock
1401 self.nTxs = nTxs
1402 self.startTime = startTime
1403 self.endTime = endTime
1404
1405
1406 #Added-----------------
1407 self.selBlocksize = blocksize
1408 self.selBlocktime = blocktime
1409
1410
1411 if not(self.setNextFile()):
1412 if (startDate!=None) and (endDate!=None):
1413 print "[Reading] No files in range: %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime())
1414 elif startDate != None:
1415 print "[Reading] No files in range: %s" %(datetime.datetime.combine(startDate,startTime).ctime())
1416 else:
1417 print "[Reading] No files"
1418
1325 self.pathList = []
1326 self.filenameList = []
1327 return
1328
1419 1329 self.fileIndex = -1
1420 self.pathList = []
1421 self.filenameList = []
1422 return
1330 self.pathList = pathList
1331 self.filenameList = filenameList
1332 file_name = os.path.basename(filenameList[-1])
1333 basename, ext = os.path.splitext(file_name)
1334 last_set = int(basename[-3:])
1335
1336 self.online = online
1337 self.realtime = realtime
1338 self.delay = delay
1339 ext = ext.lower()
1340 self.ext = ext
1341 self.getByBlock = getblock
1342 self.nTxs = nTxs
1343 self.startTime = startTime
1344 self.endTime = endTime
1345
1346 #Added-----------------
1347 self.selBlocksize = blocksize
1348 self.selBlocktime = blocktime
1349
1350 # Verbose-----------
1351 self.verbose = verbose
1352 self.warnings = warnings
1353
1354 if not(self.setNextFile()):
1355 if (startDate!=None) and (endDate!=None):
1356 print "[Reading] No files in range: %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime())
1357 elif startDate != None:
1358 print "[Reading] No files in range: %s" %(datetime.datetime.combine(startDate,startTime).ctime())
1359 else:
1360 print "[Reading] No files"
1423 1361
1424 # self.getBasicHeader()
1362 self.fileIndex = -1
1363 self.pathList = []
1364 self.filenameList = []
1365 return
1425 1366
1426 if last_set != None:
1427 self.dataOut.last_block = last_set * self.processingHeaderObj.dataBlocksPerFile + self.basicHeaderObj.dataBlock
1367 # self.getBasicHeader()
1368
1369 if last_set != None:
1370 self.dataOut.last_block = last_set * self.processingHeaderObj.dataBlocksPerFile + self.basicHeaderObj.dataBlock
1428 1371 return
1429 1372
1430 1373 def getBasicHeader(self):
@@ -1442,10 +1385,10 class JRODataReader(JRODataIO):
1442 1385 self.dataOut.useLocalTime = self.basicHeaderObj.useLocalTime
1443 1386
1444 1387 self.dataOut.ippSeconds = self.radarControllerHeaderObj.ippSeconds/self.nTxs
1445
1446 # self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock*self.nTxs
1447
1448
1388
1389 # self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock*self.nTxs
1390
1391
1449 1392 def getFirstHeader(self):
1450 1393
1451 1394 raise NotImplementedError
@@ -1517,29 +1460,27 class JRODataReader(JRODataIO):
1517 1460 server=None,
1518 1461 verbose=True, **kwargs):
1519 1462 if not(self.isConfig):
1520 # self.dataOut = dataOut
1521 self.setup( path=path,
1522 startDate=startDate,
1523 endDate=endDate,
1524 startTime=startTime,
1525 endTime=endTime,
1526 set=set,
1527 expLabel=expLabel,
1528 ext=ext,
1529 online=online,
1530 delay=delay,
1531 walk=walk,
1532 getblock=getblock,
1533 nTxs=nTxs,
1534 realtime=realtime,
1535 blocksize=blocksize,
1536 blocktime=blocktime,
1537 queue=queue,
1538 skip=skip,
1539 cursor=cursor,
1540 warnings=warnings,
1541 server=server,
1542 verbose=verbose, **kwargs)
1463 self.setup(path=path,
1464 startDate=startDate,
1465 endDate=endDate,
1466 startTime=startTime,
1467 endTime=endTime,
1468 set=set,
1469 expLabel=expLabel,
1470 ext=ext,
1471 online=online,
1472 delay=delay,
1473 walk=walk,
1474 getblock=getblock,
1475 nTxs=nTxs,
1476 realtime=realtime,
1477 blocksize=blocksize,
1478 blocktime=blocktime,
1479 skip=skip,
1480 cursor=cursor,
1481 warnings=warnings,
1482 server=server,
1483 verbose=verbose)
1543 1484 self.isConfig = True
1544 1485 if server is None:
1545 1486 self.getData()
@@ -1662,9 +1603,9 class JRODataWriter(JRODataIO):
1662 1603 Return:
1663 1604 None
1664 1605 """
1665
1666 # CALCULAR PARAMETROS
1667
1606
1607 # CALCULAR PARAMETROS
1608
1668 1609 sizeLongHeader = self.systemHeaderObj.size + self.radarControllerHeaderObj.size + self.processingHeaderObj.size
1669 1610 self.basicHeaderObj.size = self.basicHeaderSize + sizeLongHeader
1670 1611
@@ -1791,7 +1732,7 class JRODataWriter(JRODataIO):
1791 1732
1792 1733 return 1
1793 1734
1794 def setup(self, dataOut, path, blocksPerFile, profilesPerBlock=64, set=None, ext=None, datatype=4, verbose=True):
1735 def setup(self, dataOut, path, blocksPerFile, profilesPerBlock=64, set=None, ext=None, datatype=4):
1795 1736 """
1796 1737 Setea el tipo de formato en la cual sera guardada la data y escribe el First Header
1797 1738
Show file before | Show file after | Hide comments
@@ -453,7 +453,7 class FitsReader(ProcessingUnit):
453 453 # self.blockIndex = 1
454 454 return 1
455 455
456 def __searchFilesOffLine(self,
456 def searchFilesOffLine(self,
457 457 path,
458 458 startDate,
459 459 endDate,
@@ -559,7 +559,7 class FitsReader(ProcessingUnit):
559 559
560 560 if not(online):
561 561 print "Searching files in offline mode ..."
562 pathList, filenameList = self.__searchFilesOffLine(path, startDate=startDate, endDate=endDate,
562 pathList, filenameList = self.searchFilesOffLine(path, startDate=startDate, endDate=endDate,
563 563 startTime=startTime, endTime=endTime,
564 564 set=set, expLabel=expLabel, ext=ext,
565 565 walk=walk)
Show file before | Show file after | Hide comments
@@ -415,7 +415,7 class HFReader(ProcessingUnit):
415 415
416 416
417 417
418 def __searchFilesOffline(self,
418 def searchFilesOffLine(self,
419 419 path,
420 420 startDate,
421 421 endDate,
@@ -438,7 +438,7 class HFReader(ProcessingUnit):
438 438
439 439 return
440 440
441 def __searchFilesOnline(self,
441 def searchFilesOnLine(self,
442 442 path,
443 443 expLabel= "",
444 444 ext=None,
@@ -636,10 +636,10 class HFReader(ProcessingUnit):
636 636 if not(online):
637 637 print "Searching files in offline mode..."
638 638
639 self.__searchFilesOffline(path, startDate, endDate, ext, startTime, endTime, walk)
639 self.searchFilesOffLine(path, startDate, endDate, ext, startTime, endTime, walk)
640 640 else:
641 641 print "Searching files in online mode..."
642 self.__searchFilesOnline(path, walk,ext,set=set)
642 self.searchFilesOnLine(path, walk,ext,set=set)
643 643 if set==None:
644 644 pass
645 645 else:
@@ -647,7 +647,7 class HFReader(ProcessingUnit):
647 647
648 648 # for nTries in range(self.nTries):
649 649 #
650 # fullpath,file,year,month,day,set = self.__searchFilesOnline(path=path,expLabel=expLabel,ext=ext, walk=walk,set=set)
650 # fullpath,file,year,month,day,set = self.searchFilesOnLine(path=path,expLabel=expLabel,ext=ext, walk=walk,set=set)
651 651 #
652 652 # if fullpath:
653 653 # break
Show file before | Show file after | Hide comments
@@ -106,9 +106,9 class AMISRReader(ProcessingUnit):
106 106 #self.findFiles()
107 107 if not(online):
108 108 #Busqueda de archivos offline
109 self.__searchFilesOffline(path, startDate, endDate, startTime, endTime, walk)
109 self.searchFilesOffLine(path, startDate, endDate, startTime, endTime, walk)
110 110 else:
111 self.__searchFilesOnline(path, startDate, endDate, startTime,endTime,walk)
111 self.searchFilesOnLine(path, startDate, endDate, startTime,endTime,walk)
112 112
113 113 if not(self.filenameList):
114 114 print "There is no files into the folder: %s"%(path)
@@ -329,7 +329,7 class AMISRReader(ProcessingUnit):
329 329 self.dirnameList = new_dirnameList
330 330 return 1
331 331
332 def __searchFilesOnline(self, path, startDate, endDate, startTime=datetime.time(0,0,0),
332 def searchFilesOnLine(self, path, startDate, endDate, startTime=datetime.time(0,0,0),
333 333 endTime=datetime.time(23,59,59),walk=True):
334 334
335 335 if endDate ==None:
@@ -349,7 +349,7 class AMISRReader(ProcessingUnit):
349 349 return
350 350
351 351
352 def __searchFilesOffline(self,
352 def searchFilesOffLine(self,
353 353 path,
354 354 startDate,
355 355 endDate,
Show file before | Show file after | Hide comments
@@ -97,7 +97,7 class ParamReader(ProcessingUnit):
97 97 self.timezone = 'lt'
98 98
99 99 print "[Reading] Searching files in offline mode ..."
100 pathList, filenameList = self.__searchFilesOffLine(path, startDate=startDate, endDate=endDate,
100 pathList, filenameList = self.searchFilesOffLine(path, startDate=startDate, endDate=endDate,
101 101 startTime=startTime, endTime=endTime,
102 102 ext=ext, walk=walk)
103 103
@@ -115,7 +115,7 class ParamReader(ProcessingUnit):
115 115
116 116 return
117 117
118 def __searchFilesOffLine(self,
118 def searchFilesOffLine(self,
119 119 path,
120 120 startDate=None,
121 121 endDate=None,
Show file before | Show file after | Hide comments
1 NO CONTENT: modified file
The requested commit or file is too big and content was truncated. Show full diff
Show file before | Show file after | Hide comments
@@ -1,3 +1,5
1 import itertools
2
1 3 import numpy
2 4
3 5 from jroproc_base import ProcessingUnit, Operation
@@ -109,7 +111,14 class SpectraProc(ProcessingUnit):
109 111
110 112 if self.dataIn.type == "Spectra":
111 113 self.dataOut.copy(self.dataIn)
114 <<<<<<< HEAD
112 115 # self.__selectPairs(pairsList)
116 =======
117 if not pairsList:
118 pairsList = itertools.combinations(self.dataOut.channelList, 2)
119 if self.dataOut.data_cspc is not None:
120 self.__selectPairs(pairsList)
121 >>>>>>> 8048843f4edfb980d0968f42f82054783b84e1cc
113 122 return True
114 123
115 124 if self.dataIn.type == "Voltage":
@@ -178,27 +187,21 class SpectraProc(ProcessingUnit):
178 187
179 188 def __selectPairs(self, pairsList):
180 189
181 if channelList == None:
190 if not pairsList:
182 191 return
183 192
184 pairsIndexListSelected = []
185
186 for thisPair in pairsList:
193 pairs = []
194 pairsIndex = []
187 195
188 if thisPair not in self.dataOut.pairsList:
196 for pair in pairsList:
197 if pair[0] not in self.dataOut.channelList or pair[1] not in self.dataOut.channelList:
189 198 continue
190
191 pairIndex = self.dataOut.pairsList.index(thisPair)
192
193 pairsIndexListSelected.append(pairIndex)
194
195 if not pairsIndexListSelected:
196 self.dataOut.data_cspc = None
197 self.dataOut.pairsList = []
198 return
199
200 self.dataOut.data_cspc = self.dataOut.data_cspc[pairsIndexListSelected]
201 self.dataOut.pairsList = [self.dataOut.pairsList[i] for i in pairsIndexListSelected]
199 pairs.append(pair)
200 pairsIndex.append(pairs.index(pair))
201
202 self.dataOut.data_cspc = self.dataOut.data_cspc[pairsIndex]
203 self.dataOut.pairsList = pairs
204 self.dataOut.pairsIndexList = pairsIndex
202 205
203 206 return
204 207
Show file before | Show file after | Hide comments
@@ -1,6 +1,5
1 1 import sys
2 2 import numpy
3 from profilehooks import profile
4 3 from scipy import interpolate
5 4 from schainpy import cSchain
6 5 from jroproc_base import ProcessingUnit, Operation
@@ -623,13 +622,6 class Decoder(Operation):
623 622
624 623 return self.datadecTime
625 624
626 #@profile
627 def oldCorrelate(self, i, data, code_block):
628 profilesList = xrange(self.__nProfiles)
629 for j in profilesList:
630 self.datadecTime[i,j,:] = numpy.correlate(data[i,j,:], code_block[j,:], mode='full')[self.nBaud-1:]
631
632 #@profile
633 625 def __convolutionByBlockInTime(self, data):
634 626
635 627 repetitions = self.__nProfiles / self.nCode
@@ -639,25 +631,10 class Decoder(Operation):
639 631 code_block = numpy.reshape(junk, (self.nCode*repetitions, self.nBaud))
640 632 profilesList = xrange(self.__nProfiles)
641 633
642 # def toVectorize(a,b):
643 # return numpy.correlate(a,b, mode='full')
644 # vectorized = numpy.vectorize(toVectorize, signature='(n),(m)->(k)')
645 for i in range(self.__nChannels):
646 # self.datadecTime[i,:,:] = numpy.array([numpy.correlate(data[i,j,:], code_block[j,:], mode='full')[self.nBaud-1:] for j in profilesList ])
647 # def func(i, j):
648 # self.datadecTime[i,j,:] = numpy.correlate(data[i,j,:], code_block[j,:], mode='full')[self.nBaud-1:]
649 # map(lambda j: func(i, j), range(self.__nProfiles))
650 #print data[i,:,:].shape
651 # self.datadecTime[i,:,:] = vectorized(data[i,:,:], code_block[:,:])[:,self.nBaud-1:]
652 for j in profilesList:
653 self.datadecTime[i,j,:] = numpy.correlate(data[i,j,:], code_block[j,:], mode='full')[self.nBaud-1:]
654 # print data[i,:,:]
655 # print cSchain.correlateByBlock(data[i,:,:], code_block, 2)
656 # self.datadecTime[i,:,:] = cSchain.correlateByBlock(data[i,:,:], code_block, 2)
657 # print self.datadecTime[i,:,:]
658 #print self.datadecTime[i,:,:].shape
659 return self.datadecTime
660
634 for i in range(self.__nChannels):
635 for j in profilesList:
636 self.datadecTime[i,j,:] = numpy.correlate(data[i,j,:], code_block[j,:], mode='full')[self.nBaud-1:]
637 return self.datadecTime
661 638
662 639 def __convolutionByBlockInFreq(self, data):
663 640
Show file before | Show file after | Hide comments
@@ -7,7 +7,6 import json
7 7 import numpy
8 8 import paho.mqtt.client as mqtt
9 9 import zmq
10 from profilehooks import profile
11 10 import datetime
12 11 from zmq.utils.monitor import recv_monitor_message
13 12 from functools import wraps
@@ -16,6 +15,7 from multiprocessing import Process
16 15
17 16 from schainpy.model.proc.jroproc_base import Operation, ProcessingUnit
18 17 from schainpy.model.data.jrodata import JROData
18 from schainpy.utils import log
19 19
20 20 MAXNUMX = 100
21 21 MAXNUMY = 100
@@ -31,14 +31,13 def roundFloats(obj):
31 31 return round(obj, 2)
32 32
33 33 def decimate(z, MAXNUMY):
34 # dx = int(len(self.x)/self.__MAXNUMX) + 1
35
36 34 dy = int(len(z[0])/MAXNUMY) + 1
37 35
38 36 return z[::, ::dy]
39 37
40 38 class throttle(object):
41 """Decorator that prevents a function from being called more than once every
39 '''
40 Decorator that prevents a function from being called more than once every
42 41 time period.
43 42 To create a function that cannot be called more than once a minute, but
44 43 will sleep until it can be called:
@@ -49,7 +48,7 class throttle(object):
49 48 for i in range(10):
50 49 foo()
51 50 print "This function has run %s times." % i
52 """
51 '''
53 52
54 53 def __init__(self, seconds=0, minutes=0, hours=0):
55 54 self.throttle_period = datetime.timedelta(
@@ -73,9 +72,169 class throttle(object):
73 72
74 73 return wrapper
75 74
75 class Data(object):
76 '''
77 Object to hold data to be plotted
78 '''
79
80 def __init__(self, plottypes, throttle_value):
81 self.plottypes = plottypes
82 self.throttle = throttle_value
83 self.ended = False
84 self.__times = []
85
86 def __str__(self):
87 dum = ['{}{}'.format(key, self.shape(key)) for key in self.data]
88 return 'Data[{}][{}]'.format(';'.join(dum), len(self.__times))
89
90 def __len__(self):
91 return len(self.__times)
92
93 def __getitem__(self, key):
94 if key not in self.data:
95 raise KeyError(log.error('Missing key: {}'.format(key)))
96
97 if 'spc' in key:
98 ret = self.data[key]
99 else:
100 ret = numpy.array([self.data[key][x] for x in self.times])
101 if ret.ndim > 1:
102 ret = numpy.swapaxes(ret, 0, 1)
103 return ret
104
105 def setup(self):
106 '''
107 Configure object
108 '''
109
110 self.ended = False
111 self.data = {}
112 self.__times = []
113 self.__heights = []
114 self.__all_heights = set()
115 for plot in self.plottypes:
116 self.data[plot] = {}
117
118 def shape(self, key):
119 '''
120 Get the shape of the one-element data for the given key
121 '''
122
123 if len(self.data[key]):
124 if 'spc' in key:
125 return self.data[key].shape
126 return self.data[key][self.__times[0]].shape
127 return (0,)
128
129 def update(self, dataOut):
130 '''
131 Update data object with new dataOut
132 '''
133
134 tm = dataOut.utctime
135 if tm in self.__times:
136 return
137
138 self.parameters = getattr(dataOut, 'parameters', [])
139 self.pairs = dataOut.pairsList
140 self.channels = dataOut.channelList
141 self.xrange = (dataOut.getFreqRange(1)/1000. , dataOut.getAcfRange(1) , dataOut.getVelRange(1))
142 self.interval = dataOut.getTimeInterval()
143 self.__heights.append(dataOut.heightList)
144 self.__all_heights.update(dataOut.heightList)
145 self.__times.append(tm)
146
147 for plot in self.plottypes:
148 if plot == 'spc':
149 z = dataOut.data_spc/dataOut.normFactor
150 self.data[plot] = 10*numpy.log10(z)
151 if plot == 'cspc':
152 self.data[plot] = dataOut.data_cspc
153 if plot == 'noise':
154 self.data[plot][tm] = 10*numpy.log10(dataOut.getNoise()/dataOut.normFactor)
155 if plot == 'rti':
156 self.data[plot][tm] = dataOut.getPower()
157 if plot == 'snr_db':
158 self.data['snr'][tm] = dataOut.data_SNR
159 if plot == 'snr':
160 self.data[plot][tm] = 10*numpy.log10(dataOut.data_SNR)
161 if plot == 'dop':
162 self.data[plot][tm] = 10*numpy.log10(dataOut.data_DOP)
163 if plot == 'mean':
164 self.data[plot][tm] = dataOut.data_MEAN
165 if plot == 'std':
166 self.data[plot][tm] = dataOut.data_STD
167 if plot == 'coh':
168 self.data[plot][tm] = dataOut.getCoherence()
169 if plot == 'phase':
170 self.data[plot][tm] = dataOut.getCoherence(phase=True)
171 if plot == 'output':
172 self.data[plot][tm] = dataOut.data_output
173 if plot == 'param':
174 self.data[plot][tm] = dataOut.data_param
175
176 def normalize_heights(self):
177 '''
178 Ensure same-dimension of the data for different heighList
179 '''
180
181 H = numpy.array(list(self.__all_heights))
182 H.sort()
183 for key in self.data:
184 shape = self.shape(key)[:-1] + H.shape
185 for tm, obj in self.data[key].items():
186 h = self.__heights[self.__times.index(tm)]
187 if H.size == h.size:
188 continue
189 index = numpy.where(numpy.in1d(H, h))[0]
190 dummy = numpy.zeros(shape) + numpy.nan
191 if len(shape) == 2:
192 dummy[:, index] = obj
193 else:
194 dummy[index] = obj
195 self.data[key][tm] = dummy
196
197 self.__heights = [H for tm in self.__times]
198
199 def jsonify(self, decimate=False):
200 '''
201 Convert data to json
202 '''
203
204 ret = {}
205 tm = self.times[-1]
206
207 for key, value in self.data:
208 if key in ('spc', 'cspc'):
209 ret[key] = roundFloats(self.data[key].to_list())
210 else:
211 ret[key] = roundFloats(self.data[key][tm].to_list())
212
213 ret['timestamp'] = tm
214 ret['interval'] = self.interval
215
216 @property
217 def times(self):
218 '''
219 Return the list of times of the current data
220 '''
221
222 ret = numpy.array(self.__times)
223 ret.sort()
224 return ret
225
226 @property
227 def heights(self):
228 '''
229 Return the list of heights of the current data
230 '''
231
232 return numpy.array(self.__heights[-1])
76 233
77 234 class PublishData(Operation):
78 """Clase publish."""
235 '''
236 Operation to send data over zmq.
237 '''
79 238
80 239 def __init__(self, **kwargs):
81 240 """Inicio."""
@@ -87,11 +246,11 class PublishData(Operation):
87 246
88 247 def on_disconnect(self, client, userdata, rc):
89 248 if rc != 0:
90 print("Unexpected disconnection.")
249 log.warning('Unexpected disconnection.')
91 250 self.connect()
92 251
93 252 def connect(self):
94 print 'trying to connect'
253 log.warning('trying to connect')
95 254 try:
96 255 self.client.connect(
97 256 host=self.host,
@@ -105,7 +264,7 class PublishData(Operation):
105 264 # retain=True
106 265 # )
107 266 except:
108 print "MQTT Conection error."
267 log.error('MQTT Conection error.')
109 268 self.client = False
110 269
111 270 def setup(self, port=1883, username=None, password=None, clientId="user", zeromq=1, verbose=True, **kwargs):
@@ -120,8 +279,7 class PublishData(Operation):
120 279 self.zeromq = zeromq
121 280 self.mqtt = kwargs.get('plottype', 0)
122 281 self.client = None
123 self.verbose = verbose
124 self.dataOut.firstdata = True
282 self.verbose = verbose
125 283 setup = []
126 284 if mqtt is 1:
127 285 self.client = mqtt.Client(
@@ -176,7 +334,6 class PublishData(Operation):
176 334 'type': self.plottype,
177 335 'yData': yData
178 336 }
179 # print payload
180 337
181 338 elif self.plottype in ('rti', 'power'):
182 339 data = getattr(self.dataOut, 'data_spc')
@@ -230,15 +387,16 class PublishData(Operation):
230 387 'timestamp': 'None',
231 388 'type': None
232 389 }
233 # print 'Publishing data to {}'.format(self.host)
390
234 391 self.client.publish(self.topic + self.plottype, json.dumps(payload), qos=0)
235 392
236 393 if self.zeromq is 1:
237 394 if self.verbose:
238 print '[Sending] {} - {}'.format(self.dataOut.type, self.dataOut.datatime)
395 log.log(
396 '{} - {}'.format(self.dataOut.type, self.dataOut.datatime),
397 'Sending'
398 )
239 399 self.zmq_socket.send_pyobj(self.dataOut)
240 self.dataOut.firstdata = False
241
242 400
243 401 def run(self, dataOut, **kwargs):
244 402 self.dataOut = dataOut
@@ -253,6 +411,7 class PublishData(Operation):
253 411 if self.zeromq is 1:
254 412 self.dataOut.finished = True
255 413 self.zmq_socket.send_pyobj(self.dataOut)
414 time.sleep(0.1)
256 415 self.zmq_socket.close()
257 416 if self.client:
258 417 self.client.loop_stop()
@@ -281,7 +440,7 class ReceiverData(ProcessingUnit):
281 440 self.receiver = self.context.socket(zmq.PULL)
282 441 self.receiver.bind(self.address)
283 442 time.sleep(0.5)
284 print '[Starting] ReceiverData from {}'.format(self.address)
443 log.success('ReceiverData from {}'.format(self.address))
285 444
286 445
287 446 def run(self):
@@ -291,8 +450,9 class ReceiverData(ProcessingUnit):
291 450 self.isConfig = True
292 451
293 452 self.dataOut = self.receiver.recv_pyobj()
294 print '[Receiving] {} - {}'.format(self.dataOut.type,
295 self.dataOut.datatime.ctime())
453 log.log('{} - {}'.format(self.dataOut.type,
454 self.dataOut.datatime.ctime(),),
455 'Receiving')
296 456
297 457
298 458 class PlotterReceiver(ProcessingUnit, Process):
@@ -306,7 +466,6 class PlotterReceiver(ProcessingUnit, Process):
306 466 self.mp = False
307 467 self.isConfig = False
308 468 self.isWebConfig = False
309 self.plottypes = []
310 469 self.connections = 0
311 470 server = kwargs.get('server', 'zmq.pipe')
312 471 plot_server = kwargs.get('plot_server', 'zmq.web')
@@ -326,19 +485,13 class PlotterReceiver(ProcessingUnit, Process):
326 485 self.realtime = kwargs.get('realtime', False)
327 486 self.throttle_value = kwargs.get('throttle', 5)
328 487 self.sendData = self.initThrottle(self.throttle_value)
488 self.dates = []
329 489 self.setup()
330 490
331 491 def setup(self):
332 492
333 self.data = {}
334 self.data['times'] = []
335 for plottype in self.plottypes:
336 self.data[plottype] = {}
337 self.data['noise'] = {}
338 self.data['throttle'] = self.throttle_value
339 self.data['ENDED'] = False
340 self.isConfig = True
341 self.data_web = {}
493 self.data = Data(self.plottypes, self.throttle_value)
494 self.isConfig = True
342 495
343 496 def event_monitor(self, monitor):
344 497
@@ -355,15 +508,13 class PlotterReceiver(ProcessingUnit, Process):
355 508 self.connections += 1
356 509 if evt['event'] == 512:
357 510 pass
358 if self.connections == 0 and self.started is True:
359 self.ended = True
360 511
361 512 evt.update({'description': events[evt['event']]})
362 513
363 514 if evt['event'] == zmq.EVENT_MONITOR_STOPPED:
364 515 break
365 516 monitor.close()
366 print("event monitor thread done!")
517 print('event monitor thread done!')
367 518
368 519 def initThrottle(self, throttle_value):
369 520
@@ -373,65 +524,16 class PlotterReceiver(ProcessingUnit, Process):
373 524
374 525 return sendDataThrottled
375 526
376
377 527 def send(self, data):
378 # print '[sending] data=%s size=%s' % (data.keys(), len(data['times']))
528 log.success('Sending {}'.format(data), self.name)
379 529 self.sender.send_pyobj(data)
380 530
381
382 def update(self):
383 t = self.dataOut.utctime
384
385 if t in self.data['times']:
386 return
387
388 self.data['times'].append(t)
389 self.data['dataOut'] = self.dataOut
390
391 for plottype in self.plottypes:
392 if plottype == 'spc':
393 z = self.dataOut.data_spc/self.dataOut.normFactor
394 self.data[plottype] = 10*numpy.log10(z)
395 self.data['noise'][t] = 10*numpy.log10(self.dataOut.getNoise()/self.dataOut.normFactor)
396 if plottype == 'cspc':
397 jcoherence = self.dataOut.data_cspc/numpy.sqrt(self.dataOut.data_spc*self.dataOut.data_spc)
398 self.data['cspc_coh'] = numpy.abs(jcoherence)
399 self.data['cspc_phase'] = numpy.arctan2(jcoherence.imag, jcoherence.real)*180/numpy.pi
400 if plottype == 'rti':
401 self.data[plottype][t] = self.dataOut.getPower()
402 if plottype == 'snr':
403 self.data[plottype][t] = 10*numpy.log10(self.dataOut.data_SNR)
404 if plottype == 'dop':
405 self.data[plottype][t] = 10*numpy.log10(self.dataOut.data_DOP)
406 if plottype == 'mean':
407 self.data[plottype][t] = self.dataOut.data_MEAN
408 if plottype == 'std':
409 self.data[plottype][t] = self.dataOut.data_STD
410 if plottype == 'coh':
411 self.data[plottype][t] = self.dataOut.getCoherence()
412 if plottype == 'phase':
413 self.data[plottype][t] = self.dataOut.getCoherence(phase=True)
414 if plottype == 'output':
415 self.data[plottype][t] = self.dataOut.data_output
416 if plottype == 'param':
417 self.data[plottype][t] = self.dataOut.data_param
418 if self.realtime:
419 self.data_web['timestamp'] = t
420 if plottype == 'spc':
421 self.data_web[plottype] = roundFloats(decimate(self.data[plottype]).tolist())
422 elif plottype == 'cspc':
423 self.data_web['cspc_coh'] = roundFloats(decimate(self.data['cspc_coh']).tolist())
424 self.data_web['cspc_phase'] = roundFloats(decimate(self.data['cspc_phase']).tolist())
425 elif plottype == 'noise':
426 self.data_web['noise'] = roundFloats(self.data['noise'][t].tolist())
427 else:
428 self.data_web[plottype] = roundFloats(decimate(self.data[plottype][t]).tolist())
429 self.data_web['interval'] = self.dataOut.getTimeInterval()
430 self.data_web['type'] = plottype
431
432 531 def run(self):
433 532
434 print '[Starting] {} from {}'.format(self.name, self.address)
533 log.success(
534 'Starting from {}'.format(self.address),
535 self.name
536 )
435 537
436 538 self.context = zmq.Context()
437 539 self.receiver = self.context.socket(zmq.PULL)
@@ -448,39 +550,39 class PlotterReceiver(ProcessingUnit, Process):
448 550 else:
449 551 self.sender.bind("ipc:///tmp/zmq.plots")
450 552
451 time.sleep(3)
553 time.sleep(2)
452 554
453 555 t = Thread(target=self.event_monitor, args=(monitor,))
454 556 t.start()
455 557
456 558 while True:
457 self.dataOut = self.receiver.recv_pyobj()
458 # print '[Receiving] {} - {}'.format(self.dataOut.type,
459 # self.dataOut.datatime.ctime())
460
461 self.update()
559 dataOut = self.receiver.recv_pyobj()
560 dt = datetime.datetime.fromtimestamp(dataOut.utctime).date()
561 sended = False
562 if dt not in self.dates:
563 if self.data:
564 self.data.ended = True
565 self.send(self.data)
566 sended = True
567 self.data.setup()
568 self.dates.append(dt)
462 569
463 if self.dataOut.firstdata is True:
464 self.data['STARTED'] = True
570 self.data.update(dataOut)
465 571
466 if self.dataOut.finished is True:
467 self.send(self.data)
572 if dataOut.finished is True:
468 573 self.connections -= 1
469 if self.connections == 0 and self.started:
470 self.ended = True
471 self.data['ENDED'] = True
574 if self.connections == 0 and dt in self.dates:
575 self.data.ended = True
472 576 self.send(self.data)
473 self.setup()
474 self.started = False
577 self.data.setup()
475 578 else:
476 579 if self.realtime:
477 580 self.send(self.data)
478 self.sender_web.send_string(json.dumps(self.data_web))
581 # self.sender_web.send_string(self.data.jsonify())
479 582 else:
480 self.sendData(self.send, self.data)
481 self.started = True
583 if not sended:
584 self.sendData(self.send, self.data)
482 585
483 self.data['STARTED'] = False
484 586 return
485 587
486 588 def sendToWeb(self):
@@ -497,6 +599,6 class PlotterReceiver(ProcessingUnit, Process):
497 599 time.sleep(1)
498 600 for kwargs in self.operationKwargs.values():
499 601 if 'plot' in kwargs:
500 print '[Sending] Config data to web for {}'.format(kwargs['code'].upper())
602 log.success('[Sending] Config data to web for {}'.format(kwargs['code'].upper()))
501 603 sender_web_config.send_string(json.dumps(kwargs))
502 self.isWebConfig = True
604 self.isWebConfig = True No newline at end of file
Show file before | Show file after | Hide comments
@@ -1,4 +1,4
1 """
1 '''
2 2 SCHAINPY - LOG
3 3 Simple helper for log standarization
4 4 Usage:
@@ -13,47 +13,32 SCHAINPY - LOG
13 13 which will look like this:
14 14 [NEVER GONNA] - give you up
15 15 with color red as background and white as foreground.
16 """
17 import os
18 import sys
16 '''
17
19 18 import click
20 19
21 def warning(message):
22 click.echo(click.style('[WARNING] - ' + message, fg='yellow'))
20 def warning(message, tag='Warning'):
21 click.echo(click.style('[{}] {}'.format(tag, message), fg='yellow'))
22 pass
23
23 24
25 def error(message, tag='Error'):
26 click.echo(click.style('[{}] {}'.format(tag, message), fg='red'))
27 pass
24 28
25 def error(message):
26 click.echo(click.style('[ERROR] - ' + message, fg='red', bg='black'))
27 29
30 def success(message, tag='Info'):
31 click.echo(click.style('[{}] {}'.format(tag, message), fg='green'))
32 pass
28 33
29 def success(message):
30 click.echo(click.style(message, fg='green'))
31 34
35 def log(message, tag='Info'):
36 click.echo('[{}] {}'.format(tag, message))
37 pass
32 38
33 def log(message, topic='LOG'):
34 click.echo('[{}] - {}'.format(topic, message))
35 39
36 def makelogger(topic, bg='reset', fg='reset'):
40 def makelogger(tag, bg='reset', fg='reset'):
37 41 def func(message):
38 click.echo(click.style('[{}] - '.format(topic.upper()) + message,
42 click.echo(click.style('[{}] {}'.format(tag.upper(), message),
39 43 bg=bg, fg=fg))
40 44 return func
41
42 class LoggerForFile():
43 def __init__(self, filename):
44 self.old_stdout=sys.stdout
45 cwd = os.getcwd()
46 self.log_file = open(os.path.join(cwd, filename), 'w+')
47 def write(self, text):
48 text = text.rstrip()
49 if not text:
50 return
51 self.log_file.write(text + '\n')
52 self.old_stdout.write(text + '\n')
53 def flush(self):
54 self.old_stdout.flush()
55
56 def logToFile(filename='log.log'):
57 logger = LoggerForFile(filename)
58 sys.stdout = logger
59
Show file before | Show file after | Hide comments
1 NO CONTENT: file was removed
General Comments 0
You need to be logged in to leave comments. Login now