##// END OF EJS Templates
schain-jc
RSS
Merge with px1000 branch
Juan C. Espinoza -
r1164:80b8ab8c0c11 merge
parent child
Show More
Side by Side Unified
Context file:
r1164:80b8ab8c0c11 Loading diff...:
Show file before | Show file after | Hide comments
@@ -0,0 +1,350
1 '''
2 Created on Jan 15, 2018
3
4 @author: Juan C. Espinoza
5 '''
6
7 import os
8 import sys
9 import time
10 import glob
11 import datetime
12 import tarfile
13
14 import numpy
15 try:
16 from netCDF4 import Dataset
17 except:
18 log.warning(
19 'You should install "netCDF4" module if you want to read/write NCDF files'
20 )
21
22 from utils import folder_in_range
23
24 from schainpy.model.io.jroIO_base import JRODataReader
25 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation
26 from schainpy.model.data.jrodata import Parameters
27 from schainpy.utils import log
28
29 UT1970 = datetime.datetime(1970, 1, 1) - datetime.timedelta(seconds=time.timezone)
30
31
32 class PXReader(JRODataReader, ProcessingUnit):
33
34 def __init__(self, **kwargs):
35
36 ProcessingUnit.__init__(self, **kwargs)
37
38 self.dataOut = Parameters()
39 self.counter_records = 0
40 self.nrecords = None
41 self.flagNoMoreFiles = 0
42 self.isConfig = False
43 self.filename = None
44 self.intervals = set()
45 self.ext = ('.nc', '.tgz')
46 self.online_mode = False
47
48 def setup(self,
49 path=None,
50 startDate=None,
51 endDate=None,
52 format=None,
53 startTime=datetime.time(0, 0, 0),
54 endTime=datetime.time(23, 59, 59),
55 walk=False,
56 **kwargs):
57
58 self.path = path
59 self.startDate = startDate
60 self.endDate = endDate
61 self.startTime = startTime
62 self.endTime = endTime
63 self.datatime = datetime.datetime(1900,1,1)
64 self.walk = walk
65 self.nTries = kwargs.get('nTries', 10)
66 self.online = kwargs.get('online', False)
67 self.delay = kwargs.get('delay', 60)
68 self.ele = kwargs.get('ext', '')
69
70 if self.path is None:
71 raise ValueError, 'The path is not valid'
72
73 self.search_files(path, startDate, endDate, startTime, endTime, walk)
74 self.cursor = 0
75 self.counter_records = 0
76
77 if not self.files:
78 raise Warning, 'There is no files matching these date in the folder: {}. \n Check startDate and endDate'.format(path)
79
80 def search_files(self, path, startDate, endDate, startTime, endTime, walk):
81 '''
82 Searching for NCDF files in path
83 Creating a list of files to procces included in [startDate,endDate]
84
85 Input:
86 path - Path to find files
87 '''
88
89 log.log('Searching files {} in {} '.format(self.ext, path), 'PXReader')
90 if walk:
91 paths = [os.path.join(path, p) for p in os.listdir(path) if os.path.isdir(os.path.join(path, p))]
92 paths.sort()
93 else:
94 paths = [path]
95
96 fileList0 = []
97
98 for subpath in paths:
99 if not folder_in_range(subpath.split('/')[-1], startDate, endDate, '%Y%m%d'):
100 continue
101 fileList0 += [os.path.join(subpath, s) for s in glob.glob1(subpath, '*') if os.path.splitext(s)[-1] in self.ext and '{}'.format(self.ele) in s]
102
103 fileList0.sort()
104 if self.online:
105 fileList0 = fileList0[-1:]
106
107 self.files = {}
108
109 startDate = startDate - datetime.timedelta(1)
110 endDate = endDate + datetime.timedelta(1)
111
112 for fullname in fileList0:
113 thisFile = fullname.split('/')[-1]
114 year = thisFile[3:7]
115 if not year.isdigit():
116 continue
117
118 month = thisFile[7:9]
119 if not month.isdigit():
120 continue
121
122 day = thisFile[9:11]
123 if not day.isdigit():
124 continue
125
126 year, month, day = int(year), int(month), int(day)
127 dateFile = datetime.date(year, month, day)
128 timeFile = datetime.time(int(thisFile[12:14]), int(thisFile[14:16]), int(thisFile[16:18]))
129
130 if (startDate > dateFile) or (endDate < dateFile):
131 continue
132
133 dt = datetime.datetime.combine(dateFile, timeFile)
134 if dt not in self.files:
135 self.files[dt] = []
136 self.files[dt].append(fullname)
137
138 self.dates = self.files.keys()
139 self.dates.sort()
140
141 return
142
143 def search_files_online(self):
144 '''
145 Searching for NCDF files in online mode path
146 Creating a list of files to procces included in [startDate,endDate]
147
148 Input:
149 path - Path to find files
150 '''
151
152 self.files = {}
153
154 for n in range(self.nTries):
155
156 if self.walk:
157 paths = [os.path.join(self.path, p) for p in os.listdir(self.path) if os.path.isdir(os.path.join(self.path, p))]
158 paths.sort()
159 path = paths[-1]
160 else:
161 path = self.path
162
163 new_files = [os.path.join(path, s) for s in glob.glob1(path, '*') if os.path.splitext(s)[-1] in self.ext and '{}'.format(self.ele) in s]
164 new_files.sort()
165
166 for fullname in new_files:
167 thisFile = fullname.split('/')[-1]
168 year = thisFile[3:7]
169 if not year.isdigit():
170 continue
171
172 month = thisFile[7:9]
173 if not month.isdigit():
174 continue
175
176 day = thisFile[9:11]
177 if not day.isdigit():
178 continue
179
180 year, month, day = int(year), int(month), int(day)
181 dateFile = datetime.date(year, month, day)
182 timeFile = datetime.time(int(thisFile[12:14]), int(thisFile[14:16]), int(thisFile[16:18]))
183
184 dt = datetime.datetime.combine(dateFile, timeFile)
185
186 if self.dt >= dt:
187 continue
188
189 if dt not in self.files:
190 self.dt = dt
191 self.files[dt] = []
192
193 self.files[dt].append(fullname)
194 break
195
196 if self.files:
197 break
198 else:
199 log.warning('Waiting {} seconds for the next file, try {} ...'.format(self.delay, n + 1), 'PXReader')
200 time.sleep(self.delay)
201
202 if not self.files:
203 return 0
204
205 self.dates = self.files.keys()
206 self.dates.sort()
207 self.cursor = 0
208
209 return 1
210
211 def parseFile(self):
212 '''
213 '''
214
215 header = {}
216
217 for attr in self.fp.ncattrs():
218 header[str(attr)] = getattr(self.fp, attr)
219
220 self.header.append(header)
221
222 self.data[header['TypeName']] = numpy.array(self.fp.variables[header['TypeName']])
223
224 def setNextFile(self):
225 '''
226 Open next files for the current datetime
227 '''
228
229 cursor = self.cursor
230 if not self.online_mode:
231 if cursor == len(self.dates):
232 if self.online:
233 cursor = 0
234 self.dt = self.dates[cursor]
235 self.online_mode = True
236 if not self.search_files_online():
237 log.success('No more files', 'PXReader')
238 return 0
239 else:
240 log.success('No more files', 'PXReader')
241 self.flagNoMoreFiles = 1
242 return 0
243 else:
244 if not self.search_files_online():
245 return 0
246 cursor = self.cursor
247
248 self.data = {}
249 self.header = []
250
251 for fullname in self.files[self.dates[cursor]]:
252
253 log.log('Opening: {}'.format(fullname), 'PXReader')
254
255 if os.path.splitext(fullname)[-1] == '.tgz':
256 tar = tarfile.open(fullname, 'r:gz')
257 tar.extractall('/tmp')
258 files = [os.path.join('/tmp', member.name) for member in tar.getmembers()]
259 else:
260 files = [fullname]
261
262 for filename in files:
263 if self.filename is not None:
264 self.fp.close()
265
266 self.filename = filename
267 self.filedate = self.dates[cursor]
268 self.fp = Dataset(self.filename, 'r')
269 self.parseFile()
270
271 self.counter_records += 1
272 self.cursor += 1
273 return 1
274
275 def readNextFile(self):
276
277 while True:
278 self.flagDiscontinuousBlock = 0
279 if not self.setNextFile():
280 return 0
281
282 self.datatime = datetime.datetime.utcfromtimestamp(self.header[0]['Time'])
283
284 if self.online:
285 break
286
287 if (self.datatime < datetime.datetime.combine(self.startDate, self.startTime)) or \
288 (self.datatime > datetime.datetime.combine(self.endDate, self.endTime)):
289 log.warning(
290 'Reading Record No. {}/{} -> {} [Skipping]'.format(
291 self.counter_records,
292 self.nrecords,
293 self.datatime.ctime()),
294 'PXReader')
295 continue
296 break
297
298 log.log(
299 'Reading Record No. {}/{} -> {}'.format(
300 self.counter_records,
301 self.nrecords,
302 self.datatime.ctime()),
303 'PXReader')
304
305 return 1
306
307
308 def set_output(self):
309 '''
310 Storing data from buffer to dataOut object
311 '''
312
313 self.data['Elevation'] = numpy.array(self.fp.variables['Elevation'])
314 self.data['Azimuth'] = numpy.array(self.fp.variables['Azimuth'])
315 self.dataOut.range = numpy.array(self.fp.variables['GateWidth'])
316 self.dataOut.data = self.data
317 self.dataOut.units = [h['Unit-value'] for h in self.header]
318 self.dataOut.parameters = [h['TypeName'] for h in self.header]
319 self.dataOut.missing = self.header[0]['MissingData']
320 self.dataOut.max_range = self.header[0]['MaximumRange-value']
321 self.dataOut.elevation = self.header[0]['Elevation']
322 self.dataOut.azimuth = self.header[0]['Azimuth']
323 self.dataOut.latitude = self.header[0]['Latitude']
324 self.dataOut.longitude = self.header[0]['Longitude']
325 self.dataOut.utctime = self.header[0]['Time']
326 self.dataOut.utctimeInit = self.dataOut.utctime
327 self.dataOut.useLocalTime = True
328 self.dataOut.flagNoData = False
329 self.dataOut.flagDiscontinuousBlock = self.flagDiscontinuousBlock
330
331 log.log('Parameters found: {}'.format(','.join(self.dataOut.parameters)),
332 'PXReader')
333
334 def getData(self):
335 '''
336 Storing data from databuffer to dataOut object
337 '''
338 if self.flagNoMoreFiles:
339 self.dataOut.flagNoData = True
340 log.error('No file left to process', 'PXReader')
341 return 0
342
343 if not self.readNextFile():
344 self.dataOut.flagNoData = True
345 return 0
346
347 self.set_output()
348
349 return 1
350
Show file before | Show file after | Hide comments
@@ -0,0 +1,24
1 """
2 Utilities for IO modules
3 """
4
5 import os
6 from datetime import datetime
7
8 def folder_in_range(folder, start_date, end_date, pattern):
9 """
10 Check whether folder is bettwen start_date and end_date
11
12 Args:
13 folder (str): Folder to check
14 start_date (date): Initial date
15 end_date (date): Final date
16 pattern (str): Datetime format of the folder
17 Returns:
18 bool: True for success, False otherwise
19 """
20 try:
21 dt = datetime.strptime(folder, pattern)
22 except:
23 raise ValueError('Folder {} does not match {} format'.format(folder, pattern))
24 return start_date <= dt.date() <= end_date
Show file before | Show file after | Hide comments
@@ -0,0 +1,64
1 '''
2 Created on Oct 24, 2016
3
4 @author: roj- LouVD
5 '''
6
7 import numpy
8 import datetime
9 import time
10 from time import gmtime
11
12 from numpy import transpose
13
14 from jroproc_base import ProcessingUnit, Operation
15 from schainpy.model.data.jrodata import Parameters
16
17
18 class PXParametersProc(ProcessingUnit):
19 '''
20 Processing unit for PX parameters data
21 '''
22
23 def __init__(self, **kwargs):
24 """
25 Inputs: None
26 """
27 ProcessingUnit.__init__(self, **kwargs)
28 self.dataOut = Parameters()
29 self.isConfig = False
30
31 def setup(self, mode):
32 """
33 """
34 self.dataOut.mode = mode
35
36 def run(self, mode):
37 """
38 Args:
39 mode (str): select independent variable 'E' for elevation or 'A' for azimuth
40 """
41
42 if not self.isConfig:
43 self.setup(mode)
44 self.isConfig = True
45
46 if self.dataIn.type == 'Parameters':
47 self.dataOut.copy(self.dataIn)
48
49 self.dataOut.data_param = numpy.array([self.dataOut.data[var] for var in self.dataOut.parameters])
50 self.dataOut.data_param[self.dataOut.data_param == self.dataOut.missing] = numpy.nan
51
52 if mode.upper()=='E':
53 self.dataOut.heightList = self.dataOut.data['Azimuth']
54 else:
55 self.dataOut.heightList = self.dataOut.data['Elevation']
56
57 attrs = ['units', 'elevation', 'azimuth', 'max_range', 'latitude', 'longitude']
58 meta = {}
59
60 for attr in attrs:
61 meta[attr] = getattr(self.dataOut, attr)
62
63 meta['mode'] = mode
64 self.dataOut.meta = meta No newline at end of file
Show file before | Show file after | Hide comments
@@ -1,1342 +1,1342
1 1 '''
2 2 Created on September , 2012
3 3 @author:
4 4 '''
5 5
6 6 import sys
7 7 import ast
8 8 import datetime
9 9 import traceback
10 10 import math
11 11 import time
12 12 from multiprocessing import Process, cpu_count
13 13
14 14 from xml.etree.ElementTree import ElementTree, Element, SubElement, tostring
15 15 from xml.dom import minidom
16 16
17 17 import schainpy
18 18 from schainpy.admin import Alarm, SchainWarning
19 19 from schainpy.model import *
20 20 from schainpy.utils import log
21 21
22 22 DTYPES = {
23 23 'Voltage': '.r',
24 24 'Spectra': '.pdata'
25 25 }
26 26
27 27
28 28 def MPProject(project, n=cpu_count()):
29 29 '''
30 30 Project wrapper to run schain in n processes
31 31 '''
32 32
33 33 rconf = project.getReadUnitObj()
34 34 op = rconf.getOperationObj('run')
35 35 dt1 = op.getParameterValue('startDate')
36 36 dt2 = op.getParameterValue('endDate')
37 37 tm1 = op.getParameterValue('startTime')
38 38 tm2 = op.getParameterValue('endTime')
39 39 days = (dt2 - dt1).days
40 40
41 41 for day in range(days + 1):
42 42 skip = 0
43 43 cursor = 0
44 44 processes = []
45 45 dt = dt1 + datetime.timedelta(day)
46 46 dt_str = dt.strftime('%Y/%m/%d')
47 47 reader = JRODataReader()
48 48 paths, files = reader.searchFilesOffLine(path=rconf.path,
49 49 startDate=dt,
50 50 endDate=dt,
51 51 startTime=tm1,
52 52 endTime=tm2,
53 53 ext=DTYPES[rconf.datatype])
54 54 nFiles = len(files)
55 55 if nFiles == 0:
56 56 continue
57 57 skip = int(math.ceil(nFiles / n))
58 58 while nFiles > cursor * skip:
59 59 rconf.update(startDate=dt_str, endDate=dt_str, cursor=cursor,
60 60 skip=skip)
61 61 p = project.clone()
62 62 p.start()
63 63 processes.append(p)
64 64 cursor += 1
65 65
66 66 def beforeExit(exctype, value, trace):
67 67 for process in processes:
68 68 process.terminate()
69 69 process.join()
70 70 print traceback.print_tb(trace)
71 71
72 72 sys.excepthook = beforeExit
73 73
74 74 for process in processes:
75 75 process.join()
76 76 process.terminate()
77 77
78 78 time.sleep(3)
79 79
80 80
81 81 class ParameterConf():
82 82
83 83 id = None
84 84 name = None
85 85 value = None
86 86 format = None
87 87
88 88 __formated_value = None
89 89
90 90 ELEMENTNAME = 'Parameter'
91 91
92 92 def __init__(self):
93 93
94 94 self.format = 'str'
95 95
96 96 def getElementName(self):
97 97
98 98 return self.ELEMENTNAME
99 99
100 100 def getValue(self):
101 101
102 102 value = self.value
103 103 format = self.format
104 104
105 105 if self.__formated_value != None:
106 106
107 107 return self.__formated_value
108 108
109 109 if format == 'obj':
110 110 return value
111 111
112 112 if format == 'str':
113 113 self.__formated_value = str(value)
114 114 return self.__formated_value
115 115
116 116 if value == '':
117 117 raise ValueError, '%s: This parameter value is empty' % self.name
118 118
119 119 if format == 'list':
120 120 strList = value.split(',')
121 121
122 122 self.__formated_value = strList
123 123
124 124 return self.__formated_value
125 125
126 126 if format == 'intlist':
127 127 '''
128 128 Example:
129 129 value = (0,1,2)
130 130 '''
131 131
132 132 new_value = ast.literal_eval(value)
133 133
134 134 if type(new_value) not in (tuple, list):
135 135 new_value = [int(new_value)]
136 136
137 137 self.__formated_value = new_value
138 138
139 139 return self.__formated_value
140 140
141 141 if format == 'floatlist':
142 142 '''
143 143 Example:
144 144 value = (0.5, 1.4, 2.7)
145 145 '''
146 146
147 147 new_value = ast.literal_eval(value)
148 148
149 149 if type(new_value) not in (tuple, list):
150 150 new_value = [float(new_value)]
151 151
152 152 self.__formated_value = new_value
153 153
154 154 return self.__formated_value
155 155
156 156 if format == 'date':
157 157 strList = value.split('/')
158 158 intList = [int(x) for x in strList]
159 159 date = datetime.date(intList[0], intList[1], intList[2])
160 160
161 161 self.__formated_value = date
162 162
163 163 return self.__formated_value
164 164
165 165 if format == 'time':
166 166 strList = value.split(':')
167 167 intList = [int(x) for x in strList]
168 168 time = datetime.time(intList[0], intList[1], intList[2])
169 169
170 170 self.__formated_value = time
171 171
172 172 return self.__formated_value
173 173
174 174 if format == 'pairslist':
175 175 '''
176 176 Example:
177 177 value = (0,1),(1,2)
178 178 '''
179 179
180 180 new_value = ast.literal_eval(value)
181 181
182 182 if type(new_value) not in (tuple, list):
183 183 raise ValueError, '%s has to be a tuple or list of pairs' % value
184 184
185 185 if type(new_value[0]) not in (tuple, list):
186 186 if len(new_value) != 2:
187 187 raise ValueError, '%s has to be a tuple or list of pairs' % value
188 188 new_value = [new_value]
189 189
190 190 for thisPair in new_value:
191 191 if len(thisPair) != 2:
192 192 raise ValueError, '%s has to be a tuple or list of pairs' % value
193 193
194 194 self.__formated_value = new_value
195 195
196 196 return self.__formated_value
197 197
198 198 if format == 'multilist':
199 199 '''
200 200 Example:
201 201 value = (0,1,2),(3,4,5)
202 202 '''
203 203 multiList = ast.literal_eval(value)
204 204
205 205 if type(multiList[0]) == int:
206 206 multiList = ast.literal_eval('(' + value + ')')
207 207
208 208 self.__formated_value = multiList
209 209
210 210 return self.__formated_value
211 211
212 212 if format == 'bool':
213 213 value = int(value)
214 214
215 215 if format == 'int':
216 216 value = float(value)
217 217
218 218 format_func = eval(format)
219 219
220 220 self.__formated_value = format_func(value)
221 221
222 222 return self.__formated_value
223 223
224 224 def updateId(self, new_id):
225 225
226 226 self.id = str(new_id)
227 227
228 228 def setup(self, id, name, value, format='str'):
229 229 self.id = str(id)
230 230 self.name = name
231 231 if format == 'obj':
232 232 self.value = value
233 233 else:
234 234 self.value = str(value)
235 235 self.format = str.lower(format)
236 236
237 237 self.getValue()
238 238
239 239 return 1
240 240
241 241 def update(self, name, value, format='str'):
242 242
243 243 self.name = name
244 244 self.value = str(value)
245 245 self.format = format
246 246
247 247 def makeXml(self, opElement):
248 248 if self.name not in ('queue',):
249 249 parmElement = SubElement(opElement, self.ELEMENTNAME)
250 250 parmElement.set('id', str(self.id))
251 251 parmElement.set('name', self.name)
252 252 parmElement.set('value', self.value)
253 253 parmElement.set('format', self.format)
254 254
255 255 def readXml(self, parmElement):
256 256
257 257 self.id = parmElement.get('id')
258 258 self.name = parmElement.get('name')
259 259 self.value = parmElement.get('value')
260 260 self.format = str.lower(parmElement.get('format'))
261 261
262 262 # Compatible with old signal chain version
263 263 if self.format == 'int' and self.name == 'idfigure':
264 264 self.name = 'id'
265 265
266 266 def printattr(self):
267 267
268 268 print 'Parameter[%s]: name = %s, value = %s, format = %s' % (self.id, self.name, self.value, self.format)
269 269
270 270
271 271 class OperationConf():
272 272
273 273 id = None
274 274 name = None
275 275 priority = None
276 276 type = None
277 277
278 278 parmConfObjList = []
279 279
280 280 ELEMENTNAME = 'Operation'
281 281
282 282 def __init__(self):
283 283
284 284 self.id = '0'
285 285 self.name = None
286 286 self.priority = None
287 287 self.type = 'self'
288 288
289 289 def __getNewId(self):
290 290
291 291 return int(self.id) * 10 + len(self.parmConfObjList) + 1
292 292
293 293 def updateId(self, new_id):
294 294
295 295 self.id = str(new_id)
296 296
297 297 n = 1
298 298 for parmObj in self.parmConfObjList:
299 299
300 300 idParm = str(int(new_id) * 10 + n)
301 301 parmObj.updateId(idParm)
302 302
303 303 n += 1
304 304
305 305 def getElementName(self):
306 306
307 307 return self.ELEMENTNAME
308 308
309 309 def getParameterObjList(self):
310 310
311 311 return self.parmConfObjList
312 312
313 313 def getParameterObj(self, parameterName):
314 314
315 315 for parmConfObj in self.parmConfObjList:
316 316
317 317 if parmConfObj.name != parameterName:
318 318 continue
319 319
320 320 return parmConfObj
321 321
322 322 return None
323 323
324 324 def getParameterObjfromValue(self, parameterValue):
325 325
326 326 for parmConfObj in self.parmConfObjList:
327 327
328 328 if parmConfObj.getValue() != parameterValue:
329 329 continue
330 330
331 331 return parmConfObj.getValue()
332 332
333 333 return None
334 334
335 335 def getParameterValue(self, parameterName):
336 336
337 337 parameterObj = self.getParameterObj(parameterName)
338 338
339 339 # if not parameterObj:
340 340 # return None
341 341
342 342 value = parameterObj.getValue()
343 343
344 344 return value
345 345
346 346 def getKwargs(self):
347 347
348 348 kwargs = {}
349 349
350 350 for parmConfObj in self.parmConfObjList:
351 351 if self.name == 'run' and parmConfObj.name == 'datatype':
352 352 continue
353 353
354 354 kwargs[parmConfObj.name] = parmConfObj.getValue()
355 355
356 356 return kwargs
357 357
358 358 def setup(self, id, name, priority, type):
359 359
360 360 self.id = str(id)
361 361 self.name = name
362 362 self.type = type
363 363 self.priority = priority
364 364
365 365 self.parmConfObjList = []
366 366
367 367 def removeParameters(self):
368 368
369 369 for obj in self.parmConfObjList:
370 370 del obj
371 371
372 372 self.parmConfObjList = []
373 373
374 374 def addParameter(self, name, value, format='str'):
375 375
376 376 if value is None:
377 377 return None
378 378 id = self.__getNewId()
379 379
380 380 parmConfObj = ParameterConf()
381 381 if not parmConfObj.setup(id, name, value, format):
382 382 return None
383 383
384 384 self.parmConfObjList.append(parmConfObj)
385 385
386 386 return parmConfObj
387 387
388 388 def changeParameter(self, name, value, format='str'):
389 389
390 390 parmConfObj = self.getParameterObj(name)
391 391 parmConfObj.update(name, value, format)
392 392
393 393 return parmConfObj
394 394
395 395 def makeXml(self, procUnitElement):
396 396
397 397 opElement = SubElement(procUnitElement, self.ELEMENTNAME)
398 398 opElement.set('id', str(self.id))
399 399 opElement.set('name', self.name)
400 400 opElement.set('type', self.type)
401 401 opElement.set('priority', str(self.priority))
402 402
403 403 for parmConfObj in self.parmConfObjList:
404 404 parmConfObj.makeXml(opElement)
405 405
406 406 def readXml(self, opElement):
407 407
408 408 self.id = opElement.get('id')
409 409 self.name = opElement.get('name')
410 410 self.type = opElement.get('type')
411 411 self.priority = opElement.get('priority')
412 412
413 413 # Compatible with old signal chain version
414 414 # Use of 'run' method instead 'init'
415 415 if self.type == 'self' and self.name == 'init':
416 416 self.name = 'run'
417 417
418 418 self.parmConfObjList = []
419 419
420 420 parmElementList = opElement.iter(ParameterConf().getElementName())
421 421
422 422 for parmElement in parmElementList:
423 423 parmConfObj = ParameterConf()
424 424 parmConfObj.readXml(parmElement)
425 425
426 426 # Compatible with old signal chain version
427 427 # If an 'plot' OPERATION is found, changes name operation by the value of its type PARAMETER
428 428 if self.type != 'self' and self.name == 'Plot':
429 429 if parmConfObj.format == 'str' and parmConfObj.name == 'type':
430 430 self.name = parmConfObj.value
431 431 continue
432 432
433 433 self.parmConfObjList.append(parmConfObj)
434 434
435 435 def printattr(self):
436 436
437 437 print '%s[%s]: name = %s, type = %s, priority = %s' % (self.ELEMENTNAME,
438 438 self.id,
439 439 self.name,
440 440 self.type,
441 441 self.priority)
442 442
443 443 for parmConfObj in self.parmConfObjList:
444 444 parmConfObj.printattr()
445 445
446 446 def createObject(self, plotter_queue=None):
447 447
448 448 if self.type == 'self':
449 449 raise ValueError, 'This operation type cannot be created'
450 450
451 451 if self.type == 'plotter':
452 452 if not plotter_queue:
453 453 raise ValueError, 'plotter_queue is not defined. Use:\nmyProject = Project()\nmyProject.setPlotterQueue(plotter_queue)'
454 454
455 455 opObj = Plotter(self.name, plotter_queue)
456 456
457 457 if self.type == 'external' or self.type == 'other':
458 458
459 459 className = eval(self.name)
460 460 kwargs = self.getKwargs()
461 461
462 462 opObj = className(**kwargs)
463 463
464 464 return opObj
465 465
466 466
467 467 class ProcUnitConf():
468 468
469 469 id = None
470 470 name = None
471 471 datatype = None
472 472 inputId = None
473 473 parentId = None
474 474
475 475 opConfObjList = []
476 476
477 477 procUnitObj = None
478 478 opObjList = []
479 479
480 480 ELEMENTNAME = 'ProcUnit'
481 481
482 482 def __init__(self):
483 483
484 484 self.id = None
485 485 self.datatype = None
486 486 self.name = None
487 487 self.inputId = None
488 488
489 489 self.opConfObjList = []
490 490
491 491 self.procUnitObj = None
492 492 self.opObjDict = {}
493 493
494 494 def __getPriority(self):
495 495
496 496 return len(self.opConfObjList) + 1
497 497
498 498 def __getNewId(self):
499 499
500 500 return int(self.id) * 10 + len(self.opConfObjList) + 1
501 501
502 502 def getElementName(self):
503 503
504 504 return self.ELEMENTNAME
505 505
506 506 def getId(self):
507 507
508 508 return self.id
509 509
510 510 def updateId(self, new_id, parentId=parentId):
511 511
512 512 new_id = int(parentId) * 10 + (int(self.id) % 10)
513 513 new_inputId = int(parentId) * 10 + (int(self.inputId) % 10)
514 514
515 515 # If this proc unit has not inputs
516 516 if self.inputId == '0':
517 517 new_inputId = 0
518 518
519 519 n = 1
520 520 for opConfObj in self.opConfObjList:
521 521
522 522 idOp = str(int(new_id) * 10 + n)
523 523 opConfObj.updateId(idOp)
524 524
525 525 n += 1
526 526
527 527 self.parentId = str(parentId)
528 528 self.id = str(new_id)
529 529 self.inputId = str(new_inputId)
530 530
531 531 def getInputId(self):
532 532
533 533 return self.inputId
534 534
535 535 def getOperationObjList(self):
536 536
537 537 return self.opConfObjList
538 538
539 539 def getOperationObj(self, name=None):
540 540
541 541 for opConfObj in self.opConfObjList:
542 542
543 543 if opConfObj.name != name:
544 544 continue
545 545
546 546 return opConfObj
547 547
548 548 return None
549 549
550 550 def getOpObjfromParamValue(self, value=None):
551 551
552 552 for opConfObj in self.opConfObjList:
553 553 if opConfObj.getParameterObjfromValue(parameterValue=value) != value:
554 554 continue
555 555 return opConfObj
556 556 return None
557 557
558 558 def getProcUnitObj(self):
559 559
560 560 return self.procUnitObj
561 561
562 562 def setup(self, id, name, datatype, inputId, parentId=None):
563 563
564 564 # Compatible with old signal chain version
565 565 if datatype == None and name == None:
566 566 raise ValueError, 'datatype or name should be defined'
567 567
568 568 if name == None:
569 569 if 'Proc' in datatype:
570 570 name = datatype
571 571 else:
572 572 name = '%sProc' % (datatype)
573 573
574 574 if datatype == None:
575 575 datatype = name.replace('Proc', '')
576 576
577 577 self.id = str(id)
578 578 self.name = name
579 579 self.datatype = datatype
580 580 self.inputId = inputId
581 581 self.parentId = parentId
582 582
583 583 self.opConfObjList = []
584 584
585 585 self.addOperation(name='run', optype='self')
586 586
587 587 def removeOperations(self):
588 588
589 589 for obj in self.opConfObjList:
590 590 del obj
591 591
592 592 self.opConfObjList = []
593 593 self.addOperation(name='run')
594 594
595 595 def addParameter(self, **kwargs):
596 596 '''
597 597 Add parameters to 'run' operation
598 598 '''
599 599 opObj = self.opConfObjList[0]
600 600
601 601 opObj.addParameter(**kwargs)
602 602
603 603 return opObj
604 604
605 605 def addOperation(self, name, optype='self'):
606 606
607 607 id = self.__getNewId()
608 608 priority = self.__getPriority()
609 609
610 610 opConfObj = OperationConf()
611 611 opConfObj.setup(id, name=name, priority=priority, type=optype)
612 612
613 613 self.opConfObjList.append(opConfObj)
614 614
615 615 return opConfObj
616 616
617 617 def makeXml(self, projectElement):
618 618
619 619 procUnitElement = SubElement(projectElement, self.ELEMENTNAME)
620 620 procUnitElement.set('id', str(self.id))
621 621 procUnitElement.set('name', self.name)
622 622 procUnitElement.set('datatype', self.datatype)
623 623 procUnitElement.set('inputId', str(self.inputId))
624 624
625 625 for opConfObj in self.opConfObjList:
626 626 opConfObj.makeXml(procUnitElement)
627 627
628 628 def readXml(self, upElement):
629 629
630 630 self.id = upElement.get('id')
631 631 self.name = upElement.get('name')
632 632 self.datatype = upElement.get('datatype')
633 633 self.inputId = upElement.get('inputId')
634 634
635 635 if self.ELEMENTNAME == 'ReadUnit':
636 636 self.datatype = self.datatype.replace('Reader', '')
637 637
638 638 if self.ELEMENTNAME == 'ProcUnit':
639 639 self.datatype = self.datatype.replace('Proc', '')
640 640
641 641 if self.inputId == 'None':
642 642 self.inputId = '0'
643 643
644 644 self.opConfObjList = []
645 645
646 646 opElementList = upElement.iter(OperationConf().getElementName())
647 647
648 648 for opElement in opElementList:
649 649 opConfObj = OperationConf()
650 650 opConfObj.readXml(opElement)
651 651 self.opConfObjList.append(opConfObj)
652 652
653 653 def printattr(self):
654 654
655 655 print '%s[%s]: name = %s, datatype = %s, inputId = %s' % (self.ELEMENTNAME,
656 656 self.id,
657 657 self.name,
658 658 self.datatype,
659 659 self.inputId)
660 660
661 661 for opConfObj in self.opConfObjList:
662 662 opConfObj.printattr()
663 663
664 664 def getKwargs(self):
665 665
666 666 opObj = self.opConfObjList[0]
667 667 kwargs = opObj.getKwargs()
668 668
669 669 return kwargs
670 670
671 671 def createObjects(self, plotter_queue=None):
672 672
673 673 className = eval(self.name)
674 674 kwargs = self.getKwargs()
675 675 procUnitObj = className(**kwargs)
676 676
677 677 for opConfObj in self.opConfObjList:
678 678
679 679 if opConfObj.type == 'self' and self.name == 'run':
680 680 continue
681 681 elif opConfObj.type == 'self':
682 682 procUnitObj.addOperationKwargs(
683 683 opConfObj.id, **opConfObj.getKwargs())
684 684 continue
685 685
686 686 opObj = opConfObj.createObject(plotter_queue)
687 687
688 688 self.opObjDict[opConfObj.id] = opObj
689 689
690 690 procUnitObj.addOperation(opObj, opConfObj.id)
691 691
692 692 self.procUnitObj = procUnitObj
693 693
694 694 return procUnitObj
695 695
696 696 def run(self):
697 697
698 698 is_ok = False
699 699
700 700 for opConfObj in self.opConfObjList:
701 701
702 702 kwargs = {}
703 703 for parmConfObj in opConfObj.getParameterObjList():
704 704 if opConfObj.name == 'run' and parmConfObj.name == 'datatype':
705 705 continue
706 706
707 707 kwargs[parmConfObj.name] = parmConfObj.getValue()
708 708
709 709 sts = self.procUnitObj.call(opType=opConfObj.type,
710 710 opName=opConfObj.name,
711 711 opId=opConfObj.id)
712 712
713 713 is_ok = is_ok or sts
714 714
715 715 return is_ok
716 716
717 717 def close(self):
718 718
719 719 for opConfObj in self.opConfObjList:
720 720 if opConfObj.type == 'self':
721 721 continue
722 722
723 723 opObj = self.procUnitObj.getOperationObj(opConfObj.id)
724 724 opObj.close()
725 725
726 726 self.procUnitObj.close()
727 727
728 728 return
729 729
730 730
731 731 class ReadUnitConf(ProcUnitConf):
732 732
733 733 path = None
734 734 startDate = None
735 735 endDate = None
736 736 startTime = None
737 737 endTime = None
738 738
739 739 ELEMENTNAME = 'ReadUnit'
740 740
741 741 def __init__(self):
742 742
743 743 self.id = None
744 744 self.datatype = None
745 745 self.name = None
746 746 self.inputId = None
747 747
748 748 self.parentId = None
749 749
750 750 self.opConfObjList = []
751 751 self.opObjList = []
752 752
753 753 def getElementName(self):
754 754
755 755 return self.ELEMENTNAME
756 756
757 757 def setup(self, id, name, datatype, path='', startDate='', endDate='',
758 758 startTime='', endTime='', parentId=None, server=None, **kwargs):
759 759
760 760 # Compatible with old signal chain version
761 761 if datatype == None and name == None:
762 762 raise ValueError, 'datatype or name should be defined'
763 763 if name == None:
764 764 if 'Reader' in datatype:
765 765 name = datatype
766 766 datatype = name.replace('Reader','')
767 767 else:
768 768 name = '{}Reader'.format(datatype)
769 769 if datatype == None:
770 770 if 'Reader' in name:
771 771 datatype = name.replace('Reader','')
772 772 else:
773 773 datatype = name
774 774 name = '{}Reader'.format(name)
775 775
776 776 self.id = id
777 777 self.name = name
778 778 self.datatype = datatype
779 779 if path != '':
780 780 self.path = os.path.abspath(path)
781 781 self.startDate = startDate
782 782 self.endDate = endDate
783 783 self.startTime = startTime
784 784 self.endTime = endTime
785 785 self.inputId = '0'
786 786 self.parentId = parentId
787 787 self.server = server
788 788 self.addRunOperation(**kwargs)
789 789
790 790 def update(self, **kwargs):
791 791
792 792 if 'datatype' in kwargs:
793 793 datatype = kwargs.pop('datatype')
794 794 if 'Reader' in datatype:
795 795 self.name = datatype
796 796 else:
797 797 self.name = '%sReader' % (datatype)
798 798 self.datatype = self.name.replace('Reader', '')
799 799
800 800 attrs = ('path', 'startDate', 'endDate',
801 801 'startTime', 'endTime', 'parentId')
802 802
803 803 for attr in attrs:
804 804 if attr in kwargs:
805 805 setattr(self, attr, kwargs.pop(attr))
806 806
807 807 self.inputId = '0'
808 808 self.updateRunOperation(**kwargs)
809 809
810 810 def removeOperations(self):
811 811
812 812 for obj in self.opConfObjList:
813 813 del obj
814 814
815 815 self.opConfObjList = []
816 816
817 817 def addRunOperation(self, **kwargs):
818 818
819 819 opObj = self.addOperation(name='run', optype='self')
820 820
821 821 if self.server is None:
822 822 opObj.addParameter(
823 823 name='datatype', value=self.datatype, format='str')
824 824 opObj.addParameter(name='path', value=self.path, format='str')
825 825 opObj.addParameter(
826 826 name='startDate', value=self.startDate, format='date')
827 827 opObj.addParameter(
828 828 name='endDate', value=self.endDate, format='date')
829 829 opObj.addParameter(
830 830 name='startTime', value=self.startTime, format='time')
831 831 opObj.addParameter(
832 832 name='endTime', value=self.endTime, format='time')
833 833
834 834 for key, value in kwargs.items():
835 835 opObj.addParameter(name=key, value=value,
836 836 format=type(value).__name__)
837 837 else:
838 838 opObj.addParameter(name='server', value=self.server, format='str')
839 839
840 840 return opObj
841 841
842 842 def updateRunOperation(self, **kwargs):
843 843
844 844 opObj = self.getOperationObj(name='run')
845 845 opObj.removeParameters()
846 846
847 847 opObj.addParameter(name='datatype', value=self.datatype, format='str')
848 848 opObj.addParameter(name='path', value=self.path, format='str')
849 849 opObj.addParameter(
850 850 name='startDate', value=self.startDate, format='date')
851 851 opObj.addParameter(name='endDate', value=self.endDate, format='date')
852 852 opObj.addParameter(
853 853 name='startTime', value=self.startTime, format='time')
854 854 opObj.addParameter(name='endTime', value=self.endTime, format='time')
855 855
856 856 for key, value in kwargs.items():
857 857 opObj.addParameter(name=key, value=value,
858 858 format=type(value).__name__)
859 859
860 860 return opObj
861 861
862 862 def readXml(self, upElement):
863 863
864 864 self.id = upElement.get('id')
865 865 self.name = upElement.get('name')
866 866 self.datatype = upElement.get('datatype')
867 867 self.inputId = upElement.get('inputId')
868 868
869 869 if self.ELEMENTNAME == 'ReadUnit':
870 870 self.datatype = self.datatype.replace('Reader', '')
871 871
872 872 if self.inputId == 'None':
873 873 self.inputId = '0'
874 874
875 875 self.opConfObjList = []
876 876
877 877 opElementList = upElement.iter(OperationConf().getElementName())
878 878
879 879 for opElement in opElementList:
880 880 opConfObj = OperationConf()
881 881 opConfObj.readXml(opElement)
882 882 self.opConfObjList.append(opConfObj)
883 883
884 884 if opConfObj.name == 'run':
885 885 self.path = opConfObj.getParameterValue('path')
886 886 self.startDate = opConfObj.getParameterValue('startDate')
887 887 self.endDate = opConfObj.getParameterValue('endDate')
888 888 self.startTime = opConfObj.getParameterValue('startTime')
889 889 self.endTime = opConfObj.getParameterValue('endTime')
890 890
891 891
892 892 class Project(Process):
893 893
894 894 id = None
895 895 # name = None
896 896 description = None
897 897 filename = None
898 898
899 899 procUnitConfObjDict = None
900 900
901 901 ELEMENTNAME = 'Project'
902 902
903 903 plotterQueue = None
904 904
905 905 def __init__(self, plotter_queue=None):
906 906
907 907 Process.__init__(self)
908 908 self.id = None
909 909 self.description = None
910 910 self.email = None
911 911 self.alarm = None
912 912 self.plotterQueue = plotter_queue
913 913 self.procUnitConfObjDict = {}
914 914
915 915 def __getNewId(self):
916 916
917 917 idList = self.procUnitConfObjDict.keys()
918 918
919 919 id = int(self.id) * 10
920 920
921 921 while True:
922 922 id += 1
923 923
924 924 if str(id) in idList:
925 925 continue
926 926
927 927 break
928 928
929 929 return str(id)
930 930
931 931 def getElementName(self):
932 932
933 933 return self.ELEMENTNAME
934 934
935 935 def getId(self):
936 936
937 937 return self.id
938 938
939 939 def updateId(self, new_id):
940 940
941 941 self.id = str(new_id)
942 942
943 943 keyList = self.procUnitConfObjDict.keys()
944 944 keyList.sort()
945 945
946 946 n = 1
947 947 newProcUnitConfObjDict = {}
948 948
949 949 for procKey in keyList:
950 950
951 951 procUnitConfObj = self.procUnitConfObjDict[procKey]
952 952 idProcUnit = str(int(self.id) * 10 + n)
953 953 procUnitConfObj.updateId(idProcUnit, parentId=self.id)
954 954 newProcUnitConfObjDict[idProcUnit] = procUnitConfObj
955 955 n += 1
956 956
957 957 self.procUnitConfObjDict = newProcUnitConfObjDict
958 958
959 959 def setup(self, id, name='', description='', email=None, alarm=[]):
960 960
961 961 print
962 962 print '*' * 60
963 963 print ' Starting SIGNAL CHAIN PROCESSING v%s ' % schainpy.__version__
964 964 print '*' * 60
965 965 print
966 966 self.id = str(id)
967 967 self.description = description
968 968 self.email = email
969 969 self.alarm = alarm
970 970
971 971 def update(self, **kwargs):
972 972
973 973 for key, value in kwargs.items():
974 974 setattr(self, key, value)
975 975
976 976 def clone(self):
977 977
978 978 p = Project()
979 979 p.procUnitConfObjDict = self.procUnitConfObjDict
980 980 return p
981 981
982 982 def addReadUnit(self, id=None, datatype=None, name=None, **kwargs):
983 983
984 984 if id is None:
985 985 idReadUnit = self.__getNewId()
986 986 else:
987 987 idReadUnit = str(id)
988 988
989 989 readUnitConfObj = ReadUnitConf()
990 990 readUnitConfObj.setup(idReadUnit, name, datatype,
991 991 parentId=self.id, **kwargs)
992 992
993 993 self.procUnitConfObjDict[readUnitConfObj.getId()] = readUnitConfObj
994 994
995 995 return readUnitConfObj
996 996
997 997 def addProcUnit(self, inputId='0', datatype=None, name=None):
998 998
999 999 idProcUnit = self.__getNewId()
1000 1000
1001 1001 procUnitConfObj = ProcUnitConf()
1002 1002 procUnitConfObj.setup(idProcUnit, name, datatype,
1003 1003 inputId, parentId=self.id)
1004 1004
1005 1005 self.procUnitConfObjDict[procUnitConfObj.getId()] = procUnitConfObj
1006 1006
1007 1007 return procUnitConfObj
1008 1008
1009 1009 def removeProcUnit(self, id):
1010 1010
1011 1011 if id in self.procUnitConfObjDict.keys():
1012 1012 self.procUnitConfObjDict.pop(id)
1013 1013
1014 1014 def getReadUnitId(self):
1015 1015
1016 1016 readUnitConfObj = self.getReadUnitObj()
1017 1017
1018 1018 return readUnitConfObj.id
1019 1019
1020 1020 def getReadUnitObj(self):
1021 1021
1022 1022 for obj in self.procUnitConfObjDict.values():
1023 1023 if obj.getElementName() == 'ReadUnit':
1024 1024 return obj
1025 1025
1026 1026 return None
1027 1027
1028 1028 def getProcUnitObj(self, id=None, name=None):
1029 1029
1030 1030 if id != None:
1031 1031 return self.procUnitConfObjDict[id]
1032 1032
1033 1033 if name != None:
1034 1034 return self.getProcUnitObjByName(name)
1035 1035
1036 1036 return None
1037 1037
1038 1038 def getProcUnitObjByName(self, name):
1039 1039
1040 1040 for obj in self.procUnitConfObjDict.values():
1041 1041 if obj.name == name:
1042 1042 return obj
1043 1043
1044 1044 return None
1045 1045
1046 1046 def procUnitItems(self):
1047 1047
1048 1048 return self.procUnitConfObjDict.items()
1049 1049
1050 1050 def makeXml(self):
1051 1051
1052 1052 projectElement = Element('Project')
1053 1053 projectElement.set('id', str(self.id))
1054 1054 projectElement.set('name', self.name)
1055 1055 projectElement.set('description', self.description)
1056 1056
1057 1057 for procUnitConfObj in self.procUnitConfObjDict.values():
1058 1058 procUnitConfObj.makeXml(projectElement)
1059 1059
1060 1060 self.projectElement = projectElement
1061 1061
1062 1062 def writeXml(self, filename=None):
1063 1063
1064 1064 if filename == None:
1065 1065 if self.filename:
1066 1066 filename = self.filename
1067 1067 else:
1068 1068 filename = 'schain.xml'
1069 1069
1070 1070 if not filename:
1071 1071 print 'filename has not been defined. Use setFilename(filename) for do it.'
1072 1072 return 0
1073 1073
1074 1074 abs_file = os.path.abspath(filename)
1075 1075
1076 1076 if not os.access(os.path.dirname(abs_file), os.W_OK):
1077 1077 print 'No write permission on %s' % os.path.dirname(abs_file)
1078 1078 return 0
1079 1079
1080 1080 if os.path.isfile(abs_file) and not(os.access(abs_file, os.W_OK)):
1081 1081 print 'File %s already exists and it could not be overwriten' % abs_file
1082 1082 return 0
1083 1083
1084 1084 self.makeXml()
1085 1085
1086 1086 ElementTree(self.projectElement).write(abs_file, method='xml')
1087 1087
1088 1088 self.filename = abs_file
1089 1089
1090 1090 return 1
1091 1091
1092 1092 def readXml(self, filename=None):
1093 1093
1094 1094 if not filename:
1095 1095 print 'filename is not defined'
1096 1096 return 0
1097 1097
1098 1098 abs_file = os.path.abspath(filename)
1099 1099
1100 1100 if not os.path.isfile(abs_file):
1101 1101 print '%s file does not exist' % abs_file
1102 1102 return 0
1103 1103
1104 1104 self.projectElement = None
1105 1105 self.procUnitConfObjDict = {}
1106 1106
1107 1107 try:
1108 1108 self.projectElement = ElementTree().parse(abs_file)
1109 1109 except:
1110 1110 print 'Error reading %s, verify file format' % filename
1111 1111 return 0
1112 1112
1113 1113 self.project = self.projectElement.tag
1114 1114
1115 1115 self.id = self.projectElement.get('id')
1116 1116 self.name = self.projectElement.get('name')
1117 1117 self.description = self.projectElement.get('description')
1118 1118
1119 1119 readUnitElementList = self.projectElement.iter(
1120 1120 ReadUnitConf().getElementName())
1121 1121
1122 1122 for readUnitElement in readUnitElementList:
1123 1123 readUnitConfObj = ReadUnitConf()
1124 1124 readUnitConfObj.readXml(readUnitElement)
1125 1125
1126 1126 if readUnitConfObj.parentId == None:
1127 1127 readUnitConfObj.parentId = self.id
1128 1128
1129 1129 self.procUnitConfObjDict[readUnitConfObj.getId()] = readUnitConfObj
1130 1130
1131 1131 procUnitElementList = self.projectElement.iter(
1132 1132 ProcUnitConf().getElementName())
1133 1133
1134 1134 for procUnitElement in procUnitElementList:
1135 1135 procUnitConfObj = ProcUnitConf()
1136 1136 procUnitConfObj.readXml(procUnitElement)
1137 1137
1138 1138 if procUnitConfObj.parentId == None:
1139 1139 procUnitConfObj.parentId = self.id
1140 1140
1141 1141 self.procUnitConfObjDict[procUnitConfObj.getId()] = procUnitConfObj
1142 1142
1143 1143 self.filename = abs_file
1144 1144
1145 1145 return 1
1146 1146
1147 1147 def printattr(self):
1148 1148
1149 1149 print 'Project[%s]: name = %s, description = %s' % (self.id,
1150 1150 self.name,
1151 1151 self.description)
1152 1152
1153 1153 for procUnitConfObj in self.procUnitConfObjDict.values():
1154 1154 procUnitConfObj.printattr()
1155 1155
1156 1156 def createObjects(self):
1157 1157
1158 1158 for procUnitConfObj in self.procUnitConfObjDict.values():
1159 1159 procUnitConfObj.createObjects(self.plotterQueue)
1160 1160
1161 1161 def __connect(self, objIN, thisObj):
1162 1162
1163 1163 thisObj.setInput(objIN.getOutputObj())
1164 1164
1165 1165 def connectObjects(self):
1166 1166
1167 1167 for thisPUConfObj in self.procUnitConfObjDict.values():
1168 1168
1169 1169 inputId = thisPUConfObj.getInputId()
1170 1170
1171 1171 if int(inputId) == 0:
1172 1172 continue
1173 1173
1174 1174 # Get input object
1175 1175 puConfINObj = self.procUnitConfObjDict[inputId]
1176 1176 puObjIN = puConfINObj.getProcUnitObj()
1177 1177
1178 1178 # Get current object
1179 1179 thisPUObj = thisPUConfObj.getProcUnitObj()
1180 1180
1181 1181 self.__connect(puObjIN, thisPUObj)
1182 1182
1183 1183 def __handleError(self, procUnitConfObj, modes=None, stdout=True):
1184 1184
1185 1185 import socket
1186 1186
1187 1187 if modes is None:
1188 1188 modes = self.alarm
1189 1189
1190 1190 if not self.alarm:
1191 1191 modes = []
1192 1192
1193 1193 err = traceback.format_exception(sys.exc_info()[0],
1194 1194 sys.exc_info()[1],
1195 1195 sys.exc_info()[2])
1196 1196
1197 1197 log.error('{}'.format(err[-1]), procUnitConfObj.name)
1198 1198
1199 1199 message = ''.join(err)
1200 1200
1201 1201 if stdout:
1202 1202 sys.stderr.write(message)
1203 1203
1204 1204 subject = 'SChain v%s: Error running %s\n' % (
1205 1205 schainpy.__version__, procUnitConfObj.name)
1206 1206
1207 1207 subtitle = '%s: %s\n' % (
1208 1208 procUnitConfObj.getElementName(), procUnitConfObj.name)
1209 1209 subtitle += 'Hostname: %s\n' % socket.gethostbyname(
1210 1210 socket.gethostname())
1211 1211 subtitle += 'Working directory: %s\n' % os.path.abspath('./')
1212 1212 subtitle += 'Configuration file: %s\n' % self.filename
1213 1213 subtitle += 'Time: %s\n' % str(datetime.datetime.now())
1214 1214
1215 1215 readUnitConfObj = self.getReadUnitObj()
1216 1216 if readUnitConfObj:
1217 1217 subtitle += '\nInput parameters:\n'
1218 1218 subtitle += '[Data path = %s]\n' % readUnitConfObj.path
1219 1219 subtitle += '[Data type = %s]\n' % readUnitConfObj.datatype
1220 1220 subtitle += '[Start date = %s]\n' % readUnitConfObj.startDate
1221 1221 subtitle += '[End date = %s]\n' % readUnitConfObj.endDate
1222 1222 subtitle += '[Start time = %s]\n' % readUnitConfObj.startTime
1223 1223 subtitle += '[End time = %s]\n' % readUnitConfObj.endTime
1224 1224
1225 1225 a = Alarm(
1226 1226 modes=modes,
1227 1227 email=self.email,
1228 1228 message=message,
1229 1229 subject=subject,
1230 1230 subtitle=subtitle,
1231 1231 filename=self.filename
1232 1232 )
1233 1233
1234 1234 return a
1235 1235
1236 1236 def isPaused(self):
1237 1237 return 0
1238 1238
1239 1239 def isStopped(self):
1240 1240 return 0
1241 1241
1242 1242 def runController(self):
1243 1243 '''
1244 1244 returns 0 when this process has been stopped, 1 otherwise
1245 1245 '''
1246 1246
1247 1247 if self.isPaused():
1248 1248 print 'Process suspended'
1249 1249
1250 1250 while True:
1251 1251 time.sleep(0.1)
1252 1252
1253 1253 if not self.isPaused():
1254 1254 break
1255 1255
1256 1256 if self.isStopped():
1257 1257 break
1258 1258
1259 1259 print 'Process reinitialized'
1260 1260
1261 1261 if self.isStopped():
1262 1262 print 'Process stopped'
1263 1263 return 0
1264 1264
1265 1265 return 1
1266 1266
1267 1267 def setFilename(self, filename):
1268 1268
1269 1269 self.filename = filename
1270 1270
1271 1271 def setPlotterQueue(self, plotter_queue):
1272 1272
1273 1273 raise NotImplementedError, 'Use schainpy.controller_api.ControllerThread instead Project class'
1274 1274
1275 1275 def getPlotterQueue(self):
1276 1276
1277 1277 raise NotImplementedError, 'Use schainpy.controller_api.ControllerThread instead Project class'
1278 1278
1279 1279 def useExternalPlotter(self):
1280 1280
1281 1281 raise NotImplementedError, 'Use schainpy.controller_api.ControllerThread instead Project class'
1282 1282
1283 1283 def run(self):
1284 1284
1285 log.success('Starting {}'.format(self.name))
1285 log.success('Starting {}'.format(self.name), tag='')
1286 1286 self.start_time = time.time()
1287 1287 self.createObjects()
1288 1288 self.connectObjects()
1289 1289
1290 1290 keyList = self.procUnitConfObjDict.keys()
1291 1291 keyList.sort()
1292 1292
1293 1293 err = None
1294 1294
1295 1295 while(True):
1296 1296
1297 1297 is_ok = False
1298 1298
1299 1299 for procKey in keyList:
1300 1300
1301 1301 procUnitConfObj = self.procUnitConfObjDict[procKey]
1302 1302
1303 1303 try:
1304 1304 sts = procUnitConfObj.run()
1305 1305 is_ok = is_ok or sts
1306 1306 except SchainWarning:
1307 1307 err = self.__handleError(procUnitConfObj, modes=[2, 3], stdout=False)
1308 1308 is_ok = False
1309 1309 break
1310 1310 except KeyboardInterrupt:
1311 1311 is_ok = False
1312 1312 break
1313 1313 except ValueError, e:
1314 1314 time.sleep(0.5)
1315 1315 err = self.__handleError(procUnitConfObj)
1316 1316 is_ok = False
1317 1317 break
1318 1318 except:
1319 1319 time.sleep(0.5)
1320 1320 err = self.__handleError(procUnitConfObj)
1321 1321 is_ok = False
1322 1322 break
1323 1323
1324 1324 # If every process unit finished so end process
1325 1325 if not(is_ok):
1326 1326 break
1327 1327
1328 1328 if not self.runController():
1329 1329 break
1330 1330
1331 1331 # Closing every process
1332 1332 for procKey in keyList:
1333 1333 procUnitConfObj = self.procUnitConfObjDict[procKey]
1334 1334 procUnitConfObj.close()
1335 1335
1336 1336 if err is not None:
1337 1337 err.start()
1338 1338 # err.join()
1339 1339
1340 1340 log.success('{} finished (time: {}s)'.format(
1341 1341 self.name,
1342 1342 time.time()-self.start_time))
Show file before | Show file after | Hide comments
@@ -1,980 +1,1148
1 1
2 2 import os
3 3 import time
4 4 import glob
5 5 import datetime
6 6 from multiprocessing import Process
7 7
8 8 import zmq
9 9 import numpy
10 10 import matplotlib
11 11 import matplotlib.pyplot as plt
12 from matplotlib.patches import Polygon
12 13 from mpl_toolkits.axes_grid1 import make_axes_locatable
13 14 from matplotlib.ticker import FuncFormatter, LinearLocator, MultipleLocator
14 15
15 16 from schainpy.model.proc.jroproc_base import Operation
16 17 from schainpy.utils import log
17 18
18 19 jet_values = matplotlib.pyplot.get_cmap('jet', 100)(numpy.arange(100))[10:90]
19 20 blu_values = matplotlib.pyplot.get_cmap(
20 21 'seismic_r', 20)(numpy.arange(20))[10:15]
21 22 ncmap = matplotlib.colors.LinearSegmentedColormap.from_list(
22 23 'jro', numpy.vstack((blu_values, jet_values)))
23 24 matplotlib.pyplot.register_cmap(cmap=ncmap)
24 25
25 26 CMAPS = [plt.get_cmap(s) for s in ('jro', 'jet', 'viridis', 'plasma', 'inferno', 'Greys', 'seismic', 'bwr', 'coolwarm')]
26 27
28 EARTH_RADIUS = 6.3710e3
29
30 def ll2xy(lat1, lon1, lat2, lon2):
31
32 p = 0.017453292519943295
33 a = 0.5 - numpy.cos((lat2 - lat1) * p)/2 + numpy.cos(lat1 * p) * numpy.cos(lat2 * p) * (1 - numpy.cos((lon2 - lon1) * p)) / 2
34 r = 12742 * numpy.arcsin(numpy.sqrt(a))
35 theta = numpy.arctan2(numpy.sin((lon2-lon1)*p)*numpy.cos(lat2*p), numpy.cos(lat1*p)*numpy.sin(lat2*p)-numpy.sin(lat1*p)*numpy.cos(lat2*p)*numpy.cos((lon2-lon1)*p))
36 theta = -theta + numpy.pi/2
37 return r*numpy.cos(theta), r*numpy.sin(theta)
38
39 def km2deg(km):
40 '''
41 Convert distance in km to degrees
42 '''
43
44 return numpy.rad2deg(km/EARTH_RADIUS)
27 45
28 46 def figpause(interval):
29 47 backend = plt.rcParams['backend']
30 48 if backend in matplotlib.rcsetup.interactive_bk:
31 49 figManager = matplotlib._pylab_helpers.Gcf.get_active()
32 50 if figManager is not None:
33 51 canvas = figManager.canvas
34 52 if canvas.figure.stale:
35 53 canvas.draw()
36 54 try:
37 55 canvas.start_event_loop(interval)
38 56 except:
39 57 pass
40 58 return
41 59
42 60 def popup(message):
43 61 '''
44 62 '''
45 63
46 64 fig = plt.figure(figsize=(12, 8), facecolor='r')
47 65 text = '\n'.join([s.strip() for s in message.split(':')])
48 66 fig.text(0.01, 0.5, text, ha='left', va='center', size='20', weight='heavy', color='w')
49 67 fig.show()
50 68 figpause(1000)
51 69
52 70
53 71 class PlotData(Operation, Process):
54 72 '''
55 73 Base class for Schain plotting operations
56 74 '''
57 75
58 76 CODE = 'Figure'
59 77 colormap = 'jro'
60 78 bgcolor = 'white'
61 79 CONFLATE = False
62 80 __missing = 1E30
63 81
64 82 __attrs__ = ['show', 'save', 'xmin', 'xmax', 'ymin', 'ymax', 'zmin', 'zmax',
65 83 'zlimits', 'xlabel', 'ylabel', 'xaxis','cb_label', 'title',
66 84 'colorbar', 'bgcolor', 'width', 'height', 'localtime', 'oneFigure',
67 'showprofile', 'decimation']
85 'showprofile', 'decimation', 'ftp']
68 86
69 87 def __init__(self, **kwargs):
70 88
71 89 Operation.__init__(self, plot=True, **kwargs)
72 90 Process.__init__(self)
73 91
74 92 self.kwargs['code'] = self.CODE
75 93 self.mp = False
76 94 self.data = None
77 95 self.isConfig = False
78 96 self.figures = []
79 97 self.axes = []
80 98 self.cb_axes = []
81 99 self.localtime = kwargs.pop('localtime', True)
82 100 self.show = kwargs.get('show', True)
83 101 self.save = kwargs.get('save', False)
102 self.ftp = kwargs.get('ftp', False)
84 103 self.colormap = kwargs.get('colormap', self.colormap)
85 104 self.colormap_coh = kwargs.get('colormap_coh', 'jet')
86 105 self.colormap_phase = kwargs.get('colormap_phase', 'RdBu_r')
87 106 self.colormaps = kwargs.get('colormaps', None)
88 107 self.bgcolor = kwargs.get('bgcolor', self.bgcolor)
89 108 self.showprofile = kwargs.get('showprofile', False)
90 109 self.title = kwargs.get('wintitle', self.CODE.upper())
91 110 self.cb_label = kwargs.get('cb_label', None)
92 111 self.cb_labels = kwargs.get('cb_labels', None)
112 self.labels = kwargs.get('labels', None)
93 113 self.xaxis = kwargs.get('xaxis', 'frequency')
94 114 self.zmin = kwargs.get('zmin', None)
95 115 self.zmax = kwargs.get('zmax', None)
96 116 self.zlimits = kwargs.get('zlimits', None)
97 117 self.xmin = kwargs.get('xmin', None)
98 118 self.xmax = kwargs.get('xmax', None)
99 119 self.xrange = kwargs.get('xrange', 24)
120 self.xscale = kwargs.get('xscale', None)
100 121 self.ymin = kwargs.get('ymin', None)
101 122 self.ymax = kwargs.get('ymax', None)
123 self.yscale = kwargs.get('yscale', None)
102 124 self.xlabel = kwargs.get('xlabel', None)
103 125 self.decimation = kwargs.get('decimation', None)
104 126 self.showSNR = kwargs.get('showSNR', False)
105 127 self.oneFigure = kwargs.get('oneFigure', True)
106 128 self.width = kwargs.get('width', None)
107 129 self.height = kwargs.get('height', None)
108 130 self.colorbar = kwargs.get('colorbar', True)
109 131 self.factors = kwargs.get('factors', [1, 1, 1, 1, 1, 1, 1, 1])
132 self.channels = kwargs.get('channels', None)
110 133 self.titles = kwargs.get('titles', [])
111 134 self.polar = False
135 self.grid = kwargs.get('grid', False)
112 136
113 137 def __fmtTime(self, x, pos):
114 138 '''
115 139 '''
116 140
117 141 return '{}'.format(self.getDateTime(x).strftime('%H:%M'))
118 142
119 143 def __setup(self):
120 144 '''
121 145 Common setup for all figures, here figures and axes are created
122 146 '''
123 147
124 148 if self.CODE not in self.data:
125 149 raise ValueError(log.error('Missing data for {}'.format(self.CODE),
126 150 self.name))
127 151
128 152 self.setup()
129 153
130 154 self.time_label = 'LT' if self.localtime else 'UTC'
131 155 if self.data.localtime:
132 156 self.getDateTime = datetime.datetime.fromtimestamp
133 157 else:
134 158 self.getDateTime = datetime.datetime.utcfromtimestamp
135 159
136 160 if self.width is None:
137 161 self.width = 8
138 162
139 163 self.figures = []
140 164 self.axes = []
141 165 self.cb_axes = []
142 166 self.pf_axes = []
143 167 self.cmaps = []
144 168
145 169 size = '15%' if self.ncols == 1 else '30%'
146 170 pad = '4%' if self.ncols == 1 else '8%'
147 171
148 172 if self.oneFigure:
149 173 if self.height is None:
150 174 self.height = 1.4 * self.nrows + 1
151 175 fig = plt.figure(figsize=(self.width, self.height),
152 176 edgecolor='k',
153 177 facecolor='w')
154 178 self.figures.append(fig)
155 179 for n in range(self.nplots):
156 180 ax = fig.add_subplot(self.nrows, self.ncols,
157 181 n + 1, polar=self.polar)
158 182 ax.tick_params(labelsize=8)
159 183 ax.firsttime = True
160 184 ax.index = 0
161 185 ax.press = None
162 186 self.axes.append(ax)
163 187 if self.showprofile:
164 188 cax = self.__add_axes(ax, size=size, pad=pad)
165 189 cax.tick_params(labelsize=8)
166 190 self.pf_axes.append(cax)
167 191 else:
168 192 if self.height is None:
169 193 self.height = 3
170 194 for n in range(self.nplots):
171 195 fig = plt.figure(figsize=(self.width, self.height),
172 196 edgecolor='k',
173 197 facecolor='w')
174 198 ax = fig.add_subplot(1, 1, 1, polar=self.polar)
175 199 ax.tick_params(labelsize=8)
176 200 ax.firsttime = True
177 201 ax.index = 0
178 202 ax.press = None
179 203 self.figures.append(fig)
180 204 self.axes.append(ax)
181 205 if self.showprofile:
182 206 cax = self.__add_axes(ax, size=size, pad=pad)
183 207 cax.tick_params(labelsize=8)
184 208 self.pf_axes.append(cax)
185 209
186 210 for n in range(self.nrows):
187 211 if self.colormaps is not None:
188 212 cmap = plt.get_cmap(self.colormaps[n])
189 213 else:
190 214 cmap = plt.get_cmap(self.colormap)
191 215 cmap.set_bad(self.bgcolor, 1.)
192 216 self.cmaps.append(cmap)
193 217
194 218 for fig in self.figures:
195 219 fig.canvas.mpl_connect('key_press_event', self.OnKeyPress)
196 220 fig.canvas.mpl_connect('scroll_event', self.OnBtnScroll)
197 221 fig.canvas.mpl_connect('button_press_event', self.onBtnPress)
198 222 fig.canvas.mpl_connect('motion_notify_event', self.onMotion)
199 223 fig.canvas.mpl_connect('button_release_event', self.onBtnRelease)
200 224 if self.show:
201 225 fig.show()
202 226
203 227 def OnKeyPress(self, event):
204 228 '''
205 229 Event for pressing keys (up, down) change colormap
206 230 '''
207 231 ax = event.inaxes
208 232 if ax in self.axes:
209 233 if event.key == 'down':
210 234 ax.index += 1
211 235 elif event.key == 'up':
212 236 ax.index -= 1
213 237 if ax.index < 0:
214 238 ax.index = len(CMAPS) - 1
215 239 elif ax.index == len(CMAPS):
216 240 ax.index = 0
217 241 cmap = CMAPS[ax.index]
218 242 ax.cbar.set_cmap(cmap)
219 243 ax.cbar.draw_all()
220 244 ax.plt.set_cmap(cmap)
221 245 ax.cbar.patch.figure.canvas.draw()
222 246 self.colormap = cmap.name
223 247
224 248 def OnBtnScroll(self, event):
225 249 '''
226 250 Event for scrolling, scale figure
227 251 '''
228 252 cb_ax = event.inaxes
229 253 if cb_ax in [ax.cbar.ax for ax in self.axes if ax.cbar]:
230 254 ax = [ax for ax in self.axes if cb_ax == ax.cbar.ax][0]
231 255 pt = ax.cbar.ax.bbox.get_points()[:, 1]
232 256 nrm = ax.cbar.norm
233 257 vmin, vmax, p0, p1, pS = (
234 258 nrm.vmin, nrm.vmax, pt[0], pt[1], event.y)
235 259 scale = 2 if event.step == 1 else 0.5
236 260 point = vmin + (vmax - vmin) / (p1 - p0) * (pS - p0)
237 261 ax.cbar.norm.vmin = point - scale * (point - vmin)
238 262 ax.cbar.norm.vmax = point - scale * (point - vmax)
239 263 ax.plt.set_norm(ax.cbar.norm)
240 264 ax.cbar.draw_all()
241 265 ax.cbar.patch.figure.canvas.draw()
242 266
243 267 def onBtnPress(self, event):
244 268 '''
245 269 Event for mouse button press
246 270 '''
247 271 cb_ax = event.inaxes
248 272 if cb_ax is None:
249 273 return
250 274
251 275 if cb_ax in [ax.cbar.ax for ax in self.axes if ax.cbar]:
252 276 cb_ax.press = event.x, event.y
253 277 else:
254 278 cb_ax.press = None
255 279
256 280 def onMotion(self, event):
257 281 '''
258 282 Event for move inside colorbar
259 283 '''
260 284 cb_ax = event.inaxes
261 285 if cb_ax is None:
262 286 return
263 287 if cb_ax not in [ax.cbar.ax for ax in self.axes if ax.cbar]:
264 288 return
265 289 if cb_ax.press is None:
266 290 return
267 291
268 292 ax = [ax for ax in self.axes if cb_ax == ax.cbar.ax][0]
269 293 xprev, yprev = cb_ax.press
270 294 dx = event.x - xprev
271 295 dy = event.y - yprev
272 296 cb_ax.press = event.x, event.y
273 297 scale = ax.cbar.norm.vmax - ax.cbar.norm.vmin
274 298 perc = 0.03
275 299
276 300 if event.button == 1:
277 301 ax.cbar.norm.vmin -= (perc * scale) * numpy.sign(dy)
278 302 ax.cbar.norm.vmax -= (perc * scale) * numpy.sign(dy)
279 303 elif event.button == 3:
280 304 ax.cbar.norm.vmin -= (perc * scale) * numpy.sign(dy)
281 305 ax.cbar.norm.vmax += (perc * scale) * numpy.sign(dy)
282 306
283 307 ax.cbar.draw_all()
284 308 ax.plt.set_norm(ax.cbar.norm)
285 309 ax.cbar.patch.figure.canvas.draw()
286 310
287 311 def onBtnRelease(self, event):
288 312 '''
289 313 Event for mouse button release
290 314 '''
291 315 cb_ax = event.inaxes
292 316 if cb_ax is not None:
293 317 cb_ax.press = None
294 318
295 319 def __add_axes(self, ax, size='30%', pad='8%'):
296 320 '''
297 321 Add new axes to the given figure
298 322 '''
299 323 divider = make_axes_locatable(ax)
300 324 nax = divider.new_horizontal(size=size, pad=pad)
301 325 ax.figure.add_axes(nax)
302 326 return nax
303 327
304 328 self.setup()
305 329
306 330 def setup(self):
307 331 '''
308 332 This method should be implemented in the child class, the following
309 333 attributes should be set:
310 334
311 335 self.nrows: number of rows
312 336 self.ncols: number of cols
313 337 self.nplots: number of plots (channels or pairs)
314 338 self.ylabel: label for Y axes
315 339 self.titles: list of axes title
316 340
317 341 '''
318 342 raise(NotImplementedError, 'Implement this method in child class')
319 343
320 344 def fill_gaps(self, x_buffer, y_buffer, z_buffer):
321 345 '''
322 346 Create a masked array for missing data
323 347 '''
324 348 if x_buffer.shape[0] < 2:
325 349 return x_buffer, y_buffer, z_buffer
326 350
327 351 deltas = x_buffer[1:] - x_buffer[0:-1]
328 352 x_median = numpy.median(deltas)
329 353
330 354 index = numpy.where(deltas > 5 * x_median)
331 355
332 356 if len(index[0]) != 0:
333 357 z_buffer[::, index[0], ::] = self.__missing
334 358 z_buffer = numpy.ma.masked_inside(z_buffer,
335 359 0.99 * self.__missing,
336 360 1.01 * self.__missing)
337 361
338 362 return x_buffer, y_buffer, z_buffer
339 363
340 364 def decimate(self):
341 365
342 366 # dx = int(len(self.x)/self.__MAXNUMX) + 1
343 367 dy = int(len(self.y) / self.decimation) + 1
344 368
345 369 # x = self.x[::dx]
346 370 x = self.x
347 371 y = self.y[::dy]
348 372 z = self.z[::, ::, ::dy]
349 373
350 374 return x, y, z
351 375
352 376 def format(self):
353 377 '''
354 378 Set min and max values, labels, ticks and titles
355 379 '''
356 380
357 381 if self.xmin is None:
358 382 xmin = self.min_time
359 383 else:
360 384 if self.xaxis is 'time':
361 385 dt = self.getDateTime(self.min_time)
362 386 xmin = (dt.replace(hour=int(self.xmin), minute=0, second=0) -
363 387 datetime.datetime(1970, 1, 1)).total_seconds()
364 388 if self.data.localtime:
365 389 xmin += time.timezone
366 390 else:
367 391 xmin = self.xmin
368 392
369 393 if self.xmax is None:
370 394 xmax = xmin + self.xrange * 60 * 60
371 395 else:
372 396 if self.xaxis is 'time':
373 397 dt = self.getDateTime(self.max_time)
374 398 xmax = (dt.replace(hour=int(self.xmax), minute=59, second=59) -
375 399 datetime.datetime(1970, 1, 1) + datetime.timedelta(seconds=1)).total_seconds()
376 400 if self.data.localtime:
377 401 xmax += time.timezone
378 402 else:
379 403 xmax = self.xmax
380 404
381 405 ymin = self.ymin if self.ymin else numpy.nanmin(self.y)
382 406 ymax = self.ymax if self.ymax else numpy.nanmax(self.y)
383 407
384 Y = numpy.array([5, 10, 20, 50, 100, 200, 500, 1000, 2000])
385 i = 1 if numpy.where(ymax-ymin < Y)[0][0] < 0 else numpy.where(ymax-ymin < Y)[0][0]
386 ystep = Y[i] / 5
408 Y = numpy.array([1, 2, 5, 10, 20, 50, 100, 200, 500, 1000, 2000, 5000])
409 i = 1 if numpy.where(abs(ymax-ymin) <= Y)[0][0] < 0 else numpy.where(abs(ymax-ymin) <= Y)[0][0]
410 ystep = Y[i] / 10.
387 411
388 412 for n, ax in enumerate(self.axes):
389 413 if ax.firsttime:
390 414 ax.set_facecolor(self.bgcolor)
391 415 ax.yaxis.set_major_locator(MultipleLocator(ystep))
416 ax.xaxis.set_major_locator(MultipleLocator(ystep))
417 if self.xscale:
418 ax.xaxis.set_major_formatter(FuncFormatter(lambda x, pos: '{0:g}'.format(x*self.xscale)))
419 if self.xscale:
420 ax.yaxis.set_major_formatter(FuncFormatter(lambda x, pos: '{0:g}'.format(x*self.yscale)))
392 421 if self.xaxis is 'time':
393 422 ax.xaxis.set_major_formatter(FuncFormatter(self.__fmtTime))
394 423 ax.xaxis.set_major_locator(LinearLocator(9))
395 424 if self.xlabel is not None:
396 425 ax.set_xlabel(self.xlabel)
397 426 ax.set_ylabel(self.ylabel)
398 427 ax.firsttime = False
399 428 if self.showprofile:
400 429 self.pf_axes[n].set_ylim(ymin, ymax)
401 430 self.pf_axes[n].set_xlim(self.zmin, self.zmax)
402 431 self.pf_axes[n].set_xlabel('dB')
403 432 self.pf_axes[n].grid(b=True, axis='x')
404 433 [tick.set_visible(False)
405 434 for tick in self.pf_axes[n].get_yticklabels()]
406 435 if self.colorbar:
407 436 ax.cbar = plt.colorbar(
408 437 ax.plt, ax=ax, fraction=0.05, pad=0.02, aspect=10)
409 438 ax.cbar.ax.tick_params(labelsize=8)
410 439 ax.cbar.ax.press = None
411 440 if self.cb_label:
412 441 ax.cbar.set_label(self.cb_label, size=8)
413 442 elif self.cb_labels:
414 443 ax.cbar.set_label(self.cb_labels[n], size=8)
415 444 else:
416 445 ax.cbar = None
446 if self.grid:
447 ax.grid(True)
417 448
418 449 if not self.polar:
419 450 ax.set_xlim(xmin, xmax)
420 451 ax.set_ylim(ymin, ymax)
421 ax.set_title('{} - {} {}'.format(
452 ax.set_title('{} {} {}'.format(
422 453 self.titles[n],
423 self.getDateTime(self.max_time).strftime('%H:%M:%S'),
454 self.getDateTime(self.max_time).strftime('%Y-%m-%dT%H:%M:%S'),
424 455 self.time_label),
425 456 size=8)
426 457 else:
427 458 ax.set_title('{}'.format(self.titles[n]), size=8)
428 459 ax.set_ylim(0, 90)
429 460 ax.set_yticks(numpy.arange(0, 90, 20))
430 461 ax.yaxis.labelpad = 40
431 462
432 463 def __plot(self):
433 464 '''
434 465 '''
435 log.success('Plotting', self.name)
466 log.log('Plotting', self.name)
436 467
437 468 try:
438 469 self.plot()
439 470 self.format()
440 except:
471 except Exception as e:
441 472 log.warning('{} Plot could not be updated... check data'.format(self.CODE), self.name)
473 log.error(str(e), '')
474 return
442 475
443 476 for n, fig in enumerate(self.figures):
444 477 if self.nrows == 0 or self.nplots == 0:
445 478 log.warning('No data', self.name)
446 479 fig.text(0.5, 0.5, 'No Data', fontsize='large', ha='center')
447 480 fig.canvas.manager.set_window_title(self.CODE)
448 481 continue
449 482
450 483 fig.tight_layout()
451 484 fig.canvas.manager.set_window_title('{} - {}'.format(self.title,
452 485 self.getDateTime(self.max_time).strftime('%Y/%m/%d')))
453 486 fig.canvas.draw()
454 487
455 if self.save and self.data.ended:
456 channels = range(self.nrows)
488 if self.save and (self.data.ended or not self.data.buffering):
489
490 if self.save_labels:
491 labels = self.save_labels
492 else:
493 labels = range(self.nrows)
494
457 495 if self.oneFigure:
458 496 label = ''
459 497 else:
460 label = '_{}'.format(channels[n])
498 label = '-{}'.format(labels[n])
461 499 figname = os.path.join(
462 500 self.save,
501 self.CODE,
463 502 '{}{}_{}.png'.format(
464 503 self.CODE,
465 504 label,
466 505 self.getDateTime(self.saveTime).strftime(
467 506 '%Y%m%d_%H%M%S'),
468 507 )
469 508 )
470 509 log.log('Saving figure: {}'.format(figname), self.name)
510 if not os.path.isdir(os.path.dirname(figname)):
511 os.makedirs(os.path.dirname(figname))
471 512 fig.savefig(figname)
472 513
473 514 def plot(self):
474 515 '''
475 516 '''
476 517 raise(NotImplementedError, 'Implement this method in child class')
477 518
478 519 def run(self):
479 520
480 log.success('Starting', self.name)
521 log.log('Starting', self.name)
481 522
482 523 context = zmq.Context()
483 524 receiver = context.socket(zmq.SUB)
484 525 receiver.setsockopt(zmq.SUBSCRIBE, '')
485 526 receiver.setsockopt(zmq.CONFLATE, self.CONFLATE)
486 527
487 528 if 'server' in self.kwargs['parent']:
488 529 receiver.connect(
489 530 'ipc:///tmp/{}.plots'.format(self.kwargs['parent']['server']))
490 531 else:
491 532 receiver.connect("ipc:///tmp/zmq.plots")
492 533
493 534 while True:
494 535 try:
495 536 self.data = receiver.recv_pyobj(flags=zmq.NOBLOCK)
496 537 if self.data.localtime and self.localtime:
497 538 self.times = self.data.times
498 539 elif self.data.localtime and not self.localtime:
499 540 self.times = self.data.times + time.timezone
500 541 elif not self.data.localtime and self.localtime:
501 542 self.times = self.data.times - time.timezone
502 543 else:
503 544 self.times = self.data.times
504 545
505 546 self.min_time = self.times[0]
506 547 self.max_time = self.times[-1]
507 548
508 549 if self.isConfig is False:
509 550 self.__setup()
510 551 self.isConfig = True
511 552
512 553 self.__plot()
513 554
514 555 except zmq.Again as e:
515 556 if self.data and self.data.ended:
516 557 break
517 558 log.log('Waiting for data...')
518 559 if self.data:
519 560 figpause(self.data.throttle)
520 561 else:
521 562 time.sleep(2)
522 563
523 564 def close(self):
524 565 if self.data:
525 566 self.__plot()
526 567
527 568
528 569 class PlotSpectraData(PlotData):
529 570 '''
530 571 Plot for Spectra data
531 572 '''
532 573
533 574 CODE = 'spc'
534 575 colormap = 'jro'
535 576
536 577 def setup(self):
537 578 self.nplots = len(self.data.channels)
538 579 self.ncols = int(numpy.sqrt(self.nplots) + 0.9)
539 580 self.nrows = int((1.0 * self.nplots / self.ncols) + 0.9)
540 581 self.width = 3.4 * self.ncols
541 582 self.height = 3 * self.nrows
542 583 self.cb_label = 'dB'
543 584 if self.showprofile:
544 585 self.width += 0.8 * self.ncols
545 586
546 587 self.ylabel = 'Range [km]'
547 588
548 589 def plot(self):
549 590 if self.xaxis == "frequency":
550 591 x = self.data.xrange[0]
551 592 self.xlabel = "Frequency (kHz)"
552 593 elif self.xaxis == "time":
553 594 x = self.data.xrange[1]
554 595 self.xlabel = "Time (ms)"
555 596 else:
556 597 x = self.data.xrange[2]
557 598 self.xlabel = "Velocity (m/s)"
558 599
559 600 if self.CODE == 'spc_mean':
560 601 x = self.data.xrange[2]
561 602 self.xlabel = "Velocity (m/s)"
562 603
563 604 self.titles = []
564 605
565 606 y = self.data.heights
566 607 self.y = y
567 608 z = self.data['spc']
568 609
569 610 for n, ax in enumerate(self.axes):
570 611 noise = self.data['noise'][n][-1]
571 612 if self.CODE == 'spc_mean':
572 613 mean = self.data['mean'][n][-1]
573 614 if ax.firsttime:
574 615 self.xmax = self.xmax if self.xmax else numpy.nanmax(x)
575 616 self.xmin = self.xmin if self.xmin else -self.xmax
576 617 self.zmin = self.zmin if self.zmin else numpy.nanmin(z)
577 618 self.zmax = self.zmax if self.zmax else numpy.nanmax(z)
578 619 ax.plt = ax.pcolormesh(x, y, z[n].T,
579 620 vmin=self.zmin,
580 621 vmax=self.zmax,
581 622 cmap=plt.get_cmap(self.colormap)
582 623 )
583 624
584 625 if self.showprofile:
585 626 ax.plt_profile = self.pf_axes[n].plot(
586 627 self.data['rti'][n][-1], y)[0]
587 628 ax.plt_noise = self.pf_axes[n].plot(numpy.repeat(noise, len(y)), y,
588 629 color="k", linestyle="dashed", lw=1)[0]
589 630 if self.CODE == 'spc_mean':
590 631 ax.plt_mean = ax.plot(mean, y, color='k')[0]
591 632 else:
592 633 ax.plt.set_array(z[n].T.ravel())
593 634 if self.showprofile:
594 635 ax.plt_profile.set_data(self.data['rti'][n][-1], y)
595 636 ax.plt_noise.set_data(numpy.repeat(noise, len(y)), y)
596 637 if self.CODE == 'spc_mean':
597 638 ax.plt_mean.set_data(mean, y)
598 639
599 640 self.titles.append('CH {}: {:3.2f}dB'.format(n, noise))
600 641 self.saveTime = self.max_time
601 642
602 643
603 644 class PlotCrossSpectraData(PlotData):
604 645
605 646 CODE = 'cspc'
606 647 zmin_coh = None
607 648 zmax_coh = None
608 649 zmin_phase = None
609 650 zmax_phase = None
610 651
611 652 def setup(self):
612 653
613 654 self.ncols = 4
614 655 self.nrows = len(self.data.pairs)
615 656 self.nplots = self.nrows * 4
616 657 self.width = 3.4 * self.ncols
617 658 self.height = 3 * self.nrows
618 659 self.ylabel = 'Range [km]'
619 660 self.showprofile = False
620 661
621 662 def plot(self):
622 663
623 664 if self.xaxis == "frequency":
624 665 x = self.data.xrange[0]
625 666 self.xlabel = "Frequency (kHz)"
626 667 elif self.xaxis == "time":
627 668 x = self.data.xrange[1]
628 669 self.xlabel = "Time (ms)"
629 670 else:
630 671 x = self.data.xrange[2]
631 672 self.xlabel = "Velocity (m/s)"
632 673
633 674 self.titles = []
634 675
635 676 y = self.data.heights
636 677 self.y = y
637 678 spc = self.data['spc']
638 679 cspc = self.data['cspc']
639 680
640 681 for n in range(self.nrows):
641 682 noise = self.data['noise'][n][-1]
642 683 pair = self.data.pairs[n]
643 684 ax = self.axes[4 * n]
644 685 ax3 = self.axes[4 * n + 3]
645 686 if ax.firsttime:
646 687 self.xmax = self.xmax if self.xmax else numpy.nanmax(x)
647 688 self.xmin = self.xmin if self.xmin else -self.xmax
648 689 self.zmin = self.zmin if self.zmin else numpy.nanmin(spc)
649 690 self.zmax = self.zmax if self.zmax else numpy.nanmax(spc)
650 691 ax.plt = ax.pcolormesh(x, y, spc[pair[0]].T,
651 692 vmin=self.zmin,
652 693 vmax=self.zmax,
653 694 cmap=plt.get_cmap(self.colormap)
654 695 )
655 696 else:
656 697 ax.plt.set_array(spc[pair[0]].T.ravel())
657 698 self.titles.append('CH {}: {:3.2f}dB'.format(n, noise))
658 699
659 700 ax = self.axes[4 * n + 1]
660 701 if ax.firsttime:
661 702 ax.plt = ax.pcolormesh(x, y, spc[pair[1]].T,
662 703 vmin=self.zmin,
663 704 vmax=self.zmax,
664 705 cmap=plt.get_cmap(self.colormap)
665 706 )
666 707 else:
667 708 ax.plt.set_array(spc[pair[1]].T.ravel())
668 709 self.titles.append('CH {}: {:3.2f}dB'.format(n, noise))
669 710
670 711 out = cspc[n] / numpy.sqrt(spc[pair[0]] * spc[pair[1]])
671 712 coh = numpy.abs(out)
672 713 phase = numpy.arctan2(out.imag, out.real) * 180 / numpy.pi
673 714
674 715 ax = self.axes[4 * n + 2]
675 716 if ax.firsttime:
676 717 ax.plt = ax.pcolormesh(x, y, coh.T,
677 718 vmin=0,
678 719 vmax=1,
679 720 cmap=plt.get_cmap(self.colormap_coh)
680 721 )
681 722 else:
682 723 ax.plt.set_array(coh.T.ravel())
683 724 self.titles.append(
684 725 'Coherence Ch{} * Ch{}'.format(pair[0], pair[1]))
685 726
686 727 ax = self.axes[4 * n + 3]
687 728 if ax.firsttime:
688 729 ax.plt = ax.pcolormesh(x, y, phase.T,
689 730 vmin=-180,
690 731 vmax=180,
691 732 cmap=plt.get_cmap(self.colormap_phase)
692 733 )
693 734 else:
694 735 ax.plt.set_array(phase.T.ravel())
695 736 self.titles.append('Phase CH{} * CH{}'.format(pair[0], pair[1]))
696 737
697 738 self.saveTime = self.max_time
698 739
699 740
700 741 class PlotSpectraMeanData(PlotSpectraData):
701 742 '''
702 743 Plot for Spectra and Mean
703 744 '''
704 745 CODE = 'spc_mean'
705 746 colormap = 'jro'
706 747
707 748
708 749 class PlotRTIData(PlotData):
709 750 '''
710 751 Plot for RTI data
711 752 '''
712 753
713 754 CODE = 'rti'
714 755 colormap = 'jro'
715 756
716 757 def setup(self):
717 758 self.xaxis = 'time'
718 759 self.ncols = 1
719 760 self.nrows = len(self.data.channels)
720 761 self.nplots = len(self.data.channels)
721 762 self.ylabel = 'Range [km]'
722 763 self.cb_label = 'dB'
723 764 self.titles = ['{} Channel {}'.format(
724 765 self.CODE.upper(), x) for x in range(self.nrows)]
725 766
726 767 def plot(self):
727 768 self.x = self.times
728 769 self.y = self.data.heights
729 770 self.z = self.data[self.CODE]
730 771 self.z = numpy.ma.masked_invalid(self.z)
731 772
732 773 if self.decimation is None:
733 774 x, y, z = self.fill_gaps(self.x, self.y, self.z)
734 775 else:
735 776 x, y, z = self.fill_gaps(*self.decimate())
736 777
737 778 for n, ax in enumerate(self.axes):
738 779 self.zmin = self.zmin if self.zmin else numpy.min(self.z)
739 780 self.zmax = self.zmax if self.zmax else numpy.max(self.z)
740 781 if ax.firsttime:
741 782 ax.plt = ax.pcolormesh(x, y, z[n].T,
742 783 vmin=self.zmin,
743 784 vmax=self.zmax,
744 785 cmap=plt.get_cmap(self.colormap)
745 786 )
746 787 if self.showprofile:
747 788 ax.plot_profile = self.pf_axes[n].plot(
748 789 self.data['rti'][n][-1], self.y)[0]
749 790 ax.plot_noise = self.pf_axes[n].plot(numpy.repeat(self.data['noise'][n][-1], len(self.y)), self.y,
750 791 color="k", linestyle="dashed", lw=1)[0]
751 792 else:
752 793 ax.collections.remove(ax.collections[0])
753 794 ax.plt = ax.pcolormesh(x, y, z[n].T,
754 795 vmin=self.zmin,
755 796 vmax=self.zmax,
756 797 cmap=plt.get_cmap(self.colormap)
757 798 )
758 799 if self.showprofile:
759 800 ax.plot_profile.set_data(self.data['rti'][n][-1], self.y)
760 801 ax.plot_noise.set_data(numpy.repeat(
761 802 self.data['noise'][n][-1], len(self.y)), self.y)
762 803
763 804 self.saveTime = self.min_time
764 805
765 806
766 807 class PlotCOHData(PlotRTIData):
767 808 '''
768 809 Plot for Coherence data
769 810 '''
770 811
771 812 CODE = 'coh'
772 813
773 814 def setup(self):
774 815 self.xaxis = 'time'
775 816 self.ncols = 1
776 817 self.nrows = len(self.data.pairs)
777 818 self.nplots = len(self.data.pairs)
778 819 self.ylabel = 'Range [km]'
779 820 if self.CODE == 'coh':
780 821 self.cb_label = ''
781 822 self.titles = [
782 823 'Coherence Map Ch{} * Ch{}'.format(x[0], x[1]) for x in self.data.pairs]
783 824 else:
784 825 self.cb_label = 'Degrees'
785 826 self.titles = [
786 827 'Phase Map Ch{} * Ch{}'.format(x[0], x[1]) for x in self.data.pairs]
787 828
788 829
789 830 class PlotPHASEData(PlotCOHData):
790 831 '''
791 832 Plot for Phase map data
792 833 '''
793 834
794 835 CODE = 'phase'
795 836 colormap = 'seismic'
796 837
797 838
798 839 class PlotNoiseData(PlotData):
799 840 '''
800 841 Plot for noise
801 842 '''
802 843
803 844 CODE = 'noise'
804 845
805 846 def setup(self):
806 847 self.xaxis = 'time'
807 848 self.ncols = 1
808 849 self.nrows = 1
809 850 self.nplots = 1
810 851 self.ylabel = 'Intensity [dB]'
811 852 self.titles = ['Noise']
812 853 self.colorbar = False
813 854
814 855 def plot(self):
815 856
816 857 x = self.times
817 858 xmin = self.min_time
818 859 xmax = xmin + self.xrange * 60 * 60
819 860 Y = self.data[self.CODE]
820 861
821 862 if self.axes[0].firsttime:
822 863 for ch in self.data.channels:
823 864 y = Y[ch]
824 865 self.axes[0].plot(x, y, lw=1, label='Ch{}'.format(ch))
825 866 plt.legend()
826 867 else:
827 868 for ch in self.data.channels:
828 869 y = Y[ch]
829 870 self.axes[0].lines[ch].set_data(x, y)
830 871
831 872 self.ymin = numpy.nanmin(Y) - 5
832 873 self.ymax = numpy.nanmax(Y) + 5
833 874 self.saveTime = self.min_time
834 875
835 876
836 877 class PlotSNRData(PlotRTIData):
837 878 '''
838 879 Plot for SNR Data
839 880 '''
840 881
841 882 CODE = 'snr'
842 883 colormap = 'jet'
843 884
844 885
845 886 class PlotDOPData(PlotRTIData):
846 887 '''
847 888 Plot for DOPPLER Data
848 889 '''
849 890
850 891 CODE = 'dop'
851 892 colormap = 'jet'
852 893
853 894
854 895 class PlotSkyMapData(PlotData):
855 896 '''
856 897 Plot for meteors detection data
857 898 '''
858 899
859 900 CODE = 'param'
860 901
861 902 def setup(self):
862 903
863 904 self.ncols = 1
864 905 self.nrows = 1
865 906 self.width = 7.2
866 907 self.height = 7.2
867 908 self.nplots = 1
868 909 self.xlabel = 'Zonal Zenith Angle (deg)'
869 910 self.ylabel = 'Meridional Zenith Angle (deg)'
870 911 self.polar = True
871 912 self.ymin = -180
872 913 self.ymax = 180
873 914 self.colorbar = False
874 915
875 916 def plot(self):
876 917
877 918 arrayParameters = numpy.concatenate(self.data['param'])
878 919 error = arrayParameters[:, -1]
879 920 indValid = numpy.where(error == 0)[0]
880 921 finalMeteor = arrayParameters[indValid, :]
881 922 finalAzimuth = finalMeteor[:, 3]
882 923 finalZenith = finalMeteor[:, 4]
883 924
884 925 x = finalAzimuth * numpy.pi / 180
885 926 y = finalZenith
886 927
887 928 ax = self.axes[0]
888 929
889 930 if ax.firsttime:
890 931 ax.plot = ax.plot(x, y, 'bo', markersize=5)[0]
891 932 else:
892 933 ax.plot.set_data(x, y)
893 934
894 935 dt1 = self.getDateTime(self.min_time).strftime('%y/%m/%d %H:%M:%S')
895 936 dt2 = self.getDateTime(self.max_time).strftime('%y/%m/%d %H:%M:%S')
896 937 title = 'Meteor Detection Sky Map\n %s - %s \n Number of events: %5.0f\n' % (dt1,
897 938 dt2,
898 939 len(x))
899 940 self.titles[0] = title
900 941 self.saveTime = self.max_time
901 942
902 943
903 944 class PlotParamData(PlotRTIData):
904 945 '''
905 946 Plot for data_param object
906 947 '''
907 948
908 949 CODE = 'param'
909 950 colormap = 'seismic'
910 951
911 952 def setup(self):
912 953 self.xaxis = 'time'
913 954 self.ncols = 1
914 955 self.nrows = self.data.shape(self.CODE)[0]
915 956 self.nplots = self.nrows
916 957 if self.showSNR:
917 958 self.nrows += 1
918 959 self.nplots += 1
919 960
920 961 self.ylabel = 'Height [km]'
921 962 if not self.titles:
922 963 self.titles = self.data.parameters \
923 964 if self.data.parameters else ['Param {}'.format(x) for x in xrange(self.nrows)]
924 965 if self.showSNR:
925 966 self.titles.append('SNR')
926 967
927 968 def plot(self):
928 969 self.data.normalize_heights()
929 970 self.x = self.times
930 971 self.y = self.data.heights
931 972 if self.showSNR:
932 973 self.z = numpy.concatenate(
933 974 (self.data[self.CODE], self.data['snr'])
934 975 )
935 976 else:
936 977 self.z = self.data[self.CODE]
937 978
938 979 self.z = numpy.ma.masked_invalid(self.z)
939 980
940 981 if self.decimation is None:
941 982 x, y, z = self.fill_gaps(self.x, self.y, self.z)
942 983 else:
943 984 x, y, z = self.fill_gaps(*self.decimate())
944 985
945 986 for n, ax in enumerate(self.axes):
946 987
947 988 self.zmax = self.zmax if self.zmax is not None else numpy.max(
948 989 self.z[n])
949 990 self.zmin = self.zmin if self.zmin is not None else numpy.min(
950 991 self.z[n])
951 992
952 993 if ax.firsttime:
953 994 if self.zlimits is not None:
954 995 self.zmin, self.zmax = self.zlimits[n]
955 996
956 997 ax.plt = ax.pcolormesh(x, y, z[n].T * self.factors[n],
957 998 vmin=self.zmin,
958 999 vmax=self.zmax,
959 1000 cmap=self.cmaps[n]
960 1001 )
961 1002 else:
962 1003 if self.zlimits is not None:
963 1004 self.zmin, self.zmax = self.zlimits[n]
964 1005 ax.collections.remove(ax.collections[0])
965 1006 ax.plt = ax.pcolormesh(x, y, z[n].T * self.factors[n],
966 1007 vmin=self.zmin,
967 1008 vmax=self.zmax,
968 1009 cmap=self.cmaps[n]
969 1010 )
970 1011
971 1012 self.saveTime = self.min_time
972 1013
973 1014
974 1015 class PlotOutputData(PlotParamData):
975 1016 '''
976 1017 Plot data_output object
977 1018 '''
978 1019
979 1020 CODE = 'output'
980 1021 colormap = 'seismic'
1022
1023
1024 class PlotPolarMapData(PlotData):
1025 '''
1026 Plot for meteors detection data
1027 '''
1028
1029 CODE = 'param'
1030 colormap = 'seismic'
1031
1032 def setup(self):
1033 self.ncols = 1
1034 self.nrows = 1
1035 self.width = 9
1036 self.height = 8
1037 self.mode = self.data.meta['mode']
1038 if self.channels is not None:
1039 self.nplots = len(self.channels)
1040 self.nrows = len(self.channels)
1041 else:
1042 self.nplots = self.data.shape(self.CODE)[0]
1043 self.nrows = self.nplots
1044 self.channels = range(self.nplots)
1045 if self.mode == 'E':
1046 self.xlabel = 'Longitude'
1047 self.ylabel = 'Latitude'
1048 else:
1049 self.xlabel = 'Range (km)'
1050 self.ylabel = 'Height (km)'
1051 self.bgcolor = 'white'
1052 self.cb_labels = self.data.meta['units']
1053 self.lat = self.data.meta['latitude']
1054 self.lon = self.data.meta['longitude']
1055 self.xmin, self.xmax = float(km2deg(self.xmin) + self.lon), float(km2deg(self.xmax) + self.lon)
1056 self.ymin, self.ymax = float(km2deg(self.ymin) + self.lat), float(km2deg(self.ymax) + self.lat)
1057 # self.polar = True
1058
1059 def plot(self):
1060
1061 for n, ax in enumerate(self.axes):
1062 data = self.data['param'][self.channels[n]]
1063
1064 zeniths = numpy.linspace(0, self.data.meta['max_range'], data.shape[1])
1065 if self.mode == 'E':
1066 azimuths = -numpy.radians(self.data.heights)+numpy.pi/2
1067 r, theta = numpy.meshgrid(zeniths, azimuths)
1068 x, y = r*numpy.cos(theta)*numpy.cos(numpy.radians(self.data.meta['elevation'])), r*numpy.sin(theta)*numpy.cos(numpy.radians(self.data.meta['elevation']))
1069 x = km2deg(x) + self.lon
1070 y = km2deg(y) + self.lat
1071 else:
1072 azimuths = numpy.radians(self.data.heights)
1073 r, theta = numpy.meshgrid(zeniths, azimuths)
1074 x, y = r*numpy.cos(theta), r*numpy.sin(theta)
1075 self.y = zeniths
1076
1077 if ax.firsttime:
1078 if self.zlimits is not None:
1079 self.zmin, self.zmax = self.zlimits[n]
1080 ax.plt = ax.pcolormesh(#r, theta, numpy.ma.array(data, mask=numpy.isnan(data)),
1081 x, y, numpy.ma.array(data, mask=numpy.isnan(data)),
1082 vmin=self.zmin,
1083 vmax=self.zmax,
1084 cmap=self.cmaps[n])
1085 else:
1086 if self.zlimits is not None:
1087 self.zmin, self.zmax = self.zlimits[n]
1088 ax.collections.remove(ax.collections[0])
1089 ax.plt = ax.pcolormesh(# r, theta, numpy.ma.array(data, mask=numpy.isnan(data)),
1090 x, y, numpy.ma.array(data, mask=numpy.isnan(data)),
1091 vmin=self.zmin,
1092 vmax=self.zmax,
1093 cmap=self.cmaps[n])
1094
1095 if self.mode == 'A':
1096 continue
1097
1098 # plot district names
1099 f = open('/data/workspace/schain_scripts/distrito.csv')
1100 for line in f:
1101 label, lon, lat = [s.strip() for s in line.split(',') if s]
1102 lat = float(lat)
1103 lon = float(lon)
1104 # ax.plot(lon, lat, '.b', ms=2)
1105 ax.text(lon, lat, label.decode('utf8'), ha='center', va='bottom', size='8', color='black')
1106
1107 # plot limites
1108 limites =[]
1109 tmp = []
1110 for line in open('/data/workspace/schain_scripts/lima.csv'):
1111 if '#' in line:
1112 if tmp:
1113 limites.append(tmp)
1114 tmp = []
1115 continue
1116 values = line.strip().split(',')
1117 tmp.append((float(values[0]), float(values[1])))
1118 for points in limites:
1119 ax.add_patch(Polygon(points, ec='k', fc='none', ls='--', lw=0.5))
1120
1121 # plot Cuencas
1122 for cuenca in ('rimac', 'lurin', 'mala', 'chillon', 'chilca', 'chancay-huaral'):
1123 f = open('/data/workspace/schain_scripts/{}.csv'.format(cuenca))
1124 values = [line.strip().split(',') for line in f]
1125 points = [(float(s[0]), float(s[1])) for s in values]
1126 ax.add_patch(Polygon(points, ec='b', fc='none'))
1127
1128 # plot grid
1129 for r in (15, 30, 45, 60):
1130 ax.add_artist(plt.Circle((self.lon, self.lat), km2deg(r), color='0.6', fill=False, lw=0.2))
1131 ax.text(
1132 self.lon + (km2deg(r))*numpy.cos(60*numpy.pi/180),
1133 self.lat + (km2deg(r))*numpy.sin(60*numpy.pi/180),
1134 '{}km'.format(r),
1135 ha='center', va='bottom', size='8', color='0.6', weight='heavy')
1136
1137 if self.mode == 'E':
1138 title = 'El={}$^\circ$'.format(self.data.meta['elevation'])
1139 label = 'E{:02d}'.format(int(self.data.meta['elevation']))
1140 else:
1141 title = 'Az={}$^\circ$'.format(self.data.meta['azimuth'])
1142 label = 'A{:02d}'.format(int(self.data.meta['azimuth']))
1143
1144 self.save_labels = ['{}-{}'.format(lbl, label) for lbl in self.labels]
1145 self.titles = ['{} {}'.format(self.data.parameters[x], title) for x in self.channels]
1146 self.saveTime = self.max_time
1147
1148
Show file before | Show file after | Hide comments
@@ -1,1542 +1,1542
1 1 '''
2 2 Created on Jul 9, 2014
3 3
4 4 @author: roj-idl71
5 5 '''
6 6 import os
7 7 import datetime
8 8 import numpy
9 9
10 10 from figure import Figure, isRealtime, isTimeInHourRange
11 11 from plotting_codes import *
12 12
13 13
14 14 class SpectraPlot(Figure):
15 15
16 16 isConfig = None
17 17 __nsubplots = None
18 18
19 19 WIDTHPROF = None
20 20 HEIGHTPROF = None
21 21 PREFIX = 'spc'
22 22
23 23 def __init__(self, **kwargs):
24 24 Figure.__init__(self, **kwargs)
25 25 self.isConfig = False
26 26 self.__nsubplots = 1
27 27
28 self.WIDTH = 300
29 self.HEIGHT = 300
28 self.WIDTH = 250
29 self.HEIGHT = 250
30 30 self.WIDTHPROF = 120
31 31 self.HEIGHTPROF = 0
32 32 self.counter_imagwr = 0
33 33
34 34 self.PLOT_CODE = SPEC_CODE
35 35
36 36 self.FTP_WEI = None
37 37 self.EXP_CODE = None
38 38 self.SUB_EXP_CODE = None
39 39 self.PLOT_POS = None
40 40
41 41 self.__xfilter_ena = False
42 42 self.__yfilter_ena = False
43 43
44 44 def getSubplots(self):
45 45
46 46 ncol = int(numpy.sqrt(self.nplots)+0.9)
47 47 nrow = int(self.nplots*1./ncol + 0.9)
48 48
49 49 return nrow, ncol
50 50
51 51 def setup(self, id, nplots, wintitle, showprofile=True, show=True):
52 52
53 53 self.__showprofile = showprofile
54 54 self.nplots = nplots
55 55
56 56 ncolspan = 1
57 57 colspan = 1
58 58 if showprofile:
59 59 ncolspan = 3
60 60 colspan = 2
61 61 self.__nsubplots = 2
62 62
63 63 self.createFigure(id = id,
64 64 wintitle = wintitle,
65 65 widthplot = self.WIDTH + self.WIDTHPROF,
66 66 heightplot = self.HEIGHT + self.HEIGHTPROF,
67 67 show=show)
68 68
69 69 nrow, ncol = self.getSubplots()
70 70
71 71 counter = 0
72 72 for y in range(nrow):
73 73 for x in range(ncol):
74 74
75 75 if counter >= self.nplots:
76 76 break
77 77
78 78 self.addAxes(nrow, ncol*ncolspan, y, x*ncolspan, colspan, 1)
79 79
80 80 if showprofile:
81 81 self.addAxes(nrow, ncol*ncolspan, y, x*ncolspan+colspan, 1, 1)
82 82
83 83 counter += 1
84 84
85 85 def run(self, dataOut, id, wintitle="", channelList=None, showprofile=True,
86 86 xmin=None, xmax=None, ymin=None, ymax=None, zmin=None, zmax=None,
87 87 save=False, figpath='./', figfile=None, show=True, ftp=False, wr_period=1,
88 88 server=None, folder=None, username=None, password=None,
89 89 ftp_wei=0, exp_code=0, sub_exp_code=0, plot_pos=0, realtime=False,
90 90 xaxis="frequency", colormap='jet', normFactor=None):
91 91
92 92 """
93 93
94 94 Input:
95 95 dataOut :
96 96 id :
97 97 wintitle :
98 98 channelList :
99 99 showProfile :
100 100 xmin : None,
101 101 xmax : None,
102 102 ymin : None,
103 103 ymax : None,
104 104 zmin : None,
105 105 zmax : None
106 106 """
107 107 if realtime:
108 108 if not(isRealtime(utcdatatime = dataOut.utctime)):
109 109 print 'Skipping this plot function'
110 110 return
111 111
112 112 if channelList == None:
113 113 channelIndexList = dataOut.channelIndexList
114 114 else:
115 115 channelIndexList = []
116 116 for channel in channelList:
117 117 if channel not in dataOut.channelList:
118 118 raise ValueError, "Channel %d is not in dataOut.channelList" %channel
119 119 channelIndexList.append(dataOut.channelList.index(channel))
120 120
121 121 if normFactor is None:
122 122 factor = dataOut.normFactor
123 123 else:
124 124 factor = normFactor
125 125 if xaxis == "frequency":
126 126 x = dataOut.getFreqRange(1)/1000.
127 127 xlabel = "Frequency (kHz)"
128 128
129 129 elif xaxis == "time":
130 130 x = dataOut.getAcfRange(1)
131 131 xlabel = "Time (ms)"
132 132
133 133 else:
134 134 x = dataOut.getVelRange(1)
135 135 xlabel = "Velocity (m/s)"
136 136
137 137 ylabel = "Range (Km)"
138 138
139 139 y = dataOut.getHeiRange()
140 140
141 141 z = dataOut.data_spc/factor
142 142 z = numpy.where(numpy.isfinite(z), z, numpy.NAN)
143 143 zdB = 10*numpy.log10(z)
144 144
145 145 avg = numpy.average(z, axis=1)
146 146 avgdB = 10*numpy.log10(avg)
147 147
148 148 noise = dataOut.getNoise()/factor
149 149 noisedB = 10*numpy.log10(noise)
150 150
151 151 thisDatetime = datetime.datetime.utcfromtimestamp(dataOut.getTimeRange()[0])
152 152 title = wintitle + " Spectra"
153 153 if ((dataOut.azimuth!=None) and (dataOut.zenith!=None)):
154 154 title = title + '_' + 'azimuth,zenith=%2.2f,%2.2f'%(dataOut.azimuth, dataOut.zenith)
155 155
156 156 if not self.isConfig:
157 157
158 158 nplots = len(channelIndexList)
159 159
160 160 self.setup(id=id,
161 161 nplots=nplots,
162 162 wintitle=wintitle,
163 163 showprofile=showprofile,
164 164 show=show)
165 165
166 166 if xmin == None: xmin = numpy.nanmin(x)
167 167 if xmax == None: xmax = numpy.nanmax(x)
168 168 if ymin == None: ymin = numpy.nanmin(y)
169 169 if ymax == None: ymax = numpy.nanmax(y)
170 170 if zmin == None: zmin = numpy.floor(numpy.nanmin(noisedB)) - 3
171 171 if zmax == None: zmax = numpy.ceil(numpy.nanmax(avgdB)) + 3
172 172
173 173 self.FTP_WEI = ftp_wei
174 174 self.EXP_CODE = exp_code
175 175 self.SUB_EXP_CODE = sub_exp_code
176 176 self.PLOT_POS = plot_pos
177 177
178 178 self.isConfig = True
179 179
180 180 self.setWinTitle(title)
181 181
182 182 for i in range(self.nplots):
183 183 index = channelIndexList[i]
184 184 str_datetime = '%s %s'%(thisDatetime.strftime("%Y/%m/%d"),thisDatetime.strftime("%H:%M:%S"))
185 185 title = "Channel %d: %4.2fdB: %s" %(dataOut.channelList[index], noisedB[index], str_datetime)
186 186 if len(dataOut.beam.codeList) != 0:
187 187 title = "Ch%d:%4.2fdB,%2.2f,%2.2f:%s" %(dataOut.channelList[index], noisedB[index], dataOut.beam.azimuthList[index], dataOut.beam.zenithList[index], str_datetime)
188 188
189 189 axes = self.axesList[i*self.__nsubplots]
190 190 axes.pcolor(x, y, zdB[index,:,:],
191 191 xmin=xmin, xmax=xmax, ymin=ymin, ymax=ymax, zmin=zmin, zmax=zmax,
192 192 xlabel=xlabel, ylabel=ylabel, title=title, colormap=colormap,
193 193 ticksize=9, cblabel='')
194 194
195 195 if self.__showprofile:
196 196 axes = self.axesList[i*self.__nsubplots +1]
197 197 axes.pline(avgdB[index,:], y,
198 198 xmin=zmin, xmax=zmax, ymin=ymin, ymax=ymax,
199 199 xlabel='dB', ylabel='', title='',
200 200 ytick_visible=False,
201 201 grid='x')
202 202
203 203 noiseline = numpy.repeat(noisedB[index], len(y))
204 204 axes.addpline(noiseline, y, idline=1, color="black", linestyle="dashed", lw=2)
205 205
206 206 self.draw()
207 207
208 208 if figfile == None:
209 209 str_datetime = thisDatetime.strftime("%Y%m%d_%H%M%S")
210 210 name = str_datetime
211 211 if ((dataOut.azimuth!=None) and (dataOut.zenith!=None)):
212 212 name = name + '_az' + '_%2.2f'%(dataOut.azimuth) + '_zn' + '_%2.2f'%(dataOut.zenith)
213 213 figfile = self.getFilename(name)
214 214
215 215 self.save(figpath=figpath,
216 216 figfile=figfile,
217 217 save=save,
218 218 ftp=ftp,
219 219 wr_period=wr_period,
220 220 thisDatetime=thisDatetime)
221 221
222 222 class CrossSpectraPlot(Figure):
223 223
224 224 isConfig = None
225 225 __nsubplots = None
226 226
227 227 WIDTH = None
228 228 HEIGHT = None
229 229 WIDTHPROF = None
230 230 HEIGHTPROF = None
231 231 PREFIX = 'cspc'
232 232
233 233 def __init__(self, **kwargs):
234 234 Figure.__init__(self, **kwargs)
235 235 self.isConfig = False
236 236 self.__nsubplots = 4
237 237 self.counter_imagwr = 0
238 238 self.WIDTH = 250
239 239 self.HEIGHT = 250
240 240 self.WIDTHPROF = 0
241 241 self.HEIGHTPROF = 0
242 242
243 243 self.PLOT_CODE = CROSS_CODE
244 244 self.FTP_WEI = None
245 245 self.EXP_CODE = None
246 246 self.SUB_EXP_CODE = None
247 247 self.PLOT_POS = None
248 248
249 249 def getSubplots(self):
250 250
251 251 ncol = 4
252 252 nrow = self.nplots
253 253
254 254 return nrow, ncol
255 255
256 256 def setup(self, id, nplots, wintitle, showprofile=True, show=True):
257 257
258 258 self.__showprofile = showprofile
259 259 self.nplots = nplots
260 260
261 261 ncolspan = 1
262 262 colspan = 1
263 263
264 264 self.createFigure(id = id,
265 265 wintitle = wintitle,
266 266 widthplot = self.WIDTH + self.WIDTHPROF,
267 267 heightplot = self.HEIGHT + self.HEIGHTPROF,
268 268 show=True)
269 269
270 270 nrow, ncol = self.getSubplots()
271 271
272 272 counter = 0
273 273 for y in range(nrow):
274 274 for x in range(ncol):
275 275 self.addAxes(nrow, ncol*ncolspan, y, x*ncolspan, colspan, 1)
276 276
277 277 counter += 1
278 278
279 279 def run(self, dataOut, id, wintitle="", pairsList=None,
280 280 xmin=None, xmax=None, ymin=None, ymax=None, zmin=None, zmax=None,
281 281 coh_min=None, coh_max=None, phase_min=None, phase_max=None,
282 282 save=False, figpath='./', figfile=None, ftp=False, wr_period=1,
283 283 power_cmap='jet', coherence_cmap='jet', phase_cmap='RdBu_r', show=True,
284 284 server=None, folder=None, username=None, password=None,
285 285 ftp_wei=0, exp_code=0, sub_exp_code=0, plot_pos=0, normFactor=None,
286 286 xaxis='frequency'):
287 287
288 288 """
289 289
290 290 Input:
291 291 dataOut :
292 292 id :
293 293 wintitle :
294 294 channelList :
295 295 showProfile :
296 296 xmin : None,
297 297 xmax : None,
298 298 ymin : None,
299 299 ymax : None,
300 300 zmin : None,
301 301 zmax : None
302 302 """
303 303
304 304 if pairsList == None:
305 305 pairsIndexList = dataOut.pairsIndexList
306 306 else:
307 307 pairsIndexList = []
308 308 for pair in pairsList:
309 309 if pair not in dataOut.pairsList:
310 310 raise ValueError, "Pair %s is not in dataOut.pairsList" %str(pair)
311 311 pairsIndexList.append(dataOut.pairsList.index(pair))
312 312
313 313 if not pairsIndexList:
314 314 return
315 315
316 316 if len(pairsIndexList) > 4:
317 317 pairsIndexList = pairsIndexList[0:4]
318 318
319 319 if normFactor is None:
320 320 factor = dataOut.normFactor
321 321 else:
322 322 factor = normFactor
323 323 x = dataOut.getVelRange(1)
324 324 y = dataOut.getHeiRange()
325 325 z = dataOut.data_spc[:,:,:]/factor
326 326 z = numpy.where(numpy.isfinite(z), z, numpy.NAN)
327 327
328 328 noise = dataOut.noise/factor
329 329
330 330 zdB = 10*numpy.log10(z)
331 331 noisedB = 10*numpy.log10(noise)
332 332
333 333 if coh_min == None:
334 334 coh_min = 0.0
335 335 if coh_max == None:
336 336 coh_max = 1.0
337 337
338 338 if phase_min == None:
339 339 phase_min = -180
340 340 if phase_max == None:
341 341 phase_max = 180
342 342
343 343 #thisDatetime = dataOut.datatime
344 344 thisDatetime = datetime.datetime.utcfromtimestamp(dataOut.getTimeRange()[0])
345 345 title = wintitle + " Cross-Spectra: %s" %(thisDatetime.strftime("%d-%b-%Y %H:%M:%S"))
346 346 # xlabel = "Velocity (m/s)"
347 347 ylabel = "Range (Km)"
348 348
349 349 if xaxis == "frequency":
350 350 x = dataOut.getFreqRange(1)/1000.
351 351 xlabel = "Frequency (kHz)"
352 352
353 353 elif xaxis == "time":
354 354 x = dataOut.getAcfRange(1)
355 355 xlabel = "Time (ms)"
356 356
357 357 else:
358 358 x = dataOut.getVelRange(1)
359 359 xlabel = "Velocity (m/s)"
360 360
361 361 if not self.isConfig:
362 362
363 363 nplots = len(pairsIndexList)
364 364
365 365 self.setup(id=id,
366 366 nplots=nplots,
367 367 wintitle=wintitle,
368 368 showprofile=False,
369 369 show=show)
370 370
371 371 avg = numpy.abs(numpy.average(z, axis=1))
372 372 avgdB = 10*numpy.log10(avg)
373 373
374 374 if xmin == None: xmin = numpy.nanmin(x)
375 375 if xmax == None: xmax = numpy.nanmax(x)
376 376 if ymin == None: ymin = numpy.nanmin(y)
377 377 if ymax == None: ymax = numpy.nanmax(y)
378 378 if zmin == None: zmin = numpy.floor(numpy.nanmin(noisedB)) - 3
379 379 if zmax == None: zmax = numpy.ceil(numpy.nanmax(avgdB)) + 3
380 380
381 381 self.FTP_WEI = ftp_wei
382 382 self.EXP_CODE = exp_code
383 383 self.SUB_EXP_CODE = sub_exp_code
384 384 self.PLOT_POS = plot_pos
385 385
386 386 self.isConfig = True
387 387
388 388 self.setWinTitle(title)
389 389
390 390 for i in range(self.nplots):
391 391 pair = dataOut.pairsList[pairsIndexList[i]]
392 392
393 393 chan_index0 = dataOut.channelList.index(pair[0])
394 394 chan_index1 = dataOut.channelList.index(pair[1])
395 395
396 396 str_datetime = '%s %s'%(thisDatetime.strftime("%Y/%m/%d"),thisDatetime.strftime("%H:%M:%S"))
397 397 title = "Ch%d: %4.2fdB: %s" %(pair[0], noisedB[chan_index0], str_datetime)
398 398 zdB = 10.*numpy.log10(dataOut.data_spc[chan_index0,:,:]/factor)
399 399 axes0 = self.axesList[i*self.__nsubplots]
400 400 axes0.pcolor(x, y, zdB,
401 401 xmin=xmin, xmax=xmax, ymin=ymin, ymax=ymax, zmin=zmin, zmax=zmax,
402 402 xlabel=xlabel, ylabel=ylabel, title=title,
403 403 ticksize=9, colormap=power_cmap, cblabel='')
404 404
405 405 title = "Ch%d: %4.2fdB: %s" %(pair[1], noisedB[chan_index1], str_datetime)
406 406 zdB = 10.*numpy.log10(dataOut.data_spc[chan_index1,:,:]/factor)
407 407 axes0 = self.axesList[i*self.__nsubplots+1]
408 408 axes0.pcolor(x, y, zdB,
409 409 xmin=xmin, xmax=xmax, ymin=ymin, ymax=ymax, zmin=zmin, zmax=zmax,
410 410 xlabel=xlabel, ylabel=ylabel, title=title,
411 411 ticksize=9, colormap=power_cmap, cblabel='')
412 412
413 413 coherenceComplex = dataOut.data_cspc[pairsIndexList[i],:,:]/numpy.sqrt(dataOut.data_spc[chan_index0,:,:]*dataOut.data_spc[chan_index1,:,:])
414 414 coherence = numpy.abs(coherenceComplex)
415 415 # phase = numpy.arctan(-1*coherenceComplex.imag/coherenceComplex.real)*180/numpy.pi
416 416 phase = numpy.arctan2(coherenceComplex.imag, coherenceComplex.real)*180/numpy.pi
417 417
418 418 title = "Coherence Ch%d * Ch%d" %(pair[0], pair[1])
419 419 axes0 = self.axesList[i*self.__nsubplots+2]
420 420 axes0.pcolor(x, y, coherence,
421 421 xmin=xmin, xmax=xmax, ymin=ymin, ymax=ymax, zmin=coh_min, zmax=coh_max,
422 422 xlabel=xlabel, ylabel=ylabel, title=title,
423 423 ticksize=9, colormap=coherence_cmap, cblabel='')
424 424
425 425 title = "Phase Ch%d * Ch%d" %(pair[0], pair[1])
426 426 axes0 = self.axesList[i*self.__nsubplots+3]
427 427 axes0.pcolor(x, y, phase,
428 428 xmin=xmin, xmax=xmax, ymin=ymin, ymax=ymax, zmin=phase_min, zmax=phase_max,
429 429 xlabel=xlabel, ylabel=ylabel, title=title,
430 430 ticksize=9, colormap=phase_cmap, cblabel='')
431 431
432 432
433 433
434 434 self.draw()
435 435
436 436 self.save(figpath=figpath,
437 437 figfile=figfile,
438 438 save=save,
439 439 ftp=ftp,
440 440 wr_period=wr_period,
441 441 thisDatetime=thisDatetime)
442 442
443 443
444 444 class RTIPlot(Figure):
445 445
446 446 __isConfig = None
447 447 __nsubplots = None
448 448
449 449 WIDTHPROF = None
450 450 HEIGHTPROF = None
451 451 PREFIX = 'rti'
452 452
453 453 def __init__(self, **kwargs):
454 454
455 455 Figure.__init__(self, **kwargs)
456 456 self.timerange = None
457 457 self.isConfig = False
458 458 self.__nsubplots = 1
459 459
460 460 self.WIDTH = 800
461 461 self.HEIGHT = 180
462 462 self.WIDTHPROF = 120
463 463 self.HEIGHTPROF = 0
464 464 self.counter_imagwr = 0
465 465
466 466 self.PLOT_CODE = RTI_CODE
467 467
468 468 self.FTP_WEI = None
469 469 self.EXP_CODE = None
470 470 self.SUB_EXP_CODE = None
471 471 self.PLOT_POS = None
472 472 self.tmin = None
473 473 self.tmax = None
474 474
475 475 self.xmin = None
476 476 self.xmax = None
477 477
478 478 self.figfile = None
479 479
480 480 def getSubplots(self):
481 481
482 482 ncol = 1
483 483 nrow = self.nplots
484 484
485 485 return nrow, ncol
486 486
487 487 def setup(self, id, nplots, wintitle, showprofile=True, show=True):
488 488
489 489 self.__showprofile = showprofile
490 490 self.nplots = nplots
491 491
492 492 ncolspan = 1
493 493 colspan = 1
494 494 if showprofile:
495 495 ncolspan = 7
496 496 colspan = 6
497 497 self.__nsubplots = 2
498 498
499 499 self.createFigure(id = id,
500 500 wintitle = wintitle,
501 501 widthplot = self.WIDTH + self.WIDTHPROF,
502 502 heightplot = self.HEIGHT + self.HEIGHTPROF,
503 503 show=show)
504 504
505 505 nrow, ncol = self.getSubplots()
506 506
507 507 counter = 0
508 508 for y in range(nrow):
509 509 for x in range(ncol):
510 510
511 511 if counter >= self.nplots:
512 512 break
513 513
514 514 self.addAxes(nrow, ncol*ncolspan, y, x*ncolspan, colspan, 1)
515 515
516 516 if showprofile:
517 517 self.addAxes(nrow, ncol*ncolspan, y, x*ncolspan+colspan, 1, 1)
518 518
519 519 counter += 1
520 520
521 521 def run(self, dataOut, id, wintitle="", channelList=None, showprofile='True',
522 522 xmin=None, xmax=None, ymin=None, ymax=None, zmin=None, zmax=None,
523 523 timerange=None, colormap='jet',
524 524 save=False, figpath='./', lastone=0,figfile=None, ftp=False, wr_period=1, show=True,
525 525 server=None, folder=None, username=None, password=None,
526 526 ftp_wei=0, exp_code=0, sub_exp_code=0, plot_pos=0, normFactor=None, HEIGHT=None):
527 527
528 528 """
529 529
530 530 Input:
531 531 dataOut :
532 532 id :
533 533 wintitle :
534 534 channelList :
535 535 showProfile :
536 536 xmin : None,
537 537 xmax : None,
538 538 ymin : None,
539 539 ymax : None,
540 540 zmin : None,
541 541 zmax : None
542 542 """
543 543
544 544 #colormap = kwargs.get('colormap', 'jet')
545 545 if HEIGHT is not None:
546 546 self.HEIGHT = HEIGHT
547 547
548 548 if not isTimeInHourRange(dataOut.datatime, xmin, xmax):
549 549 return
550 550
551 551 if channelList == None:
552 552 channelIndexList = dataOut.channelIndexList
553 553 else:
554 554 channelIndexList = []
555 555 for channel in channelList:
556 556 if channel not in dataOut.channelList:
557 557 raise ValueError, "Channel %d is not in dataOut.channelList"
558 558 channelIndexList.append(dataOut.channelList.index(channel))
559 559
560 560 if normFactor is None:
561 561 factor = dataOut.normFactor
562 562 else:
563 563 factor = normFactor
564 564
565 565 # factor = dataOut.normFactor
566 566 x = dataOut.getTimeRange()
567 567 y = dataOut.getHeiRange()
568 568
569 569 z = dataOut.data_spc/factor
570 570 z = numpy.where(numpy.isfinite(z), z, numpy.NAN)
571 571 avg = numpy.average(z, axis=1)
572 572 avgdB = 10.*numpy.log10(avg)
573 573 # avgdB = dataOut.getPower()
574 574
575 575
576 576 thisDatetime = dataOut.datatime
577 577 # thisDatetime = datetime.datetime.utcfromtimestamp(dataOut.getTimeRange()[0])
578 578 title = wintitle + " RTI" #: %s" %(thisDatetime.strftime("%d-%b-%Y"))
579 579 xlabel = ""
580 580 ylabel = "Range (Km)"
581 581
582 582 update_figfile = False
583 583
584 584 if dataOut.ltctime >= self.xmax:
585 585 self.counter_imagwr = wr_period
586 586 self.isConfig = False
587 587 update_figfile = True
588 588
589 589 if not self.isConfig:
590 590
591 591 nplots = len(channelIndexList)
592 592
593 593 self.setup(id=id,
594 594 nplots=nplots,
595 595 wintitle=wintitle,
596 596 showprofile=showprofile,
597 597 show=show)
598 598
599 599 if timerange != None:
600 600 self.timerange = timerange
601 601
602 602 self.xmin, self.xmax = self.getTimeLim(x, xmin, xmax, timerange)
603 603
604 604 noise = dataOut.noise/factor
605 605 noisedB = 10*numpy.log10(noise)
606 606
607 607 if ymin == None: ymin = numpy.nanmin(y)
608 608 if ymax == None: ymax = numpy.nanmax(y)
609 609 if zmin == None: zmin = numpy.floor(numpy.nanmin(noisedB)) - 3
610 610 if zmax == None: zmax = numpy.ceil(numpy.nanmax(avgdB)) + 3
611 611
612 612 self.FTP_WEI = ftp_wei
613 613 self.EXP_CODE = exp_code
614 614 self.SUB_EXP_CODE = sub_exp_code
615 615 self.PLOT_POS = plot_pos
616 616
617 617 self.name = thisDatetime.strftime("%Y%m%d_%H%M%S")
618 618 self.isConfig = True
619 619 self.figfile = figfile
620 620 update_figfile = True
621 621
622 622 self.setWinTitle(title)
623 623
624 624 for i in range(self.nplots):
625 625 index = channelIndexList[i]
626 626 title = "Channel %d: %s" %(dataOut.channelList[index], thisDatetime.strftime("%Y/%m/%d %H:%M:%S"))
627 627 if ((dataOut.azimuth!=None) and (dataOut.zenith!=None)):
628 628 title = title + '_' + 'azimuth,zenith=%2.2f,%2.2f'%(dataOut.azimuth, dataOut.zenith)
629 629 axes = self.axesList[i*self.__nsubplots]
630 630 zdB = avgdB[index].reshape((1,-1))
631 631 axes.pcolorbuffer(x, y, zdB,
632 632 xmin=self.xmin, xmax=self.xmax, ymin=ymin, ymax=ymax, zmin=zmin, zmax=zmax,
633 633 xlabel=xlabel, ylabel=ylabel, title=title, rti=True, XAxisAsTime=True,
634 634 ticksize=9, cblabel='', cbsize="1%", colormap=colormap)
635 635
636 636 if self.__showprofile:
637 637 axes = self.axesList[i*self.__nsubplots +1]
638 638 axes.pline(avgdB[index], y,
639 639 xmin=zmin, xmax=zmax, ymin=ymin, ymax=ymax,
640 640 xlabel='dB', ylabel='', title='',
641 641 ytick_visible=False,
642 642 grid='x')
643 643
644 644 self.draw()
645 645
646 646 self.save(figpath=figpath,
647 647 figfile=figfile,
648 648 save=save,
649 649 ftp=ftp,
650 650 wr_period=wr_period,
651 651 thisDatetime=thisDatetime,
652 652 update_figfile=update_figfile)
653 653
654 654 class CoherenceMap(Figure):
655 655 isConfig = None
656 656 __nsubplots = None
657 657
658 658 WIDTHPROF = None
659 659 HEIGHTPROF = None
660 660 PREFIX = 'cmap'
661 661
662 662 def __init__(self, **kwargs):
663 663 Figure.__init__(self, **kwargs)
664 664 self.timerange = 2*60*60
665 665 self.isConfig = False
666 666 self.__nsubplots = 1
667 667
668 668 self.WIDTH = 800
669 669 self.HEIGHT = 180
670 670 self.WIDTHPROF = 120
671 671 self.HEIGHTPROF = 0
672 672 self.counter_imagwr = 0
673 673
674 674 self.PLOT_CODE = COH_CODE
675 675
676 676 self.FTP_WEI = None
677 677 self.EXP_CODE = None
678 678 self.SUB_EXP_CODE = None
679 679 self.PLOT_POS = None
680 680 self.counter_imagwr = 0
681 681
682 682 self.xmin = None
683 683 self.xmax = None
684 684
685 685 def getSubplots(self):
686 686 ncol = 1
687 687 nrow = self.nplots*2
688 688
689 689 return nrow, ncol
690 690
691 691 def setup(self, id, nplots, wintitle, showprofile=True, show=True):
692 692 self.__showprofile = showprofile
693 693 self.nplots = nplots
694 694
695 695 ncolspan = 1
696 696 colspan = 1
697 697 if showprofile:
698 698 ncolspan = 7
699 699 colspan = 6
700 700 self.__nsubplots = 2
701 701
702 702 self.createFigure(id = id,
703 703 wintitle = wintitle,
704 704 widthplot = self.WIDTH + self.WIDTHPROF,
705 705 heightplot = self.HEIGHT + self.HEIGHTPROF,
706 706 show=True)
707 707
708 708 nrow, ncol = self.getSubplots()
709 709
710 710 for y in range(nrow):
711 711 for x in range(ncol):
712 712
713 713 self.addAxes(nrow, ncol*ncolspan, y, x*ncolspan, colspan, 1)
714 714
715 715 if showprofile:
716 716 self.addAxes(nrow, ncol*ncolspan, y, x*ncolspan+colspan, 1, 1)
717 717
718 718 def run(self, dataOut, id, wintitle="", pairsList=None, showprofile='True',
719 719 xmin=None, xmax=None, ymin=None, ymax=None, zmin=None, zmax=None,
720 720 timerange=None, phase_min=None, phase_max=None,
721 721 save=False, figpath='./', figfile=None, ftp=False, wr_period=1,
722 722 coherence_cmap='jet', phase_cmap='RdBu_r', show=True,
723 723 server=None, folder=None, username=None, password=None,
724 724 ftp_wei=0, exp_code=0, sub_exp_code=0, plot_pos=0):
725 725
726 726 if not isTimeInHourRange(dataOut.datatime, xmin, xmax):
727 727 return
728 728
729 729 if pairsList == None:
730 730 pairsIndexList = dataOut.pairsIndexList
731 731 else:
732 732 pairsIndexList = []
733 733 for pair in pairsList:
734 734 if pair not in dataOut.pairsList:
735 735 raise ValueError, "Pair %s is not in dataOut.pairsList" %(pair)
736 736 pairsIndexList.append(dataOut.pairsList.index(pair))
737 737
738 738 if pairsIndexList == []:
739 739 return
740 740
741 741 if len(pairsIndexList) > 4:
742 742 pairsIndexList = pairsIndexList[0:4]
743 743
744 744 if phase_min == None:
745 745 phase_min = -180
746 746 if phase_max == None:
747 747 phase_max = 180
748 748
749 749 x = dataOut.getTimeRange()
750 750 y = dataOut.getHeiRange()
751 751
752 752 thisDatetime = dataOut.datatime
753 753
754 754 title = wintitle + " CoherenceMap" #: %s" %(thisDatetime.strftime("%d-%b-%Y"))
755 755 xlabel = ""
756 756 ylabel = "Range (Km)"
757 757 update_figfile = False
758 758
759 759 if not self.isConfig:
760 760 nplots = len(pairsIndexList)
761 761 self.setup(id=id,
762 762 nplots=nplots,
763 763 wintitle=wintitle,
764 764 showprofile=showprofile,
765 765 show=show)
766 766
767 767 if timerange != None:
768 768 self.timerange = timerange
769 769
770 770 self.xmin, self.xmax = self.getTimeLim(x, xmin, xmax, timerange)
771 771
772 772 if ymin == None: ymin = numpy.nanmin(y)
773 773 if ymax == None: ymax = numpy.nanmax(y)
774 774 if zmin == None: zmin = 0.
775 775 if zmax == None: zmax = 1.
776 776
777 777 self.FTP_WEI = ftp_wei
778 778 self.EXP_CODE = exp_code
779 779 self.SUB_EXP_CODE = sub_exp_code
780 780 self.PLOT_POS = plot_pos
781 781
782 782 self.name = thisDatetime.strftime("%Y%m%d_%H%M%S")
783 783
784 784 self.isConfig = True
785 785 update_figfile = True
786 786
787 787 self.setWinTitle(title)
788 788
789 789 for i in range(self.nplots):
790 790
791 791 pair = dataOut.pairsList[pairsIndexList[i]]
792 792
793 793 ccf = numpy.average(dataOut.data_cspc[pairsIndexList[i],:,:],axis=0)
794 794 powa = numpy.average(dataOut.data_spc[pair[0],:,:],axis=0)
795 795 powb = numpy.average(dataOut.data_spc[pair[1],:,:],axis=0)
796 796
797 797
798 798 avgcoherenceComplex = ccf/numpy.sqrt(powa*powb)
799 799 coherence = numpy.abs(avgcoherenceComplex)
800 800
801 801 z = coherence.reshape((1,-1))
802 802
803 803 counter = 0
804 804
805 805 title = "Coherence Ch%d * Ch%d: %s" %(pair[0], pair[1], thisDatetime.strftime("%d-%b-%Y %H:%M:%S"))
806 806 axes = self.axesList[i*self.__nsubplots*2]
807 807 axes.pcolorbuffer(x, y, z,
808 808 xmin=self.xmin, xmax=self.xmax, ymin=ymin, ymax=ymax, zmin=zmin, zmax=zmax,
809 809 xlabel=xlabel, ylabel=ylabel, title=title, rti=True, XAxisAsTime=True,
810 810 ticksize=9, cblabel='', colormap=coherence_cmap, cbsize="1%")
811 811
812 812 if self.__showprofile:
813 813 counter += 1
814 814 axes = self.axesList[i*self.__nsubplots*2 + counter]
815 815 axes.pline(coherence, y,
816 816 xmin=zmin, xmax=zmax, ymin=ymin, ymax=ymax,
817 817 xlabel='', ylabel='', title='', ticksize=7,
818 818 ytick_visible=False, nxticks=5,
819 819 grid='x')
820 820
821 821 counter += 1
822 822
823 823 phase = numpy.arctan2(avgcoherenceComplex.imag, avgcoherenceComplex.real)*180/numpy.pi
824 824
825 825 z = phase.reshape((1,-1))
826 826
827 827 title = "Phase Ch%d * Ch%d: %s" %(pair[0], pair[1], thisDatetime.strftime("%d-%b-%Y %H:%M:%S"))
828 828 axes = self.axesList[i*self.__nsubplots*2 + counter]
829 829 axes.pcolorbuffer(x, y, z,
830 830 xmin=self.xmin, xmax=self.xmax, ymin=ymin, ymax=ymax, zmin=phase_min, zmax=phase_max,
831 831 xlabel=xlabel, ylabel=ylabel, title=title, rti=True, XAxisAsTime=True,
832 832 ticksize=9, cblabel='', colormap=phase_cmap, cbsize="1%")
833 833
834 834 if self.__showprofile:
835 835 counter += 1
836 836 axes = self.axesList[i*self.__nsubplots*2 + counter]
837 837 axes.pline(phase, y,
838 838 xmin=phase_min, xmax=phase_max, ymin=ymin, ymax=ymax,
839 839 xlabel='', ylabel='', title='', ticksize=7,
840 840 ytick_visible=False, nxticks=4,
841 841 grid='x')
842 842
843 843 self.draw()
844 844
845 845 if dataOut.ltctime >= self.xmax:
846 846 self.counter_imagwr = wr_period
847 847 self.isConfig = False
848 848 update_figfile = True
849 849
850 850 self.save(figpath=figpath,
851 851 figfile=figfile,
852 852 save=save,
853 853 ftp=ftp,
854 854 wr_period=wr_period,
855 855 thisDatetime=thisDatetime,
856 856 update_figfile=update_figfile)
857 857
858 858 class PowerProfilePlot(Figure):
859 859
860 860 isConfig = None
861 861 __nsubplots = None
862 862
863 863 WIDTHPROF = None
864 864 HEIGHTPROF = None
865 865 PREFIX = 'spcprofile'
866 866
867 867 def __init__(self, **kwargs):
868 868 Figure.__init__(self, **kwargs)
869 869 self.isConfig = False
870 870 self.__nsubplots = 1
871 871
872 872 self.PLOT_CODE = POWER_CODE
873 873
874 874 self.WIDTH = 300
875 875 self.HEIGHT = 500
876 876 self.counter_imagwr = 0
877 877
878 878 def getSubplots(self):
879 879 ncol = 1
880 880 nrow = 1
881 881
882 882 return nrow, ncol
883 883
884 884 def setup(self, id, nplots, wintitle, show):
885 885
886 886 self.nplots = nplots
887 887
888 888 ncolspan = 1
889 889 colspan = 1
890 890
891 891 self.createFigure(id = id,
892 892 wintitle = wintitle,
893 893 widthplot = self.WIDTH,
894 894 heightplot = self.HEIGHT,
895 895 show=show)
896 896
897 897 nrow, ncol = self.getSubplots()
898 898
899 899 counter = 0
900 900 for y in range(nrow):
901 901 for x in range(ncol):
902 902 self.addAxes(nrow, ncol*ncolspan, y, x*ncolspan, colspan, 1)
903 903
904 904 def run(self, dataOut, id, wintitle="", channelList=None,
905 905 xmin=None, xmax=None, ymin=None, ymax=None,
906 906 save=False, figpath='./', figfile=None, show=True,
907 907 ftp=False, wr_period=1, server=None,
908 908 folder=None, username=None, password=None):
909 909
910 910
911 911 if channelList == None:
912 912 channelIndexList = dataOut.channelIndexList
913 913 channelList = dataOut.channelList
914 914 else:
915 915 channelIndexList = []
916 916 for channel in channelList:
917 917 if channel not in dataOut.channelList:
918 918 raise ValueError, "Channel %d is not in dataOut.channelList"
919 919 channelIndexList.append(dataOut.channelList.index(channel))
920 920
921 921 factor = dataOut.normFactor
922 922
923 923 y = dataOut.getHeiRange()
924 924
925 925 #for voltage
926 926 if dataOut.type == 'Voltage':
927 927 x = dataOut.data[channelIndexList,:] * numpy.conjugate(dataOut.data[channelIndexList,:])
928 928 x = x.real
929 929 x = numpy.where(numpy.isfinite(x), x, numpy.NAN)
930 930
931 931 #for spectra
932 932 if dataOut.type == 'Spectra':
933 933 x = dataOut.data_spc[channelIndexList,:,:]/factor
934 934 x = numpy.where(numpy.isfinite(x), x, numpy.NAN)
935 935 x = numpy.average(x, axis=1)
936 936
937 937
938 938 xdB = 10*numpy.log10(x)
939 939
940 940 thisDatetime = datetime.datetime.utcfromtimestamp(dataOut.getTimeRange()[0])
941 941 title = wintitle + " Power Profile %s" %(thisDatetime.strftime("%d-%b-%Y"))
942 942 xlabel = "dB"
943 943 ylabel = "Range (Km)"
944 944
945 945 if not self.isConfig:
946 946
947 947 nplots = 1
948 948
949 949 self.setup(id=id,
950 950 nplots=nplots,
951 951 wintitle=wintitle,
952 952 show=show)
953 953
954 954 if ymin == None: ymin = numpy.nanmin(y)
955 955 if ymax == None: ymax = numpy.nanmax(y)
956 956 if xmin == None: xmin = numpy.nanmin(xdB)*0.9
957 957 if xmax == None: xmax = numpy.nanmax(xdB)*1.1
958 958
959 959 self.isConfig = True
960 960
961 961 self.setWinTitle(title)
962 962
963 963 title = "Power Profile: %s" %(thisDatetime.strftime("%d-%b-%Y %H:%M:%S"))
964 964 axes = self.axesList[0]
965 965
966 966 legendlabels = ["channel %d"%x for x in channelList]
967 967 axes.pmultiline(xdB, y,
968 968 xmin=xmin, xmax=xmax, ymin=ymin, ymax=ymax,
969 969 xlabel=xlabel, ylabel=ylabel, title=title, legendlabels=legendlabels,
970 970 ytick_visible=True, nxticks=5,
971 971 grid='x')
972 972
973 973 self.draw()
974 974
975 975 self.save(figpath=figpath,
976 976 figfile=figfile,
977 977 save=save,
978 978 ftp=ftp,
979 979 wr_period=wr_period,
980 980 thisDatetime=thisDatetime)
981 981
982 982 class SpectraCutPlot(Figure):
983 983
984 984 isConfig = None
985 985 __nsubplots = None
986 986
987 987 WIDTHPROF = None
988 988 HEIGHTPROF = None
989 989 PREFIX = 'spc_cut'
990 990
991 991 def __init__(self, **kwargs):
992 992 Figure.__init__(self, **kwargs)
993 993 self.isConfig = False
994 994 self.__nsubplots = 1
995 995
996 996 self.PLOT_CODE = POWER_CODE
997 997
998 998 self.WIDTH = 700
999 999 self.HEIGHT = 500
1000 1000 self.counter_imagwr = 0
1001 1001
1002 1002 def getSubplots(self):
1003 1003 ncol = 1
1004 1004 nrow = 1
1005 1005
1006 1006 return nrow, ncol
1007 1007
1008 1008 def setup(self, id, nplots, wintitle, show):
1009 1009
1010 1010 self.nplots = nplots
1011 1011
1012 1012 ncolspan = 1
1013 1013 colspan = 1
1014 1014
1015 1015 self.createFigure(id = id,
1016 1016 wintitle = wintitle,
1017 1017 widthplot = self.WIDTH,
1018 1018 heightplot = self.HEIGHT,
1019 1019 show=show)
1020 1020
1021 1021 nrow, ncol = self.getSubplots()
1022 1022
1023 1023 counter = 0
1024 1024 for y in range(nrow):
1025 1025 for x in range(ncol):
1026 1026 self.addAxes(nrow, ncol*ncolspan, y, x*ncolspan, colspan, 1)
1027 1027
1028 1028 def run(self, dataOut, id, wintitle="", channelList=None,
1029 1029 xmin=None, xmax=None, ymin=None, ymax=None,
1030 1030 save=False, figpath='./', figfile=None, show=True,
1031 1031 ftp=False, wr_period=1, server=None,
1032 1032 folder=None, username=None, password=None,
1033 1033 xaxis="frequency"):
1034 1034
1035 1035
1036 1036 if channelList == None:
1037 1037 channelIndexList = dataOut.channelIndexList
1038 1038 channelList = dataOut.channelList
1039 1039 else:
1040 1040 channelIndexList = []
1041 1041 for channel in channelList:
1042 1042 if channel not in dataOut.channelList:
1043 1043 raise ValueError, "Channel %d is not in dataOut.channelList"
1044 1044 channelIndexList.append(dataOut.channelList.index(channel))
1045 1045
1046 1046 factor = dataOut.normFactor
1047 1047
1048 1048 y = dataOut.getHeiRange()
1049 1049
1050 1050 z = dataOut.data_spc/factor
1051 1051 z = numpy.where(numpy.isfinite(z), z, numpy.NAN)
1052 1052
1053 1053 hei_index = numpy.arange(25)*3 + 20
1054 1054
1055 1055 if xaxis == "frequency":
1056 1056 x = dataOut.getFreqRange()/1000.
1057 1057 zdB = 10*numpy.log10(z[0,:,hei_index])
1058 1058 xlabel = "Frequency (kHz)"
1059 1059 ylabel = "Power (dB)"
1060 1060
1061 1061 elif xaxis == "time":
1062 1062 x = dataOut.getAcfRange()
1063 1063 zdB = z[0,:,hei_index]
1064 1064 xlabel = "Time (ms)"
1065 1065 ylabel = "ACF"
1066 1066
1067 1067 else:
1068 1068 x = dataOut.getVelRange()
1069 1069 zdB = 10*numpy.log10(z[0,:,hei_index])
1070 1070 xlabel = "Velocity (m/s)"
1071 1071 ylabel = "Power (dB)"
1072 1072
1073 1073 thisDatetime = datetime.datetime.utcfromtimestamp(dataOut.getTimeRange()[0])
1074 1074 title = wintitle + " Range Cuts %s" %(thisDatetime.strftime("%d-%b-%Y"))
1075 1075
1076 1076 if not self.isConfig:
1077 1077
1078 1078 nplots = 1
1079 1079
1080 1080 self.setup(id=id,
1081 1081 nplots=nplots,
1082 1082 wintitle=wintitle,
1083 1083 show=show)
1084 1084
1085 1085 if xmin == None: xmin = numpy.nanmin(x)*0.9
1086 1086 if xmax == None: xmax = numpy.nanmax(x)*1.1
1087 1087 if ymin == None: ymin = numpy.nanmin(zdB)
1088 1088 if ymax == None: ymax = numpy.nanmax(zdB)
1089 1089
1090 1090 self.isConfig = True
1091 1091
1092 1092 self.setWinTitle(title)
1093 1093
1094 1094 title = "Spectra Cuts: %s" %(thisDatetime.strftime("%d-%b-%Y %H:%M:%S"))
1095 1095 axes = self.axesList[0]
1096 1096
1097 1097 legendlabels = ["Range = %dKm" %y[i] for i in hei_index]
1098 1098
1099 1099 axes.pmultilineyaxis( x, zdB,
1100 1100 xmin=xmin, xmax=xmax, ymin=ymin, ymax=ymax,
1101 1101 xlabel=xlabel, ylabel=ylabel, title=title, legendlabels=legendlabels,
1102 1102 ytick_visible=True, nxticks=5,
1103 1103 grid='x')
1104 1104
1105 1105 self.draw()
1106 1106
1107 1107 self.save(figpath=figpath,
1108 1108 figfile=figfile,
1109 1109 save=save,
1110 1110 ftp=ftp,
1111 1111 wr_period=wr_period,
1112 1112 thisDatetime=thisDatetime)
1113 1113
1114 1114 class Noise(Figure):
1115 1115
1116 1116 isConfig = None
1117 1117 __nsubplots = None
1118 1118
1119 1119 PREFIX = 'noise'
1120 1120
1121 1121
1122 1122 def __init__(self, **kwargs):
1123 1123 Figure.__init__(self, **kwargs)
1124 1124 self.timerange = 24*60*60
1125 1125 self.isConfig = False
1126 1126 self.__nsubplots = 1
1127 1127 self.counter_imagwr = 0
1128 1128 self.WIDTH = 800
1129 1129 self.HEIGHT = 400
1130 1130 self.WIDTHPROF = 120
1131 1131 self.HEIGHTPROF = 0
1132 1132 self.xdata = None
1133 1133 self.ydata = None
1134 1134
1135 1135 self.PLOT_CODE = NOISE_CODE
1136 1136
1137 1137 self.FTP_WEI = None
1138 1138 self.EXP_CODE = None
1139 1139 self.SUB_EXP_CODE = None
1140 1140 self.PLOT_POS = None
1141 1141 self.figfile = None
1142 1142
1143 1143 self.xmin = None
1144 1144 self.xmax = None
1145 1145
1146 1146 def getSubplots(self):
1147 1147
1148 1148 ncol = 1
1149 1149 nrow = 1
1150 1150
1151 1151 return nrow, ncol
1152 1152
1153 1153 def openfile(self, filename):
1154 1154 dirname = os.path.dirname(filename)
1155 1155
1156 1156 if not os.path.exists(dirname):
1157 1157 os.mkdir(dirname)
1158 1158
1159 1159 f = open(filename,'w+')
1160 1160 f.write('\n\n')
1161 1161 f.write('JICAMARCA RADIO OBSERVATORY - Noise \n')
1162 1162 f.write('DD MM YYYY HH MM SS Channel0 Channel1 Channel2 Channel3\n\n' )
1163 1163 f.close()
1164 1164
1165 1165 def save_data(self, filename_phase, data, data_datetime):
1166 1166
1167 1167 f=open(filename_phase,'a')
1168 1168
1169 1169 timetuple_data = data_datetime.timetuple()
1170 1170 day = str(timetuple_data.tm_mday)
1171 1171 month = str(timetuple_data.tm_mon)
1172 1172 year = str(timetuple_data.tm_year)
1173 1173 hour = str(timetuple_data.tm_hour)
1174 1174 minute = str(timetuple_data.tm_min)
1175 1175 second = str(timetuple_data.tm_sec)
1176 1176
1177 1177 data_msg = ''
1178 1178 for i in range(len(data)):
1179 1179 data_msg += str(data[i]) + ' '
1180 1180
1181 1181 f.write(day+' '+month+' '+year+' '+hour+' '+minute+' '+second+' ' + data_msg + '\n')
1182 1182 f.close()
1183 1183
1184 1184
1185 1185 def setup(self, id, nplots, wintitle, showprofile=True, show=True):
1186 1186
1187 1187 self.__showprofile = showprofile
1188 1188 self.nplots = nplots
1189 1189
1190 1190 ncolspan = 7
1191 1191 colspan = 6
1192 1192 self.__nsubplots = 2
1193 1193
1194 1194 self.createFigure(id = id,
1195 1195 wintitle = wintitle,
1196 1196 widthplot = self.WIDTH+self.WIDTHPROF,
1197 1197 heightplot = self.HEIGHT+self.HEIGHTPROF,
1198 1198 show=show)
1199 1199
1200 1200 nrow, ncol = self.getSubplots()
1201 1201
1202 1202 self.addAxes(nrow, ncol*ncolspan, 0, 0, colspan, 1)
1203 1203
1204 1204
1205 1205 def run(self, dataOut, id, wintitle="", channelList=None, showprofile='True',
1206 1206 xmin=None, xmax=None, ymin=None, ymax=None,
1207 1207 timerange=None,
1208 1208 save=False, figpath='./', figfile=None, show=True, ftp=False, wr_period=1,
1209 1209 server=None, folder=None, username=None, password=None,
1210 1210 ftp_wei=0, exp_code=0, sub_exp_code=0, plot_pos=0):
1211 1211
1212 1212 if not isTimeInHourRange(dataOut.datatime, xmin, xmax):
1213 1213 return
1214 1214
1215 1215 if channelList == None:
1216 1216 channelIndexList = dataOut.channelIndexList
1217 1217 channelList = dataOut.channelList
1218 1218 else:
1219 1219 channelIndexList = []
1220 1220 for channel in channelList:
1221 1221 if channel not in dataOut.channelList:
1222 1222 raise ValueError, "Channel %d is not in dataOut.channelList"
1223 1223 channelIndexList.append(dataOut.channelList.index(channel))
1224 1224
1225 1225 x = dataOut.getTimeRange()
1226 1226 #y = dataOut.getHeiRange()
1227 1227 factor = dataOut.normFactor
1228 1228 noise = dataOut.noise[channelIndexList]/factor
1229 1229 noisedB = 10*numpy.log10(noise)
1230 1230
1231 1231 thisDatetime = dataOut.datatime
1232 1232
1233 1233 title = wintitle + " Noise" # : %s" %(thisDatetime.strftime("%d-%b-%Y"))
1234 1234 xlabel = ""
1235 1235 ylabel = "Intensity (dB)"
1236 1236 update_figfile = False
1237 1237
1238 1238 if not self.isConfig:
1239 1239
1240 1240 nplots = 1
1241 1241
1242 1242 self.setup(id=id,
1243 1243 nplots=nplots,
1244 1244 wintitle=wintitle,
1245 1245 showprofile=showprofile,
1246 1246 show=show)
1247 1247
1248 1248 if timerange != None:
1249 1249 self.timerange = timerange
1250 1250
1251 1251 self.xmin, self.xmax = self.getTimeLim(x, xmin, xmax, timerange)
1252 1252
1253 1253 if ymin == None: ymin = numpy.floor(numpy.nanmin(noisedB)) - 10.0
1254 1254 if ymax == None: ymax = numpy.nanmax(noisedB) + 10.0
1255 1255
1256 1256 self.FTP_WEI = ftp_wei
1257 1257 self.EXP_CODE = exp_code
1258 1258 self.SUB_EXP_CODE = sub_exp_code
1259 1259 self.PLOT_POS = plot_pos
1260 1260
1261 1261
1262 1262 self.name = thisDatetime.strftime("%Y%m%d_%H%M%S")
1263 1263 self.isConfig = True
1264 1264 self.figfile = figfile
1265 1265 self.xdata = numpy.array([])
1266 1266 self.ydata = numpy.array([])
1267 1267
1268 1268 update_figfile = True
1269 1269
1270 1270 #open file beacon phase
1271 1271 path = '%s%03d' %(self.PREFIX, self.id)
1272 1272 noise_file = os.path.join(path,'%s.txt'%self.name)
1273 1273 self.filename_noise = os.path.join(figpath,noise_file)
1274 1274
1275 1275 self.setWinTitle(title)
1276 1276
1277 1277 title = "Noise %s" %(thisDatetime.strftime("%Y/%m/%d %H:%M:%S"))
1278 1278
1279 1279 legendlabels = ["channel %d"%(idchannel) for idchannel in channelList]
1280 1280 axes = self.axesList[0]
1281 1281
1282 1282 self.xdata = numpy.hstack((self.xdata, x[0:1]))
1283 1283
1284 1284 if len(self.ydata)==0:
1285 1285 self.ydata = noisedB.reshape(-1,1)
1286 1286 else:
1287 1287 self.ydata = numpy.hstack((self.ydata, noisedB.reshape(-1,1)))
1288 1288
1289 1289
1290 1290 axes.pmultilineyaxis(x=self.xdata, y=self.ydata,
1291 1291 xmin=self.xmin, xmax=self.xmax, ymin=ymin, ymax=ymax,
1292 1292 xlabel=xlabel, ylabel=ylabel, title=title, legendlabels=legendlabels, marker='x', markersize=8, linestyle="solid",
1293 1293 XAxisAsTime=True, grid='both'
1294 1294 )
1295 1295
1296 1296 self.draw()
1297 1297
1298 1298 if dataOut.ltctime >= self.xmax:
1299 1299 self.counter_imagwr = wr_period
1300 1300 self.isConfig = False
1301 1301 update_figfile = True
1302 1302
1303 1303 self.save(figpath=figpath,
1304 1304 figfile=figfile,
1305 1305 save=save,
1306 1306 ftp=ftp,
1307 1307 wr_period=wr_period,
1308 1308 thisDatetime=thisDatetime,
1309 1309 update_figfile=update_figfile)
1310 1310
1311 1311 #store data beacon phase
1312 1312 if save:
1313 1313 self.save_data(self.filename_noise, noisedB, thisDatetime)
1314 1314
1315 1315 class BeaconPhase(Figure):
1316 1316
1317 1317 __isConfig = None
1318 1318 __nsubplots = None
1319 1319
1320 1320 PREFIX = 'beacon_phase'
1321 1321
1322 1322 def __init__(self, **kwargs):
1323 1323 Figure.__init__(self, **kwargs)
1324 1324 self.timerange = 24*60*60
1325 1325 self.isConfig = False
1326 1326 self.__nsubplots = 1
1327 1327 self.counter_imagwr = 0
1328 1328 self.WIDTH = 800
1329 1329 self.HEIGHT = 400
1330 1330 self.WIDTHPROF = 120
1331 1331 self.HEIGHTPROF = 0
1332 1332 self.xdata = None
1333 1333 self.ydata = None
1334 1334
1335 1335 self.PLOT_CODE = BEACON_CODE
1336 1336
1337 1337 self.FTP_WEI = None
1338 1338 self.EXP_CODE = None
1339 1339 self.SUB_EXP_CODE = None
1340 1340 self.PLOT_POS = None
1341 1341
1342 1342 self.filename_phase = None
1343 1343
1344 1344 self.figfile = None
1345 1345
1346 1346 self.xmin = None
1347 1347 self.xmax = None
1348 1348
1349 1349 def getSubplots(self):
1350 1350
1351 1351 ncol = 1
1352 1352 nrow = 1
1353 1353
1354 1354 return nrow, ncol
1355 1355
1356 1356 def setup(self, id, nplots, wintitle, showprofile=True, show=True):
1357 1357
1358 1358 self.__showprofile = showprofile
1359 1359 self.nplots = nplots
1360 1360
1361 1361 ncolspan = 7
1362 1362 colspan = 6
1363 1363 self.__nsubplots = 2
1364 1364
1365 1365 self.createFigure(id = id,
1366 1366 wintitle = wintitle,
1367 1367 widthplot = self.WIDTH+self.WIDTHPROF,
1368 1368 heightplot = self.HEIGHT+self.HEIGHTPROF,
1369 1369 show=show)
1370 1370
1371 1371 nrow, ncol = self.getSubplots()
1372 1372
1373 1373 self.addAxes(nrow, ncol*ncolspan, 0, 0, colspan, 1)
1374 1374
1375 1375 def save_phase(self, filename_phase):
1376 1376 f = open(filename_phase,'w+')
1377 1377 f.write('\n\n')
1378 1378 f.write('JICAMARCA RADIO OBSERVATORY - Beacon Phase \n')
1379 1379 f.write('DD MM YYYY HH MM SS pair(2,0) pair(2,1) pair(2,3) pair(2,4)\n\n' )
1380 1380 f.close()
1381 1381
1382 1382 def save_data(self, filename_phase, data, data_datetime):
1383 1383 f=open(filename_phase,'a')
1384 1384 timetuple_data = data_datetime.timetuple()
1385 1385 day = str(timetuple_data.tm_mday)
1386 1386 month = str(timetuple_data.tm_mon)
1387 1387 year = str(timetuple_data.tm_year)
1388 1388 hour = str(timetuple_data.tm_hour)
1389 1389 minute = str(timetuple_data.tm_min)
1390 1390 second = str(timetuple_data.tm_sec)
1391 1391 f.write(day+' '+month+' '+year+' '+hour+' '+minute+' '+second+' '+str(data[0])+' '+str(data[1])+' '+str(data[2])+' '+str(data[3])+'\n')
1392 1392 f.close()
1393 1393
1394 1394
1395 1395 def run(self, dataOut, id, wintitle="", pairsList=None, showprofile='True',
1396 1396 xmin=None, xmax=None, ymin=None, ymax=None, hmin=None, hmax=None,
1397 1397 timerange=None,
1398 1398 save=False, figpath='./', figfile=None, show=True, ftp=False, wr_period=1,
1399 1399 server=None, folder=None, username=None, password=None,
1400 1400 ftp_wei=0, exp_code=0, sub_exp_code=0, plot_pos=0):
1401 1401
1402 1402 if not isTimeInHourRange(dataOut.datatime, xmin, xmax):
1403 1403 return
1404 1404
1405 1405 if pairsList == None:
1406 1406 pairsIndexList = dataOut.pairsIndexList[:10]
1407 1407 else:
1408 1408 pairsIndexList = []
1409 1409 for pair in pairsList:
1410 1410 if pair not in dataOut.pairsList:
1411 1411 raise ValueError, "Pair %s is not in dataOut.pairsList" %(pair)
1412 1412 pairsIndexList.append(dataOut.pairsList.index(pair))
1413 1413
1414 1414 if pairsIndexList == []:
1415 1415 return
1416 1416
1417 1417 # if len(pairsIndexList) > 4:
1418 1418 # pairsIndexList = pairsIndexList[0:4]
1419 1419
1420 1420 hmin_index = None
1421 1421 hmax_index = None
1422 1422
1423 1423 if hmin != None and hmax != None:
1424 1424 indexes = numpy.arange(dataOut.nHeights)
1425 1425 hmin_list = indexes[dataOut.heightList >= hmin]
1426 1426 hmax_list = indexes[dataOut.heightList <= hmax]
1427 1427
1428 1428 if hmin_list.any():
1429 1429 hmin_index = hmin_list[0]
1430 1430
1431 1431 if hmax_list.any():
1432 1432 hmax_index = hmax_list[-1]+1
1433 1433
1434 1434 x = dataOut.getTimeRange()
1435 1435 #y = dataOut.getHeiRange()
1436 1436
1437 1437
1438 1438 thisDatetime = dataOut.datatime
1439 1439
1440 1440 title = wintitle + " Signal Phase" # : %s" %(thisDatetime.strftime("%d-%b-%Y"))
1441 1441 xlabel = "Local Time"
1442 1442 ylabel = "Phase (degrees)"
1443 1443
1444 1444 update_figfile = False
1445 1445
1446 1446 nplots = len(pairsIndexList)
1447 1447 #phase = numpy.zeros((len(pairsIndexList),len(dataOut.beacon_heiIndexList)))
1448 1448 phase_beacon = numpy.zeros(len(pairsIndexList))
1449 1449 for i in range(nplots):
1450 1450 pair = dataOut.pairsList[pairsIndexList[i]]
1451 1451 ccf = numpy.average(dataOut.data_cspc[pairsIndexList[i], :, hmin_index:hmax_index], axis=0)
1452 1452 powa = numpy.average(dataOut.data_spc[pair[0], :, hmin_index:hmax_index], axis=0)
1453 1453 powb = numpy.average(dataOut.data_spc[pair[1], :, hmin_index:hmax_index], axis=0)
1454 1454 avgcoherenceComplex = ccf/numpy.sqrt(powa*powb)
1455 1455 phase = numpy.arctan2(avgcoherenceComplex.imag, avgcoherenceComplex.real)*180/numpy.pi
1456 1456
1457 1457 #print "Phase %d%d" %(pair[0], pair[1])
1458 1458 #print phase[dataOut.beacon_heiIndexList]
1459 1459
1460 1460 if dataOut.beacon_heiIndexList:
1461 1461 phase_beacon[i] = numpy.average(phase[dataOut.beacon_heiIndexList])
1462 1462 else:
1463 1463 phase_beacon[i] = numpy.average(phase)
1464 1464
1465 1465 if not self.isConfig:
1466 1466
1467 1467 nplots = len(pairsIndexList)
1468 1468
1469 1469 self.setup(id=id,
1470 1470 nplots=nplots,
1471 1471 wintitle=wintitle,
1472 1472 showprofile=showprofile,
1473 1473 show=show)
1474 1474
1475 1475 if timerange != None:
1476 1476 self.timerange = timerange
1477 1477
1478 1478 self.xmin, self.xmax = self.getTimeLim(x, xmin, xmax, timerange)
1479 1479
1480 1480 if ymin == None: ymin = 0
1481 1481 if ymax == None: ymax = 360
1482 1482
1483 1483 self.FTP_WEI = ftp_wei
1484 1484 self.EXP_CODE = exp_code
1485 1485 self.SUB_EXP_CODE = sub_exp_code
1486 1486 self.PLOT_POS = plot_pos
1487 1487
1488 1488 self.name = thisDatetime.strftime("%Y%m%d_%H%M%S")
1489 1489 self.isConfig = True
1490 1490 self.figfile = figfile
1491 1491 self.xdata = numpy.array([])
1492 1492 self.ydata = numpy.array([])
1493 1493
1494 1494 update_figfile = True
1495 1495
1496 1496 #open file beacon phase
1497 1497 path = '%s%03d' %(self.PREFIX, self.id)
1498 1498 beacon_file = os.path.join(path,'%s.txt'%self.name)
1499 1499 self.filename_phase = os.path.join(figpath,beacon_file)
1500 1500 #self.save_phase(self.filename_phase)
1501 1501
1502 1502
1503 1503 #store data beacon phase
1504 1504 #self.save_data(self.filename_phase, phase_beacon, thisDatetime)
1505 1505
1506 1506 self.setWinTitle(title)
1507 1507
1508 1508
1509 1509 title = "Phase Plot %s" %(thisDatetime.strftime("%Y/%m/%d %H:%M:%S"))
1510 1510
1511 1511 legendlabels = ["Pair (%d,%d)"%(pair[0], pair[1]) for pair in dataOut.pairsList]
1512 1512
1513 1513 axes = self.axesList[0]
1514 1514
1515 1515 self.xdata = numpy.hstack((self.xdata, x[0:1]))
1516 1516
1517 1517 if len(self.ydata)==0:
1518 1518 self.ydata = phase_beacon.reshape(-1,1)
1519 1519 else:
1520 1520 self.ydata = numpy.hstack((self.ydata, phase_beacon.reshape(-1,1)))
1521 1521
1522 1522
1523 1523 axes.pmultilineyaxis(x=self.xdata, y=self.ydata,
1524 1524 xmin=self.xmin, xmax=self.xmax, ymin=ymin, ymax=ymax,
1525 1525 xlabel=xlabel, ylabel=ylabel, title=title, legendlabels=legendlabels, marker='x', markersize=8, linestyle="solid",
1526 1526 XAxisAsTime=True, grid='both'
1527 1527 )
1528 1528
1529 1529 self.draw()
1530 1530
1531 1531 if dataOut.ltctime >= self.xmax:
1532 1532 self.counter_imagwr = wr_period
1533 1533 self.isConfig = False
1534 1534 update_figfile = True
1535 1535
1536 1536 self.save(figpath=figpath,
1537 1537 figfile=figfile,
1538 1538 save=save,
1539 1539 ftp=ftp,
1540 1540 wr_period=wr_period,
1541 1541 thisDatetime=thisDatetime,
1542 1542 update_figfile=update_figfile)
Show file before | Show file after | Hide comments
@@ -1,21 +1,23
1 1 '''
2 2
3 3 $Author: murco $
4 4 $Id: JRODataIO.py 169 2012-11-19 21:57:03Z murco $
5 5 '''
6 6
7 7 from jroIO_voltage import *
8 8 from jroIO_spectra import *
9 9 from jroIO_heispectra import *
10 10 from jroIO_usrp import *
11 11 from jroIO_digitalRF import *
12 12 from jroIO_kamisr import *
13 13 from jroIO_param import *
14 14 from jroIO_hf import *
15 15
16 16 from jroIO_madrigal import *
17 17
18 18 from bltrIO_param import *
19 19 from jroIO_bltr import *
20 20 from jroIO_mira35c import *
21 from julIO_param import * No newline at end of file
21 from julIO_param import *
22
23 from pxIO_param import * No newline at end of file
Show file before | Show file after | Hide comments
@@ -1,15 +1,16
1 1 '''
2 2
3 3 $Author: murco $
4 4 $Id: Processor.py 1 2012-11-12 18:56:07Z murco $
5 5 '''
6 6
7 7 from jroproc_voltage import *
8 8 from jroproc_spectra import *
9 9 from jroproc_heispectra import *
10 10 from jroproc_amisr import *
11 11 from jroproc_correlation import *
12 12 from jroproc_parameters import *
13 13 from jroproc_spectra_lags import *
14 14 from jroproc_spectra_acf import *
15 15 from bltrproc_parameters import *
16 from pxproc_parameters import *
Show file before | Show file after | Hide comments
@@ -1,661 +1,864
1 1 '''
2 2 @author: Juan C. Espinoza
3 3 '''
4 4
5 import os
6 import glob
5 7 import time
6 8 import json
7 9 import numpy
8 10 import paho.mqtt.client as mqtt
9 11 import zmq
10 12 import datetime
13 import ftplib
11 14 from zmq.utils.monitor import recv_monitor_message
12 15 from functools import wraps
13 16 from threading import Thread
14 17 from multiprocessing import Process
15 18
16 19 from schainpy.model.proc.jroproc_base import Operation, ProcessingUnit
17 20 from schainpy.model.data.jrodata import JROData
18 21 from schainpy.utils import log
19 22
20 MAXNUMX = 100
21 MAXNUMY = 100
22
23 class PrettyFloat(float):
24 def __repr__(self):
25 return '%.2f' % self
23 MAXNUMX = 500
24 MAXNUMY = 500
25
26 PLOT_CODES = {
27 'rti': 0, # Range time intensity (RTI).
28 'spc': 1, # Spectra (and Cross-spectra) information.
29 'cspc': 2, # Cross-Correlation information.
30 'coh': 3, # Coherence map.
31 'base': 4, # Base lines graphic.
32 'row': 5, # Row Spectra.
33 'total': 6, # Total Power.
34 'drift': 7, # Drifts graphics.
35 'height': 8, # Height profile.
36 'phase': 9, # Signal Phase.
37 'power': 16,
38 'noise': 17,
39 'beacon': 18,
40 'wind': 22,
41 'skymap': 23,
42 'Unknown': 24,
43 'V-E': 25, # PIP Velocity.
44 'Z-E': 26, # PIP Reflectivity.
45 'V-A': 27, # RHI Velocity.
46 'Z-A': 28, # RHI Reflectivity.
47 }
48
49 def get_plot_code(s):
50 label = s.split('_')[0]
51 codes = [key for key in PLOT_CODES if key in label]
52 if codes:
53 return PLOT_CODES[codes[0]]
54 else:
55 return 24
26 56
27 57 def roundFloats(obj):
28 58 if isinstance(obj, list):
29 59 return map(roundFloats, obj)
30 60 elif isinstance(obj, float):
31 61 return round(obj, 2)
32 62
33 63 def decimate(z, MAXNUMY):
34 64 dy = int(len(z[0])/MAXNUMY) + 1
35 65
36 66 return z[::, ::dy]
37 67
38 68 class throttle(object):
39 69 '''
40 70 Decorator that prevents a function from being called more than once every
41 71 time period.
42 72 To create a function that cannot be called more than once a minute, but
43 73 will sleep until it can be called:
44 74 @throttle(minutes=1)
45 75 def foo():
46 76 pass
47 77
48 78 for i in range(10):
49 79 foo()
50 80 print "This function has run %s times." % i
51 81 '''
52 82
53 83 def __init__(self, seconds=0, minutes=0, hours=0):
54 84 self.throttle_period = datetime.timedelta(
55 85 seconds=seconds, minutes=minutes, hours=hours
56 86 )
57 87
58 88 self.time_of_last_call = datetime.datetime.min
59 89
60 90 def __call__(self, fn):
61 91 @wraps(fn)
62 92 def wrapper(*args, **kwargs):
63 93 coerce = kwargs.pop('coerce', None)
64 94 if coerce:
65 95 self.time_of_last_call = datetime.datetime.now()
66 96 return fn(*args, **kwargs)
67 97 else:
68 98 now = datetime.datetime.now()
69 99 time_since_last_call = now - self.time_of_last_call
70 100 time_left = self.throttle_period - time_since_last_call
71 101
72 102 if time_left > datetime.timedelta(seconds=0):
73 103 return
74 104
75 105 self.time_of_last_call = datetime.datetime.now()
76 106 return fn(*args, **kwargs)
77 107
78 108 return wrapper
79 109
80 110 class Data(object):
81 111 '''
82 112 Object to hold data to be plotted
83 113 '''
84 114
85 def __init__(self, plottypes, throttle_value, exp_code):
115 def __init__(self, plottypes, throttle_value, exp_code, buffering=True):
86 116 self.plottypes = plottypes
87 117 self.throttle = throttle_value
88 118 self.exp_code = exp_code
119 self.buffering = buffering
89 120 self.ended = False
90 121 self.localtime = False
122 self.meta = {}
91 123 self.__times = []
92 124 self.__heights = []
93 125
94 126 def __str__(self):
95 127 dum = ['{}{}'.format(key, self.shape(key)) for key in self.data]
96 128 return 'Data[{}][{}]'.format(';'.join(dum), len(self.__times))
97 129
98 130 def __len__(self):
99 131 return len(self.__times)
100 132
101 133 def __getitem__(self, key):
102 134 if key not in self.data:
103 135 raise KeyError(log.error('Missing key: {}'.format(key)))
104 136
105 if 'spc' in key:
137 if 'spc' in key or not self.buffering:
106 138 ret = self.data[key]
107 139 else:
108 140 ret = numpy.array([self.data[key][x] for x in self.times])
109 141 if ret.ndim > 1:
110 142 ret = numpy.swapaxes(ret, 0, 1)
111 143 return ret
112 144
113 145 def __contains__(self, key):
114 146 return key in self.data
115 147
116 148 def setup(self):
117 149 '''
118 150 Configure object
119 151 '''
120 152
153 self.type = ''
121 154 self.ended = False
122 155 self.data = {}
123 156 self.__times = []
124 157 self.__heights = []
125 158 self.__all_heights = set()
126 159 for plot in self.plottypes:
127 160 if 'snr' in plot:
128 161 plot = 'snr'
129 162 self.data[plot] = {}
130 163
131 164 def shape(self, key):
132 165 '''
133 166 Get the shape of the one-element data for the given key
134 167 '''
135 168
136 169 if len(self.data[key]):
137 if 'spc' in key:
170 if 'spc' in key or not self.buffering:
138 171 return self.data[key].shape
139 172 return self.data[key][self.__times[0]].shape
140 173 return (0,)
141 174
142 175 def update(self, dataOut, tm):
143 176 '''
144 177 Update data object with new dataOut
145 178 '''
146 179
147 180 if tm in self.__times:
148 181 return
149 182
183 self.type = dataOut.type
150 184 self.parameters = getattr(dataOut, 'parameters', [])
151 185 if hasattr(dataOut, 'pairsList'):
152 186 self.pairs = dataOut.pairsList
187 if hasattr(dataOut, 'meta'):
188 self.meta = dataOut.meta
153 189 self.channels = dataOut.channelList
154 190 self.interval = dataOut.getTimeInterval()
155 191 self.localtime = dataOut.useLocalTime
156 192 if 'spc' in self.plottypes or 'cspc' in self.plottypes:
157 193 self.xrange = (dataOut.getFreqRange(1)/1000., dataOut.getAcfRange(1), dataOut.getVelRange(1))
158 194 self.__heights.append(dataOut.heightList)
159 195 self.__all_heights.update(dataOut.heightList)
160 196 self.__times.append(tm)
161 197
162 198 for plot in self.plottypes:
163 199 if plot == 'spc':
164 200 z = dataOut.data_spc/dataOut.normFactor
165 self.data[plot] = 10*numpy.log10(z)
201 buffer = 10*numpy.log10(z)
166 202 if plot == 'cspc':
167 self.data[plot] = dataOut.data_cspc
203 buffer = dataOut.data_cspc
168 204 if plot == 'noise':
169 self.data[plot][tm] = 10*numpy.log10(dataOut.getNoise()/dataOut.normFactor)
205 buffer = 10*numpy.log10(dataOut.getNoise()/dataOut.normFactor)
170 206 if plot == 'rti':
171 self.data[plot][tm] = dataOut.getPower()
207 buffer = dataOut.getPower()
172 208 if plot == 'snr_db':
173 self.data['snr'][tm] = dataOut.data_SNR
209 buffer = dataOut.data_SNR
174 210 if plot == 'snr':
175 self.data[plot][tm] = 10*numpy.log10(dataOut.data_SNR)
211 buffer = 10*numpy.log10(dataOut.data_SNR)
176 212 if plot == 'dop':
177 self.data[plot][tm] = 10*numpy.log10(dataOut.data_DOP)
213 buffer = 10*numpy.log10(dataOut.data_DOP)
178 214 if plot == 'mean':
179 self.data[plot][tm] = dataOut.data_MEAN
215 buffer = dataOut.data_MEAN
180 216 if plot == 'std':
181 self.data[plot][tm] = dataOut.data_STD
217 buffer = dataOut.data_STD
182 218 if plot == 'coh':
183 self.data[plot][tm] = dataOut.getCoherence()
219 buffer = dataOut.getCoherence()
184 220 if plot == 'phase':
185 self.data[plot][tm] = dataOut.getCoherence(phase=True)
221 buffer = dataOut.getCoherence(phase=True)
186 222 if plot == 'output':
187 self.data[plot][tm] = dataOut.data_output
223 buffer = dataOut.data_output
188 224 if plot == 'param':
189 self.data[plot][tm] = dataOut.data_param
225 buffer = dataOut.data_param
226
227 if 'spc' in plot:
228 self.data[plot] = buffer
229 else:
230 if self.buffering:
231 self.data[plot][tm] = buffer
232 else:
233 self.data[plot] = buffer
190 234
191 235 def normalize_heights(self):
192 236 '''
193 237 Ensure same-dimension of the data for different heighList
194 238 '''
195 239
196 240 H = numpy.array(list(self.__all_heights))
197 241 H.sort()
198 242 for key in self.data:
199 243 shape = self.shape(key)[:-1] + H.shape
200 244 for tm, obj in self.data[key].items():
201 245 h = self.__heights[self.__times.index(tm)]
202 246 if H.size == h.size:
203 247 continue
204 248 index = numpy.where(numpy.in1d(H, h))[0]
205 249 dummy = numpy.zeros(shape) + numpy.nan
206 250 if len(shape) == 2:
207 251 dummy[:, index] = obj
208 252 else:
209 253 dummy[index] = obj
210 254 self.data[key][tm] = dummy
211 255
212 256 self.__heights = [H for tm in self.__times]
213 257
214 258 def jsonify(self, decimate=False):
215 259 '''
216 260 Convert data to json
217 261 '''
218 262
219 263 data = {}
220 264 tm = self.times[-1]
221 265 dy = int(self.heights.size/MAXNUMY) + 1
222 266 for key in self.data:
223 if key in ('spc', 'cspc'):
267 if key in ('spc', 'cspc') or not self.buffering:
224 268 dx = int(self.data[key].shape[1]/MAXNUMX) + 1
225 269 data[key] = roundFloats(self.data[key][::, ::dx, ::dy].tolist())
226 270 else:
227 271 data[key] = roundFloats(self.data[key][tm].tolist())
228 272
229 273 ret = {'data': data}
230 274 ret['exp_code'] = self.exp_code
231 275 ret['time'] = tm
232 276 ret['interval'] = self.interval
233 277 ret['localtime'] = self.localtime
234 278 ret['yrange'] = roundFloats(self.heights[::dy].tolist())
235 279 if 'spc' in self.data or 'cspc' in self.data:
236 280 ret['xrange'] = roundFloats(self.xrange[2][::dx].tolist())
237 281 else:
238 282 ret['xrange'] = []
239 283 if hasattr(self, 'pairs'):
240 284 ret['pairs'] = self.pairs
241 285 else:
242 286 ret['pairs'] = []
287
288 for key, value in self.meta.items():
289 ret[key] = value
290
243 291 return json.dumps(ret)
244 292
245 293 @property
246 294 def times(self):
247 295 '''
248 296 Return the list of times of the current data
249 297 '''
250 298
251 299 ret = numpy.array(self.__times)
252 300 ret.sort()
253 301 return ret
254 302
255 303 @property
256 304 def heights(self):
257 305 '''
258 306 Return the list of heights of the current data
259 307 '''
260 308
261 309 return numpy.array(self.__heights[-1])
262 310
263 311 class PublishData(Operation):
264 312 '''
265 313 Operation to send data over zmq.
266 314 '''
267 315
268 316 __attrs__ = ['host', 'port', 'delay', 'zeromq', 'mqtt', 'verbose']
269 317
270 318 def __init__(self, **kwargs):
271 319 """Inicio."""
272 320 Operation.__init__(self, **kwargs)
273 321 self.isConfig = False
274 322 self.client = None
275 323 self.zeromq = None
276 324 self.mqtt = None
277 325
278 326 def on_disconnect(self, client, userdata, rc):
279 327 if rc != 0:
280 328 log.warning('Unexpected disconnection.')
281 329 self.connect()
282 330
283 331 def connect(self):
284 332 log.warning('trying to connect')
285 333 try:
286 334 self.client.connect(
287 335 host=self.host,
288 336 port=self.port,
289 337 keepalive=60*10,
290 338 bind_address='')
291 339 self.client.loop_start()
292 340 # self.client.publish(
293 341 # self.topic + 'SETUP',
294 342 # json.dumps(setup),
295 343 # retain=True
296 344 # )
297 345 except:
298 346 log.error('MQTT Conection error.')
299 347 self.client = False
300 348
301 349 def setup(self, port=1883, username=None, password=None, clientId="user", zeromq=1, verbose=True, **kwargs):
302 350 self.counter = 0
303 351 self.topic = kwargs.get('topic', 'schain')
304 352 self.delay = kwargs.get('delay', 0)
305 353 self.plottype = kwargs.get('plottype', 'spectra')
306 354 self.host = kwargs.get('host', "10.10.10.82")
307 355 self.port = kwargs.get('port', 3000)
308 356 self.clientId = clientId
309 357 self.cnt = 0
310 358 self.zeromq = zeromq
311 359 self.mqtt = kwargs.get('plottype', 0)
312 360 self.client = None
313 361 self.verbose = verbose
314 362 setup = []
315 363 if mqtt is 1:
316 364 self.client = mqtt.Client(
317 365 client_id=self.clientId + self.topic + 'SCHAIN',
318 366 clean_session=True)
319 367 self.client.on_disconnect = self.on_disconnect
320 368 self.connect()
321 369 for plot in self.plottype:
322 370 setup.append({
323 371 'plot': plot,
324 372 'topic': self.topic + plot,
325 373 'title': getattr(self, plot + '_' + 'title', False),
326 374 'xlabel': getattr(self, plot + '_' + 'xlabel', False),
327 375 'ylabel': getattr(self, plot + '_' + 'ylabel', False),
328 376 'xrange': getattr(self, plot + '_' + 'xrange', False),
329 377 'yrange': getattr(self, plot + '_' + 'yrange', False),
330 378 'zrange': getattr(self, plot + '_' + 'zrange', False),
331 379 })
332 380 if zeromq is 1:
333 381 context = zmq.Context()
334 382 self.zmq_socket = context.socket(zmq.PUSH)
335 383 server = kwargs.get('server', 'zmq.pipe')
336 384
337 385 if 'tcp://' in server:
338 386 address = server
339 387 else:
340 388 address = 'ipc:///tmp/%s' % server
341 389
342 390 self.zmq_socket.connect(address)
343 391 time.sleep(1)
344 392
345 393
346 394 def publish_data(self):
347 395 self.dataOut.finished = False
348 396 if self.mqtt is 1:
349 397 yData = self.dataOut.heightList[:2].tolist()
350 398 if self.plottype == 'spectra':
351 399 data = getattr(self.dataOut, 'data_spc')
352 400 z = data/self.dataOut.normFactor
353 401 zdB = 10*numpy.log10(z)
354 402 xlen, ylen = zdB[0].shape
355 403 dx = int(xlen/MAXNUMX) + 1
356 404 dy = int(ylen/MAXNUMY) + 1
357 405 Z = [0 for i in self.dataOut.channelList]
358 406 for i in self.dataOut.channelList:
359 407 Z[i] = zdB[i][::dx, ::dy].tolist()
360 408 payload = {
361 409 'timestamp': self.dataOut.utctime,
362 410 'data': roundFloats(Z),
363 411 'channels': ['Ch %s' % ch for ch in self.dataOut.channelList],
364 412 'interval': self.dataOut.getTimeInterval(),
365 413 'type': self.plottype,
366 414 'yData': yData
367 415 }
368 416
369 417 elif self.plottype in ('rti', 'power'):
370 418 data = getattr(self.dataOut, 'data_spc')
371 419 z = data/self.dataOut.normFactor
372 420 avg = numpy.average(z, axis=1)
373 421 avgdB = 10*numpy.log10(avg)
374 422 xlen, ylen = z[0].shape
375 423 dy = numpy.floor(ylen/self.__MAXNUMY) + 1
376 424 AVG = [0 for i in self.dataOut.channelList]
377 425 for i in self.dataOut.channelList:
378 426 AVG[i] = avgdB[i][::dy].tolist()
379 427 payload = {
380 428 'timestamp': self.dataOut.utctime,
381 429 'data': roundFloats(AVG),
382 430 'channels': ['Ch %s' % ch for ch in self.dataOut.channelList],
383 431 'interval': self.dataOut.getTimeInterval(),
384 432 'type': self.plottype,
385 433 'yData': yData
386 434 }
387 435 elif self.plottype == 'noise':
388 436 noise = self.dataOut.getNoise()/self.dataOut.normFactor
389 437 noisedB = 10*numpy.log10(noise)
390 438 payload = {
391 439 'timestamp': self.dataOut.utctime,
392 440 'data': roundFloats(noisedB.reshape(-1, 1).tolist()),
393 441 'channels': ['Ch %s' % ch for ch in self.dataOut.channelList],
394 442 'interval': self.dataOut.getTimeInterval(),
395 443 'type': self.plottype,
396 444 'yData': yData
397 445 }
398 446 elif self.plottype == 'snr':
399 447 data = getattr(self.dataOut, 'data_SNR')
400 448 avgdB = 10*numpy.log10(data)
401 449
402 450 ylen = data[0].size
403 451 dy = numpy.floor(ylen/self.__MAXNUMY) + 1
404 452 AVG = [0 for i in self.dataOut.channelList]
405 453 for i in self.dataOut.channelList:
406 454 AVG[i] = avgdB[i][::dy].tolist()
407 455 payload = {
408 456 'timestamp': self.dataOut.utctime,
409 457 'data': roundFloats(AVG),
410 458 'channels': ['Ch %s' % ch for ch in self.dataOut.channelList],
411 459 'type': self.plottype,
412 460 'yData': yData
413 461 }
414 462 else:
415 463 print "Tipo de grafico invalido"
416 464 payload = {
417 465 'data': 'None',
418 466 'timestamp': 'None',
419 467 'type': None
420 468 }
421 469
422 470 self.client.publish(self.topic + self.plottype, json.dumps(payload), qos=0)
423 471
424 472 if self.zeromq is 1:
425 473 if self.verbose:
426 474 log.log(
427 475 'Sending {} - {}'.format(self.dataOut.type, self.dataOut.datatime),
428 476 self.name
429 477 )
430 478 self.zmq_socket.send_pyobj(self.dataOut)
431 479
432 480 def run(self, dataOut, **kwargs):
433 481 self.dataOut = dataOut
434 482 if not self.isConfig:
435 483 self.setup(**kwargs)
436 484 self.isConfig = True
437 485
438 486 self.publish_data()
439 487 time.sleep(self.delay)
440 488
441 489 def close(self):
442 490 if self.zeromq is 1:
443 491 self.dataOut.finished = True
444 492 self.zmq_socket.send_pyobj(self.dataOut)
445 493 time.sleep(0.1)
446 494 self.zmq_socket.close()
447 495 if self.client:
448 496 self.client.loop_stop()
449 497 self.client.disconnect()
450 498
451 499
452 500 class ReceiverData(ProcessingUnit):
453 501
454 502 __attrs__ = ['server']
455 503
456 504 def __init__(self, **kwargs):
457 505
458 506 ProcessingUnit.__init__(self, **kwargs)
459 507
460 508 self.isConfig = False
461 509 server = kwargs.get('server', 'zmq.pipe')
462 510 if 'tcp://' in server:
463 511 address = server
464 512 else:
465 513 address = 'ipc:///tmp/%s' % server
466 514
467 515 self.address = address
468 516 self.dataOut = JROData()
469 517
470 518 def setup(self):
471 519
472 520 self.context = zmq.Context()
473 521 self.receiver = self.context.socket(zmq.PULL)
474 522 self.receiver.bind(self.address)
475 523 time.sleep(0.5)
476 524 log.success('ReceiverData from {}'.format(self.address))
477 525
478 526
479 527 def run(self):
480 528
481 529 if not self.isConfig:
482 530 self.setup()
483 531 self.isConfig = True
484 532
485 533 self.dataOut = self.receiver.recv_pyobj()
486 534 log.log('{} - {}'.format(self.dataOut.type,
487 535 self.dataOut.datatime.ctime(),),
488 536 'Receiving')
489 537
490 538
491 539 class PlotterReceiver(ProcessingUnit, Process):
492 540
493 541 throttle_value = 5
494 542 __attrs__ = ['server', 'plottypes', 'realtime', 'localtime', 'throttle',
495 'exp_code', 'web_server']
543 'exp_code', 'web_server', 'buffering']
496 544
497 545 def __init__(self, **kwargs):
498 546
499 547 ProcessingUnit.__init__(self, **kwargs)
500 548 Process.__init__(self)
501 549 self.mp = False
502 550 self.isConfig = False
503 551 self.isWebConfig = False
504 552 self.connections = 0
505 553 server = kwargs.get('server', 'zmq.pipe')
506 554 web_server = kwargs.get('web_server', None)
507 555 if 'tcp://' in server:
508 556 address = server
509 557 else:
510 558 address = 'ipc:///tmp/%s' % server
511 559 self.address = address
512 560 self.web_address = web_server
513 561 self.plottypes = [s.strip() for s in kwargs.get('plottypes', 'rti').split(',')]
514 562 self.realtime = kwargs.get('realtime', False)
515 563 self.localtime = kwargs.get('localtime', True)
564 self.buffering = kwargs.get('buffering', True)
516 565 self.throttle_value = kwargs.get('throttle', 5)
517 566 self.exp_code = kwargs.get('exp_code', None)
518 567 self.sendData = self.initThrottle(self.throttle_value)
519 568 self.dates = []
520 569 self.setup()
521 570
522 571 def setup(self):
523 572
524 self.data = Data(self.plottypes, self.throttle_value, self.exp_code)
525 self.isConfig = True
573 self.data = Data(self.plottypes, self.throttle_value, self.exp_code, self.buffering)
574 self.isConfig = True
526 575
527 576 def event_monitor(self, monitor):
528 577
529 578 events = {}
530 579
531 580 for name in dir(zmq):
532 581 if name.startswith('EVENT_'):
533 582 value = getattr(zmq, name)
534 583 events[value] = name
535 584
536 585 while monitor.poll():
537 586 evt = recv_monitor_message(monitor)
538 587 if evt['event'] == 32:
539 588 self.connections += 1
540 589 if evt['event'] == 512:
541 590 pass
542 591
543 592 evt.update({'description': events[evt['event']]})
544 593
545 594 if evt['event'] == zmq.EVENT_MONITOR_STOPPED:
546 595 break
547 596 monitor.close()
548 597 print('event monitor thread done!')
549 598
550 599 def initThrottle(self, throttle_value):
551 600
552 601 @throttle(seconds=throttle_value)
553 602 def sendDataThrottled(fn_sender, data):
554 603 fn_sender(data)
555 604
556 605 return sendDataThrottled
557 606
558 607 def send(self, data):
559 log.success('Sending {}'.format(data), self.name)
608 log.log('Sending {}'.format(data), self.name)
560 609 self.sender.send_pyobj(data)
561 610
562 611 def run(self):
563 612
564 log.success(
613 log.log(
565 614 'Starting from {}'.format(self.address),
566 615 self.name
567 616 )
568 617
569 618 self.context = zmq.Context()
570 619 self.receiver = self.context.socket(zmq.PULL)
571 620 self.receiver.bind(self.address)
572 621 monitor = self.receiver.get_monitor_socket()
573 622 self.sender = self.context.socket(zmq.PUB)
574 623 if self.web_address:
575 624 log.success(
576 625 'Sending to web: {}'.format(self.web_address),
577 626 self.name
578 627 )
579 628 self.sender_web = self.context.socket(zmq.REQ)
580 629 self.sender_web.connect(self.web_address)
581 630 self.poll = zmq.Poller()
582 631 self.poll.register(self.sender_web, zmq.POLLIN)
583 632 time.sleep(1)
584 633
585 634 if 'server' in self.kwargs:
586 635 self.sender.bind("ipc:///tmp/{}.plots".format(self.kwargs['server']))
587 636 else:
588 637 self.sender.bind("ipc:///tmp/zmq.plots")
589 638
590 639 time.sleep(2)
591 640
592 641 t = Thread(target=self.event_monitor, args=(monitor,))
593 642 t.start()
594 643
595 644 while True:
596 645 dataOut = self.receiver.recv_pyobj()
597 646 if not dataOut.flagNoData:
598 647 if dataOut.type == 'Parameters':
599 648 tm = dataOut.utctimeInit
600 649 else:
601 650 tm = dataOut.utctime
602 651 if dataOut.useLocalTime:
603 652 if not self.localtime:
604 653 tm += time.timezone
605 654 dt = datetime.datetime.fromtimestamp(tm).date()
606 655 else:
607 656 if self.localtime:
608 657 tm -= time.timezone
609 658 dt = datetime.datetime.utcfromtimestamp(tm).date()
610 659 coerce = False
611 660 if dt not in self.dates:
612 661 if self.data:
613 662 self.data.ended = True
614 663 self.send(self.data)
615 664 coerce = True
616 665 self.data.setup()
617 666 self.dates.append(dt)
618 667
619 668 self.data.update(dataOut, tm)
620 669
621 670 if dataOut.finished is True:
622 671 self.connections -= 1
623 672 if self.connections == 0 and dt in self.dates:
624 673 self.data.ended = True
625 674 self.send(self.data)
626 675 # self.data.setup()
627 676 time.sleep(1)
628 677 break
629 678 else:
630 679 if self.realtime:
631 680 self.send(self.data)
632 681 if self.web_address:
633 682 retries = 5
634 683 while True:
635 684 self.sender_web.send(self.data.jsonify())
636 685 socks = dict(self.poll.poll(5000))
637 686 if socks.get(self.sender_web) == zmq.POLLIN:
638 687 reply = self.sender_web.recv_string()
639 688 if reply == 'ok':
640 689 break
641 690 else:
642 691 print("Malformed reply from server: %s" % reply)
643 692
644 693 else:
645 694 print("No response from server, retrying...")
646 695 self.sender_web.setsockopt(zmq.LINGER, 0)
647 696 self.sender_web.close()
648 697 self.poll.unregister(self.sender_web)
649 698 retries -= 1
650 699 if retries == 0:
651 700 print("Server seems to be offline, abandoning")
652 701 break
653 702 self.sender_web = self.context.socket(zmq.REQ)
654 703 self.sender_web.connect(self.web_address)
655 704 self.poll.register(self.sender_web, zmq.POLLIN)
656 705 time.sleep(1)
657 706 else:
658 707 self.sendData(self.send, self.data, coerce=coerce)
659 708 coerce = False
660 709
661 710 return
711
712
713 class SendToFTP(Operation, Process):
714
715 '''
716 Operation to send data over FTP.
717 '''
718
719 __attrs__ = ['server', 'username', 'password', 'patterns', 'timeout']
720
721 def __init__(self, **kwargs):
722 '''
723 patterns = [(local1, remote1, ext, delay, exp_code, sub_exp_code), ...]
724 '''
725 Operation.__init__(self, **kwargs)
726 Process.__init__(self)
727 self.server = kwargs.get('server')
728 self.username = kwargs.get('username')
729 self.password = kwargs.get('password')
730 self.patterns = kwargs.get('patterns')
731 self.timeout = kwargs.get('timeout', 30)
732 self.times = [time.time() for p in self.patterns]
733 self.latest = ['' for p in self.patterns]
734 self.mp = False
735 self.ftp = None
736
737 def setup(self):
738
739 log.log('Connecting to ftp://{}'.format(self.server), self.name)
740 try:
741 self.ftp = ftplib.FTP(self.server, timeout=self.timeout)
742 except ftplib.all_errors:
743 log.error('Server connection fail: {}'.format(self.server), self.name)
744 if self.ftp is not None:
745 self.ftp.close()
746 self.ftp = None
747 self.isConfig = False
748 return
749
750 try:
751 self.ftp.login(self.username, self.password)
752 except ftplib.all_errors:
753 log.error('The given username y/o password are incorrect', self.name)
754 if self.ftp is not None:
755 self.ftp.close()
756 self.ftp = None
757 self.isConfig = False
758 return
759
760 log.success('Connection success', self.name)
761 self.isConfig = True
762 return
763
764 def check(self):
765
766 try:
767 self.ftp.voidcmd("NOOP")
768 except:
769 log.warning('Connection lost... trying to reconnect', self.name)
770 if self.ftp is not None:
771 self.ftp.close()
772 self.ftp = None
773 self.setup()
774
775 def find_files(self, path, ext):
776
777 files = glob.glob1(path, '*{}'.format(ext))
778 files.sort()
779 if files:
780 return files[-1]
781 return None
782
783 def getftpname(self, filename, exp_code, sub_exp_code):
784
785 thisDatetime = datetime.datetime.strptime(filename.split('_')[1], '%Y%m%d')
786 YEAR_STR = '%4.4d'%thisDatetime.timetuple().tm_year
787 DOY_STR = '%3.3d'%thisDatetime.timetuple().tm_yday
788 exp_code = '%3.3d'%exp_code
789 sub_exp_code = '%2.2d'%sub_exp_code
790 plot_code = '%2.2d'% get_plot_code(filename)
791 name = YEAR_STR + DOY_STR + '00' + exp_code + sub_exp_code + plot_code + '00.png'
792 return name
793
794 def upload(self, src, dst):
795
796 log.log('Uploading {} '.format(src), self.name, nl=False)
797
798 fp = open(src, 'rb')
799 command = 'STOR {}'.format(dst)
800
801 try:
802 self.ftp.storbinary(command, fp, blocksize=1024)
803 except Exception, e:
804 log.error('{}'.format(e), self.name)
805 if self.ftp is not None:
806 self.ftp.close()
807 self.ftp = None
808 return 0
809
810 try:
811 self.ftp.sendcmd('SITE CHMOD 755 {}'.format(dst))
812 except Exception, e:
813 log.error('{}'.format(e), self.name)
814 if self.ftp is not None:
815 self.ftp.close()
816 self.ftp = None
817 return 0
818
819 fp.close()
820 log.success('OK', tag='')
821 return 1
822
823 def send_files(self):
824
825 for x, pattern in enumerate(self.patterns):
826 local, remote, ext, delay, exp_code, sub_exp_code = pattern
827 if time.time()-self.times[x] >= delay:
828 srcname = self.find_files(local, ext)
829 src = os.path.join(local, srcname)
830 if os.path.getmtime(src) < time.time() - 30*60:
831 continue
832
833 if srcname is None or srcname == self.latest[x]:
834 continue
835
836 if 'png' in ext:
837 dstname = self.getftpname(srcname, exp_code, sub_exp_code)
838 else:
839 dstname = srcname
840
841 dst = os.path.join(remote, dstname)
842
843 if self.upload(src, dst):
844 self.times[x] = time.time()
845 self.latest[x] = srcname
846 else:
847 self.isConfig = False
848 break
849
850 def run(self):
851
852 while True:
853 if not self.isConfig:
854 self.setup()
855 if self.ftp is not None:
856 self.check()
857 self.send_files()
858 time.sleep(10)
859
860 def close():
861
862 if self.ftp is not None:
863 self.ftp.close()
864 self.terminate()
Show file before | Show file after | Hide comments
@@ -1,45 +1,57
1 1 '''
2 2 SCHAINPY - LOG
3 3 Simple helper for log standarization
4 4 Usage:
5 5 from schainpy.utils import log
6 6 log.error('A kitten died beacuse of you')
7 7 log.warning('You are doing it wrong but what the heck, I'll allow it)
8 8 log.succes('YOU ROCK!')
9 9 To create your own logger inside your class do it like this:
10 10 from schainpy.utils import log
11 11 awesomeLogger = log.makelogger("never gonna", bg="red", fg="white")
12 12 awesomeLogger('give you up')
13 13 which will look like this:
14 14 [NEVER GONNA] - give you up
15 15 with color red as background and white as foreground.
16 16 '''
17 17
18 18 import click
19 19
20 20
21 def warning(message, tag='Warning'):
22 click.echo(click.style('[{}] {}'.format(tag, message), fg='yellow'))
21 def warning(message, tag='Warning', nl=True):
22 if tag:
23 click.echo(click.style('[{}] {}'.format(tag, message), fg='yellow'), nl=nl)
24 else:
25 click.echo(click.style('{}'.format(message), fg='yellow'), nl=nl)
23 26 pass
24 27
25 28
26 def error(message, tag='Error'):
27 click.echo(click.style('[{}] {}'.format(tag, message), fg='red'))
29 def error(message, tag='Error', nl=True):
30 if tag:
31 click.echo(click.style('[{}] {}'.format(tag, message), fg='red'), nl=nl)
32 else:
33 click.echo(click.style('{}'.format(message), fg='red'), nl=nl)
28 34 pass
29 35
30 36
31 def success(message, tag='Info'):
32 click.echo(click.style('[{}] {}'.format(tag, message), fg='green'))
37 def success(message, tag='Success', nl=True):
38 if tag:
39 click.echo(click.style('[{}] {}'.format(tag, message), fg='green'), nl=nl)
40 else:
41 click.echo(click.style('{}'.format(message), fg='green'), nl=nl)
33 42 pass
34 43
35 44
36 def log(message, tag='Info'):
37 click.echo('[{}] {}'.format(tag, message))
45 def log(message, tag='Info', nl=True):
46 if tag:
47 click.echo('[{}] {}'.format(tag, message), nl=nl)
48 else:
49 click.echo('{}'.format(message), nl=nl)
38 50 pass
39 51
40 52
41 53 def makelogger(tag, bg='reset', fg='reset'):
42 54 def func(message):
43 55 click.echo(click.style('[{}] {}'.format(
44 56 tag.upper(), message), bg=bg, fg=fg))
45 57 return func
General Comments 0
You need to be logged in to leave comments. Login now