##// END OF EJS Templates
schain
RSS
Add python 3.8 compatibility with macOs
jespinoza -
r1342:b856bac3a9b1 v3.0.0b5
parent child
Show More
Side by Side Unified
Context file:
r1342:b856bac3a9b1 Loading diff...:
Show file before | Show file after | Hide comments
@@ -1,8 +1,8
1 """Signal chain python package"""
1 """Signal chain python package"""
2
2
3 try:
3 try:
4 from .controller import Project
4 from schainpy.controller import Project
5 except:
5 except:
6 pass
6 pass
7
7
8 __version__ = '3.0.0b4'
8 __version__ = '3.0.0b5'
Show file before | Show file after | Hide comments
@@ -1,656 +1,659
1 # Copyright (c) 2012-2020 Jicamarca Radio Observatory
1 # Copyright (c) 2012-2020 Jicamarca Radio Observatory
2 # All rights reserved.
2 # All rights reserved.
3 #
3 #
4 # Distributed under the terms of the BSD 3-clause license.
4 # Distributed under the terms of the BSD 3-clause license.
5 """API to create signal chain projects
5 """API to create signal chain projects
6
6
7 The API is provide through class: Project
7 The API is provide through class: Project
8 """
8 """
9
9
10 import re
10 import re
11 import sys
11 import sys
12 import ast
12 import ast
13 import datetime
13 import datetime
14 import traceback
14 import traceback
15 import time
15 import time
16 import multiprocessing
16 from multiprocessing import Process, Queue
17 from multiprocessing import Process, Queue
17 from threading import Thread
18 from threading import Thread
18 from xml.etree.ElementTree import ElementTree, Element, SubElement
19 from xml.etree.ElementTree import ElementTree, Element, SubElement
19
20
20 from schainpy.admin import Alarm, SchainWarning
21 from schainpy.admin import Alarm, SchainWarning
21 from schainpy.model import *
22 from schainpy.model import *
22 from schainpy.utils import log
23 from schainpy.utils import log
23
24
25 if 'darwin' in sys.platform and sys.version_info[0] == 3 and sys.version_info[1] > 7:
26 multiprocessing.set_start_method('fork')
24
27
25 class ConfBase():
28 class ConfBase():
26
29
27 def __init__(self):
30 def __init__(self):
28
31
29 self.id = '0'
32 self.id = '0'
30 self.name = None
33 self.name = None
31 self.priority = None
34 self.priority = None
32 self.parameters = {}
35 self.parameters = {}
33 self.object = None
36 self.object = None
34 self.operations = []
37 self.operations = []
35
38
36 def getId(self):
39 def getId(self):
37
40
38 return self.id
41 return self.id
39
42
40 def getNewId(self):
43 def getNewId(self):
41
44
42 return int(self.id) * 10 + len(self.operations) + 1
45 return int(self.id) * 10 + len(self.operations) + 1
43
46
44 def updateId(self, new_id):
47 def updateId(self, new_id):
45
48
46 self.id = str(new_id)
49 self.id = str(new_id)
47
50
48 n = 1
51 n = 1
49 for conf in self.operations:
52 for conf in self.operations:
50 conf_id = str(int(new_id) * 10 + n)
53 conf_id = str(int(new_id) * 10 + n)
51 conf.updateId(conf_id)
54 conf.updateId(conf_id)
52 n += 1
55 n += 1
53
56
54 def getKwargs(self):
57 def getKwargs(self):
55
58
56 params = {}
59 params = {}
57
60
58 for key, value in self.parameters.items():
61 for key, value in self.parameters.items():
59 if value not in (None, '', ' '):
62 if value not in (None, '', ' '):
60 params[key] = value
63 params[key] = value
61
64
62 return params
65 return params
63
66
64 def update(self, **kwargs):
67 def update(self, **kwargs):
65
68
66 for key, value in kwargs.items():
69 for key, value in kwargs.items():
67 self.addParameter(name=key, value=value)
70 self.addParameter(name=key, value=value)
68
71
69 def addParameter(self, name, value, format=None):
72 def addParameter(self, name, value, format=None):
70 '''
73 '''
71 '''
74 '''
72
75
73 if isinstance(value, str) and re.search(r'(\d+/\d+/\d+)', value):
76 if isinstance(value, str) and re.search(r'(\d+/\d+/\d+)', value):
74 self.parameters[name] = datetime.date(*[int(x) for x in value.split('/')])
77 self.parameters[name] = datetime.date(*[int(x) for x in value.split('/')])
75 elif isinstance(value, str) and re.search(r'(\d+:\d+:\d+)', value):
78 elif isinstance(value, str) and re.search(r'(\d+:\d+:\d+)', value):
76 self.parameters[name] = datetime.time(*[int(x) for x in value.split(':')])
79 self.parameters[name] = datetime.time(*[int(x) for x in value.split(':')])
77 else:
80 else:
78 try:
81 try:
79 self.parameters[name] = ast.literal_eval(value)
82 self.parameters[name] = ast.literal_eval(value)
80 except:
83 except:
81 if isinstance(value, str) and ',' in value:
84 if isinstance(value, str) and ',' in value:
82 self.parameters[name] = value.split(',')
85 self.parameters[name] = value.split(',')
83 else:
86 else:
84 self.parameters[name] = value
87 self.parameters[name] = value
85
88
86 def getParameters(self):
89 def getParameters(self):
87
90
88 params = {}
91 params = {}
89 for key, value in self.parameters.items():
92 for key, value in self.parameters.items():
90 s = type(value).__name__
93 s = type(value).__name__
91 if s == 'date':
94 if s == 'date':
92 params[key] = value.strftime('%Y/%m/%d')
95 params[key] = value.strftime('%Y/%m/%d')
93 elif s == 'time':
96 elif s == 'time':
94 params[key] = value.strftime('%H:%M:%S')
97 params[key] = value.strftime('%H:%M:%S')
95 else:
98 else:
96 params[key] = str(value)
99 params[key] = str(value)
97
100
98 return params
101 return params
99
102
100 def makeXml(self, element):
103 def makeXml(self, element):
101
104
102 xml = SubElement(element, self.ELEMENTNAME)
105 xml = SubElement(element, self.ELEMENTNAME)
103 for label in self.xml_labels:
106 for label in self.xml_labels:
104 xml.set(label, str(getattr(self, label)))
107 xml.set(label, str(getattr(self, label)))
105
108
106 for key, value in self.getParameters().items():
109 for key, value in self.getParameters().items():
107 xml_param = SubElement(xml, 'Parameter')
110 xml_param = SubElement(xml, 'Parameter')
108 xml_param.set('name', key)
111 xml_param.set('name', key)
109 xml_param.set('value', value)
112 xml_param.set('value', value)
110
113
111 for conf in self.operations:
114 for conf in self.operations:
112 conf.makeXml(xml)
115 conf.makeXml(xml)
113
116
114 def __str__(self):
117 def __str__(self):
115
118
116 if self.ELEMENTNAME == 'Operation':
119 if self.ELEMENTNAME == 'Operation':
117 s = ' {}[id={}]\n'.format(self.name, self.id)
120 s = ' {}[id={}]\n'.format(self.name, self.id)
118 else:
121 else:
119 s = '{}[id={}, inputId={}]\n'.format(self.name, self.id, self.inputId)
122 s = '{}[id={}, inputId={}]\n'.format(self.name, self.id, self.inputId)
120
123
121 for key, value in self.parameters.items():
124 for key, value in self.parameters.items():
122 if self.ELEMENTNAME == 'Operation':
125 if self.ELEMENTNAME == 'Operation':
123 s += ' {}: {}\n'.format(key, value)
126 s += ' {}: {}\n'.format(key, value)
124 else:
127 else:
125 s += ' {}: {}\n'.format(key, value)
128 s += ' {}: {}\n'.format(key, value)
126
129
127 for conf in self.operations:
130 for conf in self.operations:
128 s += str(conf)
131 s += str(conf)
129
132
130 return s
133 return s
131
134
132 class OperationConf(ConfBase):
135 class OperationConf(ConfBase):
133
136
134 ELEMENTNAME = 'Operation'
137 ELEMENTNAME = 'Operation'
135 xml_labels = ['id', 'name']
138 xml_labels = ['id', 'name']
136
139
137 def setup(self, id, name, priority, project_id, err_queue):
140 def setup(self, id, name, priority, project_id, err_queue):
138
141
139 self.id = str(id)
142 self.id = str(id)
140 self.project_id = project_id
143 self.project_id = project_id
141 self.name = name
144 self.name = name
142 self.type = 'other'
145 self.type = 'other'
143 self.err_queue = err_queue
146 self.err_queue = err_queue
144
147
145 def readXml(self, element, project_id, err_queue):
148 def readXml(self, element, project_id, err_queue):
146
149
147 self.id = element.get('id')
150 self.id = element.get('id')
148 self.name = element.get('name')
151 self.name = element.get('name')
149 self.type = 'other'
152 self.type = 'other'
150 self.project_id = str(project_id)
153 self.project_id = str(project_id)
151 self.err_queue = err_queue
154 self.err_queue = err_queue
152
155
153 for elm in element.iter('Parameter'):
156 for elm in element.iter('Parameter'):
154 self.addParameter(elm.get('name'), elm.get('value'))
157 self.addParameter(elm.get('name'), elm.get('value'))
155
158
156 def createObject(self):
159 def createObject(self):
157
160
158 className = eval(self.name)
161 className = eval(self.name)
159
162
160 if 'Plot' in self.name or 'Writer' in self.name or 'Send' in self.name or 'print' in self.name:
163 if 'Plot' in self.name or 'Writer' in self.name or 'Send' in self.name or 'print' in self.name:
161 kwargs = self.getKwargs()
164 kwargs = self.getKwargs()
162 opObj = className(self.id, self.id, self.project_id, self.err_queue, **kwargs)
165 opObj = className(self.id, self.id, self.project_id, self.err_queue, **kwargs)
163 opObj.start()
166 opObj.start()
164 self.type = 'external'
167 self.type = 'external'
165 else:
168 else:
166 opObj = className()
169 opObj = className()
167
170
168 self.object = opObj
171 self.object = opObj
169 return opObj
172 return opObj
170
173
171 class ProcUnitConf(ConfBase):
174 class ProcUnitConf(ConfBase):
172
175
173 ELEMENTNAME = 'ProcUnit'
176 ELEMENTNAME = 'ProcUnit'
174 xml_labels = ['id', 'inputId', 'name']
177 xml_labels = ['id', 'inputId', 'name']
175
178
176 def setup(self, project_id, id, name, datatype, inputId, err_queue):
179 def setup(self, project_id, id, name, datatype, inputId, err_queue):
177 '''
180 '''
178 '''
181 '''
179
182
180 if datatype == None and name == None:
183 if datatype == None and name == None:
181 raise ValueError('datatype or name should be defined')
184 raise ValueError('datatype or name should be defined')
182
185
183 if name == None:
186 if name == None:
184 if 'Proc' in datatype:
187 if 'Proc' in datatype:
185 name = datatype
188 name = datatype
186 else:
189 else:
187 name = '%sProc' % (datatype)
190 name = '%sProc' % (datatype)
188
191
189 if datatype == None:
192 if datatype == None:
190 datatype = name.replace('Proc', '')
193 datatype = name.replace('Proc', '')
191
194
192 self.id = str(id)
195 self.id = str(id)
193 self.project_id = project_id
196 self.project_id = project_id
194 self.name = name
197 self.name = name
195 self.datatype = datatype
198 self.datatype = datatype
196 self.inputId = inputId
199 self.inputId = inputId
197 self.err_queue = err_queue
200 self.err_queue = err_queue
198 self.operations = []
201 self.operations = []
199 self.parameters = {}
202 self.parameters = {}
200
203
201 def removeOperation(self, id):
204 def removeOperation(self, id):
202
205
203 i = [1 if x.id==id else 0 for x in self.operations]
206 i = [1 if x.id==id else 0 for x in self.operations]
204 self.operations.pop(i.index(1))
207 self.operations.pop(i.index(1))
205
208
206 def getOperation(self, id):
209 def getOperation(self, id):
207
210
208 for conf in self.operations:
211 for conf in self.operations:
209 if conf.id == id:
212 if conf.id == id:
210 return conf
213 return conf
211
214
212 def addOperation(self, name, optype='self'):
215 def addOperation(self, name, optype='self'):
213 '''
216 '''
214 '''
217 '''
215
218
216 id = self.getNewId()
219 id = self.getNewId()
217 conf = OperationConf()
220 conf = OperationConf()
218 conf.setup(id, name=name, priority='0', project_id=self.project_id, err_queue=self.err_queue)
221 conf.setup(id, name=name, priority='0', project_id=self.project_id, err_queue=self.err_queue)
219 self.operations.append(conf)
222 self.operations.append(conf)
220
223
221 return conf
224 return conf
222
225
223 def readXml(self, element, project_id, err_queue):
226 def readXml(self, element, project_id, err_queue):
224
227
225 self.id = element.get('id')
228 self.id = element.get('id')
226 self.name = element.get('name')
229 self.name = element.get('name')
227 self.inputId = None if element.get('inputId') == 'None' else element.get('inputId')
230 self.inputId = None if element.get('inputId') == 'None' else element.get('inputId')
228 self.datatype = element.get('datatype', self.name.replace(self.ELEMENTNAME.replace('Unit', ''), ''))
231 self.datatype = element.get('datatype', self.name.replace(self.ELEMENTNAME.replace('Unit', ''), ''))
229 self.project_id = str(project_id)
232 self.project_id = str(project_id)
230 self.err_queue = err_queue
233 self.err_queue = err_queue
231 self.operations = []
234 self.operations = []
232 self.parameters = {}
235 self.parameters = {}
233
236
234 for elm in element:
237 for elm in element:
235 if elm.tag == 'Parameter':
238 if elm.tag == 'Parameter':
236 self.addParameter(elm.get('name'), elm.get('value'))
239 self.addParameter(elm.get('name'), elm.get('value'))
237 elif elm.tag == 'Operation':
240 elif elm.tag == 'Operation':
238 conf = OperationConf()
241 conf = OperationConf()
239 conf.readXml(elm, project_id, err_queue)
242 conf.readXml(elm, project_id, err_queue)
240 self.operations.append(conf)
243 self.operations.append(conf)
241
244
242 def createObjects(self):
245 def createObjects(self):
243 '''
246 '''
244 Instancia de unidades de procesamiento.
247 Instancia de unidades de procesamiento.
245 '''
248 '''
246
249
247 className = eval(self.name)
250 className = eval(self.name)
248 kwargs = self.getKwargs()
251 kwargs = self.getKwargs()
249 procUnitObj = className()
252 procUnitObj = className()
250 procUnitObj.name = self.name
253 procUnitObj.name = self.name
251 log.success('creating process...', self.name)
254 log.success('creating process...', self.name)
252
255
253 for conf in self.operations:
256 for conf in self.operations:
254
257
255 opObj = conf.createObject()
258 opObj = conf.createObject()
256
259
257 log.success('adding operation: {}, type:{}'.format(
260 log.success('adding operation: {}, type:{}'.format(
258 conf.name,
261 conf.name,
259 conf.type), self.name)
262 conf.type), self.name)
260
263
261 procUnitObj.addOperation(conf, opObj)
264 procUnitObj.addOperation(conf, opObj)
262
265
263 self.object = procUnitObj
266 self.object = procUnitObj
264
267
265 def run(self):
268 def run(self):
266 '''
269 '''
267 '''
270 '''
268
271
269 return self.object.call(**self.getKwargs())
272 return self.object.call(**self.getKwargs())
270
273
271
274
272 class ReadUnitConf(ProcUnitConf):
275 class ReadUnitConf(ProcUnitConf):
273
276
274 ELEMENTNAME = 'ReadUnit'
277 ELEMENTNAME = 'ReadUnit'
275
278
276 def __init__(self):
279 def __init__(self):
277
280
278 self.id = None
281 self.id = None
279 self.datatype = None
282 self.datatype = None
280 self.name = None
283 self.name = None
281 self.inputId = None
284 self.inputId = None
282 self.operations = []
285 self.operations = []
283 self.parameters = {}
286 self.parameters = {}
284
287
285 def setup(self, project_id, id, name, datatype, err_queue, path='', startDate='', endDate='',
288 def setup(self, project_id, id, name, datatype, err_queue, path='', startDate='', endDate='',
286 startTime='', endTime='', server=None, **kwargs):
289 startTime='', endTime='', server=None, **kwargs):
287
290
288 if datatype == None and name == None:
291 if datatype == None and name == None:
289 raise ValueError('datatype or name should be defined')
292 raise ValueError('datatype or name should be defined')
290 if name == None:
293 if name == None:
291 if 'Reader' in datatype:
294 if 'Reader' in datatype:
292 name = datatype
295 name = datatype
293 datatype = name.replace('Reader','')
296 datatype = name.replace('Reader','')
294 else:
297 else:
295 name = '{}Reader'.format(datatype)
298 name = '{}Reader'.format(datatype)
296 if datatype == None:
299 if datatype == None:
297 if 'Reader' in name:
300 if 'Reader' in name:
298 datatype = name.replace('Reader','')
301 datatype = name.replace('Reader','')
299 else:
302 else:
300 datatype = name
303 datatype = name
301 name = '{}Reader'.format(name)
304 name = '{}Reader'.format(name)
302
305
303 self.id = id
306 self.id = id
304 self.project_id = project_id
307 self.project_id = project_id
305 self.name = name
308 self.name = name
306 self.datatype = datatype
309 self.datatype = datatype
307 self.err_queue = err_queue
310 self.err_queue = err_queue
308
311
309 self.addParameter(name='path', value=path)
312 self.addParameter(name='path', value=path)
310 self.addParameter(name='startDate', value=startDate)
313 self.addParameter(name='startDate', value=startDate)
311 self.addParameter(name='endDate', value=endDate)
314 self.addParameter(name='endDate', value=endDate)
312 self.addParameter(name='startTime', value=startTime)
315 self.addParameter(name='startTime', value=startTime)
313 self.addParameter(name='endTime', value=endTime)
316 self.addParameter(name='endTime', value=endTime)
314
317
315 for key, value in kwargs.items():
318 for key, value in kwargs.items():
316 self.addParameter(name=key, value=value)
319 self.addParameter(name=key, value=value)
317
320
318
321
319 class Project(Process):
322 class Project(Process):
320 """API to create signal chain projects"""
323 """API to create signal chain projects"""
321
324
322 ELEMENTNAME = 'Project'
325 ELEMENTNAME = 'Project'
323
326
324 def __init__(self, name=''):
327 def __init__(self, name=''):
325
328
326 Process.__init__(self)
329 Process.__init__(self)
327 self.id = '1'
330 self.id = '1'
328 if name:
331 if name:
329 self.name = '{} ({})'.format(Process.__name__, name)
332 self.name = '{} ({})'.format(Process.__name__, name)
330 self.filename = None
333 self.filename = None
331 self.description = None
334 self.description = None
332 self.email = None
335 self.email = None
333 self.alarm = []
336 self.alarm = []
334 self.configurations = {}
337 self.configurations = {}
335 # self.err_queue = Queue()
338 # self.err_queue = Queue()
336 self.err_queue = None
339 self.err_queue = None
337 self.started = False
340 self.started = False
338
341
339 def getNewId(self):
342 def getNewId(self):
340
343
341 idList = list(self.configurations.keys())
344 idList = list(self.configurations.keys())
342 id = int(self.id) * 10
345 id = int(self.id) * 10
343
346
344 while True:
347 while True:
345 id += 1
348 id += 1
346
349
347 if str(id) in idList:
350 if str(id) in idList:
348 continue
351 continue
349
352
350 break
353 break
351
354
352 return str(id)
355 return str(id)
353
356
354 def updateId(self, new_id):
357 def updateId(self, new_id):
355
358
356 self.id = str(new_id)
359 self.id = str(new_id)
357
360
358 keyList = list(self.configurations.keys())
361 keyList = list(self.configurations.keys())
359 keyList.sort()
362 keyList.sort()
360
363
361 n = 1
364 n = 1
362 new_confs = {}
365 new_confs = {}
363
366
364 for procKey in keyList:
367 for procKey in keyList:
365
368
366 conf = self.configurations[procKey]
369 conf = self.configurations[procKey]
367 idProcUnit = str(int(self.id) * 10 + n)
370 idProcUnit = str(int(self.id) * 10 + n)
368 conf.updateId(idProcUnit)
371 conf.updateId(idProcUnit)
369 new_confs[idProcUnit] = conf
372 new_confs[idProcUnit] = conf
370 n += 1
373 n += 1
371
374
372 self.configurations = new_confs
375 self.configurations = new_confs
373
376
374 def setup(self, id=1, name='', description='', email=None, alarm=[]):
377 def setup(self, id=1, name='', description='', email=None, alarm=[]):
375
378
376 self.id = str(id)
379 self.id = str(id)
377 self.description = description
380 self.description = description
378 self.email = email
381 self.email = email
379 self.alarm = alarm
382 self.alarm = alarm
380 if name:
383 if name:
381 self.name = '{} ({})'.format(Process.__name__, name)
384 self.name = '{} ({})'.format(Process.__name__, name)
382
385
383 def update(self, **kwargs):
386 def update(self, **kwargs):
384
387
385 for key, value in kwargs.items():
388 for key, value in kwargs.items():
386 setattr(self, key, value)
389 setattr(self, key, value)
387
390
388 def clone(self):
391 def clone(self):
389
392
390 p = Project()
393 p = Project()
391 p.id = self.id
394 p.id = self.id
392 p.name = self.name
395 p.name = self.name
393 p.description = self.description
396 p.description = self.description
394 p.configurations = self.configurations.copy()
397 p.configurations = self.configurations.copy()
395
398
396 return p
399 return p
397
400
398 def addReadUnit(self, id=None, datatype=None, name=None, **kwargs):
401 def addReadUnit(self, id=None, datatype=None, name=None, **kwargs):
399
402
400 '''
403 '''
401 '''
404 '''
402
405
403 if id is None:
406 if id is None:
404 idReadUnit = self.getNewId()
407 idReadUnit = self.getNewId()
405 else:
408 else:
406 idReadUnit = str(id)
409 idReadUnit = str(id)
407
410
408 conf = ReadUnitConf()
411 conf = ReadUnitConf()
409 conf.setup(self.id, idReadUnit, name, datatype, self.err_queue, **kwargs)
412 conf.setup(self.id, idReadUnit, name, datatype, self.err_queue, **kwargs)
410 self.configurations[conf.id] = conf
413 self.configurations[conf.id] = conf
411
414
412 return conf
415 return conf
413
416
414 def addProcUnit(self, id=None, inputId='0', datatype=None, name=None):
417 def addProcUnit(self, id=None, inputId='0', datatype=None, name=None):
415
418
416 '''
419 '''
417 '''
420 '''
418
421
419 if id is None:
422 if id is None:
420 idProcUnit = self.getNewId()
423 idProcUnit = self.getNewId()
421 else:
424 else:
422 idProcUnit = id
425 idProcUnit = id
423
426
424 conf = ProcUnitConf()
427 conf = ProcUnitConf()
425 conf.setup(self.id, idProcUnit, name, datatype, inputId, self.err_queue)
428 conf.setup(self.id, idProcUnit, name, datatype, inputId, self.err_queue)
426 self.configurations[conf.id] = conf
429 self.configurations[conf.id] = conf
427
430
428 return conf
431 return conf
429
432
430 def removeProcUnit(self, id):
433 def removeProcUnit(self, id):
431
434
432 if id in self.configurations:
435 if id in self.configurations:
433 self.configurations.pop(id)
436 self.configurations.pop(id)
434
437
435 def getReadUnit(self):
438 def getReadUnit(self):
436
439
437 for obj in list(self.configurations.values()):
440 for obj in list(self.configurations.values()):
438 if obj.ELEMENTNAME == 'ReadUnit':
441 if obj.ELEMENTNAME == 'ReadUnit':
439 return obj
442 return obj
440
443
441 return None
444 return None
442
445
443 def getProcUnit(self, id):
446 def getProcUnit(self, id):
444
447
445 return self.configurations[id]
448 return self.configurations[id]
446
449
447 def getUnits(self):
450 def getUnits(self):
448
451
449 keys = list(self.configurations)
452 keys = list(self.configurations)
450 keys.sort()
453 keys.sort()
451
454
452 for key in keys:
455 for key in keys:
453 yield self.configurations[key]
456 yield self.configurations[key]
454
457
455 def updateUnit(self, id, **kwargs):
458 def updateUnit(self, id, **kwargs):
456
459
457 conf = self.configurations[id].update(**kwargs)
460 conf = self.configurations[id].update(**kwargs)
458
461
459 def makeXml(self):
462 def makeXml(self):
460
463
461 xml = Element('Project')
464 xml = Element('Project')
462 xml.set('id', str(self.id))
465 xml.set('id', str(self.id))
463 xml.set('name', self.name)
466 xml.set('name', self.name)
464 xml.set('description', self.description)
467 xml.set('description', self.description)
465
468
466 for conf in self.configurations.values():
469 for conf in self.configurations.values():
467 conf.makeXml(xml)
470 conf.makeXml(xml)
468
471
469 self.xml = xml
472 self.xml = xml
470
473
471 def writeXml(self, filename=None):
474 def writeXml(self, filename=None):
472
475
473 if filename == None:
476 if filename == None:
474 if self.filename:
477 if self.filename:
475 filename = self.filename
478 filename = self.filename
476 else:
479 else:
477 filename = 'schain.xml'
480 filename = 'schain.xml'
478
481
479 if not filename:
482 if not filename:
480 print('filename has not been defined. Use setFilename(filename) for do it.')
483 print('filename has not been defined. Use setFilename(filename) for do it.')
481 return 0
484 return 0
482
485
483 abs_file = os.path.abspath(filename)
486 abs_file = os.path.abspath(filename)
484
487
485 if not os.access(os.path.dirname(abs_file), os.W_OK):
488 if not os.access(os.path.dirname(abs_file), os.W_OK):
486 print('No write permission on %s' % os.path.dirname(abs_file))
489 print('No write permission on %s' % os.path.dirname(abs_file))
487 return 0
490 return 0
488
491
489 if os.path.isfile(abs_file) and not(os.access(abs_file, os.W_OK)):
492 if os.path.isfile(abs_file) and not(os.access(abs_file, os.W_OK)):
490 print('File %s already exists and it could not be overwriten' % abs_file)
493 print('File %s already exists and it could not be overwriten' % abs_file)
491 return 0
494 return 0
492
495
493 self.makeXml()
496 self.makeXml()
494
497
495 ElementTree(self.xml).write(abs_file, method='xml')
498 ElementTree(self.xml).write(abs_file, method='xml')
496
499
497 self.filename = abs_file
500 self.filename = abs_file
498
501
499 return 1
502 return 1
500
503
501 def readXml(self, filename):
504 def readXml(self, filename):
502
505
503 abs_file = os.path.abspath(filename)
506 abs_file = os.path.abspath(filename)
504
507
505 self.configurations = {}
508 self.configurations = {}
506
509
507 try:
510 try:
508 self.xml = ElementTree().parse(abs_file)
511 self.xml = ElementTree().parse(abs_file)
509 except:
512 except:
510 log.error('Error reading %s, verify file format' % filename)
513 log.error('Error reading %s, verify file format' % filename)
511 return 0
514 return 0
512
515
513 self.id = self.xml.get('id')
516 self.id = self.xml.get('id')
514 self.name = self.xml.get('name')
517 self.name = self.xml.get('name')
515 self.description = self.xml.get('description')
518 self.description = self.xml.get('description')
516
519
517 for element in self.xml:
520 for element in self.xml:
518 if element.tag == 'ReadUnit':
521 if element.tag == 'ReadUnit':
519 conf = ReadUnitConf()
522 conf = ReadUnitConf()
520 conf.readXml(element, self.id, self.err_queue)
523 conf.readXml(element, self.id, self.err_queue)
521 self.configurations[conf.id] = conf
524 self.configurations[conf.id] = conf
522 elif element.tag == 'ProcUnit':
525 elif element.tag == 'ProcUnit':
523 conf = ProcUnitConf()
526 conf = ProcUnitConf()
524 input_proc = self.configurations[element.get('inputId')]
527 input_proc = self.configurations[element.get('inputId')]
525 conf.readXml(element, self.id, self.err_queue)
528 conf.readXml(element, self.id, self.err_queue)
526 self.configurations[conf.id] = conf
529 self.configurations[conf.id] = conf
527
530
528 self.filename = abs_file
531 self.filename = abs_file
529
532
530 return 1
533 return 1
531
534
532 def __str__(self):
535 def __str__(self):
533
536
534 text = '\nProject[id=%s, name=%s, description=%s]\n\n' % (
537 text = '\nProject[id=%s, name=%s, description=%s]\n\n' % (
535 self.id,
538 self.id,
536 self.name,
539 self.name,
537 self.description,
540 self.description,
538 )
541 )
539
542
540 for conf in self.configurations.values():
543 for conf in self.configurations.values():
541 text += '{}'.format(conf)
544 text += '{}'.format(conf)
542
545
543 return text
546 return text
544
547
545 def createObjects(self):
548 def createObjects(self):
546
549
547 keys = list(self.configurations.keys())
550 keys = list(self.configurations.keys())
548 keys.sort()
551 keys.sort()
549 for key in keys:
552 for key in keys:
550 conf = self.configurations[key]
553 conf = self.configurations[key]
551 conf.createObjects()
554 conf.createObjects()
552 if conf.inputId is not None:
555 if conf.inputId is not None:
553 conf.object.setInput(self.configurations[conf.inputId].object)
556 conf.object.setInput(self.configurations[conf.inputId].object)
554
557
555 def monitor(self):
558 def monitor(self):
556
559
557 t = Thread(target=self._monitor, args=(self.err_queue, self.ctx))
560 t = Thread(target=self._monitor, args=(self.err_queue, self.ctx))
558 t.start()
561 t.start()
559
562
560 def _monitor(self, queue, ctx):
563 def _monitor(self, queue, ctx):
561
564
562 import socket
565 import socket
563
566
564 procs = 0
567 procs = 0
565 err_msg = ''
568 err_msg = ''
566
569
567 while True:
570 while True:
568 msg = queue.get()
571 msg = queue.get()
569 if '#_start_#' in msg:
572 if '#_start_#' in msg:
570 procs += 1
573 procs += 1
571 elif '#_end_#' in msg:
574 elif '#_end_#' in msg:
572 procs -=1
575 procs -=1
573 else:
576 else:
574 err_msg = msg
577 err_msg = msg
575
578
576 if procs == 0 or 'Traceback' in err_msg:
579 if procs == 0 or 'Traceback' in err_msg:
577 break
580 break
578 time.sleep(0.1)
581 time.sleep(0.1)
579
582
580 if '|' in err_msg:
583 if '|' in err_msg:
581 name, err = err_msg.split('|')
584 name, err = err_msg.split('|')
582 if 'SchainWarning' in err:
585 if 'SchainWarning' in err:
583 log.warning(err.split('SchainWarning:')[-1].split('\n')[0].strip(), name)
586 log.warning(err.split('SchainWarning:')[-1].split('\n')[0].strip(), name)
584 elif 'SchainError' in err:
587 elif 'SchainError' in err:
585 log.error(err.split('SchainError:')[-1].split('\n')[0].strip(), name)
588 log.error(err.split('SchainError:')[-1].split('\n')[0].strip(), name)
586 else:
589 else:
587 log.error(err, name)
590 log.error(err, name)
588 else:
591 else:
589 name, err = self.name, err_msg
592 name, err = self.name, err_msg
590
593
591 time.sleep(1)
594 time.sleep(1)
592
595
593 ctx.term()
596 ctx.term()
594
597
595 message = ''.join(err)
598 message = ''.join(err)
596
599
597 if err_msg:
600 if err_msg:
598 subject = 'SChain v%s: Error running %s\n' % (
601 subject = 'SChain v%s: Error running %s\n' % (
599 schainpy.__version__, self.name)
602 schainpy.__version__, self.name)
600
603
601 subtitle = 'Hostname: %s\n' % socket.gethostbyname(
604 subtitle = 'Hostname: %s\n' % socket.gethostbyname(
602 socket.gethostname())
605 socket.gethostname())
603 subtitle += 'Working directory: %s\n' % os.path.abspath('./')
606 subtitle += 'Working directory: %s\n' % os.path.abspath('./')
604 subtitle += 'Configuration file: %s\n' % self.filename
607 subtitle += 'Configuration file: %s\n' % self.filename
605 subtitle += 'Time: %s\n' % str(datetime.datetime.now())
608 subtitle += 'Time: %s\n' % str(datetime.datetime.now())
606
609
607 readUnitConfObj = self.getReadUnit()
610 readUnitConfObj = self.getReadUnit()
608 if readUnitConfObj:
611 if readUnitConfObj:
609 subtitle += '\nInput parameters:\n'
612 subtitle += '\nInput parameters:\n'
610 subtitle += '[Data path = %s]\n' % readUnitConfObj.parameters['path']
613 subtitle += '[Data path = %s]\n' % readUnitConfObj.parameters['path']
611 subtitle += '[Start date = %s]\n' % readUnitConfObj.parameters['startDate']
614 subtitle += '[Start date = %s]\n' % readUnitConfObj.parameters['startDate']
612 subtitle += '[End date = %s]\n' % readUnitConfObj.parameters['endDate']
615 subtitle += '[End date = %s]\n' % readUnitConfObj.parameters['endDate']
613 subtitle += '[Start time = %s]\n' % readUnitConfObj.parameters['startTime']
616 subtitle += '[Start time = %s]\n' % readUnitConfObj.parameters['startTime']
614 subtitle += '[End time = %s]\n' % readUnitConfObj.parameters['endTime']
617 subtitle += '[End time = %s]\n' % readUnitConfObj.parameters['endTime']
615
618
616 a = Alarm(
619 a = Alarm(
617 modes=self.alarm,
620 modes=self.alarm,
618 email=self.email,
621 email=self.email,
619 message=message,
622 message=message,
620 subject=subject,
623 subject=subject,
621 subtitle=subtitle,
624 subtitle=subtitle,
622 filename=self.filename
625 filename=self.filename
623 )
626 )
624
627
625 a.start()
628 a.start()
626
629
627 def setFilename(self, filename):
630 def setFilename(self, filename):
628
631
629 self.filename = filename
632 self.filename = filename
630
633
631 def runProcs(self):
634 def runProcs(self):
632
635
633 err = False
636 err = False
634 n = len(self.configurations)
637 n = len(self.configurations)
635
638
636 while not err:
639 while not err:
637 for conf in self.getUnits():
640 for conf in self.getUnits():
638 ok = conf.run()
641 ok = conf.run()
639 if ok is 'Error':
642 if ok == 'Error':
640 n -= 1
643 n -= 1
641 continue
644 continue
642 elif not ok:
645 elif not ok:
643 break
646 break
644 if n == 0:
647 if n == 0:
645 err = True
648 err = True
646
649
647 def run(self):
650 def run(self):
648
651
649 log.success('\nStarting Project {} [id={}]'.format(self.name, self.id), tag='')
652 log.success('\nStarting Project {} [id={}]'.format(self.name, self.id), tag='')
650 self.started = True
653 self.started = True
651 self.start_time = time.time()
654 self.start_time = time.time()
652 self.createObjects()
655 self.createObjects()
653 self.runProcs()
656 self.runProcs()
654 log.success('{} Done (Time: {:4.2f}s)'.format(
657 log.success('{} Done (Time: {:4.2f}s)'.format(
655 self.name,
658 self.name,
656 time.time()-self.start_time), '')
659 time.time()-self.start_time), '')
Show file before | Show file after | Hide comments
@@ -1,86 +1,87
1 # Copyright (c) 2012-2020 Jicamarca Radio Observatory
1 # Copyright (c) 2012-2020 Jicamarca Radio Observatory
2 # All rights reserved.
2 # All rights reserved.
3 #
3 #
4 # Distributed under the terms of the BSD 3-clause license.
4 # Distributed under the terms of the BSD 3-clause license.
5 """schainpy is an open source library to read, write and process radar data
5 """schainpy is an open source library to read, write and process radar data
6
6
7 Signal Chain is a radar data processing library wich includes modules to read,
7 Signal Chain is a radar data processing library wich includes modules to read,
8 and write different files formats, besides modules to process and visualize the
8 and write different files formats, besides modules to process and visualize the
9 data.
9 data.
10 """
10 """
11
11
12 import os
12 import os
13 from setuptools import setup, Extension
13 from setuptools import setup, Extension
14 from setuptools.command.build_ext import build_ext as _build_ext
14 from setuptools.command.build_ext import build_ext as _build_ext
15 from schainpy import __version__
15 from schainpy import __version__
16
16
17 DOCLINES = __doc__.split("\n")
17 DOCLINES = __doc__.split("\n")
18
18
19 class build_ext(_build_ext):
19 class build_ext(_build_ext):
20 def finalize_options(self):
20 def finalize_options(self):
21 _build_ext.finalize_options(self)
21 _build_ext.finalize_options(self)
22 # Prevent numpy from thinking it is still in its setup process:
22 # Prevent numpy from thinking it is still in its setup process:
23 __builtins__.__NUMPY_SETUP__ = False
23 __builtins__.__NUMPY_SETUP__ = False
24 import numpy
24 import numpy
25 self.include_dirs.append(numpy.get_include())
25 self.include_dirs.append(numpy.get_include())
26
26
27 setup(
27 setup(
28 name = "schainpy",
28 name = "schainpy",
29 version = __version__,
29 version = __version__,
30 description = DOCLINES[0],
30 description = DOCLINES[0],
31 long_description = "\n".join(DOCLINES[2:]),
31 long_description = "\n".join(DOCLINES[2:]),
32 url = "https://github.com/JRO-Peru/schainpy",
32 url = "https://github.com/JRO-Peru/schainpy",
33 author = "Jicamarca Radio Observatory",
33 author = "Jicamarca Radio Observatory",
34 author_email = "jro-developers@jro.igp.gob.pe",
34 author_email = "jro-developers@jro.igp.gob.pe",
35 license="BSD-3-Clause",
35 license="BSD-3-Clause",
36 classifiers=[
36 classifiers=[
37 "Development Status :: 4 - Beta",
37 "Development Status :: 4 - Beta",
38 "Environment :: Console",
38 "Environment :: Console",
39 "Intended Audience :: Science/Research",
39 "Intended Audience :: Science/Research",
40 "License :: OSI Approved :: BSD License",
40 "License :: OSI Approved :: BSD License",
41 "Operating System :: MacOS :: MacOS X",
41 "Operating System :: MacOS :: MacOS X",
42 "Operating System :: POSIX :: Linux",
42 "Operating System :: POSIX :: Linux",
43 "Programming Language :: Python :: 2",
43 "Programming Language :: Python :: 2",
44 "Programming Language :: Python :: 2.7",
44 "Programming Language :: Python :: 2.7",
45 "Programming Language :: Python :: 3",
45 "Programming Language :: Python :: 3",
46 "Programming Language :: Python :: 3.5",
46 "Programming Language :: Python :: 3.5",
47 "Programming Language :: Python :: 3.6",
47 "Programming Language :: Python :: 3.6",
48 "Programming Language :: Python :: 3.7",
48 "Programming Language :: Python :: 3.7",
49 "Programming Language :: Python :: 3.8",
49 "Topic :: Scientific/Engineering",
50 "Topic :: Scientific/Engineering",
50 ],
51 ],
51 packages = {
52 packages = {
52 'schainpy',
53 'schainpy',
53 'schainpy.model',
54 'schainpy.model',
54 'schainpy.model.data',
55 'schainpy.model.data',
55 'schainpy.model.graphics',
56 'schainpy.model.graphics',
56 'schainpy.model.io',
57 'schainpy.model.io',
57 'schainpy.model.proc',
58 'schainpy.model.proc',
58 'schainpy.model.utils',
59 'schainpy.model.utils',
59 'schainpy.utils',
60 'schainpy.utils',
60 'schainpy.gui',
61 'schainpy.gui',
61 'schainpy.cli',
62 'schainpy.cli',
62 },
63 },
63 package_data = {'': ['schain.conf.template'],
64 package_data = {'': ['schain.conf.template'],
64 'schainpy.files': ['*.oga']
65 'schainpy.files': ['*.oga']
65 },
66 },
66 include_package_data = False,
67 include_package_data = False,
67 scripts = ['schainpy/gui/schainGUI'],
68 scripts = ['schainpy/gui/schainGUI'],
68 entry_points = {
69 entry_points = {
69 'console_scripts': [
70 'console_scripts': [
70 'schain = schainpy.cli.cli:main',
71 'schain = schainpy.cli.cli:main',
71 ],
72 ],
72 },
73 },
73 cmdclass = {'build_ext': build_ext},
74 cmdclass = {'build_ext': build_ext},
74 ext_modules=[
75 ext_modules=[
75 Extension("schainpy.model.data._noise", ["schainc/_noise.c"]),
76 Extension("schainpy.model.data._noise", ["schainc/_noise.c"]),
76 ],
77 ],
77 setup_requires = ["numpy"],
78 setup_requires = ["numpy"],
78 install_requires = [
79 install_requires = [
79 "scipy",
80 "scipy",
80 "h5py",
81 "h5py",
81 "matplotlib",
82 "matplotlib",
82 "pyzmq",
83 "pyzmq",
83 "fuzzywuzzy",
84 "fuzzywuzzy",
84 "click",
85 "click",
85 ],
86 ],
86 )
87 )
General Comments 0
You need to be logged in to leave comments. Login now