##// END OF EJS Templates
Add python 3.8 compatibility with macOs
jespinoza -
r1342:b856bac3a9b1 v3.0.0b5
parent child
Show More
@@ -1,8 +1,8
1 1 """Signal chain python package"""
2 2
3 3 try:
4 from .controller import Project
4 from schainpy.controller import Project
5 5 except:
6 6 pass
7 7
8 __version__ = '3.0.0b4'
8 __version__ = '3.0.0b5'
@@ -1,656 +1,659
1 1 # Copyright (c) 2012-2020 Jicamarca Radio Observatory
2 2 # All rights reserved.
3 3 #
4 4 # Distributed under the terms of the BSD 3-clause license.
5 5 """API to create signal chain projects
6 6
7 7 The API is provide through class: Project
8 8 """
9 9
10 10 import re
11 11 import sys
12 12 import ast
13 13 import datetime
14 14 import traceback
15 15 import time
16 import multiprocessing
16 17 from multiprocessing import Process, Queue
17 18 from threading import Thread
18 19 from xml.etree.ElementTree import ElementTree, Element, SubElement
19 20
20 21 from schainpy.admin import Alarm, SchainWarning
21 22 from schainpy.model import *
22 23 from schainpy.utils import log
23 24
25 if 'darwin' in sys.platform and sys.version_info[0] == 3 and sys.version_info[1] > 7:
26 multiprocessing.set_start_method('fork')
24 27
25 28 class ConfBase():
26 29
27 30 def __init__(self):
28 31
29 32 self.id = '0'
30 33 self.name = None
31 34 self.priority = None
32 35 self.parameters = {}
33 36 self.object = None
34 37 self.operations = []
35 38
36 39 def getId(self):
37 40
38 41 return self.id
39 42
40 43 def getNewId(self):
41 44
42 45 return int(self.id) * 10 + len(self.operations) + 1
43 46
44 47 def updateId(self, new_id):
45 48
46 49 self.id = str(new_id)
47 50
48 51 n = 1
49 52 for conf in self.operations:
50 53 conf_id = str(int(new_id) * 10 + n)
51 54 conf.updateId(conf_id)
52 55 n += 1
53 56
54 57 def getKwargs(self):
55 58
56 59 params = {}
57 60
58 61 for key, value in self.parameters.items():
59 62 if value not in (None, '', ' '):
60 63 params[key] = value
61 64
62 65 return params
63 66
64 67 def update(self, **kwargs):
65 68
66 69 for key, value in kwargs.items():
67 70 self.addParameter(name=key, value=value)
68 71
69 72 def addParameter(self, name, value, format=None):
70 73 '''
71 74 '''
72 75
73 76 if isinstance(value, str) and re.search(r'(\d+/\d+/\d+)', value):
74 77 self.parameters[name] = datetime.date(*[int(x) for x in value.split('/')])
75 78 elif isinstance(value, str) and re.search(r'(\d+:\d+:\d+)', value):
76 79 self.parameters[name] = datetime.time(*[int(x) for x in value.split(':')])
77 80 else:
78 81 try:
79 82 self.parameters[name] = ast.literal_eval(value)
80 83 except:
81 84 if isinstance(value, str) and ',' in value:
82 85 self.parameters[name] = value.split(',')
83 86 else:
84 87 self.parameters[name] = value
85 88
86 89 def getParameters(self):
87 90
88 91 params = {}
89 92 for key, value in self.parameters.items():
90 93 s = type(value).__name__
91 94 if s == 'date':
92 95 params[key] = value.strftime('%Y/%m/%d')
93 96 elif s == 'time':
94 97 params[key] = value.strftime('%H:%M:%S')
95 98 else:
96 99 params[key] = str(value)
97 100
98 101 return params
99 102
100 103 def makeXml(self, element):
101 104
102 105 xml = SubElement(element, self.ELEMENTNAME)
103 106 for label in self.xml_labels:
104 107 xml.set(label, str(getattr(self, label)))
105 108
106 109 for key, value in self.getParameters().items():
107 110 xml_param = SubElement(xml, 'Parameter')
108 111 xml_param.set('name', key)
109 112 xml_param.set('value', value)
110 113
111 114 for conf in self.operations:
112 115 conf.makeXml(xml)
113 116
114 117 def __str__(self):
115 118
116 119 if self.ELEMENTNAME == 'Operation':
117 120 s = ' {}[id={}]\n'.format(self.name, self.id)
118 121 else:
119 122 s = '{}[id={}, inputId={}]\n'.format(self.name, self.id, self.inputId)
120 123
121 124 for key, value in self.parameters.items():
122 125 if self.ELEMENTNAME == 'Operation':
123 126 s += ' {}: {}\n'.format(key, value)
124 127 else:
125 128 s += ' {}: {}\n'.format(key, value)
126 129
127 130 for conf in self.operations:
128 131 s += str(conf)
129 132
130 133 return s
131 134
132 135 class OperationConf(ConfBase):
133 136
134 137 ELEMENTNAME = 'Operation'
135 138 xml_labels = ['id', 'name']
136 139
137 140 def setup(self, id, name, priority, project_id, err_queue):
138 141
139 142 self.id = str(id)
140 143 self.project_id = project_id
141 144 self.name = name
142 145 self.type = 'other'
143 146 self.err_queue = err_queue
144 147
145 148 def readXml(self, element, project_id, err_queue):
146 149
147 150 self.id = element.get('id')
148 151 self.name = element.get('name')
149 152 self.type = 'other'
150 153 self.project_id = str(project_id)
151 154 self.err_queue = err_queue
152 155
153 156 for elm in element.iter('Parameter'):
154 157 self.addParameter(elm.get('name'), elm.get('value'))
155 158
156 159 def createObject(self):
157 160
158 161 className = eval(self.name)
159 162
160 163 if 'Plot' in self.name or 'Writer' in self.name or 'Send' in self.name or 'print' in self.name:
161 164 kwargs = self.getKwargs()
162 165 opObj = className(self.id, self.id, self.project_id, self.err_queue, **kwargs)
163 166 opObj.start()
164 167 self.type = 'external'
165 168 else:
166 169 opObj = className()
167 170
168 171 self.object = opObj
169 172 return opObj
170 173
171 174 class ProcUnitConf(ConfBase):
172 175
173 176 ELEMENTNAME = 'ProcUnit'
174 177 xml_labels = ['id', 'inputId', 'name']
175 178
176 179 def setup(self, project_id, id, name, datatype, inputId, err_queue):
177 180 '''
178 181 '''
179 182
180 183 if datatype == None and name == None:
181 184 raise ValueError('datatype or name should be defined')
182 185
183 186 if name == None:
184 187 if 'Proc' in datatype:
185 188 name = datatype
186 189 else:
187 190 name = '%sProc' % (datatype)
188 191
189 192 if datatype == None:
190 193 datatype = name.replace('Proc', '')
191 194
192 195 self.id = str(id)
193 196 self.project_id = project_id
194 197 self.name = name
195 198 self.datatype = datatype
196 199 self.inputId = inputId
197 200 self.err_queue = err_queue
198 201 self.operations = []
199 202 self.parameters = {}
200 203
201 204 def removeOperation(self, id):
202 205
203 206 i = [1 if x.id==id else 0 for x in self.operations]
204 207 self.operations.pop(i.index(1))
205 208
206 209 def getOperation(self, id):
207 210
208 211 for conf in self.operations:
209 212 if conf.id == id:
210 213 return conf
211 214
212 215 def addOperation(self, name, optype='self'):
213 216 '''
214 217 '''
215 218
216 219 id = self.getNewId()
217 220 conf = OperationConf()
218 221 conf.setup(id, name=name, priority='0', project_id=self.project_id, err_queue=self.err_queue)
219 222 self.operations.append(conf)
220 223
221 224 return conf
222 225
223 226 def readXml(self, element, project_id, err_queue):
224 227
225 228 self.id = element.get('id')
226 229 self.name = element.get('name')
227 230 self.inputId = None if element.get('inputId') == 'None' else element.get('inputId')
228 231 self.datatype = element.get('datatype', self.name.replace(self.ELEMENTNAME.replace('Unit', ''), ''))
229 232 self.project_id = str(project_id)
230 233 self.err_queue = err_queue
231 234 self.operations = []
232 235 self.parameters = {}
233 236
234 237 for elm in element:
235 238 if elm.tag == 'Parameter':
236 239 self.addParameter(elm.get('name'), elm.get('value'))
237 240 elif elm.tag == 'Operation':
238 241 conf = OperationConf()
239 242 conf.readXml(elm, project_id, err_queue)
240 243 self.operations.append(conf)
241 244
242 245 def createObjects(self):
243 246 '''
244 247 Instancia de unidades de procesamiento.
245 248 '''
246 249
247 250 className = eval(self.name)
248 251 kwargs = self.getKwargs()
249 252 procUnitObj = className()
250 253 procUnitObj.name = self.name
251 254 log.success('creating process...', self.name)
252 255
253 256 for conf in self.operations:
254 257
255 258 opObj = conf.createObject()
256 259
257 260 log.success('adding operation: {}, type:{}'.format(
258 261 conf.name,
259 262 conf.type), self.name)
260 263
261 264 procUnitObj.addOperation(conf, opObj)
262 265
263 266 self.object = procUnitObj
264 267
265 268 def run(self):
266 269 '''
267 270 '''
268 271
269 272 return self.object.call(**self.getKwargs())
270 273
271 274
272 275 class ReadUnitConf(ProcUnitConf):
273 276
274 277 ELEMENTNAME = 'ReadUnit'
275 278
276 279 def __init__(self):
277 280
278 281 self.id = None
279 282 self.datatype = None
280 283 self.name = None
281 284 self.inputId = None
282 285 self.operations = []
283 286 self.parameters = {}
284 287
285 288 def setup(self, project_id, id, name, datatype, err_queue, path='', startDate='', endDate='',
286 289 startTime='', endTime='', server=None, **kwargs):
287 290
288 291 if datatype == None and name == None:
289 292 raise ValueError('datatype or name should be defined')
290 293 if name == None:
291 294 if 'Reader' in datatype:
292 295 name = datatype
293 296 datatype = name.replace('Reader','')
294 297 else:
295 298 name = '{}Reader'.format(datatype)
296 299 if datatype == None:
297 300 if 'Reader' in name:
298 301 datatype = name.replace('Reader','')
299 302 else:
300 303 datatype = name
301 304 name = '{}Reader'.format(name)
302 305
303 306 self.id = id
304 307 self.project_id = project_id
305 308 self.name = name
306 309 self.datatype = datatype
307 310 self.err_queue = err_queue
308 311
309 312 self.addParameter(name='path', value=path)
310 313 self.addParameter(name='startDate', value=startDate)
311 314 self.addParameter(name='endDate', value=endDate)
312 315 self.addParameter(name='startTime', value=startTime)
313 316 self.addParameter(name='endTime', value=endTime)
314 317
315 318 for key, value in kwargs.items():
316 319 self.addParameter(name=key, value=value)
317 320
318 321
319 322 class Project(Process):
320 323 """API to create signal chain projects"""
321 324
322 325 ELEMENTNAME = 'Project'
323 326
324 327 def __init__(self, name=''):
325 328
326 329 Process.__init__(self)
327 330 self.id = '1'
328 331 if name:
329 332 self.name = '{} ({})'.format(Process.__name__, name)
330 333 self.filename = None
331 334 self.description = None
332 335 self.email = None
333 336 self.alarm = []
334 337 self.configurations = {}
335 338 # self.err_queue = Queue()
336 339 self.err_queue = None
337 340 self.started = False
338 341
339 342 def getNewId(self):
340 343
341 344 idList = list(self.configurations.keys())
342 345 id = int(self.id) * 10
343 346
344 347 while True:
345 348 id += 1
346 349
347 350 if str(id) in idList:
348 351 continue
349 352
350 353 break
351 354
352 355 return str(id)
353 356
354 357 def updateId(self, new_id):
355 358
356 359 self.id = str(new_id)
357 360
358 361 keyList = list(self.configurations.keys())
359 362 keyList.sort()
360 363
361 364 n = 1
362 365 new_confs = {}
363 366
364 367 for procKey in keyList:
365 368
366 369 conf = self.configurations[procKey]
367 370 idProcUnit = str(int(self.id) * 10 + n)
368 371 conf.updateId(idProcUnit)
369 372 new_confs[idProcUnit] = conf
370 373 n += 1
371 374
372 375 self.configurations = new_confs
373 376
374 377 def setup(self, id=1, name='', description='', email=None, alarm=[]):
375 378
376 379 self.id = str(id)
377 380 self.description = description
378 381 self.email = email
379 382 self.alarm = alarm
380 383 if name:
381 384 self.name = '{} ({})'.format(Process.__name__, name)
382 385
383 386 def update(self, **kwargs):
384 387
385 388 for key, value in kwargs.items():
386 389 setattr(self, key, value)
387 390
388 391 def clone(self):
389 392
390 393 p = Project()
391 394 p.id = self.id
392 395 p.name = self.name
393 396 p.description = self.description
394 397 p.configurations = self.configurations.copy()
395 398
396 399 return p
397 400
398 401 def addReadUnit(self, id=None, datatype=None, name=None, **kwargs):
399 402
400 403 '''
401 404 '''
402 405
403 406 if id is None:
404 407 idReadUnit = self.getNewId()
405 408 else:
406 409 idReadUnit = str(id)
407 410
408 411 conf = ReadUnitConf()
409 412 conf.setup(self.id, idReadUnit, name, datatype, self.err_queue, **kwargs)
410 413 self.configurations[conf.id] = conf
411 414
412 415 return conf
413 416
414 417 def addProcUnit(self, id=None, inputId='0', datatype=None, name=None):
415 418
416 419 '''
417 420 '''
418 421
419 422 if id is None:
420 423 idProcUnit = self.getNewId()
421 424 else:
422 425 idProcUnit = id
423 426
424 427 conf = ProcUnitConf()
425 428 conf.setup(self.id, idProcUnit, name, datatype, inputId, self.err_queue)
426 429 self.configurations[conf.id] = conf
427 430
428 431 return conf
429 432
430 433 def removeProcUnit(self, id):
431 434
432 435 if id in self.configurations:
433 436 self.configurations.pop(id)
434 437
435 438 def getReadUnit(self):
436 439
437 440 for obj in list(self.configurations.values()):
438 441 if obj.ELEMENTNAME == 'ReadUnit':
439 442 return obj
440 443
441 444 return None
442 445
443 446 def getProcUnit(self, id):
444 447
445 448 return self.configurations[id]
446 449
447 450 def getUnits(self):
448 451
449 452 keys = list(self.configurations)
450 453 keys.sort()
451 454
452 455 for key in keys:
453 456 yield self.configurations[key]
454 457
455 458 def updateUnit(self, id, **kwargs):
456 459
457 460 conf = self.configurations[id].update(**kwargs)
458 461
459 462 def makeXml(self):
460 463
461 464 xml = Element('Project')
462 465 xml.set('id', str(self.id))
463 466 xml.set('name', self.name)
464 467 xml.set('description', self.description)
465 468
466 469 for conf in self.configurations.values():
467 470 conf.makeXml(xml)
468 471
469 472 self.xml = xml
470 473
471 474 def writeXml(self, filename=None):
472 475
473 476 if filename == None:
474 477 if self.filename:
475 478 filename = self.filename
476 479 else:
477 480 filename = 'schain.xml'
478 481
479 482 if not filename:
480 483 print('filename has not been defined. Use setFilename(filename) for do it.')
481 484 return 0
482 485
483 486 abs_file = os.path.abspath(filename)
484 487
485 488 if not os.access(os.path.dirname(abs_file), os.W_OK):
486 489 print('No write permission on %s' % os.path.dirname(abs_file))
487 490 return 0
488 491
489 492 if os.path.isfile(abs_file) and not(os.access(abs_file, os.W_OK)):
490 493 print('File %s already exists and it could not be overwriten' % abs_file)
491 494 return 0
492 495
493 496 self.makeXml()
494 497
495 498 ElementTree(self.xml).write(abs_file, method='xml')
496 499
497 500 self.filename = abs_file
498 501
499 502 return 1
500 503
501 504 def readXml(self, filename):
502 505
503 506 abs_file = os.path.abspath(filename)
504 507
505 508 self.configurations = {}
506 509
507 510 try:
508 511 self.xml = ElementTree().parse(abs_file)
509 512 except:
510 513 log.error('Error reading %s, verify file format' % filename)
511 514 return 0
512 515
513 516 self.id = self.xml.get('id')
514 517 self.name = self.xml.get('name')
515 518 self.description = self.xml.get('description')
516 519
517 520 for element in self.xml:
518 521 if element.tag == 'ReadUnit':
519 522 conf = ReadUnitConf()
520 523 conf.readXml(element, self.id, self.err_queue)
521 524 self.configurations[conf.id] = conf
522 525 elif element.tag == 'ProcUnit':
523 526 conf = ProcUnitConf()
524 527 input_proc = self.configurations[element.get('inputId')]
525 528 conf.readXml(element, self.id, self.err_queue)
526 529 self.configurations[conf.id] = conf
527 530
528 531 self.filename = abs_file
529 532
530 533 return 1
531 534
532 535 def __str__(self):
533 536
534 537 text = '\nProject[id=%s, name=%s, description=%s]\n\n' % (
535 538 self.id,
536 539 self.name,
537 540 self.description,
538 541 )
539 542
540 543 for conf in self.configurations.values():
541 544 text += '{}'.format(conf)
542 545
543 546 return text
544 547
545 548 def createObjects(self):
546 549
547 550 keys = list(self.configurations.keys())
548 551 keys.sort()
549 552 for key in keys:
550 553 conf = self.configurations[key]
551 554 conf.createObjects()
552 555 if conf.inputId is not None:
553 556 conf.object.setInput(self.configurations[conf.inputId].object)
554 557
555 558 def monitor(self):
556 559
557 560 t = Thread(target=self._monitor, args=(self.err_queue, self.ctx))
558 561 t.start()
559 562
560 563 def _monitor(self, queue, ctx):
561 564
562 565 import socket
563 566
564 567 procs = 0
565 568 err_msg = ''
566 569
567 570 while True:
568 571 msg = queue.get()
569 572 if '#_start_#' in msg:
570 573 procs += 1
571 574 elif '#_end_#' in msg:
572 575 procs -=1
573 576 else:
574 577 err_msg = msg
575 578
576 579 if procs == 0 or 'Traceback' in err_msg:
577 580 break
578 581 time.sleep(0.1)
579 582
580 583 if '|' in err_msg:
581 584 name, err = err_msg.split('|')
582 585 if 'SchainWarning' in err:
583 586 log.warning(err.split('SchainWarning:')[-1].split('\n')[0].strip(), name)
584 587 elif 'SchainError' in err:
585 588 log.error(err.split('SchainError:')[-1].split('\n')[0].strip(), name)
586 589 else:
587 590 log.error(err, name)
588 591 else:
589 592 name, err = self.name, err_msg
590 593
591 594 time.sleep(1)
592 595
593 596 ctx.term()
594 597
595 598 message = ''.join(err)
596 599
597 600 if err_msg:
598 601 subject = 'SChain v%s: Error running %s\n' % (
599 602 schainpy.__version__, self.name)
600 603
601 604 subtitle = 'Hostname: %s\n' % socket.gethostbyname(
602 605 socket.gethostname())
603 606 subtitle += 'Working directory: %s\n' % os.path.abspath('./')
604 607 subtitle += 'Configuration file: %s\n' % self.filename
605 608 subtitle += 'Time: %s\n' % str(datetime.datetime.now())
606 609
607 610 readUnitConfObj = self.getReadUnit()
608 611 if readUnitConfObj:
609 612 subtitle += '\nInput parameters:\n'
610 613 subtitle += '[Data path = %s]\n' % readUnitConfObj.parameters['path']
611 614 subtitle += '[Start date = %s]\n' % readUnitConfObj.parameters['startDate']
612 615 subtitle += '[End date = %s]\n' % readUnitConfObj.parameters['endDate']
613 616 subtitle += '[Start time = %s]\n' % readUnitConfObj.parameters['startTime']
614 617 subtitle += '[End time = %s]\n' % readUnitConfObj.parameters['endTime']
615 618
616 619 a = Alarm(
617 620 modes=self.alarm,
618 621 email=self.email,
619 622 message=message,
620 623 subject=subject,
621 624 subtitle=subtitle,
622 625 filename=self.filename
623 626 )
624 627
625 628 a.start()
626 629
627 630 def setFilename(self, filename):
628 631
629 632 self.filename = filename
630 633
631 634 def runProcs(self):
632 635
633 636 err = False
634 637 n = len(self.configurations)
635 638
636 639 while not err:
637 640 for conf in self.getUnits():
638 641 ok = conf.run()
639 if ok is 'Error':
642 if ok == 'Error':
640 643 n -= 1
641 644 continue
642 645 elif not ok:
643 646 break
644 647 if n == 0:
645 648 err = True
646 649
647 650 def run(self):
648 651
649 652 log.success('\nStarting Project {} [id={}]'.format(self.name, self.id), tag='')
650 653 self.started = True
651 654 self.start_time = time.time()
652 655 self.createObjects()
653 656 self.runProcs()
654 657 log.success('{} Done (Time: {:4.2f}s)'.format(
655 658 self.name,
656 659 time.time()-self.start_time), '')
@@ -1,86 +1,87
1 1 # Copyright (c) 2012-2020 Jicamarca Radio Observatory
2 2 # All rights reserved.
3 3 #
4 4 # Distributed under the terms of the BSD 3-clause license.
5 5 """schainpy is an open source library to read, write and process radar data
6 6
7 7 Signal Chain is a radar data processing library wich includes modules to read,
8 8 and write different files formats, besides modules to process and visualize the
9 9 data.
10 10 """
11 11
12 12 import os
13 13 from setuptools import setup, Extension
14 14 from setuptools.command.build_ext import build_ext as _build_ext
15 15 from schainpy import __version__
16 16
17 17 DOCLINES = __doc__.split("\n")
18 18
19 19 class build_ext(_build_ext):
20 20 def finalize_options(self):
21 21 _build_ext.finalize_options(self)
22 22 # Prevent numpy from thinking it is still in its setup process:
23 23 __builtins__.__NUMPY_SETUP__ = False
24 24 import numpy
25 25 self.include_dirs.append(numpy.get_include())
26 26
27 27 setup(
28 28 name = "schainpy",
29 29 version = __version__,
30 30 description = DOCLINES[0],
31 31 long_description = "\n".join(DOCLINES[2:]),
32 32 url = "https://github.com/JRO-Peru/schainpy",
33 33 author = "Jicamarca Radio Observatory",
34 34 author_email = "jro-developers@jro.igp.gob.pe",
35 35 license="BSD-3-Clause",
36 36 classifiers=[
37 37 "Development Status :: 4 - Beta",
38 38 "Environment :: Console",
39 39 "Intended Audience :: Science/Research",
40 40 "License :: OSI Approved :: BSD License",
41 41 "Operating System :: MacOS :: MacOS X",
42 42 "Operating System :: POSIX :: Linux",
43 43 "Programming Language :: Python :: 2",
44 44 "Programming Language :: Python :: 2.7",
45 45 "Programming Language :: Python :: 3",
46 46 "Programming Language :: Python :: 3.5",
47 47 "Programming Language :: Python :: 3.6",
48 48 "Programming Language :: Python :: 3.7",
49 "Programming Language :: Python :: 3.8",
49 50 "Topic :: Scientific/Engineering",
50 51 ],
51 52 packages = {
52 53 'schainpy',
53 54 'schainpy.model',
54 55 'schainpy.model.data',
55 56 'schainpy.model.graphics',
56 57 'schainpy.model.io',
57 58 'schainpy.model.proc',
58 59 'schainpy.model.utils',
59 60 'schainpy.utils',
60 61 'schainpy.gui',
61 62 'schainpy.cli',
62 63 },
63 64 package_data = {'': ['schain.conf.template'],
64 65 'schainpy.files': ['*.oga']
65 66 },
66 67 include_package_data = False,
67 68 scripts = ['schainpy/gui/schainGUI'],
68 69 entry_points = {
69 70 'console_scripts': [
70 71 'schain = schainpy.cli.cli:main',
71 72 ],
72 73 },
73 74 cmdclass = {'build_ext': build_ext},
74 75 ext_modules=[
75 76 Extension("schainpy.model.data._noise", ["schainc/_noise.c"]),
76 77 ],
77 78 setup_requires = ["numpy"],
78 79 install_requires = [
79 80 "scipy",
80 81 "h5py",
81 82 "matplotlib",
82 83 "pyzmq",
83 84 "fuzzywuzzy",
84 85 "click",
85 86 ],
86 87 )
General Comments 0
You need to be logged in to leave comments. Login now