##// END OF EJS Templates
schain-jc
RSS
merge desde origin
José Chávez -
r1024:cfc3258ff285 merge
parent child
Show More
Side by Side Unified
Context file:
r1024:cfc3258ff285 Loading diff...:
Show file before | Show file after | Hide comments
@@ -1,113 +1,113
1 1 # Byte-compiled / optimized / DLL files
2 2 __pycache__/
3 3 *.py[cod]
4 4 *$py.class
5 5
6 6 # C extensions
7 7 *.so
8 8
9 9 # Distribution / packaging
10 10 .Python
11 11 env/
12 12 build/
13 13 develop-eggs/
14 14 dist/
15 15 downloads/
16 16 eggs/
17 17 .eggs/
18 18 lib/
19 19 lib64/
20 20 parts/
21 21 sdist/
22 22 var/
23 23 wheels/
24 24 *.egg-info/
25 25 .installed.cfg
26 26 *.egg
27 27
28 28 # PyInstaller
29 29 # Usually these files are written by a python script from a template
30 30 # before PyInstaller builds the exe, so as to inject date/other infos into it.
31 31 *.manifest
32 32 *.spec
33 33
34 34 # Installer logs
35 35 pip-log.txt
36 36 pip-delete-this-directory.txt
37 37
38 38 # Unit test / coverage reports
39 39 htmlcov/
40 40 .tox/
41 41 .coverage
42 42 .coverage.*
43 43 .cache
44 44 nosetests.xml
45 45 coverage.xml
46 46 *,cover
47 47 .hypothesis/
48 48
49 49 # Translations
50 50 *.mo
51 51 *.pot
52 52
53 53 # Django stuff:
54 54 *.log
55 55 local_settings.py
56 56
57 57 # Flask stuff:
58 58 instance/
59 59 .webassets-cache
60 60
61 61 # Scrapy stuff:
62 62 .scrapy
63 63
64 64 # Sphinx documentation
65 65 docs/_build/
66 66
67 67 # PyBuilder
68 68 target/
69 69
70 70 # Jupyter Notebook
71 71 .ipynb_checkpoints
72 72
73 73 # pyenv
74 74 .python-version
75 75
76 76 # celery beat schedule file
77 77 celerybeat-schedule
78 78
79 79 # SageMath parsed files
80 80 *.sage.py
81 81
82 82 # dotenv
83 83 .env
84 84
85 85 # virtualenv
86 86 .venv
87 87 venv/
88 88 ENV/
89 89
90 90 # Spyder project settings
91 91 .spyderproject
92 92 .spyproject
93 93
94 94 # Rope project settings
95 95 .ropeproject
96 96
97 97 # mkdocs documentation
98 98 /site
99 99
100 100 # eclipse
101 101 .project
102 102 .pydevproject
103 103
104 104 # vscode
105 105
106 106 .vscode
107 107
108 108 schainpy/scripts/
109 109 schaingui/node_modules/
110 110 .svn/
111 111 *.png
112 112 *.pyc
113 *.xml No newline at end of file
113 *.xml
Show file before | Show file after | Hide comments
@@ -1,1328 +1,1329
1 1 '''
2 2 Created on September , 2012
3 3 @author:
4 4 '''
5 5
6 6 import sys
7 7 import ast
8 8 import datetime
9 9 import traceback
10 10 import math
11 11 import time
12 12 from multiprocessing import Process, Queue, cpu_count
13 13
14 14 import schainpy
15 15 import schainpy.admin
16 16
17 17 from xml.etree.ElementTree import ElementTree, Element, SubElement, tostring
18 18 from xml.dom import minidom
19 19
20 20 from schainpy.model import *
21 21 from time import sleep
22 22
23 23
24 24
25 25 def prettify(elem):
26 26 """Return a pretty-printed XML string for the Element.
27 27 """
28 28 rough_string = tostring(elem, 'utf-8')
29 29 reparsed = minidom.parseString(rough_string)
30 30 return reparsed.toprettyxml(indent=" ")
31 31
32 32 def multiSchain(child, nProcess=cpu_count(), startDate=None, endDate=None, by_day=False):
33 33 skip = 0
34 34 cursor = 0
35 35 nFiles = None
36 36 processes = []
37 37 dt1 = datetime.datetime.strptime(startDate, '%Y/%m/%d')
38 38 dt2 = datetime.datetime.strptime(endDate, '%Y/%m/%d')
39 39 days = (dt2 - dt1).days
40 40
41 41 for day in range(days+1):
42 42 skip = 0
43 43 cursor = 0
44 44 q = Queue()
45 45 processes = []
46 46 dt = (dt1 + datetime.timedelta(day)).strftime('%Y/%m/%d')
47 47 firstProcess = Process(target=child, args=(cursor, skip, q, dt))
48 48 firstProcess.start()
49 49 if by_day:
50 50 continue
51 51 nFiles = q.get()
52 52 if nFiles==0:
53 53 continue
54 54 firstProcess.terminate()
55 55 skip = int(math.ceil(nFiles/nProcess))
56 56 while True:
57 57 processes.append(Process(target=child, args=(cursor, skip, q, dt)))
58 58 processes[cursor].start()
59 59 if nFiles < cursor*skip:
60 60 break
61 61 cursor += 1
62 62
63 63 def beforeExit(exctype, value, trace):
64 64 for process in processes:
65 65 process.terminate()
66 66 process.join()
67 67 print traceback.print_tb(trace)
68 68
69 69 sys.excepthook = beforeExit
70 70
71 71 for process in processes:
72 72 process.join()
73 73 process.terminate()
74 74
75 75 time.sleep(3)
76 76
77 77
78 78 class ParameterConf():
79 79
80 80 id = None
81 81 name = None
82 82 value = None
83 83 format = None
84 84
85 85 __formated_value = None
86 86
87 87 ELEMENTNAME = 'Parameter'
88 88
89 89 def __init__(self):
90 90
91 91 self.format = 'str'
92 92
93 93 def getElementName(self):
94 94
95 95 return self.ELEMENTNAME
96 96
97 97 def getValue(self):
98 98
99 99 value = self.value
100 100 format = self.format
101 101
102 102 if self.__formated_value != None:
103 103
104 104 return self.__formated_value
105 105
106 106 if format == 'obj':
107 107 return value
108 108
109 109 if format == 'str':
110 110 self.__formated_value = str(value)
111 111 return self.__formated_value
112 112
113 113 if value == '':
114 114 raise ValueError, "%s: This parameter value is empty" %self.name
115 115
116 116 if format == 'list':
117 117 strList = value.split(',')
118 118
119 119 self.__formated_value = strList
120 120
121 121 return self.__formated_value
122 122
123 123 if format == 'intlist':
124 124 """
125 125 Example:
126 126 value = (0,1,2)
127 127 """
128 128
129 129 new_value = ast.literal_eval(value)
130 130
131 131 if type(new_value) not in (tuple, list):
132 132 new_value = [int(new_value)]
133 133
134 134 self.__formated_value = new_value
135 135
136 136 return self.__formated_value
137 137
138 138 if format == 'floatlist':
139 139 """
140 140 Example:
141 141 value = (0.5, 1.4, 2.7)
142 142 """
143 143
144 144 new_value = ast.literal_eval(value)
145 145
146 146 if type(new_value) not in (tuple, list):
147 147 new_value = [float(new_value)]
148 148
149 149 self.__formated_value = new_value
150 150
151 151 return self.__formated_value
152 152
153 153 if format == 'date':
154 154 strList = value.split('/')
155 155 intList = [int(x) for x in strList]
156 156 date = datetime.date(intList[0], intList[1], intList[2])
157 157
158 158 self.__formated_value = date
159 159
160 160 return self.__formated_value
161 161
162 162 if format == 'time':
163 163 strList = value.split(':')
164 164 intList = [int(x) for x in strList]
165 165 time = datetime.time(intList[0], intList[1], intList[2])
166 166
167 167 self.__formated_value = time
168 168
169 169 return self.__formated_value
170 170
171 171 if format == 'pairslist':
172 172 """
173 173 Example:
174 174 value = (0,1),(1,2)
175 175 """
176 176
177 177 new_value = ast.literal_eval(value)
178 178
179 179 if type(new_value) not in (tuple, list):
180 180 raise ValueError, "%s has to be a tuple or list of pairs" %value
181 181
182 182 if type(new_value[0]) not in (tuple, list):
183 183 if len(new_value) != 2:
184 184 raise ValueError, "%s has to be a tuple or list of pairs" %value
185 185 new_value = [new_value]
186 186
187 187 for thisPair in new_value:
188 188 if len(thisPair) != 2:
189 189 raise ValueError, "%s has to be a tuple or list of pairs" %value
190 190
191 191 self.__formated_value = new_value
192 192
193 193 return self.__formated_value
194 194
195 195 if format == 'multilist':
196 196 """
197 197 Example:
198 198 value = (0,1,2),(3,4,5)
199 199 """
200 200 multiList = ast.literal_eval(value)
201 201
202 202 if type(multiList[0]) == int:
203 203 multiList = ast.literal_eval("(" + value + ")")
204 204
205 205 self.__formated_value = multiList
206 206
207 207 return self.__formated_value
208 208
209 209 if format == 'bool':
210 210 value = int(value)
211 211
212 212 if format == 'int':
213 213 value = float(value)
214 214
215 215 format_func = eval(format)
216 216
217 217 self.__formated_value = format_func(value)
218 218
219 219 return self.__formated_value
220 220
221 221 def updateId(self, new_id):
222 222
223 223 self.id = str(new_id)
224 224
225 225 def setup(self, id, name, value, format='str'):
226 226 self.id = str(id)
227 227 self.name = name
228 228 if format == 'obj':
229 229 self.value = value
230 230 else:
231 231 self.value = str(value)
232 232 self.format = str.lower(format)
233 233
234 234 self.getValue()
235 235
236 236 return 1
237 237
238 238 def update(self, name, value, format='str'):
239 239
240 240 self.name = name
241 241 self.value = str(value)
242 242 self.format = format
243 243
244 244 def makeXml(self, opElement):
245 245 if self.name not in ('queue',):
246 246 parmElement = SubElement(opElement, self.ELEMENTNAME)
247 247 parmElement.set('id', str(self.id))
248 248 parmElement.set('name', self.name)
249 249 parmElement.set('value', self.value)
250 250 parmElement.set('format', self.format)
251 251
252 252 def readXml(self, parmElement):
253 253
254 254 self.id = parmElement.get('id')
255 255 self.name = parmElement.get('name')
256 256 self.value = parmElement.get('value')
257 257 self.format = str.lower(parmElement.get('format'))
258 258
259 259 #Compatible with old signal chain version
260 260 if self.format == 'int' and self.name == 'idfigure':
261 261 self.name = 'id'
262 262
263 263 def printattr(self):
264 264
265 265 print "Parameter[%s]: name = %s, value = %s, format = %s" %(self.id, self.name, self.value, self.format)
266 266
267 267 class OperationConf():
268 268
269 269 id = None
270 270 name = None
271 271 priority = None
272 272 type = None
273 273
274 274 parmConfObjList = []
275 275
276 276 ELEMENTNAME = 'Operation'
277 277
278 278 def __init__(self):
279 279
280 280 self.id = '0'
281 281 self.name = None
282 282 self.priority = None
283 283 self.type = 'self'
284 284
285 285
286 286 def __getNewId(self):
287 287
288 288 return int(self.id)*10 + len(self.parmConfObjList) + 1
289 289
290 290 def updateId(self, new_id):
291 291
292 292 self.id = str(new_id)
293 293
294 294 n = 1
295 295 for parmObj in self.parmConfObjList:
296 296
297 297 idParm = str(int(new_id)*10 + n)
298 298 parmObj.updateId(idParm)
299 299
300 300 n += 1
301 301
302 302 def getElementName(self):
303 303
304 304 return self.ELEMENTNAME
305 305
306 306 def getParameterObjList(self):
307 307
308 308 return self.parmConfObjList
309 309
310 310 def getParameterObj(self, parameterName):
311 311
312 312 for parmConfObj in self.parmConfObjList:
313 313
314 314 if parmConfObj.name != parameterName:
315 315 continue
316 316
317 317 return parmConfObj
318 318
319 319 return None
320 320
321 321 def getParameterObjfromValue(self, parameterValue):
322 322
323 323 for parmConfObj in self.parmConfObjList:
324 324
325 325 if parmConfObj.getValue() != parameterValue:
326 326 continue
327 327
328 328 return parmConfObj.getValue()
329 329
330 330 return None
331 331
332 332 def getParameterValue(self, parameterName):
333 333
334 334 parameterObj = self.getParameterObj(parameterName)
335 335
336 336 # if not parameterObj:
337 337 # return None
338 338
339 339 value = parameterObj.getValue()
340 340
341 341 return value
342 342
343 343
344 344 def getKwargs(self):
345 345
346 346 kwargs = {}
347 347
348 348 for parmConfObj in self.parmConfObjList:
349 349 if self.name == 'run' and parmConfObj.name == 'datatype':
350 350 continue
351 351
352 352 kwargs[parmConfObj.name] = parmConfObj.getValue()
353 353
354 354 return kwargs
355 355
356 356 def setup(self, id, name, priority, type):
357 357
358 358 self.id = str(id)
359 359 self.name = name
360 360 self.type = type
361 361 self.priority = priority
362 362
363 363 self.parmConfObjList = []
364 364
365 365 def removeParameters(self):
366 366
367 367 for obj in self.parmConfObjList:
368 368 del obj
369 369
370 370 self.parmConfObjList = []
371 371
372 372 def addParameter(self, name, value, format='str'):
373 373
374 374 id = self.__getNewId()
375 375
376 376 parmConfObj = ParameterConf()
377 377 if not parmConfObj.setup(id, name, value, format):
378 378 return None
379 379
380 380 self.parmConfObjList.append(parmConfObj)
381 381
382 382 return parmConfObj
383 383
384 384 def changeParameter(self, name, value, format='str'):
385 385
386 386 parmConfObj = self.getParameterObj(name)
387 387 parmConfObj.update(name, value, format)
388 388
389 389 return parmConfObj
390 390
391 391 def makeXml(self, procUnitElement):
392 392
393 393 opElement = SubElement(procUnitElement, self.ELEMENTNAME)
394 394 opElement.set('id', str(self.id))
395 395 opElement.set('name', self.name)
396 396 opElement.set('type', self.type)
397 397 opElement.set('priority', str(self.priority))
398 398
399 399 for parmConfObj in self.parmConfObjList:
400 400 parmConfObj.makeXml(opElement)
401 401
402 402 def readXml(self, opElement):
403 403
404 404 self.id = opElement.get('id')
405 405 self.name = opElement.get('name')
406 406 self.type = opElement.get('type')
407 407 self.priority = opElement.get('priority')
408 408
409 409 #Compatible with old signal chain version
410 410 #Use of 'run' method instead 'init'
411 411 if self.type == 'self' and self.name == 'init':
412 412 self.name = 'run'
413 413
414 414 self.parmConfObjList = []
415 415
416 416 parmElementList = opElement.iter(ParameterConf().getElementName())
417 417
418 418 for parmElement in parmElementList:
419 419 parmConfObj = ParameterConf()
420 420 parmConfObj.readXml(parmElement)
421 421
422 422 #Compatible with old signal chain version
423 423 #If an 'plot' OPERATION is found, changes name operation by the value of its type PARAMETER
424 424 if self.type != 'self' and self.name == 'Plot':
425 425 if parmConfObj.format == 'str' and parmConfObj.name == 'type':
426 426 self.name = parmConfObj.value
427 427 continue
428 428
429 429 self.parmConfObjList.append(parmConfObj)
430 430
431 431 def printattr(self):
432 432
433 433 print "%s[%s]: name = %s, type = %s, priority = %s" %(self.ELEMENTNAME,
434 434 self.id,
435 435 self.name,
436 436 self.type,
437 437 self.priority)
438 438
439 439 for parmConfObj in self.parmConfObjList:
440 440 parmConfObj.printattr()
441 441
442 442 def createObject(self, plotter_queue=None):
443 443
444 444
445 445 if self.type == 'self':
446 446 raise ValueError, "This operation type cannot be created"
447 447
448 448 if self.type == 'plotter':
449 449 #Plotter(plotter_name)
450 450 if not plotter_queue:
451 451 raise ValueError, "plotter_queue is not defined. Use:\nmyProject = Project()\nmyProject.setPlotterQueue(plotter_queue)"
452 452
453 453 opObj = Plotter(self.name, plotter_queue)
454 454
455 455 if self.type == 'external' or self.type == 'other':
456 456
457 457 className = eval(self.name)
458 458 kwargs = self.getKwargs()
459 459
460 460 opObj = className(**kwargs)
461 461
462 462 return opObj
463 463
464 464
465 465 class ProcUnitConf():
466 466
467 467 id = None
468 468 name = None
469 469 datatype = None
470 470 inputId = None
471 471 parentId = None
472 472
473 473 opConfObjList = []
474 474
475 475 procUnitObj = None
476 476 opObjList = []
477 477
478 478 ELEMENTNAME = 'ProcUnit'
479 479
480 480 def __init__(self):
481 481
482 482 self.id = None
483 483 self.datatype = None
484 484 self.name = None
485 485 self.inputId = None
486 486
487 487 self.opConfObjList = []
488 488
489 489 self.procUnitObj = None
490 490 self.opObjDict = {}
491 491
492 492 def __getPriority(self):
493 493
494 494 return len(self.opConfObjList)+1
495 495
496 496 def __getNewId(self):
497 497
498 498 return int(self.id)*10 + len(self.opConfObjList) + 1
499 499
500 500 def getElementName(self):
501 501
502 502 return self.ELEMENTNAME
503 503
504 504 def getId(self):
505 505
506 506 return self.id
507 507
508 508 def updateId(self, new_id, parentId=parentId):
509 509
510 510
511 511 new_id = int(parentId)*10 + (int(self.id) % 10)
512 512 new_inputId = int(parentId)*10 + (int(self.inputId) % 10)
513 513
514 514 #If this proc unit has not inputs
515 515 if self.inputId == '0':
516 516 new_inputId = 0
517 517
518 518 n = 1
519 519 for opConfObj in self.opConfObjList:
520 520
521 521 idOp = str(int(new_id)*10 + n)
522 522 opConfObj.updateId(idOp)
523 523
524 524 n += 1
525 525
526 526 self.parentId = str(parentId)
527 527 self.id = str(new_id)
528 528 self.inputId = str(new_inputId)
529 529
530 530
531 531 def getInputId(self):
532 532
533 533 return self.inputId
534 534
535 535 def getOperationObjList(self):
536 536
537 537 return self.opConfObjList
538 538
539 539 def getOperationObj(self, name=None):
540 540
541 541 for opConfObj in self.opConfObjList:
542 542
543 543 if opConfObj.name != name:
544 544 continue
545 545
546 546 return opConfObj
547 547
548 548 return None
549 549
550 550 def getOpObjfromParamValue(self, value=None):
551 551
552 552 for opConfObj in self.opConfObjList:
553 553 if opConfObj.getParameterObjfromValue(parameterValue=value) != value:
554 554 continue
555 555 return opConfObj
556 556 return None
557 557
558 558 def getProcUnitObj(self):
559 559
560 560 return self.procUnitObj
561 561
562 562 def setup(self, id, name, datatype, inputId, parentId=None):
563 563
564 564 #Compatible with old signal chain version
565 565 if datatype==None and name==None:
566 566 raise ValueError, "datatype or name should be defined"
567 567
568 568 if name==None:
569 569 if 'Proc' in datatype:
570 570 name = datatype
571 571 else:
572 572 name = '%sProc' %(datatype)
573 573
574 574 if datatype==None:
575 575 datatype = name.replace('Proc','')
576 576
577 577 self.id = str(id)
578 578 self.name = name
579 579 self.datatype = datatype
580 580 self.inputId = inputId
581 581 self.parentId = parentId
582 582
583 583 self.opConfObjList = []
584 584
585 585 self.addOperation(name='run', optype='self')
586 586
587 587 def removeOperations(self):
588 588
589 589 for obj in self.opConfObjList:
590 590 del obj
591 591
592 592 self.opConfObjList = []
593 593 self.addOperation(name='run')
594 594
595 595 def addParameter(self, **kwargs):
596 596 '''
597 597 Add parameters to "run" operation
598 598 '''
599 599 opObj = self.opConfObjList[0]
600 600
601 601 opObj.addParameter(**kwargs)
602 602
603 603 return opObj
604 604
605 605 def addOperation(self, name, optype='self'):
606 606
607 607 id = self.__getNewId()
608 608 priority = self.__getPriority()
609 609
610 610 opConfObj = OperationConf()
611 611 opConfObj.setup(id, name=name, priority=priority, type=optype)
612 612
613 613 self.opConfObjList.append(opConfObj)
614 614
615 615 return opConfObj
616 616
617 617 def makeXml(self, projectElement):
618 618
619 619 procUnitElement = SubElement(projectElement, self.ELEMENTNAME)
620 620 procUnitElement.set('id', str(self.id))
621 621 procUnitElement.set('name', self.name)
622 622 procUnitElement.set('datatype', self.datatype)
623 623 procUnitElement.set('inputId', str(self.inputId))
624 624
625 625 for opConfObj in self.opConfObjList:
626 626 opConfObj.makeXml(procUnitElement)
627 627
628 628 def readXml(self, upElement):
629 629
630 630 self.id = upElement.get('id')
631 631 self.name = upElement.get('name')
632 632 self.datatype = upElement.get('datatype')
633 633 self.inputId = upElement.get('inputId')
634 634
635 635 if self.ELEMENTNAME == "ReadUnit":
636 636 self.datatype = self.datatype.replace("Reader", "")
637 637
638 638 if self.ELEMENTNAME == "ProcUnit":
639 639 self.datatype = self.datatype.replace("Proc", "")
640 640
641 641 if self.inputId == 'None':
642 642 self.inputId = '0'
643 643
644 644 self.opConfObjList = []
645 645
646 646 opElementList = upElement.iter(OperationConf().getElementName())
647 647
648 648 for opElement in opElementList:
649 649 opConfObj = OperationConf()
650 650 opConfObj.readXml(opElement)
651 651 self.opConfObjList.append(opConfObj)
652 652
653 653 def printattr(self):
654 654
655 655 print "%s[%s]: name = %s, datatype = %s, inputId = %s" %(self.ELEMENTNAME,
656 656 self.id,
657 657 self.name,
658 658 self.datatype,
659 659 self.inputId)
660 660
661 661 for opConfObj in self.opConfObjList:
662 662 opConfObj.printattr()
663 663
664 664
665 665 def getKwargs(self):
666 666
667 667 opObj = self.opConfObjList[0]
668 668 kwargs = opObj.getKwargs()
669 669
670 670 return kwargs
671 671
672 672 def createObjects(self, plotter_queue=None):
673 673
674 674 className = eval(self.name)
675 675 kwargs = self.getKwargs()
676 676 procUnitObj = className(**kwargs)
677 677
678 678 for opConfObj in self.opConfObjList:
679 679
680 680 if opConfObj.type=='self' and self.name=='run':
681 681 continue
682 682 elif opConfObj.type=='self':
683 683 procUnitObj.addOperationKwargs(opConfObj.id, **opConfObj.getKwargs())
684 684 continue
685 685
686 686 opObj = opConfObj.createObject(plotter_queue)
687 687
688 688 self.opObjDict[opConfObj.id] = opObj
689 689
690 690 procUnitObj.addOperation(opObj, opConfObj.id)
691 691
692 692 self.procUnitObj = procUnitObj
693 693
694 694 return procUnitObj
695 695
696 696 def run(self):
697 697
698 698 is_ok = False
699 699
700 700 for opConfObj in self.opConfObjList:
701 701
702 702 kwargs = {}
703 703 for parmConfObj in opConfObj.getParameterObjList():
704 704 if opConfObj.name == 'run' and parmConfObj.name == 'datatype':
705 705 continue
706 706
707 707 kwargs[parmConfObj.name] = parmConfObj.getValue()
708 708
709 709 #ini = time.time()
710 710
711 711 #print "\tRunning the '%s' operation with %s" %(opConfObj.name, opConfObj.id)
712 712 sts = self.procUnitObj.call(opType = opConfObj.type,
713 713 opName = opConfObj.name,
714 opId = opConfObj.id)
714 opId = opConfObj.id,
715 **kwargs)
715 716
716 717 # total_time = time.time() - ini
717 718 #
718 719 # if total_time > 0.002:
719 720 # print "%s::%s took %f seconds" %(self.name, opConfObj.name, total_time)
720 721
721 722 is_ok = is_ok or sts
722 723
723 724 return is_ok
724 725
725 726 def close(self):
726 727
727 728 for opConfObj in self.opConfObjList:
728 729 if opConfObj.type == 'self':
729 730 continue
730 731
731 732 opObj = self.procUnitObj.getOperationObj(opConfObj.id)
732 733 opObj.close()
733 734
734 735 self.procUnitObj.close()
735 736
736 737 return
737 738
738 739 class ReadUnitConf(ProcUnitConf):
739 740
740 741 path = None
741 742 startDate = None
742 743 endDate = None
743 744 startTime = None
744 745 endTime = None
745 746
746 747 ELEMENTNAME = 'ReadUnit'
747 748
748 749 def __init__(self):
749 750
750 751 self.id = None
751 752 self.datatype = None
752 753 self.name = None
753 754 self.inputId = None
754 755
755 756 self.parentId = None
756 757
757 758 self.opConfObjList = []
758 759 self.opObjList = []
759 760
760 761 def getElementName(self):
761 762
762 763 return self.ELEMENTNAME
763 764
764 765 def setup(self, id, name, datatype, path='', startDate="", endDate="", startTime="",
765 766 endTime="", parentId=None, queue=None, server=None, **kwargs):
766 767 #Compatible with old signal chain version
767 768 if datatype==None and name==None:
768 769 raise ValueError, "datatype or name should be defined"
769 770
770 771 if name==None:
771 772 if 'Reader' in datatype:
772 773 name = datatype
773 774 else:
774 775 name = '%sReader' %(datatype)
775 776 if datatype==None:
776 777 datatype = name.replace('Reader','')
777 778
778 779 self.id = id
779 780 self.name = name
780 781 self.datatype = datatype
781 782 if path != '':
782 783 self.path = os.path.abspath(path)
783 784 self.startDate = startDate
784 785 self.endDate = endDate
785 786 self.startTime = startTime
786 787 self.endTime = endTime
787 788
788 789 self.inputId = '0'
789 790 self.parentId = parentId
790 791 self.queue = queue
791 792 self.server = server
792 793 self.addRunOperation(**kwargs)
793 794
794 795 def update(self, datatype, path, startDate, endDate, startTime, endTime, parentId=None, name=None, **kwargs):
795 796
796 797 #Compatible with old signal chain version
797 798 if datatype==None and name==None:
798 799 raise ValueError, "datatype or name should be defined"
799 800
800 801 if name==None:
801 802 if 'Reader' in datatype:
802 803 name = datatype
803 804 else:
804 805 name = '%sReader' %(datatype)
805 806
806 807 if datatype==None:
807 808 datatype = name.replace('Reader','')
808 809
809 810 self.datatype = datatype
810 811 self.name = name
811 812 self.path = path
812 813 self.startDate = startDate
813 814 self.endDate = endDate
814 815 self.startTime = startTime
815 816 self.endTime = endTime
816 817
817 818 self.inputId = '0'
818 819 self.parentId = parentId
819 820
820 821 self.updateRunOperation(**kwargs)
821 822
822 823 def removeOperations(self):
823 824
824 825 for obj in self.opConfObjList:
825 826 del obj
826 827
827 828 self.opConfObjList = []
828 829
829 830 def addRunOperation(self, **kwargs):
830 831
831 832 opObj = self.addOperation(name = 'run', optype = 'self')
832 833
833 834 if self.server is None:
834 835 opObj.addParameter(name='datatype' , value=self.datatype, format='str')
835 836 opObj.addParameter(name='path' , value=self.path, format='str')
836 837 opObj.addParameter(name='startDate' , value=self.startDate, format='date')
837 838 opObj.addParameter(name='endDate' , value=self.endDate, format='date')
838 839 opObj.addParameter(name='startTime' , value=self.startTime, format='time')
839 840 opObj.addParameter(name='endTime' , value=self.endTime, format='time')
840 841 opObj.addParameter(name='queue' , value=self.queue, format='obj')
841 842 for key, value in kwargs.items():
842 843 opObj.addParameter(name=key, value=value, format=type(value).__name__)
843 844 else:
844 845 opObj.addParameter(name='server' , value=self.server, format='str')
845 846
846 847
847 848 return opObj
848 849
849 850 def updateRunOperation(self, **kwargs):
850 851
851 852 opObj = self.getOperationObj(name = 'run')
852 853 opObj.removeParameters()
853 854
854 855 opObj.addParameter(name='datatype' , value=self.datatype, format='str')
855 856 opObj.addParameter(name='path' , value=self.path, format='str')
856 857 opObj.addParameter(name='startDate' , value=self.startDate, format='date')
857 858 opObj.addParameter(name='endDate' , value=self.endDate, format='date')
858 859 opObj.addParameter(name='startTime' , value=self.startTime, format='time')
859 860 opObj.addParameter(name='endTime' , value=self.endTime, format='time')
860 861
861 862 for key, value in kwargs.items():
862 863 opObj.addParameter(name=key, value=value, format=type(value).__name__)
863 864
864 865 return opObj
865 866
866 867 # def makeXml(self, projectElement):
867 868 #
868 869 # procUnitElement = SubElement(projectElement, self.ELEMENTNAME)
869 870 # procUnitElement.set('id', str(self.id))
870 871 # procUnitElement.set('name', self.name)
871 872 # procUnitElement.set('datatype', self.datatype)
872 873 # procUnitElement.set('inputId', str(self.inputId))
873 874 #
874 875 # for opConfObj in self.opConfObjList:
875 876 # opConfObj.makeXml(procUnitElement)
876 877
877 878 def readXml(self, upElement):
878 879
879 880 self.id = upElement.get('id')
880 881 self.name = upElement.get('name')
881 882 self.datatype = upElement.get('datatype')
882 883 self.inputId = upElement.get('inputId')
883 884
884 885 if self.ELEMENTNAME == "ReadUnit":
885 886 self.datatype = self.datatype.replace("Reader", "")
886 887
887 888 if self.inputId == 'None':
888 889 self.inputId = '0'
889 890
890 891 self.opConfObjList = []
891 892
892 893 opElementList = upElement.iter(OperationConf().getElementName())
893 894
894 895 for opElement in opElementList:
895 896 opConfObj = OperationConf()
896 897 opConfObj.readXml(opElement)
897 898 self.opConfObjList.append(opConfObj)
898 899
899 900 if opConfObj.name == 'run':
900 901 self.path = opConfObj.getParameterValue('path')
901 902 self.startDate = opConfObj.getParameterValue('startDate')
902 903 self.endDate = opConfObj.getParameterValue('endDate')
903 904 self.startTime = opConfObj.getParameterValue('startTime')
904 905 self.endTime = opConfObj.getParameterValue('endTime')
905 906
906 907 class Project():
907 908
908 909 id = None
909 910 name = None
910 911 description = None
911 912 filename = None
912 913
913 914 procUnitConfObjDict = None
914 915
915 916 ELEMENTNAME = 'Project'
916 917
917 918 plotterQueue = None
918 919
919 920 def __init__(self, plotter_queue=None):
920 921
921 922 self.id = None
922 923 self.name = None
923 924 self.description = None
924 925
925 926 self.plotterQueue = plotter_queue
926 927
927 928 self.procUnitConfObjDict = {}
928 929
929 930 def __getNewId(self):
930 931
931 932 idList = self.procUnitConfObjDict.keys()
932 933
933 934 id = int(self.id)*10
934 935
935 936 while True:
936 937 id += 1
937 938
938 939 if str(id) in idList:
939 940 continue
940 941
941 942 break
942 943
943 944 return str(id)
944 945
945 946 def getElementName(self):
946 947
947 948 return self.ELEMENTNAME
948 949
949 950 def getId(self):
950 951
951 952 return self.id
952 953
953 954 def updateId(self, new_id):
954 955
955 956 self.id = str(new_id)
956 957
957 958 keyList = self.procUnitConfObjDict.keys()
958 959 keyList.sort()
959 960
960 961 n = 1
961 962 newProcUnitConfObjDict = {}
962 963
963 964 for procKey in keyList:
964 965
965 966 procUnitConfObj = self.procUnitConfObjDict[procKey]
966 967 idProcUnit = str(int(self.id)*10 + n)
967 968 procUnitConfObj.updateId(idProcUnit, parentId = self.id)
968 969
969 970 newProcUnitConfObjDict[idProcUnit] = procUnitConfObj
970 971 n += 1
971 972
972 973 self.procUnitConfObjDict = newProcUnitConfObjDict
973 974
974 975 def setup(self, id, name, description):
975 976
976 977 self.id = str(id)
977 978 self.name = name
978 979 self.description = description
979 980
980 981 def update(self, name, description):
981 982
982 983 self.name = name
983 984 self.description = description
984 985
985 986 def addReadUnit(self, id=None, datatype=None, name=None, **kwargs):
986 987 if id is None:
987 988 idReadUnit = self.__getNewId()
988 989 else:
989 990 idReadUnit = str(id)
990 991
991 992 readUnitConfObj = ReadUnitConf()
992 993 readUnitConfObj.setup(idReadUnit, name, datatype, parentId=self.id, **kwargs)
993 994
994 995 self.procUnitConfObjDict[readUnitConfObj.getId()] = readUnitConfObj
995 996
996 997 return readUnitConfObj
997 998
998 999 def addProcUnit(self, inputId='0', datatype=None, name=None):
999 1000
1000 1001 idProcUnit = self.__getNewId()
1001 1002
1002 1003 procUnitConfObj = ProcUnitConf()
1003 1004 procUnitConfObj.setup(idProcUnit, name, datatype, inputId, parentId=self.id)
1004 1005
1005 1006 self.procUnitConfObjDict[procUnitConfObj.getId()] = procUnitConfObj
1006 1007
1007 1008 return procUnitConfObj
1008 1009
1009 1010 def removeProcUnit(self, id):
1010 1011
1011 1012 if id in self.procUnitConfObjDict.keys():
1012 1013 self.procUnitConfObjDict.pop(id)
1013 1014
1014 1015 def getReadUnitId(self):
1015 1016
1016 1017 readUnitConfObj = self.getReadUnitObj()
1017 1018
1018 1019 return readUnitConfObj.id
1019 1020
1020 1021 def getReadUnitObj(self):
1021 1022
1022 1023 for obj in self.procUnitConfObjDict.values():
1023 1024 if obj.getElementName() == "ReadUnit":
1024 1025 return obj
1025 1026
1026 1027 return None
1027 1028
1028 1029 def getProcUnitObj(self, id=None, name=None):
1029 1030
1030 1031 if id != None:
1031 1032 return self.procUnitConfObjDict[id]
1032 1033
1033 1034 if name != None:
1034 1035 return self.getProcUnitObjByName(name)
1035 1036
1036 1037 return None
1037 1038
1038 1039 def getProcUnitObjByName(self, name):
1039 1040
1040 1041 for obj in self.procUnitConfObjDict.values():
1041 1042 if obj.name == name:
1042 1043 return obj
1043 1044
1044 1045 return None
1045 1046
1046 1047 def procUnitItems(self):
1047 1048
1048 1049 return self.procUnitConfObjDict.items()
1049 1050
1050 1051 def makeXml(self):
1051 1052
1052 1053 projectElement = Element('Project')
1053 1054 projectElement.set('id', str(self.id))
1054 1055 projectElement.set('name', self.name)
1055 1056 projectElement.set('description', self.description)
1056 1057
1057 1058 for procUnitConfObj in self.procUnitConfObjDict.values():
1058 1059 procUnitConfObj.makeXml(projectElement)
1059 1060
1060 1061 self.projectElement = projectElement
1061 1062
1062 1063 def writeXml(self, filename=None):
1063 1064
1064 1065 if filename == None:
1065 1066 if self.filename:
1066 1067 filename = self.filename
1067 1068 else:
1068 1069 filename = "schain.xml"
1069 1070
1070 1071 if not filename:
1071 1072 print "filename has not been defined. Use setFilename(filename) for do it."
1072 1073 return 0
1073 1074
1074 1075 abs_file = os.path.abspath(filename)
1075 1076
1076 1077 if not os.access(os.path.dirname(abs_file), os.W_OK):
1077 1078 print "No write permission on %s" %os.path.dirname(abs_file)
1078 1079 return 0
1079 1080
1080 1081 if os.path.isfile(abs_file) and not(os.access(abs_file, os.W_OK)):
1081 1082 print "File %s already exists and it could not be overwriten" %abs_file
1082 1083 return 0
1083 1084
1084 1085 self.makeXml()
1085 1086
1086 1087 ElementTree(self.projectElement).write(abs_file, method='xml')
1087 1088
1088 1089 self.filename = abs_file
1089 1090
1090 1091 return 1
1091 1092
1092 1093 def readXml(self, filename = None):
1093 1094
1094 1095 if not filename:
1095 1096 print "filename is not defined"
1096 1097 return 0
1097 1098
1098 1099 abs_file = os.path.abspath(filename)
1099 1100
1100 1101 if not os.path.isfile(abs_file):
1101 1102 print "%s file does not exist" %abs_file
1102 1103 return 0
1103 1104
1104 1105 self.projectElement = None
1105 1106 self.procUnitConfObjDict = {}
1106 1107
1107 1108 try:
1108 1109 self.projectElement = ElementTree().parse(abs_file)
1109 1110 except:
1110 1111 print "Error reading %s, verify file format" %filename
1111 1112 return 0
1112 1113
1113 1114 self.project = self.projectElement.tag
1114 1115
1115 1116 self.id = self.projectElement.get('id')
1116 1117 self.name = self.projectElement.get('name')
1117 1118 self.description = self.projectElement.get('description')
1118 1119
1119 1120 readUnitElementList = self.projectElement.iter(ReadUnitConf().getElementName())
1120 1121
1121 1122 for readUnitElement in readUnitElementList:
1122 1123 readUnitConfObj = ReadUnitConf()
1123 1124 readUnitConfObj.readXml(readUnitElement)
1124 1125
1125 1126 if readUnitConfObj.parentId == None:
1126 1127 readUnitConfObj.parentId = self.id
1127 1128
1128 1129 self.procUnitConfObjDict[readUnitConfObj.getId()] = readUnitConfObj
1129 1130
1130 1131 procUnitElementList = self.projectElement.iter(ProcUnitConf().getElementName())
1131 1132
1132 1133 for procUnitElement in procUnitElementList:
1133 1134 procUnitConfObj = ProcUnitConf()
1134 1135 procUnitConfObj.readXml(procUnitElement)
1135 1136
1136 1137 if procUnitConfObj.parentId == None:
1137 1138 procUnitConfObj.parentId = self.id
1138 1139
1139 1140 self.procUnitConfObjDict[procUnitConfObj.getId()] = procUnitConfObj
1140 1141
1141 1142 self.filename = abs_file
1142 1143
1143 1144 return 1
1144 1145
1145 1146 def printattr(self):
1146 1147
1147 1148 print "Project[%s]: name = %s, description = %s" %(self.id,
1148 1149 self.name,
1149 1150 self.description)
1150 1151
1151 1152 for procUnitConfObj in self.procUnitConfObjDict.values():
1152 1153 procUnitConfObj.printattr()
1153 1154
1154 1155 def createObjects(self):
1155 1156
1156 1157 for procUnitConfObj in self.procUnitConfObjDict.values():
1157 1158 procUnitConfObj.createObjects(self.plotterQueue)
1158 1159
1159 1160 def __connect(self, objIN, thisObj):
1160 1161
1161 1162 thisObj.setInput(objIN.getOutputObj())
1162 1163
1163 1164 def connectObjects(self):
1164 1165
1165 1166 for thisPUConfObj in self.procUnitConfObjDict.values():
1166 1167
1167 1168 inputId = thisPUConfObj.getInputId()
1168 1169
1169 1170 if int(inputId) == 0:
1170 1171 continue
1171 1172
1172 1173 #Get input object
1173 1174 puConfINObj = self.procUnitConfObjDict[inputId]
1174 1175 puObjIN = puConfINObj.getProcUnitObj()
1175 1176
1176 1177 #Get current object
1177 1178 thisPUObj = thisPUConfObj.getProcUnitObj()
1178 1179
1179 1180 self.__connect(puObjIN, thisPUObj)
1180 1181
1181 1182 def __handleError(self, procUnitConfObj, send_email=True):
1182 1183
1183 1184 import socket
1184 1185
1185 1186 err = traceback.format_exception(sys.exc_info()[0],
1186 1187 sys.exc_info()[1],
1187 1188 sys.exc_info()[2])
1188 1189
1189 1190 print "***** Error occurred in %s *****" %(procUnitConfObj.name)
1190 1191 print "***** %s" %err[-1]
1191 1192
1192 1193 message = "".join(err)
1193 1194
1194 1195 sys.stderr.write(message)
1195 1196
1196 1197 if not send_email:
1197 1198 return
1198 1199
1199 1200 subject = "SChain v%s: Error running %s\n" %(schainpy.__version__, procUnitConfObj.name)
1200 1201
1201 1202 subtitle = "%s: %s\n" %(procUnitConfObj.getElementName() ,procUnitConfObj.name)
1202 1203 subtitle += "Hostname: %s\n" %socket.gethostbyname(socket.gethostname())
1203 1204 subtitle += "Working directory: %s\n" %os.path.abspath("./")
1204 1205 subtitle += "Configuration file: %s\n" %self.filename
1205 1206 subtitle += "Time: %s\n" %str(datetime.datetime.now())
1206 1207
1207 1208 readUnitConfObj = self.getReadUnitObj()
1208 1209 if readUnitConfObj:
1209 1210 subtitle += "\nInput parameters:\n"
1210 1211 subtitle += "[Data path = %s]\n" %readUnitConfObj.path
1211 1212 subtitle += "[Data type = %s]\n" %readUnitConfObj.datatype
1212 1213 subtitle += "[Start date = %s]\n" %readUnitConfObj.startDate
1213 1214 subtitle += "[End date = %s]\n" %readUnitConfObj.endDate
1214 1215 subtitle += "[Start time = %s]\n" %readUnitConfObj.startTime
1215 1216 subtitle += "[End time = %s]\n" %readUnitConfObj.endTime
1216 1217
1217 1218 adminObj = schainpy.admin.SchainNotify()
1218 1219 adminObj.sendAlert(message=message,
1219 1220 subject=subject,
1220 1221 subtitle=subtitle,
1221 1222 filename=self.filename)
1222 1223
1223 1224 def isPaused(self):
1224 1225 return 0
1225 1226
1226 1227 def isStopped(self):
1227 1228 return 0
1228 1229
1229 1230 def runController(self):
1230 1231 """
1231 1232 returns 0 when this process has been stopped, 1 otherwise
1232 1233 """
1233 1234
1234 1235 if self.isPaused():
1235 1236 print "Process suspended"
1236 1237
1237 1238 while True:
1238 1239 sleep(0.1)
1239 1240
1240 1241 if not self.isPaused():
1241 1242 break
1242 1243
1243 1244 if self.isStopped():
1244 1245 break
1245 1246
1246 1247 print "Process reinitialized"
1247 1248
1248 1249 if self.isStopped():
1249 1250 print "Process stopped"
1250 1251 return 0
1251 1252
1252 1253 return 1
1253 1254
1254 1255 def setFilename(self, filename):
1255 1256
1256 1257 self.filename = filename
1257 1258
1258 1259 def setPlotterQueue(self, plotter_queue):
1259 1260
1260 1261 raise NotImplementedError, "Use schainpy.controller_api.ControllerThread instead Project class"
1261 1262
1262 1263 def getPlotterQueue(self):
1263 1264
1264 1265 raise NotImplementedError, "Use schainpy.controller_api.ControllerThread instead Project class"
1265 1266
1266 1267 def useExternalPlotter(self):
1267 1268
1268 1269 raise NotImplementedError, "Use schainpy.controller_api.ControllerThread instead Project class"
1269 1270
1270 1271
1271 1272 def run(self):
1272 1273
1273 1274 print
1274 1275 print "*"*60
1275 1276 print " Starting SIGNAL CHAIN PROCESSING v%s " %schainpy.__version__
1276 1277 print "*"*60
1277 1278 print
1278 1279
1279 1280 keyList = self.procUnitConfObjDict.keys()
1280 1281 keyList.sort()
1281 1282
1282 1283 while(True):
1283 1284
1284 1285 is_ok = False
1285 1286
1286 1287 for procKey in keyList:
1287 1288 # print "Running the '%s' process with %s" %(procUnitConfObj.name, procUnitConfObj.id)
1288 1289
1289 1290 procUnitConfObj = self.procUnitConfObjDict[procKey]
1290 1291
1291 1292 try:
1292 1293 sts = procUnitConfObj.run()
1293 1294 is_ok = is_ok or sts
1294 1295 except KeyboardInterrupt:
1295 1296 is_ok = False
1296 1297 break
1297 1298 except ValueError, e:
1298 1299 sleep(0.5)
1299 1300 self.__handleError(procUnitConfObj, send_email=True)
1300 1301 is_ok = False
1301 1302 break
1302 1303 except:
1303 1304 sleep(0.5)
1304 1305 self.__handleError(procUnitConfObj)
1305 1306 is_ok = False
1306 1307 break
1307 1308
1308 1309 #If every process unit finished so end process
1309 1310 if not(is_ok):
1310 1311 # print "Every process unit have finished"
1311 1312 break
1312 1313
1313 1314 if not self.runController():
1314 1315 break
1315 1316
1316 1317 #Closing every process
1317 1318 for procKey in keyList:
1318 1319 procUnitConfObj = self.procUnitConfObjDict[procKey]
1319 1320 procUnitConfObj.close()
1320 1321
1321 1322 print "Process finished"
1322 1323
1323 1324 def start(self, filename=None):
1324 1325
1325 1326 self.writeXml(filename)
1326 1327 self.createObjects()
1327 1328 self.connectObjects()
1328 1329 self.run()
Show file before | Show file after | Hide comments
1 NO CONTENT: file was removed
Show file before | Show file after | Hide comments
1 NO CONTENT: file was removed
General Comments 0
You need to be logged in to leave comments. Login now