##// END OF EJS Templates
schain-jc
RSS
Improve logger
jespinoza -
r1134:d5610a11112f
parent child
Show More
Side by Side Unified
Context file:
r1134:d5610a11112f Loading diff...:
Show file before | Show file after | Hide comments
@@ -1,1323 +1,1323
1 1 '''
2 2 Created on September , 2012
3 3 @author:
4 4 '''
5 5
6 6 import sys
7 7 import ast
8 8 import datetime
9 9 import traceback
10 10 import math
11 11 import time
12 12 from multiprocessing import Process, cpu_count
13 13
14 14 from xml.etree.ElementTree import ElementTree, Element, SubElement, tostring
15 15 from xml.dom import minidom
16 16
17 17 import schainpy
18 18 import schainpy.admin
19 19 from schainpy.model import *
20 20 from schainpy.utils import log
21 21
22 22 DTYPES = {
23 23 'Voltage': '.r',
24 24 'Spectra': '.pdata'
25 25 }
26 26
27 27
28 28 def MPProject(project, n=cpu_count()):
29 29 '''
30 30 Project wrapper to run schain in n processes
31 31 '''
32 32
33 33 rconf = project.getReadUnitObj()
34 34 op = rconf.getOperationObj('run')
35 35 dt1 = op.getParameterValue('startDate')
36 36 dt2 = op.getParameterValue('endDate')
37 37 tm1 = op.getParameterValue('startTime')
38 38 tm2 = op.getParameterValue('endTime')
39 39 days = (dt2 - dt1).days
40 40
41 41 for day in range(days + 1):
42 42 skip = 0
43 43 cursor = 0
44 44 processes = []
45 45 dt = dt1 + datetime.timedelta(day)
46 46 dt_str = dt.strftime('%Y/%m/%d')
47 47 reader = JRODataReader()
48 48 paths, files = reader.searchFilesOffLine(path=rconf.path,
49 49 startDate=dt,
50 50 endDate=dt,
51 51 startTime=tm1,
52 52 endTime=tm2,
53 53 ext=DTYPES[rconf.datatype])
54 54 nFiles = len(files)
55 55 if nFiles == 0:
56 56 continue
57 57 skip = int(math.ceil(nFiles / n))
58 58 while nFiles > cursor * skip:
59 59 rconf.update(startDate=dt_str, endDate=dt_str, cursor=cursor,
60 60 skip=skip)
61 61 p = project.clone()
62 62 p.start()
63 63 processes.append(p)
64 64 cursor += 1
65 65
66 66 def beforeExit(exctype, value, trace):
67 67 for process in processes:
68 68 process.terminate()
69 69 process.join()
70 70 print traceback.print_tb(trace)
71 71
72 72 sys.excepthook = beforeExit
73 73
74 74 for process in processes:
75 75 process.join()
76 76 process.terminate()
77 77
78 78 time.sleep(3)
79 79
80 80
81 81 class ParameterConf():
82 82
83 83 id = None
84 84 name = None
85 85 value = None
86 86 format = None
87 87
88 88 __formated_value = None
89 89
90 90 ELEMENTNAME = 'Parameter'
91 91
92 92 def __init__(self):
93 93
94 94 self.format = 'str'
95 95
96 96 def getElementName(self):
97 97
98 98 return self.ELEMENTNAME
99 99
100 100 def getValue(self):
101 101
102 102 value = self.value
103 103 format = self.format
104 104
105 105 if self.__formated_value != None:
106 106
107 107 return self.__formated_value
108 108
109 109 if format == 'obj':
110 110 return value
111 111
112 112 if format == 'str':
113 113 self.__formated_value = str(value)
114 114 return self.__formated_value
115 115
116 116 if value == '':
117 117 raise ValueError, '%s: This parameter value is empty' % self.name
118 118
119 119 if format == 'list':
120 120 strList = value.split(',')
121 121
122 122 self.__formated_value = strList
123 123
124 124 return self.__formated_value
125 125
126 126 if format == 'intlist':
127 127 '''
128 128 Example:
129 129 value = (0,1,2)
130 130 '''
131 131
132 132 new_value = ast.literal_eval(value)
133 133
134 134 if type(new_value) not in (tuple, list):
135 135 new_value = [int(new_value)]
136 136
137 137 self.__formated_value = new_value
138 138
139 139 return self.__formated_value
140 140
141 141 if format == 'floatlist':
142 142 '''
143 143 Example:
144 144 value = (0.5, 1.4, 2.7)
145 145 '''
146 146
147 147 new_value = ast.literal_eval(value)
148 148
149 149 if type(new_value) not in (tuple, list):
150 150 new_value = [float(new_value)]
151 151
152 152 self.__formated_value = new_value
153 153
154 154 return self.__formated_value
155 155
156 156 if format == 'date':
157 157 strList = value.split('/')
158 158 intList = [int(x) for x in strList]
159 159 date = datetime.date(intList[0], intList[1], intList[2])
160 160
161 161 self.__formated_value = date
162 162
163 163 return self.__formated_value
164 164
165 165 if format == 'time':
166 166 strList = value.split(':')
167 167 intList = [int(x) for x in strList]
168 168 time = datetime.time(intList[0], intList[1], intList[2])
169 169
170 170 self.__formated_value = time
171 171
172 172 return self.__formated_value
173 173
174 174 if format == 'pairslist':
175 175 '''
176 176 Example:
177 177 value = (0,1),(1,2)
178 178 '''
179 179
180 180 new_value = ast.literal_eval(value)
181 181
182 182 if type(new_value) not in (tuple, list):
183 183 raise ValueError, '%s has to be a tuple or list of pairs' % value
184 184
185 185 if type(new_value[0]) not in (tuple, list):
186 186 if len(new_value) != 2:
187 187 raise ValueError, '%s has to be a tuple or list of pairs' % value
188 188 new_value = [new_value]
189 189
190 190 for thisPair in new_value:
191 191 if len(thisPair) != 2:
192 192 raise ValueError, '%s has to be a tuple or list of pairs' % value
193 193
194 194 self.__formated_value = new_value
195 195
196 196 return self.__formated_value
197 197
198 198 if format == 'multilist':
199 199 '''
200 200 Example:
201 201 value = (0,1,2),(3,4,5)
202 202 '''
203 203 multiList = ast.literal_eval(value)
204 204
205 205 if type(multiList[0]) == int:
206 206 multiList = ast.literal_eval('(' + value + ')')
207 207
208 208 self.__formated_value = multiList
209 209
210 210 return self.__formated_value
211 211
212 212 if format == 'bool':
213 213 value = int(value)
214 214
215 215 if format == 'int':
216 216 value = float(value)
217 217
218 218 format_func = eval(format)
219 219
220 220 self.__formated_value = format_func(value)
221 221
222 222 return self.__formated_value
223 223
224 224 def updateId(self, new_id):
225 225
226 226 self.id = str(new_id)
227 227
228 228 def setup(self, id, name, value, format='str'):
229 229 self.id = str(id)
230 230 self.name = name
231 231 if format == 'obj':
232 232 self.value = value
233 233 else:
234 234 self.value = str(value)
235 235 self.format = str.lower(format)
236 236
237 237 self.getValue()
238 238
239 239 return 1
240 240
241 241 def update(self, name, value, format='str'):
242 242
243 243 self.name = name
244 244 self.value = str(value)
245 245 self.format = format
246 246
247 247 def makeXml(self, opElement):
248 248 if self.name not in ('queue',):
249 249 parmElement = SubElement(opElement, self.ELEMENTNAME)
250 250 parmElement.set('id', str(self.id))
251 251 parmElement.set('name', self.name)
252 252 parmElement.set('value', self.value)
253 253 parmElement.set('format', self.format)
254 254
255 255 def readXml(self, parmElement):
256 256
257 257 self.id = parmElement.get('id')
258 258 self.name = parmElement.get('name')
259 259 self.value = parmElement.get('value')
260 260 self.format = str.lower(parmElement.get('format'))
261 261
262 262 # Compatible with old signal chain version
263 263 if self.format == 'int' and self.name == 'idfigure':
264 264 self.name = 'id'
265 265
266 266 def printattr(self):
267 267
268 268 print 'Parameter[%s]: name = %s, value = %s, format = %s' % (self.id, self.name, self.value, self.format)
269 269
270 270
271 271 class OperationConf():
272 272
273 273 id = None
274 274 name = None
275 275 priority = None
276 276 type = None
277 277
278 278 parmConfObjList = []
279 279
280 280 ELEMENTNAME = 'Operation'
281 281
282 282 def __init__(self):
283 283
284 284 self.id = '0'
285 285 self.name = None
286 286 self.priority = None
287 287 self.type = 'self'
288 288
289 289 def __getNewId(self):
290 290
291 291 return int(self.id) * 10 + len(self.parmConfObjList) + 1
292 292
293 293 def updateId(self, new_id):
294 294
295 295 self.id = str(new_id)
296 296
297 297 n = 1
298 298 for parmObj in self.parmConfObjList:
299 299
300 300 idParm = str(int(new_id) * 10 + n)
301 301 parmObj.updateId(idParm)
302 302
303 303 n += 1
304 304
305 305 def getElementName(self):
306 306
307 307 return self.ELEMENTNAME
308 308
309 309 def getParameterObjList(self):
310 310
311 311 return self.parmConfObjList
312 312
313 313 def getParameterObj(self, parameterName):
314 314
315 315 for parmConfObj in self.parmConfObjList:
316 316
317 317 if parmConfObj.name != parameterName:
318 318 continue
319 319
320 320 return parmConfObj
321 321
322 322 return None
323 323
324 324 def getParameterObjfromValue(self, parameterValue):
325 325
326 326 for parmConfObj in self.parmConfObjList:
327 327
328 328 if parmConfObj.getValue() != parameterValue:
329 329 continue
330 330
331 331 return parmConfObj.getValue()
332 332
333 333 return None
334 334
335 335 def getParameterValue(self, parameterName):
336 336
337 337 parameterObj = self.getParameterObj(parameterName)
338 338
339 339 # if not parameterObj:
340 340 # return None
341 341
342 342 value = parameterObj.getValue()
343 343
344 344 return value
345 345
346 346 def getKwargs(self):
347 347
348 348 kwargs = {}
349 349
350 350 for parmConfObj in self.parmConfObjList:
351 351 if self.name == 'run' and parmConfObj.name == 'datatype':
352 352 continue
353 353
354 354 kwargs[parmConfObj.name] = parmConfObj.getValue()
355 355
356 356 return kwargs
357 357
358 358 def setup(self, id, name, priority, type):
359 359
360 360 self.id = str(id)
361 361 self.name = name
362 362 self.type = type
363 363 self.priority = priority
364 364
365 365 self.parmConfObjList = []
366 366
367 367 def removeParameters(self):
368 368
369 369 for obj in self.parmConfObjList:
370 370 del obj
371 371
372 372 self.parmConfObjList = []
373 373
374 374 def addParameter(self, name, value, format='str'):
375 375
376 376 if value is None:
377 377 return None
378 378 id = self.__getNewId()
379 379
380 380 parmConfObj = ParameterConf()
381 381 if not parmConfObj.setup(id, name, value, format):
382 382 return None
383 383
384 384 self.parmConfObjList.append(parmConfObj)
385 385
386 386 return parmConfObj
387 387
388 388 def changeParameter(self, name, value, format='str'):
389 389
390 390 parmConfObj = self.getParameterObj(name)
391 391 parmConfObj.update(name, value, format)
392 392
393 393 return parmConfObj
394 394
395 395 def makeXml(self, procUnitElement):
396 396
397 397 opElement = SubElement(procUnitElement, self.ELEMENTNAME)
398 398 opElement.set('id', str(self.id))
399 399 opElement.set('name', self.name)
400 400 opElement.set('type', self.type)
401 401 opElement.set('priority', str(self.priority))
402 402
403 403 for parmConfObj in self.parmConfObjList:
404 404 parmConfObj.makeXml(opElement)
405 405
406 406 def readXml(self, opElement):
407 407
408 408 self.id = opElement.get('id')
409 409 self.name = opElement.get('name')
410 410 self.type = opElement.get('type')
411 411 self.priority = opElement.get('priority')
412 412
413 413 # Compatible with old signal chain version
414 414 # Use of 'run' method instead 'init'
415 415 if self.type == 'self' and self.name == 'init':
416 416 self.name = 'run'
417 417
418 418 self.parmConfObjList = []
419 419
420 420 parmElementList = opElement.iter(ParameterConf().getElementName())
421 421
422 422 for parmElement in parmElementList:
423 423 parmConfObj = ParameterConf()
424 424 parmConfObj.readXml(parmElement)
425 425
426 426 # Compatible with old signal chain version
427 427 # If an 'plot' OPERATION is found, changes name operation by the value of its type PARAMETER
428 428 if self.type != 'self' and self.name == 'Plot':
429 429 if parmConfObj.format == 'str' and parmConfObj.name == 'type':
430 430 self.name = parmConfObj.value
431 431 continue
432 432
433 433 self.parmConfObjList.append(parmConfObj)
434 434
435 435 def printattr(self):
436 436
437 437 print '%s[%s]: name = %s, type = %s, priority = %s' % (self.ELEMENTNAME,
438 438 self.id,
439 439 self.name,
440 440 self.type,
441 441 self.priority)
442 442
443 443 for parmConfObj in self.parmConfObjList:
444 444 parmConfObj.printattr()
445 445
446 446 def createObject(self, plotter_queue=None):
447 447
448 448 if self.type == 'self':
449 449 raise ValueError, 'This operation type cannot be created'
450 450
451 451 if self.type == 'plotter':
452 452 if not plotter_queue:
453 453 raise ValueError, 'plotter_queue is not defined. Use:\nmyProject = Project()\nmyProject.setPlotterQueue(plotter_queue)'
454 454
455 455 opObj = Plotter(self.name, plotter_queue)
456 456
457 457 if self.type == 'external' or self.type == 'other':
458 458
459 459 className = eval(self.name)
460 460 kwargs = self.getKwargs()
461 461
462 462 opObj = className(**kwargs)
463 463
464 464 return opObj
465 465
466 466
467 467 class ProcUnitConf():
468 468
469 469 id = None
470 470 name = None
471 471 datatype = None
472 472 inputId = None
473 473 parentId = None
474 474
475 475 opConfObjList = []
476 476
477 477 procUnitObj = None
478 478 opObjList = []
479 479
480 480 ELEMENTNAME = 'ProcUnit'
481 481
482 482 def __init__(self):
483 483
484 484 self.id = None
485 485 self.datatype = None
486 486 self.name = None
487 487 self.inputId = None
488 488
489 489 self.opConfObjList = []
490 490
491 491 self.procUnitObj = None
492 492 self.opObjDict = {}
493 493
494 494 def __getPriority(self):
495 495
496 496 return len(self.opConfObjList) + 1
497 497
498 498 def __getNewId(self):
499 499
500 500 return int(self.id) * 10 + len(self.opConfObjList) + 1
501 501
502 502 def getElementName(self):
503 503
504 504 return self.ELEMENTNAME
505 505
506 506 def getId(self):
507 507
508 508 return self.id
509 509
510 510 def updateId(self, new_id, parentId=parentId):
511 511
512 512 new_id = int(parentId) * 10 + (int(self.id) % 10)
513 513 new_inputId = int(parentId) * 10 + (int(self.inputId) % 10)
514 514
515 515 # If this proc unit has not inputs
516 516 if self.inputId == '0':
517 517 new_inputId = 0
518 518
519 519 n = 1
520 520 for opConfObj in self.opConfObjList:
521 521
522 522 idOp = str(int(new_id) * 10 + n)
523 523 opConfObj.updateId(idOp)
524 524
525 525 n += 1
526 526
527 527 self.parentId = str(parentId)
528 528 self.id = str(new_id)
529 529 self.inputId = str(new_inputId)
530 530
531 531 def getInputId(self):
532 532
533 533 return self.inputId
534 534
535 535 def getOperationObjList(self):
536 536
537 537 return self.opConfObjList
538 538
539 539 def getOperationObj(self, name=None):
540 540
541 541 for opConfObj in self.opConfObjList:
542 542
543 543 if opConfObj.name != name:
544 544 continue
545 545
546 546 return opConfObj
547 547
548 548 return None
549 549
550 550 def getOpObjfromParamValue(self, value=None):
551 551
552 552 for opConfObj in self.opConfObjList:
553 553 if opConfObj.getParameterObjfromValue(parameterValue=value) != value:
554 554 continue
555 555 return opConfObj
556 556 return None
557 557
558 558 def getProcUnitObj(self):
559 559
560 560 return self.procUnitObj
561 561
562 562 def setup(self, id, name, datatype, inputId, parentId=None):
563 563
564 564 # Compatible with old signal chain version
565 565 if datatype == None and name == None:
566 566 raise ValueError, 'datatype or name should be defined'
567 567
568 568 if name == None:
569 569 if 'Proc' in datatype:
570 570 name = datatype
571 571 else:
572 572 name = '%sProc' % (datatype)
573 573
574 574 if datatype == None:
575 575 datatype = name.replace('Proc', '')
576 576
577 577 self.id = str(id)
578 578 self.name = name
579 579 self.datatype = datatype
580 580 self.inputId = inputId
581 581 self.parentId = parentId
582 582
583 583 self.opConfObjList = []
584 584
585 585 self.addOperation(name='run', optype='self')
586 586
587 587 def removeOperations(self):
588 588
589 589 for obj in self.opConfObjList:
590 590 del obj
591 591
592 592 self.opConfObjList = []
593 593 self.addOperation(name='run')
594 594
595 595 def addParameter(self, **kwargs):
596 596 '''
597 597 Add parameters to 'run' operation
598 598 '''
599 599 opObj = self.opConfObjList[0]
600 600
601 601 opObj.addParameter(**kwargs)
602 602
603 603 return opObj
604 604
605 605 def addOperation(self, name, optype='self'):
606 606
607 607 id = self.__getNewId()
608 608 priority = self.__getPriority()
609 609
610 610 opConfObj = OperationConf()
611 611 opConfObj.setup(id, name=name, priority=priority, type=optype)
612 612
613 613 self.opConfObjList.append(opConfObj)
614 614
615 615 return opConfObj
616 616
617 617 def makeXml(self, projectElement):
618 618
619 619 procUnitElement = SubElement(projectElement, self.ELEMENTNAME)
620 620 procUnitElement.set('id', str(self.id))
621 621 procUnitElement.set('name', self.name)
622 622 procUnitElement.set('datatype', self.datatype)
623 623 procUnitElement.set('inputId', str(self.inputId))
624 624
625 625 for opConfObj in self.opConfObjList:
626 626 opConfObj.makeXml(procUnitElement)
627 627
628 628 def readXml(self, upElement):
629 629
630 630 self.id = upElement.get('id')
631 631 self.name = upElement.get('name')
632 632 self.datatype = upElement.get('datatype')
633 633 self.inputId = upElement.get('inputId')
634 634
635 635 if self.ELEMENTNAME == 'ReadUnit':
636 636 self.datatype = self.datatype.replace('Reader', '')
637 637
638 638 if self.ELEMENTNAME == 'ProcUnit':
639 639 self.datatype = self.datatype.replace('Proc', '')
640 640
641 641 if self.inputId == 'None':
642 642 self.inputId = '0'
643 643
644 644 self.opConfObjList = []
645 645
646 646 opElementList = upElement.iter(OperationConf().getElementName())
647 647
648 648 for opElement in opElementList:
649 649 opConfObj = OperationConf()
650 650 opConfObj.readXml(opElement)
651 651 self.opConfObjList.append(opConfObj)
652 652
653 653 def printattr(self):
654 654
655 655 print '%s[%s]: name = %s, datatype = %s, inputId = %s' % (self.ELEMENTNAME,
656 656 self.id,
657 657 self.name,
658 658 self.datatype,
659 659 self.inputId)
660 660
661 661 for opConfObj in self.opConfObjList:
662 662 opConfObj.printattr()
663 663
664 664 def getKwargs(self):
665 665
666 666 opObj = self.opConfObjList[0]
667 667 kwargs = opObj.getKwargs()
668 668
669 669 return kwargs
670 670
671 671 def createObjects(self, plotter_queue=None):
672 672
673 673 className = eval(self.name)
674 674 kwargs = self.getKwargs()
675 675 procUnitObj = className(**kwargs)
676 676
677 677 for opConfObj in self.opConfObjList:
678 678
679 679 if opConfObj.type == 'self' and self.name == 'run':
680 680 continue
681 681 elif opConfObj.type == 'self':
682 682 procUnitObj.addOperationKwargs(
683 683 opConfObj.id, **opConfObj.getKwargs())
684 684 continue
685 685
686 686 opObj = opConfObj.createObject(plotter_queue)
687 687
688 688 self.opObjDict[opConfObj.id] = opObj
689 689
690 690 procUnitObj.addOperation(opObj, opConfObj.id)
691 691
692 692 self.procUnitObj = procUnitObj
693 693
694 694 return procUnitObj
695 695
696 696 def run(self):
697 697
698 698 is_ok = False
699 699
700 700 for opConfObj in self.opConfObjList:
701 701
702 702 kwargs = {}
703 703 for parmConfObj in opConfObj.getParameterObjList():
704 704 if opConfObj.name == 'run' and parmConfObj.name == 'datatype':
705 705 continue
706 706
707 707 kwargs[parmConfObj.name] = parmConfObj.getValue()
708 708
709 709 sts = self.procUnitObj.call(opType=opConfObj.type,
710 710 opName=opConfObj.name,
711 711 opId=opConfObj.id)
712 712
713 713 is_ok = is_ok or sts
714 714
715 715 return is_ok
716 716
717 717 def close(self):
718 718
719 719 for opConfObj in self.opConfObjList:
720 720 if opConfObj.type == 'self':
721 721 continue
722 722
723 723 opObj = self.procUnitObj.getOperationObj(opConfObj.id)
724 724 opObj.close()
725 725
726 726 self.procUnitObj.close()
727 727
728 728 return
729 729
730 730
731 731 class ReadUnitConf(ProcUnitConf):
732 732
733 733 path = None
734 734 startDate = None
735 735 endDate = None
736 736 startTime = None
737 737 endTime = None
738 738
739 739 ELEMENTNAME = 'ReadUnit'
740 740
741 741 def __init__(self):
742 742
743 743 self.id = None
744 744 self.datatype = None
745 745 self.name = None
746 746 self.inputId = None
747 747
748 748 self.parentId = None
749 749
750 750 self.opConfObjList = []
751 751 self.opObjList = []
752 752
753 753 def getElementName(self):
754 754
755 755 return self.ELEMENTNAME
756 756
757 757 def setup(self, id, name, datatype, path='', startDate='', endDate='',
758 758 startTime='', endTime='', parentId=None, server=None, **kwargs):
759 759
760 760 # Compatible with old signal chain version
761 761 if datatype == None and name == None:
762 762 raise ValueError, 'datatype or name should be defined'
763 763 if name == None:
764 764 if 'Reader' in datatype:
765 765 name = datatype
766 766 datatype = name.replace('Reader','')
767 767 else:
768 768 name = '{}Reader'.format(datatype)
769 769 if datatype == None:
770 770 if 'Reader' in name:
771 771 datatype = name.replace('Reader','')
772 772 else:
773 773 datatype = name
774 774 name = '{}Reader'.format(name)
775 775
776 776 self.id = id
777 777 self.name = name
778 778 self.datatype = datatype
779 779 if path != '':
780 780 self.path = os.path.abspath(path)
781 781 self.startDate = startDate
782 782 self.endDate = endDate
783 783 self.startTime = startTime
784 784 self.endTime = endTime
785 785 self.inputId = '0'
786 786 self.parentId = parentId
787 787 self.server = server
788 788 self.addRunOperation(**kwargs)
789 789
790 790 def update(self, **kwargs):
791 791
792 792 if 'datatype' in kwargs:
793 793 datatype = kwargs.pop('datatype')
794 794 if 'Reader' in datatype:
795 795 self.name = datatype
796 796 else:
797 797 self.name = '%sReader' % (datatype)
798 798 self.datatype = self.name.replace('Reader', '')
799 799
800 800 attrs = ('path', 'startDate', 'endDate',
801 801 'startTime', 'endTime', 'parentId')
802 802
803 803 for attr in attrs:
804 804 if attr in kwargs:
805 805 setattr(self, attr, kwargs.pop(attr))
806 806
807 807 self.inputId = '0'
808 808 self.updateRunOperation(**kwargs)
809 809
810 810 def removeOperations(self):
811 811
812 812 for obj in self.opConfObjList:
813 813 del obj
814 814
815 815 self.opConfObjList = []
816 816
817 817 def addRunOperation(self, **kwargs):
818 818
819 819 opObj = self.addOperation(name='run', optype='self')
820 820
821 821 if self.server is None:
822 822 opObj.addParameter(
823 823 name='datatype', value=self.datatype, format='str')
824 824 opObj.addParameter(name='path', value=self.path, format='str')
825 825 opObj.addParameter(
826 826 name='startDate', value=self.startDate, format='date')
827 827 opObj.addParameter(
828 828 name='endDate', value=self.endDate, format='date')
829 829 opObj.addParameter(
830 830 name='startTime', value=self.startTime, format='time')
831 831 opObj.addParameter(
832 832 name='endTime', value=self.endTime, format='time')
833 833
834 834 for key, value in kwargs.items():
835 835 opObj.addParameter(name=key, value=value,
836 836 format=type(value).__name__)
837 837 else:
838 838 opObj.addParameter(name='server', value=self.server, format='str')
839 839
840 840 return opObj
841 841
842 842 def updateRunOperation(self, **kwargs):
843 843
844 844 opObj = self.getOperationObj(name='run')
845 845 opObj.removeParameters()
846 846
847 847 opObj.addParameter(name='datatype', value=self.datatype, format='str')
848 848 opObj.addParameter(name='path', value=self.path, format='str')
849 849 opObj.addParameter(
850 850 name='startDate', value=self.startDate, format='date')
851 851 opObj.addParameter(name='endDate', value=self.endDate, format='date')
852 852 opObj.addParameter(
853 853 name='startTime', value=self.startTime, format='time')
854 854 opObj.addParameter(name='endTime', value=self.endTime, format='time')
855 855
856 856 for key, value in kwargs.items():
857 857 opObj.addParameter(name=key, value=value,
858 858 format=type(value).__name__)
859 859
860 860 return opObj
861 861
862 862 def readXml(self, upElement):
863 863
864 864 self.id = upElement.get('id')
865 865 self.name = upElement.get('name')
866 866 self.datatype = upElement.get('datatype')
867 867 self.inputId = upElement.get('inputId')
868 868
869 869 if self.ELEMENTNAME == 'ReadUnit':
870 870 self.datatype = self.datatype.replace('Reader', '')
871 871
872 872 if self.inputId == 'None':
873 873 self.inputId = '0'
874 874
875 875 self.opConfObjList = []
876 876
877 877 opElementList = upElement.iter(OperationConf().getElementName())
878 878
879 879 for opElement in opElementList:
880 880 opConfObj = OperationConf()
881 881 opConfObj.readXml(opElement)
882 882 self.opConfObjList.append(opConfObj)
883 883
884 884 if opConfObj.name == 'run':
885 885 self.path = opConfObj.getParameterValue('path')
886 886 self.startDate = opConfObj.getParameterValue('startDate')
887 887 self.endDate = opConfObj.getParameterValue('endDate')
888 888 self.startTime = opConfObj.getParameterValue('startTime')
889 889 self.endTime = opConfObj.getParameterValue('endTime')
890 890
891 891
892 892 class Project(Process):
893 893
894 894 id = None
895 895 # name = None
896 896 description = None
897 897 filename = None
898 898
899 899 procUnitConfObjDict = None
900 900
901 901 ELEMENTNAME = 'Project'
902 902
903 903 plotterQueue = None
904 904
905 905 def __init__(self, plotter_queue=None):
906 906
907 907 Process.__init__(self)
908 908 self.id = None
909 909 # self.name = None
910 910 self.description = None
911 911
912 912 self.plotterQueue = plotter_queue
913 913
914 914 self.procUnitConfObjDict = {}
915 915
916 916 def __getNewId(self):
917 917
918 918 idList = self.procUnitConfObjDict.keys()
919 919
920 920 id = int(self.id) * 10
921 921
922 922 while True:
923 923 id += 1
924 924
925 925 if str(id) in idList:
926 926 continue
927 927
928 928 break
929 929
930 930 return str(id)
931 931
932 932 def getElementName(self):
933 933
934 934 return self.ELEMENTNAME
935 935
936 936 def getId(self):
937 937
938 938 return self.id
939 939
940 940 def updateId(self, new_id):
941 941
942 942 self.id = str(new_id)
943 943
944 944 keyList = self.procUnitConfObjDict.keys()
945 945 keyList.sort()
946 946
947 947 n = 1
948 948 newProcUnitConfObjDict = {}
949 949
950 950 for procKey in keyList:
951 951
952 952 procUnitConfObj = self.procUnitConfObjDict[procKey]
953 953 idProcUnit = str(int(self.id) * 10 + n)
954 954 procUnitConfObj.updateId(idProcUnit, parentId=self.id)
955 955
956 956 newProcUnitConfObjDict[idProcUnit] = procUnitConfObj
957 957 n += 1
958 958
959 959 self.procUnitConfObjDict = newProcUnitConfObjDict
960 960
961 961 def setup(self, id, name='', description=''):
962 962
963 963 print
964 964 print '*' * 60
965 965 print ' Starting SIGNAL CHAIN PROCESSING v%s ' % schainpy.__version__
966 966 print '*' * 60
967 967 print
968 968 self.id = str(id)
969 969 self.description = description
970 970
971 971 def update(self, name, description):
972 972
973 973 self.description = description
974 974
975 975 def clone(self):
976 976
977 977 p = Project()
978 978 p.procUnitConfObjDict = self.procUnitConfObjDict
979 979 return p
980 980
981 981 def addReadUnit(self, id=None, datatype=None, name=None, **kwargs):
982 982
983 983 if id is None:
984 984 idReadUnit = self.__getNewId()
985 985 else:
986 986 idReadUnit = str(id)
987 987
988 988 readUnitConfObj = ReadUnitConf()
989 989 readUnitConfObj.setup(idReadUnit, name, datatype,
990 990 parentId=self.id, **kwargs)
991 991
992 992 self.procUnitConfObjDict[readUnitConfObj.getId()] = readUnitConfObj
993 993
994 994 return readUnitConfObj
995 995
996 996 def addProcUnit(self, inputId='0', datatype=None, name=None):
997 997
998 998 idProcUnit = self.__getNewId()
999 999
1000 1000 procUnitConfObj = ProcUnitConf()
1001 1001 procUnitConfObj.setup(idProcUnit, name, datatype,
1002 1002 inputId, parentId=self.id)
1003 1003
1004 1004 self.procUnitConfObjDict[procUnitConfObj.getId()] = procUnitConfObj
1005 1005
1006 1006 return procUnitConfObj
1007 1007
1008 1008 def removeProcUnit(self, id):
1009 1009
1010 1010 if id in self.procUnitConfObjDict.keys():
1011 1011 self.procUnitConfObjDict.pop(id)
1012 1012
1013 1013 def getReadUnitId(self):
1014 1014
1015 1015 readUnitConfObj = self.getReadUnitObj()
1016 1016
1017 1017 return readUnitConfObj.id
1018 1018
1019 1019 def getReadUnitObj(self):
1020 1020
1021 1021 for obj in self.procUnitConfObjDict.values():
1022 1022 if obj.getElementName() == 'ReadUnit':
1023 1023 return obj
1024 1024
1025 1025 return None
1026 1026
1027 1027 def getProcUnitObj(self, id=None, name=None):
1028 1028
1029 1029 if id != None:
1030 1030 return self.procUnitConfObjDict[id]
1031 1031
1032 1032 if name != None:
1033 1033 return self.getProcUnitObjByName(name)
1034 1034
1035 1035 return None
1036 1036
1037 1037 def getProcUnitObjByName(self, name):
1038 1038
1039 1039 for obj in self.procUnitConfObjDict.values():
1040 1040 if obj.name == name:
1041 1041 return obj
1042 1042
1043 1043 return None
1044 1044
1045 1045 def procUnitItems(self):
1046 1046
1047 1047 return self.procUnitConfObjDict.items()
1048 1048
1049 1049 def makeXml(self):
1050 1050
1051 1051 projectElement = Element('Project')
1052 1052 projectElement.set('id', str(self.id))
1053 1053 projectElement.set('name', self.name)
1054 1054 projectElement.set('description', self.description)
1055 1055
1056 1056 for procUnitConfObj in self.procUnitConfObjDict.values():
1057 1057 procUnitConfObj.makeXml(projectElement)
1058 1058
1059 1059 self.projectElement = projectElement
1060 1060
1061 1061 def writeXml(self, filename=None):
1062 1062
1063 1063 if filename == None:
1064 1064 if self.filename:
1065 1065 filename = self.filename
1066 1066 else:
1067 1067 filename = 'schain.xml'
1068 1068
1069 1069 if not filename:
1070 1070 print 'filename has not been defined. Use setFilename(filename) for do it.'
1071 1071 return 0
1072 1072
1073 1073 abs_file = os.path.abspath(filename)
1074 1074
1075 1075 if not os.access(os.path.dirname(abs_file), os.W_OK):
1076 1076 print 'No write permission on %s' % os.path.dirname(abs_file)
1077 1077 return 0
1078 1078
1079 1079 if os.path.isfile(abs_file) and not(os.access(abs_file, os.W_OK)):
1080 1080 print 'File %s already exists and it could not be overwriten' % abs_file
1081 1081 return 0
1082 1082
1083 1083 self.makeXml()
1084 1084
1085 1085 ElementTree(self.projectElement).write(abs_file, method='xml')
1086 1086
1087 1087 self.filename = abs_file
1088 1088
1089 1089 return 1
1090 1090
1091 1091 def readXml(self, filename=None):
1092 1092
1093 1093 if not filename:
1094 1094 print 'filename is not defined'
1095 1095 return 0
1096 1096
1097 1097 abs_file = os.path.abspath(filename)
1098 1098
1099 1099 if not os.path.isfile(abs_file):
1100 1100 print '%s file does not exist' % abs_file
1101 1101 return 0
1102 1102
1103 1103 self.projectElement = None
1104 1104 self.procUnitConfObjDict = {}
1105 1105
1106 1106 try:
1107 1107 self.projectElement = ElementTree().parse(abs_file)
1108 1108 except:
1109 1109 print 'Error reading %s, verify file format' % filename
1110 1110 return 0
1111 1111
1112 1112 self.project = self.projectElement.tag
1113 1113
1114 1114 self.id = self.projectElement.get('id')
1115 1115 self.name = self.projectElement.get('name')
1116 1116 self.description = self.projectElement.get('description')
1117 1117
1118 1118 readUnitElementList = self.projectElement.iter(
1119 1119 ReadUnitConf().getElementName())
1120 1120
1121 1121 for readUnitElement in readUnitElementList:
1122 1122 readUnitConfObj = ReadUnitConf()
1123 1123 readUnitConfObj.readXml(readUnitElement)
1124 1124
1125 1125 if readUnitConfObj.parentId == None:
1126 1126 readUnitConfObj.parentId = self.id
1127 1127
1128 1128 self.procUnitConfObjDict[readUnitConfObj.getId()] = readUnitConfObj
1129 1129
1130 1130 procUnitElementList = self.projectElement.iter(
1131 1131 ProcUnitConf().getElementName())
1132 1132
1133 1133 for procUnitElement in procUnitElementList:
1134 1134 procUnitConfObj = ProcUnitConf()
1135 1135 procUnitConfObj.readXml(procUnitElement)
1136 1136
1137 1137 if procUnitConfObj.parentId == None:
1138 1138 procUnitConfObj.parentId = self.id
1139 1139
1140 1140 self.procUnitConfObjDict[procUnitConfObj.getId()] = procUnitConfObj
1141 1141
1142 1142 self.filename = abs_file
1143 1143
1144 1144 return 1
1145 1145
1146 1146 def printattr(self):
1147 1147
1148 1148 print 'Project[%s]: name = %s, description = %s' % (self.id,
1149 1149 self.name,
1150 1150 self.description)
1151 1151
1152 1152 for procUnitConfObj in self.procUnitConfObjDict.values():
1153 1153 procUnitConfObj.printattr()
1154 1154
1155 1155 def createObjects(self):
1156 1156
1157 1157 for procUnitConfObj in self.procUnitConfObjDict.values():
1158 1158 procUnitConfObj.createObjects(self.plotterQueue)
1159 1159
1160 1160 def __connect(self, objIN, thisObj):
1161 1161
1162 1162 thisObj.setInput(objIN.getOutputObj())
1163 1163
1164 1164 def connectObjects(self):
1165 1165
1166 1166 for thisPUConfObj in self.procUnitConfObjDict.values():
1167 1167
1168 1168 inputId = thisPUConfObj.getInputId()
1169 1169
1170 1170 if int(inputId) == 0:
1171 1171 continue
1172 1172
1173 1173 # Get input object
1174 1174 puConfINObj = self.procUnitConfObjDict[inputId]
1175 1175 puObjIN = puConfINObj.getProcUnitObj()
1176 1176
1177 1177 # Get current object
1178 1178 thisPUObj = thisPUConfObj.getProcUnitObj()
1179 1179
1180 1180 self.__connect(puObjIN, thisPUObj)
1181 1181
1182 1182 def __handleError(self, procUnitConfObj, send_email=False):
1183 1183
1184 1184 import socket
1185 1185
1186 1186 err = traceback.format_exception(sys.exc_info()[0],
1187 1187 sys.exc_info()[1],
1188 1188 sys.exc_info()[2])
1189 1189
1190 1190 print '***** Error occurred in %s *****' % (procUnitConfObj.name)
1191 1191 print '***** %s' % err[-1]
1192 1192
1193 1193 message = ''.join(err)
1194 1194
1195 1195 sys.stderr.write(message)
1196 1196
1197 1197 if not send_email:
1198 1198 return
1199 1199
1200 1200 subject = 'SChain v%s: Error running %s\n' % (
1201 1201 schainpy.__version__, procUnitConfObj.name)
1202 1202
1203 1203 subtitle = '%s: %s\n' % (
1204 1204 procUnitConfObj.getElementName(), procUnitConfObj.name)
1205 1205 subtitle += 'Hostname: %s\n' % socket.gethostbyname(
1206 1206 socket.gethostname())
1207 1207 subtitle += 'Working directory: %s\n' % os.path.abspath('./')
1208 1208 subtitle += 'Configuration file: %s\n' % self.filename
1209 1209 subtitle += 'Time: %s\n' % str(datetime.datetime.now())
1210 1210
1211 1211 readUnitConfObj = self.getReadUnitObj()
1212 1212 if readUnitConfObj:
1213 1213 subtitle += '\nInput parameters:\n'
1214 1214 subtitle += '[Data path = %s]\n' % readUnitConfObj.path
1215 1215 subtitle += '[Data type = %s]\n' % readUnitConfObj.datatype
1216 1216 subtitle += '[Start date = %s]\n' % readUnitConfObj.startDate
1217 1217 subtitle += '[End date = %s]\n' % readUnitConfObj.endDate
1218 1218 subtitle += '[Start time = %s]\n' % readUnitConfObj.startTime
1219 1219 subtitle += '[End time = %s]\n' % readUnitConfObj.endTime
1220 1220
1221 1221 adminObj = schainpy.admin.SchainNotify()
1222 1222 adminObj.sendAlert(message=message,
1223 1223 subject=subject,
1224 1224 subtitle=subtitle,
1225 1225 filename=self.filename)
1226 1226
1227 1227 def isPaused(self):
1228 1228 return 0
1229 1229
1230 1230 def isStopped(self):
1231 1231 return 0
1232 1232
1233 1233 def runController(self):
1234 1234 '''
1235 1235 returns 0 when this process has been stopped, 1 otherwise
1236 1236 '''
1237 1237
1238 1238 if self.isPaused():
1239 1239 print 'Process suspended'
1240 1240
1241 1241 while True:
1242 1242 time.sleep(0.1)
1243 1243
1244 1244 if not self.isPaused():
1245 1245 break
1246 1246
1247 1247 if self.isStopped():
1248 1248 break
1249 1249
1250 1250 print 'Process reinitialized'
1251 1251
1252 1252 if self.isStopped():
1253 1253 print 'Process stopped'
1254 1254 return 0
1255 1255
1256 1256 return 1
1257 1257
1258 1258 def setFilename(self, filename):
1259 1259
1260 1260 self.filename = filename
1261 1261
1262 1262 def setPlotterQueue(self, plotter_queue):
1263 1263
1264 1264 raise NotImplementedError, 'Use schainpy.controller_api.ControllerThread instead Project class'
1265 1265
1266 1266 def getPlotterQueue(self):
1267 1267
1268 1268 raise NotImplementedError, 'Use schainpy.controller_api.ControllerThread instead Project class'
1269 1269
1270 1270 def useExternalPlotter(self):
1271 1271
1272 1272 raise NotImplementedError, 'Use schainpy.controller_api.ControllerThread instead Project class'
1273 1273
1274 1274 def run(self):
1275 1275
1276 log.success('Starting {}'.format(self.name))
1276 log.success('Starting {}'.format(self.name), tag='')
1277 1277 self.start_time = time.time()
1278 1278 self.createObjects()
1279 1279 self.connectObjects()
1280 1280
1281 1281 keyList = self.procUnitConfObjDict.keys()
1282 1282 keyList.sort()
1283 1283
1284 1284 while(True):
1285 1285
1286 1286 is_ok = False
1287 1287
1288 1288 for procKey in keyList:
1289 1289
1290 1290 procUnitConfObj = self.procUnitConfObjDict[procKey]
1291 1291
1292 1292 try:
1293 1293 sts = procUnitConfObj.run()
1294 1294 is_ok = is_ok or sts
1295 1295 except KeyboardInterrupt:
1296 1296 is_ok = False
1297 1297 break
1298 1298 except ValueError, e:
1299 1299 time.sleep(0.5)
1300 1300 self.__handleError(procUnitConfObj, send_email=True)
1301 1301 is_ok = False
1302 1302 break
1303 1303 except:
1304 1304 time.sleep(0.5)
1305 1305 self.__handleError(procUnitConfObj)
1306 1306 is_ok = False
1307 1307 break
1308 1308
1309 1309 # If every process unit finished so end process
1310 1310 if not(is_ok):
1311 1311 break
1312 1312
1313 1313 if not self.runController():
1314 1314 break
1315 1315
1316 1316 # Closing every process
1317 1317 for procKey in keyList:
1318 1318 procUnitConfObj = self.procUnitConfObjDict[procKey]
1319 1319 procUnitConfObj.close()
1320 1320
1321 1321 log.success('{} finished (time: {}s)'.format(
1322 1322 self.name,
1323 1323 time.time()-self.start_time))
Show file before | Show file after | Hide comments
@@ -1,45 +1,57
1 1 '''
2 2 SCHAINPY - LOG
3 3 Simple helper for log standarization
4 4 Usage:
5 5 from schainpy.utils import log
6 6 log.error('A kitten died beacuse of you')
7 7 log.warning('You are doing it wrong but what the heck, I'll allow it)
8 8 log.succes('YOU ROCK!')
9 9 To create your own logger inside your class do it like this:
10 10 from schainpy.utils import log
11 11 awesomeLogger = log.makelogger("never gonna", bg="red", fg="white")
12 12 awesomeLogger('give you up')
13 13 which will look like this:
14 14 [NEVER GONNA] - give you up
15 15 with color red as background and white as foreground.
16 16 '''
17 17
18 18 import click
19 19
20 20
21 def warning(message, tag='Warning'):
22 click.echo(click.style('[{}] {}'.format(tag, message), fg='yellow'))
21 def warning(message, tag='Warning', nl=True):
22 if tag:
23 click.echo(click.style('[{}] {}'.format(tag, message), fg='yellow'), nl=nl)
24 else:
25 click.echo(click.style('{}'.format(message), fg='yellow'), nl=nl)
23 26 pass
24 27
25 28
26 def error(message, tag='Error'):
27 click.echo(click.style('[{}] {}'.format(tag, message), fg='red'))
29 def error(message, tag='Error', nl=True):
30 if tag:
31 click.echo(click.style('[{}] {}'.format(tag, message), fg='red'), nl=nl)
32 else:
33 click.echo(click.style('{}'.format(message), fg='red'), nl=nl)
28 34 pass
29 35
30 36
31 def success(message, tag='Info'):
32 click.echo(click.style('[{}] {}'.format(tag, message), fg='green'))
37 def success(message, tag='Success', nl=True):
38 if tag:
39 click.echo(click.style('[{}] {}'.format(tag, message), fg='green'), nl=nl)
40 else:
41 click.echo(click.style('{}'.format(message), fg='green'), nl=nl)
33 42 pass
34 43
35 44
36 def log(message, tag='Info'):
37 click.echo('[{}] {}'.format(tag, message))
45 def log(message, tag='Info', nl=True):
46 if tag:
47 click.echo('[{}] {}'.format(tag, message), nl=nl)
48 else:
49 click.echo('{}'.format(message), nl=nl)
38 50 pass
39 51
40 52
41 53 def makelogger(tag, bg='reset', fg='reset'):
42 54 def func(message):
43 55 click.echo(click.style('[{}] {}'.format(
44 56 tag.upper(), message), bg=bg, fg=fg))
45 57 return func
General Comments 0
You need to be logged in to leave comments. Login now