##// END OF EJS Templates
schain
RSS
formatting, bug en controller.py
José Chávez -
r1017:934ca8cba82f
parent child
Show More
Side by Side Unified
Context file:
r1017:934ca8cba82f Loading diff...:
Show file before | Show file after | Hide comments
@@ -1,1333 +1,1328
1 1 '''
2 2 Created on September , 2012
3 3 @author:
4 4 '''
5 5
6 6 import sys
7 7 import ast
8 8 import datetime
9 9 import traceback
10 10 import math
11 11 import time
12 12 from multiprocessing import Process, Queue, cpu_count
13 from profilehooks import profile, coverage
14 13
15 14 import schainpy
16 15 import schainpy.admin
17 16
18 17 from xml.etree.ElementTree import ElementTree, Element, SubElement, tostring
19 18 from xml.dom import minidom
20 19
21 20 from schainpy.model import *
22 21 from time import sleep
23 22
24 23
25 24
26 25 def prettify(elem):
27 26 """Return a pretty-printed XML string for the Element.
28 27 """
29 28 rough_string = tostring(elem, 'utf-8')
30 29 reparsed = minidom.parseString(rough_string)
31 30 return reparsed.toprettyxml(indent=" ")
32 31
33 32 def multiSchain(child, nProcess=cpu_count(), startDate=None, endDate=None, by_day=False):
34 33 skip = 0
35 34 cursor = 0
36 35 nFiles = None
37 36 processes = []
38 37 dt1 = datetime.datetime.strptime(startDate, '%Y/%m/%d')
39 38 dt2 = datetime.datetime.strptime(endDate, '%Y/%m/%d')
40 39 days = (dt2 - dt1).days
41 40
42 41 for day in range(days+1):
43 42 skip = 0
44 43 cursor = 0
45 44 q = Queue()
46 45 processes = []
47 46 dt = (dt1 + datetime.timedelta(day)).strftime('%Y/%m/%d')
48 47 firstProcess = Process(target=child, args=(cursor, skip, q, dt))
49 48 firstProcess.start()
50 49 if by_day:
51 50 continue
52 51 nFiles = q.get()
53 52 if nFiles==0:
54 53 continue
55 54 firstProcess.terminate()
56 55 skip = int(math.ceil(nFiles/nProcess))
57 56 while True:
58 57 processes.append(Process(target=child, args=(cursor, skip, q, dt)))
59 58 processes[cursor].start()
60 59 if nFiles < cursor*skip:
61 60 break
62 61 cursor += 1
63 62
64 63 def beforeExit(exctype, value, trace):
65 64 for process in processes:
66 65 process.terminate()
67 66 process.join()
68 67 print traceback.print_tb(trace)
69 68
70 69 sys.excepthook = beforeExit
71 70
72 71 for process in processes:
73 72 process.join()
74 73 process.terminate()
75 74
76 75 time.sleep(3)
77 76
78 77
79 78 class ParameterConf():
80 79
81 80 id = None
82 81 name = None
83 82 value = None
84 83 format = None
85 84
86 85 __formated_value = None
87 86
88 87 ELEMENTNAME = 'Parameter'
89 88
90 89 def __init__(self):
91 90
92 91 self.format = 'str'
93 92
94 93 def getElementName(self):
95 94
96 95 return self.ELEMENTNAME
97 96
98 97 def getValue(self):
99 98
100 99 value = self.value
101 100 format = self.format
102 101
103 102 if self.__formated_value != None:
104 103
105 104 return self.__formated_value
106 105
107 106 if format == 'obj':
108 107 return value
109 108
110 109 if format == 'str':
111 110 self.__formated_value = str(value)
112 111 return self.__formated_value
113 112
114 113 if value == '':
115 114 raise ValueError, "%s: This parameter value is empty" %self.name
116 115
117 116 if format == 'list':
118 117 strList = value.split(',')
119 118
120 119 self.__formated_value = strList
121 120
122 121 return self.__formated_value
123 122
124 123 if format == 'intlist':
125 124 """
126 125 Example:
127 126 value = (0,1,2)
128 127 """
129 128
130 129 new_value = ast.literal_eval(value)
131 130
132 131 if type(new_value) not in (tuple, list):
133 132 new_value = [int(new_value)]
134 133
135 134 self.__formated_value = new_value
136 135
137 136 return self.__formated_value
138 137
139 138 if format == 'floatlist':
140 139 """
141 140 Example:
142 141 value = (0.5, 1.4, 2.7)
143 142 """
144 143
145 144 new_value = ast.literal_eval(value)
146 145
147 146 if type(new_value) not in (tuple, list):
148 147 new_value = [float(new_value)]
149 148
150 149 self.__formated_value = new_value
151 150
152 151 return self.__formated_value
153 152
154 153 if format == 'date':
155 154 strList = value.split('/')
156 155 intList = [int(x) for x in strList]
157 156 date = datetime.date(intList[0], intList[1], intList[2])
158 157
159 158 self.__formated_value = date
160 159
161 160 return self.__formated_value
162 161
163 162 if format == 'time':
164 163 strList = value.split(':')
165 164 intList = [int(x) for x in strList]
166 165 time = datetime.time(intList[0], intList[1], intList[2])
167 166
168 167 self.__formated_value = time
169 168
170 169 return self.__formated_value
171 170
172 171 if format == 'pairslist':
173 172 """
174 173 Example:
175 174 value = (0,1),(1,2)
176 175 """
177 176
178 177 new_value = ast.literal_eval(value)
179 178
180 179 if type(new_value) not in (tuple, list):
181 180 raise ValueError, "%s has to be a tuple or list of pairs" %value
182 181
183 182 if type(new_value[0]) not in (tuple, list):
184 183 if len(new_value) != 2:
185 184 raise ValueError, "%s has to be a tuple or list of pairs" %value
186 185 new_value = [new_value]
187 186
188 187 for thisPair in new_value:
189 188 if len(thisPair) != 2:
190 189 raise ValueError, "%s has to be a tuple or list of pairs" %value
191 190
192 191 self.__formated_value = new_value
193 192
194 193 return self.__formated_value
195 194
196 195 if format == 'multilist':
197 196 """
198 197 Example:
199 198 value = (0,1,2),(3,4,5)
200 199 """
201 200 multiList = ast.literal_eval(value)
202 201
203 202 if type(multiList[0]) == int:
204 203 multiList = ast.literal_eval("(" + value + ")")
205 204
206 205 self.__formated_value = multiList
207 206
208 207 return self.__formated_value
209 208
210 209 if format == 'bool':
211 210 value = int(value)
212 211
213 212 if format == 'int':
214 213 value = float(value)
215 214
216 215 format_func = eval(format)
217 216
218 217 self.__formated_value = format_func(value)
219 218
220 219 return self.__formated_value
221 220
222 221 def updateId(self, new_id):
223 222
224 223 self.id = str(new_id)
225 224
226 225 def setup(self, id, name, value, format='str'):
227 226 self.id = str(id)
228 227 self.name = name
229 228 if format == 'obj':
230 229 self.value = value
231 230 else:
232 231 self.value = str(value)
233 232 self.format = str.lower(format)
234 233
235 234 self.getValue()
236 235
237 236 return 1
238 237
239 238 def update(self, name, value, format='str'):
240 239
241 240 self.name = name
242 241 self.value = str(value)
243 242 self.format = format
244 243
245 244 def makeXml(self, opElement):
246 245 if self.name not in ('queue',):
247 246 parmElement = SubElement(opElement, self.ELEMENTNAME)
248 247 parmElement.set('id', str(self.id))
249 248 parmElement.set('name', self.name)
250 249 parmElement.set('value', self.value)
251 250 parmElement.set('format', self.format)
252 251
253 252 def readXml(self, parmElement):
254 253
255 254 self.id = parmElement.get('id')
256 255 self.name = parmElement.get('name')
257 256 self.value = parmElement.get('value')
258 257 self.format = str.lower(parmElement.get('format'))
259 258
260 259 #Compatible with old signal chain version
261 260 if self.format == 'int' and self.name == 'idfigure':
262 261 self.name = 'id'
263 262
264 263 def printattr(self):
265 264
266 265 print "Parameter[%s]: name = %s, value = %s, format = %s" %(self.id, self.name, self.value, self.format)
267 266
268 267 class OperationConf():
269 268
270 269 id = None
271 270 name = None
272 271 priority = None
273 272 type = None
274 273
275 274 parmConfObjList = []
276 275
277 276 ELEMENTNAME = 'Operation'
278 277
279 278 def __init__(self):
280 279
281 280 self.id = '0'
282 281 self.name = None
283 282 self.priority = None
284 283 self.type = 'self'
285 284
286 285
287 286 def __getNewId(self):
288 287
289 288 return int(self.id)*10 + len(self.parmConfObjList) + 1
290 289
291 290 def updateId(self, new_id):
292 291
293 292 self.id = str(new_id)
294 293
295 294 n = 1
296 295 for parmObj in self.parmConfObjList:
297 296
298 297 idParm = str(int(new_id)*10 + n)
299 298 parmObj.updateId(idParm)
300 299
301 300 n += 1
302 301
303 302 def getElementName(self):
304 303
305 304 return self.ELEMENTNAME
306 305
307 306 def getParameterObjList(self):
308 307
309 308 return self.parmConfObjList
310 309
311 310 def getParameterObj(self, parameterName):
312 311
313 312 for parmConfObj in self.parmConfObjList:
314 313
315 314 if parmConfObj.name != parameterName:
316 315 continue
317 316
318 317 return parmConfObj
319 318
320 319 return None
321 320
322 321 def getParameterObjfromValue(self, parameterValue):
323 322
324 323 for parmConfObj in self.parmConfObjList:
325 324
326 325 if parmConfObj.getValue() != parameterValue:
327 326 continue
328 327
329 328 return parmConfObj.getValue()
330 329
331 330 return None
332 331
333 332 def getParameterValue(self, parameterName):
334 333
335 334 parameterObj = self.getParameterObj(parameterName)
336 335
337 336 # if not parameterObj:
338 337 # return None
339 338
340 339 value = parameterObj.getValue()
341 340
342 341 return value
343 342
344 343
345 344 def getKwargs(self):
346 345
347 346 kwargs = {}
348 347
349 348 for parmConfObj in self.parmConfObjList:
350 349 if self.name == 'run' and parmConfObj.name == 'datatype':
351 350 continue
352 351
353 352 kwargs[parmConfObj.name] = parmConfObj.getValue()
354 353
355 354 return kwargs
356 355
357 356 def setup(self, id, name, priority, type):
358 357
359 358 self.id = str(id)
360 359 self.name = name
361 360 self.type = type
362 361 self.priority = priority
363 362
364 363 self.parmConfObjList = []
365 364
366 365 def removeParameters(self):
367 366
368 367 for obj in self.parmConfObjList:
369 368 del obj
370 369
371 370 self.parmConfObjList = []
372 371
373 372 def addParameter(self, name, value, format='str'):
374 373
375 374 id = self.__getNewId()
376 375
377 376 parmConfObj = ParameterConf()
378 377 if not parmConfObj.setup(id, name, value, format):
379 378 return None
380 379
381 380 self.parmConfObjList.append(parmConfObj)
382 381
383 382 return parmConfObj
384 383
385 384 def changeParameter(self, name, value, format='str'):
386 385
387 386 parmConfObj = self.getParameterObj(name)
388 387 parmConfObj.update(name, value, format)
389 388
390 389 return parmConfObj
391 390
392 391 def makeXml(self, procUnitElement):
393 392
394 393 opElement = SubElement(procUnitElement, self.ELEMENTNAME)
395 394 opElement.set('id', str(self.id))
396 395 opElement.set('name', self.name)
397 396 opElement.set('type', self.type)
398 397 opElement.set('priority', str(self.priority))
399 398
400 399 for parmConfObj in self.parmConfObjList:
401 400 parmConfObj.makeXml(opElement)
402 401
403 402 def readXml(self, opElement):
404 403
405 404 self.id = opElement.get('id')
406 405 self.name = opElement.get('name')
407 406 self.type = opElement.get('type')
408 407 self.priority = opElement.get('priority')
409 408
410 409 #Compatible with old signal chain version
411 410 #Use of 'run' method instead 'init'
412 411 if self.type == 'self' and self.name == 'init':
413 412 self.name = 'run'
414 413
415 414 self.parmConfObjList = []
416 415
417 416 parmElementList = opElement.iter(ParameterConf().getElementName())
418 417
419 418 for parmElement in parmElementList:
420 419 parmConfObj = ParameterConf()
421 420 parmConfObj.readXml(parmElement)
422 421
423 422 #Compatible with old signal chain version
424 423 #If an 'plot' OPERATION is found, changes name operation by the value of its type PARAMETER
425 424 if self.type != 'self' and self.name == 'Plot':
426 425 if parmConfObj.format == 'str' and parmConfObj.name == 'type':
427 426 self.name = parmConfObj.value
428 427 continue
429 428
430 429 self.parmConfObjList.append(parmConfObj)
431 430
432 431 def printattr(self):
433 432
434 433 print "%s[%s]: name = %s, type = %s, priority = %s" %(self.ELEMENTNAME,
435 434 self.id,
436 435 self.name,
437 436 self.type,
438 437 self.priority)
439 438
440 439 for parmConfObj in self.parmConfObjList:
441 440 parmConfObj.printattr()
442 441
443 442 def createObject(self, plotter_queue=None):
444 443
445 444
446 445 if self.type == 'self':
447 446 raise ValueError, "This operation type cannot be created"
448 447
449 448 if self.type == 'plotter':
450 449 #Plotter(plotter_name)
451 450 if not plotter_queue:
452 451 raise ValueError, "plotter_queue is not defined. Use:\nmyProject = Project()\nmyProject.setPlotterQueue(plotter_queue)"
453 452
454 453 opObj = Plotter(self.name, plotter_queue)
455 454
456 455 if self.type == 'external' or self.type == 'other':
457 456
458 457 className = eval(self.name)
459 458 kwargs = self.getKwargs()
460 459
461 460 opObj = className(**kwargs)
462 461
463 462 return opObj
464 463
465 464
466 465 class ProcUnitConf():
467 466
468 467 id = None
469 468 name = None
470 469 datatype = None
471 470 inputId = None
472 471 parentId = None
473 472
474 473 opConfObjList = []
475 474
476 475 procUnitObj = None
477 476 opObjList = []
478 477
479 478 ELEMENTNAME = 'ProcUnit'
480 479
481 480 def __init__(self):
482 481
483 482 self.id = None
484 483 self.datatype = None
485 484 self.name = None
486 485 self.inputId = None
487 486
488 487 self.opConfObjList = []
489 488
490 489 self.procUnitObj = None
491 490 self.opObjDict = {}
492 491
493 492 def __getPriority(self):
494 493
495 494 return len(self.opConfObjList)+1
496 495
497 496 def __getNewId(self):
498 497
499 498 return int(self.id)*10 + len(self.opConfObjList) + 1
500 499
501 500 def getElementName(self):
502 501
503 502 return self.ELEMENTNAME
504 503
505 504 def getId(self):
506 505
507 506 return self.id
508 507
509 508 def updateId(self, new_id, parentId=parentId):
510 509
511 510
512 511 new_id = int(parentId)*10 + (int(self.id) % 10)
513 512 new_inputId = int(parentId)*10 + (int(self.inputId) % 10)
514 513
515 514 #If this proc unit has not inputs
516 515 if self.inputId == '0':
517 516 new_inputId = 0
518 517
519 518 n = 1
520 519 for opConfObj in self.opConfObjList:
521 520
522 521 idOp = str(int(new_id)*10 + n)
523 522 opConfObj.updateId(idOp)
524 523
525 524 n += 1
526 525
527 526 self.parentId = str(parentId)
528 527 self.id = str(new_id)
529 528 self.inputId = str(new_inputId)
530 529
531 530
532 531 def getInputId(self):
533 532
534 533 return self.inputId
535 534
536 535 def getOperationObjList(self):
537 536
538 537 return self.opConfObjList
539 538
540 539 def getOperationObj(self, name=None):
541 540
542 541 for opConfObj in self.opConfObjList:
543 542
544 543 if opConfObj.name != name:
545 544 continue
546 545
547 546 return opConfObj
548 547
549 548 return None
550 549
551 550 def getOpObjfromParamValue(self, value=None):
552 551
553 552 for opConfObj in self.opConfObjList:
554 553 if opConfObj.getParameterObjfromValue(parameterValue=value) != value:
555 554 continue
556 555 return opConfObj
557 556 return None
558 557
559 558 def getProcUnitObj(self):
560 559
561 560 return self.procUnitObj
562 561
563 562 def setup(self, id, name, datatype, inputId, parentId=None):
564 563
565 564 #Compatible with old signal chain version
566 565 if datatype==None and name==None:
567 566 raise ValueError, "datatype or name should be defined"
568 567
569 568 if name==None:
570 569 if 'Proc' in datatype:
571 570 name = datatype
572 571 else:
573 572 name = '%sProc' %(datatype)
574 573
575 574 if datatype==None:
576 575 datatype = name.replace('Proc','')
577 576
578 577 self.id = str(id)
579 578 self.name = name
580 579 self.datatype = datatype
581 580 self.inputId = inputId
582 581 self.parentId = parentId
583 582
584 583 self.opConfObjList = []
585 584
586 585 self.addOperation(name='run', optype='self')
587 586
588 587 def removeOperations(self):
589 588
590 589 for obj in self.opConfObjList:
591 590 del obj
592 591
593 592 self.opConfObjList = []
594 593 self.addOperation(name='run')
595 594
596 595 def addParameter(self, **kwargs):
597 596 '''
598 597 Add parameters to "run" operation
599 598 '''
600 599 opObj = self.opConfObjList[0]
601 600
602 601 opObj.addParameter(**kwargs)
603 602
604 603 return opObj
605 604
606 605 def addOperation(self, name, optype='self'):
607 606
608 607 id = self.__getNewId()
609 608 priority = self.__getPriority()
610 609
611 610 opConfObj = OperationConf()
612 611 opConfObj.setup(id, name=name, priority=priority, type=optype)
613 612
614 613 self.opConfObjList.append(opConfObj)
615 614
616 615 return opConfObj
617 616
618 617 def makeXml(self, projectElement):
619 618
620 619 procUnitElement = SubElement(projectElement, self.ELEMENTNAME)
621 620 procUnitElement.set('id', str(self.id))
622 621 procUnitElement.set('name', self.name)
623 622 procUnitElement.set('datatype', self.datatype)
624 623 procUnitElement.set('inputId', str(self.inputId))
625 624
626 625 for opConfObj in self.opConfObjList:
627 626 opConfObj.makeXml(procUnitElement)
628 627
629 628 def readXml(self, upElement):
630 629
631 630 self.id = upElement.get('id')
632 631 self.name = upElement.get('name')
633 632 self.datatype = upElement.get('datatype')
634 633 self.inputId = upElement.get('inputId')
635 634
636 635 if self.ELEMENTNAME == "ReadUnit":
637 636 self.datatype = self.datatype.replace("Reader", "")
638 637
639 638 if self.ELEMENTNAME == "ProcUnit":
640 639 self.datatype = self.datatype.replace("Proc", "")
641 640
642 641 if self.inputId == 'None':
643 642 self.inputId = '0'
644 643
645 644 self.opConfObjList = []
646 645
647 646 opElementList = upElement.iter(OperationConf().getElementName())
648 647
649 648 for opElement in opElementList:
650 649 opConfObj = OperationConf()
651 650 opConfObj.readXml(opElement)
652 651 self.opConfObjList.append(opConfObj)
653 652
654 653 def printattr(self):
655 654
656 655 print "%s[%s]: name = %s, datatype = %s, inputId = %s" %(self.ELEMENTNAME,
657 656 self.id,
658 657 self.name,
659 658 self.datatype,
660 659 self.inputId)
661 660
662 661 for opConfObj in self.opConfObjList:
663 662 opConfObj.printattr()
664 663
665 664
666 665 def getKwargs(self):
667 666
668 667 opObj = self.opConfObjList[0]
669 668 kwargs = opObj.getKwargs()
670 669
671 670 return kwargs
672 671
673 672 def createObjects(self, plotter_queue=None):
674 673
675 674 className = eval(self.name)
676 675 kwargs = self.getKwargs()
677 676 procUnitObj = className(**kwargs)
678 677
679 678 for opConfObj in self.opConfObjList:
680 679
681 680 if opConfObj.type=='self' and self.name=='run':
682 681 continue
683 682 elif opConfObj.type=='self':
684 683 procUnitObj.addOperationKwargs(opConfObj.id, **opConfObj.getKwargs())
685 684 continue
686 685
687 686 opObj = opConfObj.createObject(plotter_queue)
688 687
689 688 self.opObjDict[opConfObj.id] = opObj
690 689
691 690 procUnitObj.addOperation(opObj, opConfObj.id)
692 691
693 692 self.procUnitObj = procUnitObj
694 693
695 694 return procUnitObj
696 695
697 ## @profile
698 696 def run(self):
699 697
700 698 is_ok = False
701 699
702 700 for opConfObj in self.opConfObjList:
703 701
704 702 kwargs = {}
705 703 for parmConfObj in opConfObj.getParameterObjList():
706 704 if opConfObj.name == 'run' and parmConfObj.name == 'datatype':
707 705 continue
708 706
709 707 kwargs[parmConfObj.name] = parmConfObj.getValue()
710 708
711 709 #ini = time.time()
712 710
713 711 #print "\tRunning the '%s' operation with %s" %(opConfObj.name, opConfObj.id)
714 712 sts = self.procUnitObj.call(opType = opConfObj.type,
715 713 opName = opConfObj.name,
716 opId = opConfObj.id,
717 **kwargs)
714 opId = opConfObj.id)
718 715
719 716 # total_time = time.time() - ini
720 717 #
721 718 # if total_time > 0.002:
722 719 # print "%s::%s took %f seconds" %(self.name, opConfObj.name, total_time)
723 720
724 721 is_ok = is_ok or sts
725 722
726 723 return is_ok
727 724
728 725 def close(self):
729 726
730 727 for opConfObj in self.opConfObjList:
731 728 if opConfObj.type == 'self':
732 729 continue
733 730
734 731 opObj = self.procUnitObj.getOperationObj(opConfObj.id)
735 732 opObj.close()
736 733
737 734 self.procUnitObj.close()
738 735
739 736 return
740 737
741 738 class ReadUnitConf(ProcUnitConf):
742 739
743 740 path = None
744 741 startDate = None
745 742 endDate = None
746 743 startTime = None
747 744 endTime = None
748 745
749 746 ELEMENTNAME = 'ReadUnit'
750 747
751 748 def __init__(self):
752 749
753 750 self.id = None
754 751 self.datatype = None
755 752 self.name = None
756 753 self.inputId = None
757 754
758 755 self.parentId = None
759 756
760 757 self.opConfObjList = []
761 758 self.opObjList = []
762 759
763 760 def getElementName(self):
764 761
765 762 return self.ELEMENTNAME
766 763
767 764 def setup(self, id, name, datatype, path='', startDate="", endDate="", startTime="",
768 765 endTime="", parentId=None, queue=None, server=None, **kwargs):
769
770 766 #Compatible with old signal chain version
771 767 if datatype==None and name==None:
772 768 raise ValueError, "datatype or name should be defined"
773 769
774 770 if name==None:
775 771 if 'Reader' in datatype:
776 772 name = datatype
777 773 else:
778 774 name = '%sReader' %(datatype)
779 775 if datatype==None:
780 776 datatype = name.replace('Reader','')
781 777
782 778 self.id = id
783 779 self.name = name
784 780 self.datatype = datatype
785 781 if path != '':
786 782 self.path = os.path.abspath(path)
787 783 self.startDate = startDate
788 784 self.endDate = endDate
789 785 self.startTime = startTime
790 786 self.endTime = endTime
791 787
792 788 self.inputId = '0'
793 789 self.parentId = parentId
794 790 self.queue = queue
795 791 self.server = server
796 792 self.addRunOperation(**kwargs)
797 793
798 794 def update(self, datatype, path, startDate, endDate, startTime, endTime, parentId=None, name=None, **kwargs):
799 795
800 796 #Compatible with old signal chain version
801 797 if datatype==None and name==None:
802 798 raise ValueError, "datatype or name should be defined"
803 799
804 800 if name==None:
805 801 if 'Reader' in datatype:
806 802 name = datatype
807 803 else:
808 804 name = '%sReader' %(datatype)
809 805
810 806 if datatype==None:
811 807 datatype = name.replace('Reader','')
812 808
813 809 self.datatype = datatype
814 810 self.name = name
815 811 self.path = path
816 812 self.startDate = startDate
817 813 self.endDate = endDate
818 814 self.startTime = startTime
819 815 self.endTime = endTime
820 816
821 817 self.inputId = '0'
822 818 self.parentId = parentId
823 819
824 820 self.updateRunOperation(**kwargs)
825 821
826 822 def removeOperations(self):
827 823
828 824 for obj in self.opConfObjList:
829 825 del obj
830 826
831 827 self.opConfObjList = []
832 828
833 829 def addRunOperation(self, **kwargs):
834 830
835 831 opObj = self.addOperation(name = 'run', optype = 'self')
836 832
837 833 if self.server is None:
838 834 opObj.addParameter(name='datatype' , value=self.datatype, format='str')
839 835 opObj.addParameter(name='path' , value=self.path, format='str')
840 836 opObj.addParameter(name='startDate' , value=self.startDate, format='date')
841 837 opObj.addParameter(name='endDate' , value=self.endDate, format='date')
842 838 opObj.addParameter(name='startTime' , value=self.startTime, format='time')
843 839 opObj.addParameter(name='endTime' , value=self.endTime, format='time')
844 840 opObj.addParameter(name='queue' , value=self.queue, format='obj')
845 841 for key, value in kwargs.items():
846 842 opObj.addParameter(name=key, value=value, format=type(value).__name__)
847 843 else:
848 844 opObj.addParameter(name='server' , value=self.server, format='str')
849 845
850 846
851 847 return opObj
852 848
853 849 def updateRunOperation(self, **kwargs):
854 850
855 851 opObj = self.getOperationObj(name = 'run')
856 852 opObj.removeParameters()
857 853
858 854 opObj.addParameter(name='datatype' , value=self.datatype, format='str')
859 855 opObj.addParameter(name='path' , value=self.path, format='str')
860 856 opObj.addParameter(name='startDate' , value=self.startDate, format='date')
861 857 opObj.addParameter(name='endDate' , value=self.endDate, format='date')
862 858 opObj.addParameter(name='startTime' , value=self.startTime, format='time')
863 859 opObj.addParameter(name='endTime' , value=self.endTime, format='time')
864 860
865 861 for key, value in kwargs.items():
866 862 opObj.addParameter(name=key, value=value, format=type(value).__name__)
867 863
868 864 return opObj
869 865
870 866 # def makeXml(self, projectElement):
871 867 #
872 868 # procUnitElement = SubElement(projectElement, self.ELEMENTNAME)
873 869 # procUnitElement.set('id', str(self.id))
874 870 # procUnitElement.set('name', self.name)
875 871 # procUnitElement.set('datatype', self.datatype)
876 872 # procUnitElement.set('inputId', str(self.inputId))
877 873 #
878 874 # for opConfObj in self.opConfObjList:
879 875 # opConfObj.makeXml(procUnitElement)
880 876
881 877 def readXml(self, upElement):
882 878
883 879 self.id = upElement.get('id')
884 880 self.name = upElement.get('name')
885 881 self.datatype = upElement.get('datatype')
886 882 self.inputId = upElement.get('inputId')
887 883
888 884 if self.ELEMENTNAME == "ReadUnit":
889 885 self.datatype = self.datatype.replace("Reader", "")
890 886
891 887 if self.inputId == 'None':
892 888 self.inputId = '0'
893 889
894 890 self.opConfObjList = []
895 891
896 892 opElementList = upElement.iter(OperationConf().getElementName())
897 893
898 894 for opElement in opElementList:
899 895 opConfObj = OperationConf()
900 896 opConfObj.readXml(opElement)
901 897 self.opConfObjList.append(opConfObj)
902 898
903 899 if opConfObj.name == 'run':
904 900 self.path = opConfObj.getParameterValue('path')
905 901 self.startDate = opConfObj.getParameterValue('startDate')
906 902 self.endDate = opConfObj.getParameterValue('endDate')
907 903 self.startTime = opConfObj.getParameterValue('startTime')
908 904 self.endTime = opConfObj.getParameterValue('endTime')
909 905
910 906 class Project():
911 907
912 908 id = None
913 909 name = None
914 910 description = None
915 911 filename = None
916 912
917 913 procUnitConfObjDict = None
918 914
919 915 ELEMENTNAME = 'Project'
920 916
921 917 plotterQueue = None
922 918
923 919 def __init__(self, plotter_queue=None):
924 920
925 921 self.id = None
926 922 self.name = None
927 923 self.description = None
928 924
929 925 self.plotterQueue = plotter_queue
930 926
931 927 self.procUnitConfObjDict = {}
932 928
933 929 def __getNewId(self):
934 930
935 931 idList = self.procUnitConfObjDict.keys()
936 932
937 933 id = int(self.id)*10
938 934
939 935 while True:
940 936 id += 1
941 937
942 938 if str(id) in idList:
943 939 continue
944 940
945 941 break
946 942
947 943 return str(id)
948 944
949 945 def getElementName(self):
950 946
951 947 return self.ELEMENTNAME
952 948
953 949 def getId(self):
954 950
955 951 return self.id
956 952
957 953 def updateId(self, new_id):
958 954
959 955 self.id = str(new_id)
960 956
961 957 keyList = self.procUnitConfObjDict.keys()
962 958 keyList.sort()
963 959
964 960 n = 1
965 961 newProcUnitConfObjDict = {}
966 962
967 963 for procKey in keyList:
968 964
969 965 procUnitConfObj = self.procUnitConfObjDict[procKey]
970 966 idProcUnit = str(int(self.id)*10 + n)
971 967 procUnitConfObj.updateId(idProcUnit, parentId = self.id)
972 968
973 969 newProcUnitConfObjDict[idProcUnit] = procUnitConfObj
974 970 n += 1
975 971
976 972 self.procUnitConfObjDict = newProcUnitConfObjDict
977 973
978 974 def setup(self, id, name, description):
979 975
980 976 self.id = str(id)
981 977 self.name = name
982 978 self.description = description
983 979
984 980 def update(self, name, description):
985 981
986 982 self.name = name
987 983 self.description = description
988 984
989 985 def addReadUnit(self, id=None, datatype=None, name=None, **kwargs):
990
991 986 if id is None:
992 987 idReadUnit = self.__getNewId()
993 988 else:
994 989 idReadUnit = str(id)
995 990
996 991 readUnitConfObj = ReadUnitConf()
997 992 readUnitConfObj.setup(idReadUnit, name, datatype, parentId=self.id, **kwargs)
998 993
999 994 self.procUnitConfObjDict[readUnitConfObj.getId()] = readUnitConfObj
1000 995
1001 996 return readUnitConfObj
1002 997
1003 998 def addProcUnit(self, inputId='0', datatype=None, name=None):
1004 999
1005 1000 idProcUnit = self.__getNewId()
1006 1001
1007 1002 procUnitConfObj = ProcUnitConf()
1008 1003 procUnitConfObj.setup(idProcUnit, name, datatype, inputId, parentId=self.id)
1009 1004
1010 1005 self.procUnitConfObjDict[procUnitConfObj.getId()] = procUnitConfObj
1011 1006
1012 1007 return procUnitConfObj
1013 1008
1014 1009 def removeProcUnit(self, id):
1015 1010
1016 1011 if id in self.procUnitConfObjDict.keys():
1017 1012 self.procUnitConfObjDict.pop(id)
1018 1013
1019 1014 def getReadUnitId(self):
1020 1015
1021 1016 readUnitConfObj = self.getReadUnitObj()
1022 1017
1023 1018 return readUnitConfObj.id
1024 1019
1025 1020 def getReadUnitObj(self):
1026 1021
1027 1022 for obj in self.procUnitConfObjDict.values():
1028 1023 if obj.getElementName() == "ReadUnit":
1029 1024 return obj
1030 1025
1031 1026 return None
1032 1027
1033 1028 def getProcUnitObj(self, id=None, name=None):
1034 1029
1035 1030 if id != None:
1036 1031 return self.procUnitConfObjDict[id]
1037 1032
1038 1033 if name != None:
1039 1034 return self.getProcUnitObjByName(name)
1040 1035
1041 1036 return None
1042 1037
1043 1038 def getProcUnitObjByName(self, name):
1044 1039
1045 1040 for obj in self.procUnitConfObjDict.values():
1046 1041 if obj.name == name:
1047 1042 return obj
1048 1043
1049 1044 return None
1050 1045
1051 1046 def procUnitItems(self):
1052 1047
1053 1048 return self.procUnitConfObjDict.items()
1054 1049
1055 1050 def makeXml(self):
1056 1051
1057 1052 projectElement = Element('Project')
1058 1053 projectElement.set('id', str(self.id))
1059 1054 projectElement.set('name', self.name)
1060 1055 projectElement.set('description', self.description)
1061 1056
1062 1057 for procUnitConfObj in self.procUnitConfObjDict.values():
1063 1058 procUnitConfObj.makeXml(projectElement)
1064 1059
1065 1060 self.projectElement = projectElement
1066 1061
1067 1062 def writeXml(self, filename=None):
1068 1063
1069 1064 if filename == None:
1070 1065 if self.filename:
1071 1066 filename = self.filename
1072 1067 else:
1073 1068 filename = "schain.xml"
1074 1069
1075 1070 if not filename:
1076 1071 print "filename has not been defined. Use setFilename(filename) for do it."
1077 1072 return 0
1078 1073
1079 1074 abs_file = os.path.abspath(filename)
1080 1075
1081 1076 if not os.access(os.path.dirname(abs_file), os.W_OK):
1082 1077 print "No write permission on %s" %os.path.dirname(abs_file)
1083 1078 return 0
1084 1079
1085 1080 if os.path.isfile(abs_file) and not(os.access(abs_file, os.W_OK)):
1086 1081 print "File %s already exists and it could not be overwriten" %abs_file
1087 1082 return 0
1088 1083
1089 1084 self.makeXml()
1090 1085
1091 1086 ElementTree(self.projectElement).write(abs_file, method='xml')
1092 1087
1093 1088 self.filename = abs_file
1094 1089
1095 1090 return 1
1096 1091
1097 1092 def readXml(self, filename = None):
1098 1093
1099 1094 if not filename:
1100 1095 print "filename is not defined"
1101 1096 return 0
1102 1097
1103 1098 abs_file = os.path.abspath(filename)
1104 1099
1105 1100 if not os.path.isfile(abs_file):
1106 1101 print "%s file does not exist" %abs_file
1107 1102 return 0
1108 1103
1109 1104 self.projectElement = None
1110 1105 self.procUnitConfObjDict = {}
1111 1106
1112 1107 try:
1113 1108 self.projectElement = ElementTree().parse(abs_file)
1114 1109 except:
1115 1110 print "Error reading %s, verify file format" %filename
1116 1111 return 0
1117 1112
1118 1113 self.project = self.projectElement.tag
1119 1114
1120 1115 self.id = self.projectElement.get('id')
1121 1116 self.name = self.projectElement.get('name')
1122 1117 self.description = self.projectElement.get('description')
1123 1118
1124 1119 readUnitElementList = self.projectElement.iter(ReadUnitConf().getElementName())
1125 1120
1126 1121 for readUnitElement in readUnitElementList:
1127 1122 readUnitConfObj = ReadUnitConf()
1128 1123 readUnitConfObj.readXml(readUnitElement)
1129 1124
1130 1125 if readUnitConfObj.parentId == None:
1131 1126 readUnitConfObj.parentId = self.id
1132 1127
1133 1128 self.procUnitConfObjDict[readUnitConfObj.getId()] = readUnitConfObj
1134 1129
1135 1130 procUnitElementList = self.projectElement.iter(ProcUnitConf().getElementName())
1136 1131
1137 1132 for procUnitElement in procUnitElementList:
1138 1133 procUnitConfObj = ProcUnitConf()
1139 1134 procUnitConfObj.readXml(procUnitElement)
1140 1135
1141 1136 if procUnitConfObj.parentId == None:
1142 1137 procUnitConfObj.parentId = self.id
1143 1138
1144 1139 self.procUnitConfObjDict[procUnitConfObj.getId()] = procUnitConfObj
1145 1140
1146 1141 self.filename = abs_file
1147 1142
1148 1143 return 1
1149 1144
1150 1145 def printattr(self):
1151 1146
1152 1147 print "Project[%s]: name = %s, description = %s" %(self.id,
1153 1148 self.name,
1154 1149 self.description)
1155 1150
1156 1151 for procUnitConfObj in self.procUnitConfObjDict.values():
1157 1152 procUnitConfObj.printattr()
1158 1153
1159 1154 def createObjects(self):
1160 1155
1161 1156 for procUnitConfObj in self.procUnitConfObjDict.values():
1162 1157 procUnitConfObj.createObjects(self.plotterQueue)
1163 1158
1164 1159 def __connect(self, objIN, thisObj):
1165 1160
1166 1161 thisObj.setInput(objIN.getOutputObj())
1167 1162
1168 1163 def connectObjects(self):
1169 1164
1170 1165 for thisPUConfObj in self.procUnitConfObjDict.values():
1171 1166
1172 1167 inputId = thisPUConfObj.getInputId()
1173 1168
1174 1169 if int(inputId) == 0:
1175 1170 continue
1176 1171
1177 1172 #Get input object
1178 1173 puConfINObj = self.procUnitConfObjDict[inputId]
1179 1174 puObjIN = puConfINObj.getProcUnitObj()
1180 1175
1181 1176 #Get current object
1182 1177 thisPUObj = thisPUConfObj.getProcUnitObj()
1183 1178
1184 1179 self.__connect(puObjIN, thisPUObj)
1185 1180
1186 1181 def __handleError(self, procUnitConfObj, send_email=True):
1187 1182
1188 1183 import socket
1189 1184
1190 1185 err = traceback.format_exception(sys.exc_info()[0],
1191 1186 sys.exc_info()[1],
1192 1187 sys.exc_info()[2])
1193 1188
1194 1189 print "***** Error occurred in %s *****" %(procUnitConfObj.name)
1195 1190 print "***** %s" %err[-1]
1196 1191
1197 1192 message = "".join(err)
1198 1193
1199 1194 sys.stderr.write(message)
1200 1195
1201 1196 if not send_email:
1202 1197 return
1203 1198
1204 1199 subject = "SChain v%s: Error running %s\n" %(schainpy.__version__, procUnitConfObj.name)
1205 1200
1206 1201 subtitle = "%s: %s\n" %(procUnitConfObj.getElementName() ,procUnitConfObj.name)
1207 1202 subtitle += "Hostname: %s\n" %socket.gethostbyname(socket.gethostname())
1208 1203 subtitle += "Working directory: %s\n" %os.path.abspath("./")
1209 1204 subtitle += "Configuration file: %s\n" %self.filename
1210 1205 subtitle += "Time: %s\n" %str(datetime.datetime.now())
1211 1206
1212 1207 readUnitConfObj = self.getReadUnitObj()
1213 1208 if readUnitConfObj:
1214 1209 subtitle += "\nInput parameters:\n"
1215 1210 subtitle += "[Data path = %s]\n" %readUnitConfObj.path
1216 1211 subtitle += "[Data type = %s]\n" %readUnitConfObj.datatype
1217 1212 subtitle += "[Start date = %s]\n" %readUnitConfObj.startDate
1218 1213 subtitle += "[End date = %s]\n" %readUnitConfObj.endDate
1219 1214 subtitle += "[Start time = %s]\n" %readUnitConfObj.startTime
1220 1215 subtitle += "[End time = %s]\n" %readUnitConfObj.endTime
1221 1216
1222 1217 adminObj = schainpy.admin.SchainNotify()
1223 1218 adminObj.sendAlert(message=message,
1224 1219 subject=subject,
1225 1220 subtitle=subtitle,
1226 1221 filename=self.filename)
1227 1222
1228 1223 def isPaused(self):
1229 1224 return 0
1230 1225
1231 1226 def isStopped(self):
1232 1227 return 0
1233 1228
1234 1229 def runController(self):
1235 1230 """
1236 1231 returns 0 when this process has been stopped, 1 otherwise
1237 1232 """
1238 1233
1239 1234 if self.isPaused():
1240 1235 print "Process suspended"
1241 1236
1242 1237 while True:
1243 1238 sleep(0.1)
1244 1239
1245 1240 if not self.isPaused():
1246 1241 break
1247 1242
1248 1243 if self.isStopped():
1249 1244 break
1250 1245
1251 1246 print "Process reinitialized"
1252 1247
1253 1248 if self.isStopped():
1254 1249 print "Process stopped"
1255 1250 return 0
1256 1251
1257 1252 return 1
1258 1253
1259 1254 def setFilename(self, filename):
1260 1255
1261 1256 self.filename = filename
1262 1257
1263 1258 def setPlotterQueue(self, plotter_queue):
1264 1259
1265 1260 raise NotImplementedError, "Use schainpy.controller_api.ControllerThread instead Project class"
1266 1261
1267 1262 def getPlotterQueue(self):
1268 1263
1269 1264 raise NotImplementedError, "Use schainpy.controller_api.ControllerThread instead Project class"
1270 1265
1271 1266 def useExternalPlotter(self):
1272 1267
1273 1268 raise NotImplementedError, "Use schainpy.controller_api.ControllerThread instead Project class"
1274 1269
1275 1270
1276 1271 def run(self):
1277 1272
1278 1273 print
1279 1274 print "*"*60
1280 1275 print " Starting SIGNAL CHAIN PROCESSING v%s " %schainpy.__version__
1281 1276 print "*"*60
1282 1277 print
1283 1278
1284 1279 keyList = self.procUnitConfObjDict.keys()
1285 1280 keyList.sort()
1286 1281
1287 1282 while(True):
1288 1283
1289 1284 is_ok = False
1290 1285
1291 1286 for procKey in keyList:
1292 1287 # print "Running the '%s' process with %s" %(procUnitConfObj.name, procUnitConfObj.id)
1293 1288
1294 1289 procUnitConfObj = self.procUnitConfObjDict[procKey]
1295 1290
1296 1291 try:
1297 1292 sts = procUnitConfObj.run()
1298 1293 is_ok = is_ok or sts
1299 1294 except KeyboardInterrupt:
1300 1295 is_ok = False
1301 1296 break
1302 1297 except ValueError, e:
1303 1298 sleep(0.5)
1304 1299 self.__handleError(procUnitConfObj, send_email=True)
1305 1300 is_ok = False
1306 1301 break
1307 1302 except:
1308 1303 sleep(0.5)
1309 1304 self.__handleError(procUnitConfObj)
1310 1305 is_ok = False
1311 1306 break
1312 1307
1313 1308 #If every process unit finished so end process
1314 1309 if not(is_ok):
1315 1310 # print "Every process unit have finished"
1316 1311 break
1317 1312
1318 1313 if not self.runController():
1319 1314 break
1320 1315
1321 1316 #Closing every process
1322 1317 for procKey in keyList:
1323 1318 procUnitConfObj = self.procUnitConfObjDict[procKey]
1324 1319 procUnitConfObj.close()
1325 1320
1326 1321 print "Process finished"
1327 1322
1328 1323 def start(self, filename=None):
1329 1324
1330 1325 self.writeXml(filename)
1331 1326 self.createObjects()
1332 1327 self.connectObjects()
1333 1328 self.run()
Show file before | Show file after | Hide comments
1 NO CONTENT: modified file
Show file before | Show file after | Hide comments
1 NO CONTENT: modified file
Show file before | Show file after | Hide comments
@@ -1,349 +1,348
1 1 '''
2 2
3 3 $Author: murco $
4 4 $Id: jroproc_base.py 1 2012-11-12 18:56:07Z murco $
5 5 '''
6 6 import inspect
7 7 from fuzzywuzzy import process
8 8
9 9 def checkKwargs(method, kwargs):
10 10 currentKwargs = kwargs
11 11 choices = inspect.getargspec(method).args
12 12 try:
13 13 choices.remove('self')
14 14 except Exception as e:
15 15 pass
16 16
17 17 try:
18 18 choices.remove('dataOut')
19 19 except Exception as e:
20 20 pass
21 21
22 22 for kwarg in kwargs:
23 23 fuzz = process.extractOne(kwarg, choices)
24 24 if fuzz is None:
25 25 continue
26 26 if fuzz[1] < 100:
27 27 raise Exception('\x1b[0;32;40mDid you mean {} instead of {} in {}? \x1b[0m'.
28 28 format(fuzz[0], kwarg, method.__self__.__class__.__name__))
29 29
30 30 class ProcessingUnit(object):
31 31
32 32 """
33 33 Esta es la clase base para el procesamiento de datos.
34 34
35 35 Contiene el metodo "call" para llamar operaciones. Las operaciones pueden ser:
36 36 - Metodos internos (callMethod)
37 37 - Objetos del tipo Operation (callObject). Antes de ser llamados, estos objetos
38 38 tienen que ser agreagados con el metodo "add".
39 39
40 40 """
41 41 # objeto de datos de entrada (Voltage, Spectra o Correlation)
42 42 dataIn = None
43 43 dataInList = []
44 44
45 45 # objeto de datos de entrada (Voltage, Spectra o Correlation)
46 46 dataOut = None
47 47
48 48 operations2RunDict = None
49 49
50 50 isConfig = False
51 51
52 52
53 53 def __init__(self, *args, **kwargs):
54 54
55 55 self.dataIn = None
56 56 self.dataInList = []
57 57
58 58 self.dataOut = None
59 59
60 60 self.operations2RunDict = {}
61 61 self.operationKwargs = {}
62 62
63 63 self.isConfig = False
64 64
65 65 self.args = args
66 66 self.kwargs = kwargs
67 67 checkKwargs(self.run, kwargs)
68 68
69 69 def getAllowedArgs(self):
70 70 return inspect.getargspec(self.run).args
71 71
72 72 def addOperationKwargs(self, objId, **kwargs):
73 73 '''
74 74 '''
75 75
76 76 self.operationKwargs[objId] = kwargs
77 77
78 78
79 79 def addOperation(self, opObj, objId):
80 80
81 81 """
82 82 Agrega un objeto del tipo "Operation" (opObj) a la lista de objetos "self.objectList" y retorna el
83 83 identificador asociado a este objeto.
84 84
85 85 Input:
86 86
87 87 object : objeto de la clase "Operation"
88 88
89 89 Return:
90 90
91 91 objId : identificador del objeto, necesario para ejecutar la operacion
92 92 """
93 93
94 94 self.operations2RunDict[objId] = opObj
95 95
96 96 return objId
97 97
98 98 def getOperationObj(self, objId):
99 99
100 100 if objId not in self.operations2RunDict.keys():
101 101 return None
102 102
103 103 return self.operations2RunDict[objId]
104 104
105 105 def operation(self, **kwargs):
106 106
107 107 """
108 108 Operacion directa sobre la data (dataOut.data). Es necesario actualizar los valores de los
109 109 atributos del objeto dataOut
110 110
111 111 Input:
112 112
113 113 **kwargs : Diccionario de argumentos de la funcion a ejecutar
114 114 """
115 115
116 116 raise NotImplementedError
117 117
118 118 def callMethod(self, name, opId):
119 119
120 120 """
121 121 Ejecuta el metodo con el nombre "name" y con argumentos **kwargs de la propia clase.
122 122
123 123 Input:
124 124 name : nombre del metodo a ejecutar
125 125
126 126 **kwargs : diccionario con los nombres y valores de la funcion a ejecutar.
127 127
128 128 """
129 129
130 130 #Checking the inputs
131 131 if name == 'run':
132 132
133 133 if not self.checkInputs():
134 134 self.dataOut.flagNoData = True
135 135 return False
136 136 else:
137 137 #Si no es un metodo RUN la entrada es la misma dataOut (interna)
138 138 if self.dataOut is not None and self.dataOut.isEmpty():
139 139 return False
140 140
141 141 #Getting the pointer to method
142 142 methodToCall = getattr(self, name)
143 143
144 144 #Executing the self method
145 145
146 146 if hasattr(self, 'mp'):
147 147 if name=='run':
148 148 if self.mp is False:
149 149 self.mp = True
150 150 self.start()
151 151 else:
152 152 self.operationKwargs[opId]['parent'] = self.kwargs
153 153 methodToCall(**self.operationKwargs[opId])
154 154 else:
155 155 if name=='run':
156 156 methodToCall(**self.kwargs)
157 157 else:
158 158 methodToCall(**self.operationKwargs[opId])
159 159
160 160 if self.dataOut is None:
161 161 return False
162 162
163 163 if self.dataOut.isEmpty():
164 164 return False
165 165
166 166 return True
167 167
168 168 def callObject(self, objId):
169 169
170 170 """
171 171 Ejecuta la operacion asociada al identificador del objeto "objId"
172 172
173 173 Input:
174 174
175 175 objId : identificador del objeto a ejecutar
176 176
177 177 **kwargs : diccionario con los nombres y valores de la funcion a ejecutar.
178 178
179 179 Return:
180 180
181 181 None
182 182 """
183 183
184 184 if self.dataOut is not None and self.dataOut.isEmpty():
185 185 return False
186 186
187 187 externalProcObj = self.operations2RunDict[objId]
188 188
189 189 if hasattr(externalProcObj, 'mp'):
190 190 if externalProcObj.mp is False:
191 191 externalProcObj.kwargs['parent'] = self.kwargs
192 192 self.operationKwargs[objId] = externalProcObj.kwargs
193 193 externalProcObj.mp = True
194 194 externalProcObj.start()
195 195 else:
196 196 externalProcObj.run(self.dataOut, **externalProcObj.kwargs)
197 197 self.operationKwargs[objId] = externalProcObj.kwargs
198 198
199 199
200 200 return True
201 201
202 202 def call(self, opType, opName=None, opId=None):
203
204 203 """
205 204 Return True si ejecuta la operacion interna nombrada "opName" o la operacion externa
206 205 identificada con el id "opId"; con los argumentos "**kwargs".
207 206
208 207 False si la operacion no se ha ejecutado.
209 208
210 209 Input:
211 210
212 211 opType : Puede ser "self" o "external"
213 212
214 213 Depende del tipo de operacion para llamar a:callMethod or callObject:
215 214
216 215 1. If opType = "self": Llama a un metodo propio de esta clase:
217 216
218 217 name_method = getattr(self, name)
219 218 name_method(**kwargs)
220 219
221 220
222 221 2. If opType = "other" o"external": Llama al metodo "run()" de una instancia de la
223 222 clase "Operation" o de un derivado de ella:
224 223
225 224 instanceName = self.operationList[opId]
226 225 instanceName.run(**kwargs)
227 226
228 227 opName : Si la operacion es interna (opType = 'self'), entonces el "opName" sera
229 228 usada para llamar a un metodo interno de la clase Processing
230 229
231 230 opId : Si la operacion es externa (opType = 'other' o 'external), entonces el
232 231 "opId" sera usada para llamar al metodo "run" de la clase Operation
233 232 registrada anteriormente con ese Id
234 233
235 234 Exception:
236 235 Este objeto de tipo Operation debe de haber sido agregado antes con el metodo:
237 236 "addOperation" e identificado con el valor "opId" = el id de la operacion.
238 237 De lo contrario retornara un error del tipo ValueError
239 238
240 239 """
241 240
242 241 if opType == 'self':
243 242
244 243 if not opName:
245 244 raise ValueError, "opName parameter should be defined"
246 245
247 246 sts = self.callMethod(opName, opId)
248 247
249 248 elif opType == 'other' or opType == 'external' or opType == 'plotter':
250 249
251 250 if not opId:
252 251 raise ValueError, "opId parameter should be defined"
253 252
254 253 if opId not in self.operations2RunDict.keys():
255 254 raise ValueError, "Any operation with id=%s has been added" %str(opId)
256 255
257 256 sts = self.callObject(opId)
258 257
259 258 else:
260 259 raise ValueError, "opType should be 'self', 'external' or 'plotter'; and not '%s'" %opType
261 260
262 261 return sts
263 262
264 263 def setInput(self, dataIn):
265 264
266 265 self.dataIn = dataIn
267 266 self.dataInList.append(dataIn)
268 267
269 268 def getOutputObj(self):
270 269
271 270 return self.dataOut
272 271
273 272 def checkInputs(self):
274 273
275 274 for thisDataIn in self.dataInList:
276 275
277 276 if thisDataIn.isEmpty():
278 277 return False
279 278
280 279 return True
281 280
282 281 def setup(self):
283 282
284 283 raise NotImplementedError
285 284
286 285 def run(self):
287 286
288 287 raise NotImplementedError
289 288
290 289 def close(self):
291 290 #Close every thread, queue or any other object here is it is neccesary.
292 291 return
293 292
294 293 class Operation(object):
295 294
296 295 """
297 296 Clase base para definir las operaciones adicionales que se pueden agregar a la clase ProcessingUnit
298 297 y necesiten acumular informacion previa de los datos a procesar. De preferencia usar un buffer de
299 298 acumulacion dentro de esta clase
300 299
301 300 Ejemplo: Integraciones coherentes, necesita la informacion previa de los n perfiles anteriores (bufffer)
302 301
303 302 """
304 303
305 304 __buffer = None
306 305 isConfig = False
307 306
308 307 def __init__(self, **kwargs):
309 308
310 309 self.__buffer = None
311 310 self.isConfig = False
312 311 self.kwargs = kwargs
313 312 checkKwargs(self.run, kwargs)
314 313
315 314 def getAllowedArgs(self):
316 315 return inspect.getargspec(self.run).args
317 316
318 317 def setup(self):
319 318
320 319 self.isConfig = True
321 320
322 321 raise NotImplementedError
323 322
324 323 def run(self, dataIn, **kwargs):
325 324
326 325 """
327 326 Realiza las operaciones necesarias sobre la dataIn.data y actualiza los
328 327 atributos del objeto dataIn.
329 328
330 329 Input:
331 330
332 331 dataIn : objeto del tipo JROData
333 332
334 333 Return:
335 334
336 335 None
337 336
338 337 Affected:
339 338 __buffer : buffer de recepcion de datos.
340 339
341 340 """
342 341 if not self.isConfig:
343 342 self.setup(**kwargs)
344 343
345 344 raise NotImplementedError
346 345
347 346 def close(self):
348 347
349 348 pass
Show file before | Show file after | Hide comments
@@ -1,64 +1,64
1 1 """.
2 2
3 3 Created on Jul 16, 2014
4 4
5 5 @author: Miguel Urco
6 6 """
7 7
8 8 import numpy
9 9 from setuptools import setup, Extension
10 import numpy
10 from schainpy import __version__
11 11
12 12 setup(name="schainpy",
13 13 version=__version__,
14 14 description="Python tools to read, write and process Jicamarca data",
15 15 author="Miguel Urco",
16 16 author_email="miguel.urco@jro.igp.gob.pe",
17 17 url="http://jro.igp.gob.pe",
18 18 packages={'schainpy',
19 19 'schainpy.model',
20 20 'schainpy.model.data',
21 21 'schainpy.model.graphics',
22 22 'schainpy.model.io',
23 23 'schainpy.model.proc',
24 24 'schainpy.model.serializer',
25 25 'schainpy.model.utils',
26 26 'schainpy.gui',
27 27 'schainpy.gui.figures',
28 28 'schainpy.gui.viewcontroller',
29 29 'schainpy.gui.viewer',
30 30 'schainpy.gui.viewer.windows'},
31 31 ext_package='schainpy',
32 32 py_modules=[''],
33 33 package_data={'': ['schain.conf.template'],
34 34 'schainpy.gui.figures': ['*.png', '*.jpg'],
35 35 },
36 36 include_package_data=False,
37 37 entry_points={
38 38 'console_scripts': [
39 39 'schain = schaincli.cli:main',
40 40 ],
41 41 },
42 42 scripts=['schainpy/gui/schainGUI'],
43 43 ext_modules=[
44 44 Extension("cSchain",
45 45 ["schainpy/model/proc/extensions.c"],
46 46 include_dirs=[numpy.get_include()],
47 47 #extra_compile_args=['-Werror'],
48 48 )
49 49 ],
50 50 install_requires=[
51 51 "scipy >= 0.14.0",
52 52 "h5py >= 2.2.1",
53 53 "matplotlib >= 1.4.2",
54 54 "pyfits >= 3.4",
55 55 "paramiko >= 2.1.2",
56 56 "paho-mqtt >= 1.2",
57 57 "zmq",
58 58 "fuzzywuzzy",
59 59 "click",
60 60 "colorama",
61 61 "python-Levenshtein"
62 62 ],
63 63 )
64 64
Show file before | Show file after | Hide comments
1 NO CONTENT: file was removed
General Comments 0
You need to be logged in to leave comments. Login now