##// END OF EJS Templates
schain-jc
RSS
Bug fixed in SignalChain GUI:...
Miguel Valdez -
r581:cd9db1f6f595
parent child
Show More

The requested changes are too big and content was truncated. Show full diff

Show file before | Show file after | Hide comments
@@ -1,1026 +1,1034
1 1 '''
2 2 Created on September , 2012
3 3 @author:
4 4 '''
5 5 from xml.etree.ElementTree import Element, SubElement
6 6 from xml.etree import ElementTree as ET
7 7 from xml.dom import minidom
8 8
9 9 #import datetime
10 10 from model import *
11 11
12 12 import ast
13 13
14 14 def prettify(elem):
15 15 """Return a pretty-printed XML string for the Element.
16 16 """
17 17 rough_string = ET.tostring(elem, 'utf-8')
18 18 reparsed = minidom.parseString(rough_string)
19 19 return reparsed.toprettyxml(indent=" ")
20 20
21 21 class ParameterConf():
22 22
23 23 id = None
24 24 name = None
25 25 value = None
26 26 format = None
27 27
28 28 __formated_value = None
29 29
30 30 ELEMENTNAME = 'Parameter'
31 31
32 32 def __init__(self):
33 33
34 34 self.format = 'str'
35 35
36 36 def getElementName(self):
37 37
38 38 return self.ELEMENTNAME
39 39
40 40 def getValue(self):
41 41
42 42 if self.__formated_value != None:
43 43
44 44 return self.__formated_value
45 45
46 46 value = self.value
47 47
48 48 if self.format == 'bool':
49 49 value = int(value)
50 50
51 51 if self.format == 'list':
52 52 strList = value.split(',')
53 53
54 54 self.__formated_value = strList
55 55
56 56 return self.__formated_value
57 57
58 58 if self.format == 'intlist':
59 59 """
60 60 Example:
61 61 value = (0,1,2)
62 62 """
63 63 strList = value.split(',')
64 64 intList = [int(x) for x in strList]
65 65
66 66 self.__formated_value = intList
67 67
68 68 return self.__formated_value
69 69
70 70 if self.format == 'floatlist':
71 71 """
72 72 Example:
73 73 value = (0.5, 1.4, 2.7)
74 74 """
75 75 strList = value.split(',')
76 76 floatList = [float(x) for x in strList]
77 77
78 78 self.__formated_value = floatList
79 79
80 80 return self.__formated_value
81 81
82 82 if self.format == 'date':
83 83 strList = value.split('/')
84 84 intList = [int(x) for x in strList]
85 85 date = datetime.date(intList[0], intList[1], intList[2])
86 86
87 87 self.__formated_value = date
88 88
89 89 return self.__formated_value
90 90
91 91 if self.format == 'time':
92 92 strList = value.split(':')
93 93 intList = [int(x) for x in strList]
94 94 time = datetime.time(intList[0], intList[1], intList[2])
95 95
96 96 self.__formated_value = time
97 97
98 98 return self.__formated_value
99 99
100 100 if self.format == 'pairslist':
101 101 """
102 102 Example:
103 103 value = (0,1),(1,2)
104 104 """
105 105
106 106 value = value.replace('(', '')
107 107 value = value.replace(')', '')
108 108
109 109 strList = value.split(',')
110 110 intList = [int(item) for item in strList]
111 111 pairList = []
112 112 for i in range(len(intList)/2):
113 113 pairList.append((intList[i*2], intList[i*2 + 1]))
114 114
115 115 self.__formated_value = pairList
116 116
117 117 return self.__formated_value
118 118
119 119 if self.format == 'multilist':
120 120 """
121 121 Example:
122 122 value = (0,1,2),(3,4,5)
123 123 """
124 124 multiList = ast.literal_eval(value)
125 125
126 126 self.__formated_value = multiList
127 127
128 128 return self.__formated_value
129 129
130 130 format_func = eval(self.format)
131 131
132 132 self.__formated_value = format_func(value)
133 133
134 134 return self.__formated_value
135 135
136 136 def setup(self, id, name, value, format='str'):
137 137
138 138 self.id = id
139 139 self.name = name
140 140 self.value = str(value)
141 141 self.format = str.lower(format)
142 142
143 143 def update(self, name, value, format='str'):
144 144
145 145 self.name = name
146 146 self.value = str(value)
147 147 self.format = format
148 148
149 149 def makeXml(self, opElement):
150 150
151 151 parmElement = SubElement(opElement, self.ELEMENTNAME)
152 152 parmElement.set('id', str(self.id))
153 153 parmElement.set('name', self.name)
154 154 parmElement.set('value', self.value)
155 155 parmElement.set('format', self.format)
156 156
157 157 def readXml(self, parmElement):
158 158
159 159 self.id = parmElement.get('id')
160 160 self.name = parmElement.get('name')
161 161 self.value = parmElement.get('value')
162 162 self.format = str.lower(parmElement.get('format'))
163 163
164 164 #Compatible with old signal chain version
165 165 if self.format == 'int' and self.name == 'idfigure':
166 166 self.name = 'id'
167 167
168 168 def printattr(self):
169 169
170 170 print "Parameter[%s]: name = %s, value = %s, format = %s" %(self.id, self.name, self.value, self.format)
171 171
172 172 class OperationConf():
173 173
174 174 id = None
175 175 name = None
176 176 priority = None
177 177 type = None
178 178
179 179 parmConfObjList = []
180 180
181 181 ELEMENTNAME = 'Operation'
182 182
183 183 def __init__(self):
184 184
185 185 self.id = 0
186 186 self.name = None
187 187 self.priority = None
188 188 self.type = 'self'
189 189
190 190
191 191 def __getNewId(self):
192 192
193 193 return int(self.id)*10 + len(self.parmConfObjList) + 1
194 194
195 195 def getElementName(self):
196 196
197 197 return self.ELEMENTNAME
198 198
199 199 def getParameterObjList(self):
200 200
201 201 return self.parmConfObjList
202 202
203 203 def getParameterObj(self, parameterName):
204 204
205 205 for parmConfObj in self.parmConfObjList:
206 206
207 207 if parmConfObj.name != parameterName:
208 208 continue
209 209
210 210 return parmConfObj
211 211
212 212 return None
213 213
214 214 def getParameterObjfromValue(self,parameterValue):
215 215 for parmConfObj in self.parmConfObjList:
216 216
217 217 if parmConfObj.getValue() != parameterValue:
218 218 continue
219 219
220 220 return parmConfObj.getValue()
221 221
222 222 return None
223 223
224 224 def getParameterValue(self, parameterName):
225 225
226 226 parameterObj = self.getParameterObj(parameterName)
227 227 value = parameterObj.getValue()
228 228
229 229 return value
230 230
231 231 def setup(self, id, name, priority, type):
232 232
233 233 self.id = id
234 234 self.name = name
235 235 self.type = type
236 236 self.priority = priority
237 237
238 238 self.parmConfObjList = []
239 239
240 240 def removeParameters(self):
241 241
242 242 for obj in self.parmConfObjList:
243 243 del obj
244 244
245 245 self.parmConfObjList = []
246 246
247 247 def addParameter(self, name, value, format='str'):
248 248
249 249 id = self.__getNewId()
250 250
251 251 parmConfObj = ParameterConf()
252 252 parmConfObj.setup(id, name, value, format)
253 253
254 254 self.parmConfObjList.append(parmConfObj)
255 255
256 256 return parmConfObj
257 257
258 258 def changeParameter(self, name, value, format='str'):
259 259
260 260 parmConfObj = self.getParameterObj(name)
261 261 parmConfObj.update(name, value, format)
262 262
263 263 return parmConfObj
264 264
265 265 def makeXml(self, upElement):
266 266
267 267 opElement = SubElement(upElement, self.ELEMENTNAME)
268 268 opElement.set('id', str(self.id))
269 269 opElement.set('name', self.name)
270 270 opElement.set('type', self.type)
271 271 opElement.set('priority', str(self.priority))
272 272
273 273 for parmConfObj in self.parmConfObjList:
274 274 parmConfObj.makeXml(opElement)
275 275
276 276 def readXml(self, opElement):
277 277
278 278 self.id = opElement.get('id')
279 279 self.name = opElement.get('name')
280 280 self.type = opElement.get('type')
281 281 self.priority = opElement.get('priority')
282 282
283 283 #Compatible with old signal chain version
284 284 #Use of 'run' method instead 'init'
285 285 if self.type == 'self' and self.name == 'init':
286 286 self.name = 'run'
287 287
288 288 self.parmConfObjList = []
289 289
290 290 parmElementList = opElement.getiterator(ParameterConf().getElementName())
291 291
292 292 for parmElement in parmElementList:
293 293 parmConfObj = ParameterConf()
294 294 parmConfObj.readXml(parmElement)
295 295
296 296 #Compatible with old signal chain version
297 297 #If an 'plot' OPERATION is found, changes name operation by the value of its type PARAMETER
298 298 if self.type != 'self' and self.name == 'Plot':
299 299 if parmConfObj.format == 'str' and parmConfObj.name == 'type':
300 300 self.name = parmConfObj.value
301 301 continue
302 302
303 303 self.parmConfObjList.append(parmConfObj)
304 304
305 305 def printattr(self):
306 306
307 307 print "%s[%s]: name = %s, type = %s, priority = %s" %(self.ELEMENTNAME,
308 308 self.id,
309 309 self.name,
310 310 self.type,
311 311 self.priority)
312 312
313 313 for parmConfObj in self.parmConfObjList:
314 314 parmConfObj.printattr()
315 315
316 316 def createObject(self):
317 317
318 318 if self.type == 'self':
319 319 raise ValueError, "This operation type cannot be created"
320 320
321 321 if self.type == 'external' or self.type == 'other':
322 322 className = eval(self.name)
323 323 opObj = className()
324 324
325 325 return opObj
326 326
327 327 class ProcUnitConf():
328 328
329 329 id = None
330 330 name = None
331 331 datatype = None
332 332 inputId = None
333 333 parentId = None
334 334
335 335 opConfObjList = []
336 336
337 337 procUnitObj = None
338 338 opObjList = []
339 339
340 340 ELEMENTNAME = 'ProcUnit'
341 341
342 342 def __init__(self):
343 343
344 344 self.id = None
345 345 self.datatype = None
346 346 self.name = None
347 347 self.inputId = None
348 348
349 349 self.opConfObjList = []
350 350
351 351 self.procUnitObj = None
352 352 self.opObjDict = {}
353 353
354 354 def __getPriority(self):
355 355
356 356 return len(self.opConfObjList)+1
357 357
358 358 def __getNewId(self):
359 359
360 360 return int(self.id)*10 + len(self.opConfObjList) + 1
361 361
362 362 def getElementName(self):
363 363
364 364 return self.ELEMENTNAME
365 365
366 366 def getId(self):
367 367
368 368 return str(self.id)
369 369
370 370 def getInputId(self):
371 371
372 372 return str(self.inputId)
373 373
374 374 def getOperationObjList(self):
375 375
376 376 return self.opConfObjList
377 377
378 378 def getOperationObj(self, name=None):
379 379
380 380 for opConfObj in self.opConfObjList:
381 381
382 382 if opConfObj.name != name:
383 383 continue
384 384
385 385 return opConfObj
386 386
387 387 return None
388 388
389 389 def getOpObjfromParamValue(self,value=None):
390 390
391 391 for opConfObj in self.opConfObjList:
392 392 if opConfObj.getParameterObjfromValue(parameterValue=value) != value:
393 393 continue
394 394 return opConfObj
395 395 return None
396 396
397 397 def getProcUnitObj(self):
398 398
399 399 return self.procUnitObj
400 400
401 401 def setup(self, id, name, datatype, inputId, parentId=None):
402 402
403 403 self.id = id
404 404 self.name = name
405 405 self.datatype = datatype
406 406 self.inputId = inputId
407 407 self.parentId = parentId
408 408
409 409 self.opConfObjList = []
410 410
411 411 self.addOperation(name='run', optype='self')
412 412
413 413 def removeOperations(self):
414 414
415 415 for obj in self.opConfObjList:
416 416 del obj
417 417
418 418 self.opConfObjList = []
419 419 self.addOperation(name='run')
420 420
421 421 def addParameter(self, **kwargs):
422 422 '''
423 423 Add parameters to "run" operation
424 424 '''
425 425 opObj = self.opConfObjList[0]
426 426
427 427 opObj.addParameter(**kwargs)
428 428
429 429 return opObj
430 430
431 431 def addOperation(self, name, optype='self'):
432 432
433 433 id = self.__getNewId()
434 434 priority = self.__getPriority()
435 435
436 436 opConfObj = OperationConf()
437 437 opConfObj.setup(id, name=name, priority=priority, type=optype)
438 438
439 439 self.opConfObjList.append(opConfObj)
440 440
441 441 return opConfObj
442 442
443 443 def makeXml(self, procUnitElement):
444 444
445 445 upElement = SubElement(procUnitElement, self.ELEMENTNAME)
446 446 upElement.set('id', str(self.id))
447 447 upElement.set('name', self.name)
448 448 upElement.set('datatype', self.datatype)
449 449 upElement.set('inputId', str(self.inputId))
450 450
451 451 for opConfObj in self.opConfObjList:
452 452 opConfObj.makeXml(upElement)
453 453
454 454 def readXml(self, upElement):
455 455
456 456 self.id = upElement.get('id')
457 457 self.name = upElement.get('name')
458 458 self.datatype = upElement.get('datatype')
459 459 self.inputId = upElement.get('inputId')
460 460
461 461 self.opConfObjList = []
462 462
463 463 opElementList = upElement.getiterator(OperationConf().getElementName())
464 464
465 465 for opElement in opElementList:
466 466 opConfObj = OperationConf()
467 467 opConfObj.readXml(opElement)
468 468 self.opConfObjList.append(opConfObj)
469 469
470 470 def printattr(self):
471 471
472 472 print "%s[%s]: name = %s, datatype = %s, inputId = %s" %(self.ELEMENTNAME,
473 473 self.id,
474 474 self.name,
475 475 self.datatype,
476 476 self.inputId)
477 477
478 478 for opConfObj in self.opConfObjList:
479 479 opConfObj.printattr()
480 480
481 481 def createObjects(self):
482 482
483 483 className = eval(self.name)
484 484 procUnitObj = className()
485 485
486 486 for opConfObj in self.opConfObjList:
487 487
488 488 if opConfObj.type == 'self':
489 489 continue
490 490
491 491 opObj = opConfObj.createObject()
492 492
493 493 self.opObjDict[opConfObj.id] = opObj
494 494 procUnitObj.addOperation(opObj, opConfObj.id)
495 495
496 496 self.procUnitObj = procUnitObj
497 497
498 498 return procUnitObj
499 499
500 500 def run(self):
501 501
502 502 finalSts = False
503 503
504 504 for opConfObj in self.opConfObjList:
505 505
506 506 kwargs = {}
507 507 for parmConfObj in opConfObj.getParameterObjList():
508 508 if opConfObj.name == 'run' and parmConfObj.name == 'datatype':
509 509 continue
510 510
511 511 kwargs[parmConfObj.name] = parmConfObj.getValue()
512 512
513 513 #print "\tRunning the '%s' operation with %s" %(opConfObj.name, opConfObj.id)
514 514 sts = self.procUnitObj.call(opType = opConfObj.type,
515 515 opName = opConfObj.name,
516 516 opId = opConfObj.id,
517 517 **kwargs)
518 518 finalSts = finalSts or sts
519 519
520 520 return finalSts
521 521
522 522 def close(self):
523 523
524 524 for opConfObj in self.opConfObjList:
525 525 if opConfObj.type == 'self':
526 526 continue
527 527
528 528 opObj = self.procUnitObj.getOperationObj(opConfObj.id)
529 529 opObj.close()
530 530
531 531 self.procUnitObj.close()
532 532
533 533 return
534 534
535 535 class ReadUnitConf(ProcUnitConf):
536 536
537 537 path = None
538 538 startDate = None
539 539 endDate = None
540 540 startTime = None
541 541 endTime = None
542 542
543 543 ELEMENTNAME = 'ReadUnit'
544 544
545 545 def __init__(self):
546 546
547 547 self.id = None
548 548 self.datatype = None
549 549 self.name = None
550 550 self.inputId = 0
551 551
552 552 self.opConfObjList = []
553 553 self.opObjList = []
554 554
555 555 def getElementName(self):
556 556
557 557 return self.ELEMENTNAME
558 558
559 559 def setup(self, id, name, datatype, path, startDate="", endDate="", startTime="", endTime="", parentId=None, **kwargs):
560 560
561 561 self.id = id
562 562 self.name = name
563 563 self.datatype = datatype
564 564
565 565 self.path = path
566 566 self.startDate = startDate
567 567 self.endDate = endDate
568 568 self.startTime = startTime
569 569 self.endTime = endTime
570 570
571 571 self.addRunOperation(**kwargs)
572 572
573 573 def update(self, datatype, path, startDate, endDate, startTime, endTime, parentId=None, **kwargs):
574 574
575 575 self.datatype = datatype
576 576 self.path = path
577 577 self.startDate = startDate
578 578 self.endDate = endDate
579 579 self.startTime = startTime
580 580 self.endTime = endTime
581 581
582 582 self.updateRunOperation(**kwargs)
583 583
584 584 def addRunOperation(self, **kwargs):
585 585
586 586 opObj = self.addOperation(name = 'run', optype = 'self')
587 587
588 588 opObj.addParameter(name='datatype' , value=self.datatype, format='str')
589 589 opObj.addParameter(name='path' , value=self.path, format='str')
590 590 opObj.addParameter(name='startDate' , value=self.startDate, format='date')
591 591 opObj.addParameter(name='endDate' , value=self.endDate, format='date')
592 592 opObj.addParameter(name='startTime' , value=self.startTime, format='time')
593 593 opObj.addParameter(name='endTime' , value=self.endTime, format='time')
594 594
595 595 for key, value in kwargs.items():
596 596 opObj.addParameter(name=key, value=value, format=type(value).__name__)
597 597
598 598 return opObj
599 599
600 600 def updateRunOperation(self, **kwargs):
601 601
602 602 opObj = self.getOperationObj(name = 'run')
603 603 opObj.removeParameters()
604 604
605 605 opObj.addParameter(name='datatype' , value=self.datatype, format='str')
606 606 opObj.addParameter(name='path' , value=self.path, format='str')
607 607 opObj.addParameter(name='startDate' , value=self.startDate, format='date')
608 608 opObj.addParameter(name='endDate' , value=self.endDate, format='date')
609 609 opObj.addParameter(name='startTime' , value=self.startTime, format='time')
610 610 opObj.addParameter(name='endTime' , value=self.endTime, format='time')
611 611
612 612 for key, value in kwargs.items():
613 613 opObj.addParameter(name=key, value=value, format=type(value).__name__)
614 614
615 615 return opObj
616 616
617 617 class Project():
618 618
619 619 id = None
620 620 name = None
621 621 description = None
622 622 # readUnitConfObjList = None
623 623 procUnitConfObjDict = None
624 624
625 625 ELEMENTNAME = 'Project'
626 626
627 627 def __init__(self, control=None, dataq=None):
628 628
629 629 self.id = None
630 630 self.name = None
631 631 self.description = None
632 632
633 633 self.procUnitConfObjDict = {}
634 634
635 635 #global data_q
636 636 #data_q = dataq
637 637
638 638 if control==None:
639 639 control = {}
640 640 control['stop'] = False
641 641 control['pause'] = False
642 642
643 643 self.control = control
644 644
645 645 def __getNewId(self):
646 646
647 647 id = int(self.id)*10 + len(self.procUnitConfObjDict) + 1
648 648
649 649 return str(id)
650 650
651 651 def getElementName(self):
652 652
653 653 return self.ELEMENTNAME
654 654
655 655 def getId(self):
656 656
657 657 return self.id
658 658
659 659 def setup(self, id, name, description):
660 660
661 661 self.id = id
662 662 self.name = name
663 663 self.description = description
664 664
665 665 def update(self, name, description):
666 666
667 667 self.name = name
668 668 self.description = description
669 669
670 670 def addReadUnit(self, datatype=None, name=None, **kwargs):
671 671
672 672 #Compatible with old signal chain version
673 673 if datatype==None and name==None:
674 674 raise ValueError, "datatype or name should be defined"
675 675
676 676 if name==None:
677 677 if 'Reader' in datatype:
678 678 name = datatype
679 679 else:
680 680 name = '%sReader' %(datatype)
681 681
682 682 if datatype==None:
683 683 datatype = name.replace('Reader','')
684 684
685 685 id = self.__getNewId()
686 686
687 687 readUnitConfObj = ReadUnitConf()
688 688 readUnitConfObj.setup(id, name, datatype, parentId=self.id, **kwargs)
689 689
690 690 self.procUnitConfObjDict[readUnitConfObj.getId()] = readUnitConfObj
691 691
692 692 return readUnitConfObj
693 693
694 694 def addProcUnit(self, inputId=0, datatype=None, name=None):
695 695
696 696 #Compatible with old signal chain version
697 697 if datatype==None and name==None:
698 698 raise ValueError, "datatype or name should be defined"
699 699
700 700 if name==None:
701 701 if 'Proc' in datatype:
702 702 name = datatype
703 703 else:
704 704 name = '%sProc' %(datatype)
705 705
706 706 if datatype==None:
707 707 datatype = name.replace('Proc','')
708 708
709 709 id = self.__getNewId()
710 710
711 711 procUnitConfObj = ProcUnitConf()
712 712 procUnitConfObj.setup(id, name, datatype, inputId, parentId=self.id)
713 713
714 714 self.procUnitConfObjDict[procUnitConfObj.getId()] = procUnitConfObj
715 715
716 716 return procUnitConfObj
717 717
718 718 def getReadUnitId(self):
719 719
720 720 readUnitConfObj = self.getReadUnitObj()
721 721
722 722 return readUnitConfObj.id
723 723
724 724 def getReadUnitObj(self):
725 725
726 726 for obj in self.procUnitConfObjDict.values():
727 727 if obj.getElementName() == "ReadUnit":
728 728 return obj
729 729
730 730 return None
731 731
732 732 def getProcUnitObj(self, id):
733 733
734 734 return self.procUnitConfObjDict[id]
735
735
736 def getProcUnitObjByName(self, name):
737
738 for obj in self.procUnitConfObjDict.values():
739 if obj.name == name:
740 return obj
741
742 return None
743
736 744 def makeXml(self):
737 745
738 746 projectElement = Element('Project')
739 747 projectElement.set('id', str(self.id))
740 748 projectElement.set('name', self.name)
741 749 projectElement.set('description', self.description)
742 750
743 751 # for readUnitConfObj in self.readUnitConfObjList:
744 752 # readUnitConfObj.makeXml(projectElement)
745 753
746 754 for procUnitConfObj in self.procUnitConfObjDict.values():
747 755 procUnitConfObj.makeXml(projectElement)
748 756
749 757 self.projectElement = projectElement
750 758
751 759 def writeXml(self, filename):
752 760
753 761 self.makeXml()
754 762
755 763 #print prettify(self.projectElement)
756 764
757 765 ElementTree(self.projectElement).write(filename, method='xml')
758 766
759 767 def readXml(self, filename):
760 768
761 769 #tree = ET.parse(filename)
762 770 self.projectElement = None
763 771 # self.readUnitConfObjList = []
764 772 self.procUnitConfObjDict = {}
765 773
766 774 self.projectElement = ElementTree().parse(filename)
767 775
768 776 self.project = self.projectElement.tag
769 777
770 778 self.id = self.projectElement.get('id')
771 779 self.name = self.projectElement.get('name')
772 780 self.description = self.projectElement.get('description')
773 781
774 782 readUnitElementList = self.projectElement.getiterator(ReadUnitConf().getElementName())
775 783
776 784 for readUnitElement in readUnitElementList:
777 785 readUnitConfObj = ReadUnitConf()
778 786 readUnitConfObj.readXml(readUnitElement)
779 787
780 788 self.procUnitConfObjDict[readUnitConfObj.getId()] = readUnitConfObj
781 789
782 790 procUnitElementList = self.projectElement.getiterator(ProcUnitConf().getElementName())
783 791
784 792 for procUnitElement in procUnitElementList:
785 793 procUnitConfObj = ProcUnitConf()
786 794 procUnitConfObj.readXml(procUnitElement)
787 795
788 796 self.procUnitConfObjDict[procUnitConfObj.getId()] = procUnitConfObj
789 797
790 798 def printattr(self):
791 799
792 800 print "Project[%s]: name = %s, description = %s" %(self.id,
793 801 self.name,
794 802 self.description)
795 803
796 804 # for readUnitConfObj in self.readUnitConfObjList:
797 805 # readUnitConfObj.printattr()
798 806
799 807 for procUnitConfObj in self.procUnitConfObjDict.values():
800 808 procUnitConfObj.printattr()
801 809
802 810 def createObjects(self):
803 811
804 812 # for readUnitConfObj in self.readUnitConfObjList:
805 813 # readUnitConfObj.createObjects()
806 814
807 815 for procUnitConfObj in self.procUnitConfObjDict.values():
808 816 procUnitConfObj.createObjects()
809 817
810 818 def __connect(self, objIN, thisObj):
811 819
812 820 thisObj.setInput(objIN.getOutputObj())
813 821
814 822 def connectObjects(self):
815 823
816 824 for thisPUConfObj in self.procUnitConfObjDict.values():
817 825
818 826 inputId = thisPUConfObj.getInputId()
819 827
820 828 if int(inputId) == 0:
821 829 continue
822 830
823 831 #Get input object
824 832 puConfINObj = self.procUnitConfObjDict[inputId]
825 833 puObjIN = puConfINObj.getProcUnitObj()
826 834
827 835 #Get current object
828 836 thisPUObj = thisPUConfObj.getProcUnitObj()
829 837
830 838 self.__connect(puObjIN, thisPUObj)
831 839
832 840 def run(self):
833 841
834 842 # for readUnitConfObj in self.readUnitConfObjList:
835 843 # readUnitConfObj.run()
836 844 print
837 845 print "*"*40
838 846 print " Starting SIGNAL CHAIN PROCESSING "
839 847 print "*"*40
840 848 print
841 849
842 850 keyList = self.procUnitConfObjDict.keys()
843 851 keyList.sort()
844 852
845 853 while(True):
846 854
847 855 finalSts = False
848 856
849 857 for procKey in keyList:
850 858 # print "Running the '%s' process with %s" %(procUnitConfObj.name, procUnitConfObj.id)
851 859
852 860 procUnitConfObj = self.procUnitConfObjDict[procKey]
853 861 sts = procUnitConfObj.run()
854 862 finalSts = finalSts or sts
855 863
856 864 #If every process unit finished so end process
857 865 if not(finalSts):
858 866 print "Every process unit have finished"
859 867 break
860 868
861 869 if self.control['pause']:
862 870 print "Pause..."
863 871
864 872 while True:
865 873 time.sleep(0.1)
866 874
867 875 if not self.control['pause']:
868 876 break
869 877
870 878 if self.control['stop']:
871 879 break
872 880
873 881 if self.control['stop']:
874 882 print "Stopping process"
875 883 break
876 884
877 885 #Closing every process
878 886 for procKey in keyList:
879 887 procUnitConfObj = self.procUnitConfObjDict[procKey]
880 888 procUnitConfObj.close()
881 889
882 890 print "Process stopped"
883 891
884 892 def start(self, filename):
885 893
886 894 self.writeXml(filename)
887 895 self.readXml(filename)
888 896
889 897 self.createObjects()
890 898 self.connectObjects()
891 899 self.run()
892 900
893 901 if __name__ == '__main__':
894 902
895 903 desc = "Segundo Test"
896 904 filename = "schain.xml"
897 905
898 906 controllerObj = Project()
899 907
900 908 controllerObj.setup(id = '191', name='test01', description=desc)
901 909
902 910 readUnitConfObj = controllerObj.addReadUnit(datatype='Voltage',
903 911 path='data/rawdata/',
904 912 startDate='2011/01/01',
905 913 endDate='2012/12/31',
906 914 startTime='00:00:00',
907 915 endTime='23:59:59',
908 916 online=1,
909 917 walk=1)
910 918
911 919 # opObj00 = readUnitConfObj.addOperation(name='printInfo')
912 920
913 921 procUnitConfObj0 = controllerObj.addProcUnit(datatype='Voltage', inputId=readUnitConfObj.getId())
914 922
915 923 opObj10 = procUnitConfObj0.addOperation(name='selectChannels')
916 924 opObj10.addParameter(name='channelList', value='3,4,5', format='intlist')
917 925
918 926 opObj10 = procUnitConfObj0.addOperation(name='selectHeights')
919 927 opObj10.addParameter(name='minHei', value='90', format='float')
920 928 opObj10.addParameter(name='maxHei', value='180', format='float')
921 929
922 930 opObj12 = procUnitConfObj0.addOperation(name='CohInt', optype='external')
923 931 opObj12.addParameter(name='n', value='10', format='int')
924 932
925 933 procUnitConfObj1 = controllerObj.addProcUnit(datatype='Spectra', inputId=procUnitConfObj0.getId())
926 934 procUnitConfObj1.addParameter(name='nFFTPoints', value='32', format='int')
927 935 # procUnitConfObj1.addParameter(name='pairList', value='(0,1),(0,2),(1,2)', format='')
928 936
929 937
930 938 opObj11 = procUnitConfObj1.addOperation(name='SpectraPlot', optype='external')
931 939 opObj11.addParameter(name='idfigure', value='1', format='int')
932 940 opObj11.addParameter(name='wintitle', value='SpectraPlot0', format='str')
933 941 opObj11.addParameter(name='zmin', value='40', format='int')
934 942 opObj11.addParameter(name='zmax', value='90', format='int')
935 943 opObj11.addParameter(name='showprofile', value='1', format='int')
936 944
937 945 # opObj11 = procUnitConfObj1.addOperation(name='CrossSpectraPlot', optype='external')
938 946 # opObj11.addParameter(name='idfigure', value='2', format='int')
939 947 # opObj11.addParameter(name='wintitle', value='CrossSpectraPlot', format='str')
940 948 # opObj11.addParameter(name='zmin', value='40', format='int')
941 949 # opObj11.addParameter(name='zmax', value='90', format='int')
942 950
943 951
944 952 # procUnitConfObj2 = controllerObj.addProcUnit(datatype='Voltage', inputId=procUnitConfObj0.getId())
945 953 #
946 954 # opObj12 = procUnitConfObj2.addOperation(name='CohInt', optype='external')
947 955 # opObj12.addParameter(name='n', value='2', format='int')
948 956 # opObj12.addParameter(name='overlapping', value='1', format='int')
949 957 #
950 958 # procUnitConfObj3 = controllerObj.addProcUnit(datatype='Spectra', inputId=procUnitConfObj2.getId())
951 959 # procUnitConfObj3.addParameter(name='nFFTPoints', value='32', format='int')
952 960 #
953 961 # opObj11 = procUnitConfObj3.addOperation(name='SpectraPlot', optype='external')
954 962 # opObj11.addParameter(name='idfigure', value='2', format='int')
955 963 # opObj11.addParameter(name='wintitle', value='SpectraPlot1', format='str')
956 964 # opObj11.addParameter(name='zmin', value='40', format='int')
957 965 # opObj11.addParameter(name='zmax', value='90', format='int')
958 966 # opObj11.addParameter(name='showprofile', value='1', format='int')
959 967
960 968 # opObj11 = procUnitConfObj1.addOperation(name='RTIPlot', optype='external')
961 969 # opObj11.addParameter(name='idfigure', value='10', format='int')
962 970 # opObj11.addParameter(name='wintitle', value='RTI', format='str')
963 971 ## opObj11.addParameter(name='xmin', value='21', format='float')
964 972 ## opObj11.addParameter(name='xmax', value='22', format='float')
965 973 # opObj11.addParameter(name='zmin', value='40', format='int')
966 974 # opObj11.addParameter(name='zmax', value='90', format='int')
967 975 # opObj11.addParameter(name='showprofile', value='1', format='int')
968 976 # opObj11.addParameter(name='timerange', value=str(60), format='int')
969 977
970 978 # opObj10 = procUnitConfObj1.addOperation(name='selectChannels')
971 979 # opObj10.addParameter(name='channelList', value='0,2,4,6', format='intlist')
972 980 #
973 981 # opObj12 = procUnitConfObj1.addOperation(name='IncohInt', optype='external')
974 982 # opObj12.addParameter(name='n', value='2', format='int')
975 983 #
976 984 # opObj11 = procUnitConfObj1.addOperation(name='SpectraPlot', optype='external')
977 985 # opObj11.addParameter(name='idfigure', value='2', format='int')
978 986 # opObj11.addParameter(name='wintitle', value='SpectraPlot10', format='str')
979 987 # opObj11.addParameter(name='zmin', value='70', format='int')
980 988 # opObj11.addParameter(name='zmax', value='90', format='int')
981 989 #
982 990 # opObj10 = procUnitConfObj1.addOperation(name='selectChannels')
983 991 # opObj10.addParameter(name='channelList', value='2,6', format='intlist')
984 992 #
985 993 # opObj12 = procUnitConfObj1.addOperation(name='IncohInt', optype='external')
986 994 # opObj12.addParameter(name='n', value='2', format='int')
987 995 #
988 996 # opObj11 = procUnitConfObj1.addOperation(name='SpectraPlot', optype='external')
989 997 # opObj11.addParameter(name='idfigure', value='3', format='int')
990 998 # opObj11.addParameter(name='wintitle', value='SpectraPlot10', format='str')
991 999 # opObj11.addParameter(name='zmin', value='70', format='int')
992 1000 # opObj11.addParameter(name='zmax', value='90', format='int')
993 1001
994 1002
995 1003 # opObj12 = procUnitConfObj1.addOperation(name='decoder')
996 1004 # opObj12.addParameter(name='ncode', value='2', format='int')
997 1005 # opObj12.addParameter(name='nbauds', value='8', format='int')
998 1006 # opObj12.addParameter(name='code0', value='001110011', format='int')
999 1007 # opObj12.addParameter(name='code1', value='001110011', format='int')
1000 1008
1001 1009
1002 1010
1003 1011 # procUnitConfObj2 = controllerObj.addProcUnit(datatype='Spectra', inputId=procUnitConfObj1.getId())
1004 1012 #
1005 1013 # opObj21 = procUnitConfObj2.addOperation(name='IncohInt', optype='external')
1006 1014 # opObj21.addParameter(name='n', value='2', format='int')
1007 1015 #
1008 1016 # opObj11 = procUnitConfObj2.addOperation(name='SpectraPlot', optype='external')
1009 1017 # opObj11.addParameter(name='idfigure', value='4', format='int')
1010 1018 # opObj11.addParameter(name='wintitle', value='SpectraPlot OBJ 2', format='str')
1011 1019 # opObj11.addParameter(name='zmin', value='70', format='int')
1012 1020 # opObj11.addParameter(name='zmax', value='90', format='int')
1013 1021
1014 1022 print "Escribiendo el archivo XML"
1015 1023
1016 1024 controllerObj.writeXml(filename)
1017 1025
1018 1026 print "Leyendo el archivo XML"
1019 1027 controllerObj.readXml(filename)
1020 1028 #controllerObj.printattr()
1021 1029
1022 1030 controllerObj.createObjects()
1023 1031 controllerObj.connectObjects()
1024 1032 controllerObj.run()
1025 1033
1026 1034 No newline at end of file
Show file before | Show file after | Hide comments
1 NO CONTENT: modified file
The requested commit or file is too big and content was truncated. Show full diff
Show file before | Show file after | Hide comments
@@ -1,949 +1,949
1 1 '''
2 2 @author: Daniel Suarez
3 3 '''
4 4 import os
5 5 import glob
6 6 import ftplib
7 7
8 8 try:
9 9 import paramiko
10 10 import scp
11 11 except:
12 12 print "You should install paramiko if you will use SSH protocol to upload files to a server"
13 13
14 14 import multiprocessing
15 15
16 16 import time
17 17 import threading
18 18
19 19
20 20 try:
21 21 from gevent import sleep
22 22 except:
23 23 from time import sleep
24 24
25 25 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation
26 26
27 27 class Remote(threading.Thread):
28 28 """
29 29 Remote is a parent class used to define the behaviour of FTP and SSH class. These clases are
30 30 used to upload or download files remotely.
31 31
32 32 Non-standard Python modules used:
33 33 None
34 34
35 35 Written by:
36 36
37 37 "Miguel Urco":mailto:miguel.urco@jro.igp.gob.pe Jun. 03, 2015
38 38
39 39 """
40 40
41 41 server = None
42 42 username = None
43 43 password = None
44 44 remotefolder = None
45 45
46 46 period = 60
47 47 fileList = []
48 48 bussy = False
49 49
50 50 def __init__(self, server, username, password, remotefolder, period=60):
51 51
52 52 threading.Thread.__init__(self)
53 53 self._stop = threading.Event()
54 54
55 55 self.status = 0
56 56
57 57 self.period = period
58 58 self.fileList = []
59 59 self.bussy = False
60 60
61 61 self.stopFlag = False
62 62
63 63 print "[Remote Server] Opening server: %s" %server
64 64 if self.open(server, username, password, remotefolder):
65 65 print "[Remote Server] %s server was opened successfully" %server
66 66
67 67 def stop(self):
68 68
69 69 self.stopFlag = True
70 70
71 71 def open(self):
72 72 """
73 73 Connect to server and create a connection class (FTP or SSH) to remote server.
74 74 """
75 75 raise NotImplementedError, "Implement this method in child class"
76 76
77 77 def close(self):
78 78 """
79 79 Close connection to server
80 80 """
81 81 raise NotImplementedError, "Implement this method in child class"
82 82
83 83 def mkdir(self, remotefolder):
84 84 """
85 85 Create a folder remotely
86 86 """
87 87 raise NotImplementedError, "Implement this method in child class"
88 88
89 89 def cd(self, remotefolder):
90 90 """
91 91 Change working directory in remote server
92 92 """
93 93 raise NotImplementedError, "Implement this method in child class"
94 94
95 95 def download(self, filename, localfolder=None):
96 96 """
97 97 Download a file from server to local host
98 98 """
99 99 raise NotImplementedError, "Implement this method in child class"
100 100
101 101 def sendFile(self, fullfilename):
102 102 """
103 103 sendFile method is used to upload a local file to the current directory in remote server
104 104
105 105 Inputs:
106 106 fullfilename - full path name of local file to store in remote directory
107 107
108 108 Returns:
109 109 0 in error case else 1
110 110 """
111 111 raise NotImplementedError, "Implement this method in child class"
112 112
113 113 def upload(self, fullfilename, remotefolder=None):
114 114 """
115 115 upload method is used to upload a local file to remote directory. This method changes
116 116 working directory before sending a file.
117 117
118 118 Inputs:
119 119 fullfilename - full path name of local file to store in remote directory
120 120
121 121 remotefolder - remote directory
122 122
123 123 Returns:
124 124 0 in error case else 1
125 125 """
126 126 print "[Remote Server] Uploading %s to %s:%s" %(fullfilename, self.server, self.remotefolder)
127 127
128 128 if not self.status:
129 129 return 0
130 130
131 131 if remotefolder == None:
132 132 remotefolder = self.remotefolder
133 133
134 134 if not self.cd(remotefolder):
135 135 return 0
136 136
137 137 if not self.sendFile(fullfilename):
138 138 print "[Remote Server] Error uploading file %s" %fullfilename
139 139 return 0
140 140
141 141 print "[Remote Server] upload finished successfully"
142 142
143 143 return 1
144 144
145 145 def delete(self, filename):
146 146 """
147 147 Remove a file from remote server
148 148 """
149 149 pass
150 150
151 151 def updateFileList(self, fileList):
152 152 """
153 153 Remove a file from remote server
154 154 """
155 155
156 156 if fileList == self.fileList:
157 157 return 1
158 158
159 159 init = time.time()
160 160
161 161 while(self.bussy):
162 162 sleep(0.1)
163 163 if time.time() - init > 2*self.period:
164 164 return 0
165 165
166 166 self.fileList = fileList
167 167
168 168 return 1
169 169
170 170 def run(self):
171 171
172 172 if not self.status:
173 173 print "Finishing FTP service"
174 174 return
175 175
176 176 if not self.cd(self.remotefolder):
177 177 raise ValueError, "Could not access to the new remote directory: %s" %self.remotefolder
178 178
179 179 sts = True
180 180
181 181 while True:
182 182
183 183 sleep(self.period)
184 184
185 185 self.bussy = True
186 186
187 187 for thisFile in self.fileList:
188 188 sts = self.upload(thisFile, self.remotefolder)
189 189 if not sts: break
190 190
191 191 if not sts: break
192 192
193 193 self.bussy = False
194 194
195 195 if self.stopFlag:
196 196 break
197 197
198 198 print "[Remote Server] Thread stopped successfully"
199 199
200 200 class FTPClient(Remote):
201 201
202 202 __ftpClientObj = None
203 203
204 204 def __init__(self, server, username, password, remotefolder, period=60):
205 205 """
206 206 """
207 207 Remote.__init__(self, server, username, password, remotefolder, period)
208 208
209 209 def open(self, server, username, password, remotefolder):
210 210
211 211 """
212 212 This method is used to set FTP parameters and establish a connection to remote server
213 213
214 214 Inputs:
215 215 server - remote server IP Address
216 216
217 217 username - remote server Username
218 218
219 219 password - remote server password
220 220
221 221 remotefolder - remote server current working directory
222 222
223 223 Return: void
224 224
225 225 Affects:
226 226 self.status - in case of error or fail connection this parameter is set to 0 else 1
227 227
228 228 """
229 229
230 230 if server == None:
231 231 raise ValueError, "FTP server should be defined"
232 232
233 233 if username == None:
234 234 raise ValueError, "FTP username should be defined"
235 235
236 236 if password == None:
237 237 raise ValueError, "FTP password should be defined"
238 238
239 239 if remotefolder == None:
240 240 raise ValueError, "FTP remote folder should be defined"
241 241
242 242 try:
243 243 ftpClientObj = ftplib.FTP(server)
244 244 except ftplib.all_errors:
245 245 print "FTP server connection fail: %s" %server
246 246 self.status = 0
247 247 return 0
248 248
249 249 try:
250 250 ftpClientObj.login(username, password)
251 251 except ftplib.all_errors:
252 252 print "FTP username or password are incorrect"
253 253 self.status = 0
254 254 return 0
255 255
256 256 if remotefolder == None:
257 257 remotefolder = ftpClientObj.pwd()
258 258 else:
259 259 try:
260 260 ftpClientObj.cwd(remotefolder)
261 261 except ftplib.all_errors:
262 262 print "FTP remote folder is invalid: %s" %remotefolder
263 263 remotefolder = ftpClientObj.pwd()
264 264
265 265 self.server = server
266 266 self.username = username
267 267 self.password = password
268 268 self.remotefolder = remotefolder
269 269 self.__ftpClientObj = ftpClientObj
270 270 self.status = 1
271 271
272 272 return 1
273 273
274 274 def close(self):
275 275 """
276 276 Close connection to remote server
277 277 """
278 278 if not self.status:
279 279 return 0
280 280
281 281 self.__ftpClientObj.close()
282 282
283 283 def mkdir(self, remotefolder):
284 284 """
285 285 mkdir is used to make a new directory in remote server
286 286
287 287 Input:
288 288 remotefolder - directory name
289 289
290 290 Return:
291 291 0 in error case else 1
292 292 """
293 293 if not self.status:
294 294 return 0
295 295
296 296 try:
297 297 self.__ftpClientObj.mkd(dirname)
298 298 except ftplib.all_errors:
299 299 print "Error creating remote folder: %s" %remotefolder
300 300 return 0
301 301
302 302 return 1
303 303
304 304 def cd(self, remotefolder):
305 305 """
306 306 cd is used to change remote working directory on server
307 307
308 308 Input:
309 309 remotefolder - current working directory
310 310
311 311 Affects:
312 312 self.remotefolder
313 313
314 314 Return:
315 315 0 in case of error else 1
316 316 """
317 317 if not self.status:
318 318 return 0
319 319
320 320 if remotefolder == self.remotefolder:
321 321 return 1
322 322
323 323 try:
324 324 self.__ftpClientObj.cwd(remotefolder)
325 325 except ftplib.all_errors:
326 326 print 'Error changing to %s' %remotefolder
327 327 print 'Trying to create remote folder'
328 328
329 329 if not self.mkdir(remotefolder):
330 330 print 'Remote folder could not be created'
331 331 return 0
332 332
333 333 try:
334 334 self.__ftpClientObj.cwd(remotefolder)
335 335 except ftplib.all_errors:
336 336 return 0
337 337
338 338 self.remotefolder = remotefolder
339 339
340 340 return 1
341 341
342 342 def sendFile(self, fullfilename):
343 343
344 344 if not self.status:
345 345 return 0
346 346
347 347 file = open(fullfilename, 'rb')
348 348
349 349 filename = os.path.split(fullfilename)[-1]
350 350
351 351 command = "STOR %s" %filename
352 352
353 353 try:
354 354 self.__ftpClientObj.storbinary(command, file)
355 355 except ftplib.all_errors:
356 356 return 0
357 357
358 358 try:
359 359 self.__ftpClientObj.sendcmd('SITE CHMOD 755 ' + filename)
360 360 except ftplib.all_errors, e:
361 361 print e
362 362
363 363 file.close()
364 364
365 365 return 1
366 366
367 367 class SSHClient(Remote):
368 368
369 369 __sshClientObj = None
370 370 __scpClientObj = None
371 371
372 372 def __init__(self, server, username, password, remotefolder, period=60):
373 373 """
374 374 """
375 375 Remote.__init__(self, server, username, password, remotefolder, period)
376 376
377 377 def open(self, server, username, password, remotefolder, port=22):
378 378
379 379 """
380 380 This method is used to set SSH parameters and establish a connection to a remote server
381 381
382 382 Inputs:
383 383 server - remote server IP Address
384 384
385 385 username - remote server Username
386 386
387 387 password - remote server password
388 388
389 389 remotefolder - remote server current working directory
390 390
391 391 Return: void
392 392
393 393 Affects:
394 394 self.status - in case of error or fail connection this parameter is set to 0 else 1
395 395
396 396 """
397 397
398 398 if server == None:
399 399 raise ValueError, "SSH server should be defined"
400 400
401 401 if username == None:
402 402 raise ValueError, "SSH username should be defined"
403 403
404 404 if password == None:
405 405 raise ValueError, "SSH password should be defined"
406 406
407 407 if remotefolder == None:
408 408 raise ValueError, "SSH remote folder should be defined"
409 409
410 410 try:
411 411 sshClientObj = paramiko.SSHClient()
412 412 except:
413 413 print "SSH server connection fail: %s" %server
414 414 self.status = 0
415 415 return 0
416 416
417 417 sshClientObj.load_system_host_keys()
418 418 sshClientObj.set_missing_host_key_policy(paramiko.WarningPolicy())
419 419
420 420 try:
421 421 sshClientObj.connect(server, username=username, password=password, port=port)
422 422 except :
423 423 print "SSH username or password are incorrect: %s"
424 424 self.status = 0
425 425 return 0
426 426
427 427 scpClientObj = scp.SCPClient(sshClientObj.get_transport(), socket_timeout=30)
428 428
429 429 if remotefolder == None:
430 430 remotefolder = self.pwd()
431 431
432 432 self.server = server
433 433 self.username = username
434 434 self.password = password
435 435 self.__sshClientObj = sshClientObj
436 436 self.__scpClientObj = scpClientObj
437 437 self.status = 1
438 438
439 439 if not self.cd(remotefolder):
440 440 raise ValueError, "Could not access to remote folder: %s" %remotefolder
441 441 return 0
442 442
443 443 self.remotefolder = remotefolder
444 444
445 445 return 1
446 446
447 447 def close(self):
448 448 """
449 449 Close connection to remote server
450 450 """
451 451 if not self.status:
452 452 return 0
453 453
454 454 self.__sshObj.close()
455 455
456 456 def __execute(self, command):
457 457 """
458 458 __execute a command on remote server
459 459
460 460 Input:
461 461 command - Exmaple 'ls -l'
462 462
463 463 Return:
464 464 0 in error case else 1
465 465 """
466 466 if not self.status:
467 467 return 0
468 468
469 469 stdin, stdout, stderr = self.__sshClientObj.exec_command(command)
470 470
471 471 result = stderr.readlines()
472 472 if len(result) > 1:
473 473 return 0
474 474
475 475 result = stdout.readlines()
476 476 if len(result) > 1:
477 477 return result[0][:-1]
478 478
479 479 return 1
480 480
481 481 def mkdir(self, remotefolder):
482 482 """
483 483 mkdir is used to make a new directory in remote server
484 484
485 485 Input:
486 486 remotefolder - directory name
487 487
488 488 Return:
489 489 0 in error case else 1
490 490 """
491 491
492 492 command = 'mkdir %s' %remotefolder
493 493
494 494 return self.__execute(command)
495 495
496 496 def pwd(self):
497 497
498 498 command = 'pwd'
499 499
500 500 return self.__execute(command)
501 501
502 502 def cd(self, remotefolder):
503 503 """
504 504 cd is used to change remote working directory on server
505 505
506 506 Input:
507 507 remotefolder - current working directory
508 508
509 509 Affects:
510 510 self.remotefolder
511 511
512 512 Return:
513 513 0 in case of error else 1
514 514 """
515 515 if not self.status:
516 516 return 0
517 517
518 518 if remotefolder == self.remotefolder:
519 519 return 1
520 520
521 521 chk_command = "cd %s; pwd" %remotefolder
522 522 mkdir_command = "mkdir %s" %remotefolder
523 523
524 524 if not self.__execute(chk_command):
525 525 if not self.__execute(mkdir_command):
526 526 self.remotefolder = None
527 527 return 0
528 528
529 529 self.remotefolder = remotefolder
530 530
531 531 return 1
532 532
533 533 def sendFile(self, fullfilename):
534 534
535 535 if not self.status:
536 536 return 0
537 537
538 538 try:
539 539 self.__scpClientObj.put(fullfilename, remote_path=self.remotefolder)
540 540 except:
541 541 return 0
542 542
543 543 remotefile = os.path.join(self.remotefolder, os.path.split(fullfilename)[-1])
544 544 command = 'chmod 775 %s' %remotefile
545 545
546 546 return self.__execute(command)
547 547
548 548 class SendToServer(ProcessingUnit):
549 549
550 550 def __init__(self):
551 551
552 552 ProcessingUnit.__init__(self)
553 553
554 554 self.isConfig = False
555 555 self.clientObj = None
556 556
557 def setup(self, server, username, password, remotefolder, localfolder, ext='.png', period=60, protocol='ftp'):
557 def setup(self, server, username, password, remotefolder, localfolder, ext='.png', period=60, protocol='ftp', **kwargs):
558 558
559 559 self.clientObj = None
560 560 self.localfolder = localfolder
561 561 self.ext = ext
562 562 self.period = period
563 563
564 564 if str.lower(protocol) == 'ftp':
565 565 self.clientObj = FTPClient(server, username, password, remotefolder, period)
566 566
567 567 if str.lower(protocol) == 'ssh':
568 568 self.clientObj = SSHClient(server, username, password, remotefolder, period)
569 569
570 570 if not self.clientObj:
571 571 raise ValueError, "%s has been chosen as remote access protocol but it is not valid" %protocol
572 572
573 573 self.clientObj.start()
574 574
575 575 def findFiles(self):
576 576
577 577 filenameList = glob.glob1(self.localfolder, '*%s' %self.ext)
578 578
579 579 if len(filenameList) < 1:
580 580 return []
581 581
582 582 fullfilenameList = [os.path.join(self.localfolder, thisFile) for thisFile in filenameList]
583 583
584 584 return fullfilenameList
585 585
586 586 def run(self, **kwargs):
587 587
588 588 if not self.isConfig:
589 589 self.init = time.time()
590 590 self.setup(**kwargs)
591 591 self.isConfig = True
592 592
593 593 if time.time() - self.init >= self.period:
594 594 fullfilenameList = self.findFiles()
595 595 self.clientObj.updateFileList(fullfilenameList)
596 596 self.init = time.time()
597 597
598 598 def close(self):
599 599 print "[Remote Server] Stopping thread"
600 600 self.clientObj.stop()
601 601
602 602
603 603 class FTP(object):
604 604 """
605 605 Ftp is a public class used to define custom File Transfer Protocol from "ftplib" python module
606 606
607 607 Non-standard Python modules used: None
608 608
609 609 Written by "Daniel Suarez":mailto:daniel.suarez@jro.igp.gob.pe Oct. 26, 2010
610 610 """
611 611
612 612 def __init__(self,server = None, username=None, password=None, remotefolder=None):
613 613 """
614 614 This method is used to setting parameters for FTP and establishing connection to remote server
615 615
616 616 Inputs:
617 617 server - remote server IP Address
618 618
619 619 username - remote server Username
620 620
621 621 password - remote server password
622 622
623 623 remotefolder - remote server current working directory
624 624
625 625 Return: void
626 626
627 627 Affects:
628 628 self.status - in Error Case or Connection Failed this parameter is set to 1 else 0
629 629
630 630 self.folderList - sub-folder list of remote folder
631 631
632 632 self.fileList - file list of remote folder
633 633
634 634
635 635 """
636 636
637 637 if ((server == None) and (username==None) and (password==None) and (remotefolder==None)):
638 638 server, username, password, remotefolder = self.parmsByDefault()
639 639
640 640 self.server = server
641 641 self.username = username
642 642 self.password = password
643 643 self.remotefolder = remotefolder
644 644 self.file = None
645 645 self.ftp = None
646 646 self.status = 0
647 647
648 648 try:
649 649 self.ftp = ftplib.FTP(self.server)
650 650 self.ftp.login(self.username,self.password)
651 651 self.ftp.cwd(self.remotefolder)
652 652 # print 'Connect to FTP Server: Successfully'
653 653
654 654 except ftplib.all_errors:
655 655 print 'Error FTP Service'
656 656 self.status = 1
657 657 return
658 658
659 659
660 660
661 661 self.dirList = []
662 662
663 663 try:
664 664 self.dirList = self.ftp.nlst()
665 665
666 666 except ftplib.error_perm, resp:
667 667 if str(resp) == "550 No files found":
668 668 print "no files in this directory"
669 669 self.status = 1
670 670 return
671 671
672 672 except ftplib.all_errors:
673 673 print 'Error Displaying Dir-Files'
674 674 self.status = 1
675 675 return
676 676
677 677 self.fileList = []
678 678 self.folderList = []
679 679 #only for test
680 680 for f in self.dirList:
681 681 name, ext = os.path.splitext(f)
682 682 if ext != '':
683 683 self.fileList.append(f)
684 684 # print 'filename: %s - size: %d'%(f,self.ftp.size(f))
685 685
686 686 def parmsByDefault(self):
687 687 server = 'jro-app.igp.gob.pe'
688 688 username = 'wmaster'
689 689 password = 'mst2010vhf'
690 690 remotefolder = '/home/wmaster/graficos'
691 691
692 692 return server, username, password, remotefolder
693 693
694 694
695 695 def mkd(self,dirname):
696 696 """
697 697 mkd is used to make directory in remote server
698 698
699 699 Input:
700 700 dirname - directory name
701 701
702 702 Return:
703 703 1 in error case else 0
704 704 """
705 705 try:
706 706 self.ftp.mkd(dirname)
707 707 except:
708 708 print 'Error creating remote folder:%s'%dirname
709 709 return 1
710 710
711 711 return 0
712 712
713 713
714 714 def delete(self,filename):
715 715 """
716 716 delete is used to delete file in current working directory of remote server
717 717
718 718 Input:
719 719 filename - filename to delete in remote folder
720 720
721 721 Return:
722 722 1 in error case else 0
723 723 """
724 724
725 725 try:
726 726 self.ftp.delete(filename)
727 727 except:
728 728 print 'Error deleting remote file:%s'%filename
729 729 return 1
730 730
731 731 return 0
732 732
733 733 def download(self,filename,localfolder):
734 734 """
735 735 download is used to downloading file from remote folder into local folder
736 736
737 737 Inputs:
738 738 filename - filename to donwload
739 739
740 740 localfolder - directory local to store filename
741 741
742 742 Returns:
743 743 self.status - 1 in error case else 0
744 744 """
745 745
746 746 self.status = 0
747 747
748 748
749 749 if not(filename in self.fileList):
750 750 print 'filename:%s not exists'%filename
751 751 self.status = 1
752 752 return self.status
753 753
754 754 newfilename = os.path.join(localfolder,filename)
755 755
756 756 self.file = open(newfilename, 'wb')
757 757
758 758 try:
759 759 print 'Download: ' + filename
760 760 self.ftp.retrbinary('RETR ' + filename, self.__handleDownload)
761 761 print 'Download Complete'
762 762 except ftplib.all_errors:
763 763 print 'Error Downloading ' + filename
764 764 self.status = 1
765 765 return self.status
766 766
767 767 self.file.close()
768 768
769 769 return self.status
770 770
771 771
772 772 def __handleDownload(self,block):
773 773 """
774 774 __handleDownload is used to handle writing file
775 775 """
776 776 self.file.write(block)
777 777
778 778
779 779 def upload(self,filename,remotefolder=None):
780 780 """
781 781 upload is used to uploading local file to remote directory
782 782
783 783 Inputs:
784 784 filename - full path name of local file to store in remote directory
785 785
786 786 remotefolder - remote directory
787 787
788 788 Returns:
789 789 self.status - 1 in error case else 0
790 790 """
791 791
792 792 if remotefolder == None:
793 793 remotefolder = self.remotefolder
794 794
795 795 self.status = 0
796 796
797 797 try:
798 798 self.ftp.cwd(remotefolder)
799 799
800 800 self.file = open(filename, 'rb')
801 801
802 802 (head, tail) = os.path.split(filename)
803 803
804 804 command = "STOR " + tail
805 805
806 806 print 'Uploading: ' + tail
807 807 self.ftp.storbinary(command, self.file)
808 808 print 'Upload Completed'
809 809
810 810 except ftplib.all_errors:
811 811 print 'Error Uploading ' + tail
812 812 self.status = 1
813 813 return self.status
814 814
815 815 self.file.close()
816 816
817 817 #back to initial directory in __init__()
818 818 self.ftp.cwd(self.remotefolder)
819 819
820 820 return self.status
821 821
822 822
823 823 def dir(self,remotefolder):
824 824 """
825 825 dir is used to change working directory of remote server and get folder and file list
826 826
827 827 Input:
828 828 remotefolder - current working directory
829 829
830 830 Affects:
831 831 self.fileList - file list of working directory
832 832
833 833 Return:
834 834 infoList - list with filenames and size of file in bytes
835 835
836 836 self.folderList - folder list
837 837 """
838 838
839 839 self.remotefolder = remotefolder
840 840 print 'Change to ' + self.remotefolder
841 841 try:
842 842 self.ftp.cwd(remotefolder)
843 843 except ftplib.all_errors:
844 844 print 'Error Change to ' + self.remotefolder
845 845 infoList = None
846 846 self.folderList = None
847 847 return infoList,self.folderList
848 848
849 849 self.dirList = []
850 850
851 851 try:
852 852 self.dirList = self.ftp.nlst()
853 853
854 854 except ftplib.error_perm, resp:
855 855 if str(resp) == "550 No files found":
856 856 print "no files in this directory"
857 857 infoList = None
858 858 self.folderList = None
859 859 return infoList,self.folderList
860 860 except ftplib.all_errors:
861 861 print 'Error Displaying Dir-Files'
862 862 infoList = None
863 863 self.folderList = None
864 864 return infoList,self.folderList
865 865
866 866 infoList = []
867 867 self.fileList = []
868 868 self.folderList = []
869 869 for f in self.dirList:
870 870 name,ext = os.path.splitext(f)
871 871 if ext != '':
872 872 self.fileList.append(f)
873 873 value = (f,self.ftp.size(f))
874 874 infoList.append(value)
875 875
876 876 if ext == '':
877 877 self.folderList.append(f)
878 878
879 879 return infoList,self.folderList
880 880
881 881
882 882 def close(self):
883 883 """
884 884 close is used to close and end FTP connection
885 885
886 886 Inputs: None
887 887
888 888 Return: void
889 889
890 890 """
891 891 self.ftp.close()
892 892
893 893 class SendByFTP(Operation):
894 894
895 895 def __init__(self):
896 896
897 897 self.status = 1
898 898 self.counter = 0
899 899
900 900 def error_print(self, ValueError):
901 901
902 902 print ValueError, 'Error FTP'
903 903 print "don't worry the program is running..."
904 904
905 905 def worker_ftp(self, server, username, password, remotefolder, filenameList):
906 906
907 907 self.ftpClientObj = FTP(server, username, password, remotefolder)
908 908 for filename in filenameList:
909 909 self.ftpClientObj.upload(filename)
910 910 self.ftpClientObj.close()
911 911
912 912 def ftp_thread(self, server, username, password, remotefolder):
913 913 if not(self.status):
914 914 return
915 915
916 916 p = multiprocessing.Process(target=self.worker_ftp, args=(server, username, password, remotefolder, self.filenameList,))
917 917 p.start()
918 918
919 919 p.join(3)
920 920
921 921 if p.is_alive():
922 922 p.terminate()
923 923 p.join()
924 924 print 'killing ftp process...'
925 925 self.status = 0
926 926 return
927 927
928 928 self.status = 1
929 929 return
930 930
931 931 def filterByExt(self, ext, localfolder):
932 932 fnameList = glob.glob1(localfolder,ext)
933 933 self.filenameList = [os.path.join(localfolder,x) for x in fnameList]
934 934
935 935 if len(self.filenameList) == 0:
936 936 self.status = 0
937 937
938 938 def run(self, dataOut, ext, localfolder, remotefolder, server, username, password, period=1):
939 939
940 940 self.counter += 1
941 941 if self.counter >= period:
942 942 self.filterByExt(ext, localfolder)
943 943
944 944 self.ftp_thread(server, username, password, remotefolder)
945 945
946 946 self.counter = 0
947 947
948 948 self.status = 1
949 949
General Comments 0
You need to be logged in to leave comments. Login now