##// END OF EJS Templates
schain
RSS
this fixes bug #1074 and bug #1101
José Chávez -
r1011:6db081ca94fe
parent child
Show More
Side by Side Unified
Context file:
r1011:6db081ca94fe Loading diff...:
Show file before | Show file after | Hide comments
@@ -0,0 +1,69
1 {
2 "configurations": [
3 {
4 "name": "Mac",
5 "includePath": [
6 "/usr/include",
7 "/usr/local/include",
8 "${workspaceRoot}"
9 ],
10 "defines": [],
11 "intelliSenseMode": "clang-x64",
12 "browse": {
13 "path": [
14 "/usr/include",
15 "/usr/local/include",
16 "${workspaceRoot}"
17 ],
18 "limitSymbolsToIncludedHeaders": true,
19 "databaseFilename": ""
20 }
21 },
22 {
23 "name": "Linux",
24 "includePath": [
25 "/usr/include/c++/4.8.4",
26 "/usr/include/x86_64-linux-gnu/c++/4.8",
27 "/usr/local/include",
28 "/usr/include",
29 "/usr/include/x86_64-linux-gnu",
30 "${workspaceRoot}"
31 ],
32 "defines": [],
33 "intelliSenseMode": "clang-x64",
34 "browse": {
35 "path": [
36 "/usr/include/c++/4.8.4",
37 "/usr/include/x86_64-linux-gnu/c++/4.8",
38 "/usr/local/include",
39 "/usr/include",
40 "/usr/include/x86_64-linux-gnu",
41 "${workspaceRoot}"
42 ],
43 "limitSymbolsToIncludedHeaders": true,
44 "databaseFilename": ""
45 }
46 },
47 {
48 "name": "Win32",
49 "includePath": [
50 "C:/Program Files (x86)/Microsoft Visual Studio 14.0/VC/include",
51 "${workspaceRoot}"
52 ],
53 "defines": [
54 "_DEBUG",
55 "UNICODE"
56 ],
57 "intelliSenseMode": "msvc-x64",
58 "browse": {
59 "path": [
60 "C:/Program Files (x86)/Microsoft Visual Studio 14.0/VC/include/*",
61 "${workspaceRoot}"
62 ],
63 "limitSymbolsToIncludedHeaders": true,
64 "databaseFilename": ""
65 }
66 }
67 ],
68 "version": 2
69 } No newline at end of file
Show file before | Show file after | Hide comments
@@ -0,0 +1,7
1 {
2 "files.associations": {
3 "arrayobject.h": "c",
4 "ndarrayobject.h": "c",
5 "complex": "c"
6 }
7 } No newline at end of file
Show file before | Show file after | Hide comments
@@ -1,13 +1,108
1 .project
2 .pydevproject
3 build/
4 *.pyc
1 # Byte-compiled / optimized / DLL files
2 __pycache__/
3 *.py[cod]
4 *$py.class
5
6 # C extensions
5 7 *.so
8
9 # Distribution / packaging
10 .Python
11 env/
12 build/
13 develop-eggs/
14 dist/
15 downloads/
16 eggs/
17 .eggs/
18 lib/
19 lib64/
20 parts/
21 sdist/
22 var/
23 wheels/
6 24 *.egg-info/
25 .installed.cfg
26 *.egg
27
28 # PyInstaller
29 # Usually these files are written by a python script from a template
30 # before PyInstaller builds the exe, so as to inject date/other infos into it.
31 *.manifest
32 *.spec
33
34 # Installer logs
35 pip-log.txt
36 pip-delete-this-directory.txt
37
38 # Unit test / coverage reports
39 htmlcov/
40 .tox/
41 .coverage
42 .coverage.*
43 .cache
44 nosetests.xml
45 coverage.xml
46 *,cover
47 .hypothesis/
48
49 # Translations
50 *.mo
51 *.pot
52
53 # Django stuff:
54 *.log
55 local_settings.py
56
57 # Flask stuff:
58 instance/
59 .webassets-cache
60
61 # Scrapy stuff:
62 .scrapy
63
64 # Sphinx documentation
65 docs/_build/
66
67 # PyBuilder
68 target/
69
70 # Jupyter Notebook
71 .ipynb_checkpoints
72
73 # pyenv
74 .python-version
75
76 # celery beat schedule file
77 celerybeat-schedule
78
79 # SageMath parsed files
80 *.sage.py
81
82 # dotenv
83 .env
84
85 # virtualenv
86 .venv
87 venv/
88 ENV/
89
90 # Spyder project settings
91 .spyderproject
92 .spyproject
93
94 # Rope project settings
95 .ropeproject
96
97 # mkdocs documentation
98 /site
7 99
8 100 # eclipse
9 101 .project
10 102 .pydevproject
103 .svn/
104 *.png
105 *.pyc
106 schainpy/scripts
11 107
12 # vscode
13 .vscode No newline at end of file
108 schaingui/node_modules No newline at end of file
Show file before | Show file after | Hide comments
@@ -1,1294 +1,1294
1 1 '''
2 2 Created on September , 2012
3 3 @author:
4 4 '''
5 5
6 6 import sys
7 7 import ast
8 8 import datetime
9 9 import traceback
10 10 import schainpy
11 11 import schainpy.admin
12 12
13 13 from xml.etree.ElementTree import ElementTree, Element, SubElement, tostring
14 14 from xml.dom import minidom
15 15
16 16 from schainpy.model import *
17 17 from time import sleep
18 18
19 19 def prettify(elem):
20 20 """Return a pretty-printed XML string for the Element.
21 21 """
22 22 rough_string = tostring(elem, 'utf-8')
23 23 reparsed = minidom.parseString(rough_string)
24 24 return reparsed.toprettyxml(indent=" ")
25 25
26 26 class ParameterConf():
27 27
28 28 id = None
29 29 name = None
30 30 value = None
31 31 format = None
32 32
33 33 __formated_value = None
34 34
35 35 ELEMENTNAME = 'Parameter'
36 36
37 37 def __init__(self):
38 38
39 39 self.format = 'str'
40 40
41 41 def getElementName(self):
42 42
43 43 return self.ELEMENTNAME
44 44
45 45 def getValue(self):
46 46
47 47 value = self.value
48 48 format = self.format
49 49
50 50 if self.__formated_value != None:
51 51
52 52 return self.__formated_value
53 53
54 54 if format == 'str':
55 55 self.__formated_value = str(value)
56 56 return self.__formated_value
57 57
58 58 if value == '':
59 59 raise ValueError, "%s: This parameter value is empty" %self.name
60 60
61 61 if format == 'list':
62 62 strList = value.split(',')
63 63
64 64 self.__formated_value = strList
65 65
66 66 return self.__formated_value
67 67
68 68 if format == 'intlist':
69 69 """
70 70 Example:
71 71 value = (0,1,2)
72 72 """
73 73
74 74 new_value = ast.literal_eval(value)
75 75
76 76 if type(new_value) not in (tuple, list):
77 77 new_value = [int(new_value)]
78 78
79 79 self.__formated_value = new_value
80 80
81 81 return self.__formated_value
82 82
83 83 if format == 'floatlist':
84 84 """
85 85 Example:
86 86 value = (0.5, 1.4, 2.7)
87 87 """
88 88
89 89 new_value = ast.literal_eval(value)
90 90
91 91 if type(new_value) not in (tuple, list):
92 92 new_value = [float(new_value)]
93 93
94 94 self.__formated_value = new_value
95 95
96 96 return self.__formated_value
97 97
98 98 if format == 'date':
99 99 strList = value.split('/')
100 100 intList = [int(x) for x in strList]
101 101 date = datetime.date(intList[0], intList[1], intList[2])
102 102
103 103 self.__formated_value = date
104 104
105 105 return self.__formated_value
106 106
107 107 if format == 'time':
108 108 strList = value.split(':')
109 109 intList = [int(x) for x in strList]
110 110 time = datetime.time(intList[0], intList[1], intList[2])
111 111
112 112 self.__formated_value = time
113 113
114 114 return self.__formated_value
115 115
116 116 if format == 'pairslist':
117 117 """
118 118 Example:
119 119 value = (0,1),(1,2)
120 120 """
121 121
122 122 new_value = ast.literal_eval(value)
123 123
124 124 if type(new_value) not in (tuple, list):
125 125 raise ValueError, "%s has to be a tuple or list of pairs" %value
126 126
127 127 if type(new_value[0]) not in (tuple, list):
128 128 if len(new_value) != 2:
129 129 raise ValueError, "%s has to be a tuple or list of pairs" %value
130 130 new_value = [new_value]
131 131
132 132 for thisPair in new_value:
133 133 if len(thisPair) != 2:
134 134 raise ValueError, "%s has to be a tuple or list of pairs" %value
135 135
136 136 self.__formated_value = new_value
137 137
138 138 return self.__formated_value
139 139
140 140 if format == 'multilist':
141 141 """
142 142 Example:
143 143 value = (0,1,2),(3,4,5)
144 144 """
145 145 multiList = ast.literal_eval(value)
146 146
147 147 if type(multiList[0]) == int:
148 148 multiList = ast.literal_eval("(" + value + ")")
149 149
150 150 self.__formated_value = multiList
151 151
152 152 return self.__formated_value
153 153
154 154 if format == 'bool':
155 155 value = int(value)
156 156
157 157 if format == 'int':
158 158 value = float(value)
159 159
160 160 format_func = eval(format)
161 161
162 162 self.__formated_value = format_func(value)
163 163
164 164 return self.__formated_value
165 165
166 166 def updateId(self, new_id):
167 167
168 168 self.id = str(new_id)
169 169
170 170 def setup(self, id, name, value, format='str'):
171 171
172 172 self.id = str(id)
173 173 self.name = name
174 174 self.value = str(value)
175 175 self.format = str.lower(format)
176 176
177 177 self.getValue()
178 178
179 179 return 1
180 180
181 181 def update(self, name, value, format='str'):
182 182
183 183 self.name = name
184 184 self.value = str(value)
185 185 self.format = format
186 186
187 187 def makeXml(self, opElement):
188 188
189 189 parmElement = SubElement(opElement, self.ELEMENTNAME)
190 190 parmElement.set('id', str(self.id))
191 191 parmElement.set('name', self.name)
192 192 parmElement.set('value', self.value)
193 193 parmElement.set('format', self.format)
194 194
195 195 def readXml(self, parmElement):
196 196
197 197 self.id = parmElement.get('id')
198 198 self.name = parmElement.get('name')
199 199 self.value = parmElement.get('value')
200 200 self.format = str.lower(parmElement.get('format'))
201 201
202 202 #Compatible with old signal chain version
203 203 if self.format == 'int' and self.name == 'idfigure':
204 204 self.name = 'id'
205 205
206 206 def printattr(self):
207 207
208 208 print "Parameter[%s]: name = %s, value = %s, format = %s" %(self.id, self.name, self.value, self.format)
209 209
210 210 class OperationConf():
211 211
212 212 id = None
213 213 name = None
214 214 priority = None
215 215 type = None
216 216
217 217 parmConfObjList = []
218 218
219 219 ELEMENTNAME = 'Operation'
220 220
221 221 def __init__(self):
222 222
223 223 self.id = '0'
224 224 self.name = None
225 225 self.priority = None
226 226 self.type = 'self'
227 227
228 228
229 229 def __getNewId(self):
230 230
231 231 return int(self.id)*10 + len(self.parmConfObjList) + 1
232 232
233 233 def updateId(self, new_id):
234 234
235 235 self.id = str(new_id)
236 236
237 237 n = 1
238 238 for parmObj in self.parmConfObjList:
239 239
240 240 idParm = str(int(new_id)*10 + n)
241 241 parmObj.updateId(idParm)
242 242
243 243 n += 1
244 244
245 245 def getElementName(self):
246 246
247 247 return self.ELEMENTNAME
248 248
249 249 def getParameterObjList(self):
250 250
251 251 return self.parmConfObjList
252 252
253 253 def getParameterObj(self, parameterName):
254 254
255 255 for parmConfObj in self.parmConfObjList:
256 256
257 257 if parmConfObj.name != parameterName:
258 258 continue
259 259
260 260 return parmConfObj
261 261
262 262 return None
263 263
264 264 def getParameterObjfromValue(self, parameterValue):
265 265
266 266 for parmConfObj in self.parmConfObjList:
267 267
268 268 if parmConfObj.getValue() != parameterValue:
269 269 continue
270 270
271 271 return parmConfObj.getValue()
272 272
273 273 return None
274 274
275 275 def getParameterValue(self, parameterName):
276 276
277 277 parameterObj = self.getParameterObj(parameterName)
278 278
279 # if not parameterObj:
280 # return None
279 # if not parameterObj:
280 # return None
281 281
282 282 value = parameterObj.getValue()
283 283
284 284 return value
285 285
286 286 def setup(self, id, name, priority, type):
287 287
288 288 self.id = str(id)
289 289 self.name = name
290 290 self.type = type
291 291 self.priority = priority
292 292
293 293 self.parmConfObjList = []
294 294
295 295 def removeParameters(self):
296 296
297 297 for obj in self.parmConfObjList:
298 298 del obj
299 299
300 300 self.parmConfObjList = []
301 301
302 302 def addParameter(self, name, value, format='str'):
303 303
304 304 id = self.__getNewId()
305 305
306 306 parmConfObj = ParameterConf()
307 307 if not parmConfObj.setup(id, name, value, format):
308 308 return None
309 309
310 310 self.parmConfObjList.append(parmConfObj)
311 311
312 312 return parmConfObj
313 313
314 314 def changeParameter(self, name, value, format='str'):
315 315
316 316 parmConfObj = self.getParameterObj(name)
317 317 parmConfObj.update(name, value, format)
318 318
319 319 return parmConfObj
320 320
321 321 def makeXml(self, procUnitElement):
322 322
323 323 opElement = SubElement(procUnitElement, self.ELEMENTNAME)
324 324 opElement.set('id', str(self.id))
325 325 opElement.set('name', self.name)
326 326 opElement.set('type', self.type)
327 327 opElement.set('priority', str(self.priority))
328 328
329 329 for parmConfObj in self.parmConfObjList:
330 330 parmConfObj.makeXml(opElement)
331 331
332 332 def readXml(self, opElement):
333 333
334 334 self.id = opElement.get('id')
335 335 self.name = opElement.get('name')
336 336 self.type = opElement.get('type')
337 337 self.priority = opElement.get('priority')
338 338
339 339 #Compatible with old signal chain version
340 340 #Use of 'run' method instead 'init'
341 341 if self.type == 'self' and self.name == 'init':
342 342 self.name = 'run'
343 343
344 344 self.parmConfObjList = []
345 345
346 346 parmElementList = opElement.iter(ParameterConf().getElementName())
347 347
348 348 for parmElement in parmElementList:
349 349 parmConfObj = ParameterConf()
350 350 parmConfObj.readXml(parmElement)
351 351
352 352 #Compatible with old signal chain version
353 353 #If an 'plot' OPERATION is found, changes name operation by the value of its type PARAMETER
354 354 if self.type != 'self' and self.name == 'Plot':
355 355 if parmConfObj.format == 'str' and parmConfObj.name == 'type':
356 356 self.name = parmConfObj.value
357 357 continue
358 358
359 359 self.parmConfObjList.append(parmConfObj)
360 360
361 361 def printattr(self):
362 362
363 363 print "%s[%s]: name = %s, type = %s, priority = %s" %(self.ELEMENTNAME,
364 364 self.id,
365 365 self.name,
366 366 self.type,
367 367 self.priority)
368 368
369 369 for parmConfObj in self.parmConfObjList:
370 370 parmConfObj.printattr()
371 371
372 372 def createObject(self, plotter_queue=None):
373 373
374 374 if self.type == 'self':
375 375 raise ValueError, "This operation type cannot be created"
376 376
377 377 if self.type == 'plotter':
378 378 #Plotter(plotter_name)
379 379 if not plotter_queue:
380 380 raise ValueError, "plotter_queue is not defined. Use:\nmyProject = Project()\nmyProject.setPlotterQueue(plotter_queue)"
381 381
382 382 opObj = Plotter(self.name, plotter_queue)
383 383
384 384 if self.type == 'external' or self.type == 'other':
385 385 className = eval(self.name)
386 386 opObj = className()
387 387
388 388 return opObj
389 389
390 390 class ProcUnitConf():
391 391
392 392 id = None
393 393 name = None
394 394 datatype = None
395 395 inputId = None
396 396 parentId = None
397 397
398 398 opConfObjList = []
399 399
400 400 procUnitObj = None
401 401 opObjList = []
402 402
403 403 ELEMENTNAME = 'ProcUnit'
404 404
405 405 def __init__(self):
406 406
407 407 self.id = None
408 408 self.datatype = None
409 409 self.name = None
410 410 self.inputId = None
411 411
412 412 self.opConfObjList = []
413 413
414 414 self.procUnitObj = None
415 415 self.opObjDict = {}
416 416
417 417 def __getPriority(self):
418 418
419 419 return len(self.opConfObjList)+1
420 420
421 421 def __getNewId(self):
422 422
423 423 return int(self.id)*10 + len(self.opConfObjList) + 1
424 424
425 425 def getElementName(self):
426 426
427 427 return self.ELEMENTNAME
428 428
429 429 def getId(self):
430 430
431 431 return self.id
432 432
433 433 def updateId(self, new_id, parentId=parentId):
434 434
435 435
436 436 new_id = int(parentId)*10 + (int(self.id) % 10)
437 437 new_inputId = int(parentId)*10 + (int(self.inputId) % 10)
438 438
439 439 #If this proc unit has not inputs
440 440 if self.inputId == '0':
441 441 new_inputId = 0
442 442
443 443 n = 1
444 444 for opConfObj in self.opConfObjList:
445 445
446 446 idOp = str(int(new_id)*10 + n)
447 447 opConfObj.updateId(idOp)
448 448
449 449 n += 1
450 450
451 451 self.parentId = str(parentId)
452 452 self.id = str(new_id)
453 453 self.inputId = str(new_inputId)
454 454
455 455
456 456 def getInputId(self):
457 457
458 458 return self.inputId
459 459
460 460 def getOperationObjList(self):
461 461
462 462 return self.opConfObjList
463 463
464 464 def getOperationObj(self, name=None):
465 465
466 466 for opConfObj in self.opConfObjList:
467 467
468 468 if opConfObj.name != name:
469 469 continue
470 470
471 471 return opConfObj
472 472
473 473 return None
474 474
475 475 def getOpObjfromParamValue(self, value=None):
476 476
477 477 for opConfObj in self.opConfObjList:
478 478 if opConfObj.getParameterObjfromValue(parameterValue=value) != value:
479 479 continue
480 480 return opConfObj
481 481 return None
482 482
483 483 def getProcUnitObj(self):
484 484
485 485 return self.procUnitObj
486 486
487 487 def setup(self, id, name, datatype, inputId, parentId=None):
488 488
489 489 #Compatible with old signal chain version
490 490 if datatype==None and name==None:
491 491 raise ValueError, "datatype or name should be defined"
492 492
493 493 if name==None:
494 494 if 'Proc' in datatype:
495 495 name = datatype
496 496 else:
497 497 name = '%sProc' %(datatype)
498 498
499 499 if datatype==None:
500 500 datatype = name.replace('Proc','')
501 501
502 502 self.id = str(id)
503 503 self.name = name
504 504 self.datatype = datatype
505 505 self.inputId = inputId
506 506 self.parentId = parentId
507 507
508 508 self.opConfObjList = []
509 509
510 510 self.addOperation(name='run', optype='self')
511 511
512 512 def removeOperations(self):
513 513
514 514 for obj in self.opConfObjList:
515 515 del obj
516 516
517 517 self.opConfObjList = []
518 518 self.addOperation(name='run')
519 519
520 520 def addParameter(self, **kwargs):
521 521 '''
522 522 Add parameters to "run" operation
523 523 '''
524 524 opObj = self.opConfObjList[0]
525 525
526 526 opObj.addParameter(**kwargs)
527 527
528 528 return opObj
529 529
530 530 def addOperation(self, name, optype='self'):
531 531
532 532 id = self.__getNewId()
533 533 priority = self.__getPriority()
534 534
535 535 opConfObj = OperationConf()
536 536 opConfObj.setup(id, name=name, priority=priority, type=optype)
537 537
538 538 self.opConfObjList.append(opConfObj)
539 539
540 540 return opConfObj
541 541
542 542 def makeXml(self, projectElement):
543 543
544 544 procUnitElement = SubElement(projectElement, self.ELEMENTNAME)
545 545 procUnitElement.set('id', str(self.id))
546 546 procUnitElement.set('name', self.name)
547 547 procUnitElement.set('datatype', self.datatype)
548 548 procUnitElement.set('inputId', str(self.inputId))
549 549
550 550 for opConfObj in self.opConfObjList:
551 551 opConfObj.makeXml(procUnitElement)
552 552
553 553 def readXml(self, upElement):
554 554
555 555 self.id = upElement.get('id')
556 556 self.name = upElement.get('name')
557 557 self.datatype = upElement.get('datatype')
558 558 self.inputId = upElement.get('inputId')
559 559
560 560 if self.ELEMENTNAME == "ReadUnit":
561 561 self.datatype = self.datatype.replace("Reader", "")
562 562
563 563 if self.ELEMENTNAME == "ProcUnit":
564 564 self.datatype = self.datatype.replace("Proc", "")
565 565
566 566 if self.inputId == 'None':
567 567 self.inputId = '0'
568 568
569 569 self.opConfObjList = []
570 570
571 571 opElementList = upElement.iter(OperationConf().getElementName())
572 572
573 573 for opElement in opElementList:
574 574 opConfObj = OperationConf()
575 575 opConfObj.readXml(opElement)
576 576 self.opConfObjList.append(opConfObj)
577 577
578 578 def printattr(self):
579 579
580 580 print "%s[%s]: name = %s, datatype = %s, inputId = %s" %(self.ELEMENTNAME,
581 self.id,
582 self.name,
583 self.datatype,
584 self.inputId)
581 self.id,
582 self.name,
583 self.datatype,
584 self.inputId)
585 585
586 586 for opConfObj in self.opConfObjList:
587 587 opConfObj.printattr()
588 588
589 589 def createObjects(self, plotter_queue=None):
590 590
591 591 className = eval(self.name)
592 592 procUnitObj = className()
593 593
594 594 for opConfObj in self.opConfObjList:
595 595
596 596 if opConfObj.type == 'self':
597 597 continue
598 598
599 599 opObj = opConfObj.createObject(plotter_queue)
600 600
601 601 self.opObjDict[opConfObj.id] = opObj
602 602 procUnitObj.addOperation(opObj, opConfObj.id)
603 603
604 604 self.procUnitObj = procUnitObj
605 605
606 606 return procUnitObj
607 607
608 608 def run(self):
609 609
610 610 is_ok = False
611 611
612 612 for opConfObj in self.opConfObjList:
613 613
614 614 kwargs = {}
615 615 for parmConfObj in opConfObj.getParameterObjList():
616 616 if opConfObj.name == 'run' and parmConfObj.name == 'datatype':
617 617 continue
618 618
619 619 kwargs[parmConfObj.name] = parmConfObj.getValue()
620 620
621 621 #ini = time.time()
622 622
623 623 #print "\tRunning the '%s' operation with %s" %(opConfObj.name, opConfObj.id)
624 624 sts = self.procUnitObj.call(opType = opConfObj.type,
625 625 opName = opConfObj.name,
626 626 opId = opConfObj.id,
627 **kwargs)
628
629 # total_time = time.time() - ini
630 #
631 # if total_time > 0.002:
632 # print "%s::%s took %f seconds" %(self.name, opConfObj.name, total_time)
627 **kwargs)
633 628
629 # total_time = time.time() - ini
630 #
631 # if total_time > 0.002:
632 # print "%s::%s took %f seconds" %(self.name, opConfObj.name, total_time)
633
634 634 is_ok = is_ok or sts
635 635
636 636 return is_ok
637 637
638 638 def close(self):
639 639
640 640 for opConfObj in self.opConfObjList:
641 641 if opConfObj.type == 'self':
642 642 continue
643 643
644 644 opObj = self.procUnitObj.getOperationObj(opConfObj.id)
645 645 opObj.close()
646 646
647 647 self.procUnitObj.close()
648 648
649 649 return
650
650
651 651 class ReadUnitConf(ProcUnitConf):
652 652
653 653 path = None
654 654 startDate = None
655 655 endDate = None
656 656 startTime = None
657 657 endTime = None
658 658
659 659 ELEMENTNAME = 'ReadUnit'
660 660
661 661 def __init__(self):
662 662
663 663 self.id = None
664 664 self.datatype = None
665 665 self.name = None
666 666 self.inputId = None
667 667
668 668 self.parentId = None
669 669
670 670 self.opConfObjList = []
671 671 self.opObjList = []
672 672
673 673 def getElementName(self):
674 674
675 675 return self.ELEMENTNAME
676 676
677 677 def setup(self, id, name, datatype, path, startDate="", endDate="", startTime="", endTime="", parentId=None, **kwargs):
678 678
679 679 #Compatible with old signal chain version
680 680 if datatype==None and name==None:
681 681 raise ValueError, "datatype or name should be defined"
682 682
683 683 if name==None:
684 684 if 'Reader' in datatype:
685 685 name = datatype
686 686 else:
687 687 name = '%sReader' %(datatype)
688 688
689 689 if datatype==None:
690 690 datatype = name.replace('Reader','')
691 691
692 692 self.id = id
693 693 self.name = name
694 694 self.datatype = datatype
695 695
696 696 self.path = os.path.abspath(path)
697 697 self.startDate = startDate
698 698 self.endDate = endDate
699 699 self.startTime = startTime
700 700 self.endTime = endTime
701 701
702 702 self.inputId = '0'
703 703 self.parentId = parentId
704 704
705 705 self.addRunOperation(**kwargs)
706 706
707 707 def update(self, datatype, path, startDate, endDate, startTime, endTime, parentId=None, name=None, **kwargs):
708 708
709 709 #Compatible with old signal chain version
710 710 if datatype==None and name==None:
711 711 raise ValueError, "datatype or name should be defined"
712 712
713 713 if name==None:
714 714 if 'Reader' in datatype:
715 715 name = datatype
716 716 else:
717 717 name = '%sReader' %(datatype)
718 718
719 719 if datatype==None:
720 720 datatype = name.replace('Reader','')
721 721
722 722 self.datatype = datatype
723 723 self.name = name
724 724 self.path = path
725 725 self.startDate = startDate
726 726 self.endDate = endDate
727 727 self.startTime = startTime
728 728 self.endTime = endTime
729 729
730 730 self.inputId = '0'
731 731 self.parentId = parentId
732 732
733 733 self.updateRunOperation(**kwargs)
734 734
735 735 def removeOperations(self):
736 736
737 737 for obj in self.opConfObjList:
738 738 del obj
739 739
740 740 self.opConfObjList = []
741 741
742 742 def addRunOperation(self, **kwargs):
743 743
744 744 opObj = self.addOperation(name = 'run', optype = 'self')
745 745
746 746 opObj.addParameter(name='datatype' , value=self.datatype, format='str')
747 747 opObj.addParameter(name='path' , value=self.path, format='str')
748 748 opObj.addParameter(name='startDate' , value=self.startDate, format='date')
749 749 opObj.addParameter(name='endDate' , value=self.endDate, format='date')
750 750 opObj.addParameter(name='startTime' , value=self.startTime, format='time')
751 751 opObj.addParameter(name='endTime' , value=self.endTime, format='time')
752 752
753 753 for key, value in kwargs.items():
754 754 opObj.addParameter(name=key, value=value, format=type(value).__name__)
755 755
756 756 return opObj
757 757
758 758 def updateRunOperation(self, **kwargs):
759 759
760 760 opObj = self.getOperationObj(name = 'run')
761 761 opObj.removeParameters()
762 762
763 763 opObj.addParameter(name='datatype' , value=self.datatype, format='str')
764 764 opObj.addParameter(name='path' , value=self.path, format='str')
765 765 opObj.addParameter(name='startDate' , value=self.startDate, format='date')
766 766 opObj.addParameter(name='endDate' , value=self.endDate, format='date')
767 767 opObj.addParameter(name='startTime' , value=self.startTime, format='time')
768 768 opObj.addParameter(name='endTime' , value=self.endTime, format='time')
769 769
770 770 for key, value in kwargs.items():
771 771 opObj.addParameter(name=key, value=value, format=type(value).__name__)
772 772
773 773 return opObj
774 774
775 # def makeXml(self, projectElement):
776 #
777 # procUnitElement = SubElement(projectElement, self.ELEMENTNAME)
778 # procUnitElement.set('id', str(self.id))
779 # procUnitElement.set('name', self.name)
780 # procUnitElement.set('datatype', self.datatype)
781 # procUnitElement.set('inputId', str(self.inputId))
782 #
783 # for opConfObj in self.opConfObjList:
784 # opConfObj.makeXml(procUnitElement)
775 # def makeXml(self, projectElement):
776 #
777 # procUnitElement = SubElement(projectElement, self.ELEMENTNAME)
778 # procUnitElement.set('id', str(self.id))
779 # procUnitElement.set('name', self.name)
780 # procUnitElement.set('datatype', self.datatype)
781 # procUnitElement.set('inputId', str(self.inputId))
782 #
783 # for opConfObj in self.opConfObjList:
784 # opConfObj.makeXml(procUnitElement)
785 785
786 786 def readXml(self, upElement):
787 787
788 788 self.id = upElement.get('id')
789 789 self.name = upElement.get('name')
790 790 self.datatype = upElement.get('datatype')
791 791 self.inputId = upElement.get('inputId')
792 792
793 793 if self.ELEMENTNAME == "ReadUnit":
794 794 self.datatype = self.datatype.replace("Reader", "")
795 795
796 796 if self.inputId == 'None':
797 797 self.inputId = '0'
798 798
799 799 self.opConfObjList = []
800 800
801 801 opElementList = upElement.iter(OperationConf().getElementName())
802 802
803 803 for opElement in opElementList:
804 804 opConfObj = OperationConf()
805 805 opConfObj.readXml(opElement)
806 806 self.opConfObjList.append(opConfObj)
807 807
808 808 if opConfObj.name == 'run':
809 809 self.path = opConfObj.getParameterValue('path')
810 810 self.startDate = opConfObj.getParameterValue('startDate')
811 811 self.endDate = opConfObj.getParameterValue('endDate')
812 812 self.startTime = opConfObj.getParameterValue('startTime')
813 813 self.endTime = opConfObj.getParameterValue('endTime')
814 814
815 815 class Project():
816 816
817 817 id = None
818 818 name = None
819 819 description = None
820 820 filename = None
821 821
822 822 procUnitConfObjDict = None
823 823
824 824 ELEMENTNAME = 'Project'
825 825
826 826 plotterQueue = None
827 827
828 828 def __init__(self, plotter_queue=None):
829 829
830 830 self.id = None
831 831 self.name = None
832 832 self.description = None
833 833
834 834 self.plotterQueue = plotter_queue
835 835
836 836 self.procUnitConfObjDict = {}
837 837
838 838 def __getNewId(self):
839 839
840 840 idList = self.procUnitConfObjDict.keys()
841 841
842 842 id = int(self.id)*10
843 843
844 844 while True:
845 845 id += 1
846 846
847 847 if str(id) in idList:
848 848 continue
849 849
850 850 break
851 851
852 852 return str(id)
853 853
854 854 def getElementName(self):
855 855
856 856 return self.ELEMENTNAME
857 857
858 858 def getId(self):
859 859
860 860 return self.id
861 861
862 862 def updateId(self, new_id):
863 863
864 864 self.id = str(new_id)
865 865
866 866 keyList = self.procUnitConfObjDict.keys()
867 867 keyList.sort()
868 868
869 869 n = 1
870 870 newProcUnitConfObjDict = {}
871 871
872 872 for procKey in keyList:
873 873
874 874 procUnitConfObj = self.procUnitConfObjDict[procKey]
875 875 idProcUnit = str(int(self.id)*10 + n)
876 876 procUnitConfObj.updateId(idProcUnit, parentId = self.id)
877 877
878 878 newProcUnitConfObjDict[idProcUnit] = procUnitConfObj
879 879 n += 1
880 880
881 881 self.procUnitConfObjDict = newProcUnitConfObjDict
882 882
883 883 def setup(self, id, name, description):
884 884
885 885 self.id = str(id)
886 886 self.name = name
887 887 self.description = description
888 888
889 889 def update(self, name, description):
890 890
891 891 self.name = name
892 892 self.description = description
893 893
894 894 def addReadUnit(self, id=None, datatype=None, name=None, **kwargs):
895 895
896 896 if id is None:
897 897 idReadUnit = self.__getNewId()
898 898 else:
899 899 idReadUnit = str(id)
900 900
901 901 readUnitConfObj = ReadUnitConf()
902 902 readUnitConfObj.setup(idReadUnit, name, datatype, parentId=self.id, **kwargs)
903 903
904 904 self.procUnitConfObjDict[readUnitConfObj.getId()] = readUnitConfObj
905 905
906 906 return readUnitConfObj
907 907
908 908 def addProcUnit(self, inputId='0', datatype=None, name=None):
909 909
910 910 idProcUnit = self.__getNewId()
911 911
912 912 procUnitConfObj = ProcUnitConf()
913 913 procUnitConfObj.setup(idProcUnit, name, datatype, inputId, parentId=self.id)
914 914
915 915 self.procUnitConfObjDict[procUnitConfObj.getId()] = procUnitConfObj
916 916
917 917 return procUnitConfObj
918 918
919 919 def removeProcUnit(self, id):
920 920
921 921 if id in self.procUnitConfObjDict.keys():
922 922 self.procUnitConfObjDict.pop(id)
923 923
924 924 def getReadUnitId(self):
925 925
926 926 readUnitConfObj = self.getReadUnitObj()
927 927
928 928 return readUnitConfObj.id
929 929
930 930 def getReadUnitObj(self):
931 931
932 932 for obj in self.procUnitConfObjDict.values():
933 933 if obj.getElementName() == "ReadUnit":
934 934 return obj
935 935
936 936 return None
937 937
938 938 def getProcUnitObj(self, id=None, name=None):
939 939
940 940 if id != None:
941 941 return self.procUnitConfObjDict[id]
942 942
943 943 if name != None:
944 944 return self.getProcUnitObjByName(name)
945 945
946 946 return None
947 947
948 948 def getProcUnitObjByName(self, name):
949 949
950 950 for obj in self.procUnitConfObjDict.values():
951 951 if obj.name == name:
952 952 return obj
953 953
954 954 return None
955 955
956 956 def procUnitItems(self):
957 957
958 958 return self.procUnitConfObjDict.items()
959 959
960 960 def makeXml(self):
961 961
962 962 projectElement = Element('Project')
963 963 projectElement.set('id', str(self.id))
964 964 projectElement.set('name', self.name)
965 965 projectElement.set('description', self.description)
966 966
967 967 for procUnitConfObj in self.procUnitConfObjDict.values():
968 968 procUnitConfObj.makeXml(projectElement)
969 969
970 970 self.projectElement = projectElement
971 971
972 972 def writeXml(self, filename=None):
973 973
974 974 if filename == None:
975 975 if self.filename:
976 976 filename = self.filename
977 977 else:
978 978 filename = "schain.xml"
979 979
980 980 if not filename:
981 981 print "filename has not been defined. Use setFilename(filename) for do it."
982 982 return 0
983 983
984 984 abs_file = os.path.abspath(filename)
985 985
986 986 if not os.access(os.path.dirname(abs_file), os.W_OK):
987 987 print "No write permission on %s" %os.path.dirname(abs_file)
988 988 return 0
989 989
990 990 if os.path.isfile(abs_file) and not(os.access(abs_file, os.W_OK)):
991 991 print "File %s already exists and it could not be overwriten" %abs_file
992 992 return 0
993 993
994 994 self.makeXml()
995 995
996 996 ElementTree(self.projectElement).write(abs_file, method='xml')
997 997
998 998 self.filename = abs_file
999 999
1000 1000 return 1
1001 1001
1002 1002 def readXml(self, filename = None):
1003 1003
1004 1004 if not filename:
1005 1005 print "filename is not defined"
1006 1006 return 0
1007 1007
1008 1008 abs_file = os.path.abspath(filename)
1009 1009
1010 1010 if not os.path.isfile(abs_file):
1011 1011 print "%s file does not exist" %abs_file
1012 1012 return 0
1013 1013
1014 1014 self.projectElement = None
1015 1015 self.procUnitConfObjDict = {}
1016 1016
1017 1017 try:
1018 1018 self.projectElement = ElementTree().parse(abs_file)
1019 1019 except:
1020 1020 print "Error reading %s, verify file format" %filename
1021 1021 return 0
1022 1022
1023 1023 self.project = self.projectElement.tag
1024 1024
1025 1025 self.id = self.projectElement.get('id')
1026 1026 self.name = self.projectElement.get('name')
1027 self.description = self.projectElement.get('description')
1027 self.description = self.projectElement.get('description')
1028 1028
1029 1029 readUnitElementList = self.projectElement.iter(ReadUnitConf().getElementName())
1030 1030
1031 1031 for readUnitElement in readUnitElementList:
1032 1032 readUnitConfObj = ReadUnitConf()
1033 1033 readUnitConfObj.readXml(readUnitElement)
1034 1034
1035 1035 if readUnitConfObj.parentId == None:
1036 1036 readUnitConfObj.parentId = self.id
1037 1037
1038 1038 self.procUnitConfObjDict[readUnitConfObj.getId()] = readUnitConfObj
1039 1039
1040 1040 procUnitElementList = self.projectElement.iter(ProcUnitConf().getElementName())
1041 1041
1042 1042 for procUnitElement in procUnitElementList:
1043 1043 procUnitConfObj = ProcUnitConf()
1044 1044 procUnitConfObj.readXml(procUnitElement)
1045 1045
1046 1046 if procUnitConfObj.parentId == None:
1047 1047 procUnitConfObj.parentId = self.id
1048 1048
1049 1049 self.procUnitConfObjDict[procUnitConfObj.getId()] = procUnitConfObj
1050 1050
1051 1051 self.filename = abs_file
1052 1052
1053 1053 return 1
1054 1054
1055 1055 def printattr(self):
1056 1056
1057 1057 print "Project[%s]: name = %s, description = %s" %(self.id,
1058 self.name,
1059 self.description)
1058 self.name,
1059 self.description)
1060 1060
1061 1061 for procUnitConfObj in self.procUnitConfObjDict.values():
1062 1062 procUnitConfObj.printattr()
1063 1063
1064 1064 def createObjects(self):
1065 1065
1066 1066 for procUnitConfObj in self.procUnitConfObjDict.values():
1067 1067 procUnitConfObj.createObjects(self.plotterQueue)
1068 1068
1069 1069 def __connect(self, objIN, thisObj):
1070 1070
1071 1071 thisObj.setInput(objIN.getOutputObj())
1072 1072
1073 1073 def connectObjects(self):
1074 1074
1075 1075 for thisPUConfObj in self.procUnitConfObjDict.values():
1076 1076
1077 1077 inputId = thisPUConfObj.getInputId()
1078 1078
1079 1079 if int(inputId) == 0:
1080 1080 continue
1081 1081
1082 1082 #Get input object
1083 1083 puConfINObj = self.procUnitConfObjDict[inputId]
1084 1084 puObjIN = puConfINObj.getProcUnitObj()
1085 1085
1086 1086 #Get current object
1087 1087 thisPUObj = thisPUConfObj.getProcUnitObj()
1088 1088
1089 1089 self.__connect(puObjIN, thisPUObj)
1090 1090
1091 1091 def __handleError(self, procUnitConfObj, send_email=True):
1092 1092
1093 1093 import socket
1094 1094
1095 1095 err = traceback.format_exception(sys.exc_info()[0],
1096 sys.exc_info()[1],
1097 sys.exc_info()[2])
1096 sys.exc_info()[1],
1097 sys.exc_info()[2])
1098 1098
1099 1099 print "***** Error occurred in %s *****" %(procUnitConfObj.name)
1100 1100 print "***** %s" %err[-1]
1101 1101
1102 1102 message = "".join(err)
1103 1103
1104 1104 sys.stderr.write(message)
1105 1105
1106 1106 if not send_email:
1107 1107 return
1108 1108
1109 1109 subject = "SChain v%s: Error running %s\n" %(schainpy.__version__, procUnitConfObj.name)
1110 1110
1111 1111 subtitle = "%s: %s\n" %(procUnitConfObj.getElementName() ,procUnitConfObj.name)
1112 1112 subtitle += "Hostname: %s\n" %socket.gethostbyname(socket.gethostname())
1113 1113 subtitle += "Working directory: %s\n" %os.path.abspath("./")
1114 1114 subtitle += "Configuration file: %s\n" %self.filename
1115 1115 subtitle += "Time: %s\n" %str(datetime.datetime.now())
1116 1116
1117 1117 readUnitConfObj = self.getReadUnitObj()
1118 1118 if readUnitConfObj:
1119 1119 subtitle += "\nInput parameters:\n"
1120 1120 subtitle += "[Data path = %s]\n" %readUnitConfObj.path
1121 1121 subtitle += "[Data type = %s]\n" %readUnitConfObj.datatype
1122 1122 subtitle += "[Start date = %s]\n" %readUnitConfObj.startDate
1123 1123 subtitle += "[End date = %s]\n" %readUnitConfObj.endDate
1124 1124 subtitle += "[Start time = %s]\n" %readUnitConfObj.startTime
1125 1125 subtitle += "[End time = %s]\n" %readUnitConfObj.endTime
1126 1126
1127 1127 adminObj = schainpy.admin.SchainNotify()
1128 1128 adminObj.sendAlert(message=message,
1129 subject=subject,
1130 subtitle=subtitle,
1131 filename=self.filename)
1129 subject=subject,
1130 subtitle=subtitle,
1131 filename=self.filename)
1132 1132
1133 1133 def isPaused(self):
1134 1134 return 0
1135 1135
1136 1136 def isStopped(self):
1137 1137 return 0
1138 1138
1139 1139 def runController(self):
1140 1140 """
1141 1141 returns 0 when this process has been stopped, 1 otherwise
1142 1142 """
1143 1143
1144 1144 if self.isPaused():
1145 1145 print "Process suspended"
1146 1146
1147 1147 while True:
1148 1148 sleep(0.1)
1149 1149
1150 1150 if not self.isPaused():
1151 1151 break
1152 1152
1153 1153 if self.isStopped():
1154 1154 break
1155 1155
1156 1156 print "Process reinitialized"
1157 1157
1158 1158 if self.isStopped():
1159 1159 print "Process stopped"
1160 1160 return 0
1161 1161
1162 1162 return 1
1163 1163
1164 1164 def setFilename(self, filename):
1165 1165
1166 1166 self.filename = filename
1167 1167
1168 1168 def setPlotterQueue(self, plotter_queue):
1169 1169
1170 1170 raise NotImplementedError, "Use schainpy.controller_api.ControllerThread instead Project class"
1171 1171
1172 1172 def getPlotterQueue(self):
1173 1173
1174 1174 raise NotImplementedError, "Use schainpy.controller_api.ControllerThread instead Project class"
1175 1175
1176 1176 def useExternalPlotter(self):
1177 1177
1178 1178 raise NotImplementedError, "Use schainpy.controller_api.ControllerThread instead Project class"
1179 1179
1180 1180 def run(self):
1181 1181
1182 1182 print
1183 1183 print "*"*60
1184 1184 print " Starting SIGNAL CHAIN PROCESSING v%s " %schainpy.__version__
1185 1185 print "*"*60
1186 1186 print
1187 1187
1188 1188 keyList = self.procUnitConfObjDict.keys()
1189 1189 keyList.sort()
1190 1190
1191 1191 while(True):
1192 1192
1193 1193 is_ok = False
1194 1194
1195 1195 for procKey in keyList:
1196 1196 # print "Running the '%s' process with %s" %(procUnitConfObj.name, procUnitConfObj.id)
1197 1197
1198 1198 procUnitConfObj = self.procUnitConfObjDict[procKey]
1199 1199
1200 1200 try:
1201 1201 sts = procUnitConfObj.run()
1202 1202 is_ok = is_ok or sts
1203 1203 except KeyboardInterrupt:
1204 1204 is_ok = False
1205 1205 break
1206 1206 except ValueError, e:
1207 1207 sleep(0.5)
1208 1208 self.__handleError(procUnitConfObj, send_email=True)
1209 1209 is_ok = False
1210 1210 break
1211 1211 except:
1212 1212 sleep(0.5)
1213 1213 self.__handleError(procUnitConfObj)
1214 1214 is_ok = False
1215 1215 break
1216 1216
1217 1217 #If every process unit finished so end process
1218 1218 if not(is_ok):
1219 1219 # print "Every process unit have finished"
1220 1220 break
1221 1221
1222 1222 if not self.runController():
1223 1223 break
1224 1224
1225 1225 #Closing every process
1226 1226 for procKey in keyList:
1227 1227 procUnitConfObj = self.procUnitConfObjDict[procKey]
1228 1228 procUnitConfObj.close()
1229 1229
1230 1230 print "Process finished"
1231 1231
1232 1232 def start(self):
1233 1233
1234 1234 self.writeXml()
1235 1235
1236 1236 self.createObjects()
1237 1237 self.connectObjects()
1238 1238 self.run()
1239 1239
1240 1240 if __name__ == '__main__':
1241 1241
1242 1242 desc = "Segundo Test"
1243 1243 filename = "schain.xml"
1244 1244
1245 1245 controllerObj = Project()
1246 1246
1247 1247 controllerObj.setup(id = '191', name='test01', description=desc)
1248 1248
1249 1249 readUnitConfObj = controllerObj.addReadUnit(datatype='Voltage',
1250 1250 path='data/rawdata/',
1251 1251 startDate='2011/01/01',
1252 1252 endDate='2012/12/31',
1253 1253 startTime='00:00:00',
1254 1254 endTime='23:59:59',
1255 1255 online=1,
1256 1256 walk=1)
1257 1257
1258 1258 procUnitConfObj0 = controllerObj.addProcUnit(datatype='Voltage', inputId=readUnitConfObj.getId())
1259 1259
1260 1260 opObj10 = procUnitConfObj0.addOperation(name='selectChannels')
1261 1261 opObj10.addParameter(name='channelList', value='3,4,5', format='intlist')
1262 1262
1263 1263 opObj10 = procUnitConfObj0.addOperation(name='selectHeights')
1264 1264 opObj10.addParameter(name='minHei', value='90', format='float')
1265 1265 opObj10.addParameter(name='maxHei', value='180', format='float')
1266 1266
1267 1267 opObj12 = procUnitConfObj0.addOperation(name='CohInt', optype='external')
1268 1268 opObj12.addParameter(name='n', value='10', format='int')
1269 1269
1270 1270 procUnitConfObj1 = controllerObj.addProcUnit(datatype='Spectra', inputId=procUnitConfObj0.getId())
1271 1271 procUnitConfObj1.addParameter(name='nFFTPoints', value='32', format='int')
1272 1272 # procUnitConfObj1.addParameter(name='pairList', value='(0,1),(0,2),(1,2)', format='')
1273 1273
1274 1274
1275 1275 opObj11 = procUnitConfObj1.addOperation(name='SpectraPlot', optype='external')
1276 1276 opObj11.addParameter(name='idfigure', value='1', format='int')
1277 1277 opObj11.addParameter(name='wintitle', value='SpectraPlot0', format='str')
1278 1278 opObj11.addParameter(name='zmin', value='40', format='int')
1279 1279 opObj11.addParameter(name='zmax', value='90', format='int')
1280 1280 opObj11.addParameter(name='showprofile', value='1', format='int')
1281 1281
1282 1282 print "Escribiendo el archivo XML"
1283 1283
1284 1284 controllerObj.writeXml(filename)
1285 1285
1286 1286 print "Leyendo el archivo XML"
1287 1287 controllerObj.readXml(filename)
1288 1288 #controllerObj.printattr()
1289 1289
1290 1290 controllerObj.createObjects()
1291 1291 controllerObj.connectObjects()
1292 1292 controllerObj.run()
1293 1293
1294 1294 No newline at end of file
Show file before | Show file after | Hide comments
@@ -1,179 +1,179
1 1 import threading
2 2 from Queue import Queue
3 3
4 4 from schainpy.controller import Project
5 5 from schainpy.model.graphics.jroplotter import PlotManager
6 6
7 7 class ControllerThread(threading.Thread, Project):
8 8
9 9 def __init__(self, plotter_queue=None):
10 10
11 11 threading.Thread.__init__(self)
12 12 Project.__init__(self, plotter_queue)
13 13
14 14 self.setDaemon(True)
15 15
16 16 self.lock = threading.Lock()
17 self.control = {'stop':False, 'pause':False}
17 self.control = { 'stop':False, 'pause':False }
18 18
19 19 def __del__(self):
20 20
21 21 self.control['stop'] = True
22 22
23 23 def stop(self):
24 24
25 25 self.lock.acquire()
26 26
27 27 self.control['stop'] = True
28 28
29 29 self.lock.release()
30 30
31 31 def pause(self):
32 32
33 33 self.lock.acquire()
34 34
35 35 self.control['pause'] = not(self.control['pause'])
36 36 paused = self.control['pause']
37 37
38 38 self.lock.release()
39 39
40 40 return paused
41 41
42 42 def isPaused(self):
43 43
44 44 self.lock.acquire()
45 45 paused = self.control['pause']
46 46 self.lock.release()
47 47
48 48 return paused
49 49
50 50 def isStopped(self):
51 51
52 52 self.lock.acquire()
53 53 stopped = self.control['stop']
54 54 self.lock.release()
55 55
56 56 return stopped
57 57
58 58 def run(self):
59 59 self.control['stop'] = False
60 60 self.control['pause'] = False
61 61
62 62 self.writeXml()
63 63
64 64 self.createObjects()
65 65 self.connectObjects()
66 66 Project.run(self)
67 67
68 68 def isRunning(self):
69 69
70 70 return self.is_alive()
71 71
72 72 def isFinished(self):
73 73
74 74 return not self.is_alive()
75 75
76 76 def setPlotters(self):
77 77
78 78 plotterList = PlotManager.plotterList
79 79
80 80 for thisPUConfObj in self.procUnitConfObjDict.values():
81 81
82 82 inputId = thisPUConfObj.getInputId()
83 83
84 84 if int(inputId) == 0:
85 85 continue
86 86
87 87 for thisOpObj in thisPUConfObj.getOperationObjList():
88 88
89 89 if thisOpObj.type == "self":
90 90 continue
91 91
92 92 if thisOpObj.name in plotterList:
93 thisOpObj.type = "plotter"
93 thisOpObj.type = "other"
94 94
95 95 def setPlotterQueue(self, plotter_queue):
96 96
97 97 self.plotterQueue = plotter_queue
98 98
99 99 def getPlotterQueue(self):
100 100
101 101 return self.plotterQueue
102 102
103 103 def useExternalPlotter(self):
104 104
105 105 self.plotterQueue = Queue(10)
106 106 self.setPlotters()
107 107
108 108 plotManagerObj = PlotManager(self.plotterQueue)
109 109 plotManagerObj.setController(self)
110 110
111 111 return plotManagerObj
112 112
113 113 # from PyQt4 import QtCore
114 114 # from PyQt4.QtCore import SIGNAL
115 115 #
116 116 # class ControllerQThread(QtCore.QThread, Project):
117 117 #
118 118 # def __init__(self, filename):
119 119 #
120 120 # QtCore.QThread.__init__(self)
121 121 # Project.__init__(self)
122 122 #
123 123 # self.filename = filename
124 124 #
125 125 # self.lock = threading.Lock()
126 126 # self.control = {'stop':False, 'pause':False}
127 127 #
128 128 # def __del__(self):
129 129 #
130 130 # self.control['stop'] = True
131 131 # self.wait()
132 132 #
133 133 # def stop(self):
134 134 #
135 135 # self.lock.acquire()
136 136 #
137 137 # self.control['stop'] = True
138 138 #
139 139 # self.lock.release()
140 140 #
141 141 # def pause(self):
142 142 #
143 143 # self.lock.acquire()
144 144 #
145 145 # self.control['pause'] = not(self.control['pause'])
146 146 # paused = self.control['pause']
147 147 #
148 148 # self.lock.release()
149 149 #
150 150 # return paused
151 151 #
152 152 # def isPaused(self):
153 153 #
154 154 # self.lock.acquire()
155 155 # paused = self.control['pause']
156 156 # self.lock.release()
157 157 #
158 158 # return paused
159 159 #
160 160 # def isStopped(self):
161 161 #
162 162 # self.lock.acquire()
163 163 # stopped = self.control['stop']
164 164 # self.lock.release()
165 165 #
166 166 # return stopped
167 167 #
168 168 # def run(self):
169 169 #
170 170 # self.control['stop'] = False
171 171 # self.control['pause'] = False
172 172 #
173 173 # self.readXml(self.filename)
174 174 # self.createObjects()
175 175 # self.connectObjects()
176 176 # self.emit( SIGNAL( "jobStarted( PyQt_PyObject )" ), 1)
177 177 # Project.run(self)
178 178 # self.emit( SIGNAL( "jobFinished( PyQt_PyObject )" ), 1)
179 179 # No newline at end of file
Show file before | Show file after | Hide comments
@@ -1,1008 +1,1012
1 1 '''
2 2 @author: Daniel Suarez
3 3 '''
4 4 import os
5 5 import glob
6 6 import ftplib
7 7
8 8 try:
9 9 import paramiko
10 10 import scp
11 11 except:
12 12 print "You should install paramiko and scp libraries \nif you want to use SSH protocol to upload files to the server"
13 13
14 14 import time
15 15
16 16 import threading
17 17 Thread = threading.Thread
18 18
19 19 # try:
20 20 # from gevent import sleep
21 21 # except:
22 22 from time import sleep
23 23
24 24 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation
25 25
26 26 class Remote(Thread):
27 27 """
28 28 Remote is a parent class used to define the behaviour of FTP and SSH class. These clases are
29 29 used to upload or download files remotely.
30 30
31 31 Non-standard Python modules used:
32 32 None
33 33
34 34 Written by:
35 35
36 36 "Miguel Urco":mailto:miguel.urco@jro.igp.gob.pe Jun. 03, 2015
37 37
38 38 """
39 39
40 40 server = None
41 41 username = None
42 42 password = None
43 43 remotefolder = None
44 44
45 45 period = 60
46 46 fileList = []
47 47 bussy = False
48 48
49 49 def __init__(self, server, username, password, remotefolder, period=60):
50 50
51 51 Thread.__init__(self)
52 52
53 53 self.setDaemon(True)
54 54
55 55 self.status = 0
56 56
57 57 self.__server = server
58 58 self.__username = username
59 59 self.__password = password
60 60 self.__remotefolder = remotefolder
61 61
62 62 self.period = period
63 63
64 64 self.fileList = []
65 65 self.bussy = False
66 66
67 67 self.stopFlag = False
68 68
69 69 print "[Remote Server] Opening server: %s" %self.__server
70 70 if self.open(self.__server, self.__username, self.__password, self.__remotefolder):
71 71 print "[Remote Server] %s server was opened successfully" %self.__server
72 72
73 73 self.close()
74 74
75 75 self.mutex = threading.Lock()
76 76
77 77 def stop(self):
78 78
79 79 self.stopFlag = True
80 80 self.join(10)
81 81
82 82 def open(self):
83 83 """
84 84 Connect to server and create a connection class (FTP or SSH) to remote server.
85 85 """
86 86 raise NotImplementedError, "Implement this method in child class"
87 87
88 88 def close(self):
89 89 """
90 90 Close connection to server
91 91 """
92 92 raise NotImplementedError, "Implement this method in child class"
93 93
94 94 def mkdir(self, remotefolder):
95 95 """
96 96 Create a folder remotely
97 97 """
98 98 raise NotImplementedError, "Implement this method in child class"
99 99
100 100 def cd(self, remotefolder):
101 101 """
102 102 Change working directory in remote server
103 103 """
104 104 raise NotImplementedError, "Implement this method in child class"
105 105
106 106 def download(self, filename, localfolder=None):
107 107 """
108 108 Download a file from server to local host
109 109 """
110 110 raise NotImplementedError, "Implement this method in child class"
111 111
112 112 def sendFile(self, fullfilename):
113 113 """
114 114 sendFile method is used to upload a local file to the current directory in remote server
115 115
116 116 Inputs:
117 117 fullfilename - full path name of local file to store in remote directory
118 118
119 119 Returns:
120 120 0 in error case else 1
121 121 """
122 122 raise NotImplementedError, "Implement this method in child class"
123 123
124 124 def upload(self, fullfilename, remotefolder=None):
125 125 """
126 126 upload method is used to upload a local file to remote directory. This method changes
127 127 working directory before sending a file.
128 128
129 129 Inputs:
130 130 fullfilename - full path name of local file to store in remote directory
131 131
132 132 remotefolder - remote directory
133 133
134 134 Returns:
135 135 0 in error case else 1
136 136 """
137 137 print "[Remote Server] Uploading %s to %s:%s" %(fullfilename, self.server, self.remotefolder)
138 138
139 139 if not self.status:
140 140 return 0
141 141
142 142 if remotefolder == None:
143 143 remotefolder = self.remotefolder
144 144
145 145 if not self.cd(remotefolder):
146 146 return 0
147 147
148 148 if not self.sendFile(fullfilename):
149 149 print "[Remote Server] Error uploading file %s" %fullfilename
150 150 return 0
151 151
152 152 print "[Remote Server] upload finished successfully"
153 153
154 154 return 1
155 155
156 156 def delete(self, filename):
157 157 """
158 158 Remove a file from remote server
159 159 """
160 160 pass
161 161
162 162 def updateFileList(self, fileList):
163 163 """
164 164 Remove a file from remote server
165 165 """
166 166
167 167 if fileList == self.fileList:
168 168 return 0
169 169
170 170 self.mutex.acquire()
171 171 # init = time.time()
172 172 #
173 173 # while(self.bussy):
174 174 # sleep(0.1)
175 175 # if time.time() - init > 2*self.period:
176 176 # return 0
177 177
178 178 self.fileList = fileList
179 179 self.mutex.release()
180 180 return 1
181 181
182 182 def run(self):
183 183
184 184 if not self.status:
185 185 print "Finishing FTP service"
186 186 return
187 187
188 188 if not self.cd(self.remotefolder):
189 189 raise ValueError, "Could not access to the new remote directory: %s" %self.remotefolder
190 190
191 191 while True:
192 192
193 193 for i in range(self.period):
194 194 if self.stopFlag:
195 195 break
196 196 sleep(1)
197 197
198 198 if self.stopFlag:
199 199 break
200 200
201 201 # self.bussy = True
202 202 self.mutex.acquire()
203 203
204 204 print "[Remote Server] Opening %s" %self.__server
205 205 if not self.open(self.__server, self.__username, self.__password, self.__remotefolder):
206 206 self.mutex.release()
207 207 continue
208 208
209 209 for thisFile in self.fileList:
210 210 self.upload(thisFile, self.remotefolder)
211 211
212 212 print "[Remote Server] Closing %s" %self.__server
213 213 self.close()
214 214
215 215 self.mutex.release()
216 216 # self.bussy = False
217 217
218 218 print "[Remote Server] Thread stopped successfully"
219 219
220 220 class FTPClient(Remote):
221 221
222 222 __ftpClientObj = None
223 223
224 224 def __init__(self, server, username, password, remotefolder, period=60):
225 225 """
226 226 """
227 227 Remote.__init__(self, server, username, password, remotefolder, period)
228 228
229 229 def open(self, server, username, password, remotefolder):
230 230
231 231 """
232 232 This method is used to set FTP parameters and establish a connection to remote server
233 233
234 234 Inputs:
235 235 server - remote server IP Address
236 236
237 237 username - remote server Username
238 238
239 239 password - remote server password
240 240
241 241 remotefolder - remote server current working directory
242 242
243 243 Return:
244 244 Boolean - Returns 1 if a connection has been established, 0 otherwise
245 245
246 246 Affects:
247 247 self.status - in case of error or fail connection this parameter is set to 0 else 1
248 248
249 249 """
250 250
251 251 if server == None:
252 252 raise ValueError, "FTP server should be defined"
253 253
254 254 if username == None:
255 255 raise ValueError, "FTP username should be defined"
256 256
257 257 if password == None:
258 258 raise ValueError, "FTP password should be defined"
259 259
260 260 if remotefolder == None:
261 261 raise ValueError, "FTP remote folder should be defined"
262 262
263 263 try:
264 264 ftpClientObj = ftplib.FTP(server)
265 265 except ftplib.all_errors, e:
266 266 print "[FTP Server]: FTP server connection fail: %s" %server
267 267 print "[FTP Server]:", e
268 268 self.status = 0
269 269 return 0
270 270
271 271 try:
272 272 ftpClientObj.login(username, password)
273 273 except ftplib.all_errors:
274 274 print "[FTP Server]: FTP username or password are incorrect"
275 275 self.status = 0
276 276 return 0
277 277
278 278 if remotefolder == None:
279 279 remotefolder = ftpClientObj.pwd()
280 280 else:
281 281 try:
282 282 ftpClientObj.cwd(remotefolder)
283 283 except ftplib.all_errors:
284 284 print "[FTP Server]: FTP remote folder is invalid: %s" %remotefolder
285 285 remotefolder = ftpClientObj.pwd()
286 286
287 287 self.server = server
288 288 self.username = username
289 289 self.password = password
290 290 self.remotefolder = remotefolder
291 291 self.__ftpClientObj = ftpClientObj
292 292 self.status = 1
293 293
294 294 return 1
295 295
296 296 def close(self):
297 297 """
298 298 Close connection to remote server
299 299 """
300 300 if not self.status:
301 301 return 0
302 302
303 303 self.__ftpClientObj.close()
304 304
305 305 def mkdir(self, remotefolder):
306 306 """
307 307 mkdir is used to make a new directory in remote server
308 308
309 309 Input:
310 310 remotefolder - directory name
311 311
312 312 Return:
313 313 0 in error case else 1
314 314 """
315 315 if not self.status:
316 316 return 0
317 317
318 318 try:
319 319 self.__ftpClientObj.mkd(dirname)
320 320 except ftplib.all_errors:
321 321 print "[FTP Server]: Error creating remote folder: %s" %remotefolder
322 322 return 0
323 323
324 324 return 1
325 325
326 326 def cd(self, remotefolder):
327 327 """
328 328 cd is used to change remote working directory on server
329 329
330 330 Input:
331 331 remotefolder - current working directory
332 332
333 333 Affects:
334 334 self.remotefolder
335 335
336 336 Return:
337 337 0 in case of error else 1
338 338 """
339 339 if not self.status:
340 340 return 0
341 341
342 342 if remotefolder == self.remotefolder:
343 343 return 1
344 344
345 345 try:
346 346 self.__ftpClientObj.cwd(remotefolder)
347 347 except ftplib.all_errors:
348 348 print '[FTP Server]: Error changing to %s' %remotefolder
349 349 print '[FTP Server]: Trying to create remote folder'
350 350
351 351 if not self.mkdir(remotefolder):
352 352 print '[FTP Server]: Remote folder could not be created'
353 353 return 0
354 354
355 355 try:
356 356 self.__ftpClientObj.cwd(remotefolder)
357 357 except ftplib.all_errors:
358 358 return 0
359 359
360 360 self.remotefolder = remotefolder
361 361
362 362 return 1
363 363
364 364 def sendFile(self, fullfilename):
365 365
366 366 if not self.status:
367 367 return 0
368 368
369 369 fp = open(fullfilename, 'rb')
370 370
371 371 filename = os.path.basename(fullfilename)
372 372
373 373 command = "STOR %s" %filename
374 374
375 375 try:
376 376 self.__ftpClientObj.storbinary(command, fp)
377 377 except ftplib.all_errors, e:
378 378 print "[FTP Server]:", e
379 379 return 0
380 380
381 381 try:
382 382 self.__ftpClientObj.sendcmd('SITE CHMOD 755 ' + filename)
383 383 except ftplib.all_errors, e:
384 384 print "[FTP Server]:", e
385 385
386 386 fp.close()
387 387
388 388 return 1
389 389
390 390 class SSHClient(Remote):
391 391
392 392 __sshClientObj = None
393 393 __scpClientObj = None
394 394
395 395 def __init__(self, server, username, password, remotefolder, period=60):
396 396 """
397 397 """
398 398 Remote.__init__(self, server, username, password, remotefolder, period)
399 399
400 400 def open(self, server, username, password, remotefolder, port=22):
401 401
402 402 """
403 403 This method is used to set SSH parameters and establish a connection to a remote server
404 404
405 405 Inputs:
406 406 server - remote server IP Address
407 407
408 408 username - remote server Username
409 409
410 410 password - remote server password
411 411
412 412 remotefolder - remote server current working directory
413 413
414 414 Return: void
415 415
416 416 Affects:
417 417 self.status - in case of error or fail connection this parameter is set to 0 else 1
418 418
419 419 """
420 420 import socket
421 421
422 422 if server == None:
423 423 raise ValueError, "SSH server should be defined"
424 424
425 425 if username == None:
426 426 raise ValueError, "SSH username should be defined"
427 427
428 428 if password == None:
429 429 raise ValueError, "SSH password should be defined"
430 430
431 431 if remotefolder == None:
432 432 raise ValueError, "SSH remote folder should be defined"
433 433
434 434 sshClientObj = paramiko.SSHClient()
435 435
436 436 sshClientObj.load_system_host_keys()
437 437 sshClientObj.set_missing_host_key_policy(paramiko.WarningPolicy())
438 438
439 439 self.status = 0
440 440 try:
441 441 sshClientObj.connect(server, username=username, password=password, port=port)
442 442 except paramiko.AuthenticationException, e:
443 443 # print "SSH username or password are incorrect: %s"
444 444 print "[SSH Server]:", e
445 445 return 0
446 446 except SSHException, e:
447 447 print "[SSH Server]:", e
448 448 return 0
449 449 except socket.error:
450 450 self.status = 0
451 451 print "[SSH Server]:", e
452 452 return 0
453 453
454 454 self.status = 1
455 455 scpClientObj = scp.SCPClient(sshClientObj.get_transport(), socket_timeout=30)
456 456
457 457 if remotefolder == None:
458 458 remotefolder = self.pwd()
459 459
460 460 self.server = server
461 461 self.username = username
462 462 self.password = password
463 463 self.__sshClientObj = sshClientObj
464 464 self.__scpClientObj = scpClientObj
465 465 self.status = 1
466 466
467 467 if not self.cd(remotefolder):
468 468 raise ValueError, "[SSH Server]: Could not access to remote folder: %s" %remotefolder
469 469 return 0
470 470
471 471 self.remotefolder = remotefolder
472 472
473 473 return 1
474 474
475 475 def close(self):
476 476 """
477 477 Close connection to remote server
478 478 """
479 479 if not self.status:
480 480 return 0
481 481
482 482 self.__scpClientObj.close()
483 483 self.__sshClientObj.close()
484 484
485 485 def __execute(self, command):
486 486 """
487 487 __execute a command on remote server
488 488
489 489 Input:
490 490 command - Exmaple 'ls -l'
491 491
492 492 Return:
493 493 0 in error case else 1
494 494 """
495 495 if not self.status:
496 496 return 0
497 497
498 498 stdin, stdout, stderr = self.__sshClientObj.exec_command(command)
499 499
500 500 result = stderr.readlines()
501 501 if len(result) > 1:
502 502 return 0
503 503
504 504 result = stdout.readlines()
505 505 if len(result) > 1:
506 506 return result[0][:-1]
507 507
508 508 return 1
509 509
510 510 def mkdir(self, remotefolder):
511 511 """
512 512 mkdir is used to make a new directory in remote server
513 513
514 514 Input:
515 515 remotefolder - directory name
516 516
517 517 Return:
518 518 0 in error case else 1
519 519 """
520 520
521 521 command = 'mkdir %s' %remotefolder
522 522
523 523 return self.__execute(command)
524 524
525 525 def pwd(self):
526 526
527 527 command = 'pwd'
528 528
529 529 return self.__execute(command)
530 530
531 531 def cd(self, remotefolder):
532 532 """
533 533 cd is used to change remote working directory on server
534 534
535 535 Input:
536 536 remotefolder - current working directory
537 537
538 538 Affects:
539 539 self.remotefolder
540 540
541 541 Return:
542 542 0 in case of error else 1
543 543 """
544 544 if not self.status:
545 545 return 0
546 546
547 547 if remotefolder == self.remotefolder:
548 548 return 1
549 549
550 550 chk_command = "cd %s; pwd" %remotefolder
551 551 mkdir_command = "mkdir %s" %remotefolder
552 552
553 553 if not self.__execute(chk_command):
554 554 if not self.__execute(mkdir_command):
555 555 self.remotefolder = None
556 556 return 0
557 557
558 558 self.remotefolder = remotefolder
559 559
560 560 return 1
561 561
562 562 def sendFile(self, fullfilename):
563 563
564 564 if not self.status:
565 565 return 0
566 566
567 567 try:
568 568 self.__scpClientObj.put(fullfilename, remote_path=self.remotefolder)
569 569 except scp.ScpError, e:
570 570 print "[SSH Server]", str(e)
571 571 return 0
572 572
573 573 remotefile = os.path.join(self.remotefolder, os.path.split(fullfilename)[-1])
574 574 command = 'chmod 775 %s' %remotefile
575 575
576 576 return self.__execute(command)
577 577
578 578 class SendToServer(ProcessingUnit):
579 579
580 580 def __init__(self):
581 581
582 582 ProcessingUnit.__init__(self)
583 583
584 584 self.isConfig = False
585 585 self.clientObj = None
586
587
586 588
587 589 def setup(self, server, username, password, remotefolder, localfolder, ext='.png', period=60, protocol='ftp', **kwargs):
588 590
589 591 self.clientObj = None
590 592 self.localfolder = localfolder
591 593 self.ext = ext
592 594 self.period = period
593 595
594 596 if str.lower(protocol) == 'ftp':
595 597 self.clientObj = FTPClient(server, username, password, remotefolder, period)
596 598
597 599 if str.lower(protocol) == 'ssh':
598 600 self.clientObj = SSHClient(server, username, password, remotefolder, period)
599 601
600 602 if not self.clientObj:
601 603 raise ValueError, "%s has been chosen as remote access protocol but it is not valid" %protocol
602 604
603 605 self.clientObj.start()
604 606
605 607 def findFiles(self):
606 608
607 609 if not type(self.localfolder) == list:
608 610 folderList = [self.localfolder]
609 611 else:
610 612 folderList = self.localfolder
611 613
612 614 #Remove duplicate items
613 615 folderList = list(set(folderList))
614 616
615 617 fullfilenameList = []
616 618
617 619 for thisFolder in folderList:
618 620
619 621 print "[Remote Server]: Searching files on %s" %thisFolder
620 622
621 623 filenameList = glob.glob1(thisFolder, '*%s' %self.ext)
622 624
623 625 if len(filenameList) < 1:
624 626 continue
625 627
626 628 for thisFile in filenameList:
627 629 fullfilename = os.path.join(thisFolder, thisFile)
628 630
629 631 if fullfilename in fullfilenameList:
630 632 continue
631 633
632 634 #Only files modified in the last 30 minutes are considered
633 635 if os.path.getmtime(fullfilename) < time.time() - 30*60:
634 636 continue
635 637
636 638 fullfilenameList.append(fullfilename)
637 639
638 640 return fullfilenameList
639 641
640 642 def run(self, **kwargs):
641
642 643 if not self.isConfig:
643 644 self.init = time.time()
644 645 self.setup(**kwargs)
645 646 self.isConfig = True
646 647
648 if not self.clientObj.is_alive():
649 print "[Remote Server]: Restarting connection "
650 self.setup(**kwargs)
651
647 652 if time.time() - self.init >= self.period:
648 653 fullfilenameList = self.findFiles()
649 654
650 655 if self.clientObj.updateFileList(fullfilenameList):
651 656 print "[Remote Server]: Sending the next files ", str(fullfilenameList)
652
653 657 self.init = time.time()
654 658
655 659 def close(self):
656 660 print "[Remote Server] Stopping thread"
657 661 self.clientObj.stop()
658 662
659 663
660 664 class FTP(object):
661 665 """
662 666 Ftp is a public class used to define custom File Transfer Protocol from "ftplib" python module
663 667
664 668 Non-standard Python modules used: None
665 669
666 670 Written by "Daniel Suarez":mailto:daniel.suarez@jro.igp.gob.pe Oct. 26, 2010
667 671 """
668 672
669 673 def __init__(self,server = None, username=None, password=None, remotefolder=None):
670 674 """
671 675 This method is used to setting parameters for FTP and establishing connection to remote server
672 676
673 677 Inputs:
674 678 server - remote server IP Address
675 679
676 680 username - remote server Username
677 681
678 682 password - remote server password
679 683
680 684 remotefolder - remote server current working directory
681 685
682 686 Return: void
683 687
684 688 Affects:
685 689 self.status - in Error Case or Connection Failed this parameter is set to 1 else 0
686 690
687 691 self.folderList - sub-folder list of remote folder
688 692
689 693 self.fileList - file list of remote folder
690 694
691 695
692 696 """
693 697
694 698 if ((server == None) and (username==None) and (password==None) and (remotefolder==None)):
695 699 server, username, password, remotefolder = self.parmsByDefault()
696 700
697 701 self.server = server
698 702 self.username = username
699 703 self.password = password
700 704 self.remotefolder = remotefolder
701 705 self.file = None
702 706 self.ftp = None
703 707 self.status = 0
704 708
705 709 try:
706 710 self.ftp = ftplib.FTP(self.server)
707 711 self.ftp.login(self.username,self.password)
708 712 self.ftp.cwd(self.remotefolder)
709 713 # print 'Connect to FTP Server: Successfully'
710 714
711 715 except ftplib.all_errors:
712 716 print 'Error FTP Service'
713 717 self.status = 1
714 718 return
715 719
716 720
717 721
718 722 self.dirList = []
719 723
720 724 try:
721 725 self.dirList = self.ftp.nlst()
722 726
723 727 except ftplib.error_perm, resp:
724 728 if str(resp) == "550 No files found":
725 729 print "no files in this directory"
726 730 self.status = 1
727 731 return
728 732
729 733 except ftplib.all_errors:
730 734 print 'Error Displaying Dir-Files'
731 735 self.status = 1
732 736 return
733 737
734 738 self.fileList = []
735 739 self.folderList = []
736 740 #only for test
737 741 for f in self.dirList:
738 742 name, ext = os.path.splitext(f)
739 743 if ext != '':
740 744 self.fileList.append(f)
741 745 # print 'filename: %s - size: %d'%(f,self.ftp.size(f))
742 746
743 747 def parmsByDefault(self):
744 748 server = 'jro-app.igp.gob.pe'
745 749 username = 'wmaster'
746 750 password = 'mst2010vhf'
747 751 remotefolder = '/home/wmaster/graficos'
748 752
749 753 return server, username, password, remotefolder
750 754
751 755
752 756 def mkd(self,dirname):
753 757 """
754 758 mkd is used to make directory in remote server
755 759
756 760 Input:
757 761 dirname - directory name
758 762
759 763 Return:
760 764 1 in error case else 0
761 765 """
762 766 try:
763 767 self.ftp.mkd(dirname)
764 768 except:
765 769 print 'Error creating remote folder:%s'%dirname
766 770 return 1
767 771
768 772 return 0
769 773
770 774
771 775 def delete(self,filename):
772 776 """
773 777 delete is used to delete file in current working directory of remote server
774 778
775 779 Input:
776 780 filename - filename to delete in remote folder
777 781
778 782 Return:
779 783 1 in error case else 0
780 784 """
781 785
782 786 try:
783 787 self.ftp.delete(filename)
784 788 except:
785 789 print 'Error deleting remote file:%s'%filename
786 790 return 1
787 791
788 792 return 0
789 793
790 794 def download(self,filename,localfolder):
791 795 """
792 796 download is used to downloading file from remote folder into local folder
793 797
794 798 Inputs:
795 799 filename - filename to donwload
796 800
797 801 localfolder - directory local to store filename
798 802
799 803 Returns:
800 804 self.status - 1 in error case else 0
801 805 """
802 806
803 807 self.status = 0
804 808
805 809
806 810 if not(filename in self.fileList):
807 811 print 'filename:%s not exists'%filename
808 812 self.status = 1
809 813 return self.status
810 814
811 815 newfilename = os.path.join(localfolder,filename)
812 816
813 817 self.file = open(newfilename, 'wb')
814 818
815 819 try:
816 820 print 'Download: ' + filename
817 821 self.ftp.retrbinary('RETR ' + filename, self.__handleDownload)
818 822 print 'Download Complete'
819 823 except ftplib.all_errors:
820 824 print 'Error Downloading ' + filename
821 825 self.status = 1
822 826 return self.status
823 827
824 828 self.file.close()
825 829
826 830 return self.status
827 831
828 832
829 833 def __handleDownload(self,block):
830 834 """
831 835 __handleDownload is used to handle writing file
832 836 """
833 837 self.file.write(block)
834 838
835 839
836 840 def upload(self,filename,remotefolder=None):
837 841 """
838 842 upload is used to uploading local file to remote directory
839 843
840 844 Inputs:
841 845 filename - full path name of local file to store in remote directory
842 846
843 847 remotefolder - remote directory
844 848
845 849 Returns:
846 850 self.status - 1 in error case else 0
847 851 """
848 852
849 853 if remotefolder == None:
850 854 remotefolder = self.remotefolder
851 855
852 856 self.status = 0
853 857
854 858 try:
855 859 self.ftp.cwd(remotefolder)
856 860
857 861 self.file = open(filename, 'rb')
858 862
859 863 (head, tail) = os.path.split(filename)
860 864
861 865 command = "STOR " + tail
862 866
863 867 print 'Uploading: ' + tail
864 868 self.ftp.storbinary(command, self.file)
865 869 print 'Upload Completed'
866 870
867 871 except ftplib.all_errors:
868 872 print 'Error Uploading ' + tail
869 873 self.status = 1
870 874 return self.status
871 875
872 876 self.file.close()
873 877
874 878 #back to initial directory in __init__()
875 879 self.ftp.cwd(self.remotefolder)
876 880
877 881 return self.status
878 882
879 883
880 884 def dir(self,remotefolder):
881 885 """
882 886 dir is used to change working directory of remote server and get folder and file list
883 887
884 888 Input:
885 889 remotefolder - current working directory
886 890
887 891 Affects:
888 892 self.fileList - file list of working directory
889 893
890 894 Return:
891 895 infoList - list with filenames and size of file in bytes
892 896
893 897 self.folderList - folder list
894 898 """
895 899
896 900 self.remotefolder = remotefolder
897 901 print 'Change to ' + self.remotefolder
898 902 try:
899 903 self.ftp.cwd(remotefolder)
900 904 except ftplib.all_errors:
901 905 print 'Error Change to ' + self.remotefolder
902 906 infoList = None
903 907 self.folderList = None
904 908 return infoList,self.folderList
905 909
906 910 self.dirList = []
907 911
908 912 try:
909 913 self.dirList = self.ftp.nlst()
910 914
911 915 except ftplib.error_perm, resp:
912 916 if str(resp) == "550 No files found":
913 917 print "no files in this directory"
914 918 infoList = None
915 919 self.folderList = None
916 920 return infoList,self.folderList
917 921 except ftplib.all_errors:
918 922 print 'Error Displaying Dir-Files'
919 923 infoList = None
920 924 self.folderList = None
921 925 return infoList,self.folderList
922 926
923 927 infoList = []
924 928 self.fileList = []
925 929 self.folderList = []
926 930 for f in self.dirList:
927 931 name,ext = os.path.splitext(f)
928 932 if ext != '':
929 933 self.fileList.append(f)
930 934 value = (f,self.ftp.size(f))
931 935 infoList.append(value)
932 936
933 937 if ext == '':
934 938 self.folderList.append(f)
935 939
936 940 return infoList,self.folderList
937 941
938 942
939 943 def close(self):
940 944 """
941 945 close is used to close and end FTP connection
942 946
943 947 Inputs: None
944 948
945 949 Return: void
946 950
947 951 """
948 952 self.ftp.close()
949 953
950 954 class SendByFTP(Operation):
951 955
952 956 def __init__(self):
953 957
954 958 self.status = 1
955 959 self.counter = 0
956 960
957 961 def error_print(self, ValueError):
958 962
959 963 print ValueError, 'Error FTP'
960 964 print "don't worry the program is running..."
961 965
962 966 def worker_ftp(self, server, username, password, remotefolder, filenameList):
963 967
964 968 self.ftpClientObj = FTP(server, username, password, remotefolder)
965 969 for filename in filenameList:
966 970 self.ftpClientObj.upload(filename)
967 971 self.ftpClientObj.close()
968 972
969 973 def ftp_thread(self, server, username, password, remotefolder):
970 974 if not(self.status):
971 975 return
972 976
973 977 import multiprocessing
974 978
975 979 p = multiprocessing.Process(target=self.worker_ftp, args=(server, username, password, remotefolder, self.filenameList,))
976 980 p.start()
977 981
978 982 p.join(3)
979 983
980 984 if p.is_alive():
981 985 p.terminate()
982 986 p.join()
983 987 print 'killing ftp process...'
984 988 self.status = 0
985 989 return
986 990
987 991 self.status = 1
988 992 return
989 993
990 994 def filterByExt(self, ext, localfolder):
991 995 fnameList = glob.glob1(localfolder,ext)
992 996 self.filenameList = [os.path.join(localfolder,x) for x in fnameList]
993 997
994 998 if len(self.filenameList) == 0:
995 999 self.status = 0
996 1000
997 1001 def run(self, dataOut, ext, localfolder, remotefolder, server, username, password, period=1):
998 1002
999 1003 self.counter += 1
1000 1004 if self.counter >= period:
1001 1005 self.filterByExt(ext, localfolder)
1002 1006
1003 1007 self.ftp_thread(server, username, password, remotefolder)
1004 1008
1005 1009 self.counter = 0
1006 1010
1007 1011 self.status = 1
1008 1012
General Comments 0
You need to be logged in to leave comments. Login now