##// END OF EJS Templates
schain
RSS
jroutils_ftp.py: Remote Server is opened and closed each time a file is sent
Miguel Valdez -
r653:ae332cfe81c4
parent child
Show More
Show file before | Show file after | Hide comments
@@ -1,7 +1,8
1 1 VERSIONS:
2 2
3 3 2.1.2:
4 -jroutils_ftp.py: Server thread stops when an error sending file has been detected
4 -jroutils_ftp.py: Bug fixed, Any error sending file stopped the Server Thread
5 Server thread opens and closes remote server each time file list is sent
5 6 -jroplot_spectra.py: Noise path was not being created saving noise data.
6 7 -jroIO_base.py: startTime can be greater than endTime
7 8 -jroplot_heispectra.py: SpectraHeisScope was not showing the right channel
Show file before | Show file after | Hide comments
@@ -1,982 +1,1007
1 1 '''
2 2 @author: Daniel Suarez
3 3 '''
4 4 import os
5 5 import glob
6 6 import ftplib
7 7
8 8 try:
9 9 import paramiko
10 10 import scp
11 11 except:
12 12 print "You should install paramiko and scp libraries \nif you want to use SSH protocol to upload files to the server"
13 13
14 14 import time
15 15
16 16 import threading
17 17 Thread = threading.Thread
18 18
19 19 # try:
20 20 # from gevent import sleep
21 21 # except:
22 22 from time import sleep
23 23
24 24 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation
25 25
26 26 class Remote(Thread):
27 27 """
28 28 Remote is a parent class used to define the behaviour of FTP and SSH class. These clases are
29 29 used to upload or download files remotely.
30 30
31 31 Non-standard Python modules used:
32 32 None
33 33
34 34 Written by:
35 35
36 36 "Miguel Urco":mailto:miguel.urco@jro.igp.gob.pe Jun. 03, 2015
37 37
38 38 """
39 39
40 40 server = None
41 41 username = None
42 42 password = None
43 43 remotefolder = None
44 44
45 45 period = 60
46 46 fileList = []
47 47 bussy = False
48 48
49 49 def __init__(self, server, username, password, remotefolder, period=60):
50 50
51 51 Thread.__init__(self)
52 52
53 53 self.setDaemon(True)
54 54
55 55 self.status = 0
56 56
57 self.__server = server
58 self.__username = username
59 self.__password = password
60 self.__remotefolder = remotefolder
61
57 62 self.period = period
63
58 64 self.fileList = []
59 65 self.bussy = False
60 66
61 67 self.stopFlag = False
62 68
63 print "[Remote Server] Opening server: %s" %server
64 if self.open(server, username, password, remotefolder):
65 print "[Remote Server] %s server was opened successfully" %server
69 print "[Remote Server] Opening server: %s" %self.__server
70 if self.open(self.__server, self.__username, self.__password, self.__remotefolder):
71 print "[Remote Server] %s server was opened successfully" %self.__server
72
73 self.close()
66 74
67 75 self.mutex = threading.Lock()
68 76
69 77 def stop(self):
70 78
71 79 self.stopFlag = True
72 80 self.join(10)
73 81
74 82 def open(self):
75 83 """
76 84 Connect to server and create a connection class (FTP or SSH) to remote server.
77 85 """
78 86 raise NotImplementedError, "Implement this method in child class"
79 87
80 88 def close(self):
81 89 """
82 90 Close connection to server
83 91 """
84 92 raise NotImplementedError, "Implement this method in child class"
85 93
86 94 def mkdir(self, remotefolder):
87 95 """
88 96 Create a folder remotely
89 97 """
90 98 raise NotImplementedError, "Implement this method in child class"
91 99
92 100 def cd(self, remotefolder):
93 101 """
94 102 Change working directory in remote server
95 103 """
96 104 raise NotImplementedError, "Implement this method in child class"
97 105
98 106 def download(self, filename, localfolder=None):
99 107 """
100 108 Download a file from server to local host
101 109 """
102 110 raise NotImplementedError, "Implement this method in child class"
103 111
104 112 def sendFile(self, fullfilename):
105 113 """
106 114 sendFile method is used to upload a local file to the current directory in remote server
107 115
108 116 Inputs:
109 117 fullfilename - full path name of local file to store in remote directory
110 118
111 119 Returns:
112 120 0 in error case else 1
113 121 """
114 122 raise NotImplementedError, "Implement this method in child class"
115 123
116 124 def upload(self, fullfilename, remotefolder=None):
117 125 """
118 126 upload method is used to upload a local file to remote directory. This method changes
119 127 working directory before sending a file.
120 128
121 129 Inputs:
122 130 fullfilename - full path name of local file to store in remote directory
123 131
124 132 remotefolder - remote directory
125 133
126 134 Returns:
127 135 0 in error case else 1
128 136 """
129 137 print "[Remote Server] Uploading %s to %s:%s" %(fullfilename, self.server, self.remotefolder)
130 138
131 139 if not self.status:
132 140 return 0
133 141
134 142 if remotefolder == None:
135 143 remotefolder = self.remotefolder
136 144
137 145 if not self.cd(remotefolder):
138 146 return 0
139 147
140 148 if not self.sendFile(fullfilename):
141 149 print "[Remote Server] Error uploading file %s" %fullfilename
142 150 return 0
143 151
144 152 print "[Remote Server] upload finished successfully"
145 153
146 154 return 1
147 155
148 156 def delete(self, filename):
149 157 """
150 158 Remove a file from remote server
151 159 """
152 160 pass
153 161
154 162 def updateFileList(self, fileList):
155 163 """
156 164 Remove a file from remote server
157 165 """
158 166
159 167 if fileList == self.fileList:
160 168 return 0
161 169
162 170 self.mutex.acquire()
163 171 # init = time.time()
164 172 #
165 173 # while(self.bussy):
166 174 # sleep(0.1)
167 175 # if time.time() - init > 2*self.period:
168 176 # return 0
169 177
170 178 self.fileList = fileList
171 179 self.mutex.release()
172 180 return 1
173 181
174 182 def run(self):
175 183
176 184 if not self.status:
177 185 print "Finishing FTP service"
178 186 return
179 187
180 188 if not self.cd(self.remotefolder):
181 189 raise ValueError, "Could not access to the new remote directory: %s" %self.remotefolder
182 190
183 191 while True:
184 192
185 193 for i in range(self.period):
186 194 if self.stopFlag:
187 195 break
188 196 sleep(1)
189 197
190 198 if self.stopFlag:
191 199 break
192 200
193 201 # self.bussy = True
194 202 self.mutex.acquire()
195 203
204 print "[Remote Server] Opening %s" %self.__server
205 if not self.open(self.__server, self.__username, self.__password, self.__remotefolder):
206 return 0
207
196 208 for thisFile in self.fileList:
197 209 self.upload(thisFile, self.remotefolder)
198 210
211 print "[Remote Server] Closing %s" %self.__server
212 self.close()
213
199 214 self.mutex.release()
200 215 # self.bussy = False
201 216
202 217 print "[Remote Server] Thread stopped successfully"
203 218
204 219 class FTPClient(Remote):
205 220
206 221 __ftpClientObj = None
207 222
208 223 def __init__(self, server, username, password, remotefolder, period=60):
209 224 """
210 225 """
211 226 Remote.__init__(self, server, username, password, remotefolder, period)
212 227
213 228 def open(self, server, username, password, remotefolder):
214 229
215 230 """
216 231 This method is used to set FTP parameters and establish a connection to remote server
217 232
218 233 Inputs:
219 234 server - remote server IP Address
220 235
221 236 username - remote server Username
222 237
223 238 password - remote server password
224 239
225 240 remotefolder - remote server current working directory
226 241
227 Return: void
242 Return:
243 Boolean - Returns 1 if a connection has been established, 0 otherwise
228 244
229 245 Affects:
230 246 self.status - in case of error or fail connection this parameter is set to 0 else 1
231 247
232 248 """
233 249
234 250 if server == None:
235 251 raise ValueError, "FTP server should be defined"
236 252
237 253 if username == None:
238 254 raise ValueError, "FTP username should be defined"
239 255
240 256 if password == None:
241 257 raise ValueError, "FTP password should be defined"
242 258
243 259 if remotefolder == None:
244 260 raise ValueError, "FTP remote folder should be defined"
245 261
246 262 try:
247 263 ftpClientObj = ftplib.FTP(server)
248 except ftplib.all_errors:
249 print "FTP server connection fail: %s" %server
264 except ftplib.all_errors, e:
265 print "[FTP Server]: FTP server connection fail: %s" %server
266 print "[FTP Server]:", e
250 267 self.status = 0
251 268 return 0
252 269
253 270 try:
254 271 ftpClientObj.login(username, password)
255 272 except ftplib.all_errors:
256 print "FTP username or password are incorrect"
273 print "[FTP Server]: FTP username or password are incorrect"
257 274 self.status = 0
258 275 return 0
259 276
260 277 if remotefolder == None:
261 278 remotefolder = ftpClientObj.pwd()
262 279 else:
263 280 try:
264 281 ftpClientObj.cwd(remotefolder)
265 282 except ftplib.all_errors:
266 print "FTP remote folder is invalid: %s" %remotefolder
283 print "[FTP Server]: FTP remote folder is invalid: %s" %remotefolder
267 284 remotefolder = ftpClientObj.pwd()
268 285
269 286 self.server = server
270 287 self.username = username
271 288 self.password = password
272 289 self.remotefolder = remotefolder
273 290 self.__ftpClientObj = ftpClientObj
274 291 self.status = 1
275 292
276 293 return 1
277 294
278 295 def close(self):
279 296 """
280 297 Close connection to remote server
281 298 """
282 299 if not self.status:
283 300 return 0
284 301
285 302 self.__ftpClientObj.close()
286 303
287 304 def mkdir(self, remotefolder):
288 305 """
289 306 mkdir is used to make a new directory in remote server
290 307
291 308 Input:
292 309 remotefolder - directory name
293 310
294 311 Return:
295 312 0 in error case else 1
296 313 """
297 314 if not self.status:
298 315 return 0
299 316
300 317 try:
301 318 self.__ftpClientObj.mkd(dirname)
302 319 except ftplib.all_errors:
303 print "Error creating remote folder: %s" %remotefolder
320 print "[FTP Server]: Error creating remote folder: %s" %remotefolder
304 321 return 0
305 322
306 323 return 1
307 324
308 325 def cd(self, remotefolder):
309 326 """
310 327 cd is used to change remote working directory on server
311 328
312 329 Input:
313 330 remotefolder - current working directory
314 331
315 332 Affects:
316 333 self.remotefolder
317 334
318 335 Return:
319 336 0 in case of error else 1
320 337 """
321 338 if not self.status:
322 339 return 0
323 340
324 341 if remotefolder == self.remotefolder:
325 342 return 1
326 343
327 344 try:
328 345 self.__ftpClientObj.cwd(remotefolder)
329 346 except ftplib.all_errors:
330 print 'Error changing to %s' %remotefolder
331 print 'Trying to create remote folder'
347 print '[FTP Server]: Error changing to %s' %remotefolder
348 print '[FTP Server]: Trying to create remote folder'
332 349
333 350 if not self.mkdir(remotefolder):
334 print 'Remote folder could not be created'
351 print '[FTP Server]: Remote folder could not be created'
335 352 return 0
336 353
337 354 try:
338 355 self.__ftpClientObj.cwd(remotefolder)
339 356 except ftplib.all_errors:
340 357 return 0
341 358
342 359 self.remotefolder = remotefolder
343 360
344 361 return 1
345 362
346 363 def sendFile(self, fullfilename):
347 364
348 365 if not self.status:
349 366 return 0
350 367
351 file = open(fullfilename, 'rb')
368 fp = open(fullfilename, 'rb')
352 369
353 370 filename = os.path.basename(fullfilename)
354 371
355 372 command = "STOR %s" %filename
356 373
357 374 try:
358 self.__ftpClientObj.storbinary(command, file)
359 except ftplib.all_errors:
375 self.__ftpClientObj.storbinary(command, fp)
376 except ftplib.all_errors, e:
377 print "[FTP Server]:", e
360 378 return 0
361 379
362 380 try:
363 381 self.__ftpClientObj.sendcmd('SITE CHMOD 755 ' + filename)
364 382 except ftplib.all_errors, e:
365 print e
383 print "[FTP Server]:", e
366 384
367 file.close()
385 fp.close()
368 386
369 387 return 1
370 388
371 389 class SSHClient(Remote):
372 390
373 391 __sshClientObj = None
374 392 __scpClientObj = None
375 393
376 394 def __init__(self, server, username, password, remotefolder, period=60):
377 395 """
378 396 """
379 397 Remote.__init__(self, server, username, password, remotefolder, period)
380 398
381 399 def open(self, server, username, password, remotefolder, port=22):
382 400
383 401 """
384 402 This method is used to set SSH parameters and establish a connection to a remote server
385 403
386 404 Inputs:
387 405 server - remote server IP Address
388 406
389 407 username - remote server Username
390 408
391 409 password - remote server password
392 410
393 411 remotefolder - remote server current working directory
394 412
395 413 Return: void
396 414
397 415 Affects:
398 416 self.status - in case of error or fail connection this parameter is set to 0 else 1
399 417
400 418 """
419 import socket
401 420
402 421 if server == None:
403 422 raise ValueError, "SSH server should be defined"
404 423
405 424 if username == None:
406 425 raise ValueError, "SSH username should be defined"
407 426
408 427 if password == None:
409 428 raise ValueError, "SSH password should be defined"
410 429
411 430 if remotefolder == None:
412 431 raise ValueError, "SSH remote folder should be defined"
413 432
414 try:
415 433 sshClientObj = paramiko.SSHClient()
416 except:
417 print "SSH server connection fail: %s" %server
418 self.status = 0
419 return 0
420 434
421 435 sshClientObj.load_system_host_keys()
422 436 sshClientObj.set_missing_host_key_policy(paramiko.WarningPolicy())
423 437
438 self.status = 0
424 439 try:
425 440 sshClientObj.connect(server, username=username, password=password, port=port)
426 except :
427 print "SSH username or password are incorrect: %s"
441 except paramiko.AuthenticationException, e:
442 # print "SSH username or password are incorrect: %s"
443 print "[SSH Server]:", e
444 return 0
445 except SSHException, e:
446 print "[SSH Server]:", e
447 return 0
448 except socket.error:
428 449 self.status = 0
450 print "[SSH Server]:", e
429 451 return 0
430 452
453 self.status = 1
431 454 scpClientObj = scp.SCPClient(sshClientObj.get_transport(), socket_timeout=30)
432 455
433 456 if remotefolder == None:
434 457 remotefolder = self.pwd()
435 458
436 459 self.server = server
437 460 self.username = username
438 461 self.password = password
439 462 self.__sshClientObj = sshClientObj
440 463 self.__scpClientObj = scpClientObj
441 464 self.status = 1
442 465
443 466 if not self.cd(remotefolder):
444 raise ValueError, "Could not access to remote folder: %s" %remotefolder
467 raise ValueError, "[SSH Server]: Could not access to remote folder: %s" %remotefolder
445 468 return 0
446 469
447 470 self.remotefolder = remotefolder
448 471
449 472 return 1
450 473
451 474 def close(self):
452 475 """
453 476 Close connection to remote server
454 477 """
455 478 if not self.status:
456 479 return 0
457 480
458 self.__sshObj.close()
481 self.__scpClientObj.close()
482 self.__sshClientObj.close()
459 483
460 484 def __execute(self, command):
461 485 """
462 486 __execute a command on remote server
463 487
464 488 Input:
465 489 command - Exmaple 'ls -l'
466 490
467 491 Return:
468 492 0 in error case else 1
469 493 """
470 494 if not self.status:
471 495 return 0
472 496
473 497 stdin, stdout, stderr = self.__sshClientObj.exec_command(command)
474 498
475 499 result = stderr.readlines()
476 500 if len(result) > 1:
477 501 return 0
478 502
479 503 result = stdout.readlines()
480 504 if len(result) > 1:
481 505 return result[0][:-1]
482 506
483 507 return 1
484 508
485 509 def mkdir(self, remotefolder):
486 510 """
487 511 mkdir is used to make a new directory in remote server
488 512
489 513 Input:
490 514 remotefolder - directory name
491 515
492 516 Return:
493 517 0 in error case else 1
494 518 """
495 519
496 520 command = 'mkdir %s' %remotefolder
497 521
498 522 return self.__execute(command)
499 523
500 524 def pwd(self):
501 525
502 526 command = 'pwd'
503 527
504 528 return self.__execute(command)
505 529
506 530 def cd(self, remotefolder):
507 531 """
508 532 cd is used to change remote working directory on server
509 533
510 534 Input:
511 535 remotefolder - current working directory
512 536
513 537 Affects:
514 538 self.remotefolder
515 539
516 540 Return:
517 541 0 in case of error else 1
518 542 """
519 543 if not self.status:
520 544 return 0
521 545
522 546 if remotefolder == self.remotefolder:
523 547 return 1
524 548
525 549 chk_command = "cd %s; pwd" %remotefolder
526 550 mkdir_command = "mkdir %s" %remotefolder
527 551
528 552 if not self.__execute(chk_command):
529 553 if not self.__execute(mkdir_command):
530 554 self.remotefolder = None
531 555 return 0
532 556
533 557 self.remotefolder = remotefolder
534 558
535 559 return 1
536 560
537 561 def sendFile(self, fullfilename):
538 562
539 563 if not self.status:
540 564 return 0
541 565
542 566 try:
543 567 self.__scpClientObj.put(fullfilename, remote_path=self.remotefolder)
544 except:
568 except scp.ScpError, e:
569 print "[SSH Server]", str(e)
545 570 return 0
546 571
547 572 remotefile = os.path.join(self.remotefolder, os.path.split(fullfilename)[-1])
548 573 command = 'chmod 775 %s' %remotefile
549 574
550 575 return self.__execute(command)
551 576
552 577 class SendToServer(ProcessingUnit):
553 578
554 579 def __init__(self):
555 580
556 581 ProcessingUnit.__init__(self)
557 582
558 583 self.isConfig = False
559 584 self.clientObj = None
560 585
561 586 def setup(self, server, username, password, remotefolder, localfolder, ext='.png', period=60, protocol='ftp', **kwargs):
562 587
563 588 self.clientObj = None
564 589 self.localfolder = localfolder
565 590 self.ext = ext
566 591 self.period = period
567 592
568 593 if str.lower(protocol) == 'ftp':
569 594 self.clientObj = FTPClient(server, username, password, remotefolder, period)
570 595
571 596 if str.lower(protocol) == 'ssh':
572 597 self.clientObj = SSHClient(server, username, password, remotefolder, period)
573 598
574 599 if not self.clientObj:
575 600 raise ValueError, "%s has been chosen as remote access protocol but it is not valid" %protocol
576 601
577 602 self.clientObj.start()
578 603
579 604 def findFiles(self):
580 605
581 606 if not type(self.localfolder) == list:
582 607 folderList = [self.localfolder]
583 608 else:
584 609 folderList = self.localfolder
585 610
586 611 #Remove duplicate items
587 612 folderList = list(set(folderList))
588 613
589 614 fullfilenameList = []
590 615
591 616 for thisFolder in folderList:
592 617
593 618 print "[Remote Server]: Searching files on %s" %thisFolder
594 619
595 620 filenameList = glob.glob1(thisFolder, '*%s' %self.ext)
596 621
597 622 if len(filenameList) < 1:
598 623 continue
599 624
600 625 for thisFile in filenameList:
601 626 fullfilename = os.path.join(thisFolder, thisFile)
602 627
603 628 if fullfilename in fullfilenameList:
604 629 continue
605 630
606 631 #Only files modified in the last 30 minutes are considered
607 632 if os.path.getmtime(fullfilename) < time.time() - 30*60:
608 633 continue
609 634
610 635 fullfilenameList.append(fullfilename)
611 636
612 637 return fullfilenameList
613 638
614 639 def run(self, **kwargs):
615 640
616 641 if not self.isConfig:
617 642 self.init = time.time()
618 643 self.setup(**kwargs)
619 644 self.isConfig = True
620 645
621 646 if time.time() - self.init >= self.period:
622 647 fullfilenameList = self.findFiles()
623 648
624 649 if self.clientObj.updateFileList(fullfilenameList):
625 650 print "[Remote Server]: Sending the next files ", str(fullfilenameList)
626 651
627 652 self.init = time.time()
628 653
629 654 def close(self):
630 655 print "[Remote Server] Stopping thread"
631 656 self.clientObj.stop()
632 657
633 658
634 659 class FTP(object):
635 660 """
636 661 Ftp is a public class used to define custom File Transfer Protocol from "ftplib" python module
637 662
638 663 Non-standard Python modules used: None
639 664
640 665 Written by "Daniel Suarez":mailto:daniel.suarez@jro.igp.gob.pe Oct. 26, 2010
641 666 """
642 667
643 668 def __init__(self,server = None, username=None, password=None, remotefolder=None):
644 669 """
645 670 This method is used to setting parameters for FTP and establishing connection to remote server
646 671
647 672 Inputs:
648 673 server - remote server IP Address
649 674
650 675 username - remote server Username
651 676
652 677 password - remote server password
653 678
654 679 remotefolder - remote server current working directory
655 680
656 681 Return: void
657 682
658 683 Affects:
659 684 self.status - in Error Case or Connection Failed this parameter is set to 1 else 0
660 685
661 686 self.folderList - sub-folder list of remote folder
662 687
663 688 self.fileList - file list of remote folder
664 689
665 690
666 691 """
667 692
668 693 if ((server == None) and (username==None) and (password==None) and (remotefolder==None)):
669 694 server, username, password, remotefolder = self.parmsByDefault()
670 695
671 696 self.server = server
672 697 self.username = username
673 698 self.password = password
674 699 self.remotefolder = remotefolder
675 700 self.file = None
676 701 self.ftp = None
677 702 self.status = 0
678 703
679 704 try:
680 705 self.ftp = ftplib.FTP(self.server)
681 706 self.ftp.login(self.username,self.password)
682 707 self.ftp.cwd(self.remotefolder)
683 708 # print 'Connect to FTP Server: Successfully'
684 709
685 710 except ftplib.all_errors:
686 711 print 'Error FTP Service'
687 712 self.status = 1
688 713 return
689 714
690 715
691 716
692 717 self.dirList = []
693 718
694 719 try:
695 720 self.dirList = self.ftp.nlst()
696 721
697 722 except ftplib.error_perm, resp:
698 723 if str(resp) == "550 No files found":
699 724 print "no files in this directory"
700 725 self.status = 1
701 726 return
702 727
703 728 except ftplib.all_errors:
704 729 print 'Error Displaying Dir-Files'
705 730 self.status = 1
706 731 return
707 732
708 733 self.fileList = []
709 734 self.folderList = []
710 735 #only for test
711 736 for f in self.dirList:
712 737 name, ext = os.path.splitext(f)
713 738 if ext != '':
714 739 self.fileList.append(f)
715 740 # print 'filename: %s - size: %d'%(f,self.ftp.size(f))
716 741
717 742 def parmsByDefault(self):
718 743 server = 'jro-app.igp.gob.pe'
719 744 username = 'wmaster'
720 745 password = 'mst2010vhf'
721 746 remotefolder = '/home/wmaster/graficos'
722 747
723 748 return server, username, password, remotefolder
724 749
725 750
726 751 def mkd(self,dirname):
727 752 """
728 753 mkd is used to make directory in remote server
729 754
730 755 Input:
731 756 dirname - directory name
732 757
733 758 Return:
734 759 1 in error case else 0
735 760 """
736 761 try:
737 762 self.ftp.mkd(dirname)
738 763 except:
739 764 print 'Error creating remote folder:%s'%dirname
740 765 return 1
741 766
742 767 return 0
743 768
744 769
745 770 def delete(self,filename):
746 771 """
747 772 delete is used to delete file in current working directory of remote server
748 773
749 774 Input:
750 775 filename - filename to delete in remote folder
751 776
752 777 Return:
753 778 1 in error case else 0
754 779 """
755 780
756 781 try:
757 782 self.ftp.delete(filename)
758 783 except:
759 784 print 'Error deleting remote file:%s'%filename
760 785 return 1
761 786
762 787 return 0
763 788
764 789 def download(self,filename,localfolder):
765 790 """
766 791 download is used to downloading file from remote folder into local folder
767 792
768 793 Inputs:
769 794 filename - filename to donwload
770 795
771 796 localfolder - directory local to store filename
772 797
773 798 Returns:
774 799 self.status - 1 in error case else 0
775 800 """
776 801
777 802 self.status = 0
778 803
779 804
780 805 if not(filename in self.fileList):
781 806 print 'filename:%s not exists'%filename
782 807 self.status = 1
783 808 return self.status
784 809
785 810 newfilename = os.path.join(localfolder,filename)
786 811
787 812 self.file = open(newfilename, 'wb')
788 813
789 814 try:
790 815 print 'Download: ' + filename
791 816 self.ftp.retrbinary('RETR ' + filename, self.__handleDownload)
792 817 print 'Download Complete'
793 818 except ftplib.all_errors:
794 819 print 'Error Downloading ' + filename
795 820 self.status = 1
796 821 return self.status
797 822
798 823 self.file.close()
799 824
800 825 return self.status
801 826
802 827
803 828 def __handleDownload(self,block):
804 829 """
805 830 __handleDownload is used to handle writing file
806 831 """
807 832 self.file.write(block)
808 833
809 834
810 835 def upload(self,filename,remotefolder=None):
811 836 """
812 837 upload is used to uploading local file to remote directory
813 838
814 839 Inputs:
815 840 filename - full path name of local file to store in remote directory
816 841
817 842 remotefolder - remote directory
818 843
819 844 Returns:
820 845 self.status - 1 in error case else 0
821 846 """
822 847
823 848 if remotefolder == None:
824 849 remotefolder = self.remotefolder
825 850
826 851 self.status = 0
827 852
828 853 try:
829 854 self.ftp.cwd(remotefolder)
830 855
831 856 self.file = open(filename, 'rb')
832 857
833 858 (head, tail) = os.path.split(filename)
834 859
835 860 command = "STOR " + tail
836 861
837 862 print 'Uploading: ' + tail
838 863 self.ftp.storbinary(command, self.file)
839 864 print 'Upload Completed'
840 865
841 866 except ftplib.all_errors:
842 867 print 'Error Uploading ' + tail
843 868 self.status = 1
844 869 return self.status
845 870
846 871 self.file.close()
847 872
848 873 #back to initial directory in __init__()
849 874 self.ftp.cwd(self.remotefolder)
850 875
851 876 return self.status
852 877
853 878
854 879 def dir(self,remotefolder):
855 880 """
856 881 dir is used to change working directory of remote server and get folder and file list
857 882
858 883 Input:
859 884 remotefolder - current working directory
860 885
861 886 Affects:
862 887 self.fileList - file list of working directory
863 888
864 889 Return:
865 890 infoList - list with filenames and size of file in bytes
866 891
867 892 self.folderList - folder list
868 893 """
869 894
870 895 self.remotefolder = remotefolder
871 896 print 'Change to ' + self.remotefolder
872 897 try:
873 898 self.ftp.cwd(remotefolder)
874 899 except ftplib.all_errors:
875 900 print 'Error Change to ' + self.remotefolder
876 901 infoList = None
877 902 self.folderList = None
878 903 return infoList,self.folderList
879 904
880 905 self.dirList = []
881 906
882 907 try:
883 908 self.dirList = self.ftp.nlst()
884 909
885 910 except ftplib.error_perm, resp:
886 911 if str(resp) == "550 No files found":
887 912 print "no files in this directory"
888 913 infoList = None
889 914 self.folderList = None
890 915 return infoList,self.folderList
891 916 except ftplib.all_errors:
892 917 print 'Error Displaying Dir-Files'
893 918 infoList = None
894 919 self.folderList = None
895 920 return infoList,self.folderList
896 921
897 922 infoList = []
898 923 self.fileList = []
899 924 self.folderList = []
900 925 for f in self.dirList:
901 926 name,ext = os.path.splitext(f)
902 927 if ext != '':
903 928 self.fileList.append(f)
904 929 value = (f,self.ftp.size(f))
905 930 infoList.append(value)
906 931
907 932 if ext == '':
908 933 self.folderList.append(f)
909 934
910 935 return infoList,self.folderList
911 936
912 937
913 938 def close(self):
914 939 """
915 940 close is used to close and end FTP connection
916 941
917 942 Inputs: None
918 943
919 944 Return: void
920 945
921 946 """
922 947 self.ftp.close()
923 948
924 949 class SendByFTP(Operation):
925 950
926 951 def __init__(self):
927 952
928 953 self.status = 1
929 954 self.counter = 0
930 955
931 956 def error_print(self, ValueError):
932 957
933 958 print ValueError, 'Error FTP'
934 959 print "don't worry the program is running..."
935 960
936 961 def worker_ftp(self, server, username, password, remotefolder, filenameList):
937 962
938 963 self.ftpClientObj = FTP(server, username, password, remotefolder)
939 964 for filename in filenameList:
940 965 self.ftpClientObj.upload(filename)
941 966 self.ftpClientObj.close()
942 967
943 968 def ftp_thread(self, server, username, password, remotefolder):
944 969 if not(self.status):
945 970 return
946 971
947 972 import multiprocessing
948 973
949 974 p = multiprocessing.Process(target=self.worker_ftp, args=(server, username, password, remotefolder, self.filenameList,))
950 975 p.start()
951 976
952 977 p.join(3)
953 978
954 979 if p.is_alive():
955 980 p.terminate()
956 981 p.join()
957 982 print 'killing ftp process...'
958 983 self.status = 0
959 984 return
960 985
961 986 self.status = 1
962 987 return
963 988
964 989 def filterByExt(self, ext, localfolder):
965 990 fnameList = glob.glob1(localfolder,ext)
966 991 self.filenameList = [os.path.join(localfolder,x) for x in fnameList]
967 992
968 993 if len(self.filenameList) == 0:
969 994 self.status = 0
970 995
971 996 def run(self, dataOut, ext, localfolder, remotefolder, server, username, password, period=1):
972 997
973 998 self.counter += 1
974 999 if self.counter >= period:
975 1000 self.filterByExt(ext, localfolder)
976 1001
977 1002 self.ftp_thread(server, username, password, remotefolder)
978 1003
979 1004 self.counter = 0
980 1005
981 1006 self.status = 1
982 1007
General Comments 0
You need to be logged in to leave comments. Login now