##// END OF EJS Templates
schain
RSS
SendToServer:...
Miguel Valdez -
r633:bcf0a5d4812c
parent child
Show More
Show file before | Show file after | Hide comments
@@ -1,965 +1,988
1 '''
1 '''
2 @author: Daniel Suarez
2 @author: Daniel Suarez
3 '''
3 '''
4 import os
4 import os
5 import glob
5 import glob
6 import ftplib
6 import ftplib
7
7
8 try:
8 try:
9 import paramiko
9 import paramiko
10 import scp
10 import scp
11 except:
11 except:
12 print "You should install paramiko if you will use SSH protocol to upload files to a server"
12 print "You should install paramiko if you will use SSH protocol to upload files to a server"
13
13
14 import multiprocessing
15
16 import time
14 import time
17 import threading
18
19
15
20 try:
16 import threading
21 from gevent import sleep
17 Thread = threading.Thread
22 except:
18
23 from time import sleep
19 # try:
20 # from gevent import sleep
21 # except:
22 from time import sleep
24
23
25 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation
24 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation
26
25
27 class Remote(threading.Thread):
26 class Remote(Thread):
28 """
27 """
29 Remote is a parent class used to define the behaviour of FTP and SSH class. These clases are
28 Remote is a parent class used to define the behaviour of FTP and SSH class. These clases are
30 used to upload or download files remotely.
29 used to upload or download files remotely.
31
30
32 Non-standard Python modules used:
31 Non-standard Python modules used:
33 None
32 None
34
33
35 Written by:
34 Written by:
36
35
37 "Miguel Urco":mailto:miguel.urco@jro.igp.gob.pe Jun. 03, 2015
36 "Miguel Urco":mailto:miguel.urco@jro.igp.gob.pe Jun. 03, 2015
38
37
39 """
38 """
40
39
41 server = None
40 server = None
42 username = None
41 username = None
43 password = None
42 password = None
44 remotefolder = None
43 remotefolder = None
45
44
46 period = 60
45 period = 60
47 fileList = []
46 fileList = []
48 bussy = False
47 bussy = False
49
48
50 def __init__(self, server, username, password, remotefolder, period=60):
49 def __init__(self, server, username, password, remotefolder, period=60):
51
50
52 threading.Thread.__init__(self)
51 Thread.__init__(self)
53 self._stop = threading.Event()
54
52
55 self.setDaemon(True)
53 self.setDaemon(True)
56
54
57 self.status = 0
55 self.status = 0
58
56
59 self.period = period
57 self.period = period
60 self.fileList = []
58 self.fileList = []
61 self.bussy = False
59 self.bussy = False
62
60
63 self.stopFlag = False
61 self.stopFlag = False
64
62
65 print "[Remote Server] Opening server: %s" %server
63 print "[Remote Server] Opening server: %s" %server
66 if self.open(server, username, password, remotefolder):
64 if self.open(server, username, password, remotefolder):
67 print "[Remote Server] %s server was opened successfully" %server
65 print "[Remote Server] %s server was opened successfully" %server
68
66
67 self.mutex = threading.Lock()
68
69 def stop(self):
69 def stop(self):
70
70
71 self.stopFlag = True
71 self.stopFlag = True
72 self.join(10)
72
73
73 def open(self):
74 def open(self):
74 """
75 """
75 Connect to server and create a connection class (FTP or SSH) to remote server.
76 Connect to server and create a connection class (FTP or SSH) to remote server.
76 """
77 """
77 raise NotImplementedError, "Implement this method in child class"
78 raise NotImplementedError, "Implement this method in child class"
78
79
79 def close(self):
80 def close(self):
80 """
81 """
81 Close connection to server
82 Close connection to server
82 """
83 """
83 raise NotImplementedError, "Implement this method in child class"
84 raise NotImplementedError, "Implement this method in child class"
84
85
85 def mkdir(self, remotefolder):
86 def mkdir(self, remotefolder):
86 """
87 """
87 Create a folder remotely
88 Create a folder remotely
88 """
89 """
89 raise NotImplementedError, "Implement this method in child class"
90 raise NotImplementedError, "Implement this method in child class"
90
91
91 def cd(self, remotefolder):
92 def cd(self, remotefolder):
92 """
93 """
93 Change working directory in remote server
94 Change working directory in remote server
94 """
95 """
95 raise NotImplementedError, "Implement this method in child class"
96 raise NotImplementedError, "Implement this method in child class"
96
97
97 def download(self, filename, localfolder=None):
98 def download(self, filename, localfolder=None):
98 """
99 """
99 Download a file from server to local host
100 Download a file from server to local host
100 """
101 """
101 raise NotImplementedError, "Implement this method in child class"
102 raise NotImplementedError, "Implement this method in child class"
102
103
103 def sendFile(self, fullfilename):
104 def sendFile(self, fullfilename):
104 """
105 """
105 sendFile method is used to upload a local file to the current directory in remote server
106 sendFile method is used to upload a local file to the current directory in remote server
106
107
107 Inputs:
108 Inputs:
108 fullfilename - full path name of local file to store in remote directory
109 fullfilename - full path name of local file to store in remote directory
109
110
110 Returns:
111 Returns:
111 0 in error case else 1
112 0 in error case else 1
112 """
113 """
113 raise NotImplementedError, "Implement this method in child class"
114 raise NotImplementedError, "Implement this method in child class"
114
115
115 def upload(self, fullfilename, remotefolder=None):
116 def upload(self, fullfilename, remotefolder=None):
116 """
117 """
117 upload method is used to upload a local file to remote directory. This method changes
118 upload method is used to upload a local file to remote directory. This method changes
118 working directory before sending a file.
119 working directory before sending a file.
119
120
120 Inputs:
121 Inputs:
121 fullfilename - full path name of local file to store in remote directory
122 fullfilename - full path name of local file to store in remote directory
122
123
123 remotefolder - remote directory
124 remotefolder - remote directory
124
125
125 Returns:
126 Returns:
126 0 in error case else 1
127 0 in error case else 1
127 """
128 """
128 print "[Remote Server] Uploading %s to %s:%s" %(fullfilename, self.server, self.remotefolder)
129 print "[Remote Server] Uploading %s to %s:%s" %(fullfilename, self.server, self.remotefolder)
129
130
130 if not self.status:
131 if not self.status:
131 return 0
132 return 0
132
133
133 if remotefolder == None:
134 if remotefolder == None:
134 remotefolder = self.remotefolder
135 remotefolder = self.remotefolder
135
136
136 if not self.cd(remotefolder):
137 if not self.cd(remotefolder):
137 return 0
138 return 0
138
139
139 if not self.sendFile(fullfilename):
140 if not self.sendFile(fullfilename):
140 print "[Remote Server] Error uploading file %s" %fullfilename
141 print "[Remote Server] Error uploading file %s" %fullfilename
141 return 0
142 return 0
142
143
143 print "[Remote Server] upload finished successfully"
144 print "[Remote Server] upload finished successfully"
144
145
145 return 1
146 return 1
146
147
147 def delete(self, filename):
148 def delete(self, filename):
148 """
149 """
149 Remove a file from remote server
150 Remove a file from remote server
150 """
151 """
151 pass
152 pass
152
153
153 def updateFileList(self, fileList):
154 def updateFileList(self, fileList):
154 """
155 """
155 Remove a file from remote server
156 Remove a file from remote server
156 """
157 """
157
158
158 if fileList == self.fileList:
159 if fileList == self.fileList:
159 return 0
160 return 0
160
161
161 init = time.time()
162 self.mutex.acquire()
162
163 # init = time.time()
163 while(self.bussy):
164 #
164 sleep(0.1)
165 # while(self.bussy):
165 if time.time() - init > 2*self.period:
166 # sleep(0.1)
166 return 0
167 # if time.time() - init > 2*self.period:
168 # return 0
167
169
168 self.fileList = fileList
170 self.fileList = fileList
169
171 self.mutex.release()
170 return 1
172 return 1
171
173
172 def run(self):
174 def run(self):
173
175
174 if not self.status:
176 if not self.status:
175 print "Finishing FTP service"
177 print "Finishing FTP service"
176 return
178 return
177
179
178 if not self.cd(self.remotefolder):
180 if not self.cd(self.remotefolder):
179 raise ValueError, "Could not access to the new remote directory: %s" %self.remotefolder
181 raise ValueError, "Could not access to the new remote directory: %s" %self.remotefolder
180
182
181 sts = True
183 sts = True
182
184
183 while True:
185 while True:
184
186
185 sleep(self.period)
187 for i in range(self.period):
188 if self.stopFlag:
189 break
190 sleep(1)
186
191
187 self.bussy = True
192 if self.stopFlag:
193 break
194
195 # self.bussy = True
196 self.mutex.acquire()
188
197
189 for thisFile in self.fileList:
198 for thisFile in self.fileList:
190 sts = self.upload(thisFile, self.remotefolder)
199 sts = self.upload(thisFile, self.remotefolder)
191 if not sts: break
200 if not sts: break
192
201
193 self.bussy = False
202 self.mutex.release()
194
203 # self.bussy = False
195 if not sts: break
196
204
197 if self.stopFlag:
205 if not sts:
198 break
206 break
199
207
200 print "[Remote Server] Thread stopped successfully"
208 print "[Remote Server] Thread stopped successfully"
201
209
202 class FTPClient(Remote):
210 class FTPClient(Remote):
203
211
204 __ftpClientObj = None
212 __ftpClientObj = None
205
213
206 def __init__(self, server, username, password, remotefolder, period=60):
214 def __init__(self, server, username, password, remotefolder, period=60):
207 """
215 """
208 """
216 """
209 Remote.__init__(self, server, username, password, remotefolder, period)
217 Remote.__init__(self, server, username, password, remotefolder, period)
210
218
211 def open(self, server, username, password, remotefolder):
219 def open(self, server, username, password, remotefolder):
212
220
213 """
221 """
214 This method is used to set FTP parameters and establish a connection to remote server
222 This method is used to set FTP parameters and establish a connection to remote server
215
223
216 Inputs:
224 Inputs:
217 server - remote server IP Address
225 server - remote server IP Address
218
226
219 username - remote server Username
227 username - remote server Username
220
228
221 password - remote server password
229 password - remote server password
222
230
223 remotefolder - remote server current working directory
231 remotefolder - remote server current working directory
224
232
225 Return: void
233 Return: void
226
234
227 Affects:
235 Affects:
228 self.status - in case of error or fail connection this parameter is set to 0 else 1
236 self.status - in case of error or fail connection this parameter is set to 0 else 1
229
237
230 """
238 """
231
239
232 if server == None:
240 if server == None:
233 raise ValueError, "FTP server should be defined"
241 raise ValueError, "FTP server should be defined"
234
242
235 if username == None:
243 if username == None:
236 raise ValueError, "FTP username should be defined"
244 raise ValueError, "FTP username should be defined"
237
245
238 if password == None:
246 if password == None:
239 raise ValueError, "FTP password should be defined"
247 raise ValueError, "FTP password should be defined"
240
248
241 if remotefolder == None:
249 if remotefolder == None:
242 raise ValueError, "FTP remote folder should be defined"
250 raise ValueError, "FTP remote folder should be defined"
243
251
244 try:
252 try:
245 ftpClientObj = ftplib.FTP(server)
253 ftpClientObj = ftplib.FTP(server)
246 except ftplib.all_errors:
254 except ftplib.all_errors:
247 print "FTP server connection fail: %s" %server
255 print "FTP server connection fail: %s" %server
248 self.status = 0
256 self.status = 0
249 return 0
257 return 0
250
258
251 try:
259 try:
252 ftpClientObj.login(username, password)
260 ftpClientObj.login(username, password)
253 except ftplib.all_errors:
261 except ftplib.all_errors:
254 print "FTP username or password are incorrect"
262 print "FTP username or password are incorrect"
255 self.status = 0
263 self.status = 0
256 return 0
264 return 0
257
265
258 if remotefolder == None:
266 if remotefolder == None:
259 remotefolder = ftpClientObj.pwd()
267 remotefolder = ftpClientObj.pwd()
260 else:
268 else:
261 try:
269 try:
262 ftpClientObj.cwd(remotefolder)
270 ftpClientObj.cwd(remotefolder)
263 except ftplib.all_errors:
271 except ftplib.all_errors:
264 print "FTP remote folder is invalid: %s" %remotefolder
272 print "FTP remote folder is invalid: %s" %remotefolder
265 remotefolder = ftpClientObj.pwd()
273 remotefolder = ftpClientObj.pwd()
266
274
267 self.server = server
275 self.server = server
268 self.username = username
276 self.username = username
269 self.password = password
277 self.password = password
270 self.remotefolder = remotefolder
278 self.remotefolder = remotefolder
271 self.__ftpClientObj = ftpClientObj
279 self.__ftpClientObj = ftpClientObj
272 self.status = 1
280 self.status = 1
273
281
274 return 1
282 return 1
275
283
276 def close(self):
284 def close(self):
277 """
285 """
278 Close connection to remote server
286 Close connection to remote server
279 """
287 """
280 if not self.status:
288 if not self.status:
281 return 0
289 return 0
282
290
283 self.__ftpClientObj.close()
291 self.__ftpClientObj.close()
284
292
285 def mkdir(self, remotefolder):
293 def mkdir(self, remotefolder):
286 """
294 """
287 mkdir is used to make a new directory in remote server
295 mkdir is used to make a new directory in remote server
288
296
289 Input:
297 Input:
290 remotefolder - directory name
298 remotefolder - directory name
291
299
292 Return:
300 Return:
293 0 in error case else 1
301 0 in error case else 1
294 """
302 """
295 if not self.status:
303 if not self.status:
296 return 0
304 return 0
297
305
298 try:
306 try:
299 self.__ftpClientObj.mkd(dirname)
307 self.__ftpClientObj.mkd(dirname)
300 except ftplib.all_errors:
308 except ftplib.all_errors:
301 print "Error creating remote folder: %s" %remotefolder
309 print "Error creating remote folder: %s" %remotefolder
302 return 0
310 return 0
303
311
304 return 1
312 return 1
305
313
306 def cd(self, remotefolder):
314 def cd(self, remotefolder):
307 """
315 """
308 cd is used to change remote working directory on server
316 cd is used to change remote working directory on server
309
317
310 Input:
318 Input:
311 remotefolder - current working directory
319 remotefolder - current working directory
312
320
313 Affects:
321 Affects:
314 self.remotefolder
322 self.remotefolder
315
323
316 Return:
324 Return:
317 0 in case of error else 1
325 0 in case of error else 1
318 """
326 """
319 if not self.status:
327 if not self.status:
320 return 0
328 return 0
321
329
322 if remotefolder == self.remotefolder:
330 if remotefolder == self.remotefolder:
323 return 1
331 return 1
324
332
325 try:
333 try:
326 self.__ftpClientObj.cwd(remotefolder)
334 self.__ftpClientObj.cwd(remotefolder)
327 except ftplib.all_errors:
335 except ftplib.all_errors:
328 print 'Error changing to %s' %remotefolder
336 print 'Error changing to %s' %remotefolder
329 print 'Trying to create remote folder'
337 print 'Trying to create remote folder'
330
338
331 if not self.mkdir(remotefolder):
339 if not self.mkdir(remotefolder):
332 print 'Remote folder could not be created'
340 print 'Remote folder could not be created'
333 return 0
341 return 0
334
342
335 try:
343 try:
336 self.__ftpClientObj.cwd(remotefolder)
344 self.__ftpClientObj.cwd(remotefolder)
337 except ftplib.all_errors:
345 except ftplib.all_errors:
338 return 0
346 return 0
339
347
340 self.remotefolder = remotefolder
348 self.remotefolder = remotefolder
341
349
342 return 1
350 return 1
343
351
344 def sendFile(self, fullfilename):
352 def sendFile(self, fullfilename):
345
353
346 if not self.status:
354 if not self.status:
347 return 0
355 return 0
348
356
349 file = open(fullfilename, 'rb')
357 file = open(fullfilename, 'rb')
350
358
351 filename = os.path.basename(fullfilename)
359 filename = os.path.basename(fullfilename)
352
360
353 command = "STOR %s" %filename
361 command = "STOR %s" %filename
354
362
355 try:
363 try:
356 self.__ftpClientObj.storbinary(command, file)
364 self.__ftpClientObj.storbinary(command, file)
357 except ftplib.all_errors:
365 except ftplib.all_errors:
358 return 0
366 return 0
359
367
360 try:
368 try:
361 self.__ftpClientObj.sendcmd('SITE CHMOD 755 ' + filename)
369 self.__ftpClientObj.sendcmd('SITE CHMOD 755 ' + filename)
362 except ftplib.all_errors, e:
370 except ftplib.all_errors, e:
363 print e
371 print e
364
372
365 file.close()
373 file.close()
366
374
367 return 1
375 return 1
368
376
369 class SSHClient(Remote):
377 class SSHClient(Remote):
370
378
371 __sshClientObj = None
379 __sshClientObj = None
372 __scpClientObj = None
380 __scpClientObj = None
373
381
374 def __init__(self, server, username, password, remotefolder, period=60):
382 def __init__(self, server, username, password, remotefolder, period=60):
375 """
383 """
376 """
384 """
377 Remote.__init__(self, server, username, password, remotefolder, period)
385 Remote.__init__(self, server, username, password, remotefolder, period)
378
386
379 def open(self, server, username, password, remotefolder, port=22):
387 def open(self, server, username, password, remotefolder, port=22):
380
388
381 """
389 """
382 This method is used to set SSH parameters and establish a connection to a remote server
390 This method is used to set SSH parameters and establish a connection to a remote server
383
391
384 Inputs:
392 Inputs:
385 server - remote server IP Address
393 server - remote server IP Address
386
394
387 username - remote server Username
395 username - remote server Username
388
396
389 password - remote server password
397 password - remote server password
390
398
391 remotefolder - remote server current working directory
399 remotefolder - remote server current working directory
392
400
393 Return: void
401 Return: void
394
402
395 Affects:
403 Affects:
396 self.status - in case of error or fail connection this parameter is set to 0 else 1
404 self.status - in case of error or fail connection this parameter is set to 0 else 1
397
405
398 """
406 """
399
407
400 if server == None:
408 if server == None:
401 raise ValueError, "SSH server should be defined"
409 raise ValueError, "SSH server should be defined"
402
410
403 if username == None:
411 if username == None:
404 raise ValueError, "SSH username should be defined"
412 raise ValueError, "SSH username should be defined"
405
413
406 if password == None:
414 if password == None:
407 raise ValueError, "SSH password should be defined"
415 raise ValueError, "SSH password should be defined"
408
416
409 if remotefolder == None:
417 if remotefolder == None:
410 raise ValueError, "SSH remote folder should be defined"
418 raise ValueError, "SSH remote folder should be defined"
411
419
412 try:
420 try:
413 sshClientObj = paramiko.SSHClient()
421 sshClientObj = paramiko.SSHClient()
414 except:
422 except:
415 print "SSH server connection fail: %s" %server
423 print "SSH server connection fail: %s" %server
416 self.status = 0
424 self.status = 0
417 return 0
425 return 0
418
426
419 sshClientObj.load_system_host_keys()
427 sshClientObj.load_system_host_keys()
420 sshClientObj.set_missing_host_key_policy(paramiko.WarningPolicy())
428 sshClientObj.set_missing_host_key_policy(paramiko.WarningPolicy())
421
429
422 try:
430 try:
423 sshClientObj.connect(server, username=username, password=password, port=port)
431 sshClientObj.connect(server, username=username, password=password, port=port)
424 except :
432 except :
425 print "SSH username or password are incorrect: %s"
433 print "SSH username or password are incorrect: %s"
426 self.status = 0
434 self.status = 0
427 return 0
435 return 0
428
436
429 scpClientObj = scp.SCPClient(sshClientObj.get_transport(), socket_timeout=30)
437 scpClientObj = scp.SCPClient(sshClientObj.get_transport(), socket_timeout=30)
430
438
431 if remotefolder == None:
439 if remotefolder == None:
432 remotefolder = self.pwd()
440 remotefolder = self.pwd()
433
441
434 self.server = server
442 self.server = server
435 self.username = username
443 self.username = username
436 self.password = password
444 self.password = password
437 self.__sshClientObj = sshClientObj
445 self.__sshClientObj = sshClientObj
438 self.__scpClientObj = scpClientObj
446 self.__scpClientObj = scpClientObj
439 self.status = 1
447 self.status = 1
440
448
441 if not self.cd(remotefolder):
449 if not self.cd(remotefolder):
442 raise ValueError, "Could not access to remote folder: %s" %remotefolder
450 raise ValueError, "Could not access to remote folder: %s" %remotefolder
443 return 0
451 return 0
444
452
445 self.remotefolder = remotefolder
453 self.remotefolder = remotefolder
446
454
447 return 1
455 return 1
448
456
449 def close(self):
457 def close(self):
450 """
458 """
451 Close connection to remote server
459 Close connection to remote server
452 """
460 """
453 if not self.status:
461 if not self.status:
454 return 0
462 return 0
455
463
456 self.__sshObj.close()
464 self.__sshObj.close()
457
465
458 def __execute(self, command):
466 def __execute(self, command):
459 """
467 """
460 __execute a command on remote server
468 __execute a command on remote server
461
469
462 Input:
470 Input:
463 command - Exmaple 'ls -l'
471 command - Exmaple 'ls -l'
464
472
465 Return:
473 Return:
466 0 in error case else 1
474 0 in error case else 1
467 """
475 """
468 if not self.status:
476 if not self.status:
469 return 0
477 return 0
470
478
471 stdin, stdout, stderr = self.__sshClientObj.exec_command(command)
479 stdin, stdout, stderr = self.__sshClientObj.exec_command(command)
472
480
473 result = stderr.readlines()
481 result = stderr.readlines()
474 if len(result) > 1:
482 if len(result) > 1:
475 return 0
483 return 0
476
484
477 result = stdout.readlines()
485 result = stdout.readlines()
478 if len(result) > 1:
486 if len(result) > 1:
479 return result[0][:-1]
487 return result[0][:-1]
480
488
481 return 1
489 return 1
482
490
483 def mkdir(self, remotefolder):
491 def mkdir(self, remotefolder):
484 """
492 """
485 mkdir is used to make a new directory in remote server
493 mkdir is used to make a new directory in remote server
486
494
487 Input:
495 Input:
488 remotefolder - directory name
496 remotefolder - directory name
489
497
490 Return:
498 Return:
491 0 in error case else 1
499 0 in error case else 1
492 """
500 """
493
501
494 command = 'mkdir %s' %remotefolder
502 command = 'mkdir %s' %remotefolder
495
503
496 return self.__execute(command)
504 return self.__execute(command)
497
505
498 def pwd(self):
506 def pwd(self):
499
507
500 command = 'pwd'
508 command = 'pwd'
501
509
502 return self.__execute(command)
510 return self.__execute(command)
503
511
504 def cd(self, remotefolder):
512 def cd(self, remotefolder):
505 """
513 """
506 cd is used to change remote working directory on server
514 cd is used to change remote working directory on server
507
515
508 Input:
516 Input:
509 remotefolder - current working directory
517 remotefolder - current working directory
510
518
511 Affects:
519 Affects:
512 self.remotefolder
520 self.remotefolder
513
521
514 Return:
522 Return:
515 0 in case of error else 1
523 0 in case of error else 1
516 """
524 """
517 if not self.status:
525 if not self.status:
518 return 0
526 return 0
519
527
520 if remotefolder == self.remotefolder:
528 if remotefolder == self.remotefolder:
521 return 1
529 return 1
522
530
523 chk_command = "cd %s; pwd" %remotefolder
531 chk_command = "cd %s; pwd" %remotefolder
524 mkdir_command = "mkdir %s" %remotefolder
532 mkdir_command = "mkdir %s" %remotefolder
525
533
526 if not self.__execute(chk_command):
534 if not self.__execute(chk_command):
527 if not self.__execute(mkdir_command):
535 if not self.__execute(mkdir_command):
528 self.remotefolder = None
536 self.remotefolder = None
529 return 0
537 return 0
530
538
531 self.remotefolder = remotefolder
539 self.remotefolder = remotefolder
532
540
533 return 1
541 return 1
534
542
535 def sendFile(self, fullfilename):
543 def sendFile(self, fullfilename):
536
544
537 if not self.status:
545 if not self.status:
538 return 0
546 return 0
539
547
540 try:
548 try:
541 self.__scpClientObj.put(fullfilename, remote_path=self.remotefolder)
549 self.__scpClientObj.put(fullfilename, remote_path=self.remotefolder)
542 except:
550 except:
543 return 0
551 return 0
544
552
545 remotefile = os.path.join(self.remotefolder, os.path.split(fullfilename)[-1])
553 remotefile = os.path.join(self.remotefolder, os.path.split(fullfilename)[-1])
546 command = 'chmod 775 %s' %remotefile
554 command = 'chmod 775 %s' %remotefile
547
555
548 return self.__execute(command)
556 return self.__execute(command)
549
557
550 class SendToServer(ProcessingUnit):
558 class SendToServer(ProcessingUnit):
551
559
552 def __init__(self):
560 def __init__(self):
553
561
554 ProcessingUnit.__init__(self)
562 ProcessingUnit.__init__(self)
555
563
556 self.isConfig = False
564 self.isConfig = False
557 self.clientObj = None
565 self.clientObj = None
558
566
559 def setup(self, server, username, password, remotefolder, localfolder, ext='.png', period=60, protocol='ftp', **kwargs):
567 def setup(self, server, username, password, remotefolder, localfolder, ext='.png', period=60, protocol='ftp', **kwargs):
560
568
561 self.clientObj = None
569 self.clientObj = None
562 self.localfolder = localfolder
570 self.localfolder = localfolder
563 self.ext = ext
571 self.ext = ext
564 self.period = period
572 self.period = period
565
573
566 if str.lower(protocol) == 'ftp':
574 if str.lower(protocol) == 'ftp':
567 self.clientObj = FTPClient(server, username, password, remotefolder, period)
575 self.clientObj = FTPClient(server, username, password, remotefolder, period)
568
576
569 if str.lower(protocol) == 'ssh':
577 if str.lower(protocol) == 'ssh':
570 self.clientObj = SSHClient(server, username, password, remotefolder, period)
578 self.clientObj = SSHClient(server, username, password, remotefolder, period)
571
579
572 if not self.clientObj:
580 if not self.clientObj:
573 raise ValueError, "%s has been chosen as remote access protocol but it is not valid" %protocol
581 raise ValueError, "%s has been chosen as remote access protocol but it is not valid" %protocol
574
582
575 self.clientObj.start()
583 self.clientObj.start()
576
584
577 def findFiles(self):
585 def findFiles(self):
578
586
579 if not type(self.localfolder) == list:
587 if not type(self.localfolder) == list:
580 folderList = [self.localfolder]
588 folderList = [self.localfolder]
581 else:
589 else:
582 folderList = self.localfolder
590 folderList = self.localfolder
583
591
592 #Remove duplicate items
593 folderList = list(set(folderList))
594
584 fullfilenameList = []
595 fullfilenameList = []
585
596
586 for thisFolder in folderList:
597 for thisFolder in folderList:
587
598
599 print "[Remote Server]: Searching files on %s" %thisFolder
600
588 filenameList = glob.glob1(thisFolder, '*%s' %self.ext)
601 filenameList = glob.glob1(thisFolder, '*%s' %self.ext)
589
602
590 if len(filenameList) < 1:
603 if len(filenameList) < 1:
591 continue
604 continue
592
605
593 for thisFile in filenameList:
606 for thisFile in filenameList:
594 fullfilename = os.path.join(thisFolder, thisFile)
607 fullfilename = os.path.join(thisFolder, thisFile)
608
609 if fullfilename in fullfilenameList:
610 continue
611
612 #Only files modified in the last 30 minutes are considered
613 if os.path.getmtime(fullfilename) < time.time() - 30*60:
614 continue
615
595 fullfilenameList.append(fullfilename)
616 fullfilenameList.append(fullfilename)
596
617
597 return fullfilenameList
618 return fullfilenameList
598
619
599 def run(self, **kwargs):
620 def run(self, **kwargs):
600
621
601 if not self.isConfig:
622 if not self.isConfig:
602 self.init = time.time()
623 self.init = time.time()
603 self.setup(**kwargs)
624 self.setup(**kwargs)
604 self.isConfig = True
625 self.isConfig = True
605
626
606 if time.time() - self.init >= self.period:
627 if time.time() - self.init >= self.period:
607 fullfilenameList = self.findFiles()
628 fullfilenameList = self.findFiles()
608
629
609 if self.clientObj.updateFileList(fullfilenameList):
630 if self.clientObj.updateFileList(fullfilenameList):
610 print "[Remote Server]: Sending the next files ", str(fullfilenameList)
631 print "[Remote Server]: Sending the next files ", str(fullfilenameList)
611
632
612 self.init = time.time()
633 self.init = time.time()
613
634
614 def close(self):
635 def close(self):
615 print "[Remote Server] Stopping thread"
636 print "[Remote Server] Stopping thread"
616 self.clientObj.stop()
637 self.clientObj.stop()
617
638
618
639
619 class FTP(object):
640 class FTP(object):
620 """
641 """
621 Ftp is a public class used to define custom File Transfer Protocol from "ftplib" python module
642 Ftp is a public class used to define custom File Transfer Protocol from "ftplib" python module
622
643
623 Non-standard Python modules used: None
644 Non-standard Python modules used: None
624
645
625 Written by "Daniel Suarez":mailto:daniel.suarez@jro.igp.gob.pe Oct. 26, 2010
646 Written by "Daniel Suarez":mailto:daniel.suarez@jro.igp.gob.pe Oct. 26, 2010
626 """
647 """
627
648
628 def __init__(self,server = None, username=None, password=None, remotefolder=None):
649 def __init__(self,server = None, username=None, password=None, remotefolder=None):
629 """
650 """
630 This method is used to setting parameters for FTP and establishing connection to remote server
651 This method is used to setting parameters for FTP and establishing connection to remote server
631
652
632 Inputs:
653 Inputs:
633 server - remote server IP Address
654 server - remote server IP Address
634
655
635 username - remote server Username
656 username - remote server Username
636
657
637 password - remote server password
658 password - remote server password
638
659
639 remotefolder - remote server current working directory
660 remotefolder - remote server current working directory
640
661
641 Return: void
662 Return: void
642
663
643 Affects:
664 Affects:
644 self.status - in Error Case or Connection Failed this parameter is set to 1 else 0
665 self.status - in Error Case or Connection Failed this parameter is set to 1 else 0
645
666
646 self.folderList - sub-folder list of remote folder
667 self.folderList - sub-folder list of remote folder
647
668
648 self.fileList - file list of remote folder
669 self.fileList - file list of remote folder
649
670
650
671
651 """
672 """
652
673
653 if ((server == None) and (username==None) and (password==None) and (remotefolder==None)):
674 if ((server == None) and (username==None) and (password==None) and (remotefolder==None)):
654 server, username, password, remotefolder = self.parmsByDefault()
675 server, username, password, remotefolder = self.parmsByDefault()
655
676
656 self.server = server
677 self.server = server
657 self.username = username
678 self.username = username
658 self.password = password
679 self.password = password
659 self.remotefolder = remotefolder
680 self.remotefolder = remotefolder
660 self.file = None
681 self.file = None
661 self.ftp = None
682 self.ftp = None
662 self.status = 0
683 self.status = 0
663
684
664 try:
685 try:
665 self.ftp = ftplib.FTP(self.server)
686 self.ftp = ftplib.FTP(self.server)
666 self.ftp.login(self.username,self.password)
687 self.ftp.login(self.username,self.password)
667 self.ftp.cwd(self.remotefolder)
688 self.ftp.cwd(self.remotefolder)
668 # print 'Connect to FTP Server: Successfully'
689 # print 'Connect to FTP Server: Successfully'
669
690
670 except ftplib.all_errors:
691 except ftplib.all_errors:
671 print 'Error FTP Service'
692 print 'Error FTP Service'
672 self.status = 1
693 self.status = 1
673 return
694 return
674
695
675
696
676
697
677 self.dirList = []
698 self.dirList = []
678
699
679 try:
700 try:
680 self.dirList = self.ftp.nlst()
701 self.dirList = self.ftp.nlst()
681
702
682 except ftplib.error_perm, resp:
703 except ftplib.error_perm, resp:
683 if str(resp) == "550 No files found":
704 if str(resp) == "550 No files found":
684 print "no files in this directory"
705 print "no files in this directory"
685 self.status = 1
706 self.status = 1
686 return
707 return
687
708
688 except ftplib.all_errors:
709 except ftplib.all_errors:
689 print 'Error Displaying Dir-Files'
710 print 'Error Displaying Dir-Files'
690 self.status = 1
711 self.status = 1
691 return
712 return
692
713
693 self.fileList = []
714 self.fileList = []
694 self.folderList = []
715 self.folderList = []
695 #only for test
716 #only for test
696 for f in self.dirList:
717 for f in self.dirList:
697 name, ext = os.path.splitext(f)
718 name, ext = os.path.splitext(f)
698 if ext != '':
719 if ext != '':
699 self.fileList.append(f)
720 self.fileList.append(f)
700 # print 'filename: %s - size: %d'%(f,self.ftp.size(f))
721 # print 'filename: %s - size: %d'%(f,self.ftp.size(f))
701
722
702 def parmsByDefault(self):
723 def parmsByDefault(self):
703 server = 'jro-app.igp.gob.pe'
724 server = 'jro-app.igp.gob.pe'
704 username = 'wmaster'
725 username = 'wmaster'
705 password = 'mst2010vhf'
726 password = 'mst2010vhf'
706 remotefolder = '/home/wmaster/graficos'
727 remotefolder = '/home/wmaster/graficos'
707
728
708 return server, username, password, remotefolder
729 return server, username, password, remotefolder
709
730
710
731
711 def mkd(self,dirname):
732 def mkd(self,dirname):
712 """
733 """
713 mkd is used to make directory in remote server
734 mkd is used to make directory in remote server
714
735
715 Input:
736 Input:
716 dirname - directory name
737 dirname - directory name
717
738
718 Return:
739 Return:
719 1 in error case else 0
740 1 in error case else 0
720 """
741 """
721 try:
742 try:
722 self.ftp.mkd(dirname)
743 self.ftp.mkd(dirname)
723 except:
744 except:
724 print 'Error creating remote folder:%s'%dirname
745 print 'Error creating remote folder:%s'%dirname
725 return 1
746 return 1
726
747
727 return 0
748 return 0
728
749
729
750
730 def delete(self,filename):
751 def delete(self,filename):
731 """
752 """
732 delete is used to delete file in current working directory of remote server
753 delete is used to delete file in current working directory of remote server
733
754
734 Input:
755 Input:
735 filename - filename to delete in remote folder
756 filename - filename to delete in remote folder
736
757
737 Return:
758 Return:
738 1 in error case else 0
759 1 in error case else 0
739 """
760 """
740
761
741 try:
762 try:
742 self.ftp.delete(filename)
763 self.ftp.delete(filename)
743 except:
764 except:
744 print 'Error deleting remote file:%s'%filename
765 print 'Error deleting remote file:%s'%filename
745 return 1
766 return 1
746
767
747 return 0
768 return 0
748
769
749 def download(self,filename,localfolder):
770 def download(self,filename,localfolder):
750 """
771 """
751 download is used to downloading file from remote folder into local folder
772 download is used to downloading file from remote folder into local folder
752
773
753 Inputs:
774 Inputs:
754 filename - filename to donwload
775 filename - filename to donwload
755
776
756 localfolder - directory local to store filename
777 localfolder - directory local to store filename
757
778
758 Returns:
779 Returns:
759 self.status - 1 in error case else 0
780 self.status - 1 in error case else 0
760 """
781 """
761
782
762 self.status = 0
783 self.status = 0
763
784
764
785
765 if not(filename in self.fileList):
786 if not(filename in self.fileList):
766 print 'filename:%s not exists'%filename
787 print 'filename:%s not exists'%filename
767 self.status = 1
788 self.status = 1
768 return self.status
789 return self.status
769
790
770 newfilename = os.path.join(localfolder,filename)
791 newfilename = os.path.join(localfolder,filename)
771
792
772 self.file = open(newfilename, 'wb')
793 self.file = open(newfilename, 'wb')
773
794
774 try:
795 try:
775 print 'Download: ' + filename
796 print 'Download: ' + filename
776 self.ftp.retrbinary('RETR ' + filename, self.__handleDownload)
797 self.ftp.retrbinary('RETR ' + filename, self.__handleDownload)
777 print 'Download Complete'
798 print 'Download Complete'
778 except ftplib.all_errors:
799 except ftplib.all_errors:
779 print 'Error Downloading ' + filename
800 print 'Error Downloading ' + filename
780 self.status = 1
801 self.status = 1
781 return self.status
802 return self.status
782
803
783 self.file.close()
804 self.file.close()
784
805
785 return self.status
806 return self.status
786
807
787
808
788 def __handleDownload(self,block):
809 def __handleDownload(self,block):
789 """
810 """
790 __handleDownload is used to handle writing file
811 __handleDownload is used to handle writing file
791 """
812 """
792 self.file.write(block)
813 self.file.write(block)
793
814
794
815
795 def upload(self,filename,remotefolder=None):
816 def upload(self,filename,remotefolder=None):
796 """
817 """
797 upload is used to uploading local file to remote directory
818 upload is used to uploading local file to remote directory
798
819
799 Inputs:
820 Inputs:
800 filename - full path name of local file to store in remote directory
821 filename - full path name of local file to store in remote directory
801
822
802 remotefolder - remote directory
823 remotefolder - remote directory
803
824
804 Returns:
825 Returns:
805 self.status - 1 in error case else 0
826 self.status - 1 in error case else 0
806 """
827 """
807
828
808 if remotefolder == None:
829 if remotefolder == None:
809 remotefolder = self.remotefolder
830 remotefolder = self.remotefolder
810
831
811 self.status = 0
832 self.status = 0
812
833
813 try:
834 try:
814 self.ftp.cwd(remotefolder)
835 self.ftp.cwd(remotefolder)
815
836
816 self.file = open(filename, 'rb')
837 self.file = open(filename, 'rb')
817
838
818 (head, tail) = os.path.split(filename)
839 (head, tail) = os.path.split(filename)
819
840
820 command = "STOR " + tail
841 command = "STOR " + tail
821
842
822 print 'Uploading: ' + tail
843 print 'Uploading: ' + tail
823 self.ftp.storbinary(command, self.file)
844 self.ftp.storbinary(command, self.file)
824 print 'Upload Completed'
845 print 'Upload Completed'
825
846
826 except ftplib.all_errors:
847 except ftplib.all_errors:
827 print 'Error Uploading ' + tail
848 print 'Error Uploading ' + tail
828 self.status = 1
849 self.status = 1
829 return self.status
850 return self.status
830
851
831 self.file.close()
852 self.file.close()
832
853
833 #back to initial directory in __init__()
854 #back to initial directory in __init__()
834 self.ftp.cwd(self.remotefolder)
855 self.ftp.cwd(self.remotefolder)
835
856
836 return self.status
857 return self.status
837
858
838
859
839 def dir(self,remotefolder):
860 def dir(self,remotefolder):
840 """
861 """
841 dir is used to change working directory of remote server and get folder and file list
862 dir is used to change working directory of remote server and get folder and file list
842
863
843 Input:
864 Input:
844 remotefolder - current working directory
865 remotefolder - current working directory
845
866
846 Affects:
867 Affects:
847 self.fileList - file list of working directory
868 self.fileList - file list of working directory
848
869
849 Return:
870 Return:
850 infoList - list with filenames and size of file in bytes
871 infoList - list with filenames and size of file in bytes
851
872
852 self.folderList - folder list
873 self.folderList - folder list
853 """
874 """
854
875
855 self.remotefolder = remotefolder
876 self.remotefolder = remotefolder
856 print 'Change to ' + self.remotefolder
877 print 'Change to ' + self.remotefolder
857 try:
878 try:
858 self.ftp.cwd(remotefolder)
879 self.ftp.cwd(remotefolder)
859 except ftplib.all_errors:
880 except ftplib.all_errors:
860 print 'Error Change to ' + self.remotefolder
881 print 'Error Change to ' + self.remotefolder
861 infoList = None
882 infoList = None
862 self.folderList = None
883 self.folderList = None
863 return infoList,self.folderList
884 return infoList,self.folderList
864
885
865 self.dirList = []
886 self.dirList = []
866
887
867 try:
888 try:
868 self.dirList = self.ftp.nlst()
889 self.dirList = self.ftp.nlst()
869
890
870 except ftplib.error_perm, resp:
891 except ftplib.error_perm, resp:
871 if str(resp) == "550 No files found":
892 if str(resp) == "550 No files found":
872 print "no files in this directory"
893 print "no files in this directory"
873 infoList = None
894 infoList = None
874 self.folderList = None
895 self.folderList = None
875 return infoList,self.folderList
896 return infoList,self.folderList
876 except ftplib.all_errors:
897 except ftplib.all_errors:
877 print 'Error Displaying Dir-Files'
898 print 'Error Displaying Dir-Files'
878 infoList = None
899 infoList = None
879 self.folderList = None
900 self.folderList = None
880 return infoList,self.folderList
901 return infoList,self.folderList
881
902
882 infoList = []
903 infoList = []
883 self.fileList = []
904 self.fileList = []
884 self.folderList = []
905 self.folderList = []
885 for f in self.dirList:
906 for f in self.dirList:
886 name,ext = os.path.splitext(f)
907 name,ext = os.path.splitext(f)
887 if ext != '':
908 if ext != '':
888 self.fileList.append(f)
909 self.fileList.append(f)
889 value = (f,self.ftp.size(f))
910 value = (f,self.ftp.size(f))
890 infoList.append(value)
911 infoList.append(value)
891
912
892 if ext == '':
913 if ext == '':
893 self.folderList.append(f)
914 self.folderList.append(f)
894
915
895 return infoList,self.folderList
916 return infoList,self.folderList
896
917
897
918
898 def close(self):
919 def close(self):
899 """
920 """
900 close is used to close and end FTP connection
921 close is used to close and end FTP connection
901
922
902 Inputs: None
923 Inputs: None
903
924
904 Return: void
925 Return: void
905
926
906 """
927 """
907 self.ftp.close()
928 self.ftp.close()
908
929
909 class SendByFTP(Operation):
930 class SendByFTP(Operation):
910
931
911 def __init__(self):
932 def __init__(self):
912
933
913 self.status = 1
934 self.status = 1
914 self.counter = 0
935 self.counter = 0
915
936
916 def error_print(self, ValueError):
937 def error_print(self, ValueError):
917
938
918 print ValueError, 'Error FTP'
939 print ValueError, 'Error FTP'
919 print "don't worry the program is running..."
940 print "don't worry the program is running..."
920
941
921 def worker_ftp(self, server, username, password, remotefolder, filenameList):
942 def worker_ftp(self, server, username, password, remotefolder, filenameList):
922
943
923 self.ftpClientObj = FTP(server, username, password, remotefolder)
944 self.ftpClientObj = FTP(server, username, password, remotefolder)
924 for filename in filenameList:
945 for filename in filenameList:
925 self.ftpClientObj.upload(filename)
946 self.ftpClientObj.upload(filename)
926 self.ftpClientObj.close()
947 self.ftpClientObj.close()
927
948
928 def ftp_thread(self, server, username, password, remotefolder):
949 def ftp_thread(self, server, username, password, remotefolder):
929 if not(self.status):
950 if not(self.status):
930 return
951 return
931
952
953 import multiprocessing
954
932 p = multiprocessing.Process(target=self.worker_ftp, args=(server, username, password, remotefolder, self.filenameList,))
955 p = multiprocessing.Process(target=self.worker_ftp, args=(server, username, password, remotefolder, self.filenameList,))
933 p.start()
956 p.start()
934
957
935 p.join(3)
958 p.join(3)
936
959
937 if p.is_alive():
960 if p.is_alive():
938 p.terminate()
961 p.terminate()
939 p.join()
962 p.join()
940 print 'killing ftp process...'
963 print 'killing ftp process...'
941 self.status = 0
964 self.status = 0
942 return
965 return
943
966
944 self.status = 1
967 self.status = 1
945 return
968 return
946
969
947 def filterByExt(self, ext, localfolder):
970 def filterByExt(self, ext, localfolder):
948 fnameList = glob.glob1(localfolder,ext)
971 fnameList = glob.glob1(localfolder,ext)
949 self.filenameList = [os.path.join(localfolder,x) for x in fnameList]
972 self.filenameList = [os.path.join(localfolder,x) for x in fnameList]
950
973
951 if len(self.filenameList) == 0:
974 if len(self.filenameList) == 0:
952 self.status = 0
975 self.status = 0
953
976
954 def run(self, dataOut, ext, localfolder, remotefolder, server, username, password, period=1):
977 def run(self, dataOut, ext, localfolder, remotefolder, server, username, password, period=1):
955
978
956 self.counter += 1
979 self.counter += 1
957 if self.counter >= period:
980 if self.counter >= period:
958 self.filterByExt(ext, localfolder)
981 self.filterByExt(ext, localfolder)
959
982
960 self.ftp_thread(server, username, password, remotefolder)
983 self.ftp_thread(server, username, password, remotefolder)
961
984
962 self.counter = 0
985 self.counter = 0
963
986
964 self.status = 1
987 self.status = 1
965
988
General Comments 0
You need to be logged in to leave comments. Login now