##// END OF EJS Templates
schain
RSS
jroutils_ftp.py: Remote Server is opened and closed each time a file is sent
Miguel Valdez -
r653:ae332cfe81c4
parent child
Show More
Show file before | Show file after | Hide comments
@@ -1,7 +1,8
1 VERSIONS:
1 VERSIONS:
2
2
3 2.1.2:
3 2.1.2:
4 -jroutils_ftp.py: Server thread stops when an error sending file has been detected
4 -jroutils_ftp.py: Bug fixed, Any error sending file stopped the Server Thread
5 Server thread opens and closes remote server each time file list is sent
5 -jroplot_spectra.py: Noise path was not being created saving noise data.
6 -jroplot_spectra.py: Noise path was not being created saving noise data.
6 -jroIO_base.py: startTime can be greater than endTime
7 -jroIO_base.py: startTime can be greater than endTime
7 -jroplot_heispectra.py: SpectraHeisScope was not showing the right channel
8 -jroplot_heispectra.py: SpectraHeisScope was not showing the right channel
Show file before | Show file after | Hide comments
@@ -1,982 +1,1007
1 '''
1 '''
2 @author: Daniel Suarez
2 @author: Daniel Suarez
3 '''
3 '''
4 import os
4 import os
5 import glob
5 import glob
6 import ftplib
6 import ftplib
7
7
8 try:
8 try:
9 import paramiko
9 import paramiko
10 import scp
10 import scp
11 except:
11 except:
12 print "You should install paramiko and scp libraries \nif you want to use SSH protocol to upload files to the server"
12 print "You should install paramiko and scp libraries \nif you want to use SSH protocol to upload files to the server"
13
13
14 import time
14 import time
15
15
16 import threading
16 import threading
17 Thread = threading.Thread
17 Thread = threading.Thread
18
18
19 # try:
19 # try:
20 # from gevent import sleep
20 # from gevent import sleep
21 # except:
21 # except:
22 from time import sleep
22 from time import sleep
23
23
24 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation
24 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation
25
25
26 class Remote(Thread):
26 class Remote(Thread):
27 """
27 """
28 Remote is a parent class used to define the behaviour of FTP and SSH class. These clases are
28 Remote is a parent class used to define the behaviour of FTP and SSH class. These clases are
29 used to upload or download files remotely.
29 used to upload or download files remotely.
30
30
31 Non-standard Python modules used:
31 Non-standard Python modules used:
32 None
32 None
33
33
34 Written by:
34 Written by:
35
35
36 "Miguel Urco":mailto:miguel.urco@jro.igp.gob.pe Jun. 03, 2015
36 "Miguel Urco":mailto:miguel.urco@jro.igp.gob.pe Jun. 03, 2015
37
37
38 """
38 """
39
39
40 server = None
40 server = None
41 username = None
41 username = None
42 password = None
42 password = None
43 remotefolder = None
43 remotefolder = None
44
44
45 period = 60
45 period = 60
46 fileList = []
46 fileList = []
47 bussy = False
47 bussy = False
48
48
49 def __init__(self, server, username, password, remotefolder, period=60):
49 def __init__(self, server, username, password, remotefolder, period=60):
50
50
51 Thread.__init__(self)
51 Thread.__init__(self)
52
52
53 self.setDaemon(True)
53 self.setDaemon(True)
54
54
55 self.status = 0
55 self.status = 0
56
56
57 self.__server = server
58 self.__username = username
59 self.__password = password
60 self.__remotefolder = remotefolder
61
57 self.period = period
62 self.period = period
63
58 self.fileList = []
64 self.fileList = []
59 self.bussy = False
65 self.bussy = False
60
66
61 self.stopFlag = False
67 self.stopFlag = False
62
68
63 print "[Remote Server] Opening server: %s" %server
69 print "[Remote Server] Opening server: %s" %self.__server
64 if self.open(server, username, password, remotefolder):
70 if self.open(self.__server, self.__username, self.__password, self.__remotefolder):
65 print "[Remote Server] %s server was opened successfully" %server
71 print "[Remote Server] %s server was opened successfully" %self.__server
66
72
73 self.close()
74
67 self.mutex = threading.Lock()
75 self.mutex = threading.Lock()
68
76
69 def stop(self):
77 def stop(self):
70
78
71 self.stopFlag = True
79 self.stopFlag = True
72 self.join(10)
80 self.join(10)
73
81
74 def open(self):
82 def open(self):
75 """
83 """
76 Connect to server and create a connection class (FTP or SSH) to remote server.
84 Connect to server and create a connection class (FTP or SSH) to remote server.
77 """
85 """
78 raise NotImplementedError, "Implement this method in child class"
86 raise NotImplementedError, "Implement this method in child class"
79
87
80 def close(self):
88 def close(self):
81 """
89 """
82 Close connection to server
90 Close connection to server
83 """
91 """
84 raise NotImplementedError, "Implement this method in child class"
92 raise NotImplementedError, "Implement this method in child class"
85
93
86 def mkdir(self, remotefolder):
94 def mkdir(self, remotefolder):
87 """
95 """
88 Create a folder remotely
96 Create a folder remotely
89 """
97 """
90 raise NotImplementedError, "Implement this method in child class"
98 raise NotImplementedError, "Implement this method in child class"
91
99
92 def cd(self, remotefolder):
100 def cd(self, remotefolder):
93 """
101 """
94 Change working directory in remote server
102 Change working directory in remote server
95 """
103 """
96 raise NotImplementedError, "Implement this method in child class"
104 raise NotImplementedError, "Implement this method in child class"
97
105
98 def download(self, filename, localfolder=None):
106 def download(self, filename, localfolder=None):
99 """
107 """
100 Download a file from server to local host
108 Download a file from server to local host
101 """
109 """
102 raise NotImplementedError, "Implement this method in child class"
110 raise NotImplementedError, "Implement this method in child class"
103
111
104 def sendFile(self, fullfilename):
112 def sendFile(self, fullfilename):
105 """
113 """
106 sendFile method is used to upload a local file to the current directory in remote server
114 sendFile method is used to upload a local file to the current directory in remote server
107
115
108 Inputs:
116 Inputs:
109 fullfilename - full path name of local file to store in remote directory
117 fullfilename - full path name of local file to store in remote directory
110
118
111 Returns:
119 Returns:
112 0 in error case else 1
120 0 in error case else 1
113 """
121 """
114 raise NotImplementedError, "Implement this method in child class"
122 raise NotImplementedError, "Implement this method in child class"
115
123
116 def upload(self, fullfilename, remotefolder=None):
124 def upload(self, fullfilename, remotefolder=None):
117 """
125 """
118 upload method is used to upload a local file to remote directory. This method changes
126 upload method is used to upload a local file to remote directory. This method changes
119 working directory before sending a file.
127 working directory before sending a file.
120
128
121 Inputs:
129 Inputs:
122 fullfilename - full path name of local file to store in remote directory
130 fullfilename - full path name of local file to store in remote directory
123
131
124 remotefolder - remote directory
132 remotefolder - remote directory
125
133
126 Returns:
134 Returns:
127 0 in error case else 1
135 0 in error case else 1
128 """
136 """
129 print "[Remote Server] Uploading %s to %s:%s" %(fullfilename, self.server, self.remotefolder)
137 print "[Remote Server] Uploading %s to %s:%s" %(fullfilename, self.server, self.remotefolder)
130
138
131 if not self.status:
139 if not self.status:
132 return 0
140 return 0
133
141
134 if remotefolder == None:
142 if remotefolder == None:
135 remotefolder = self.remotefolder
143 remotefolder = self.remotefolder
136
144
137 if not self.cd(remotefolder):
145 if not self.cd(remotefolder):
138 return 0
146 return 0
139
147
140 if not self.sendFile(fullfilename):
148 if not self.sendFile(fullfilename):
141 print "[Remote Server] Error uploading file %s" %fullfilename
149 print "[Remote Server] Error uploading file %s" %fullfilename
142 return 0
150 return 0
143
151
144 print "[Remote Server] upload finished successfully"
152 print "[Remote Server] upload finished successfully"
145
153
146 return 1
154 return 1
147
155
148 def delete(self, filename):
156 def delete(self, filename):
149 """
157 """
150 Remove a file from remote server
158 Remove a file from remote server
151 """
159 """
152 pass
160 pass
153
161
154 def updateFileList(self, fileList):
162 def updateFileList(self, fileList):
155 """
163 """
156 Remove a file from remote server
164 Remove a file from remote server
157 """
165 """
158
166
159 if fileList == self.fileList:
167 if fileList == self.fileList:
160 return 0
168 return 0
161
169
162 self.mutex.acquire()
170 self.mutex.acquire()
163 # init = time.time()
171 # init = time.time()
164 #
172 #
165 # while(self.bussy):
173 # while(self.bussy):
166 # sleep(0.1)
174 # sleep(0.1)
167 # if time.time() - init > 2*self.period:
175 # if time.time() - init > 2*self.period:
168 # return 0
176 # return 0
169
177
170 self.fileList = fileList
178 self.fileList = fileList
171 self.mutex.release()
179 self.mutex.release()
172 return 1
180 return 1
173
181
174 def run(self):
182 def run(self):
175
183
176 if not self.status:
184 if not self.status:
177 print "Finishing FTP service"
185 print "Finishing FTP service"
178 return
186 return
179
187
180 if not self.cd(self.remotefolder):
188 if not self.cd(self.remotefolder):
181 raise ValueError, "Could not access to the new remote directory: %s" %self.remotefolder
189 raise ValueError, "Could not access to the new remote directory: %s" %self.remotefolder
182
190
183 while True:
191 while True:
184
192
185 for i in range(self.period):
193 for i in range(self.period):
186 if self.stopFlag:
194 if self.stopFlag:
187 break
195 break
188 sleep(1)
196 sleep(1)
189
197
190 if self.stopFlag:
198 if self.stopFlag:
191 break
199 break
192
200
193 # self.bussy = True
201 # self.bussy = True
194 self.mutex.acquire()
202 self.mutex.acquire()
195
203
204 print "[Remote Server] Opening %s" %self.__server
205 if not self.open(self.__server, self.__username, self.__password, self.__remotefolder):
206 return 0
207
196 for thisFile in self.fileList:
208 for thisFile in self.fileList:
197 self.upload(thisFile, self.remotefolder)
209 self.upload(thisFile, self.remotefolder)
198
210
211 print "[Remote Server] Closing %s" %self.__server
212 self.close()
213
199 self.mutex.release()
214 self.mutex.release()
200 # self.bussy = False
215 # self.bussy = False
201
216
202 print "[Remote Server] Thread stopped successfully"
217 print "[Remote Server] Thread stopped successfully"
203
218
204 class FTPClient(Remote):
219 class FTPClient(Remote):
205
220
206 __ftpClientObj = None
221 __ftpClientObj = None
207
222
208 def __init__(self, server, username, password, remotefolder, period=60):
223 def __init__(self, server, username, password, remotefolder, period=60):
209 """
224 """
210 """
225 """
211 Remote.__init__(self, server, username, password, remotefolder, period)
226 Remote.__init__(self, server, username, password, remotefolder, period)
212
227
213 def open(self, server, username, password, remotefolder):
228 def open(self, server, username, password, remotefolder):
214
229
215 """
230 """
216 This method is used to set FTP parameters and establish a connection to remote server
231 This method is used to set FTP parameters and establish a connection to remote server
217
232
218 Inputs:
233 Inputs:
219 server - remote server IP Address
234 server - remote server IP Address
220
235
221 username - remote server Username
236 username - remote server Username
222
237
223 password - remote server password
238 password - remote server password
224
239
225 remotefolder - remote server current working directory
240 remotefolder - remote server current working directory
226
241
227 Return: void
242 Return:
243 Boolean - Returns 1 if a connection has been established, 0 otherwise
228
244
229 Affects:
245 Affects:
230 self.status - in case of error or fail connection this parameter is set to 0 else 1
246 self.status - in case of error or fail connection this parameter is set to 0 else 1
231
247
232 """
248 """
233
249
234 if server == None:
250 if server == None:
235 raise ValueError, "FTP server should be defined"
251 raise ValueError, "FTP server should be defined"
236
252
237 if username == None:
253 if username == None:
238 raise ValueError, "FTP username should be defined"
254 raise ValueError, "FTP username should be defined"
239
255
240 if password == None:
256 if password == None:
241 raise ValueError, "FTP password should be defined"
257 raise ValueError, "FTP password should be defined"
242
258
243 if remotefolder == None:
259 if remotefolder == None:
244 raise ValueError, "FTP remote folder should be defined"
260 raise ValueError, "FTP remote folder should be defined"
245
261
246 try:
262 try:
247 ftpClientObj = ftplib.FTP(server)
263 ftpClientObj = ftplib.FTP(server)
248 except ftplib.all_errors:
264 except ftplib.all_errors, e:
249 print "FTP server connection fail: %s" %server
265 print "[FTP Server]: FTP server connection fail: %s" %server
266 print "[FTP Server]:", e
250 self.status = 0
267 self.status = 0
251 return 0
268 return 0
252
269
253 try:
270 try:
254 ftpClientObj.login(username, password)
271 ftpClientObj.login(username, password)
255 except ftplib.all_errors:
272 except ftplib.all_errors:
256 print "FTP username or password are incorrect"
273 print "[FTP Server]: FTP username or password are incorrect"
257 self.status = 0
274 self.status = 0
258 return 0
275 return 0
259
276
260 if remotefolder == None:
277 if remotefolder == None:
261 remotefolder = ftpClientObj.pwd()
278 remotefolder = ftpClientObj.pwd()
262 else:
279 else:
263 try:
280 try:
264 ftpClientObj.cwd(remotefolder)
281 ftpClientObj.cwd(remotefolder)
265 except ftplib.all_errors:
282 except ftplib.all_errors:
266 print "FTP remote folder is invalid: %s" %remotefolder
283 print "[FTP Server]: FTP remote folder is invalid: %s" %remotefolder
267 remotefolder = ftpClientObj.pwd()
284 remotefolder = ftpClientObj.pwd()
268
285
269 self.server = server
286 self.server = server
270 self.username = username
287 self.username = username
271 self.password = password
288 self.password = password
272 self.remotefolder = remotefolder
289 self.remotefolder = remotefolder
273 self.__ftpClientObj = ftpClientObj
290 self.__ftpClientObj = ftpClientObj
274 self.status = 1
291 self.status = 1
275
292
276 return 1
293 return 1
277
294
278 def close(self):
295 def close(self):
279 """
296 """
280 Close connection to remote server
297 Close connection to remote server
281 """
298 """
282 if not self.status:
299 if not self.status:
283 return 0
300 return 0
284
301
285 self.__ftpClientObj.close()
302 self.__ftpClientObj.close()
286
303
287 def mkdir(self, remotefolder):
304 def mkdir(self, remotefolder):
288 """
305 """
289 mkdir is used to make a new directory in remote server
306 mkdir is used to make a new directory in remote server
290
307
291 Input:
308 Input:
292 remotefolder - directory name
309 remotefolder - directory name
293
310
294 Return:
311 Return:
295 0 in error case else 1
312 0 in error case else 1
296 """
313 """
297 if not self.status:
314 if not self.status:
298 return 0
315 return 0
299
316
300 try:
317 try:
301 self.__ftpClientObj.mkd(dirname)
318 self.__ftpClientObj.mkd(dirname)
302 except ftplib.all_errors:
319 except ftplib.all_errors:
303 print "Error creating remote folder: %s" %remotefolder
320 print "[FTP Server]: Error creating remote folder: %s" %remotefolder
304 return 0
321 return 0
305
322
306 return 1
323 return 1
307
324
308 def cd(self, remotefolder):
325 def cd(self, remotefolder):
309 """
326 """
310 cd is used to change remote working directory on server
327 cd is used to change remote working directory on server
311
328
312 Input:
329 Input:
313 remotefolder - current working directory
330 remotefolder - current working directory
314
331
315 Affects:
332 Affects:
316 self.remotefolder
333 self.remotefolder
317
334
318 Return:
335 Return:
319 0 in case of error else 1
336 0 in case of error else 1
320 """
337 """
321 if not self.status:
338 if not self.status:
322 return 0
339 return 0
323
340
324 if remotefolder == self.remotefolder:
341 if remotefolder == self.remotefolder:
325 return 1
342 return 1
326
343
327 try:
344 try:
328 self.__ftpClientObj.cwd(remotefolder)
345 self.__ftpClientObj.cwd(remotefolder)
329 except ftplib.all_errors:
346 except ftplib.all_errors:
330 print 'Error changing to %s' %remotefolder
347 print '[FTP Server]: Error changing to %s' %remotefolder
331 print 'Trying to create remote folder'
348 print '[FTP Server]: Trying to create remote folder'
332
349
333 if not self.mkdir(remotefolder):
350 if not self.mkdir(remotefolder):
334 print 'Remote folder could not be created'
351 print '[FTP Server]: Remote folder could not be created'
335 return 0
352 return 0
336
353
337 try:
354 try:
338 self.__ftpClientObj.cwd(remotefolder)
355 self.__ftpClientObj.cwd(remotefolder)
339 except ftplib.all_errors:
356 except ftplib.all_errors:
340 return 0
357 return 0
341
358
342 self.remotefolder = remotefolder
359 self.remotefolder = remotefolder
343
360
344 return 1
361 return 1
345
362
346 def sendFile(self, fullfilename):
363 def sendFile(self, fullfilename):
347
364
348 if not self.status:
365 if not self.status:
349 return 0
366 return 0
350
367
351 file = open(fullfilename, 'rb')
368 fp = open(fullfilename, 'rb')
352
369
353 filename = os.path.basename(fullfilename)
370 filename = os.path.basename(fullfilename)
354
371
355 command = "STOR %s" %filename
372 command = "STOR %s" %filename
356
373
357 try:
374 try:
358 self.__ftpClientObj.storbinary(command, file)
375 self.__ftpClientObj.storbinary(command, fp)
359 except ftplib.all_errors:
376 except ftplib.all_errors, e:
377 print "[FTP Server]:", e
360 return 0
378 return 0
361
379
362 try:
380 try:
363 self.__ftpClientObj.sendcmd('SITE CHMOD 755 ' + filename)
381 self.__ftpClientObj.sendcmd('SITE CHMOD 755 ' + filename)
364 except ftplib.all_errors, e:
382 except ftplib.all_errors, e:
365 print e
383 print "[FTP Server]:", e
366
384
367 file.close()
385 fp.close()
368
386
369 return 1
387 return 1
370
388
371 class SSHClient(Remote):
389 class SSHClient(Remote):
372
390
373 __sshClientObj = None
391 __sshClientObj = None
374 __scpClientObj = None
392 __scpClientObj = None
375
393
376 def __init__(self, server, username, password, remotefolder, period=60):
394 def __init__(self, server, username, password, remotefolder, period=60):
377 """
395 """
378 """
396 """
379 Remote.__init__(self, server, username, password, remotefolder, period)
397 Remote.__init__(self, server, username, password, remotefolder, period)
380
398
381 def open(self, server, username, password, remotefolder, port=22):
399 def open(self, server, username, password, remotefolder, port=22):
382
400
383 """
401 """
384 This method is used to set SSH parameters and establish a connection to a remote server
402 This method is used to set SSH parameters and establish a connection to a remote server
385
403
386 Inputs:
404 Inputs:
387 server - remote server IP Address
405 server - remote server IP Address
388
406
389 username - remote server Username
407 username - remote server Username
390
408
391 password - remote server password
409 password - remote server password
392
410
393 remotefolder - remote server current working directory
411 remotefolder - remote server current working directory
394
412
395 Return: void
413 Return: void
396
414
397 Affects:
415 Affects:
398 self.status - in case of error or fail connection this parameter is set to 0 else 1
416 self.status - in case of error or fail connection this parameter is set to 0 else 1
399
417
400 """
418 """
401
419 import socket
420
402 if server == None:
421 if server == None:
403 raise ValueError, "SSH server should be defined"
422 raise ValueError, "SSH server should be defined"
404
423
405 if username == None:
424 if username == None:
406 raise ValueError, "SSH username should be defined"
425 raise ValueError, "SSH username should be defined"
407
426
408 if password == None:
427 if password == None:
409 raise ValueError, "SSH password should be defined"
428 raise ValueError, "SSH password should be defined"
410
429
411 if remotefolder == None:
430 if remotefolder == None:
412 raise ValueError, "SSH remote folder should be defined"
431 raise ValueError, "SSH remote folder should be defined"
413
432
414 try:
433 sshClientObj = paramiko.SSHClient()
415 sshClientObj = paramiko.SSHClient()
416 except:
417 print "SSH server connection fail: %s" %server
418 self.status = 0
419 return 0
420
434
421 sshClientObj.load_system_host_keys()
435 sshClientObj.load_system_host_keys()
422 sshClientObj.set_missing_host_key_policy(paramiko.WarningPolicy())
436 sshClientObj.set_missing_host_key_policy(paramiko.WarningPolicy())
423
437
438 self.status = 0
424 try:
439 try:
425 sshClientObj.connect(server, username=username, password=password, port=port)
440 sshClientObj.connect(server, username=username, password=password, port=port)
426 except :
441 except paramiko.AuthenticationException, e:
427 print "SSH username or password are incorrect: %s"
442 # print "SSH username or password are incorrect: %s"
443 print "[SSH Server]:", e
444 return 0
445 except SSHException, e:
446 print "[SSH Server]:", e
447 return 0
448 except socket.error:
428 self.status = 0
449 self.status = 0
450 print "[SSH Server]:", e
429 return 0
451 return 0
430
452
453 self.status = 1
431 scpClientObj = scp.SCPClient(sshClientObj.get_transport(), socket_timeout=30)
454 scpClientObj = scp.SCPClient(sshClientObj.get_transport(), socket_timeout=30)
432
455
433 if remotefolder == None:
456 if remotefolder == None:
434 remotefolder = self.pwd()
457 remotefolder = self.pwd()
435
458
436 self.server = server
459 self.server = server
437 self.username = username
460 self.username = username
438 self.password = password
461 self.password = password
439 self.__sshClientObj = sshClientObj
462 self.__sshClientObj = sshClientObj
440 self.__scpClientObj = scpClientObj
463 self.__scpClientObj = scpClientObj
441 self.status = 1
464 self.status = 1
442
465
443 if not self.cd(remotefolder):
466 if not self.cd(remotefolder):
444 raise ValueError, "Could not access to remote folder: %s" %remotefolder
467 raise ValueError, "[SSH Server]: Could not access to remote folder: %s" %remotefolder
445 return 0
468 return 0
446
469
447 self.remotefolder = remotefolder
470 self.remotefolder = remotefolder
448
471
449 return 1
472 return 1
450
473
451 def close(self):
474 def close(self):
452 """
475 """
453 Close connection to remote server
476 Close connection to remote server
454 """
477 """
455 if not self.status:
478 if not self.status:
456 return 0
479 return 0
457
480
458 self.__sshObj.close()
481 self.__scpClientObj.close()
482 self.__sshClientObj.close()
459
483
460 def __execute(self, command):
484 def __execute(self, command):
461 """
485 """
462 __execute a command on remote server
486 __execute a command on remote server
463
487
464 Input:
488 Input:
465 command - Exmaple 'ls -l'
489 command - Exmaple 'ls -l'
466
490
467 Return:
491 Return:
468 0 in error case else 1
492 0 in error case else 1
469 """
493 """
470 if not self.status:
494 if not self.status:
471 return 0
495 return 0
472
496
473 stdin, stdout, stderr = self.__sshClientObj.exec_command(command)
497 stdin, stdout, stderr = self.__sshClientObj.exec_command(command)
474
498
475 result = stderr.readlines()
499 result = stderr.readlines()
476 if len(result) > 1:
500 if len(result) > 1:
477 return 0
501 return 0
478
502
479 result = stdout.readlines()
503 result = stdout.readlines()
480 if len(result) > 1:
504 if len(result) > 1:
481 return result[0][:-1]
505 return result[0][:-1]
482
506
483 return 1
507 return 1
484
508
485 def mkdir(self, remotefolder):
509 def mkdir(self, remotefolder):
486 """
510 """
487 mkdir is used to make a new directory in remote server
511 mkdir is used to make a new directory in remote server
488
512
489 Input:
513 Input:
490 remotefolder - directory name
514 remotefolder - directory name
491
515
492 Return:
516 Return:
493 0 in error case else 1
517 0 in error case else 1
494 """
518 """
495
519
496 command = 'mkdir %s' %remotefolder
520 command = 'mkdir %s' %remotefolder
497
521
498 return self.__execute(command)
522 return self.__execute(command)
499
523
500 def pwd(self):
524 def pwd(self):
501
525
502 command = 'pwd'
526 command = 'pwd'
503
527
504 return self.__execute(command)
528 return self.__execute(command)
505
529
506 def cd(self, remotefolder):
530 def cd(self, remotefolder):
507 """
531 """
508 cd is used to change remote working directory on server
532 cd is used to change remote working directory on server
509
533
510 Input:
534 Input:
511 remotefolder - current working directory
535 remotefolder - current working directory
512
536
513 Affects:
537 Affects:
514 self.remotefolder
538 self.remotefolder
515
539
516 Return:
540 Return:
517 0 in case of error else 1
541 0 in case of error else 1
518 """
542 """
519 if not self.status:
543 if not self.status:
520 return 0
544 return 0
521
545
522 if remotefolder == self.remotefolder:
546 if remotefolder == self.remotefolder:
523 return 1
547 return 1
524
548
525 chk_command = "cd %s; pwd" %remotefolder
549 chk_command = "cd %s; pwd" %remotefolder
526 mkdir_command = "mkdir %s" %remotefolder
550 mkdir_command = "mkdir %s" %remotefolder
527
551
528 if not self.__execute(chk_command):
552 if not self.__execute(chk_command):
529 if not self.__execute(mkdir_command):
553 if not self.__execute(mkdir_command):
530 self.remotefolder = None
554 self.remotefolder = None
531 return 0
555 return 0
532
556
533 self.remotefolder = remotefolder
557 self.remotefolder = remotefolder
534
558
535 return 1
559 return 1
536
560
537 def sendFile(self, fullfilename):
561 def sendFile(self, fullfilename):
538
562
539 if not self.status:
563 if not self.status:
540 return 0
564 return 0
541
565
542 try:
566 try:
543 self.__scpClientObj.put(fullfilename, remote_path=self.remotefolder)
567 self.__scpClientObj.put(fullfilename, remote_path=self.remotefolder)
544 except:
568 except scp.ScpError, e:
569 print "[SSH Server]", str(e)
545 return 0
570 return 0
546
571
547 remotefile = os.path.join(self.remotefolder, os.path.split(fullfilename)[-1])
572 remotefile = os.path.join(self.remotefolder, os.path.split(fullfilename)[-1])
548 command = 'chmod 775 %s' %remotefile
573 command = 'chmod 775 %s' %remotefile
549
574
550 return self.__execute(command)
575 return self.__execute(command)
551
576
552 class SendToServer(ProcessingUnit):
577 class SendToServer(ProcessingUnit):
553
578
554 def __init__(self):
579 def __init__(self):
555
580
556 ProcessingUnit.__init__(self)
581 ProcessingUnit.__init__(self)
557
582
558 self.isConfig = False
583 self.isConfig = False
559 self.clientObj = None
584 self.clientObj = None
560
585
561 def setup(self, server, username, password, remotefolder, localfolder, ext='.png', period=60, protocol='ftp', **kwargs):
586 def setup(self, server, username, password, remotefolder, localfolder, ext='.png', period=60, protocol='ftp', **kwargs):
562
587
563 self.clientObj = None
588 self.clientObj = None
564 self.localfolder = localfolder
589 self.localfolder = localfolder
565 self.ext = ext
590 self.ext = ext
566 self.period = period
591 self.period = period
567
592
568 if str.lower(protocol) == 'ftp':
593 if str.lower(protocol) == 'ftp':
569 self.clientObj = FTPClient(server, username, password, remotefolder, period)
594 self.clientObj = FTPClient(server, username, password, remotefolder, period)
570
595
571 if str.lower(protocol) == 'ssh':
596 if str.lower(protocol) == 'ssh':
572 self.clientObj = SSHClient(server, username, password, remotefolder, period)
597 self.clientObj = SSHClient(server, username, password, remotefolder, period)
573
598
574 if not self.clientObj:
599 if not self.clientObj:
575 raise ValueError, "%s has been chosen as remote access protocol but it is not valid" %protocol
600 raise ValueError, "%s has been chosen as remote access protocol but it is not valid" %protocol
576
601
577 self.clientObj.start()
602 self.clientObj.start()
578
603
579 def findFiles(self):
604 def findFiles(self):
580
605
581 if not type(self.localfolder) == list:
606 if not type(self.localfolder) == list:
582 folderList = [self.localfolder]
607 folderList = [self.localfolder]
583 else:
608 else:
584 folderList = self.localfolder
609 folderList = self.localfolder
585
610
586 #Remove duplicate items
611 #Remove duplicate items
587 folderList = list(set(folderList))
612 folderList = list(set(folderList))
588
613
589 fullfilenameList = []
614 fullfilenameList = []
590
615
591 for thisFolder in folderList:
616 for thisFolder in folderList:
592
617
593 print "[Remote Server]: Searching files on %s" %thisFolder
618 print "[Remote Server]: Searching files on %s" %thisFolder
594
619
595 filenameList = glob.glob1(thisFolder, '*%s' %self.ext)
620 filenameList = glob.glob1(thisFolder, '*%s' %self.ext)
596
621
597 if len(filenameList) < 1:
622 if len(filenameList) < 1:
598 continue
623 continue
599
624
600 for thisFile in filenameList:
625 for thisFile in filenameList:
601 fullfilename = os.path.join(thisFolder, thisFile)
626 fullfilename = os.path.join(thisFolder, thisFile)
602
627
603 if fullfilename in fullfilenameList:
628 if fullfilename in fullfilenameList:
604 continue
629 continue
605
630
606 #Only files modified in the last 30 minutes are considered
631 #Only files modified in the last 30 minutes are considered
607 if os.path.getmtime(fullfilename) < time.time() - 30*60:
632 if os.path.getmtime(fullfilename) < time.time() - 30*60:
608 continue
633 continue
609
634
610 fullfilenameList.append(fullfilename)
635 fullfilenameList.append(fullfilename)
611
636
612 return fullfilenameList
637 return fullfilenameList
613
638
614 def run(self, **kwargs):
639 def run(self, **kwargs):
615
640
616 if not self.isConfig:
641 if not self.isConfig:
617 self.init = time.time()
642 self.init = time.time()
618 self.setup(**kwargs)
643 self.setup(**kwargs)
619 self.isConfig = True
644 self.isConfig = True
620
645
621 if time.time() - self.init >= self.period:
646 if time.time() - self.init >= self.period:
622 fullfilenameList = self.findFiles()
647 fullfilenameList = self.findFiles()
623
648
624 if self.clientObj.updateFileList(fullfilenameList):
649 if self.clientObj.updateFileList(fullfilenameList):
625 print "[Remote Server]: Sending the next files ", str(fullfilenameList)
650 print "[Remote Server]: Sending the next files ", str(fullfilenameList)
626
651
627 self.init = time.time()
652 self.init = time.time()
628
653
629 def close(self):
654 def close(self):
630 print "[Remote Server] Stopping thread"
655 print "[Remote Server] Stopping thread"
631 self.clientObj.stop()
656 self.clientObj.stop()
632
657
633
658
634 class FTP(object):
659 class FTP(object):
635 """
660 """
636 Ftp is a public class used to define custom File Transfer Protocol from "ftplib" python module
661 Ftp is a public class used to define custom File Transfer Protocol from "ftplib" python module
637
662
638 Non-standard Python modules used: None
663 Non-standard Python modules used: None
639
664
640 Written by "Daniel Suarez":mailto:daniel.suarez@jro.igp.gob.pe Oct. 26, 2010
665 Written by "Daniel Suarez":mailto:daniel.suarez@jro.igp.gob.pe Oct. 26, 2010
641 """
666 """
642
667
643 def __init__(self,server = None, username=None, password=None, remotefolder=None):
668 def __init__(self,server = None, username=None, password=None, remotefolder=None):
644 """
669 """
645 This method is used to setting parameters for FTP and establishing connection to remote server
670 This method is used to setting parameters for FTP and establishing connection to remote server
646
671
647 Inputs:
672 Inputs:
648 server - remote server IP Address
673 server - remote server IP Address
649
674
650 username - remote server Username
675 username - remote server Username
651
676
652 password - remote server password
677 password - remote server password
653
678
654 remotefolder - remote server current working directory
679 remotefolder - remote server current working directory
655
680
656 Return: void
681 Return: void
657
682
658 Affects:
683 Affects:
659 self.status - in Error Case or Connection Failed this parameter is set to 1 else 0
684 self.status - in Error Case or Connection Failed this parameter is set to 1 else 0
660
685
661 self.folderList - sub-folder list of remote folder
686 self.folderList - sub-folder list of remote folder
662
687
663 self.fileList - file list of remote folder
688 self.fileList - file list of remote folder
664
689
665
690
666 """
691 """
667
692
668 if ((server == None) and (username==None) and (password==None) and (remotefolder==None)):
693 if ((server == None) and (username==None) and (password==None) and (remotefolder==None)):
669 server, username, password, remotefolder = self.parmsByDefault()
694 server, username, password, remotefolder = self.parmsByDefault()
670
695
671 self.server = server
696 self.server = server
672 self.username = username
697 self.username = username
673 self.password = password
698 self.password = password
674 self.remotefolder = remotefolder
699 self.remotefolder = remotefolder
675 self.file = None
700 self.file = None
676 self.ftp = None
701 self.ftp = None
677 self.status = 0
702 self.status = 0
678
703
679 try:
704 try:
680 self.ftp = ftplib.FTP(self.server)
705 self.ftp = ftplib.FTP(self.server)
681 self.ftp.login(self.username,self.password)
706 self.ftp.login(self.username,self.password)
682 self.ftp.cwd(self.remotefolder)
707 self.ftp.cwd(self.remotefolder)
683 # print 'Connect to FTP Server: Successfully'
708 # print 'Connect to FTP Server: Successfully'
684
709
685 except ftplib.all_errors:
710 except ftplib.all_errors:
686 print 'Error FTP Service'
711 print 'Error FTP Service'
687 self.status = 1
712 self.status = 1
688 return
713 return
689
714
690
715
691
716
692 self.dirList = []
717 self.dirList = []
693
718
694 try:
719 try:
695 self.dirList = self.ftp.nlst()
720 self.dirList = self.ftp.nlst()
696
721
697 except ftplib.error_perm, resp:
722 except ftplib.error_perm, resp:
698 if str(resp) == "550 No files found":
723 if str(resp) == "550 No files found":
699 print "no files in this directory"
724 print "no files in this directory"
700 self.status = 1
725 self.status = 1
701 return
726 return
702
727
703 except ftplib.all_errors:
728 except ftplib.all_errors:
704 print 'Error Displaying Dir-Files'
729 print 'Error Displaying Dir-Files'
705 self.status = 1
730 self.status = 1
706 return
731 return
707
732
708 self.fileList = []
733 self.fileList = []
709 self.folderList = []
734 self.folderList = []
710 #only for test
735 #only for test
711 for f in self.dirList:
736 for f in self.dirList:
712 name, ext = os.path.splitext(f)
737 name, ext = os.path.splitext(f)
713 if ext != '':
738 if ext != '':
714 self.fileList.append(f)
739 self.fileList.append(f)
715 # print 'filename: %s - size: %d'%(f,self.ftp.size(f))
740 # print 'filename: %s - size: %d'%(f,self.ftp.size(f))
716
741
717 def parmsByDefault(self):
742 def parmsByDefault(self):
718 server = 'jro-app.igp.gob.pe'
743 server = 'jro-app.igp.gob.pe'
719 username = 'wmaster'
744 username = 'wmaster'
720 password = 'mst2010vhf'
745 password = 'mst2010vhf'
721 remotefolder = '/home/wmaster/graficos'
746 remotefolder = '/home/wmaster/graficos'
722
747
723 return server, username, password, remotefolder
748 return server, username, password, remotefolder
724
749
725
750
726 def mkd(self,dirname):
751 def mkd(self,dirname):
727 """
752 """
728 mkd is used to make directory in remote server
753 mkd is used to make directory in remote server
729
754
730 Input:
755 Input:
731 dirname - directory name
756 dirname - directory name
732
757
733 Return:
758 Return:
734 1 in error case else 0
759 1 in error case else 0
735 """
760 """
736 try:
761 try:
737 self.ftp.mkd(dirname)
762 self.ftp.mkd(dirname)
738 except:
763 except:
739 print 'Error creating remote folder:%s'%dirname
764 print 'Error creating remote folder:%s'%dirname
740 return 1
765 return 1
741
766
742 return 0
767 return 0
743
768
744
769
745 def delete(self,filename):
770 def delete(self,filename):
746 """
771 """
747 delete is used to delete file in current working directory of remote server
772 delete is used to delete file in current working directory of remote server
748
773
749 Input:
774 Input:
750 filename - filename to delete in remote folder
775 filename - filename to delete in remote folder
751
776
752 Return:
777 Return:
753 1 in error case else 0
778 1 in error case else 0
754 """
779 """
755
780
756 try:
781 try:
757 self.ftp.delete(filename)
782 self.ftp.delete(filename)
758 except:
783 except:
759 print 'Error deleting remote file:%s'%filename
784 print 'Error deleting remote file:%s'%filename
760 return 1
785 return 1
761
786
762 return 0
787 return 0
763
788
764 def download(self,filename,localfolder):
789 def download(self,filename,localfolder):
765 """
790 """
766 download is used to downloading file from remote folder into local folder
791 download is used to downloading file from remote folder into local folder
767
792
768 Inputs:
793 Inputs:
769 filename - filename to donwload
794 filename - filename to donwload
770
795
771 localfolder - directory local to store filename
796 localfolder - directory local to store filename
772
797
773 Returns:
798 Returns:
774 self.status - 1 in error case else 0
799 self.status - 1 in error case else 0
775 """
800 """
776
801
777 self.status = 0
802 self.status = 0
778
803
779
804
780 if not(filename in self.fileList):
805 if not(filename in self.fileList):
781 print 'filename:%s not exists'%filename
806 print 'filename:%s not exists'%filename
782 self.status = 1
807 self.status = 1
783 return self.status
808 return self.status
784
809
785 newfilename = os.path.join(localfolder,filename)
810 newfilename = os.path.join(localfolder,filename)
786
811
787 self.file = open(newfilename, 'wb')
812 self.file = open(newfilename, 'wb')
788
813
789 try:
814 try:
790 print 'Download: ' + filename
815 print 'Download: ' + filename
791 self.ftp.retrbinary('RETR ' + filename, self.__handleDownload)
816 self.ftp.retrbinary('RETR ' + filename, self.__handleDownload)
792 print 'Download Complete'
817 print 'Download Complete'
793 except ftplib.all_errors:
818 except ftplib.all_errors:
794 print 'Error Downloading ' + filename
819 print 'Error Downloading ' + filename
795 self.status = 1
820 self.status = 1
796 return self.status
821 return self.status
797
822
798 self.file.close()
823 self.file.close()
799
824
800 return self.status
825 return self.status
801
826
802
827
803 def __handleDownload(self,block):
828 def __handleDownload(self,block):
804 """
829 """
805 __handleDownload is used to handle writing file
830 __handleDownload is used to handle writing file
806 """
831 """
807 self.file.write(block)
832 self.file.write(block)
808
833
809
834
810 def upload(self,filename,remotefolder=None):
835 def upload(self,filename,remotefolder=None):
811 """
836 """
812 upload is used to uploading local file to remote directory
837 upload is used to uploading local file to remote directory
813
838
814 Inputs:
839 Inputs:
815 filename - full path name of local file to store in remote directory
840 filename - full path name of local file to store in remote directory
816
841
817 remotefolder - remote directory
842 remotefolder - remote directory
818
843
819 Returns:
844 Returns:
820 self.status - 1 in error case else 0
845 self.status - 1 in error case else 0
821 """
846 """
822
847
823 if remotefolder == None:
848 if remotefolder == None:
824 remotefolder = self.remotefolder
849 remotefolder = self.remotefolder
825
850
826 self.status = 0
851 self.status = 0
827
852
828 try:
853 try:
829 self.ftp.cwd(remotefolder)
854 self.ftp.cwd(remotefolder)
830
855
831 self.file = open(filename, 'rb')
856 self.file = open(filename, 'rb')
832
857
833 (head, tail) = os.path.split(filename)
858 (head, tail) = os.path.split(filename)
834
859
835 command = "STOR " + tail
860 command = "STOR " + tail
836
861
837 print 'Uploading: ' + tail
862 print 'Uploading: ' + tail
838 self.ftp.storbinary(command, self.file)
863 self.ftp.storbinary(command, self.file)
839 print 'Upload Completed'
864 print 'Upload Completed'
840
865
841 except ftplib.all_errors:
866 except ftplib.all_errors:
842 print 'Error Uploading ' + tail
867 print 'Error Uploading ' + tail
843 self.status = 1
868 self.status = 1
844 return self.status
869 return self.status
845
870
846 self.file.close()
871 self.file.close()
847
872
848 #back to initial directory in __init__()
873 #back to initial directory in __init__()
849 self.ftp.cwd(self.remotefolder)
874 self.ftp.cwd(self.remotefolder)
850
875
851 return self.status
876 return self.status
852
877
853
878
854 def dir(self,remotefolder):
879 def dir(self,remotefolder):
855 """
880 """
856 dir is used to change working directory of remote server and get folder and file list
881 dir is used to change working directory of remote server and get folder and file list
857
882
858 Input:
883 Input:
859 remotefolder - current working directory
884 remotefolder - current working directory
860
885
861 Affects:
886 Affects:
862 self.fileList - file list of working directory
887 self.fileList - file list of working directory
863
888
864 Return:
889 Return:
865 infoList - list with filenames and size of file in bytes
890 infoList - list with filenames and size of file in bytes
866
891
867 self.folderList - folder list
892 self.folderList - folder list
868 """
893 """
869
894
870 self.remotefolder = remotefolder
895 self.remotefolder = remotefolder
871 print 'Change to ' + self.remotefolder
896 print 'Change to ' + self.remotefolder
872 try:
897 try:
873 self.ftp.cwd(remotefolder)
898 self.ftp.cwd(remotefolder)
874 except ftplib.all_errors:
899 except ftplib.all_errors:
875 print 'Error Change to ' + self.remotefolder
900 print 'Error Change to ' + self.remotefolder
876 infoList = None
901 infoList = None
877 self.folderList = None
902 self.folderList = None
878 return infoList,self.folderList
903 return infoList,self.folderList
879
904
880 self.dirList = []
905 self.dirList = []
881
906
882 try:
907 try:
883 self.dirList = self.ftp.nlst()
908 self.dirList = self.ftp.nlst()
884
909
885 except ftplib.error_perm, resp:
910 except ftplib.error_perm, resp:
886 if str(resp) == "550 No files found":
911 if str(resp) == "550 No files found":
887 print "no files in this directory"
912 print "no files in this directory"
888 infoList = None
913 infoList = None
889 self.folderList = None
914 self.folderList = None
890 return infoList,self.folderList
915 return infoList,self.folderList
891 except ftplib.all_errors:
916 except ftplib.all_errors:
892 print 'Error Displaying Dir-Files'
917 print 'Error Displaying Dir-Files'
893 infoList = None
918 infoList = None
894 self.folderList = None
919 self.folderList = None
895 return infoList,self.folderList
920 return infoList,self.folderList
896
921
897 infoList = []
922 infoList = []
898 self.fileList = []
923 self.fileList = []
899 self.folderList = []
924 self.folderList = []
900 for f in self.dirList:
925 for f in self.dirList:
901 name,ext = os.path.splitext(f)
926 name,ext = os.path.splitext(f)
902 if ext != '':
927 if ext != '':
903 self.fileList.append(f)
928 self.fileList.append(f)
904 value = (f,self.ftp.size(f))
929 value = (f,self.ftp.size(f))
905 infoList.append(value)
930 infoList.append(value)
906
931
907 if ext == '':
932 if ext == '':
908 self.folderList.append(f)
933 self.folderList.append(f)
909
934
910 return infoList,self.folderList
935 return infoList,self.folderList
911
936
912
937
913 def close(self):
938 def close(self):
914 """
939 """
915 close is used to close and end FTP connection
940 close is used to close and end FTP connection
916
941
917 Inputs: None
942 Inputs: None
918
943
919 Return: void
944 Return: void
920
945
921 """
946 """
922 self.ftp.close()
947 self.ftp.close()
923
948
924 class SendByFTP(Operation):
949 class SendByFTP(Operation):
925
950
926 def __init__(self):
951 def __init__(self):
927
952
928 self.status = 1
953 self.status = 1
929 self.counter = 0
954 self.counter = 0
930
955
931 def error_print(self, ValueError):
956 def error_print(self, ValueError):
932
957
933 print ValueError, 'Error FTP'
958 print ValueError, 'Error FTP'
934 print "don't worry the program is running..."
959 print "don't worry the program is running..."
935
960
936 def worker_ftp(self, server, username, password, remotefolder, filenameList):
961 def worker_ftp(self, server, username, password, remotefolder, filenameList):
937
962
938 self.ftpClientObj = FTP(server, username, password, remotefolder)
963 self.ftpClientObj = FTP(server, username, password, remotefolder)
939 for filename in filenameList:
964 for filename in filenameList:
940 self.ftpClientObj.upload(filename)
965 self.ftpClientObj.upload(filename)
941 self.ftpClientObj.close()
966 self.ftpClientObj.close()
942
967
943 def ftp_thread(self, server, username, password, remotefolder):
968 def ftp_thread(self, server, username, password, remotefolder):
944 if not(self.status):
969 if not(self.status):
945 return
970 return
946
971
947 import multiprocessing
972 import multiprocessing
948
973
949 p = multiprocessing.Process(target=self.worker_ftp, args=(server, username, password, remotefolder, self.filenameList,))
974 p = multiprocessing.Process(target=self.worker_ftp, args=(server, username, password, remotefolder, self.filenameList,))
950 p.start()
975 p.start()
951
976
952 p.join(3)
977 p.join(3)
953
978
954 if p.is_alive():
979 if p.is_alive():
955 p.terminate()
980 p.terminate()
956 p.join()
981 p.join()
957 print 'killing ftp process...'
982 print 'killing ftp process...'
958 self.status = 0
983 self.status = 0
959 return
984 return
960
985
961 self.status = 1
986 self.status = 1
962 return
987 return
963
988
964 def filterByExt(self, ext, localfolder):
989 def filterByExt(self, ext, localfolder):
965 fnameList = glob.glob1(localfolder,ext)
990 fnameList = glob.glob1(localfolder,ext)
966 self.filenameList = [os.path.join(localfolder,x) for x in fnameList]
991 self.filenameList = [os.path.join(localfolder,x) for x in fnameList]
967
992
968 if len(self.filenameList) == 0:
993 if len(self.filenameList) == 0:
969 self.status = 0
994 self.status = 0
970
995
971 def run(self, dataOut, ext, localfolder, remotefolder, server, username, password, period=1):
996 def run(self, dataOut, ext, localfolder, remotefolder, server, username, password, period=1):
972
997
973 self.counter += 1
998 self.counter += 1
974 if self.counter >= period:
999 if self.counter >= period:
975 self.filterByExt(ext, localfolder)
1000 self.filterByExt(ext, localfolder)
976
1001
977 self.ftp_thread(server, username, password, remotefolder)
1002 self.ftp_thread(server, username, password, remotefolder)
978
1003
979 self.counter = 0
1004 self.counter = 0
980
1005
981 self.status = 1
1006 self.status = 1
982
1007
General Comments 0
You need to be logged in to leave comments. Login now