##// END OF EJS Templates
schain
RSS
This update consider connect and send FTP using multiprocessing with timeout
Daniel Valdez -
r517:014ca0bd8892
parent child
Show More
Show file before | Show file after | Hide comments
@@ -1,354 +1,353
1 '''
1 '''
2 @author: Daniel Suarez
2 @author: Daniel Suarez
3 '''
3 '''
4 import os
4 import os
5 import glob
5 import glob
6 import ftplib
6 import ftplib
7 import multiprocessing
7 from model.proc.jroproc_base import ProcessingUnit, Operation
8 from model.proc.jroproc_base import ProcessingUnit, Operation
8
9
9 class FTP():
10 class FTP(object):
10 """
11 """
11 Ftp is a public class used to define custom File Transfer Protocol from "ftplib" python module
12 Ftp is a public class used to define custom File Transfer Protocol from "ftplib" python module
12
13
13 Non-standard Python modules used: None
14 Non-standard Python modules used: None
14
15
15 Written by "Daniel Suarez":mailto:daniel.suarez@jro.igp.gob.pe Oct. 26, 2010
16 Written by "Daniel Suarez":mailto:daniel.suarez@jro.igp.gob.pe Oct. 26, 2010
16 """
17 """
17
18
18 def __init__(self,server = None, username=None, password=None, remotefolder=None):
19 def __init__(self,server = None, username=None, password=None, remotefolder=None):
19 """
20 """
20 This method is used to setting parameters for FTP and establishing connection to remote server
21 This method is used to setting parameters for FTP and establishing connection to remote server
21
22
22 Inputs:
23 Inputs:
23 server - remote server IP Address
24 server - remote server IP Address
24
25
25 username - remote server Username
26 username - remote server Username
26
27
27 password - remote server password
28 password - remote server password
28
29
29 remotefolder - remote server current working directory
30 remotefolder - remote server current working directory
30
31
31 Return: void
32 Return: void
32
33
33 Affects:
34 Affects:
34 self.status - in Error Case or Connection Failed this parameter is set to 1 else 0
35 self.status - in Error Case or Connection Failed this parameter is set to 1 else 0
35
36
36 self.folderList - sub-folder list of remote folder
37 self.folderList - sub-folder list of remote folder
37
38
38 self.fileList - file list of remote folder
39 self.fileList - file list of remote folder
39
40
40
41
41 """
42 """
42
43
43 if ((server == None) and (username==None) and (password==None) and (remotefolder==None)):
44 if ((server == None) and (username==None) and (password==None) and (remotefolder==None)):
44 server, username, password, remotefolder = self.parmsByDefault()
45 server, username, password, remotefolder = self.parmsByDefault()
45
46
46 self.server = server
47 self.server = server
47 self.username = username
48 self.username = username
48 self.password = password
49 self.password = password
49 self.remotefolder = remotefolder
50 self.remotefolder = remotefolder
50 self.file = None
51 self.file = None
51 self.ftp = None
52 self.ftp = None
52 self.status = 0
53 self.status = 0
53
54
54 try:
55 try:
55 self.ftp = ftplib.FTP(self.server)
56 self.ftp = ftplib.FTP(self.server)
56 self.ftp.login(self.username,self.password)
57 self.ftp.login(self.username,self.password)
57 self.ftp.cwd(self.remotefolder)
58 self.ftp.cwd(self.remotefolder)
58 # print 'Connect to FTP Server: Successfully'
59 # print 'Connect to FTP Server: Successfully'
59
60
60 except ftplib.all_errors:
61 except ftplib.all_errors:
61 print 'Error FTP Service'
62 print 'Error FTP Service'
62 self.status = 1
63 self.status = 1
63 return
64 return
64
65
65
66
66
67
67 self.dirList = []
68 self.dirList = []
68
69
69 try:
70 try:
70 self.dirList = self.ftp.nlst()
71 self.dirList = self.ftp.nlst()
71
72
72 except ftplib.error_perm, resp:
73 except ftplib.error_perm, resp:
73 if str(resp) == "550 No files found":
74 if str(resp) == "550 No files found":
74 print "no files in this directory"
75 print "no files in this directory"
75 self.status = 1
76 self.status = 1
76 return
77 return
77
78
78 except ftplib.all_errors:
79 except ftplib.all_errors:
79 print 'Error Displaying Dir-Files'
80 print 'Error Displaying Dir-Files'
80 self.status = 1
81 self.status = 1
81 return
82 return
82
83
83 self.fileList = []
84 self.fileList = []
84 self.folderList = []
85 self.folderList = []
85 #only for test
86 #only for test
86 for f in self.dirList:
87 for f in self.dirList:
87 name, ext = os.path.splitext(f)
88 name, ext = os.path.splitext(f)
88 if ext != '':
89 if ext != '':
89 self.fileList.append(f)
90 self.fileList.append(f)
90 # print 'filename: %s - size: %d'%(f,self.ftp.size(f))
91 # print 'filename: %s - size: %d'%(f,self.ftp.size(f))
91
92
92 def parmsByDefault(self):
93 def parmsByDefault(self):
93 server = 'jro-app.igp.gob.pe'
94 server = 'jro-app.igp.gob.pe'
94 username = 'wmaster'
95 username = 'wmaster'
95 password = 'mst2010vhf'
96 password = 'mst2010vhf'
96 remotefolder = '/home/wmaster/graficos'
97 remotefolder = '/home/wmaster/graficos'
97
98
98 return server, username, password, remotefolder
99 return server, username, password, remotefolder
99
100
100
101
101 def mkd(self,dirname):
102 def mkd(self,dirname):
102 """
103 """
103 mkd is used to make directory in remote server
104 mkd is used to make directory in remote server
104
105
105 Input:
106 Input:
106 dirname - directory name
107 dirname - directory name
107
108
108 Return:
109 Return:
109 1 in error case else 0
110 1 in error case else 0
110 """
111 """
111 try:
112 try:
112 self.ftp.mkd(dirname)
113 self.ftp.mkd(dirname)
113 except:
114 except:
114 print 'Error creating remote folder:%s'%dirname
115 print 'Error creating remote folder:%s'%dirname
115 return 1
116 return 1
116
117
117 return 0
118 return 0
118
119
119
120
120 def delete(self,filename):
121 def delete(self,filename):
121 """
122 """
122 delete is used to delete file in current working directory of remote server
123 delete is used to delete file in current working directory of remote server
123
124
124 Input:
125 Input:
125 filename - filename to delete in remote folder
126 filename - filename to delete in remote folder
126
127
127 Return:
128 Return:
128 1 in error case else 0
129 1 in error case else 0
129 """
130 """
130
131
131 try:
132 try:
132 self.ftp.delete(filename)
133 self.ftp.delete(filename)
133 except:
134 except:
134 print 'Error deleting remote file:%s'%filename
135 print 'Error deleting remote file:%s'%filename
135 return 1
136 return 1
136
137
137 return 0
138 return 0
138
139
139 def download(self,filename,localfolder):
140 def download(self,filename,localfolder):
140 """
141 """
141 download is used to downloading file from remote folder into local folder
142 download is used to downloading file from remote folder into local folder
142
143
143 Inputs:
144 Inputs:
144 filename - filename to donwload
145 filename - filename to donwload
145
146
146 localfolder - directory local to store filename
147 localfolder - directory local to store filename
147
148
148 Returns:
149 Returns:
149 self.status - 1 in error case else 0
150 self.status - 1 in error case else 0
150 """
151 """
151
152
152 self.status = 0
153 self.status = 0
153
154
154
155
155 if not(filename in self.fileList):
156 if not(filename in self.fileList):
156 print 'filename:%s not exists'%filename
157 print 'filename:%s not exists'%filename
157 self.status = 1
158 self.status = 1
158 return self.status
159 return self.status
159
160
160 newfilename = os.path.join(localfolder,filename)
161 newfilename = os.path.join(localfolder,filename)
161
162
162 self.file = open(newfilename, 'wb')
163 self.file = open(newfilename, 'wb')
163
164
164 try:
165 try:
165 print 'Download: ' + filename
166 print 'Download: ' + filename
166 self.ftp.retrbinary('RETR ' + filename, self.__handleDownload)
167 self.ftp.retrbinary('RETR ' + filename, self.__handleDownload)
167 print 'Download Complete'
168 print 'Download Complete'
168 except ftplib.all_errors:
169 except ftplib.all_errors:
169 print 'Error Downloading ' + filename
170 print 'Error Downloading ' + filename
170 self.status = 1
171 self.status = 1
171 return self.status
172 return self.status
172
173
173 self.file.close()
174 self.file.close()
174
175
175 return self.status
176 return self.status
176
177
177
178
178 def __handleDownload(self,block):
179 def __handleDownload(self,block):
179 """
180 """
180 __handleDownload is used to handle writing file
181 __handleDownload is used to handle writing file
181 """
182 """
182 self.file.write(block)
183 self.file.write(block)
183
184
184
185
185 def upload(self,filename,remotefolder=None):
186 def upload(self,filename,remotefolder=None):
186 """
187 """
187 upload is used to uploading local file to remote directory
188 upload is used to uploading local file to remote directory
188
189
189 Inputs:
190 Inputs:
190 filename - full path name of local file to store in remote directory
191 filename - full path name of local file to store in remote directory
191
192
192 remotefolder - remote directory
193 remotefolder - remote directory
193
194
194 Returns:
195 Returns:
195 self.status - 1 in error case else 0
196 self.status - 1 in error case else 0
196 """
197 """
197
198
198 if remotefolder == None:
199 if remotefolder == None:
199 remotefolder = self.remotefolder
200 remotefolder = self.remotefolder
200
201
201 self.status = 0
202 self.status = 0
202
203
203 try:
204 try:
204 self.ftp.cwd(remotefolder)
205 self.ftp.cwd(remotefolder)
205
206
206 self.file = open(filename, 'rb')
207 self.file = open(filename, 'rb')
207
208
208 (head, tail) = os.path.split(filename)
209 (head, tail) = os.path.split(filename)
209
210
210 command = "STOR " + tail
211 command = "STOR " + tail
211
212
212 print 'Uploading: ' + tail
213 print 'Uploading: ' + tail
213 self.ftp.storbinary(command, self.file)
214 self.ftp.storbinary(command, self.file)
214 print 'Upload Completed'
215 print 'Upload Completed'
215
216
216 except ftplib.all_errors:
217 except ftplib.all_errors:
217 print 'Error Uploading ' + tail
218 print 'Error Uploading ' + tail
218 self.status = 1
219 self.status = 1
219 return self.status
220 return self.status
220
221
221 self.file.close()
222 self.file.close()
222
223
223 #back to initial directory in __init__()
224 #back to initial directory in __init__()
224 self.ftp.cwd(self.remotefolder)
225 self.ftp.cwd(self.remotefolder)
225
226
226 return self.status
227 return self.status
227
228
228
229
229 def dir(self,remotefolder):
230 def dir(self,remotefolder):
230 """
231 """
231 dir is used to change working directory of remote server and get folder and file list
232 dir is used to change working directory of remote server and get folder and file list
232
233
233 Input:
234 Input:
234 remotefolder - current working directory
235 remotefolder - current working directory
235
236
236 Affects:
237 Affects:
237 self.fileList - file list of working directory
238 self.fileList - file list of working directory
238
239
239 Return:
240 Return:
240 infoList - list with filenames and size of file in bytes
241 infoList - list with filenames and size of file in bytes
241
242
242 self.folderList - folder list
243 self.folderList - folder list
243 """
244 """
244
245
245 self.remotefolder = remotefolder
246 self.remotefolder = remotefolder
246 print 'Change to ' + self.remotefolder
247 print 'Change to ' + self.remotefolder
247 try:
248 try:
248 self.ftp.cwd(remotefolder)
249 self.ftp.cwd(remotefolder)
249 except ftplib.all_errors:
250 except ftplib.all_errors:
250 print 'Error Change to ' + self.remotefolder
251 print 'Error Change to ' + self.remotefolder
251 infoList = None
252 infoList = None
252 self.folderList = None
253 self.folderList = None
253 return infoList,self.folderList
254 return infoList,self.folderList
254
255
255 self.dirList = []
256 self.dirList = []
256
257
257 try:
258 try:
258 self.dirList = self.ftp.nlst()
259 self.dirList = self.ftp.nlst()
259
260
260 except ftplib.error_perm, resp:
261 except ftplib.error_perm, resp:
261 if str(resp) == "550 No files found":
262 if str(resp) == "550 No files found":
262 print "no files in this directory"
263 print "no files in this directory"
263 infoList = None
264 infoList = None
264 self.folderList = None
265 self.folderList = None
265 return infoList,self.folderList
266 return infoList,self.folderList
266 except ftplib.all_errors:
267 except ftplib.all_errors:
267 print 'Error Displaying Dir-Files'
268 print 'Error Displaying Dir-Files'
268 infoList = None
269 infoList = None
269 self.folderList = None
270 self.folderList = None
270 return infoList,self.folderList
271 return infoList,self.folderList
271
272
272 infoList = []
273 infoList = []
273 self.fileList = []
274 self.fileList = []
274 self.folderList = []
275 self.folderList = []
275 for f in self.dirList:
276 for f in self.dirList:
276 name,ext = os.path.splitext(f)
277 name,ext = os.path.splitext(f)
277 if ext != '':
278 if ext != '':
278 self.fileList.append(f)
279 self.fileList.append(f)
279 value = (f,self.ftp.size(f))
280 value = (f,self.ftp.size(f))
280 infoList.append(value)
281 infoList.append(value)
281
282
282 if ext == '':
283 if ext == '':
283 self.folderList.append(f)
284 self.folderList.append(f)
284
285
285 return infoList,self.folderList
286 return infoList,self.folderList
286
287
287
288
288 def close(self):
289 def close(self):
289 """
290 """
290 close is used to close and end FTP connection
291 close is used to close and end FTP connection
291
292
292 Inputs: None
293 Inputs: None
293
294
294 Return: void
295 Return: void
295
296
296 """
297 """
297 self.ftp.close()
298 self.ftp.close()
298
299
299 class SendByFTP(Operation):
300 class SendByFTP(Operation):
300 def __init__(self):
301 def __init__(self):
301 self.status = 1
302 self.status = 1
302 self.counter = 0
303 self.counter = 0
303
304
304 def error_print(self, ValueError):
305 def error_print(self, ValueError):
305 print ValueError, 'Error FTP'
306 print ValueError, 'Error FTP'
306 print "don't worry the program is running..."
307 print "don't worry the program is running..."
307
308
308 def connect(self, server, username, password, remotefolder):
309 def worker_ftp(self, server, username, password, remotefolder, filenameList):
309 if not(self.status):
310
310 return
311 try:
312 self.ftpObj = FTP(server, username, password, remotefolder)
311 self.ftpObj = FTP(server, username, password, remotefolder)
313 except:
312 for filename in filenameList:
314 self.error_print(ValueError)
313 self.ftpObj.upload(filename)
315 self.status = 0
314 self.ftpObj.close()
316
315
317 def put(self):
316 def ftp_thread(self, server, username, password, remotefolder):
318 if not(self.status):
317 if not(self.status):
319 return
318 return
320
319
321 try:
320 p = multiprocessing.Process(target=self.worker_ftp, args=(server, username, password, remotefolder, self.filenameList,))
322 for filename in self.filenameList:
321 p.start()
323 self.ftpObj.upload(filename)
324 except:
325 self.error_print(ValueError)
326 self.status = 0
327
322
328 def close(self):
323 p.join(3)
329 if not(self.status):
324
325 if p.is_alive():
326 p.terminate()
327 p.join()
328 print 'killing ftp process...'
329 self.status = 0
330 return
330 return
331
331
332 self.ftpObj.close()
332 self.status = 1
333 return
333
334
334 def filterByExt(self, ext, localfolder):
335 def filterByExt(self, ext, localfolder):
335 fnameList = glob.glob1(localfolder,ext)
336 fnameList = glob.glob1(localfolder,ext)
336 self.filenameList = [os.path.join(localfolder,x) for x in fnameList]
337 self.filenameList = [os.path.join(localfolder,x) for x in fnameList]
337
338
338 if len(self.filenameList) == 0:
339 if len(self.filenameList) == 0:
339 self.status = 0
340 self.status = 0
340
341
341 def run(self, dataOut, ext, localfolder, remotefolder, server, username, password, period=1):
342 def run(self, dataOut, ext, localfolder, remotefolder, server, username, password, period=1):
342
343
343 self.counter += 1
344 self.counter += 1
344 if self.counter >= period:
345 if self.counter >= period:
345 self.filterByExt(ext, localfolder)
346 self.filterByExt(ext, localfolder)
346
347
347 self.connect(server, username, password, remotefolder)
348 self.ftp_thread(server, username, password, remotefolder)
348
349 self.put()
350
351 self.close()
352
349
353 self.counter = 0
350 self.counter = 0
354
351
352 self.status = 1
353
General Comments 0
You need to be logged in to leave comments. Login now