##// END OF EJS Templates
schain
RSS
This update consider connect and send FTP using multiprocessing with timeout
Daniel Valdez -
r517:014ca0bd8892
parent child
Show More
Show file before | Show file after | Hide comments
@@ -1,354 +1,353
1 1 '''
2 2 @author: Daniel Suarez
3 3 '''
4 4 import os
5 5 import glob
6 6 import ftplib
7 import multiprocessing
7 8 from model.proc.jroproc_base import ProcessingUnit, Operation
8 9
9 class FTP():
10 class FTP(object):
10 11 """
11 12 Ftp is a public class used to define custom File Transfer Protocol from "ftplib" python module
12 13
13 14 Non-standard Python modules used: None
14 15
15 16 Written by "Daniel Suarez":mailto:daniel.suarez@jro.igp.gob.pe Oct. 26, 2010
16 17 """
17 18
18 19 def __init__(self,server = None, username=None, password=None, remotefolder=None):
19 20 """
20 21 This method is used to setting parameters for FTP and establishing connection to remote server
21 22
22 23 Inputs:
23 24 server - remote server IP Address
24 25
25 26 username - remote server Username
26 27
27 28 password - remote server password
28 29
29 30 remotefolder - remote server current working directory
30 31
31 32 Return: void
32 33
33 34 Affects:
34 35 self.status - in Error Case or Connection Failed this parameter is set to 1 else 0
35 36
36 37 self.folderList - sub-folder list of remote folder
37 38
38 39 self.fileList - file list of remote folder
39 40
40 41
41 42 """
42 43
43 44 if ((server == None) and (username==None) and (password==None) and (remotefolder==None)):
44 45 server, username, password, remotefolder = self.parmsByDefault()
45 46
46 47 self.server = server
47 48 self.username = username
48 49 self.password = password
49 50 self.remotefolder = remotefolder
50 51 self.file = None
51 52 self.ftp = None
52 53 self.status = 0
53 54
54 55 try:
55 56 self.ftp = ftplib.FTP(self.server)
56 57 self.ftp.login(self.username,self.password)
57 58 self.ftp.cwd(self.remotefolder)
58 59 # print 'Connect to FTP Server: Successfully'
59 60
60 61 except ftplib.all_errors:
61 62 print 'Error FTP Service'
62 63 self.status = 1
63 64 return
64 65
65 66
66 67
67 68 self.dirList = []
68 69
69 70 try:
70 71 self.dirList = self.ftp.nlst()
71 72
72 73 except ftplib.error_perm, resp:
73 74 if str(resp) == "550 No files found":
74 75 print "no files in this directory"
75 76 self.status = 1
76 77 return
77 78
78 79 except ftplib.all_errors:
79 80 print 'Error Displaying Dir-Files'
80 81 self.status = 1
81 82 return
82 83
83 84 self.fileList = []
84 85 self.folderList = []
85 86 #only for test
86 87 for f in self.dirList:
87 88 name, ext = os.path.splitext(f)
88 89 if ext != '':
89 90 self.fileList.append(f)
90 91 # print 'filename: %s - size: %d'%(f,self.ftp.size(f))
91 92
92 93 def parmsByDefault(self):
93 94 server = 'jro-app.igp.gob.pe'
94 95 username = 'wmaster'
95 96 password = 'mst2010vhf'
96 97 remotefolder = '/home/wmaster/graficos'
97 98
98 99 return server, username, password, remotefolder
99 100
100 101
101 102 def mkd(self,dirname):
102 103 """
103 104 mkd is used to make directory in remote server
104 105
105 106 Input:
106 107 dirname - directory name
107 108
108 109 Return:
109 110 1 in error case else 0
110 111 """
111 112 try:
112 113 self.ftp.mkd(dirname)
113 114 except:
114 115 print 'Error creating remote folder:%s'%dirname
115 116 return 1
116 117
117 118 return 0
118 119
119 120
120 121 def delete(self,filename):
121 122 """
122 123 delete is used to delete file in current working directory of remote server
123 124
124 125 Input:
125 126 filename - filename to delete in remote folder
126 127
127 128 Return:
128 129 1 in error case else 0
129 130 """
130 131
131 132 try:
132 133 self.ftp.delete(filename)
133 134 except:
134 135 print 'Error deleting remote file:%s'%filename
135 136 return 1
136 137
137 138 return 0
138 139
139 140 def download(self,filename,localfolder):
140 141 """
141 142 download is used to downloading file from remote folder into local folder
142 143
143 144 Inputs:
144 145 filename - filename to donwload
145 146
146 147 localfolder - directory local to store filename
147 148
148 149 Returns:
149 150 self.status - 1 in error case else 0
150 151 """
151 152
152 153 self.status = 0
153 154
154 155
155 156 if not(filename in self.fileList):
156 157 print 'filename:%s not exists'%filename
157 158 self.status = 1
158 159 return self.status
159 160
160 161 newfilename = os.path.join(localfolder,filename)
161 162
162 163 self.file = open(newfilename, 'wb')
163 164
164 165 try:
165 166 print 'Download: ' + filename
166 167 self.ftp.retrbinary('RETR ' + filename, self.__handleDownload)
167 168 print 'Download Complete'
168 169 except ftplib.all_errors:
169 170 print 'Error Downloading ' + filename
170 171 self.status = 1
171 172 return self.status
172 173
173 174 self.file.close()
174 175
175 176 return self.status
176 177
177 178
178 179 def __handleDownload(self,block):
179 180 """
180 181 __handleDownload is used to handle writing file
181 182 """
182 183 self.file.write(block)
183 184
184 185
185 186 def upload(self,filename,remotefolder=None):
186 187 """
187 188 upload is used to uploading local file to remote directory
188 189
189 190 Inputs:
190 191 filename - full path name of local file to store in remote directory
191 192
192 193 remotefolder - remote directory
193 194
194 195 Returns:
195 196 self.status - 1 in error case else 0
196 197 """
197 198
198 199 if remotefolder == None:
199 200 remotefolder = self.remotefolder
200 201
201 202 self.status = 0
202 203
203 204 try:
204 205 self.ftp.cwd(remotefolder)
205 206
206 207 self.file = open(filename, 'rb')
207 208
208 209 (head, tail) = os.path.split(filename)
209 210
210 211 command = "STOR " + tail
211 212
212 213 print 'Uploading: ' + tail
213 214 self.ftp.storbinary(command, self.file)
214 215 print 'Upload Completed'
215 216
216 217 except ftplib.all_errors:
217 218 print 'Error Uploading ' + tail
218 219 self.status = 1
219 220 return self.status
220 221
221 222 self.file.close()
222 223
223 224 #back to initial directory in __init__()
224 225 self.ftp.cwd(self.remotefolder)
225 226
226 227 return self.status
227 228
228 229
229 230 def dir(self,remotefolder):
230 231 """
231 232 dir is used to change working directory of remote server and get folder and file list
232 233
233 234 Input:
234 235 remotefolder - current working directory
235 236
236 237 Affects:
237 238 self.fileList - file list of working directory
238 239
239 240 Return:
240 241 infoList - list with filenames and size of file in bytes
241 242
242 243 self.folderList - folder list
243 244 """
244 245
245 246 self.remotefolder = remotefolder
246 247 print 'Change to ' + self.remotefolder
247 248 try:
248 249 self.ftp.cwd(remotefolder)
249 250 except ftplib.all_errors:
250 251 print 'Error Change to ' + self.remotefolder
251 252 infoList = None
252 253 self.folderList = None
253 254 return infoList,self.folderList
254 255
255 256 self.dirList = []
256 257
257 258 try:
258 259 self.dirList = self.ftp.nlst()
259 260
260 261 except ftplib.error_perm, resp:
261 262 if str(resp) == "550 No files found":
262 263 print "no files in this directory"
263 264 infoList = None
264 265 self.folderList = None
265 266 return infoList,self.folderList
266 267 except ftplib.all_errors:
267 268 print 'Error Displaying Dir-Files'
268 269 infoList = None
269 270 self.folderList = None
270 271 return infoList,self.folderList
271 272
272 273 infoList = []
273 274 self.fileList = []
274 275 self.folderList = []
275 276 for f in self.dirList:
276 277 name,ext = os.path.splitext(f)
277 278 if ext != '':
278 279 self.fileList.append(f)
279 280 value = (f,self.ftp.size(f))
280 281 infoList.append(value)
281 282
282 283 if ext == '':
283 284 self.folderList.append(f)
284 285
285 286 return infoList,self.folderList
286 287
287 288
288 289 def close(self):
289 290 """
290 291 close is used to close and end FTP connection
291 292
292 293 Inputs: None
293 294
294 295 Return: void
295 296
296 297 """
297 298 self.ftp.close()
298 299
299 300 class SendByFTP(Operation):
300 301 def __init__(self):
301 302 self.status = 1
302 303 self.counter = 0
303 304
304 305 def error_print(self, ValueError):
305 306 print ValueError, 'Error FTP'
306 307 print "don't worry the program is running..."
307 308
308 def connect(self, server, username, password, remotefolder):
309 if not(self.status):
310 return
311 try:
312 self.ftpObj = FTP(server, username, password, remotefolder)
313 except:
314 self.error_print(ValueError)
315 self.status = 0
309 def worker_ftp(self, server, username, password, remotefolder, filenameList):
310
311 self.ftpObj = FTP(server, username, password, remotefolder)
312 for filename in filenameList:
313 self.ftpObj.upload(filename)
314 self.ftpObj.close()
316 315
317 def put(self):
316 def ftp_thread(self, server, username, password, remotefolder):
318 317 if not(self.status):
319 318 return
320 319
321 try:
322 for filename in self.filenameList:
323 self.ftpObj.upload(filename)
324 except:
325 self.error_print(ValueError)
320 p = multiprocessing.Process(target=self.worker_ftp, args=(server, username, password, remotefolder, self.filenameList,))
321 p.start()
322
323 p.join(3)
324
325 if p.is_alive():
326 p.terminate()
327 p.join()
328 print 'killing ftp process...'
326 329 self.status = 0
327
328 def close(self):
329 if not(self.status):
330 330 return
331 331
332 self.ftpObj.close()
332 self.status = 1
333 return
333 334
334 335 def filterByExt(self, ext, localfolder):
335 336 fnameList = glob.glob1(localfolder,ext)
336 337 self.filenameList = [os.path.join(localfolder,x) for x in fnameList]
337 338
338 339 if len(self.filenameList) == 0:
339 340 self.status = 0
340 341
341 342 def run(self, dataOut, ext, localfolder, remotefolder, server, username, password, period=1):
342 343
343 344 self.counter += 1
344 345 if self.counter >= period:
345 346 self.filterByExt(ext, localfolder)
346 347
347 self.connect(server, username, password, remotefolder)
348
349 self.put()
350
351 self.close()
352
348 self.ftp_thread(server, username, password, remotefolder)
349
353 350 self.counter = 0
354 351
352 self.status = 1
353
General Comments 0
You need to be logged in to leave comments. Login now