##// END OF EJS Templates
DATABASES » ckanext-jro » api-cliente
RSS
v2.9.2 :: Add 'upload_multiple_files_advance' function
eynilupu -
r2:f8e143a2e63e
parent child
Show More
Show file before | Show file after | Hide comments
1 NO CONTENT: modified file, binary diff hidden
NO CONTENT: modified file, binary diff hidden
Show file before | Show file after | Hide comments
@@ -1,704 +1,813
1 from ckanapi import RemoteCKAN
1 from ckanapi import RemoteCKAN
2 #from ckanapi.errors import NotAuthorized, NotFound, ValidationError, SearchQueryError, SearchError, CKANAPIError, ServerIncompatibleError
2 #from ckanapi.errors import NotAuthorized, NotFound, ValidationError, SearchQueryError, SearchError, CKANAPIError, ServerIncompatibleError
3 import sys
3 import sys
4 import platform
4 import platform
5 import os
5 import os
6 import tempfile
6 import tempfile
7 import shutil
7 import shutil
8 import zipfile
8 import zipfile
9 import concurrent.futures
9 import concurrent.futures
10 import requests
10 import requests
11 import json
11 import json
12 import ast
12 import pathlib
13 import uuid
13 from datetime import datetime
14 from datetime import datetime
14 from tqdm import tqdm
15 from tqdm import tqdm
15 from six import string_types
16
16
17 class JROAPI():
17 class JROAPI():
18 """
18 """
19 FINALIDAD:
19 FINALIDAD:
20 Script para administrar y obtener la data del repositorio por medio de APIs.
20 Script para administrar y obtener la data del repositorio por medio de APIs.
21
21
22 REQUISITIOS PREVIOS:
22 REQUISITIOS PREVIOS:
23 - Paso 1: Tener "pip [Python 2]" o "pip3 [Python 3]" instalado:
23 - Paso 1: Tener "pip [Python 2]" o "pip3 [Python 3]" instalado:
24 - Paso 2: Instalar lo siguiente como admininstrador:
24 - Paso 2: Instalar lo siguiente como admininstrador:
25 En Python 2
25 En Python 2
26 - pip install ckanapi==4.5
26 - pip install ckanapi==4.5
27 - pip install requests
27 - pip install requests
28 - pip install tqdm
28 - pip install tqdm
29 En Python 3
29 En Python 3
30 - pip3 install ckanapi==4.5
30 - pip3 install ckanapi==4.5
31 - pip3 install requests
31 - pip3 install requests
32 - pip3 install tqdm
32 - pip3 install tqdm
33
33
34 FUNCIONES DISPONIBLES:
34 FUNCIONES DISPONIBLES:
35 - action
35 - action
36 - upload_file
36 - upload_file
37 - upload_multiple_files
37 - upload_multiple_files
38 - show
38 - show
39 - search
39 - search
40 - create
40 - create
41 - patch
41 - patch
42 - delete
42 - delete
43 - download_files
43 - download_files
44
44
45 EJEMPLOS:
45 EJEMPLOS:
46 #1:
46 #1:
47 with JROAPI('http://demo.example.com', Authorization='#########') as <access_name>:
47 with JROAPI('http://demo.example.com', Authorization='#########') as <access_name>:
48 ... some operation(s) ...
48 ... some operation(s) ...
49 #2:
49 #2:
50 <access_name> = JROAPI('http://example.com', Authorization='#########')
50 <access_name> = JROAPI('http://example.com', Authorization='#########')
51 ... some operation(s) ...
51 ... some operation(s) ...
52 <access_name>.ckan.close()
52 <access_name>.ckan.close()
53
53
54 REPORTAR ALGUN PROBLEMA:
54 REPORTAR ALGUN PROBLEMA:
55 Debe enviar un correo a eynilupu@igp.gob.pe detallando los siguientes pasos:
55 Debe enviar un correo a eynilupu@igp.gob.pe detallando los siguientes pasos:
56 1) Identifiquese
56 1) Identifiquese
57 2) Describir el problema
57 2) Describir el problema
58 3) ¿En que funcion esta el problema?
58 3) ¿En que funcion esta el problema?
59 4) ¿Que esperaba que hiciera la funcion sin el problema?
59 4) ¿Que esperaba que hiciera la funcion sin el problema?
60 """
60 """
61 def __init__(self, url, Authorization=None):
61 def __init__(self, url, Authorization=None):
62 ua = 'CKAN_JRO/1.1 (+'+str(url)+')'
62 ua = 'CKAN_JRO/1.1 (+'+str(url)+')'
63 #ua = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36'
63 #ua = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36'
64 self.ckan = RemoteCKAN(url, apikey=Authorization, user_agent=ua)
64 self.ckan = RemoteCKAN(url, apikey=Authorization, user_agent=ua)
65 #self.ckan = RemoteCKAN(url, apikey=Authorization)
65 #self.ckan = RemoteCKAN(url, apikey=Authorization)
66 self.Authorization = Authorization
66 self.Authorization = Authorization
67 if platform.system() == 'Windows':
67 if platform.system() == 'Windows':
68 self.separator = '\\'
68 self.separator = '\\'
69 else:
69 else:
70 self.separator = '/'
70 self.separator = '/'
71
71
72 self.chunk_size = 1024
72 self.chunk_size = 1024
73 self.list = []
73 self.list = []
74 self.dict = {}
74 self.dict = {}
75 self.str = ''
75 self.str = ''
76 self.check = 1
76 self.check = 1
77 self.cont = 0
77 self.cont = 0
78
78
79 def __enter__(self):
79 def __enter__(self):
80 return self
80 return self
81
81
82 def __exit__(self, *args):
82 def __exit__(self, *args):
83 self.ckan.close()
83 self.ckan.close()
84
84
85 def action(self, action, **kwargs):
85 def action(self, action, **kwargs):
86 """
86 """
87 FINALIDAD:
87 FINALIDAD:
88 Funcion para llamar a las APIs disponibles
88 Funcion para llamar a las APIs disponibles
89
89
90 APIs DISPONIBLES:
90 APIs DISPONIBLES:
91 CONSULTAR: "GUIA DE SCRIPT.pdf"
91 CONSULTAR: "GUIA DE SCRIPT.pdf"
92
92
93 EJEMPLO:
93 EJEMPLO:
94 <access_name>.action(<consuming API>, param_1 = <class 'param_1'>, ...)
94 <access_name>.action(<consuming API>, param_1 = <class 'param_1'>, ...)
95 """
95 """
96 #--------------- CASE: PACKAGE SEARCH ---------------#
96 #--------------- CASE: PACKAGE SEARCH ---------------#
97 if kwargs is not None:
97 if kwargs is not None:
98 if action == 'package_search':
98 if action == 'package_search':
99 self.list = ['facet_mincount', 'facet_limit', 'facet_field']
99 self.list = ['facet_mincount', 'facet_limit', 'facet_field']
100 for facet in self.list:
100 for facet in self.list:
101 if facet in kwargs:
101 if facet in kwargs:
102 kwargs[facet.replace('_', '.')] = kwargs[facet]
102 kwargs[facet.replace('_', '.')] = kwargs[facet]
103 kwargs.pop(facet)
103 kwargs.pop(facet)
104 #----------------------------------------------------#
104 #----------------------------------------------------#
105 try:
105 try:
106 return getattr(self.ckan.action, action)(**kwargs)
106 return getattr(self.ckan.action, action)(**kwargs)
107 except:
107 except:
108 _, exc_value, _ = sys.exc_info()
108 _, exc_value, _ = sys.exc_info()
109 return exc_value
109 return exc_value
110
110
111 def upload_file(self, dataset_id, file_path, file_date, file_type, **kwargs):
111 def upload_file(self, dataset_id, file_path, file_date, file_type, **kwargs):
112 '''
112 '''
113 FINALIDAD:
113 FINALIDAD:
114 Funcion para subir un unico archivo al repositorio del ROJ.
114 Funcion para subir un unico archivo al repositorio del ROJ.
115
115
116 PARAMETROS DISPONIBLES:
116 PARAMETROS DISPONIBLES:
117 CONSULTAR: "GUIA DE SCRIPT.pdf"
117 CONSULTAR: "GUIA DE SCRIPT.pdf"
118
118
119 ESTRUCTURA:
119 ESTRUCTURA:
120 <access_name>.upload_file(dataset_id = <class 'str'>, file_date = <class 'str'>, file_path = <class 'str'>, file_type = <class 'str'>, param_1 = <class 'param_1'>, ...)
120 <access_name>.upload_file(dataset_id = <class 'str'>, file_date = <class 'str'>, file_path = <class 'str'>, file_type = <class 'str'>, param_1 = <class 'param_1'>, ...)
121 '''
121 '''
122 self.list = ['package_id', 'upload', 'voc_file_type', 'name'] #file_date
122 self.list = ['package_id', 'upload', 'voc_file_type', 'name'] #file_date
123 for key1, value1 in kwargs.items():
123 for key1, value1 in kwargs.items():
124 if not key1 in self.list:
124 if not key1 in self.list:
125 self.dict[key1] = value1
125 self.dict[key1] = value1
126
126
127 #---------------------------#
127 #---------------------------#
128 if not 'others' in kwargs:
128 if not 'others' in kwargs:
129 self.dict['others'] = ''
129 self.dict['others'] = ''
130 else:
130 else:
131 if isinstance(kwargs['others'], list):
131 if isinstance(kwargs['others'], list):
132 self.dict['others'] = json.dumps(kwargs['others'])
132 self.dict['others'] = json.dumps(kwargs['others'])
133 #---------------------------#
133 #---------------------------#
134
134
135 if not os.path.isfile(file_path):
135 if not os.path.isfile(file_path):
136 return 'File "%s" not exist' % (file_path)
136 return 'File "%s" not exist' % (file_path)
137
138 if not 'format' in self.dict:
139 self.str = ''.join(pathlib.Path(file_path).suffixes)
140 if len(self.str) > 0:
141 self.dict['format'] = self.str.upper()[1:]
137
142
138 try:
143 try:
139 return getattr(self.ckan.action, 'resource_create')(package_id=dataset_id, file_date=file_date, upload=open(file_path, 'rb'), voc_file_type=file_type, name=file_path.split(self.separator)[-1], **self.dict)
144 return getattr(self.ckan.action, 'resource_create')(package_id=dataset_id, file_date=file_date, upload=open(file_path, 'rb'), voc_file_type=file_type, name=pathlib.Path(file_path).name, **self.dict)
140 except:
145 except:
141 _, exc_value, _ = sys.exc_info()
146 _, exc_value, _ = sys.exc_info()
142 return exc_value
147 return exc_value
148
149
150 def upload_multiple_files_advance(self, dataset_id, path_files, file_date, file_type, **kwargs):
151 '''
152 FINALIDAD:
153 Funcion para subir multiples archivos al repositorio del ROJ.
154
155 PARAMETROS DISPONIBLES:
156 CONSULTAR: "GUIA DE SCRIPT.pdf"
157
158 ESTRUCTURA:
159 <access_name>.upload_multiple_files_advance(dataset_id = <class 'str'>, path_files = <class 'list of strings'>, file_date = <class 'str'>, file_type = <class 'str'>, param_1 = <class 'param_1'>, ...)
160 '''
161 self.list = ['package_id', 'upload', 'voc_file_type', 'name']
162
163 for key1, value1 in kwargs.items():
164 if not key1 in self.list:
165 self.dict[key1] = value1
166
167 #---------------------------#
168 if not 'others' in kwargs:
169 self.dict['others'] = ''
170 else:
171 if isinstance(kwargs['others'], list):
172 self.dict['others'] = json.dumps(kwargs['others'])
173 #---------------------------#
174
175 params_dict = {'upload':[], 'name':[]}
176 #---------------CASO : "path" or "path_list"-----------------#
177 if type(path_files) is list:
178 if len(path_files) != 0:
179 path_files.sort()
180 for u in path_files:
181 if os.path.isfile(u):
182 params_dict['upload'].append(open(u, 'rb'))
183 params_dict['name'].append(pathlib.Path(u).name)
184 else:
185 return 'File "%s" does not exist' % (u)
186 else:
187 return 'ERROR:: "path_list is empty"'
188 elif type(path_files) is str:
189 if os.path.isdir(path_files):
190 path_order = [f for f in os.listdir(path_files) if os.path.isfile(os.path.join(path_files, f))]
191 path_order.sort()
192 if path_order:
193 for name in path_order:
194 params_dict['upload'].append(open(os.path.join(path_files, name), 'rb'))
195 params_dict['name'].append(name)
196 else:
197 return "ERROR:: There aren't files in this directory"
198 else:
199 return 'ERROR:: Directory "%s" does not exist' % (path_files)
200 else:
201 return 'ERROR:: "path_files" must be a str or list'
202 #------------------------------------------------------------#
203 resource_extend = []
204 files_dict = {}
205 for count, name in enumerate(params_dict['name']):
206 param_list = {"name": name, "file_date": file_date, "voc_file_type": file_type}
207 param_list.update(self.dict)
208
209 if not 'format' in param_list:
210 format = ''.join(pathlib.Path(name).suffixes)
211 if len(format) > 0:
212 param_list['format'] = format.upper()[1:]
143
213
214 resource_extend.append(param_list)
215 files_dict['update__resources__-'+ str(len(params_dict['name'])-count) +'__upload'] = (name, params_dict['upload'][count])
216
217 #------------------------------------------------------------#
218 try:
219 uuid.UUID(str(dataset_id), version=4)
220 package_id_or_name = '"id": "' + str(dataset_id) + '"'
221 except ValueError:
222 package_id_or_name = '"name": "' + str(dataset_id) + '"'
223 #------------------------------------------------------------#
224 print('"{}" file(s) found >> uploading'.format(len(params_dict['name'])))
225 try:
226 result = self.ckan.call_action(
227 'package_revise',
228 {'match': '{'+ str(package_id_or_name) +'}', 'update__resources__extend': json.dumps(resource_extend)},
229 files=files_dict
230 )
231 print('Uploaded file(s) successfully')
232 return result
233 except:
234 print('ERROR :: Use the "print" option for more information')
235 _, exc_value, _ = sys.exc_info()
236 return exc_value
237
144 def upload_multiple_files(self, dataset_id, path_files, date_files, type_files, **kwargs):
238 def upload_multiple_files(self, dataset_id, path_files, date_files, type_files, **kwargs):
145 '''
239 '''
146 FINALIDAD:
240 FINALIDAD:
147 Funcion para subir multiples archivos al repositorio del ROJ.
241 Funcion para subir multiples archivos al repositorio del ROJ.
148
242
149 PARAMETROS DISPONIBLES:
243 PARAMETROS DISPONIBLES:
150 CONSULTAR: "GUIA DE SCRIPT.pdf"
244 CONSULTAR: "GUIA DE SCRIPT.pdf"
151
245
152 ESTRUCTURA:
246 ESTRUCTURA:
153 <access_name>.upload_multiple_files(dataset_id = <class 'str'>, path_files = <class 'str'> or <class 'list of strings'>, date_files = <class 'str'> or <class 'list of strings'>, type_files = <class 'str'> or <class 'list of strings'>, param_1 = <class 'param_1'>, ...)
247 <access_name>.upload_multiple_files(dataset_id = <class 'str'>, path_files = <class 'str'> or <class 'list of strings'>, date_files = <class 'str'> or <class 'list of strings'>, type_files = <class 'str'> or <class 'list of strings'>, param_1 = <class 'param_1'>, ...)
154 '''
248 '''
155 params_dict = {'upload':[], 'name':[]} #ADD FORMAT
249
250 params_dict = {'upload':[], 'name':[]}
251 if not 'format' in kwargs:
252 params_dict.update({'format':[]})
156 #---------------CASO : "path" or "path_list"-----------------#
253 #---------------CASO : "path" or "path_list"-----------------#
157 if type(path_files) is list:
254 if type(path_files) is list:
158 if len(path_files) != 0:
255 if len(path_files) != 0:
159 path_files.sort()
256 path_files.sort()
160 for u in path_files:
257 for u in path_files:
161 if os.path.isfile(u):
258 if os.path.isfile(u):
162 params_dict['upload'].append(open(u, 'rb'))
259 params_dict['upload'].append(open(u, 'rb'))
163 params_dict['name'].append(u.split(self.separator)[-1])
260 params_dict['name'].append(pathlib.Path(u).name)
261 if not 'format' in kwargs:
262 format = ''.join(pathlib.Path(u).suffixes)
263 if len(format) > 0:
264 params_dict['format'].append(format.upper()[1:])
265 else:
266 params_dict['format'].append('')
164 else:
267 else:
165 return 'File "%s" not exist' % (u)
268 return 'File "%s" does not exist' % (u)
166 else:
269 else:
167 return 'ERROR:: "path_list is empty"'
270 return 'ERROR:: "path_list is empty"'
168 elif type(path_files) is str:
271 elif type(path_files) is str:
169 if os.path.isdir(path_files):
272 if os.path.isdir(path_files):
170 path_order = [f for f in os.listdir(path_files) if os.path.isfile(os.path.join(path_files, f))]
273 path_order = [f for f in os.listdir(path_files) if os.path.isfile(os.path.join(path_files, f))]
171 path_order.sort()
274 path_order.sort()
172 if path_order:
275 if path_order:
173 for name in path_order:
276 for name in path_order:
174 params_dict['upload'].append(open(os.path.join(path_files, name), 'rb'))
277 params_dict['upload'].append(open(os.path.join(path_files, name), 'rb'))
175 params_dict['name'].append(name)
278 params_dict['name'].append(name)
279 if not 'format' in kwargs:
280 format = ''.join(pathlib.Path(name).suffixes)
281 if len(format) > 0:
282 params_dict['format'].append(format.upper()[1:])
283 else:
284 params_dict['format'].append('')
176 else:
285 else:
177 return 'ERROR:: Directory is empty'
286 return "ERROR:: There aren't files in this directory"
178 else:
287 else:
179 return 'ERROR:: Directory "%s" not exist' % (path_files)
288 return 'ERROR:: Directory "%s" does not exist' % (path_files)
180 else:
289 else:
181 return 'ERROR:: "path_files" must be a str or list'
290 return 'ERROR:: "path_files" must be a str or list'
182 #------------------------------------------------------------#
291 #------------------------------------------------------------#
183 params_no_dict = {'package_id': dataset_id}
292 params_no_dict = {'package_id': dataset_id}
184 if type(date_files) is list:
293 if type(date_files) is list:
185 params_dict['file_date'] = date_files
294 params_dict['file_date'] = date_files
186 else:
295 else:
187 params_no_dict['file_date'] = date_files
296 params_no_dict['file_date'] = date_files
188
297
189 if type(type_files) is list:
298 if type(type_files) is list:
190 params_dict['voc_file_type'] = type_files
299 params_dict['voc_file_type'] = type_files
191 else:
300 else:
192 params_no_dict['voc_file_type'] = type_files
301 params_no_dict['voc_file_type'] = type_files
193
302
194 for key1, value1 in kwargs.items():
303 for key1, value1 in kwargs.items():
195 if not key1 in params_dict and not key1 in params_no_dict and key1 != 'others':
304 if not key1 in params_dict and not key1 in params_no_dict and key1 != 'others':
196 if type(value1) is list:
305 if type(value1) is list:
197 params_dict[key1] = value1
306 params_dict[key1] = value1
198 else:
307 else:
199 params_no_dict[key1] = value1
308 params_no_dict[key1] = value1
200 #------------------------------------------#
309 #------------------------------------------#
201 if not 'others' in kwargs:
310 if not 'others' in kwargs:
202 params_no_dict['others'] = ''
311 params_no_dict['others'] = ''
203 else:
312 else:
204 if isinstance(kwargs['others'], tuple):
313 if isinstance(kwargs['others'], tuple):
205 params_dict['others'] = [json.dumps(w) for w in kwargs['others']]
314 params_dict['others'] = [json.dumps(w) for w in kwargs['others']]
206 elif isinstance(kwargs['others'], list):
315 elif isinstance(kwargs['others'], list):
207 params_no_dict['others'] = json.dumps(kwargs['others'])
316 params_no_dict['others'] = json.dumps(kwargs['others'])
208 elif isinstance(kwargs['others'], str):
317 elif isinstance(kwargs['others'], str):
209 params_no_dict['others'] = kwargs['others']
318 params_no_dict['others'] = kwargs['others']
210 else:
319 else:
211 return 'ERROR:: "others" must be a tuple, list or str'
320 return 'ERROR:: "others" must be a tuple, list or str'
212 #------------------------------------------#
321 #------------------------------------------#
213 len_params_dict = []
322 len_params_dict = []
214 for value2 in params_dict.values():
323 for value2 in params_dict.values():
215 len_params_dict.append(len(value2))
324 len_params_dict.append(len(value2))
216
325
217 if len(list(set(len_params_dict))) > 1:
326 if len(list(set(len_params_dict))) > 1:
218 return 'ERROR:: All lists must be the same length: %s' % (len(params_dict['name']))
327 return 'ERROR:: All lists must be the same length: %s' % (len(params_dict['name']))
219 #------------------------------------------------------------#
328 #------------------------------------------------------------#
220
329 print('"{}" file(s) found >> uploading'.format(len(params_dict['name'])))
221 for v in range(len(params_dict['name'])):
330 for v in range(len(params_dict['name'])):
222 try:
331 try:
223 send = {}
332 send = {}
224 for key_dict, value_dict in params_dict.items():
333 for key_dict, value_dict in params_dict.items():
225 send[key_dict] = value_dict[v]
334 send[key_dict] = value_dict[v]
226 for key_no_dict, value_no_dict in params_no_dict.items():
335 for key_no_dict, value_no_dict in params_no_dict.items():
227 send[key_no_dict] = value_no_dict
336 send[key_no_dict] = value_no_dict
228
337
229 self.list.append(getattr(self.ckan.action, 'resource_create')(**send))
338 self.list.append(getattr(self.ckan.action, 'resource_create')(**send))
230 print('File "{}" was uploaded successfully'.format(params_dict['name'][v]))
339 print('File #{} :: "{}" was uploaded successfully'.format(v+1, params_dict['name'][v]))
231 except:
340 except:
232 _, exc_value, _ = sys.exc_info()
341 _, exc_value, _ = sys.exc_info()
233 self.list.append(exc_value)
342 self.list.append(exc_value)
234 print('Error uploading "{}" file'.format(params_dict['name'][v]))
343 print('File #{} :: Error uploading "{}" file'.format(v+1, params_dict['name'][v]))
235 return self.list
344 return self.list
236 #------------------------------------------------------------#
345 #------------------------------------------------------------#
237
346
238 def show(self, type_option, id, **kwargs):
347 def show(self, type_option, id, **kwargs):
239 '''
348 '''
240 FINALIDAD:
349 FINALIDAD:
241 Funcion personalizada para una busqueda en especifico.
350 Funcion personalizada para una busqueda en especifico.
242
351
243 PARAMETROS DISPONIBLES:
352 PARAMETROS DISPONIBLES:
244 CONSULTAR: "GUIA DE SCRIPT.pdf"
353 CONSULTAR: "GUIA DE SCRIPT.pdf"
245
354
246 ESTRUCTURA:
355 ESTRUCTURA:
247 <access_name>.show(type_option = <class 'str'>, id = <class 'str'>, param_1 = <class 'param_1'>, ...)
356 <access_name>.show(type_option = <class 'str'>, id = <class 'str'>, param_1 = <class 'param_1'>, ...)
248 '''
357 '''
249 if type(type_option) is str:
358 if type(type_option) is str:
250 try:
359 try:
251 if type_option == 'dataset':
360 if type_option == 'dataset':
252 return getattr(self.ckan.action, 'package_show')(id=id, **kwargs)
361 return getattr(self.ckan.action, 'package_show')(id=id, **kwargs)
253 elif type_option == 'resource':
362 elif type_option == 'resource':
254 return getattr(self.ckan.action, 'resource_show')(id=id, **kwargs)
363 return getattr(self.ckan.action, 'resource_show')(id=id, **kwargs)
255 elif type_option == 'project':
364 elif type_option == 'project':
256 return getattr(self.ckan.action, 'organization_show')(id=id, **kwargs)
365 return getattr(self.ckan.action, 'organization_show')(id=id, **kwargs)
257 elif type_option == 'collaborator':
366 elif type_option == 'collaborator':
258 return getattr(self.ckan.action, 'package_collaborator_list_for_user')(id=id, **kwargs)
367 return getattr(self.ckan.action, 'package_collaborator_list_for_user')(id=id, **kwargs)
259 elif type_option == 'member':
368 elif type_option == 'member':
260 return getattr(self.ckan.action, 'organization_list_for_user')(id=id, **kwargs)
369 return getattr(self.ckan.action, 'organization_list_for_user')(id=id, **kwargs)
261 elif type_option == 'vocabulary':
370 elif type_option == 'vocabulary':
262 return getattr(self.ckan.action, 'vocabulary_show')(id=id, **kwargs)
371 return getattr(self.ckan.action, 'vocabulary_show')(id=id, **kwargs)
263 elif type_option == 'tag':
372 elif type_option == 'tag':
264 if not 'vocabulary_id' in kwargs:
373 if not 'vocabulary_id' in kwargs:
265 print('Missing "vocabulary_id" value: assume it is a free tag')
374 print('Missing "vocabulary_id" value: assume it is a free tag')
266 return getattr(self.ckan.action, 'tag_show')(id=id, **kwargs)
375 return getattr(self.ckan.action, 'tag_show')(id=id, **kwargs)
267 elif type_option == 'user':
376 elif type_option == 'user':
268 return getattr(self.ckan.action, 'user_show')(id=id, **kwargs)
377 return getattr(self.ckan.action, 'user_show')(id=id, **kwargs)
269 elif type_option == 'job':
378 elif type_option == 'job':
270 return getattr(self.ckan.action, 'job_show')(id=id, **kwargs)
379 return getattr(self.ckan.action, 'job_show')(id=id, **kwargs)
271 else:
380 else:
272 return 'ERROR:: "%s" is not accepted' % (type_option)
381 return 'ERROR:: "%s" is not accepted' % (type_option)
273 except:
382 except:
274 _, exc_value, _ = sys.exc_info()
383 _, exc_value, _ = sys.exc_info()
275 return exc_value
384 return exc_value
276 else:
385 else:
277 return 'ERROR:: "type_option" must be a str'
386 return 'ERROR:: "type_option" must be a str'
278
387
279 def search(self, type_option, query=None, **kwargs):
388 def search(self, type_option, query=None, **kwargs):
280 '''
389 '''
281 FINALIDAD:
390 FINALIDAD:
282 Funcion personalizada para busquedas que satisfagan algun criterio.
391 Funcion personalizada para busquedas que satisfagan algun criterio.
283
392
284 PARAMETROS DISPONIBLES:
393 PARAMETROS DISPONIBLES:
285 CONSULTAR: "GUIA DE SCRIPT.pdf"
394 CONSULTAR: "GUIA DE SCRIPT.pdf"
286
395
287 ESTRUCTURA:
396 ESTRUCTURA:
288 <access_name>.search(type_option = <class 'str'>, query = <class 'dict'>, param_1 = <class 'param_1'>, ...)
397 <access_name>.search(type_option = <class 'str'>, query = <class 'dict'>, param_1 = <class 'param_1'>, ...)
289 '''
398 '''
290 if type(type_option) is str:
399 if type(type_option) is str:
291 try:
400 try:
292 if type_option == 'dataset':
401 if type_option == 'dataset':
293 key_replace = ['fq', 'fq_list', 'include_private']
402 key_replace = ['fq', 'fq_list', 'include_private']
294 key_point = ['facet_mincount', 'facet_limit', 'facet_field']
403 key_point = ['facet_mincount', 'facet_limit', 'facet_field']
295 for key1, value1 in kwargs.items():
404 for key1, value1 in kwargs.items():
296 if not key1 in key_replace:
405 if not key1 in key_replace:
297 if key1 in key_point:
406 if key1 in key_point:
298 self.dict[key1.replace('_', '.')] = value1
407 self.dict[key1.replace('_', '.')] = value1
299 else:
408 else:
300 self.dict[key1] = value1
409 self.dict[key1] = value1
301
410
302 if query is not None:
411 if query is not None:
303 if type(query) is dict:
412 if type(query) is dict:
304 self.dict['fq_list'] = []
413 self.dict['fq_list'] = []
305 #NUM_RESOURCES_MIN / NUM_RESOURCES_MAX
414 #NUM_RESOURCES_MIN / NUM_RESOURCES_MAX
306 #----------------------------------------------------#
415 #----------------------------------------------------#
307 if 'dataset_start_date' in query:
416 if 'dataset_start_date' in query:
308 if type(query['dataset_start_date']) is str:
417 if type(query['dataset_start_date']) is str:
309 try:
418 try:
310 datetime.strptime(query['dataset_start_date'], '%Y-%m-%d')
419 datetime.strptime(query['dataset_start_date'], '%Y-%m-%d')
311 if len(query['dataset_start_date']) != 10:
420 if len(query['dataset_start_date']) != 10:
312 return '"dataset_start_date", must be: <YYYY-MM-DD>'
421 return '"dataset_start_date", must be: <YYYY-MM-DD>'
313 self.dict['fq_list'].append('dataset_start_date:"'+query['dataset_start_date']+'"')
422 self.dict['fq_list'].append('dataset_start_date:"'+query['dataset_start_date']+'"')
314 self.list.append('dataset_start_date')
423 self.list.append('dataset_start_date')
315 except:
424 except:
316 return '"dataset_start_date" incorrect: "%s"' % (query['dataset_start_date'])
425 return '"dataset_start_date" incorrect: "%s"' % (query['dataset_start_date'])
317 else:
426 else:
318 return '"dataset_start_date" must be <str>'
427 return '"dataset_start_date" must be <str>'
319 #----------------------------------------------------#
428 #----------------------------------------------------#
320 if 'dataset_end_date' in query:
429 if 'dataset_end_date' in query:
321 if type(query['dataset_end_date']) is str:
430 if type(query['dataset_end_date']) is str:
322 try:
431 try:
323 datetime.strptime(query['dataset_end_date'], '%Y-%m-%d')
432 datetime.strptime(query['dataset_end_date'], '%Y-%m-%d')
324 if len(query['dataset_end_date']) != 10:
433 if len(query['dataset_end_date']) != 10:
325 return '"dataset_end_date", must be: <YYYY-MM-DD>'
434 return '"dataset_end_date", must be: <YYYY-MM-DD>'
326
435
327 if 'dataset_start_date' in query:
436 if 'dataset_start_date' in query:
328 if query['dataset_start_date'] > query['dataset_end_date']:
437 if query['dataset_start_date'] > query['dataset_end_date']:
329 return '"dataset_end_date" must be greater than "dataset_start_date"'
438 return '"dataset_end_date" must be greater than "dataset_start_date"'
330
439
331 self.dict['fq_list'].append('dataset_end_date:"'+query['dataset_end_date']+'"')
440 self.dict['fq_list'].append('dataset_end_date:"'+query['dataset_end_date']+'"')
332 self.list.append('dataset_end_date')
441 self.list.append('dataset_end_date')
333 except:
442 except:
334 return '"dataset_end_date" incorrect: "%s"' % (query['dataset_end_date'])
443 return '"dataset_end_date" incorrect: "%s"' % (query['dataset_end_date'])
335 else:
444 else:
336 return '"dataset_end_date" must be <str>'
445 return '"dataset_end_date" must be <str>'
337 #----------------------------------------------------#
446 #----------------------------------------------------#
338 for key, value in query.items():
447 for key, value in query.items():
339 if value is not None and not key in self.list:
448 if value is not None and not key in self.list:
340 self.dict['fq_list'].append(str(key)+':"'+str(value)+'"')
449 self.dict['fq_list'].append(str(key)+':"'+str(value)+'"')
341 else:
450 else:
342 return '"query" must be <dict>'
451 return '"query" must be <dict>'
343
452
344 return getattr(self.ckan.action, 'package_search')(include_private=True, **self.dict)
453 return getattr(self.ckan.action, 'package_search')(include_private=True, **self.dict)
345
454
346 elif type_option == 'resource':
455 elif type_option == 'resource':
347 for key1, value1 in kwargs.items():
456 for key1, value1 in kwargs.items():
348 if key1 != 'fields':
457 if key1 != 'fields':
349 self.dict[key1] = value1
458 self.dict[key1] = value1
350
459
351 if query is not None:
460 if query is not None:
352 if type(query) is dict:
461 if type(query) is dict:
353 #----------------------------------------------------#
462 #----------------------------------------------------#
354 if 'file_date_min' in query:
463 if 'file_date_min' in query:
355 if type(query['file_date_min']) is str:
464 if type(query['file_date_min']) is str:
356 try:
465 try:
357 datetime.strptime(query['file_date_min'], '%Y-%m-%d')
466 datetime.strptime(query['file_date_min'], '%Y-%m-%d')
358 if len(query['file_date_min']) != 10:
467 if len(query['file_date_min']) != 10:
359 return '"file_date_min", must be: <YYYY-MM-DD>'
468 return '"file_date_min", must be: <YYYY-MM-DD>'
360 except:
469 except:
361 return '"file_date_min" incorrect: "%s"' % (query['file_date_min'])
470 return '"file_date_min" incorrect: "%s"' % (query['file_date_min'])
362 else:
471 else:
363 return '"file_date_min" must be <str>'
472 return '"file_date_min" must be <str>'
364 #----------------------------------------------------#
473 #----------------------------------------------------#
365 if 'file_date_max' in query:
474 if 'file_date_max' in query:
366 if type(query['file_date_max']) is str:
475 if type(query['file_date_max']) is str:
367 try:
476 try:
368 datetime.strptime(query['file_date_max'], '%Y-%m-%d')
477 datetime.strptime(query['file_date_max'], '%Y-%m-%d')
369 if len(query['file_date_max']) != 10:
478 if len(query['file_date_max']) != 10:
370 return '"file_date_max", must be: <YYYY-MM-DD>'
479 return '"file_date_max", must be: <YYYY-MM-DD>'
371
480
372 if 'file_date_min' in query:
481 if 'file_date_min' in query:
373 if query['file_date_min'] > query['file_date_max']:
482 if query['file_date_min'] > query['file_date_max']:
374 return '"file_date_max" must be greater than "file_date_min"'
483 return '"file_date_max" must be greater than "file_date_min"'
375 except:
484 except:
376 return '"file_date_max" incorrect: "%s"' % (query['file_date_max'])
485 return '"file_date_max" incorrect: "%s"' % (query['file_date_max'])
377 else:
486 else:
378 return '"file_date_max" must be <str>'
487 return '"file_date_max" must be <str>'
379 #----------------------------------------------------#
488 #----------------------------------------------------#
380 self.dict['query'] = query
489 self.dict['query'] = query
381 else:
490 else:
382 return '"query" must be <dict>'
491 return '"query" must be <dict>'
383 return getattr(self.ckan.action, 'resources_search')(**self.dict)
492 return getattr(self.ckan.action, 'resources_search')(**self.dict)
384
493
385 elif type_option == 'tag':
494 elif type_option == 'tag':
386 for key1, value1 in kwargs.items():
495 for key1, value1 in kwargs.items():
387 if key1 != 'fields':
496 if key1 != 'fields':
388 self.dict[key1] = value1
497 self.dict[key1] = value1
389
498
390 if not 'vocabulary_id' in kwargs:
499 if not 'vocabulary_id' in kwargs:
391 print('Missing "vocabulary_id" value: tags that don’t belong to any vocabulary')
500 print('Missing "vocabulary_id" value: tags that don’t belong to any vocabulary')
392 else:
501 else:
393 print('Only tags that belong to "{}" vocabulary'.format(kwargs['vocabulary_id']))
502 print('Only tags that belong to "{}" vocabulary'.format(kwargs['vocabulary_id']))
394
503
395 if query is not None:
504 if query is not None:
396 if type(query) is dict:
505 if type(query) is dict:
397 if 'search' in query:
506 if 'search' in query:
398 if type(query['search']) is list or type(query['search']) is str:
507 if type(query['search']) is list or type(query['search']) is str:
399 self.dict['query'] = query['search']
508 self.dict['query'] = query['search']
400 else:
509 else:
401 return '"search" must be <list> or <str>'
510 return '"search" must be <list> or <str>'
402 else:
511 else:
403 return '"query" must be <dict>'
512 return '"query" must be <dict>'
404 return getattr(self.ckan.action, 'tag_search')(**self.dict)
513 return getattr(self.ckan.action, 'tag_search')(**self.dict)
405
514
406 else:
515 else:
407 return 'ERROR:: "%s" is not accepted' % (type_option)
516 return 'ERROR:: "%s" is not accepted' % (type_option)
408
517
409 except:
518 except:
410 _, exc_value, _ = sys.exc_info()
519 _, exc_value, _ = sys.exc_info()
411 return exc_value
520 return exc_value
412 else:
521 else:
413 return 'ERROR:: "type_option" must be <str>'
522 return 'ERROR:: "type_option" must be <str>'
414
523
415 def create(self, type_option, **kwargs):
524 def create(self, type_option, **kwargs):
416 '''
525 '''
417 FINALIDAD:
526 FINALIDAD:
418 Funcion personalizada para crear.
527 Funcion personalizada para crear.
419
528
420 PARAMETROS DISPONIBLES:
529 PARAMETROS DISPONIBLES:
421 CONSULTAR: "GUIA DE SCRIPT.pdf"
530 CONSULTAR: "GUIA DE SCRIPT.pdf"
422
531
423 ESTRUCTURA:
532 ESTRUCTURA:
424 <access_name>.create(type_option = <class 'str'>, param_1 = <class 'param_1'>, ...)
533 <access_name>.create(type_option = <class 'str'>, param_1 = <class 'param_1'>, ...)
425 '''
534 '''
426 if type(type_option) is str:
535 if type(type_option) is str:
427 try:
536 try:
428 if type_option == 'dataset':
537 if type_option == 'dataset':
429 return getattr(self.ckan.action, 'package_create')(**kwargs)
538 return getattr(self.ckan.action, 'package_create')(**kwargs)
430 elif type_option == 'project':
539 elif type_option == 'project':
431 return getattr(self.ckan.action, 'organization_create')(**kwargs)
540 return getattr(self.ckan.action, 'organization_create')(**kwargs)
432 elif type_option == 'member':
541 elif type_option == 'member':
433 return getattr(self.ckan.action, 'organization_member_create')(**kwargs)
542 return getattr(self.ckan.action, 'organization_member_create')(**kwargs)
434 elif type_option == 'collaborator':
543 elif type_option == 'collaborator':
435 return getattr(self.ckan.action, 'package_collaborator_create')(**kwargs)
544 return getattr(self.ckan.action, 'package_collaborator_create')(**kwargs)
436 elif type_option == 'vocabulary':
545 elif type_option == 'vocabulary':
437 return getattr(self.ckan.action, 'vocabulary_create')(**kwargs)
546 return getattr(self.ckan.action, 'vocabulary_create')(**kwargs)
438 elif type_option == 'tag':
547 elif type_option == 'tag':
439 return getattr(self.ckan.action, 'tag_create')(**kwargs)
548 return getattr(self.ckan.action, 'tag_create')(**kwargs)
440 elif type_option == 'user':
549 elif type_option == 'user':
441 return getattr(self.ckan.action, 'user_create')(**kwargs)
550 return getattr(self.ckan.action, 'user_create')(**kwargs)
442 else:
551 else:
443 return 'ERROR:: "%s" is not accepted' % (type_option)
552 return 'ERROR:: "%s" is not accepted' % (type_option)
444 except:
553 except:
445 _, exc_value, _ = sys.exc_info()
554 _, exc_value, _ = sys.exc_info()
446 return exc_value
555 return exc_value
447 else:
556 else:
448 return 'ERROR:: "type_option" must be <str>'
557 return 'ERROR:: "type_option" must be <str>'
449
558
450 def patch(self, type_option, **kwargs):
559 def patch(self, type_option, **kwargs):
451 '''
560 '''
452 FINALIDAD:
561 FINALIDAD:
453 Funciones personalizadas para actualizar
562 Funciones personalizadas para actualizar
454
563
455 PARAMETROS DISPONIBLES:
564 PARAMETROS DISPONIBLES:
456 CONSULTAR: "GUIA DE SCRIPT.pdf"
565 CONSULTAR: "GUIA DE SCRIPT.pdf"
457
566
458 ESTRUCTURA:
567 ESTRUCTURA:
459 <access_name>.patch(type_option = <class 'str'>, param_1 = <class 'param_1'>, ...)
568 <access_name>.patch(type_option = <class 'str'>, param_1 = <class 'param_1'>, ...)
460 '''
569 '''
461 if type(type_option) is str:
570 if type(type_option) is str:
462 try:
571 try:
463 if type_option == 'dataset':
572 if type_option == 'dataset':
464 return getattr(self.ckan.action, 'package_patch')(**kwargs)
573 return getattr(self.ckan.action, 'package_patch')(**kwargs)
465 elif type_option == 'project':
574 elif type_option == 'project':
466 return getattr(self.ckan.action, 'organization_patch')(**kwargs)
575 return getattr(self.ckan.action, 'organization_patch')(**kwargs)
467 elif type_option == 'resource':
576 elif type_option == 'resource':
468 return getattr(self.ckan.action, 'resource_patch')(**kwargs)
577 return getattr(self.ckan.action, 'resource_patch')(**kwargs)
469 elif type_option == 'member':
578 elif type_option == 'member':
470 return getattr(self.ckan.action, 'organization_member_create')(**kwargs)
579 return getattr(self.ckan.action, 'organization_member_create')(**kwargs)
471 elif type_option == 'collaborator':
580 elif type_option == 'collaborator':
472 return getattr(self.ckan.action, 'package_collaborator_create')(**kwargs)
581 return getattr(self.ckan.action, 'package_collaborator_create')(**kwargs)
473 else:
582 else:
474 return 'ERROR:: "%s" is not accepted' % (type_option)
583 return 'ERROR:: "%s" is not accepted' % (type_option)
475 except:
584 except:
476 _, exc_value, _ = sys.exc_info()
585 _, exc_value, _ = sys.exc_info()
477 return exc_value
586 return exc_value
478 else:
587 else:
479 return 'ERROR:: "type_option" must be <str>'
588 return 'ERROR:: "type_option" must be <str>'
480
589
481 def delete(self, type_option, select=None, **kwargs):
590 def delete(self, type_option, select=None, **kwargs):
482 '''
591 '''
483 FINALIDAD:
592 FINALIDAD:
484 Función personalizada para eliminar y/o purgar.
593 Función personalizada para eliminar y/o purgar.
485
594
486 PARAMETROS DISPONIBLES:
595 PARAMETROS DISPONIBLES:
487 CONSULTAR: "GUIA DE SCRIPT.pdf"
596 CONSULTAR: "GUIA DE SCRIPT.pdf"
488
597
489 ESTRUCTURA:
598 ESTRUCTURA:
490 <access_name>.delete(type_option = <class 'str'>, param_1 = <class 'param_1'>, ...)
599 <access_name>.delete(type_option = <class 'str'>, param_1 = <class 'param_1'>, ...)
491 '''
600 '''
492 if type(type_option) is str:
601 if type(type_option) is str:
493 try:
602 try:
494 if type_option == 'dataset':
603 if type_option == 'dataset':
495 if select is None:
604 if select is None:
496 return 'ERROR:: "select" must not be "None"'
605 return 'ERROR:: "select" must not be "None"'
497 else:
606 else:
498 if 'delete' == select:
607 if 'delete' == select:
499 return getattr(self.ckan.action, 'package_delete')(**kwargs)
608 return getattr(self.ckan.action, 'package_delete')(**kwargs)
500 elif 'purge' == select:
609 elif 'purge' == select:
501 return getattr(self.ckan.action, 'dataset_purge')(**kwargs)
610 return getattr(self.ckan.action, 'dataset_purge')(**kwargs)
502 else:
611 else:
503 return 'ERROR:: "%s" is not accepted' % (select)
612 return 'ERROR:: "%s" is not accepted' % (select)
504 elif type_option == 'project':
613 elif type_option == 'project':
505 if select is None:
614 if select is None:
506 return 'ERROR:: "select" must not be "None"'
615 return 'ERROR:: "select" must not be "None"'
507 else:
616 else:
508 if 'delete' == select:
617 if 'delete' == select:
509 return getattr(self.ckan.action, 'organization_delete')(**kwargs)
618 return getattr(self.ckan.action, 'organization_delete')(**kwargs)
510 elif 'purge' == select:
619 elif 'purge' == select:
511 return getattr(self.ckan.action, 'organization_purge')(**kwargs)
620 return getattr(self.ckan.action, 'organization_purge')(**kwargs)
512 else:
621 else:
513 return 'ERROR:: "%s" is not accepted' % (select)
622 return 'ERROR:: "%s" is not accepted' % (select)
514 elif type_option == 'resource':
623 elif type_option == 'resource':
515 return getattr(self.ckan.action, 'resource_delete')(**kwargs)
624 return getattr(self.ckan.action, 'resource_delete')(**kwargs)
516 elif type_option == 'vocabulary':
625 elif type_option == 'vocabulary':
517 return getattr(self.ckan.action, 'vocabulary_delete')(**kwargs)
626 return getattr(self.ckan.action, 'vocabulary_delete')(**kwargs)
518 elif type_option == 'tag':
627 elif type_option == 'tag':
519 return getattr(self.ckan.action, 'tag_delete')(**kwargs)
628 return getattr(self.ckan.action, 'tag_delete')(**kwargs)
520 elif type_option == 'user':
629 elif type_option == 'user':
521 return getattr(self.ckan.action, 'user_delete')(**kwargs)
630 return getattr(self.ckan.action, 'user_delete')(**kwargs)
522 else:
631 else:
523 return 'ERROR:: "%s" is not accepted' % (type_option)
632 return 'ERROR:: "%s" is not accepted' % (type_option)
524 except:
633 except:
525 _, exc_value, _ = sys.exc_info()
634 _, exc_value, _ = sys.exc_info()
526 return exc_value
635 return exc_value
527 else:
636 else:
528 return 'ERROR:: "type_option" must be <str>'
637 return 'ERROR:: "type_option" must be <str>'
529
638
530 def f_status_note(self, total, result, path):
639 def f_status_note(self, total, result, path):
531 file_txt = open(path+'status_note.txt', 'w')
640 file_txt = open(path+'status_note.txt', 'w')
532 file_txt = open(path+'status_note.txt', 'a')
641 file_txt = open(path+'status_note.txt', 'a')
533
642
534 file_txt.write('DOWNLOADED FILE(S): "%s"' % (len(result['name'])))
643 file_txt.write('DOWNLOADED FILE(S): "%s"' % (len(result['name'])))
535 file_txt.write(''+ os.linesep)
644 file_txt.write(''+ os.linesep)
536 for u in result['name']:
645 for u in result['name']:
537 file_txt.write(' - '+ u + os.linesep)
646 file_txt.write(' - '+ u + os.linesep)
538 file_txt.write(''+ os.linesep)
647 file_txt.write(''+ os.linesep)
539
648
540 file_txt.write('FAILED FILE(S): "%s"' % (len(total['name'])-len(result['name'])))
649 file_txt.write('FAILED FILE(S): "%s"' % (len(total['name'])-len(result['name'])))
541 file_txt.write(''+ os.linesep)
650 file_txt.write(''+ os.linesep)
542 if len(total['name'])-len(result['name']) != 0:
651 if len(total['name'])-len(result['name']) != 0:
543 for u in total['name']:
652 for u in total['name']:
544 if not u in result['name']:
653 if not u in result['name']:
545 file_txt.write(' - '+ u + os.linesep)
654 file_txt.write(' - '+ u + os.linesep)
546 else:
655 else:
547 file_txt.write(' "None"'+ os.linesep)
656 file_txt.write(' "None"'+ os.linesep)
548
657
549 def f_name(self, name_dataset, ext, tempdir):
658 def f_name(self, name_dataset, ext, tempdir):
550 while self.check:
659 while self.check:
551 self.str = ''
660 self.str = ''
552 if self.cont == 0:
661 if self.cont == 0:
553 if os.path.exists(tempdir + name_dataset + ext):
662 if os.path.exists(tempdir + name_dataset + ext):
554 self.str = name_dataset+'('+str(self.cont+1)+')'+ext
663 self.str = name_dataset+'('+str(self.cont+1)+')'+ext
555 else:
664 else:
556 self.check = self.check * 0
665 self.check = self.check * 0
557 self.str = name_dataset + ext
666 self.str = name_dataset + ext
558 else:
667 else:
559 if not os.path.exists(tempdir + name_dataset+'('+str(self.cont)+')'+ext):
668 if not os.path.exists(tempdir + name_dataset+'('+str(self.cont)+')'+ext):
560 self.check = self.check * 0
669 self.check = self.check * 0
561 self.str = name_dataset+'('+str(self.cont)+')'+ ext
670 self.str = name_dataset+'('+str(self.cont)+')'+ ext
562 self.cont = self.cont+1
671 self.cont = self.cont+1
563 return self.str
672 return self.str
564
673
565 def f_zipdir(self, path, ziph, zip_name):
674 def f_zipdir(self, path, ziph, zip_name):
566 for root, _, files in os.walk(path):
675 for root, _, files in os.walk(path):
567 print('.....')
676 print('.....')
568 print('Creating: "{}" >>'.format(zip_name))
677 print('Creating: "{}" >>'.format(zip_name))
569 for __file in tqdm(iterable=files, total=len(files)):
678 for __file in tqdm(iterable=files, total=len(files)):
570 new_dir = os.path.relpath(os.path.join(root, __file), os.path.join(path, '..'))
679 new_dir = os.path.relpath(os.path.join(root, __file), os.path.join(path, '..'))
571 ziph.write(os.path.join(root, __file), new_dir)
680 ziph.write(os.path.join(root, __file), new_dir)
572 print('Created >>')
681 print('Created >>')
573
682
574 def download_by_step(self, response, tempdir_name):
683 def download_by_step(self, response, tempdir_name):
575 try:
684 try:
576 with requests.get(response['url'], stream=True, headers={'Authorization': self.Authorization}) as resp:
685 with requests.get(response['url'], stream=True, headers={'Authorization': self.Authorization}) as resp:
577 if resp.status_code == 200:
686 if resp.status_code == 200:
578 with open(tempdir_name+response['name'], 'wb') as file:
687 with open(tempdir_name+response['name'], 'wb') as file:
579 for chunk in resp.iter_content(chunk_size = self.chunk_size):
688 for chunk in resp.iter_content(chunk_size = self.chunk_size):
580 if chunk:
689 if chunk:
581 file.write(chunk)
690 file.write(chunk)
582 except requests.exceptions.RequestException:
691 except requests.exceptions.RequestException:
583 pass
692 pass
584
693
585 def download_files(self, **kwargs):
694 def download_files(self, **kwargs):
586 '''
695 '''
587 FINALIDAD:
696 FINALIDAD:
588 Funcion personalizada para la descarga de archivos existentes de un dataset.
697 Funcion personalizada para la descarga de archivos existentes de un dataset.
589
698
590 PARAMETROS DISPONIBLES:
699 PARAMETROS DISPONIBLES:
591 CONSULTAR: "GUIA DE SCRIPT.pdf"
700 CONSULTAR: "GUIA DE SCRIPT.pdf"
592
701
593 ESTRUCTURA:
702 ESTRUCTURA:
594 <access_name>.download_files(id = <class 'str'>, param_1 = <class 'param_1'>, ...)
703 <access_name>.download_files(id = <class 'str'>, param_1 = <class 'param_1'>, ...)
595 '''
704 '''
596 dict_local = {}
705 dict_local = {}
597 #----------------------------------------------#
706 #----------------------------------------------#
598 if 'zip' in kwargs:
707 if 'zip' in kwargs:
599 if type(kwargs['zip']) is not bool:
708 if type(kwargs['zip']) is not bool:
600 return 'ERROR:: "zip" must be: <class "bool">'
709 return 'ERROR:: "zip" must be: <class "bool">'
601 else:
710 else:
602 dict_local['zip'] = kwargs['zip']
711 dict_local['zip'] = kwargs['zip']
603 else:
712 else:
604 dict_local['zip'] = False
713 dict_local['zip'] = False
605 #----------------------------------------------#
714 #----------------------------------------------#
606 if 'status_note' in kwargs:
715 if 'status_note' in kwargs:
607 if type(kwargs['status_note']) is not bool:
716 if type(kwargs['status_note']) is not bool:
608 return 'ERROR:: "status_note" must be: <class "bool">'
717 return 'ERROR:: "status_note" must be: <class "bool">'
609 else:
718 else:
610 dict_local['status_note'] = kwargs['status_note']
719 dict_local['status_note'] = kwargs['status_note']
611 else:
720 else:
612 dict_local['status_note'] = False
721 dict_local['status_note'] = False
613 #----------------------------------------------#
722 #----------------------------------------------#
614 if 'path' in kwargs:
723 if 'path' in kwargs:
615 if type(kwargs['path']) is str:
724 if type(kwargs['path']) is str:
616 if os.path.isdir(kwargs['path']) == False:
725 if os.path.isdir(kwargs['path']) == False:
617 return 'ERROR:: "path" does not exist'
726 return 'ERROR:: "path" does not exist'
618 else:
727 else:
619 if kwargs['path'][-1:] != self.separator:
728 if kwargs['path'][-1:] != self.separator:
620 dict_local['path'] = kwargs['path']+self.separator
729 dict_local['path'] = kwargs['path']+self.separator
621 else:
730 else:
622 dict_local['path'] = kwargs['path']
731 dict_local['path'] = kwargs['path']
623
732
624 txt = dict_local['path']+datetime.now().strftime("%Y_%m_%d_%H_%M_%S_%f")+'.txt'
733 txt = dict_local['path']+datetime.now().strftime("%Y_%m_%d_%H_%M_%S_%f")+'.txt'
625 if int(platform.python_version()[0]) == 3:
734 if int(platform.python_version()[0]) == 3:
626 try:
735 try:
627 file_txt = open(txt, 'w')
736 file_txt = open(txt, 'w')
628 file_txt.close()
737 file_txt.close()
629 os.remove(txt)
738 os.remove(txt)
630 except PermissionError:
739 except PermissionError:
631 return 'ERROR:: Access denied, you are not authorized to write files: "%s"' % (dict_local['path'])
740 return 'ERROR:: Access denied, you are not authorized to write files: "%s"' % (dict_local['path'])
632 else:
741 else:
633 try:
742 try:
634 file_txt = open(txt, 'w')
743 file_txt = open(txt, 'w')
635 file_txt.close()
744 file_txt.close()
636 os.remove(txt)
745 os.remove(txt)
637 except:
746 except:
638 return 'ERROR:: Access denied, you are not authorized to write files: "%s"' % (dict_local['path'])
747 return 'ERROR:: Access denied, you are not authorized to write files: "%s"' % (dict_local['path'])
639 else:
748 else:
640 return 'ERROR:: "path" must be: <class "str">'
749 return 'ERROR:: "path" must be: <class "str">'
641 else:
750 else:
642 dict_local['path'] = ''
751 dict_local['path'] = ''
643 #----------------------------------------------#
752 #----------------------------------------------#
644 for key, value in kwargs.items():
753 for key, value in kwargs.items():
645 if not key in dict_local:
754 if not key in dict_local:
646 self.dict[key] = value
755 self.dict[key] = value
647 try:
756 try:
648 response = getattr(self.ckan.action, 'url_resources')(**self.dict)
757 response = getattr(self.ckan.action, 'url_resources')(**self.dict)
649 except:
758 except:
650 _, exc_value, _ = sys.exc_info()
759 _, exc_value, _ = sys.exc_info()
651 return exc_value
760 return exc_value
652
761
653 if len(response) != 0:
762 if len(response) != 0:
654 #--------------TEMP PATH---------------#
763 #--------------TEMP PATH---------------#
655 if dict_local['zip']:
764 if dict_local['zip']:
656 tempdir = tempfile.mkdtemp(prefix=kwargs['id']+'-')+self.separator
765 tempdir = tempfile.mkdtemp(prefix=kwargs['id']+'-')+self.separator
657 os.mkdir(tempdir+kwargs['id'])
766 os.mkdir(tempdir+kwargs['id'])
658 dir_name = tempdir + kwargs['id'] + self.separator
767 dir_name = tempdir + kwargs['id'] + self.separator
659 else:
768 else:
660 dir = self.f_name(kwargs['id'], '', dict_local['path'])
769 dir = self.f_name(kwargs['id'], '', dict_local['path'])
661 os.mkdir(dict_local['path'] + dir)
770 os.mkdir(dict_local['path'] + dir)
662 dir_name = dict_local['path'] + dir + self.separator
771 dir_name = dict_local['path'] + dir + self.separator
663 #-----------DOWNLOAD FILES-------------#
772 #-----------DOWNLOAD FILES-------------#
664 print('.....')
773 print('.....')
665 print('Downloading "{}" file(s) >>'.format(len(response)))
774 print('Downloading "{}" file(s) >>'.format(len(response)))
666 name_total = {'name': []}
775 name_total = {'name': []}
667 with concurrent.futures.ThreadPoolExecutor() as executor:
776 with concurrent.futures.ThreadPoolExecutor() as executor:
668 for u in tqdm(iterable=response, total=len(response)):
777 for u in tqdm(iterable=response, total=len(response)):
669 name_total['name'].append(u['name'])
778 name_total['name'].append(u['name'])
670 executor.submit(self.download_by_step, u, dir_name)
779 executor.submit(self.download_by_step, u, dir_name)
671 name_check = {}
780 name_check = {}
672 name_check['name'] = [f for f in os.listdir(dir_name) if os.path.isfile(os.path.join(dir_name, f))]
781 name_check['name'] = [f for f in os.listdir(dir_name) if os.path.isfile(os.path.join(dir_name, f))]
673 print('"{}" downloaded file(s) successfully >>'.format(len(name_check['name'])))
782 print('"{}" downloaded file(s) successfully >>'.format(len(name_check['name'])))
674 #--------------------------------------#
783 #--------------------------------------#
675 if len(name_check['name']) != 0:
784 if len(name_check['name']) != 0:
676 #----------Status Note---------#
785 #----------Status Note---------#
677 if dict_local['status_note']:
786 if dict_local['status_note']:
678 print('.....')
787 print('.....')
679 print('Creating: "status_note.txt" >>')
788 print('Creating: "status_note.txt" >>')
680 self.f_status_note(name_total, name_check, dir_name)
789 self.f_status_note(name_total, name_check, dir_name)
681 print('Created>>')
790 print('Created>>')
682 #----------ZIP CREATE----------#
791 #----------ZIP CREATE----------#
683 if dict_local['zip']:
792 if dict_local['zip']:
684 zip_name = self.f_name(kwargs['id'], '.zip', dict_local['path'])
793 zip_name = self.f_name(kwargs['id'], '.zip', dict_local['path'])
685 ziph = zipfile.ZipFile(dict_local['path'] + zip_name, 'w', zipfile.ZIP_DEFLATED, allowZip64=True)
794 ziph = zipfile.ZipFile(dict_local['path'] + zip_name, 'w', zipfile.ZIP_DEFLATED, allowZip64=True)
686 self.f_zipdir(dir_name, ziph, zip_name)
795 self.f_zipdir(dir_name, ziph, zip_name)
687 ziph.close()
796 ziph.close()
688 #Delete Temporal Path
797 #Delete Temporal Path
689 if os.path.exists(tempdir[:-1]):
798 if os.path.exists(tempdir[:-1]):
690 shutil.rmtree(tempdir[:-1])
799 shutil.rmtree(tempdir[:-1])
691 #------------------------------#
800 #------------------------------#
692 print('.....')
801 print('.....')
693 return 'DOWNLOAD FINISHED'
802 return 'DOWNLOAD FINISHED'
694 else:
803 else:
695 #Delete Temporal Path
804 #Delete Temporal Path
696 if dict_local['zip']:
805 if dict_local['zip']:
697 if os.path.exists(tempdir[:-1]):
806 if os.path.exists(tempdir[:-1]):
698 shutil.rmtree(tempdir[:-1])
807 shutil.rmtree(tempdir[:-1])
699 else:
808 else:
700 if os.path.exists(dir_name[:-1]):
809 if os.path.exists(dir_name[:-1]):
701 shutil.rmtree(dir_name[:-1])
810 shutil.rmtree(dir_name[:-1])
702 return 'NO FILES WERE DOWNLOADED'
811 return 'NO FILES WERE DOWNLOADED'
703 else:
812 else:
704 return 'FILES NOT FOUND' No newline at end of file
813 return 'FILES NOT FOUND'
General Comments 0
You need to be logged in to leave comments. Login now