##// END OF EJS Templates
DATABASES » ckanext-jro » api-cliente
RSS
v2.9.2 :: Add 'resource_views' create function
eynilupu -
r3:ab4b435c96a4
parent child
Show More
Show file before | Show file after | Hide comments
1 NO CONTENT: modified file, binary diff hidden
NO CONTENT: modified file, binary diff hidden
Show file before | Show file after | Hide comments
@@ -1,813 +1,825
1 from ckanapi import RemoteCKAN
1 from ckanapi import RemoteCKAN
2 #from ckanapi.errors import NotAuthorized, NotFound, ValidationError, SearchQueryError, SearchError, CKANAPIError, ServerIncompatibleError
2 #from ckanapi.errors import NotAuthorized, NotFound, ValidationError, SearchQueryError, SearchError, CKANAPIError, ServerIncompatibleError
3 import sys
3 import sys
4 import platform
4 import platform
5 import os
5 import os
6 import tempfile
6 import tempfile
7 import shutil
7 import shutil
8 import zipfile
8 import zipfile
9 import concurrent.futures
9 import concurrent.futures
10 import requests
10 import requests
11 import json
11 import json
12 import pathlib
12 import pathlib
13 import uuid
13 import uuid
14 from datetime import datetime
14 from datetime import datetime
15 from tqdm import tqdm
15 from tqdm import tqdm
16
16
17 class JROAPI():
17 class JROAPI():
18 """
18 """
19 FINALIDAD:
19 FINALIDAD:
20 Script para administrar y obtener la data del repositorio por medio de APIs.
20 Script para administrar y obtener la data del repositorio por medio de APIs.
21
21
22 REQUISITIOS PREVIOS:
22 REQUISITIOS PREVIOS:
23 - Paso 1: Tener "pip [Python 2]" o "pip3 [Python 3]" instalado:
23 - Paso 1: Tener "pip [Python 2]" o "pip3 [Python 3]" instalado:
24 - Paso 2: Instalar lo siguiente como admininstrador:
24 - Paso 2: Instalar lo siguiente como admininstrador:
25 En Python 2
25 En Python 2
26 - pip install ckanapi==4.5
26 - pip install ckanapi==4.5
27 - pip install requests
27 - pip install requests
28 - pip install tqdm
28 - pip install tqdm
29 En Python 3
29 En Python 3
30 - pip3 install ckanapi==4.5
30 - pip3 install ckanapi==4.5
31 - pip3 install requests
31 - pip3 install requests
32 - pip3 install tqdm
32 - pip3 install tqdm
33
33
34 FUNCIONES DISPONIBLES:
34 FUNCIONES DISPONIBLES:
35 - action
35 - action
36 - upload_file
36 - upload_file
37 - upload_multiple_files
37 - upload_multiple_files
38 - upload_multiple_files_advance
38 - show
39 - show
39 - search
40 - search
40 - create
41 - create
41 - patch
42 - patch
42 - delete
43 - delete
43 - download_files
44 - download_files
44
45
45 EJEMPLOS:
46 EJEMPLOS:
46 #1:
47 #1:
47 with JROAPI('http://demo.example.com', Authorization='#########') as <access_name>:
48 with JROAPI('http://demo.example.com', Authorization='#########') as <access_name>:
48 ... some operation(s) ...
49 ... some operation(s) ...
49 #2:
50 #2:
50 <access_name> = JROAPI('http://example.com', Authorization='#########')
51 <access_name> = JROAPI('http://example.com', Authorization='#########')
51 ... some operation(s) ...
52 ... some operation(s) ...
52 <access_name>.ckan.close()
53 <access_name>.ckan.close()
53
54
54 REPORTAR ALGUN PROBLEMA:
55 REPORTAR ALGUN PROBLEMA:
55 Debe enviar un correo a eynilupu@igp.gob.pe detallando los siguientes pasos:
56 Debe enviar un correo a eynilupu@igp.gob.pe detallando los siguientes pasos:
56 1) Identifiquese
57 1) Identifiquese
57 2) Describir el problema
58 2) Describir el problema
58 3) ¿En que funcion esta el problema?
59 3) ¿En que funcion esta el problema?
59 4) ¿Que esperaba que hiciera la funcion sin el problema?
60 4) ¿Que esperaba que hiciera la funcion sin el problema?
60 """
61 """
61 def __init__(self, url, Authorization=None):
62 def __init__(self, url, Authorization=None):
62 ua = 'CKAN_JRO/1.1 (+'+str(url)+')'
63 ua = 'CKAN_JRO/1.1 (+'+str(url)+')'
63 #ua = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36'
64 #ua = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36'
64 self.ckan = RemoteCKAN(url, apikey=Authorization, user_agent=ua)
65 self.ckan = RemoteCKAN(url, apikey=Authorization, user_agent=ua)
65 #self.ckan = RemoteCKAN(url, apikey=Authorization)
66 #self.ckan = RemoteCKAN(url, apikey=Authorization)
66 self.Authorization = Authorization
67 self.Authorization = Authorization
67 if platform.system() == 'Windows':
68 if platform.system() == 'Windows':
68 self.separator = '\\'
69 self.separator = '\\'
69 else:
70 else:
70 self.separator = '/'
71 self.separator = '/'
71
72
72 self.chunk_size = 1024
73 self.chunk_size = 1024
73 self.list = []
74 self.list = []
74 self.dict = {}
75 self.dict = {}
75 self.str = ''
76 self.str = ''
76 self.check = 1
77 self.check = 1
77 self.cont = 0
78 self.cont = 0
78
79
79 def __enter__(self):
80 def __enter__(self):
80 return self
81 return self
81
82
82 def __exit__(self, *args):
83 def __exit__(self, *args):
83 self.ckan.close()
84 self.ckan.close()
84
85
85 def action(self, action, **kwargs):
86 def action(self, action, **kwargs):
86 """
87 """
87 FINALIDAD:
88 FINALIDAD:
88 Funcion para llamar a las APIs disponibles
89 Funcion para llamar a las APIs disponibles
89
90
90 APIs DISPONIBLES:
91 APIs DISPONIBLES:
91 CONSULTAR: "GUIA DE SCRIPT.pdf"
92 CONSULTAR: "GUIA DE SCRIPT.pdf"
92
93
93 EJEMPLO:
94 EJEMPLO:
94 <access_name>.action(<consuming API>, param_1 = <class 'param_1'>, ...)
95 <access_name>.action(<consuming API>, param_1 = <class 'param_1'>, ...)
95 """
96 """
96 #--------------- CASE: PACKAGE SEARCH ---------------#
97 #--------------- CASE: PACKAGE SEARCH ---------------#
97 if kwargs is not None:
98 if kwargs is not None:
98 if action == 'package_search':
99 if action == 'package_search':
99 self.list = ['facet_mincount', 'facet_limit', 'facet_field']
100 self.list = ['facet_mincount', 'facet_limit', 'facet_field']
100 for facet in self.list:
101 for facet in self.list:
101 if facet in kwargs:
102 if facet in kwargs:
102 kwargs[facet.replace('_', '.')] = kwargs[facet]
103 kwargs[facet.replace('_', '.')] = kwargs[facet]
103 kwargs.pop(facet)
104 kwargs.pop(facet)
104 #----------------------------------------------------#
105 #----------------------------------------------------#
105 try:
106 try:
106 return getattr(self.ckan.action, action)(**kwargs)
107 return getattr(self.ckan.action, action)(**kwargs)
107 except:
108 except:
108 _, exc_value, _ = sys.exc_info()
109 _, exc_value, _ = sys.exc_info()
109 return exc_value
110 return exc_value
110
111
111 def upload_file(self, dataset_id, file_path, file_date, file_type, **kwargs):
112 def upload_file(self, dataset_id, file_path, file_date, file_type, **kwargs):
112 '''
113 '''
113 FINALIDAD:
114 FINALIDAD:
114 Funcion para subir un unico archivo al repositorio del ROJ.
115 Funcion para subir un unico archivo al repositorio del ROJ.
115
116
116 PARAMETROS DISPONIBLES:
117 PARAMETROS DISPONIBLES:
117 CONSULTAR: "GUIA DE SCRIPT.pdf"
118 CONSULTAR: "GUIA DE SCRIPT.pdf"
118
119
119 ESTRUCTURA:
120 ESTRUCTURA:
120 <access_name>.upload_file(dataset_id = <class 'str'>, file_date = <class 'str'>, file_path = <class 'str'>, file_type = <class 'str'>, param_1 = <class 'param_1'>, ...)
121 <access_name>.upload_file(dataset_id = <class 'str'>, file_date = <class 'str'>, file_path = <class 'str'>, file_type = <class 'str'>, param_1 = <class 'param_1'>, ...)
121 '''
122 '''
122 self.list = ['package_id', 'upload', 'voc_file_type', 'name'] #file_date
123 self.list = ['package_id', 'upload', 'voc_file_type', 'name'] #file_date
123 for key1, value1 in kwargs.items():
124 for key1, value1 in kwargs.items():
124 if not key1 in self.list:
125 if not key1 in self.list:
125 self.dict[key1] = value1
126 self.dict[key1] = value1
126
127
127 #---------------------------#
128 #---------------------------#
128 if not 'others' in kwargs:
129 if not 'others' in kwargs:
129 self.dict['others'] = ''
130 self.dict['others'] = ''
130 else:
131 else:
131 if isinstance(kwargs['others'], list):
132 if isinstance(kwargs['others'], list):
132 self.dict['others'] = json.dumps(kwargs['others'])
133 self.dict['others'] = json.dumps(kwargs['others'])
133 #---------------------------#
134 #---------------------------#
134
135
135 if not os.path.isfile(file_path):
136 if not os.path.isfile(file_path):
136 return 'File "%s" not exist' % (file_path)
137 return 'File "%s" not exist' % (file_path)
137
138
138 if not 'format' in self.dict:
139 if not 'format' in self.dict:
139 self.str = ''.join(pathlib.Path(file_path).suffixes)
140 self.str = ''.join(pathlib.Path(file_path).suffixes)
140 if len(self.str) > 0:
141 if len(self.str) > 0:
141 self.dict['format'] = self.str.upper()[1:]
142 self.dict['format'] = self.str.upper()[1:]
142
143
143 try:
144 try:
144 return getattr(self.ckan.action, 'resource_create')(package_id=dataset_id, file_date=file_date, upload=open(file_path, 'rb'), voc_file_type=file_type, name=pathlib.Path(file_path).name, **self.dict)
145 return getattr(self.ckan.action, 'resource_create')(package_id=dataset_id, file_date=file_date, upload=open(file_path, 'rb'), voc_file_type=file_type, name=pathlib.Path(file_path).name, **self.dict)
145 except:
146 except:
146 _, exc_value, _ = sys.exc_info()
147 _, exc_value, _ = sys.exc_info()
147 return exc_value
148 return exc_value
148
149
149
150
150 def upload_multiple_files_advance(self, dataset_id, path_files, file_date, file_type, **kwargs):
151 def upload_multiple_files_advance(self, dataset_id, path_files, file_date, file_type, **kwargs):
151 '''
152 '''
152 FINALIDAD:
153 FINALIDAD:
153 Funcion para subir multiples archivos al repositorio del ROJ.
154 Funcion para subir multiples archivos al repositorio del ROJ.
154
155
155 PARAMETROS DISPONIBLES:
156 PARAMETROS DISPONIBLES:
156 CONSULTAR: "GUIA DE SCRIPT.pdf"
157 CONSULTAR: "GUIA DE SCRIPT.pdf"
157
158
158 ESTRUCTURA:
159 ESTRUCTURA:
159 <access_name>.upload_multiple_files_advance(dataset_id = <class 'str'>, path_files = <class 'list of strings'>, file_date = <class 'str'>, file_type = <class 'str'>, param_1 = <class 'param_1'>, ...)
160 <access_name>.upload_multiple_files_advance(dataset_id = <class 'str'>, path_files = <class 'list of strings'>, file_date = <class 'str'>, file_type = <class 'str'>, param_1 = <class 'param_1'>, ...)
160 '''
161 '''
161 self.list = ['package_id', 'upload', 'voc_file_type', 'name']
162 self.list = ['package_id', 'upload', 'voc_file_type', 'name']
162
163
163 for key1, value1 in kwargs.items():
164 for key1, value1 in kwargs.items():
164 if not key1 in self.list:
165 if not key1 in self.list:
165 self.dict[key1] = value1
166 self.dict[key1] = value1
166
167
167 #---------------------------#
168 #---------------------------#
168 if not 'others' in kwargs:
169 if not 'others' in kwargs:
169 self.dict['others'] = ''
170 self.dict['others'] = ''
170 else:
171 else:
171 if isinstance(kwargs['others'], list):
172 if isinstance(kwargs['others'], list):
172 self.dict['others'] = json.dumps(kwargs['others'])
173 self.dict['others'] = json.dumps(kwargs['others'])
173 #---------------------------#
174 #---------------------------#
174
175
175 params_dict = {'upload':[], 'name':[]}
176 params_dict = {'upload':[], 'name':[]}
176 #---------------CASO : "path" or "path_list"-----------------#
177 #---------------CASO : "path" or "path_list"-----------------#
177 if type(path_files) is list:
178 if type(path_files) is list:
178 if len(path_files) != 0:
179 if len(path_files) != 0:
179 path_files.sort()
180 path_files.sort()
180 for u in path_files:
181 for u in path_files:
181 if os.path.isfile(u):
182 if os.path.isfile(u):
182 params_dict['upload'].append(open(u, 'rb'))
183 params_dict['upload'].append(open(u, 'rb'))
183 params_dict['name'].append(pathlib.Path(u).name)
184 params_dict['name'].append(pathlib.Path(u).name)
184 else:
185 else:
185 return 'File "%s" does not exist' % (u)
186 return 'File "%s" does not exist' % (u)
186 else:
187 else:
187 return 'ERROR:: "path_list is empty"'
188 return 'ERROR:: "path_list is empty"'
188 elif type(path_files) is str:
189 elif type(path_files) is str:
189 if os.path.isdir(path_files):
190 if os.path.isdir(path_files):
190 path_order = [f for f in os.listdir(path_files) if os.path.isfile(os.path.join(path_files, f))]
191 path_order = [f for f in os.listdir(path_files) if os.path.isfile(os.path.join(path_files, f))]
191 path_order.sort()
192 path_order.sort()
192 if path_order:
193 if path_order:
193 for name in path_order:
194 for name in path_order:
194 params_dict['upload'].append(open(os.path.join(path_files, name), 'rb'))
195 params_dict['upload'].append(open(os.path.join(path_files, name), 'rb'))
195 params_dict['name'].append(name)
196 params_dict['name'].append(name)
196 else:
197 else:
197 return "ERROR:: There aren't files in this directory"
198 return "ERROR:: There aren't files in this directory"
198 else:
199 else:
199 return 'ERROR:: Directory "%s" does not exist' % (path_files)
200 return 'ERROR:: Directory "%s" does not exist' % (path_files)
200 else:
201 else:
201 return 'ERROR:: "path_files" must be a str or list'
202 return 'ERROR:: "path_files" must be a str or list'
202 #------------------------------------------------------------#
203 #------------------------------------------------------------#
203 resource_extend = []
204 resource_extend = []
204 files_dict = {}
205 files_dict = {}
205 for count, name in enumerate(params_dict['name']):
206 for count, name in enumerate(params_dict['name']):
206 param_list = {"name": name, "file_date": file_date, "voc_file_type": file_type}
207 param_list = {"name": name, "file_date": file_date, "voc_file_type": file_type}
207 param_list.update(self.dict)
208 param_list.update(self.dict)
208
209
209 if not 'format' in param_list:
210 if not 'format' in param_list:
210 format = ''.join(pathlib.Path(name).suffixes)
211 format = ''.join(pathlib.Path(name).suffixes)
211 if len(format) > 0:
212 if len(format) > 0:
212 param_list['format'] = format.upper()[1:]
213 param_list['format'] = format.upper()[1:]
213
214
214 resource_extend.append(param_list)
215 resource_extend.append(param_list)
215 files_dict['update__resources__-'+ str(len(params_dict['name'])-count) +'__upload'] = (name, params_dict['upload'][count])
216 files_dict['update__resources__-'+ str(len(params_dict['name'])-count) +'__upload'] = (name, params_dict['upload'][count])
216
217
217 #------------------------------------------------------------#
218 #------------------------------------------------------------#
218 try:
219 try:
219 uuid.UUID(str(dataset_id), version=4)
220 uuid.UUID(str(dataset_id), version=4)
220 package_id_or_name = '"id": "' + str(dataset_id) + '"'
221 package_id_or_name = '"id": "' + str(dataset_id) + '"'
221 except ValueError:
222 except ValueError:
222 package_id_or_name = '"name": "' + str(dataset_id) + '"'
223 package_id_or_name = '"name": "' + str(dataset_id) + '"'
223 #------------------------------------------------------------#
224 #------------------------------------------------------------#
224 print('"{}" file(s) found >> uploading'.format(len(params_dict['name'])))
225 print('"{}" file(s) found >> uploading'.format(len(params_dict['name'])))
225 try:
226 try:
226 result = self.ckan.call_action(
227 result = self.ckan.call_action(
227 'package_revise',
228 'package_revise',
228 {'match': '{'+ str(package_id_or_name) +'}', 'update__resources__extend': json.dumps(resource_extend)},
229 {'match': '{'+ str(package_id_or_name) +'}', 'update__resources__extend': json.dumps(resource_extend)},
229 files=files_dict
230 files=files_dict
230 )
231 )
231 print('Uploaded file(s) successfully')
232 print('Uploaded file(s) successfully')
232 return result
233 return result
233 except:
234 except:
234 print('ERROR :: Use the "print" option for more information')
235 print('ERROR :: Use the "print" option for more information')
235 _, exc_value, _ = sys.exc_info()
236 _, exc_value, _ = sys.exc_info()
236 return exc_value
237 return exc_value
237
238
238 def upload_multiple_files(self, dataset_id, path_files, date_files, type_files, **kwargs):
239 def upload_multiple_files(self, dataset_id, path_files, date_files, type_files, **kwargs):
239 '''
240 '''
240 FINALIDAD:
241 FINALIDAD:
241 Funcion para subir multiples archivos al repositorio del ROJ.
242 Funcion para subir multiples archivos al repositorio del ROJ.
242
243
243 PARAMETROS DISPONIBLES:
244 PARAMETROS DISPONIBLES:
244 CONSULTAR: "GUIA DE SCRIPT.pdf"
245 CONSULTAR: "GUIA DE SCRIPT.pdf"
245
246
246 ESTRUCTURA:
247 ESTRUCTURA:
247 <access_name>.upload_multiple_files(dataset_id = <class 'str'>, path_files = <class 'str'> or <class 'list of strings'>, date_files = <class 'str'> or <class 'list of strings'>, type_files = <class 'str'> or <class 'list of strings'>, param_1 = <class 'param_1'>, ...)
248 <access_name>.upload_multiple_files(dataset_id = <class 'str'>, path_files = <class 'str'> or <class 'list of strings'>, date_files = <class 'str'> or <class 'list of strings'>, type_files = <class 'str'> or <class 'list of strings'>, param_1 = <class 'param_1'>, ...)
248 '''
249 '''
249
250
250 params_dict = {'upload':[], 'name':[]}
251 params_dict = {'upload':[], 'name':[]}
251 if not 'format' in kwargs:
252 if not 'format' in kwargs:
252 params_dict.update({'format':[]})
253 params_dict.update({'format':[]})
253 #---------------CASO : "path" or "path_list"-----------------#
254 #---------------CASO : "path" or "path_list"-----------------#
254 if type(path_files) is list:
255 if type(path_files) is list:
255 if len(path_files) != 0:
256 if len(path_files) != 0:
256 path_files.sort()
257 path_files.sort()
257 for u in path_files:
258 for u in path_files:
258 if os.path.isfile(u):
259 if os.path.isfile(u):
259 params_dict['upload'].append(open(u, 'rb'))
260 params_dict['upload'].append(open(u, 'rb'))
260 params_dict['name'].append(pathlib.Path(u).name)
261 params_dict['name'].append(pathlib.Path(u).name)
261 if not 'format' in kwargs:
262 if not 'format' in kwargs:
262 format = ''.join(pathlib.Path(u).suffixes)
263 format = ''.join(pathlib.Path(u).suffixes)
263 if len(format) > 0:
264 if len(format) > 0:
264 params_dict['format'].append(format.upper()[1:])
265 params_dict['format'].append(format.upper()[1:])
265 else:
266 else:
266 params_dict['format'].append('')
267 params_dict['format'].append('')
267 else:
268 else:
268 return 'File "%s" does not exist' % (u)
269 return 'File "%s" does not exist' % (u)
269 else:
270 else:
270 return 'ERROR:: "path_list is empty"'
271 return 'ERROR:: "path_list is empty"'
271 elif type(path_files) is str:
272 elif type(path_files) is str:
272 if os.path.isdir(path_files):
273 if os.path.isdir(path_files):
273 path_order = [f for f in os.listdir(path_files) if os.path.isfile(os.path.join(path_files, f))]
274 path_order = [f for f in os.listdir(path_files) if os.path.isfile(os.path.join(path_files, f))]
274 path_order.sort()
275 path_order.sort()
275 if path_order:
276 if path_order:
276 for name in path_order:
277 for name in path_order:
277 params_dict['upload'].append(open(os.path.join(path_files, name), 'rb'))
278 params_dict['upload'].append(open(os.path.join(path_files, name), 'rb'))
278 params_dict['name'].append(name)
279 params_dict['name'].append(name)
279 if not 'format' in kwargs:
280 if not 'format' in kwargs:
280 format = ''.join(pathlib.Path(name).suffixes)
281 format = ''.join(pathlib.Path(name).suffixes)
281 if len(format) > 0:
282 if len(format) > 0:
282 params_dict['format'].append(format.upper()[1:])
283 params_dict['format'].append(format.upper()[1:])
283 else:
284 else:
284 params_dict['format'].append('')
285 params_dict['format'].append('')
285 else:
286 else:
286 return "ERROR:: There aren't files in this directory"
287 return "ERROR:: There aren't files in this directory"
287 else:
288 else:
288 return 'ERROR:: Directory "%s" does not exist' % (path_files)
289 return 'ERROR:: Directory "%s" does not exist' % (path_files)
289 else:
290 else:
290 return 'ERROR:: "path_files" must be a str or list'
291 return 'ERROR:: "path_files" must be a str or list'
291 #------------------------------------------------------------#
292 #------------------------------------------------------------#
292 params_no_dict = {'package_id': dataset_id}
293 params_no_dict = {'package_id': dataset_id}
293 if type(date_files) is list:
294 if type(date_files) is list:
294 params_dict['file_date'] = date_files
295 params_dict['file_date'] = date_files
295 else:
296 else:
296 params_no_dict['file_date'] = date_files
297 params_no_dict['file_date'] = date_files
297
298
298 if type(type_files) is list:
299 if type(type_files) is list:
299 params_dict['voc_file_type'] = type_files
300 params_dict['voc_file_type'] = type_files
300 else:
301 else:
301 params_no_dict['voc_file_type'] = type_files
302 params_no_dict['voc_file_type'] = type_files
302
303
303 for key1, value1 in kwargs.items():
304 for key1, value1 in kwargs.items():
304 if not key1 in params_dict and not key1 in params_no_dict and key1 != 'others':
305 if not key1 in params_dict and not key1 in params_no_dict and key1 != 'others':
305 if type(value1) is list:
306 if type(value1) is list:
306 params_dict[key1] = value1
307 params_dict[key1] = value1
307 else:
308 else:
308 params_no_dict[key1] = value1
309 params_no_dict[key1] = value1
309 #------------------------------------------#
310 #------------------------------------------#
310 if not 'others' in kwargs:
311 if not 'others' in kwargs:
311 params_no_dict['others'] = ''
312 params_no_dict['others'] = ''
312 else:
313 else:
313 if isinstance(kwargs['others'], tuple):
314 if isinstance(kwargs['others'], tuple):
314 params_dict['others'] = [json.dumps(w) for w in kwargs['others']]
315 params_dict['others'] = [json.dumps(w) for w in kwargs['others']]
315 elif isinstance(kwargs['others'], list):
316 elif isinstance(kwargs['others'], list):
316 params_no_dict['others'] = json.dumps(kwargs['others'])
317 params_no_dict['others'] = json.dumps(kwargs['others'])
317 elif isinstance(kwargs['others'], str):
318 elif isinstance(kwargs['others'], str):
318 params_no_dict['others'] = kwargs['others']
319 params_no_dict['others'] = kwargs['others']
319 else:
320 else:
320 return 'ERROR:: "others" must be a tuple, list or str'
321 return 'ERROR:: "others" must be a tuple, list or str'
321 #------------------------------------------#
322 #------------------------------------------#
322 len_params_dict = []
323 len_params_dict = []
323 for value2 in params_dict.values():
324 for value2 in params_dict.values():
324 len_params_dict.append(len(value2))
325 len_params_dict.append(len(value2))
325
326
326 if len(list(set(len_params_dict))) > 1:
327 if len(list(set(len_params_dict))) > 1:
327 return 'ERROR:: All lists must be the same length: %s' % (len(params_dict['name']))
328 return 'ERROR:: All lists must be the same length: %s' % (len(params_dict['name']))
328 #------------------------------------------------------------#
329 #------------------------------------------------------------#
329 print('"{}" file(s) found >> uploading'.format(len(params_dict['name'])))
330 print('"{}" file(s) found >> uploading'.format(len(params_dict['name'])))
330 for v in range(len(params_dict['name'])):
331 for v in range(len(params_dict['name'])):
331 try:
332 try:
332 send = {}
333 send = {}
333 for key_dict, value_dict in params_dict.items():
334 for key_dict, value_dict in params_dict.items():
334 send[key_dict] = value_dict[v]
335 send[key_dict] = value_dict[v]
335 for key_no_dict, value_no_dict in params_no_dict.items():
336 for key_no_dict, value_no_dict in params_no_dict.items():
336 send[key_no_dict] = value_no_dict
337 send[key_no_dict] = value_no_dict
337
338
338 self.list.append(getattr(self.ckan.action, 'resource_create')(**send))
339 self.list.append(getattr(self.ckan.action, 'resource_create')(**send))
339 print('File #{} :: "{}" was uploaded successfully'.format(v+1, params_dict['name'][v]))
340 print('File #{} :: "{}" was uploaded successfully'.format(v+1, params_dict['name'][v]))
340 except:
341 except:
341 _, exc_value, _ = sys.exc_info()
342 _, exc_value, _ = sys.exc_info()
342 self.list.append(exc_value)
343 self.list.append(exc_value)
343 print('File #{} :: Error uploading "{}" file'.format(v+1, params_dict['name'][v]))
344 print('File #{} :: Error uploading "{}" file'.format(v+1, params_dict['name'][v]))
344 return self.list
345 return self.list
345 #------------------------------------------------------------#
346 #------------------------------------------------------------#
346
347
347 def show(self, type_option, id, **kwargs):
348 def show(self, type_option, id, **kwargs):
348 '''
349 '''
349 FINALIDAD:
350 FINALIDAD:
350 Funcion personalizada para una busqueda en especifico.
351 Funcion personalizada para una busqueda en especifico.
351
352
352 PARAMETROS DISPONIBLES:
353 PARAMETROS DISPONIBLES:
353 CONSULTAR: "GUIA DE SCRIPT.pdf"
354 CONSULTAR: "GUIA DE SCRIPT.pdf"
354
355
355 ESTRUCTURA:
356 ESTRUCTURA:
356 <access_name>.show(type_option = <class 'str'>, id = <class 'str'>, param_1 = <class 'param_1'>, ...)
357 <access_name>.show(type_option = <class 'str'>, id = <class 'str'>, param_1 = <class 'param_1'>, ...)
357 '''
358 '''
358 if type(type_option) is str:
359 if type(type_option) is str:
359 try:
360 try:
360 if type_option == 'dataset':
361 if type_option == 'dataset':
361 return getattr(self.ckan.action, 'package_show')(id=id, **kwargs)
362 return getattr(self.ckan.action, 'package_show')(id=id, **kwargs)
362 elif type_option == 'resource':
363 elif type_option == 'resource':
363 return getattr(self.ckan.action, 'resource_show')(id=id, **kwargs)
364 return getattr(self.ckan.action, 'resource_show')(id=id, **kwargs)
364 elif type_option == 'project':
365 elif type_option == 'project':
365 return getattr(self.ckan.action, 'organization_show')(id=id, **kwargs)
366 return getattr(self.ckan.action, 'organization_show')(id=id, **kwargs)
366 elif type_option == 'collaborator':
367 elif type_option == 'collaborator':
367 return getattr(self.ckan.action, 'package_collaborator_list_for_user')(id=id, **kwargs)
368 return getattr(self.ckan.action, 'package_collaborator_list_for_user')(id=id, **kwargs)
368 elif type_option == 'member':
369 elif type_option == 'member':
369 return getattr(self.ckan.action, 'organization_list_for_user')(id=id, **kwargs)
370 return getattr(self.ckan.action, 'organization_list_for_user')(id=id, **kwargs)
370 elif type_option == 'vocabulary':
371 elif type_option == 'vocabulary':
371 return getattr(self.ckan.action, 'vocabulary_show')(id=id, **kwargs)
372 return getattr(self.ckan.action, 'vocabulary_show')(id=id, **kwargs)
372 elif type_option == 'tag':
373 elif type_option == 'tag':
373 if not 'vocabulary_id' in kwargs:
374 if not 'vocabulary_id' in kwargs:
374 print('Missing "vocabulary_id" value: assume it is a free tag')
375 print('Missing "vocabulary_id" value: assume it is a free tag')
375 return getattr(self.ckan.action, 'tag_show')(id=id, **kwargs)
376 return getattr(self.ckan.action, 'tag_show')(id=id, **kwargs)
376 elif type_option == 'user':
377 elif type_option == 'user':
377 return getattr(self.ckan.action, 'user_show')(id=id, **kwargs)
378 return getattr(self.ckan.action, 'user_show')(id=id, **kwargs)
378 elif type_option == 'job':
379 elif type_option == 'job':
379 return getattr(self.ckan.action, 'job_show')(id=id, **kwargs)
380 return getattr(self.ckan.action, 'job_show')(id=id, **kwargs)
380 else:
381 else:
381 return 'ERROR:: "%s" is not accepted' % (type_option)
382 return 'ERROR:: "type_option = %s" is not accepted' % (type_option)
382 except:
383 except:
383 _, exc_value, _ = sys.exc_info()
384 _, exc_value, _ = sys.exc_info()
384 return exc_value
385 return exc_value
385 else:
386 else:
386 return 'ERROR:: "type_option" must be a str'
387 return 'ERROR:: "type_option" must be a str'
387
388
388 def search(self, type_option, query=None, **kwargs):
389 def search(self, type_option, query=None, **kwargs):
389 '''
390 '''
390 FINALIDAD:
391 FINALIDAD:
391 Funcion personalizada para busquedas que satisfagan algun criterio.
392 Funcion personalizada para busquedas que satisfagan algun criterio.
392
393
393 PARAMETROS DISPONIBLES:
394 PARAMETROS DISPONIBLES:
394 CONSULTAR: "GUIA DE SCRIPT.pdf"
395 CONSULTAR: "GUIA DE SCRIPT.pdf"
395
396
396 ESTRUCTURA:
397 ESTRUCTURA:
397 <access_name>.search(type_option = <class 'str'>, query = <class 'dict'>, param_1 = <class 'param_1'>, ...)
398 <access_name>.search(type_option = <class 'str'>, query = <class 'dict'>, param_1 = <class 'param_1'>, ...)
398 '''
399 '''
399 if type(type_option) is str:
400 if type(type_option) is str:
400 try:
401 try:
401 if type_option == 'dataset':
402 if type_option == 'dataset':
402 key_replace = ['fq', 'fq_list', 'include_private']
403 key_replace = ['fq', 'fq_list', 'include_private']
403 key_point = ['facet_mincount', 'facet_limit', 'facet_field']
404 key_point = ['facet_mincount', 'facet_limit', 'facet_field']
404 for key1, value1 in kwargs.items():
405 for key1, value1 in kwargs.items():
405 if not key1 in key_replace:
406 if not key1 in key_replace:
406 if key1 in key_point:
407 if key1 in key_point:
407 self.dict[key1.replace('_', '.')] = value1
408 self.dict[key1.replace('_', '.')] = value1
408 else:
409 else:
409 self.dict[key1] = value1
410 self.dict[key1] = value1
410
411
411 if query is not None:
412 if query is not None:
412 if type(query) is dict:
413 if type(query) is dict:
413 self.dict['fq_list'] = []
414 self.dict['fq_list'] = []
414 #NUM_RESOURCES_MIN / NUM_RESOURCES_MAX
415 #NUM_RESOURCES_MIN / NUM_RESOURCES_MAX
415 #----------------------------------------------------#
416 #----------------------------------------------------#
416 if 'dataset_start_date' in query:
417 if 'dataset_start_date' in query:
417 if type(query['dataset_start_date']) is str:
418 if type(query['dataset_start_date']) is str:
418 try:
419 try:
419 datetime.strptime(query['dataset_start_date'], '%Y-%m-%d')
420 datetime.strptime(query['dataset_start_date'], '%Y-%m-%d')
420 if len(query['dataset_start_date']) != 10:
421 if len(query['dataset_start_date']) != 10:
421 return '"dataset_start_date", must be: <YYYY-MM-DD>'
422 return '"dataset_start_date", must be: <YYYY-MM-DD>'
422 self.dict['fq_list'].append('dataset_start_date:"'+query['dataset_start_date']+'"')
423 self.dict['fq_list'].append('dataset_start_date:"'+query['dataset_start_date']+'"')
423 self.list.append('dataset_start_date')
424 self.list.append('dataset_start_date')
424 except:
425 except:
425 return '"dataset_start_date" incorrect: "%s"' % (query['dataset_start_date'])
426 return '"dataset_start_date" incorrect: "%s"' % (query['dataset_start_date'])
426 else:
427 else:
427 return '"dataset_start_date" must be <str>'
428 return '"dataset_start_date" must be <str>'
428 #----------------------------------------------------#
429 #----------------------------------------------------#
429 if 'dataset_end_date' in query:
430 if 'dataset_end_date' in query:
430 if type(query['dataset_end_date']) is str:
431 if type(query['dataset_end_date']) is str:
431 try:
432 try:
432 datetime.strptime(query['dataset_end_date'], '%Y-%m-%d')
433 datetime.strptime(query['dataset_end_date'], '%Y-%m-%d')
433 if len(query['dataset_end_date']) != 10:
434 if len(query['dataset_end_date']) != 10:
434 return '"dataset_end_date", must be: <YYYY-MM-DD>'
435 return '"dataset_end_date", must be: <YYYY-MM-DD>'
435
436
436 if 'dataset_start_date' in query:
437 if 'dataset_start_date' in query:
437 if query['dataset_start_date'] > query['dataset_end_date']:
438 if query['dataset_start_date'] > query['dataset_end_date']:
438 return '"dataset_end_date" must be greater than "dataset_start_date"'
439 return '"dataset_end_date" must be greater than "dataset_start_date"'
439
440
440 self.dict['fq_list'].append('dataset_end_date:"'+query['dataset_end_date']+'"')
441 self.dict['fq_list'].append('dataset_end_date:"'+query['dataset_end_date']+'"')
441 self.list.append('dataset_end_date')
442 self.list.append('dataset_end_date')
442 except:
443 except:
443 return '"dataset_end_date" incorrect: "%s"' % (query['dataset_end_date'])
444 return '"dataset_end_date" incorrect: "%s"' % (query['dataset_end_date'])
444 else:
445 else:
445 return '"dataset_end_date" must be <str>'
446 return '"dataset_end_date" must be <str>'
446 #----------------------------------------------------#
447 #----------------------------------------------------#
447 for key, value in query.items():
448 for key, value in query.items():
448 if value is not None and not key in self.list:
449 if value is not None and not key in self.list:
449 self.dict['fq_list'].append(str(key)+':"'+str(value)+'"')
450 self.dict['fq_list'].append(str(key)+':"'+str(value)+'"')
450 else:
451 else:
451 return '"query" must be <dict>'
452 return '"query" must be <dict>'
452
453
453 return getattr(self.ckan.action, 'package_search')(include_private=True, **self.dict)
454 return getattr(self.ckan.action, 'package_search')(include_private=True, **self.dict)
454
455
455 elif type_option == 'resource':
456 elif type_option == 'resource':
456 for key1, value1 in kwargs.items():
457 for key1, value1 in kwargs.items():
457 if key1 != 'fields':
458 if key1 != 'fields':
458 self.dict[key1] = value1
459 self.dict[key1] = value1
459
460
460 if query is not None:
461 if query is not None:
461 if type(query) is dict:
462 if type(query) is dict:
462 #----------------------------------------------------#
463 #----------------------------------------------------#
463 if 'file_date_min' in query:
464 if 'file_date_min' in query:
464 if type(query['file_date_min']) is str:
465 if type(query['file_date_min']) is str:
465 try:
466 try:
466 datetime.strptime(query['file_date_min'], '%Y-%m-%d')
467 datetime.strptime(query['file_date_min'], '%Y-%m-%d')
467 if len(query['file_date_min']) != 10:
468 if len(query['file_date_min']) != 10:
468 return '"file_date_min", must be: <YYYY-MM-DD>'
469 return '"file_date_min", must be: <YYYY-MM-DD>'
469 except:
470 except:
470 return '"file_date_min" incorrect: "%s"' % (query['file_date_min'])
471 return '"file_date_min" incorrect: "%s"' % (query['file_date_min'])
471 else:
472 else:
472 return '"file_date_min" must be <str>'
473 return '"file_date_min" must be <str>'
473 #----------------------------------------------------#
474 #----------------------------------------------------#
474 if 'file_date_max' in query:
475 if 'file_date_max' in query:
475 if type(query['file_date_max']) is str:
476 if type(query['file_date_max']) is str:
476 try:
477 try:
477 datetime.strptime(query['file_date_max'], '%Y-%m-%d')
478 datetime.strptime(query['file_date_max'], '%Y-%m-%d')
478 if len(query['file_date_max']) != 10:
479 if len(query['file_date_max']) != 10:
479 return '"file_date_max", must be: <YYYY-MM-DD>'
480 return '"file_date_max", must be: <YYYY-MM-DD>'
480
481
481 if 'file_date_min' in query:
482 if 'file_date_min' in query:
482 if query['file_date_min'] > query['file_date_max']:
483 if query['file_date_min'] > query['file_date_max']:
483 return '"file_date_max" must be greater than "file_date_min"'
484 return '"file_date_max" must be greater than "file_date_min"'
484 except:
485 except:
485 return '"file_date_max" incorrect: "%s"' % (query['file_date_max'])
486 return '"file_date_max" incorrect: "%s"' % (query['file_date_max'])
486 else:
487 else:
487 return '"file_date_max" must be <str>'
488 return '"file_date_max" must be <str>'
488 #----------------------------------------------------#
489 #----------------------------------------------------#
489 self.dict['query'] = query
490 self.dict['query'] = query
490 else:
491 else:
491 return '"query" must be <dict>'
492 return '"query" must be <dict>'
492 return getattr(self.ckan.action, 'resources_search')(**self.dict)
493 return getattr(self.ckan.action, 'resources_search')(**self.dict)
493
494
494 elif type_option == 'tag':
495 elif type_option == 'tag':
495 for key1, value1 in kwargs.items():
496 for key1, value1 in kwargs.items():
496 if key1 != 'fields':
497 if key1 != 'fields':
497 self.dict[key1] = value1
498 self.dict[key1] = value1
498
499
499 if not 'vocabulary_id' in kwargs:
500 if not 'vocabulary_id' in kwargs:
500 print('Missing "vocabulary_id" value: tags that don’t belong to any vocabulary')
501 print('Missing "vocabulary_id" value: tags that don’t belong to any vocabulary')
501 else:
502 else:
502 print('Only tags that belong to "{}" vocabulary'.format(kwargs['vocabulary_id']))
503 print('Only tags that belong to "{}" vocabulary'.format(kwargs['vocabulary_id']))
503
504
504 if query is not None:
505 if query is not None:
505 if type(query) is dict:
506 if type(query) is dict:
506 if 'search' in query:
507 if 'search' in query:
507 if type(query['search']) is list or type(query['search']) is str:
508 if type(query['search']) is list or type(query['search']) is str:
508 self.dict['query'] = query['search']
509 self.dict['query'] = query['search']
509 else:
510 else:
510 return '"search" must be <list> or <str>'
511 return '"search" must be <list> or <str>'
511 else:
512 else:
512 return '"query" must be <dict>'
513 return '"query" must be <dict>'
513 return getattr(self.ckan.action, 'tag_search')(**self.dict)
514 return getattr(self.ckan.action, 'tag_search')(**self.dict)
514
515
515 else:
516 else:
516 return 'ERROR:: "%s" is not accepted' % (type_option)
517 return 'ERROR:: "type_option = %s" is not accepted' % (type_option)
517
518
518 except:
519 except:
519 _, exc_value, _ = sys.exc_info()
520 _, exc_value, _ = sys.exc_info()
520 return exc_value
521 return exc_value
521 else:
522 else:
522 return 'ERROR:: "type_option" must be <str>'
523 return 'ERROR:: "type_option" must be <str>'
523
524
524 def create(self, type_option, **kwargs):
525 def create(self, type_option, select=None, **kwargs):
525 '''
526 '''
526 FINALIDAD:
527 FINALIDAD:
527 Funcion personalizada para crear.
528 Funcion personalizada para crear.
528
529
529 PARAMETROS DISPONIBLES:
530 PARAMETROS DISPONIBLES:
530 CONSULTAR: "GUIA DE SCRIPT.pdf"
531 CONSULTAR: "GUIA DE SCRIPT.pdf"
531
532
532 ESTRUCTURA:
533 ESTRUCTURA:
533 <access_name>.create(type_option = <class 'str'>, param_1 = <class 'param_1'>, ...)
534 <access_name>.create(type_option = <class 'str'>, param_1 = <class 'param_1'>, ...)
534 '''
535 '''
535 if type(type_option) is str:
536 if type(type_option) is str:
536 try:
537 try:
537 if type_option == 'dataset':
538 if type_option == 'dataset':
538 return getattr(self.ckan.action, 'package_create')(**kwargs)
539 return getattr(self.ckan.action, 'package_create')(**kwargs)
539 elif type_option == 'project':
540 elif type_option == 'project':
540 return getattr(self.ckan.action, 'organization_create')(**kwargs)
541 return getattr(self.ckan.action, 'organization_create')(**kwargs)
541 elif type_option == 'member':
542 elif type_option == 'member':
542 return getattr(self.ckan.action, 'organization_member_create')(**kwargs)
543 return getattr(self.ckan.action, 'organization_member_create')(**kwargs)
543 elif type_option == 'collaborator':
544 elif type_option == 'collaborator':
544 return getattr(self.ckan.action, 'package_collaborator_create')(**kwargs)
545 return getattr(self.ckan.action, 'package_collaborator_create')(**kwargs)
545 elif type_option == 'vocabulary':
546 elif type_option == 'vocabulary':
546 return getattr(self.ckan.action, 'vocabulary_create')(**kwargs)
547 return getattr(self.ckan.action, 'vocabulary_create')(**kwargs)
547 elif type_option == 'tag':
548 elif type_option == 'tag':
548 return getattr(self.ckan.action, 'tag_create')(**kwargs)
549 return getattr(self.ckan.action, 'tag_create')(**kwargs)
549 elif type_option == 'user':
550 elif type_option == 'user':
550 return getattr(self.ckan.action, 'user_create')(**kwargs)
551 return getattr(self.ckan.action, 'user_create')(**kwargs)
552 elif type_option == 'views':
553 if 'resource' == select:
554 self.list = ['package']
555 for key1, value1 in kwargs.items():
556 if not key1 in self.list:
557 self.dict[key1] = value1
558 return getattr(self.ckan.action, 'resource_create_default_resource_views')(**self.dict)
559 elif 'dataset' == select:
560 return getattr(self.ckan.action, 'package_create_default_resource_views')(**kwargs)
561 else:
562 return 'ERROR:: "select = %s" is not accepted' % (select)
551 else:
563 else:
552 return 'ERROR:: "%s" is not accepted' % (type_option)
564 return 'ERROR:: "type_option = %s" is not accepted' % (type_option)
553 except:
565 except:
554 _, exc_value, _ = sys.exc_info()
566 _, exc_value, _ = sys.exc_info()
555 return exc_value
567 return exc_value
556 else:
568 else:
557 return 'ERROR:: "type_option" must be <str>'
569 return 'ERROR:: "type_option" must be <str>'
558
570
559 def patch(self, type_option, **kwargs):
571 def patch(self, type_option, **kwargs):
560 '''
572 '''
561 FINALIDAD:
573 FINALIDAD:
562 Funciones personalizadas para actualizar
574 Funciones personalizadas para actualizar
563
575
564 PARAMETROS DISPONIBLES:
576 PARAMETROS DISPONIBLES:
565 CONSULTAR: "GUIA DE SCRIPT.pdf"
577 CONSULTAR: "GUIA DE SCRIPT.pdf"
566
578
567 ESTRUCTURA:
579 ESTRUCTURA:
568 <access_name>.patch(type_option = <class 'str'>, param_1 = <class 'param_1'>, ...)
580 <access_name>.patch(type_option = <class 'str'>, param_1 = <class 'param_1'>, ...)
569 '''
581 '''
570 if type(type_option) is str:
582 if type(type_option) is str:
571 try:
583 try:
572 if type_option == 'dataset':
584 if type_option == 'dataset':
573 return getattr(self.ckan.action, 'package_patch')(**kwargs)
585 return getattr(self.ckan.action, 'package_patch')(**kwargs)
574 elif type_option == 'project':
586 elif type_option == 'project':
575 return getattr(self.ckan.action, 'organization_patch')(**kwargs)
587 return getattr(self.ckan.action, 'organization_patch')(**kwargs)
576 elif type_option == 'resource':
588 elif type_option == 'resource':
577 return getattr(self.ckan.action, 'resource_patch')(**kwargs)
589 return getattr(self.ckan.action, 'resource_patch')(**kwargs)
578 elif type_option == 'member':
590 elif type_option == 'member':
579 return getattr(self.ckan.action, 'organization_member_create')(**kwargs)
591 return getattr(self.ckan.action, 'organization_member_create')(**kwargs)
580 elif type_option == 'collaborator':
592 elif type_option == 'collaborator':
581 return getattr(self.ckan.action, 'package_collaborator_create')(**kwargs)
593 return getattr(self.ckan.action, 'package_collaborator_create')(**kwargs)
582 else:
594 else:
583 return 'ERROR:: "%s" is not accepted' % (type_option)
595 return 'ERROR:: "type_option = %s" is not accepted' % (type_option)
584 except:
596 except:
585 _, exc_value, _ = sys.exc_info()
597 _, exc_value, _ = sys.exc_info()
586 return exc_value
598 return exc_value
587 else:
599 else:
588 return 'ERROR:: "type_option" must be <str>'
600 return 'ERROR:: "type_option" must be <str>'
589
601
590 def delete(self, type_option, select=None, **kwargs):
602 def delete(self, type_option, select=None, **kwargs):
591 '''
603 '''
592 FINALIDAD:
604 FINALIDAD:
593 Función personalizada para eliminar y/o purgar.
605 Función personalizada para eliminar y/o purgar.
594
606
595 PARAMETROS DISPONIBLES:
607 PARAMETROS DISPONIBLES:
596 CONSULTAR: "GUIA DE SCRIPT.pdf"
608 CONSULTAR: "GUIA DE SCRIPT.pdf"
597
609
598 ESTRUCTURA:
610 ESTRUCTURA:
599 <access_name>.delete(type_option = <class 'str'>, param_1 = <class 'param_1'>, ...)
611 <access_name>.delete(type_option = <class 'str'>, param_1 = <class 'param_1'>, ...)
600 '''
612 '''
601 if type(type_option) is str:
613 if type(type_option) is str:
602 try:
614 try:
603 if type_option == 'dataset':
615 if type_option == 'dataset':
604 if select is None:
616 if select is None:
605 return 'ERROR:: "select" must not be "None"'
617 return 'ERROR:: "select" must not be "None"'
606 else:
618 else:
607 if 'delete' == select:
619 if 'delete' == select:
608 return getattr(self.ckan.action, 'package_delete')(**kwargs)
620 return getattr(self.ckan.action, 'package_delete')(**kwargs)
609 elif 'purge' == select:
621 elif 'purge' == select:
610 return getattr(self.ckan.action, 'dataset_purge')(**kwargs)
622 return getattr(self.ckan.action, 'dataset_purge')(**kwargs)
611 else:
623 else:
612 return 'ERROR:: "%s" is not accepted' % (select)
624 return 'ERROR:: "select = %s" is not accepted' % (select)
613 elif type_option == 'project':
625 elif type_option == 'project':
614 if select is None:
626 if select is None:
615 return 'ERROR:: "select" must not be "None"'
627 return 'ERROR:: "select" must not be "None"'
616 else:
628 else:
617 if 'delete' == select:
629 if 'delete' == select:
618 return getattr(self.ckan.action, 'organization_delete')(**kwargs)
630 return getattr(self.ckan.action, 'organization_delete')(**kwargs)
619 elif 'purge' == select:
631 elif 'purge' == select:
620 return getattr(self.ckan.action, 'organization_purge')(**kwargs)
632 return getattr(self.ckan.action, 'organization_purge')(**kwargs)
621 else:
633 else:
622 return 'ERROR:: "%s" is not accepted' % (select)
634 return 'ERROR:: "select = %s" is not accepted' % (select)
623 elif type_option == 'resource':
635 elif type_option == 'resource':
624 return getattr(self.ckan.action, 'resource_delete')(**kwargs)
636 return getattr(self.ckan.action, 'resource_delete')(**kwargs)
625 elif type_option == 'vocabulary':
637 elif type_option == 'vocabulary':
626 return getattr(self.ckan.action, 'vocabulary_delete')(**kwargs)
638 return getattr(self.ckan.action, 'vocabulary_delete')(**kwargs)
627 elif type_option == 'tag':
639 elif type_option == 'tag':
628 return getattr(self.ckan.action, 'tag_delete')(**kwargs)
640 return getattr(self.ckan.action, 'tag_delete')(**kwargs)
629 elif type_option == 'user':
641 elif type_option == 'user':
630 return getattr(self.ckan.action, 'user_delete')(**kwargs)
642 return getattr(self.ckan.action, 'user_delete')(**kwargs)
631 else:
643 else:
632 return 'ERROR:: "%s" is not accepted' % (type_option)
644 return 'ERROR:: "type_option = %s" is not accepted' % (type_option)
633 except:
645 except:
634 _, exc_value, _ = sys.exc_info()
646 _, exc_value, _ = sys.exc_info()
635 return exc_value
647 return exc_value
636 else:
648 else:
637 return 'ERROR:: "type_option" must be <str>'
649 return 'ERROR:: "type_option" must be <str>'
638
650
639 def f_status_note(self, total, result, path):
651 def f_status_note(self, total, result, path):
640 file_txt = open(path+'status_note.txt', 'w')
652 file_txt = open(path+'status_note.txt', 'w')
641 file_txt = open(path+'status_note.txt', 'a')
653 file_txt = open(path+'status_note.txt', 'a')
642
654
643 file_txt.write('DOWNLOADED FILE(S): "%s"' % (len(result['name'])))
655 file_txt.write('DOWNLOADED FILE(S): "%s"' % (len(result['name'])))
644 file_txt.write(''+ os.linesep)
656 file_txt.write(''+ os.linesep)
645 for u in result['name']:
657 for u in result['name']:
646 file_txt.write(' - '+ u + os.linesep)
658 file_txt.write(' - '+ u + os.linesep)
647 file_txt.write(''+ os.linesep)
659 file_txt.write(''+ os.linesep)
648
660
649 file_txt.write('FAILED FILE(S): "%s"' % (len(total['name'])-len(result['name'])))
661 file_txt.write('FAILED FILE(S): "%s"' % (len(total['name'])-len(result['name'])))
650 file_txt.write(''+ os.linesep)
662 file_txt.write(''+ os.linesep)
651 if len(total['name'])-len(result['name']) != 0:
663 if len(total['name'])-len(result['name']) != 0:
652 for u in total['name']:
664 for u in total['name']:
653 if not u in result['name']:
665 if not u in result['name']:
654 file_txt.write(' - '+ u + os.linesep)
666 file_txt.write(' - '+ u + os.linesep)
655 else:
667 else:
656 file_txt.write(' "None"'+ os.linesep)
668 file_txt.write(' "None"'+ os.linesep)
657
669
658 def f_name(self, name_dataset, ext, tempdir):
670 def f_name(self, name_dataset, ext, tempdir):
659 while self.check:
671 while self.check:
660 self.str = ''
672 self.str = ''
661 if self.cont == 0:
673 if self.cont == 0:
662 if os.path.exists(tempdir + name_dataset + ext):
674 if os.path.exists(tempdir + name_dataset + ext):
663 self.str = name_dataset+'('+str(self.cont+1)+')'+ext
675 self.str = name_dataset+'('+str(self.cont+1)+')'+ext
664 else:
676 else:
665 self.check = self.check * 0
677 self.check = self.check * 0
666 self.str = name_dataset + ext
678 self.str = name_dataset + ext
667 else:
679 else:
668 if not os.path.exists(tempdir + name_dataset+'('+str(self.cont)+')'+ext):
680 if not os.path.exists(tempdir + name_dataset+'('+str(self.cont)+')'+ext):
669 self.check = self.check * 0
681 self.check = self.check * 0
670 self.str = name_dataset+'('+str(self.cont)+')'+ ext
682 self.str = name_dataset+'('+str(self.cont)+')'+ ext
671 self.cont = self.cont+1
683 self.cont = self.cont+1
672 return self.str
684 return self.str
673
685
674 def f_zipdir(self, path, ziph, zip_name):
686 def f_zipdir(self, path, ziph, zip_name):
675 for root, _, files in os.walk(path):
687 for root, _, files in os.walk(path):
676 print('.....')
688 print('.....')
677 print('Creating: "{}" >>'.format(zip_name))
689 print('Creating: "{}" >>'.format(zip_name))
678 for __file in tqdm(iterable=files, total=len(files)):
690 for __file in tqdm(iterable=files, total=len(files)):
679 new_dir = os.path.relpath(os.path.join(root, __file), os.path.join(path, '..'))
691 new_dir = os.path.relpath(os.path.join(root, __file), os.path.join(path, '..'))
680 ziph.write(os.path.join(root, __file), new_dir)
692 ziph.write(os.path.join(root, __file), new_dir)
681 print('Created >>')
693 print('Created >>')
682
694
683 def download_by_step(self, response, tempdir_name):
695 def download_by_step(self, response, tempdir_name):
684 try:
696 try:
685 with requests.get(response['url'], stream=True, headers={'Authorization': self.Authorization}) as resp:
697 with requests.get(response['url'], stream=True, headers={'Authorization': self.Authorization}) as resp:
686 if resp.status_code == 200:
698 if resp.status_code == 200:
687 with open(tempdir_name+response['name'], 'wb') as file:
699 with open(tempdir_name+response['name'], 'wb') as file:
688 for chunk in resp.iter_content(chunk_size = self.chunk_size):
700 for chunk in resp.iter_content(chunk_size = self.chunk_size):
689 if chunk:
701 if chunk:
690 file.write(chunk)
702 file.write(chunk)
691 except requests.exceptions.RequestException:
703 except requests.exceptions.RequestException:
692 pass
704 pass
693
705
694 def download_files(self, **kwargs):
706 def download_files(self, **kwargs):
695 '''
707 '''
696 FINALIDAD:
708 FINALIDAD:
697 Funcion personalizada para la descarga de archivos existentes de un dataset.
709 Funcion personalizada para la descarga de archivos existentes de un dataset.
698
710
699 PARAMETROS DISPONIBLES:
711 PARAMETROS DISPONIBLES:
700 CONSULTAR: "GUIA DE SCRIPT.pdf"
712 CONSULTAR: "GUIA DE SCRIPT.pdf"
701
713
702 ESTRUCTURA:
714 ESTRUCTURA:
703 <access_name>.download_files(id = <class 'str'>, param_1 = <class 'param_1'>, ...)
715 <access_name>.download_files(id = <class 'str'>, param_1 = <class 'param_1'>, ...)
704 '''
716 '''
705 dict_local = {}
717 dict_local = {}
706 #----------------------------------------------#
718 #----------------------------------------------#
707 if 'zip' in kwargs:
719 if 'zip' in kwargs:
708 if type(kwargs['zip']) is not bool:
720 if type(kwargs['zip']) is not bool:
709 return 'ERROR:: "zip" must be: <class "bool">'
721 return 'ERROR:: "zip" must be: <class "bool">'
710 else:
722 else:
711 dict_local['zip'] = kwargs['zip']
723 dict_local['zip'] = kwargs['zip']
712 else:
724 else:
713 dict_local['zip'] = False
725 dict_local['zip'] = False
714 #----------------------------------------------#
726 #----------------------------------------------#
715 if 'status_note' in kwargs:
727 if 'status_note' in kwargs:
716 if type(kwargs['status_note']) is not bool:
728 if type(kwargs['status_note']) is not bool:
717 return 'ERROR:: "status_note" must be: <class "bool">'
729 return 'ERROR:: "status_note" must be: <class "bool">'
718 else:
730 else:
719 dict_local['status_note'] = kwargs['status_note']
731 dict_local['status_note'] = kwargs['status_note']
720 else:
732 else:
721 dict_local['status_note'] = False
733 dict_local['status_note'] = False
722 #----------------------------------------------#
734 #----------------------------------------------#
723 if 'path' in kwargs:
735 if 'path' in kwargs:
724 if type(kwargs['path']) is str:
736 if type(kwargs['path']) is str:
725 if os.path.isdir(kwargs['path']) == False:
737 if os.path.isdir(kwargs['path']) == False:
726 return 'ERROR:: "path" does not exist'
738 return 'ERROR:: "path" does not exist'
727 else:
739 else:
728 if kwargs['path'][-1:] != self.separator:
740 if kwargs['path'][-1:] != self.separator:
729 dict_local['path'] = kwargs['path']+self.separator
741 dict_local['path'] = kwargs['path']+self.separator
730 else:
742 else:
731 dict_local['path'] = kwargs['path']
743 dict_local['path'] = kwargs['path']
732
744
733 txt = dict_local['path']+datetime.now().strftime("%Y_%m_%d_%H_%M_%S_%f")+'.txt'
745 txt = dict_local['path']+datetime.now().strftime("%Y_%m_%d_%H_%M_%S_%f")+'.txt'
734 if int(platform.python_version()[0]) == 3:
746 if int(platform.python_version()[0]) == 3:
735 try:
747 try:
736 file_txt = open(txt, 'w')
748 file_txt = open(txt, 'w')
737 file_txt.close()
749 file_txt.close()
738 os.remove(txt)
750 os.remove(txt)
739 except PermissionError:
751 except PermissionError:
740 return 'ERROR:: Access denied, you are not authorized to write files: "%s"' % (dict_local['path'])
752 return 'ERROR:: Access denied, you are not authorized to write files: "%s"' % (dict_local['path'])
741 else:
753 else:
742 try:
754 try:
743 file_txt = open(txt, 'w')
755 file_txt = open(txt, 'w')
744 file_txt.close()
756 file_txt.close()
745 os.remove(txt)
757 os.remove(txt)
746 except:
758 except:
747 return 'ERROR:: Access denied, you are not authorized to write files: "%s"' % (dict_local['path'])
759 return 'ERROR:: Access denied, you are not authorized to write files: "%s"' % (dict_local['path'])
748 else:
760 else:
749 return 'ERROR:: "path" must be: <class "str">'
761 return 'ERROR:: "path" must be: <class "str">'
750 else:
762 else:
751 dict_local['path'] = ''
763 dict_local['path'] = ''
752 #----------------------------------------------#
764 #----------------------------------------------#
753 for key, value in kwargs.items():
765 for key, value in kwargs.items():
754 if not key in dict_local:
766 if not key in dict_local:
755 self.dict[key] = value
767 self.dict[key] = value
756 try:
768 try:
757 response = getattr(self.ckan.action, 'url_resources')(**self.dict)
769 response = getattr(self.ckan.action, 'url_resources')(**self.dict)
758 except:
770 except:
759 _, exc_value, _ = sys.exc_info()
771 _, exc_value, _ = sys.exc_info()
760 return exc_value
772 return exc_value
761
773
762 if len(response) != 0:
774 if len(response) != 0:
763 #--------------TEMP PATH---------------#
775 #--------------TEMP PATH---------------#
764 if dict_local['zip']:
776 if dict_local['zip']:
765 tempdir = tempfile.mkdtemp(prefix=kwargs['id']+'-')+self.separator
777 tempdir = tempfile.mkdtemp(prefix=kwargs['id']+'-')+self.separator
766 os.mkdir(tempdir+kwargs['id'])
778 os.mkdir(tempdir+kwargs['id'])
767 dir_name = tempdir + kwargs['id'] + self.separator
779 dir_name = tempdir + kwargs['id'] + self.separator
768 else:
780 else:
769 dir = self.f_name(kwargs['id'], '', dict_local['path'])
781 dir = self.f_name(kwargs['id'], '', dict_local['path'])
770 os.mkdir(dict_local['path'] + dir)
782 os.mkdir(dict_local['path'] + dir)
771 dir_name = dict_local['path'] + dir + self.separator
783 dir_name = dict_local['path'] + dir + self.separator
772 #-----------DOWNLOAD FILES-------------#
784 #-----------DOWNLOAD FILES-------------#
773 print('.....')
785 print('.....')
774 print('Downloading "{}" file(s) >>'.format(len(response)))
786 print('Downloading "{}" file(s) >>'.format(len(response)))
775 name_total = {'name': []}
787 name_total = {'name': []}
776 with concurrent.futures.ThreadPoolExecutor() as executor:
788 with concurrent.futures.ThreadPoolExecutor() as executor:
777 for u in tqdm(iterable=response, total=len(response)):
789 for u in tqdm(iterable=response, total=len(response)):
778 name_total['name'].append(u['name'])
790 name_total['name'].append(u['name'])
779 executor.submit(self.download_by_step, u, dir_name)
791 executor.submit(self.download_by_step, u, dir_name)
780 name_check = {}
792 name_check = {}
781 name_check['name'] = [f for f in os.listdir(dir_name) if os.path.isfile(os.path.join(dir_name, f))]
793 name_check['name'] = [f for f in os.listdir(dir_name) if os.path.isfile(os.path.join(dir_name, f))]
782 print('"{}" downloaded file(s) successfully >>'.format(len(name_check['name'])))
794 print('"{}" downloaded file(s) successfully >>'.format(len(name_check['name'])))
783 #--------------------------------------#
795 #--------------------------------------#
784 if len(name_check['name']) != 0:
796 if len(name_check['name']) != 0:
785 #----------Status Note---------#
797 #----------Status Note---------#
786 if dict_local['status_note']:
798 if dict_local['status_note']:
787 print('.....')
799 print('.....')
788 print('Creating: "status_note.txt" >>')
800 print('Creating: "status_note.txt" >>')
789 self.f_status_note(name_total, name_check, dir_name)
801 self.f_status_note(name_total, name_check, dir_name)
790 print('Created>>')
802 print('Created>>')
791 #----------ZIP CREATE----------#
803 #----------ZIP CREATE----------#
792 if dict_local['zip']:
804 if dict_local['zip']:
793 zip_name = self.f_name(kwargs['id'], '.zip', dict_local['path'])
805 zip_name = self.f_name(kwargs['id'], '.zip', dict_local['path'])
794 ziph = zipfile.ZipFile(dict_local['path'] + zip_name, 'w', zipfile.ZIP_DEFLATED, allowZip64=True)
806 ziph = zipfile.ZipFile(dict_local['path'] + zip_name, 'w', zipfile.ZIP_DEFLATED, allowZip64=True)
795 self.f_zipdir(dir_name, ziph, zip_name)
807 self.f_zipdir(dir_name, ziph, zip_name)
796 ziph.close()
808 ziph.close()
797 #Delete Temporal Path
809 #Delete Temporal Path
798 if os.path.exists(tempdir[:-1]):
810 if os.path.exists(tempdir[:-1]):
799 shutil.rmtree(tempdir[:-1])
811 shutil.rmtree(tempdir[:-1])
800 #------------------------------#
812 #------------------------------#
801 print('.....')
813 print('.....')
802 return 'DOWNLOAD FINISHED'
814 return 'DOWNLOAD FINISHED'
803 else:
815 else:
804 #Delete Temporal Path
816 #Delete Temporal Path
805 if dict_local['zip']:
817 if dict_local['zip']:
806 if os.path.exists(tempdir[:-1]):
818 if os.path.exists(tempdir[:-1]):
807 shutil.rmtree(tempdir[:-1])
819 shutil.rmtree(tempdir[:-1])
808 else:
820 else:
809 if os.path.exists(dir_name[:-1]):
821 if os.path.exists(dir_name[:-1]):
810 shutil.rmtree(dir_name[:-1])
822 shutil.rmtree(dir_name[:-1])
811 return 'NO FILES WERE DOWNLOADED'
823 return 'NO FILES WERE DOWNLOADED'
812 else:
824 else:
813 return 'FILES NOT FOUND' No newline at end of file
825 return 'FILES NOT FOUND'
General Comments 0
You need to be logged in to leave comments. Login now