##// END OF EJS Templates
DATABASES » ckanext-jro » api-cliente
RSS
v2.9.2 :: Update 'upload_multiple_files_advance' api and Add views resource
eynilupu -
r4:27e924dd52b0
parent child
Show More
Show file before | Show file after | Hide comments
1 NO CONTENT: modified file, binary diff hidden
NO CONTENT: modified file, binary diff hidden
Show file before | Show file after | Hide comments
@@ -1,825 +1,871
1 from ckanapi import RemoteCKAN
1 from ckanapi import RemoteCKAN
2 from datetime import datetime
3 from tqdm import tqdm
2 #from ckanapi.errors import NotAuthorized, NotFound, ValidationError, SearchQueryError, SearchError, CKANAPIError, ServerIncompatibleError
4 #from ckanapi.errors import NotAuthorized, NotFound, ValidationError, SearchQueryError, SearchError, CKANAPIError, ServerIncompatibleError
3 import sys
5 import sys
4 import platform
6 import platform
5 import os
7 import os
6 import tempfile
8 import tempfile
7 import shutil
9 import shutil
8 import zipfile
10 import zipfile
9 import concurrent.futures
11 import concurrent.futures
10 import requests
12 import requests
11 import json
13 import json
12 import pathlib
14 import pathlib
13 import uuid
15 import uuid
14 from datetime import datetime
15 from tqdm import tqdm
16
16
17 class JROAPI():
17 class JROAPI():
18 """
18 """
19 FINALIDAD:
19 FINALIDAD:
20 Script para administrar y obtener la data del repositorio por medio de APIs.
20 Script para administrar y obtener la data del repositorio por medio de APIs.
21
21
22 REQUISITIOS PREVIOS:
22 REQUISITIOS PREVIOS:
23 - Paso 1: Tener "pip [Python 2]" o "pip3 [Python 3]" instalado:
23 - Paso 1: Tener "pip [Python 2]" o "pip3 [Python 3]" instalado:
24 - Paso 2: Instalar lo siguiente como admininstrador:
24 - Paso 2: Instalar lo siguiente como admininstrador:
25 En Python 2
25 En Python 2
26 - pip install ckanapi==4.5
26 - pip install ckanapi==4.5
27 - pip install requests
27 - pip install requests
28 - pip install pathlib
29 - pip install futures
28 - pip install tqdm
30 - pip install tqdm
29 En Python 3
31 En Python > 3
30 - pip3 install ckanapi==4.5
32 - pip3 install ckanapi==4.5
31 - pip3 install requests
33 - pip3 install requests
32 - pip3 install tqdm
34 - pip3 install tqdm
33
35
34 FUNCIONES DISPONIBLES:
36 FUNCIONES DISPONIBLES:
35 - action
37 - action
36 - upload_file
38 - upload_file
37 - upload_multiple_files
39 - upload_multiple_files
38 - upload_multiple_files_advance
40 - upload_multiple_files_advance
39 - show
41 - show
40 - search
42 - search
41 - create
43 - create
42 - patch
44 - patch
43 - delete
45 - delete
44 - download_files
46 - download_files
45
47
46 EJEMPLOS:
48 EJEMPLOS:
47 #1:
49 #1:
48 with JROAPI('http://demo.example.com', Authorization='#########') as <access_name>:
50 with JROAPI('http://demo.example.com', Authorization='#########') as <access_name>:
49 ... some operation(s) ...
51 ... some operation(s) ...
50 #2:
52 #2:
51 <access_name> = JROAPI('http://example.com', Authorization='#########')
53 <access_name> = JROAPI('http://example.com', Authorization='#########')
52 ... some operation(s) ...
54 ... some operation(s) ...
53 <access_name>.ckan.close()
55 <access_name>.ckan.close()
54
56
55 REPORTAR ALGUN PROBLEMA:
57 REPORTAR ALGUN PROBLEMA:
56 Debe enviar un correo a eynilupu@igp.gob.pe detallando los siguientes pasos:
58 Debe enviar un correo a eynilupu@igp.gob.pe detallando los siguientes pasos:
57 1) Identifiquese
59 1) Correo para contactarlo
58 2) Describir el problema
60 2) Descripcion del problema
59 3) ¿En que funcion esta el problema?
61 3) ¿En que paso o seccion encontro el problema?
60 4) ¿Que esperaba que hiciera la funcion sin el problema?
62 4) ¿Cual era el resultado que usted esperaba?
61 """
63 """
62 def __init__(self, url, Authorization=None):
64 def __init__(self, url, Authorization=None):
63 ua = 'CKAN_JRO/1.1 (+'+str(url)+')'
65 ua = 'CKAN_JRO/1.1 (+'+str(url)+')'
64 #ua = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36'
66 #ua = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36'
65 self.ckan = RemoteCKAN(url, apikey=Authorization, user_agent=ua)
67 self.ckan = RemoteCKAN(url, apikey=Authorization, user_agent=ua)
66 #self.ckan = RemoteCKAN(url, apikey=Authorization)
68 #self.ckan = RemoteCKAN(url, apikey=Authorization)
67 self.Authorization = Authorization
69 self.Authorization = Authorization
68 if platform.system() == 'Windows':
70 if platform.system() == 'Windows':
69 self.separator = '\\'
71 self.separator = '\\'
70 else:
72 else:
71 self.separator = '/'
73 self.separator = '/'
72
74
73 self.chunk_size = 1024
75 self.chunk_size = 1024
74 self.list = []
76 self.list = []
75 self.dict = {}
77 self.dict = {}
76 self.str = ''
78 self.str = ''
77 self.check = 1
79 self.check = 1
78 self.cont = 0
80 self.cont = 0
79
81
80 def __enter__(self):
82 def __enter__(self):
81 return self
83 return self
82
84
83 def __exit__(self, *args):
85 def __exit__(self, *args):
84 self.ckan.close()
86 self.ckan.close()
85
87
86 def action(self, action, **kwargs):
88 def action(self, action, **kwargs):
87 """
89 """
88 FINALIDAD:
90 FINALIDAD:
89 Funcion para llamar a las APIs disponibles
91 Funcion para llamar a las APIs disponibles
90
92
91 APIs DISPONIBLES:
93 APIs DISPONIBLES:
92 CONSULTAR: "GUIA DE SCRIPT.pdf"
94 CONSULTAR: "GUIA DE SCRIPT.pdf"
93
95
94 EJEMPLO:
96 EJEMPLO:
95 <access_name>.action(<consuming API>, param_1 = <class 'param_1'>, ...)
97 <access_name>.action(<consuming API>, param_1 = <class 'param_1'>, ...)
96 """
98 """
97 #--------------- CASE: PACKAGE SEARCH ---------------#
99 #--------------- CASE: PACKAGE SEARCH ---------------#
98 if kwargs is not None:
100 if kwargs is not None:
99 if action == 'package_search':
101 if action == 'package_search':
100 self.list = ['facet_mincount', 'facet_limit', 'facet_field']
102 self.list = ['facet_mincount', 'facet_limit', 'facet_field']
101 for facet in self.list:
103 for facet in self.list:
102 if facet in kwargs:
104 if facet in kwargs:
103 kwargs[facet.replace('_', '.')] = kwargs[facet]
105 kwargs[facet.replace('_', '.')] = kwargs[facet]
104 kwargs.pop(facet)
106 kwargs.pop(facet)
105 #----------------------------------------------------#
107 #----------------------------------------------------#
106 try:
108 try:
107 return getattr(self.ckan.action, action)(**kwargs)
109 return getattr(self.ckan.action, action)(**kwargs)
108 except:
110 except:
109 _, exc_value, _ = sys.exc_info()
111 _, exc_value, _ = sys.exc_info()
110 return exc_value
112 return exc_value
111
113
112 def upload_file(self, dataset_id, file_path, file_date, file_type, **kwargs):
114 def upload_file(self, dataset_id, file_path, file_date, file_type, **kwargs):
115 # Agregar si es interruptido por teclado
113 '''
116 '''
114 FINALIDAD:
117 FINALIDAD:
115 Funcion para subir un unico archivo al repositorio del ROJ.
118 Funcion para subir un unico archivo al repositorio del ROJ.
116
119
117 PARAMETROS DISPONIBLES:
120 PARAMETROS DISPONIBLES:
118 CONSULTAR: "GUIA DE SCRIPT.pdf"
121 CONSULTAR: "GUIA DE SCRIPT.pdf"
119
122
120 ESTRUCTURA:
123 ESTRUCTURA:
121 <access_name>.upload_file(dataset_id = <class 'str'>, file_date = <class 'str'>, file_path = <class 'str'>, file_type = <class 'str'>, param_1 = <class 'param_1'>, ...)
124 <access_name>.upload_file(dataset_id = <class 'str'>, file_date = <class 'str'>, file_path = <class 'str'>, file_type = <class 'str'>, param_1 = <class 'param_1'>, ...)
122 '''
125 '''
123 self.list = ['package_id', 'upload', 'voc_file_type', 'name'] #file_date
126 self.list = ['package_id', 'upload', 'voc_file_type', 'name'] #file_date
124 for key1, value1 in kwargs.items():
127 for key1, value1 in kwargs.items():
125 if not key1 in self.list:
128 if not key1 in self.list:
126 self.dict[key1] = value1
129 self.dict[key1] = value1
127
130
128 #---------------------------#
131 #---------------------------#
129 if not 'others' in kwargs:
132 if not 'others' in kwargs:
130 self.dict['others'] = ''
133 self.dict['others'] = ''
131 else:
134 else:
132 if isinstance(kwargs['others'], list):
135 if isinstance(kwargs['others'], list):
133 self.dict['others'] = json.dumps(kwargs['others'])
136 self.dict['others'] = json.dumps(kwargs['others'])
134 #---------------------------#
137 #---------------------------#
135
138
136 if not os.path.isfile(file_path):
139 if not os.path.isfile(file_path):
137 return 'File "%s" not exist' % (file_path)
140 return 'File "%s" not exist' % (file_path)
138
141
139 if not 'format' in self.dict:
142 if not 'format' in self.dict:
140 self.str = ''.join(pathlib.Path(file_path).suffixes)
143 self.str = ''.join(pathlib.Path(file_path).suffixes)
141 if len(self.str) > 0:
144 if len(self.str) > 0:
142 self.dict['format'] = self.str.upper()[1:]
145 self.dict['format'] = self.str.upper()[1:]
143
146
144 try:
147 try:
145 return getattr(self.ckan.action, 'resource_create')(package_id=dataset_id, file_date=file_date, upload=open(file_path, 'rb'), voc_file_type=file_type, name=pathlib.Path(file_path).name, **self.dict)
148 return getattr(self.ckan.action, 'resource_create')(package_id=dataset_id, file_date=file_date, upload=open(file_path, 'rb'), voc_file_type=file_type, name=pathlib.Path(file_path).name, **self.dict)
146 except:
149 except:
147 _, exc_value, _ = sys.exc_info()
150 _, exc_value, _ = sys.exc_info()
148 return exc_value
151 return exc_value
149
152
150
153 def upload_multiple_files_advance(self, dataset_id, path_files, file_date, file_type, max_size=100, ignore_repetition=False, **kwargs):
151 def upload_multiple_files_advance(self, dataset_id, path_files, file_date, file_type, **kwargs):
154 # Agregar si es interruptido por teclado
152 '''
155 '''
153 FINALIDAD:
156 FINALIDAD:
154 Funcion para subir multiples archivos al repositorio del ROJ.
157 Funcion para subir multiples archivos al repositorio del ROJ.
155
158
156 PARAMETROS DISPONIBLES:
159 PARAMETROS DISPONIBLES:
157 CONSULTAR: "GUIA DE SCRIPT.pdf"
160 CONSULTAR: "GUIA DE SCRIPT.pdf"
158
161
159 ESTRUCTURA:
162 ESTRUCTURA:
160 <access_name>.upload_multiple_files_advance(dataset_id = <class 'str'>, path_files = <class 'list of strings'>, file_date = <class 'str'>, file_type = <class 'str'>, param_1 = <class 'param_1'>, ...)
163 <access_name>.upload_multiple_files_advance(dataset_id = <class 'str'>, path_files = <class 'list of strings'>, file_date = <class 'str'>, file_type = <class 'str'>, param_1 = <class 'param_1'>, ...)
161 '''
164 '''
165 #-------------------------PACKAGE SHOW-----------------------#
166 try:
167 dataset_show = getattr(self.ckan.action, 'package_show')(id=dataset_id)['resources']
168 except:
169 _, exc_value, _ = sys.exc_info()
170 print('ERROR obtaining metadata dataset:: Use the "print" for more information')
171 return exc_value
172 #------------------------------------------------------------#
173 resources_name = []
174 for u in dataset_show:
175 resources_name.append(u['name'].lower())
176 #------------------------------------------------------------#
162 self.list = ['package_id', 'upload', 'voc_file_type', 'name']
177 self.list = ['package_id', 'upload', 'voc_file_type', 'name']
163
164 for key1, value1 in kwargs.items():
178 for key1, value1 in kwargs.items():
165 if not key1 in self.list:
179 if not key1 in self.list:
166 self.dict[key1] = value1
180 self.dict[key1] = value1
167
181 #------------------------------------------------------------#
168 #---------------------------#
169 if not 'others' in kwargs:
182 if not 'others' in kwargs:
170 self.dict['others'] = ''
183 self.dict['others'] = ''
171 else:
184 else:
172 if isinstance(kwargs['others'], list):
185 if isinstance(kwargs['others'], list):
173 self.dict['others'] = json.dumps(kwargs['others'])
186 self.dict['others'] = json.dumps(kwargs['others'])
174 #---------------------------#
187 #------------------------------------------------------------#
175
188 total_list = []
176 params_dict = {'upload':[], 'name':[]}
177 #---------------CASO : "path" or "path_list"-----------------#
189 #---------------CASO : "path" or "path_list"-----------------#
178 if type(path_files) is list:
190 if type(path_files) is list:
179 if len(path_files) != 0:
191 if len(path_files) != 0:
180 path_files.sort()
192 path_files.sort()
181 for u in path_files:
193 for u in path_files:
182 if os.path.isfile(u):
194 if os.path.isfile(u):
183 params_dict['upload'].append(open(u, 'rb'))
195 if pathlib.Path(u).name.lower() in resources_name:
184 params_dict['name'].append(pathlib.Path(u).name)
196 if not ignore_repetition:
197 return 'ERROR:: "%s" file already exist in this dataset' % (pathlib.Path(u).name)
198 print('WARRING:: "'+ str(pathlib.Path(u).name) +'" file was ignored because already exist in this dataset')
199 else:
200 total_list.append({'name':pathlib.Path(u).name, 'size': os.stat(u).st_size, 'upload':open(u, 'rb')})
185 else:
201 else:
186 return 'File "%s" does not exist' % (u)
202 return 'File "%s" does not exist' % (u)
187 else:
203 else:
188 return 'ERROR:: "path_list is empty"'
204 return 'ERROR:: "path_list is empty"'
205
189 elif type(path_files) is str:
206 elif type(path_files) is str:
190 if os.path.isdir(path_files):
207 if os.path.isdir(path_files):
191 path_order = [f for f in os.listdir(path_files) if os.path.isfile(os.path.join(path_files, f))]
208 path_order = [f for f in os.listdir(path_files) if os.path.isfile(os.path.join(path_files, f))]
192 path_order.sort()
209 path_order.sort()
193 if path_order:
210 if path_order:
194 for name in path_order:
211 for name in path_order:
195 params_dict['upload'].append(open(os.path.join(path_files, name), 'rb'))
212 if name.lower() in resources_name:
196 params_dict['name'].append(name)
213 if not ignore_repetition:
214 return 'ERROR:: "%s" file already exist in this dataset' % (name)
215 print('WARRING:: "'+ name +'" file was ignored because already exist in this dataset')
216 else:
217 total_list.append({'name':name, 'size': os.stat(os.path.join(path_files, name)).st_size, 'upload':open(os.path.join(path_files, name), 'rb')})
197 else:
218 else:
198 return "ERROR:: There aren't files in this directory"
219 return "ERROR:: There aren't files in this directory"
199 else:
220 else:
200 return 'ERROR:: Directory "%s" does not exist' % (path_files)
221 return 'ERROR:: Directory "%s" does not exist' % (path_files)
201 else:
222 else:
202 return 'ERROR:: "path_files" must be a str or list'
223 return 'ERROR:: "path_files" must be a str or list'
203 #------------------------------------------------------------#
224 #------------------------------------------------------------#
204 resource_extend = []
205 files_dict = {}
206 for count, name in enumerate(params_dict['name']):
207 param_list = {"name": name, "file_date": file_date, "voc_file_type": file_type}
208 param_list.update(self.dict)
209
210 if not 'format' in param_list:
211 format = ''.join(pathlib.Path(name).suffixes)
212 if len(format) > 0:
213 param_list['format'] = format.upper()[1:]
214
215 resource_extend.append(param_list)
216 files_dict['update__resources__-'+ str(len(params_dict['name'])-count) +'__upload'] = (name, params_dict['upload'][count])
217
218 #------------------------------------------------------------#
219 try:
225 try:
220 uuid.UUID(str(dataset_id), version=4)
226 uuid.UUID(str(dataset_id), version=4)
221 package_id_or_name = '"id": "' + str(dataset_id) + '"'
227 package_id_or_name = '"id": "' + str(dataset_id) + '"'
222 except ValueError:
228 except ValueError:
223 package_id_or_name = '"name": "' + str(dataset_id) + '"'
229 package_id_or_name = '"name": "' + str(dataset_id) + '"'
224 #------------------------------------------------------------#
230 #------------------------------------------------------------#
225 print('"{}" file(s) found >> uploading'.format(len(params_dict['name'])))
231 blocks = [[]]
226 try:
232 size_file = 0
227 result = self.ckan.call_action(
233 inter_num = 0
228 'package_revise',
234 for value in total_list:
229 {'match': '{'+ str(package_id_or_name) +'}', 'update__resources__extend': json.dumps(resource_extend)},
235 if value['size'] > 1048576 * float(max_size):
230 files=files_dict
236 return 'ERROR:: The size of the "%s" file is %sMB, please change "max_size" value' % (value['name'], str(round(value['size']/1048576, 1)))
231 )
237 size_file = size_file + value['size']
232 print('Uploaded file(s) successfully')
238 if size_file <= 1048576 * float(max_size):
233 return result
239 del value['size']
234 except:
240 blocks[inter_num].append(value)
235 print('ERROR :: Use the "print" option for more information')
241 else:
236 _, exc_value, _ = sys.exc_info()
242 inter_num = inter_num + 1
237 return exc_value
243 blocks.append([])
244 del value['size']
245 blocks[inter_num].append(value)
246 #------------------------------------------------------------#
247 if len(blocks[0]) > 0:
248 print('BLOCK(S) IN TOTAL:: {}'.format(len(blocks)))
249 for count1, block in enumerate(blocks):
250 print('---- BLOCK N°{} ----'.format(count1 + 1))
251 resource_extend = []
252 files_dict = {}
253 for count2, value2 in enumerate(block):
254 value2['file_date'] = file_date
255 value2['voc_file_type'] = file_type
256 value2.update(self.dict)
257
258 if not 'format' in value2:
259 format = ''.join(pathlib.Path(value2['name']).suffixes)
260 if len(format) > 0:
261 value2['format'] = format.upper()[1:]
262
263 files_dict['update__resources__-'+ str(len(block)-count2) +'__upload'] = (value2['name'], value2['upload'])
264 del value2['upload']
265 resource_extend.append(value2)
266
267 print('BLOCK N°{} :: "{}" file(s) found >> uploading'.format(count1 + 1, len(block)))
268 try:
269 result = self.ckan.call_action(
270 'package_revise',
271 {'match': '{'+ str(package_id_or_name) +'}', 'update__resources__extend': json.dumps(resource_extend)},
272 files=files_dict
273 )
274 print('BLOCK N°{} :: Uploaded file(s) successfully'.format(count1 + 1))
275 if len(blocks) == count1 + 1:
276 return result
277 except:
278 print('ERROR :: Use the "print" for more information')
279 _, exc_value, _ = sys.exc_info()
280 return exc_value
281 else:
282 return "ERROR:: No file(s) found to upload"
238
283
239 def upload_multiple_files(self, dataset_id, path_files, date_files, type_files, **kwargs):
284 def upload_multiple_files(self, dataset_id, path_files, date_files, type_files, **kwargs):
285 # Agregar si es interruptido por teclado
240 '''
286 '''
241 FINALIDAD:
287 FINALIDAD:
242 Funcion para subir multiples archivos al repositorio del ROJ.
288 Funcion para subir multiples archivos al repositorio del ROJ.
243
289
244 PARAMETROS DISPONIBLES:
290 PARAMETROS DISPONIBLES:
245 CONSULTAR: "GUIA DE SCRIPT.pdf"
291 CONSULTAR: "GUIA DE SCRIPT.pdf"
246
292
247 ESTRUCTURA:
293 ESTRUCTURA:
248 <access_name>.upload_multiple_files(dataset_id = <class 'str'>, path_files = <class 'str'> or <class 'list of strings'>, date_files = <class 'str'> or <class 'list of strings'>, type_files = <class 'str'> or <class 'list of strings'>, param_1 = <class 'param_1'>, ...)
294 <access_name>.upload_multiple_files(dataset_id = <class 'str'>, path_files = <class 'str'> or <class 'list of strings'>, date_files = <class 'str'> or <class 'list of strings'>, type_files = <class 'str'> or <class 'list of strings'>, param_1 = <class 'param_1'>, ...)
249 '''
295 '''
250
296
251 params_dict = {'upload':[], 'name':[]}
297 params_dict = {'upload':[], 'name':[]}
252 if not 'format' in kwargs:
298 if not 'format' in kwargs:
253 params_dict.update({'format':[]})
299 params_dict.update({'format':[]})
254 #---------------CASO : "path" or "path_list"-----------------#
300 #---------------CASO : "path" or "path_list"-----------------#
255 if type(path_files) is list:
301 if type(path_files) is list:
256 if len(path_files) != 0:
302 if len(path_files) != 0:
257 path_files.sort()
303 path_files.sort()
258 for u in path_files:
304 for u in path_files:
259 if os.path.isfile(u):
305 if os.path.isfile(u):
260 params_dict['upload'].append(open(u, 'rb'))
306 params_dict['upload'].append(open(u, 'rb'))
261 params_dict['name'].append(pathlib.Path(u).name)
307 params_dict['name'].append(pathlib.Path(u).name)
262 if not 'format' in kwargs:
308 if not 'format' in kwargs:
263 format = ''.join(pathlib.Path(u).suffixes)
309 format = ''.join(pathlib.Path(u).suffixes)
264 if len(format) > 0:
310 if len(format) > 0:
265 params_dict['format'].append(format.upper()[1:])
311 params_dict['format'].append(format.upper()[1:])
266 else:
312 else:
267 params_dict['format'].append('')
313 params_dict['format'].append('')
268 else:
314 else:
269 return 'File "%s" does not exist' % (u)
315 return 'File "%s" does not exist' % (u)
270 else:
316 else:
271 return 'ERROR:: "path_list is empty"'
317 return 'ERROR:: "path_list is empty"'
272 elif type(path_files) is str:
318 elif type(path_files) is str:
273 if os.path.isdir(path_files):
319 if os.path.isdir(path_files):
274 path_order = [f for f in os.listdir(path_files) if os.path.isfile(os.path.join(path_files, f))]
320 path_order = [f for f in os.listdir(path_files) if os.path.isfile(os.path.join(path_files, f))]
275 path_order.sort()
321 path_order.sort()
276 if path_order:
322 if path_order:
277 for name in path_order:
323 for name in path_order:
278 params_dict['upload'].append(open(os.path.join(path_files, name), 'rb'))
324 params_dict['upload'].append(open(os.path.join(path_files, name), 'rb'))
279 params_dict['name'].append(name)
325 params_dict['name'].append(name)
280 if not 'format' in kwargs:
326 if not 'format' in kwargs:
281 format = ''.join(pathlib.Path(name).suffixes)
327 format = ''.join(pathlib.Path(name).suffixes)
282 if len(format) > 0:
328 if len(format) > 0:
283 params_dict['format'].append(format.upper()[1:])
329 params_dict['format'].append(format.upper()[1:])
284 else:
330 else:
285 params_dict['format'].append('')
331 params_dict['format'].append('')
286 else:
332 else:
287 return "ERROR:: There aren't files in this directory"
333 return "ERROR:: There aren't files in this directory"
288 else:
334 else:
289 return 'ERROR:: Directory "%s" does not exist' % (path_files)
335 return 'ERROR:: Directory "%s" does not exist' % (path_files)
290 else:
336 else:
291 return 'ERROR:: "path_files" must be a str or list'
337 return 'ERROR:: "path_files" must be a str or list'
292 #------------------------------------------------------------#
338 #------------------------------------------------------------#
293 params_no_dict = {'package_id': dataset_id}
339 params_no_dict = {'package_id': dataset_id}
294 if type(date_files) is list:
340 if type(date_files) is list:
295 params_dict['file_date'] = date_files
341 params_dict['file_date'] = date_files
296 else:
342 else:
297 params_no_dict['file_date'] = date_files
343 params_no_dict['file_date'] = date_files
298
344
299 if type(type_files) is list:
345 if type(type_files) is list:
300 params_dict['voc_file_type'] = type_files
346 params_dict['voc_file_type'] = type_files
301 else:
347 else:
302 params_no_dict['voc_file_type'] = type_files
348 params_no_dict['voc_file_type'] = type_files
303
349
304 for key1, value1 in kwargs.items():
350 for key1, value1 in kwargs.items():
305 if not key1 in params_dict and not key1 in params_no_dict and key1 != 'others':
351 if not key1 in params_dict and not key1 in params_no_dict and key1 != 'others':
306 if type(value1) is list:
352 if type(value1) is list:
307 params_dict[key1] = value1
353 params_dict[key1] = value1
308 else:
354 else:
309 params_no_dict[key1] = value1
355 params_no_dict[key1] = value1
310 #------------------------------------------#
356 #------------------------------------------#
311 if not 'others' in kwargs:
357 if not 'others' in kwargs:
312 params_no_dict['others'] = ''
358 params_no_dict['others'] = ''
313 else:
359 else:
314 if isinstance(kwargs['others'], tuple):
360 if isinstance(kwargs['others'], tuple):
315 params_dict['others'] = [json.dumps(w) for w in kwargs['others']]
361 params_dict['others'] = [json.dumps(w) for w in kwargs['others']]
316 elif isinstance(kwargs['others'], list):
362 elif isinstance(kwargs['others'], list):
317 params_no_dict['others'] = json.dumps(kwargs['others'])
363 params_no_dict['others'] = json.dumps(kwargs['others'])
318 elif isinstance(kwargs['others'], str):
364 elif isinstance(kwargs['others'], str):
319 params_no_dict['others'] = kwargs['others']
365 params_no_dict['others'] = kwargs['others']
320 else:
366 else:
321 return 'ERROR:: "others" must be a tuple, list or str'
367 return 'ERROR:: "others" must be a tuple, list or str'
322 #------------------------------------------#
368 #------------------------------------------#
323 len_params_dict = []
369 len_params_dict = []
324 for value2 in params_dict.values():
370 for value2 in params_dict.values():
325 len_params_dict.append(len(value2))
371 len_params_dict.append(len(value2))
326
372
327 if len(list(set(len_params_dict))) > 1:
373 if len(list(set(len_params_dict))) > 1:
328 return 'ERROR:: All lists must be the same length: %s' % (len(params_dict['name']))
374 return 'ERROR:: All lists must be the same length: %s' % (len(params_dict['name']))
329 #------------------------------------------------------------#
375 #------------------------------------------------------------#
330 print('"{}" file(s) found >> uploading'.format(len(params_dict['name'])))
376 print('"{}" file(s) found >> uploading'.format(len(params_dict['name'])))
331 for v in range(len(params_dict['name'])):
377 for v in range(len(params_dict['name'])):
332 try:
378 try:
333 send = {}
379 send = {}
334 for key_dict, value_dict in params_dict.items():
380 for key_dict, value_dict in params_dict.items():
335 send[key_dict] = value_dict[v]
381 send[key_dict] = value_dict[v]
336 for key_no_dict, value_no_dict in params_no_dict.items():
382 for key_no_dict, value_no_dict in params_no_dict.items():
337 send[key_no_dict] = value_no_dict
383 send[key_no_dict] = value_no_dict
338
384
339 self.list.append(getattr(self.ckan.action, 'resource_create')(**send))
385 self.list.append(getattr(self.ckan.action, 'resource_create')(**send))
340 print('File #{} :: "{}" was uploaded successfully'.format(v+1, params_dict['name'][v]))
386 print('File #{} :: "{}" was uploaded successfully'.format(v+1, params_dict['name'][v]))
341 except:
387 except:
342 _, exc_value, _ = sys.exc_info()
388 _, exc_value, _ = sys.exc_info()
343 self.list.append(exc_value)
389 self.list.append(exc_value)
344 print('File #{} :: Error uploading "{}" file'.format(v+1, params_dict['name'][v]))
390 print('File #{} :: Error uploading "{}" file'.format(v+1, params_dict['name'][v]))
345 return self.list
391 return self.list
346 #------------------------------------------------------------#
392 #------------------------------------------------------------#
347
393
348 def show(self, type_option, id, **kwargs):
394 def show(self, type_option, id, **kwargs):
349 '''
395 '''
350 FINALIDAD:
396 FINALIDAD:
351 Funcion personalizada para una busqueda en especifico.
397 Funcion personalizada para una busqueda en especifico.
352
398
353 PARAMETROS DISPONIBLES:
399 PARAMETROS DISPONIBLES:
354 CONSULTAR: "GUIA DE SCRIPT.pdf"
400 CONSULTAR: "GUIA DE SCRIPT.pdf"
355
401
356 ESTRUCTURA:
402 ESTRUCTURA:
357 <access_name>.show(type_option = <class 'str'>, id = <class 'str'>, param_1 = <class 'param_1'>, ...)
403 <access_name>.show(type_option = <class 'str'>, id = <class 'str'>, param_1 = <class 'param_1'>, ...)
358 '''
404 '''
359 if type(type_option) is str:
405 if type(type_option) is str:
360 try:
406 try:
361 if type_option == 'dataset':
407 if type_option == 'dataset':
362 return getattr(self.ckan.action, 'package_show')(id=id, **kwargs)
408 return getattr(self.ckan.action, 'package_show')(id=id, **kwargs)
363 elif type_option == 'resource':
409 elif type_option == 'resource':
364 return getattr(self.ckan.action, 'resource_show')(id=id, **kwargs)
410 return getattr(self.ckan.action, 'resource_show')(id=id, **kwargs)
365 elif type_option == 'project':
411 elif type_option == 'project':
366 return getattr(self.ckan.action, 'organization_show')(id=id, **kwargs)
412 return getattr(self.ckan.action, 'organization_show')(id=id, **kwargs)
367 elif type_option == 'collaborator':
413 elif type_option == 'collaborator':
368 return getattr(self.ckan.action, 'package_collaborator_list_for_user')(id=id, **kwargs)
414 return getattr(self.ckan.action, 'package_collaborator_list_for_user')(id=id, **kwargs)
369 elif type_option == 'member':
415 elif type_option == 'member':
370 return getattr(self.ckan.action, 'organization_list_for_user')(id=id, **kwargs)
416 return getattr(self.ckan.action, 'organization_list_for_user')(id=id, **kwargs)
371 elif type_option == 'vocabulary':
417 elif type_option == 'vocabulary':
372 return getattr(self.ckan.action, 'vocabulary_show')(id=id, **kwargs)
418 return getattr(self.ckan.action, 'vocabulary_show')(id=id, **kwargs)
373 elif type_option == 'tag':
419 elif type_option == 'tag':
374 if not 'vocabulary_id' in kwargs:
420 if not 'vocabulary_id' in kwargs:
375 print('Missing "vocabulary_id" value: assume it is a free tag')
421 print('Missing "vocabulary_id" value: assume it is a free tag')
376 return getattr(self.ckan.action, 'tag_show')(id=id, **kwargs)
422 return getattr(self.ckan.action, 'tag_show')(id=id, **kwargs)
377 elif type_option == 'user':
423 elif type_option == 'user':
378 return getattr(self.ckan.action, 'user_show')(id=id, **kwargs)
424 return getattr(self.ckan.action, 'user_show')(id=id, **kwargs)
379 elif type_option == 'job':
425 elif type_option == 'job':
380 return getattr(self.ckan.action, 'job_show')(id=id, **kwargs)
426 return getattr(self.ckan.action, 'job_show')(id=id, **kwargs)
381 else:
427 else:
382 return 'ERROR:: "type_option = %s" is not accepted' % (type_option)
428 return 'ERROR:: "type_option = %s" is not accepted' % (type_option)
383 except:
429 except:
384 _, exc_value, _ = sys.exc_info()
430 _, exc_value, _ = sys.exc_info()
385 return exc_value
431 return exc_value
386 else:
432 else:
387 return 'ERROR:: "type_option" must be a str'
433 return 'ERROR:: "type_option" must be a str'
388
434
389 def search(self, type_option, query=None, **kwargs):
435 def search(self, type_option, query=None, **kwargs):
390 '''
436 '''
391 FINALIDAD:
437 FINALIDAD:
392 Funcion personalizada para busquedas que satisfagan algun criterio.
438 Funcion personalizada para busquedas que satisfagan algun criterio.
393
439
394 PARAMETROS DISPONIBLES:
440 PARAMETROS DISPONIBLES:
395 CONSULTAR: "GUIA DE SCRIPT.pdf"
441 CONSULTAR: "GUIA DE SCRIPT.pdf"
396
442
397 ESTRUCTURA:
443 ESTRUCTURA:
398 <access_name>.search(type_option = <class 'str'>, query = <class 'dict'>, param_1 = <class 'param_1'>, ...)
444 <access_name>.search(type_option = <class 'str'>, query = <class 'dict'>, param_1 = <class 'param_1'>, ...)
399 '''
445 '''
400 if type(type_option) is str:
446 if type(type_option) is str:
401 try:
447 try:
402 if type_option == 'dataset':
448 if type_option == 'dataset':
403 key_replace = ['fq', 'fq_list', 'include_private']
449 key_replace = ['fq', 'fq_list', 'include_private']
404 key_point = ['facet_mincount', 'facet_limit', 'facet_field']
450 key_point = ['facet_mincount', 'facet_limit', 'facet_field']
405 for key1, value1 in kwargs.items():
451 for key1, value1 in kwargs.items():
406 if not key1 in key_replace:
452 if not key1 in key_replace:
407 if key1 in key_point:
453 if key1 in key_point:
408 self.dict[key1.replace('_', '.')] = value1
454 self.dict[key1.replace('_', '.')] = value1
409 else:
455 else:
410 self.dict[key1] = value1
456 self.dict[key1] = value1
411
457
412 if query is not None:
458 if query is not None:
413 if type(query) is dict:
459 if type(query) is dict:
414 self.dict['fq_list'] = []
460 self.dict['fq_list'] = []
415 #NUM_RESOURCES_MIN / NUM_RESOURCES_MAX
461 #NUM_RESOURCES_MIN / NUM_RESOURCES_MAX
416 #----------------------------------------------------#
462 #----------------------------------------------------#
417 if 'dataset_start_date' in query:
463 if 'dataset_start_date' in query:
418 if type(query['dataset_start_date']) is str:
464 if type(query['dataset_start_date']) is str:
419 try:
465 try:
420 datetime.strptime(query['dataset_start_date'], '%Y-%m-%d')
466 datetime.strptime(query['dataset_start_date'], '%Y-%m-%d')
421 if len(query['dataset_start_date']) != 10:
467 if len(query['dataset_start_date']) != 10:
422 return '"dataset_start_date", must be: <YYYY-MM-DD>'
468 return '"dataset_start_date", must be: <YYYY-MM-DD>'
423 self.dict['fq_list'].append('dataset_start_date:"'+query['dataset_start_date']+'"')
469 self.dict['fq_list'].append('dataset_start_date:"'+query['dataset_start_date']+'"')
424 self.list.append('dataset_start_date')
470 self.list.append('dataset_start_date')
425 except:
471 except:
426 return '"dataset_start_date" incorrect: "%s"' % (query['dataset_start_date'])
472 return '"dataset_start_date" incorrect: "%s"' % (query['dataset_start_date'])
427 else:
473 else:
428 return '"dataset_start_date" must be <str>'
474 return '"dataset_start_date" must be <str>'
429 #----------------------------------------------------#
475 #----------------------------------------------------#
430 if 'dataset_end_date' in query:
476 if 'dataset_end_date' in query:
431 if type(query['dataset_end_date']) is str:
477 if type(query['dataset_end_date']) is str:
432 try:
478 try:
433 datetime.strptime(query['dataset_end_date'], '%Y-%m-%d')
479 datetime.strptime(query['dataset_end_date'], '%Y-%m-%d')
434 if len(query['dataset_end_date']) != 10:
480 if len(query['dataset_end_date']) != 10:
435 return '"dataset_end_date", must be: <YYYY-MM-DD>'
481 return '"dataset_end_date", must be: <YYYY-MM-DD>'
436
482
437 if 'dataset_start_date' in query:
483 if 'dataset_start_date' in query:
438 if query['dataset_start_date'] > query['dataset_end_date']:
484 if query['dataset_start_date'] > query['dataset_end_date']:
439 return '"dataset_end_date" must be greater than "dataset_start_date"'
485 return '"dataset_end_date" must be greater than "dataset_start_date"'
440
486
441 self.dict['fq_list'].append('dataset_end_date:"'+query['dataset_end_date']+'"')
487 self.dict['fq_list'].append('dataset_end_date:"'+query['dataset_end_date']+'"')
442 self.list.append('dataset_end_date')
488 self.list.append('dataset_end_date')
443 except:
489 except:
444 return '"dataset_end_date" incorrect: "%s"' % (query['dataset_end_date'])
490 return '"dataset_end_date" incorrect: "%s"' % (query['dataset_end_date'])
445 else:
491 else:
446 return '"dataset_end_date" must be <str>'
492 return '"dataset_end_date" must be <str>'
447 #----------------------------------------------------#
493 #----------------------------------------------------#
448 for key, value in query.items():
494 for key, value in query.items():
449 if value is not None and not key in self.list:
495 if value is not None and not key in self.list:
450 self.dict['fq_list'].append(str(key)+':"'+str(value)+'"')
496 self.dict['fq_list'].append(str(key)+':"'+str(value)+'"')
451 else:
497 else:
452 return '"query" must be <dict>'
498 return '"query" must be <dict>'
453
499
454 return getattr(self.ckan.action, 'package_search')(include_private=True, **self.dict)
500 return getattr(self.ckan.action, 'package_search')(include_private=True, **self.dict)
455
501
456 elif type_option == 'resource':
502 elif type_option == 'resource':
457 for key1, value1 in kwargs.items():
503 for key1, value1 in kwargs.items():
458 if key1 != 'fields':
504 if key1 != 'fields':
459 self.dict[key1] = value1
505 self.dict[key1] = value1
460
506
461 if query is not None:
507 if query is not None:
462 if type(query) is dict:
508 if type(query) is dict:
463 #----------------------------------------------------#
509 #----------------------------------------------------#
464 if 'file_date_min' in query:
510 if 'file_date_min' in query:
465 if type(query['file_date_min']) is str:
511 if type(query['file_date_min']) is str:
466 try:
512 try:
467 datetime.strptime(query['file_date_min'], '%Y-%m-%d')
513 datetime.strptime(query['file_date_min'], '%Y-%m-%d')
468 if len(query['file_date_min']) != 10:
514 if len(query['file_date_min']) != 10:
469 return '"file_date_min", must be: <YYYY-MM-DD>'
515 return '"file_date_min", must be: <YYYY-MM-DD>'
470 except:
516 except:
471 return '"file_date_min" incorrect: "%s"' % (query['file_date_min'])
517 return '"file_date_min" incorrect: "%s"' % (query['file_date_min'])
472 else:
518 else:
473 return '"file_date_min" must be <str>'
519 return '"file_date_min" must be <str>'
474 #----------------------------------------------------#
520 #----------------------------------------------------#
475 if 'file_date_max' in query:
521 if 'file_date_max' in query:
476 if type(query['file_date_max']) is str:
522 if type(query['file_date_max']) is str:
477 try:
523 try:
478 datetime.strptime(query['file_date_max'], '%Y-%m-%d')
524 datetime.strptime(query['file_date_max'], '%Y-%m-%d')
479 if len(query['file_date_max']) != 10:
525 if len(query['file_date_max']) != 10:
480 return '"file_date_max", must be: <YYYY-MM-DD>'
526 return '"file_date_max", must be: <YYYY-MM-DD>'
481
527
482 if 'file_date_min' in query:
528 if 'file_date_min' in query:
483 if query['file_date_min'] > query['file_date_max']:
529 if query['file_date_min'] > query['file_date_max']:
484 return '"file_date_max" must be greater than "file_date_min"'
530 return '"file_date_max" must be greater than "file_date_min"'
485 except:
531 except:
486 return '"file_date_max" incorrect: "%s"' % (query['file_date_max'])
532 return '"file_date_max" incorrect: "%s"' % (query['file_date_max'])
487 else:
533 else:
488 return '"file_date_max" must be <str>'
534 return '"file_date_max" must be <str>'
489 #----------------------------------------------------#
535 #----------------------------------------------------#
490 self.dict['query'] = query
536 self.dict['query'] = query
491 else:
537 else:
492 return '"query" must be <dict>'
538 return '"query" must be <dict>'
493 return getattr(self.ckan.action, 'resources_search')(**self.dict)
539 return getattr(self.ckan.action, 'resources_search')(**self.dict)
494
540
495 elif type_option == 'tag':
541 elif type_option == 'tag':
496 for key1, value1 in kwargs.items():
542 for key1, value1 in kwargs.items():
497 if key1 != 'fields':
543 if key1 != 'fields':
498 self.dict[key1] = value1
544 self.dict[key1] = value1
499
545
500 if not 'vocabulary_id' in kwargs:
546 if not 'vocabulary_id' in kwargs:
501 print('Missing "vocabulary_id" value: tags that don’t belong to any vocabulary')
547 print('Missing "vocabulary_id" value: tags that don’t belong to any vocabulary')
502 else:
548 else:
503 print('Only tags that belong to "{}" vocabulary'.format(kwargs['vocabulary_id']))
549 print('Only tags that belong to "{}" vocabulary'.format(kwargs['vocabulary_id']))
504
550
505 if query is not None:
551 if query is not None:
506 if type(query) is dict:
552 if type(query) is dict:
507 if 'search' in query:
553 if 'search' in query:
508 if type(query['search']) is list or type(query['search']) is str:
554 if type(query['search']) is list or type(query['search']) is str:
509 self.dict['query'] = query['search']
555 self.dict['query'] = query['search']
510 else:
556 else:
511 return '"search" must be <list> or <str>'
557 return '"search" must be <list> or <str>'
512 else:
558 else:
513 return '"query" must be <dict>'
559 return '"query" must be <dict>'
514 return getattr(self.ckan.action, 'tag_search')(**self.dict)
560 return getattr(self.ckan.action, 'tag_search')(**self.dict)
515
561
516 else:
562 else:
517 return 'ERROR:: "type_option = %s" is not accepted' % (type_option)
563 return 'ERROR:: "type_option = %s" is not accepted' % (type_option)
518
564
519 except:
565 except:
520 _, exc_value, _ = sys.exc_info()
566 _, exc_value, _ = sys.exc_info()
521 return exc_value
567 return exc_value
522 else:
568 else:
523 return 'ERROR:: "type_option" must be <str>'
569 return 'ERROR:: "type_option" must be <str>'
524
570
525 def create(self, type_option, select=None, **kwargs):
571 def create(self, type_option, select=None, **kwargs):
526 '''
572 '''
527 FINALIDAD:
573 FINALIDAD:
528 Funcion personalizada para crear.
574 Funcion personalizada para crear.
529
575
530 PARAMETROS DISPONIBLES:
576 PARAMETROS DISPONIBLES:
531 CONSULTAR: "GUIA DE SCRIPT.pdf"
577 CONSULTAR: "GUIA DE SCRIPT.pdf"
532
578
533 ESTRUCTURA:
579 ESTRUCTURA:
534 <access_name>.create(type_option = <class 'str'>, param_1 = <class 'param_1'>, ...)
580 <access_name>.create(type_option = <class 'str'>, param_1 = <class 'param_1'>, ...)
535 '''
581 '''
536 if type(type_option) is str:
582 if type(type_option) is str:
537 try:
583 try:
538 if type_option == 'dataset':
584 if type_option == 'dataset':
539 return getattr(self.ckan.action, 'package_create')(**kwargs)
585 return getattr(self.ckan.action, 'package_create')(**kwargs)
540 elif type_option == 'project':
586 elif type_option == 'project':
541 return getattr(self.ckan.action, 'organization_create')(**kwargs)
587 return getattr(self.ckan.action, 'organization_create')(**kwargs)
542 elif type_option == 'member':
588 elif type_option == 'member':
543 return getattr(self.ckan.action, 'organization_member_create')(**kwargs)
589 return getattr(self.ckan.action, 'organization_member_create')(**kwargs)
544 elif type_option == 'collaborator':
590 elif type_option == 'collaborator':
545 return getattr(self.ckan.action, 'package_collaborator_create')(**kwargs)
591 return getattr(self.ckan.action, 'package_collaborator_create')(**kwargs)
546 elif type_option == 'vocabulary':
592 elif type_option == 'vocabulary':
547 return getattr(self.ckan.action, 'vocabulary_create')(**kwargs)
593 return getattr(self.ckan.action, 'vocabulary_create')(**kwargs)
548 elif type_option == 'tag':
594 elif type_option == 'tag':
549 return getattr(self.ckan.action, 'tag_create')(**kwargs)
595 return getattr(self.ckan.action, 'tag_create')(**kwargs)
550 elif type_option == 'user':
596 elif type_option == 'user':
551 return getattr(self.ckan.action, 'user_create')(**kwargs)
597 return getattr(self.ckan.action, 'user_create')(**kwargs)
552 elif type_option == 'views':
598 elif type_option == 'views':
553 if 'resource' == select:
599 if 'resource' == select:
554 self.list = ['package']
600 self.list = ['package']
555 for key1, value1 in kwargs.items():
601 for key1, value1 in kwargs.items():
556 if not key1 in self.list:
602 if not key1 in self.list:
557 self.dict[key1] = value1
603 self.dict[key1] = value1
558 return getattr(self.ckan.action, 'resource_create_default_resource_views')(**self.dict)
604 return getattr(self.ckan.action, 'resource_create_default_resource_views')(**self.dict)
559 elif 'dataset' == select:
605 elif 'dataset' == select:
560 return getattr(self.ckan.action, 'package_create_default_resource_views')(**kwargs)
606 return getattr(self.ckan.action, 'package_create_default_resource_views')(**kwargs)
561 else:
607 else:
562 return 'ERROR:: "select = %s" is not accepted' % (select)
608 return 'ERROR:: "select = %s" is not accepted' % (select)
563 else:
609 else:
564 return 'ERROR:: "type_option = %s" is not accepted' % (type_option)
610 return 'ERROR:: "type_option = %s" is not accepted' % (type_option)
565 except:
611 except:
566 _, exc_value, _ = sys.exc_info()
612 _, exc_value, _ = sys.exc_info()
567 return exc_value
613 return exc_value
568 else:
614 else:
569 return 'ERROR:: "type_option" must be <str>'
615 return 'ERROR:: "type_option" must be <str>'
570
616
571 def patch(self, type_option, **kwargs):
617 def patch(self, type_option, **kwargs):
572 '''
618 '''
573 FINALIDAD:
619 FINALIDAD:
574 Funciones personalizadas para actualizar
620 Funciones personalizadas para actualizar
575
621
576 PARAMETROS DISPONIBLES:
622 PARAMETROS DISPONIBLES:
577 CONSULTAR: "GUIA DE SCRIPT.pdf"
623 CONSULTAR: "GUIA DE SCRIPT.pdf"
578
624
579 ESTRUCTURA:
625 ESTRUCTURA:
580 <access_name>.patch(type_option = <class 'str'>, param_1 = <class 'param_1'>, ...)
626 <access_name>.patch(type_option = <class 'str'>, param_1 = <class 'param_1'>, ...)
581 '''
627 '''
582 if type(type_option) is str:
628 if type(type_option) is str:
583 try:
629 try:
584 if type_option == 'dataset':
630 if type_option == 'dataset':
585 return getattr(self.ckan.action, 'package_patch')(**kwargs)
631 return getattr(self.ckan.action, 'package_patch')(**kwargs)
586 elif type_option == 'project':
632 elif type_option == 'project':
587 return getattr(self.ckan.action, 'organization_patch')(**kwargs)
633 return getattr(self.ckan.action, 'organization_patch')(**kwargs)
588 elif type_option == 'resource':
634 elif type_option == 'resource':
589 return getattr(self.ckan.action, 'resource_patch')(**kwargs)
635 return getattr(self.ckan.action, 'resource_patch')(**kwargs)
590 elif type_option == 'member':
636 elif type_option == 'member':
591 return getattr(self.ckan.action, 'organization_member_create')(**kwargs)
637 return getattr(self.ckan.action, 'organization_member_create')(**kwargs)
592 elif type_option == 'collaborator':
638 elif type_option == 'collaborator':
593 return getattr(self.ckan.action, 'package_collaborator_create')(**kwargs)
639 return getattr(self.ckan.action, 'package_collaborator_create')(**kwargs)
594 else:
640 else:
595 return 'ERROR:: "type_option = %s" is not accepted' % (type_option)
641 return 'ERROR:: "type_option = %s" is not accepted' % (type_option)
596 except:
642 except:
597 _, exc_value, _ = sys.exc_info()
643 _, exc_value, _ = sys.exc_info()
598 return exc_value
644 return exc_value
599 else:
645 else:
600 return 'ERROR:: "type_option" must be <str>'
646 return 'ERROR:: "type_option" must be <str>'
601
647
602 def delete(self, type_option, select=None, **kwargs):
648 def delete(self, type_option, select=None, **kwargs):
603 '''
649 '''
604 FINALIDAD:
650 FINALIDAD:
605 Función personalizada para eliminar y/o purgar.
651 Función personalizada para eliminar y/o purgar.
606
652
607 PARAMETROS DISPONIBLES:
653 PARAMETROS DISPONIBLES:
608 CONSULTAR: "GUIA DE SCRIPT.pdf"
654 CONSULTAR: "GUIA DE SCRIPT.pdf"
609
655
610 ESTRUCTURA:
656 ESTRUCTURA:
611 <access_name>.delete(type_option = <class 'str'>, param_1 = <class 'param_1'>, ...)
657 <access_name>.delete(type_option = <class 'str'>, param_1 = <class 'param_1'>, ...)
612 '''
658 '''
613 if type(type_option) is str:
659 if type(type_option) is str:
614 try:
660 try:
615 if type_option == 'dataset':
661 if type_option == 'dataset':
616 if select is None:
662 if select is None:
617 return 'ERROR:: "select" must not be "None"'
663 return 'ERROR:: "select" must not be "None"'
618 else:
664 else:
619 if 'delete' == select:
665 if 'delete' == select:
620 return getattr(self.ckan.action, 'package_delete')(**kwargs)
666 return getattr(self.ckan.action, 'package_delete')(**kwargs)
621 elif 'purge' == select:
667 elif 'purge' == select:
622 return getattr(self.ckan.action, 'dataset_purge')(**kwargs)
668 return getattr(self.ckan.action, 'dataset_purge')(**kwargs)
623 else:
669 else:
624 return 'ERROR:: "select = %s" is not accepted' % (select)
670 return 'ERROR:: "select = %s" is not accepted' % (select)
625 elif type_option == 'project':
671 elif type_option == 'project':
626 if select is None:
672 if select is None:
627 return 'ERROR:: "select" must not be "None"'
673 return 'ERROR:: "select" must not be "None"'
628 else:
674 else:
629 if 'delete' == select:
675 if 'delete' == select:
630 return getattr(self.ckan.action, 'organization_delete')(**kwargs)
676 return getattr(self.ckan.action, 'organization_delete')(**kwargs)
631 elif 'purge' == select:
677 elif 'purge' == select:
632 return getattr(self.ckan.action, 'organization_purge')(**kwargs)
678 return getattr(self.ckan.action, 'organization_purge')(**kwargs)
633 else:
679 else:
634 return 'ERROR:: "select = %s" is not accepted' % (select)
680 return 'ERROR:: "select = %s" is not accepted' % (select)
635 elif type_option == 'resource':
681 elif type_option == 'resource':
636 return getattr(self.ckan.action, 'resource_delete')(**kwargs)
682 return getattr(self.ckan.action, 'resource_delete')(**kwargs)
637 elif type_option == 'vocabulary':
683 elif type_option == 'vocabulary':
638 return getattr(self.ckan.action, 'vocabulary_delete')(**kwargs)
684 return getattr(self.ckan.action, 'vocabulary_delete')(**kwargs)
639 elif type_option == 'tag':
685 elif type_option == 'tag':
640 return getattr(self.ckan.action, 'tag_delete')(**kwargs)
686 return getattr(self.ckan.action, 'tag_delete')(**kwargs)
641 elif type_option == 'user':
687 elif type_option == 'user':
642 return getattr(self.ckan.action, 'user_delete')(**kwargs)
688 return getattr(self.ckan.action, 'user_delete')(**kwargs)
643 else:
689 else:
644 return 'ERROR:: "type_option = %s" is not accepted' % (type_option)
690 return 'ERROR:: "type_option = %s" is not accepted' % (type_option)
645 except:
691 except:
646 _, exc_value, _ = sys.exc_info()
692 _, exc_value, _ = sys.exc_info()
647 return exc_value
693 return exc_value
648 else:
694 else:
649 return 'ERROR:: "type_option" must be <str>'
695 return 'ERROR:: "type_option" must be <str>'
650
696
651 def f_status_note(self, total, result, path):
697 def f_status_note(self, total, result, path):
652 file_txt = open(path+'status_note.txt', 'w')
698 file_txt = open(path+'status_note.txt', 'w')
653 file_txt = open(path+'status_note.txt', 'a')
699 file_txt = open(path+'status_note.txt', 'a')
654
700
655 file_txt.write('DOWNLOADED FILE(S): "%s"' % (len(result['name'])))
701 file_txt.write('DOWNLOADED FILE(S): "%s"' % (len(result['name'])))
656 file_txt.write(''+ os.linesep)
702 file_txt.write(''+ os.linesep)
657 for u in result['name']:
703 for u in result['name']:
658 file_txt.write(' - '+ u + os.linesep)
704 file_txt.write(' - '+ u + os.linesep)
659 file_txt.write(''+ os.linesep)
705 file_txt.write(''+ os.linesep)
660
706
661 file_txt.write('FAILED FILE(S): "%s"' % (len(total['name'])-len(result['name'])))
707 file_txt.write('FAILED FILE(S): "%s"' % (len(total['name'])-len(result['name'])))
662 file_txt.write(''+ os.linesep)
708 file_txt.write(''+ os.linesep)
663 if len(total['name'])-len(result['name']) != 0:
709 if len(total['name'])-len(result['name']) != 0:
664 for u in total['name']:
710 for u in total['name']:
665 if not u in result['name']:
711 if not u in result['name']:
666 file_txt.write(' - '+ u + os.linesep)
712 file_txt.write(' - '+ u + os.linesep)
667 else:
713 else:
668 file_txt.write(' "None"'+ os.linesep)
714 file_txt.write(' "None"'+ os.linesep)
669
715
670 def f_name(self, name_dataset, ext, tempdir):
716 def f_name(self, name_dataset, ext, tempdir):
671 while self.check:
717 while self.check:
672 self.str = ''
718 self.str = ''
673 if self.cont == 0:
719 if self.cont == 0:
674 if os.path.exists(tempdir + name_dataset + ext):
720 if os.path.exists(tempdir + name_dataset + ext):
675 self.str = name_dataset+'('+str(self.cont+1)+')'+ext
721 self.str = name_dataset+'('+str(self.cont+1)+')'+ext
676 else:
722 else:
677 self.check = self.check * 0
723 self.check = self.check * 0
678 self.str = name_dataset + ext
724 self.str = name_dataset + ext
679 else:
725 else:
680 if not os.path.exists(tempdir + name_dataset+'('+str(self.cont)+')'+ext):
726 if not os.path.exists(tempdir + name_dataset+'('+str(self.cont)+')'+ext):
681 self.check = self.check * 0
727 self.check = self.check * 0
682 self.str = name_dataset+'('+str(self.cont)+')'+ ext
728 self.str = name_dataset+'('+str(self.cont)+')'+ ext
683 self.cont = self.cont+1
729 self.cont = self.cont+1
684 return self.str
730 return self.str
685
731
686 def f_zipdir(self, path, ziph, zip_name):
732 def f_zipdir(self, path, ziph, zip_name):
687 for root, _, files in os.walk(path):
733 for root, _, files in os.walk(path):
688 print('.....')
734 print('.....')
689 print('Creating: "{}" >>'.format(zip_name))
735 print('Creating: "{}" >>'.format(zip_name))
690 for __file in tqdm(iterable=files, total=len(files)):
736 for __file in tqdm(iterable=files, total=len(files)):
691 new_dir = os.path.relpath(os.path.join(root, __file), os.path.join(path, '..'))
737 new_dir = os.path.relpath(os.path.join(root, __file), os.path.join(path, '..'))
692 ziph.write(os.path.join(root, __file), new_dir)
738 ziph.write(os.path.join(root, __file), new_dir)
693 print('Created >>')
739 print('Created >>')
694
740
695 def download_by_step(self, response, tempdir_name):
741 def download_by_step(self, response, tempdir_name):
696 try:
742 try:
697 with requests.get(response['url'], stream=True, headers={'Authorization': self.Authorization}) as resp:
743 with requests.get(response['url'], stream=True, headers={'Authorization': self.Authorization}) as resp:
698 if resp.status_code == 200:
744 if resp.status_code == 200:
699 with open(tempdir_name+response['name'], 'wb') as file:
745 with open(tempdir_name+response['name'], 'wb') as file:
700 for chunk in resp.iter_content(chunk_size = self.chunk_size):
746 for chunk in resp.iter_content(chunk_size = self.chunk_size):
701 if chunk:
747 if chunk:
702 file.write(chunk)
748 file.write(chunk)
703 except requests.exceptions.RequestException:
749 except requests.exceptions.RequestException:
704 pass
750 pass
705
751
706 def download_files(self, **kwargs):
752 def download_files(self, **kwargs):
707 '''
753 '''
708 FINALIDAD:
754 FINALIDAD:
709 Funcion personalizada para la descarga de archivos existentes de un dataset.
755 Funcion personalizada para la descarga de archivos existentes de un dataset.
710
756
711 PARAMETROS DISPONIBLES:
757 PARAMETROS DISPONIBLES:
712 CONSULTAR: "GUIA DE SCRIPT.pdf"
758 CONSULTAR: "GUIA DE SCRIPT.pdf"
713
759
714 ESTRUCTURA:
760 ESTRUCTURA:
715 <access_name>.download_files(id = <class 'str'>, param_1 = <class 'param_1'>, ...)
761 <access_name>.download_files(id = <class 'str'>, param_1 = <class 'param_1'>, ...)
716 '''
762 '''
717 dict_local = {}
763 dict_local = {}
718 #----------------------------------------------#
764 #----------------------------------------------#
719 if 'zip' in kwargs:
765 if 'zip' in kwargs:
720 if type(kwargs['zip']) is not bool:
766 if type(kwargs['zip']) is not bool:
721 return 'ERROR:: "zip" must be: <class "bool">'
767 return 'ERROR:: "zip" must be: <class "bool">'
722 else:
768 else:
723 dict_local['zip'] = kwargs['zip']
769 dict_local['zip'] = kwargs['zip']
724 else:
770 else:
725 dict_local['zip'] = False
771 dict_local['zip'] = False
726 #----------------------------------------------#
772 #----------------------------------------------#
727 if 'status_note' in kwargs:
773 if 'status_note' in kwargs:
728 if type(kwargs['status_note']) is not bool:
774 if type(kwargs['status_note']) is not bool:
729 return 'ERROR:: "status_note" must be: <class "bool">'
775 return 'ERROR:: "status_note" must be: <class "bool">'
730 else:
776 else:
731 dict_local['status_note'] = kwargs['status_note']
777 dict_local['status_note'] = kwargs['status_note']
732 else:
778 else:
733 dict_local['status_note'] = False
779 dict_local['status_note'] = False
734 #----------------------------------------------#
780 #----------------------------------------------#
735 if 'path' in kwargs:
781 if 'path' in kwargs:
736 if type(kwargs['path']) is str:
782 if type(kwargs['path']) is str:
737 if os.path.isdir(kwargs['path']) == False:
783 if os.path.isdir(kwargs['path']) == False:
738 return 'ERROR:: "path" does not exist'
784 return 'ERROR:: "path" does not exist'
739 else:
785 else:
740 if kwargs['path'][-1:] != self.separator:
786 if kwargs['path'][-1:] != self.separator:
741 dict_local['path'] = kwargs['path']+self.separator
787 dict_local['path'] = kwargs['path']+self.separator
742 else:
788 else:
743 dict_local['path'] = kwargs['path']
789 dict_local['path'] = kwargs['path']
744
790
745 txt = dict_local['path']+datetime.now().strftime("%Y_%m_%d_%H_%M_%S_%f")+'.txt'
791 txt = dict_local['path']+datetime.now().strftime("%Y_%m_%d_%H_%M_%S_%f")+'.txt'
746 if int(platform.python_version()[0]) == 3:
792 if int(platform.python_version()[0]) == 3:
747 try:
793 try:
748 file_txt = open(txt, 'w')
794 file_txt = open(txt, 'w')
749 file_txt.close()
795 file_txt.close()
750 os.remove(txt)
796 os.remove(txt)
751 except PermissionError:
797 except PermissionError:
752 return 'ERROR:: Access denied, you are not authorized to write files: "%s"' % (dict_local['path'])
798 return 'ERROR:: Access denied, you are not authorized to write files: "%s"' % (dict_local['path'])
753 else:
799 else:
754 try:
800 try:
755 file_txt = open(txt, 'w')
801 file_txt = open(txt, 'w')
756 file_txt.close()
802 file_txt.close()
757 os.remove(txt)
803 os.remove(txt)
758 except:
804 except:
759 return 'ERROR:: Access denied, you are not authorized to write files: "%s"' % (dict_local['path'])
805 return 'ERROR:: Access denied, you are not authorized to write files: "%s"' % (dict_local['path'])
760 else:
806 else:
761 return 'ERROR:: "path" must be: <class "str">'
807 return 'ERROR:: "path" must be: <class "str">'
762 else:
808 else:
763 dict_local['path'] = ''
809 dict_local['path'] = ''
764 #----------------------------------------------#
810 #----------------------------------------------#
765 for key, value in kwargs.items():
811 for key, value in kwargs.items():
766 if not key in dict_local:
812 if not key in dict_local:
767 self.dict[key] = value
813 self.dict[key] = value
768 try:
814 try:
769 response = getattr(self.ckan.action, 'url_resources')(**self.dict)
815 response = getattr(self.ckan.action, 'url_resources')(**self.dict)
770 except:
816 except:
771 _, exc_value, _ = sys.exc_info()
817 _, exc_value, _ = sys.exc_info()
772 return exc_value
818 return exc_value
773
819
774 if len(response) != 0:
820 if len(response) != 0:
775 #--------------TEMP PATH---------------#
821 #--------------TEMP PATH---------------#
776 if dict_local['zip']:
822 if dict_local['zip']:
777 tempdir = tempfile.mkdtemp(prefix=kwargs['id']+'-')+self.separator
823 tempdir = tempfile.mkdtemp(prefix=kwargs['id']+'-')+self.separator
778 os.mkdir(tempdir+kwargs['id'])
824 os.mkdir(tempdir+kwargs['id'])
779 dir_name = tempdir + kwargs['id'] + self.separator
825 dir_name = tempdir + kwargs['id'] + self.separator
780 else:
826 else:
781 dir = self.f_name(kwargs['id'], '', dict_local['path'])
827 dir = self.f_name(kwargs['id'], '', dict_local['path'])
782 os.mkdir(dict_local['path'] + dir)
828 os.mkdir(dict_local['path'] + dir)
783 dir_name = dict_local['path'] + dir + self.separator
829 dir_name = dict_local['path'] + dir + self.separator
784 #-----------DOWNLOAD FILES-------------#
830 #-----------DOWNLOAD FILES-------------#
785 print('.....')
831 print('.....')
786 print('Downloading "{}" file(s) >>'.format(len(response)))
832 print('Downloading "{}" file(s) >>'.format(len(response)))
787 name_total = {'name': []}
833 name_total = {'name': []}
788 with concurrent.futures.ThreadPoolExecutor() as executor:
834 with concurrent.futures.ThreadPoolExecutor() as executor:
789 for u in tqdm(iterable=response, total=len(response)):
835 for u in tqdm(iterable=response, total=len(response)):
790 name_total['name'].append(u['name'])
836 name_total['name'].append(u['name'])
791 executor.submit(self.download_by_step, u, dir_name)
837 executor.submit(self.download_by_step, u, dir_name)
792 name_check = {}
838 name_check = {}
793 name_check['name'] = [f for f in os.listdir(dir_name) if os.path.isfile(os.path.join(dir_name, f))]
839 name_check['name'] = [f for f in os.listdir(dir_name) if os.path.isfile(os.path.join(dir_name, f))]
794 print('"{}" downloaded file(s) successfully >>'.format(len(name_check['name'])))
840 print('"{}" downloaded file(s) successfully >>'.format(len(name_check['name'])))
795 #--------------------------------------#
841 #--------------------------------------#
796 if len(name_check['name']) != 0:
842 if len(name_check['name']) != 0:
797 #----------Status Note---------#
843 #----------Status Note---------#
798 if dict_local['status_note']:
844 if dict_local['status_note']:
799 print('.....')
845 print('.....')
800 print('Creating: "status_note.txt" >>')
846 print('Creating: "status_note.txt" >>')
801 self.f_status_note(name_total, name_check, dir_name)
847 self.f_status_note(name_total, name_check, dir_name)
802 print('Created>>')
848 print('Created>>')
803 #----------ZIP CREATE----------#
849 #----------ZIP CREATE----------#
804 if dict_local['zip']:
850 if dict_local['zip']:
805 zip_name = self.f_name(kwargs['id'], '.zip', dict_local['path'])
851 zip_name = self.f_name(kwargs['id'], '.zip', dict_local['path'])
806 ziph = zipfile.ZipFile(dict_local['path'] + zip_name, 'w', zipfile.ZIP_DEFLATED, allowZip64=True)
852 ziph = zipfile.ZipFile(dict_local['path'] + zip_name, 'w', zipfile.ZIP_DEFLATED, allowZip64=True)
807 self.f_zipdir(dir_name, ziph, zip_name)
853 self.f_zipdir(dir_name, ziph, zip_name)
808 ziph.close()
854 ziph.close()
809 #Delete Temporal Path
855 #Delete Temporal Path
810 if os.path.exists(tempdir[:-1]):
856 if os.path.exists(tempdir[:-1]):
811 shutil.rmtree(tempdir[:-1])
857 shutil.rmtree(tempdir[:-1])
812 #------------------------------#
858 #------------------------------#
813 print('.....')
859 print('.....')
814 return 'DOWNLOAD FINISHED'
860 return 'DOWNLOAD FINISHED'
815 else:
861 else:
816 #Delete Temporal Path
862 #Delete Temporal Path
817 if dict_local['zip']:
863 if dict_local['zip']:
818 if os.path.exists(tempdir[:-1]):
864 if os.path.exists(tempdir[:-1]):
819 shutil.rmtree(tempdir[:-1])
865 shutil.rmtree(tempdir[:-1])
820 else:
866 else:
821 if os.path.exists(dir_name[:-1]):
867 if os.path.exists(dir_name[:-1]):
822 shutil.rmtree(dir_name[:-1])
868 shutil.rmtree(dir_name[:-1])
823 return 'NO FILES WERE DOWNLOADED'
869 return 'NO FILES WERE DOWNLOADED'
824 else:
870 else:
825 return 'FILES NOT FOUND' No newline at end of file
871 return 'FILES NOT FOUND'
General Comments 0
You need to be logged in to leave comments. Login now