##// END OF EJS Templates
DATABASES » ckanext-jro » api-cliente
RSS
v2.9.2 :: Add 'tags' in search option - dataset
eynilupu -
r22:3ecf3c51fc50
parent child
Show More
Show file before | Show file after | Hide comments
@@ -1,502 +1,514
1 1 from ckanapi import RemoteCKAN
2 2 from datetime import datetime
3 3 from jrodb import download
4 4 from jrodb import resource
5 5 #from ckanapi.errors import NotAuthorized, NotFound, ValidationError, SearchQueryError, SearchError, CKANAPIError, ServerIncompatibleError
6 6 import sys
7 7 import platform
8 8 import os
9 9 import requests
10 10
11 11 class Api():
12 12 """
13 13 FINALIDAD:
14 14 Script para administrar y obtener la data del repositorio por medio de APIs.
15 15
16 16 REQUISITIOS PREVIOS:
17 17 - Paso 1: Tener "pip [Python 2]" o "pip3 [Python 3]" instalado:
18 18 - Paso 2: Instalar los siguientes paquetes:
19 19 En Python 2
20 20 - pip install -e git+http://intranet.igp.gob.pe:8082/DATABASES/ckanext-jro/api-cliente#egg=jrodb
21 21 En Python 3
22 22 - pip3 install -e git+http://intranet.igp.gob.pe:8082/DATABASES/ckanext-jro/api-cliente#egg=jrodb
23 23
24 24 FUNCIONES DISPONIBLES:
25 25 - action
26 26 - show
27 27 - search
28 28 - create
29 29 - patch
30 30 - delete
31 31 - download
32 32
33 33 EJEMPLOS:
34 34 #1:
35 35 with Api('http://demo.example.com', Authorization='#########') as <access_name>:
36 36 ... some operation(s) ...
37 37 #2:
38 38 <access_name> = Api('http://example.com', Authorization='#########')
39 39 ... some operation(s) ...
40 40 <access_name>.ckan.close()
41 41
42 42 REPORTAR ALGUN PROBLEMA:
43 43 Debe enviar un correo a eynilupu@igp.gob.pe detallando los siguientes pasos:
44 44 1) Correo para contactarlo
45 45 2) Descripcion del problema
46 46 3) ¿En que paso o seccion encontro el problema?
47 47 4) ¿Cual era el resultado que usted esperaba?
48 48 """
49 49 def __init__(self, url, Authorization=None, secure=True):
50 50 #-------- Check Secure -------#
51 51 self.verify = secure
52 52 if not secure and isinstance(secure, bool):
53 53 session = requests.Session()
54 54 session.verify = False
55 55 else:
56 56 session = None
57 57 #------------------------------#
58 58 self.url = url
59 59 ua = 'CKAN_JRO/2.9.2 (+'+str(self.url)+')'
60 60 #ua = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36'
61 61 self.ckan = RemoteCKAN(self.url, apikey=Authorization, user_agent=ua, session=session)
62 62 #self.ckan = RemoteCKAN(self.url, apikey=Authorization)
63 63 self.Authorization = Authorization
64 64 # Change for --> self.separator = os.sep
65 65 if platform.system() == 'Windows':
66 66 self.separator = '\\'
67 67 else:
68 68 self.separator = '/'
69 69
70 70 self.chunk_size = 1024
71 71 self.list = []
72 72 self.dict = {}
73 73 self.str = ''
74 74 self.check = 1
75 75 self.cont = 0
76 76
77 77 def __enter__(self):
78 78 return self
79 79
80 80 def __exit__(self, *args):
81 81 self.ckan.close()
82 82
83 83 def action(self, action, **kwargs):
84 84 """
85 85 FINALIDAD:
86 86 Funcion para llamar a las APIs disponibles
87 87
88 88 APIs DISPONIBLES:
89 89 CONSULTAR: "GUIA DE SCRIPT.pdf"
90 90
91 91 EJEMPLO:
92 92 <access_name>.action(<consuming API>, param_1 = <class 'param_1'>, ...)
93 93 """
94 94 #--------------- CASE: PACKAGE SEARCH ---------------#
95 95 if kwargs is not None:
96 96 if action == 'package_search':
97 97 self.list = ['facet_mincount', 'facet_limit', 'facet_field']
98 98 for facet in self.list:
99 99 if facet in kwargs:
100 100 kwargs[facet.replace('_', '.')] = kwargs[facet]
101 101 kwargs.pop(facet)
102 102 #----------------------------------------------------#
103 103 try:
104 104 return getattr(self.ckan.action, action)(**kwargs)
105 105 except:
106 106 _, exc_value, _ = sys.exc_info()
107 107 return exc_value
108 108
109 109 def show(self, type_option, id, **kwargs):
110 110 '''
111 111 FINALIDAD:
112 112 Funcion personalizada para una busqueda en especifico.
113 113
114 114 PARAMETROS DISPONIBLES:
115 115 CONSULTAR: "GUIA DE SCRIPT.pdf"
116 116
117 117 ESTRUCTURA:
118 118 <access_name>.show(type_option = <class 'str'>, id = <class 'str'>, param_1 = <class 'param_1'>, ...)
119 119 '''
120 120 if type(type_option) is str:
121 121 try:
122 122 if type_option == 'dataset':
123 123 return getattr(self.ckan.action, 'package_show')(id=id, **kwargs)
124 124 elif type_option == 'resource':
125 125 return getattr(self.ckan.action, 'resource_show')(id=id, **kwargs)
126 126 elif type_option == 'project':
127 127 return getattr(self.ckan.action, 'organization_show')(id=id, **kwargs)
128 128 elif type_option == 'collaborator':
129 129 return getattr(self.ckan.action, 'package_collaborator_list_for_user')(id=id, **kwargs)
130 130 elif type_option == 'member':
131 131 return getattr(self.ckan.action, 'organization_list_for_user')(id=id, **kwargs)
132 132 elif type_option == 'vocabulary':
133 133 return getattr(self.ckan.action, 'vocabulary_show')(id=id, **kwargs)
134 134 elif type_option == 'tag':
135 135 if not 'vocabulary_id' in kwargs:
136 136 print('Missing "vocabulary_id" value: assume it is a free tag')
137 137 return getattr(self.ckan.action, 'tag_show')(id=id, **kwargs)
138 138 elif type_option == 'user':
139 139 return getattr(self.ckan.action, 'user_show')(id=id, **kwargs)
140 140 elif type_option == 'job':
141 141 return getattr(self.ckan.action, 'job_show')(id=id, **kwargs)
142 142 else:
143 143 return 'ERROR:: "type_option = %s" is not accepted' % (type_option)
144 144 except:
145 145 _, exc_value, _ = sys.exc_info()
146 146 return exc_value
147 147 else:
148 148 return 'ERROR:: "type_option" must be a str'
149 149
150 150 def search(self, type_option, query=None, **kwargs):
151 151 '''
152 152 FINALIDAD:
153 153 Funcion personalizada para busquedas que satisfagan algun criterio.
154 154
155 155 PARAMETROS DISPONIBLES:
156 156 CONSULTAR: "GUIA DE SCRIPT.pdf"
157 157
158 158 ESTRUCTURA:
159 159 <access_name>.search(type_option = <class 'str'>, query = <class 'dict'>, param_1 = <class 'param_1'>, ...)
160 160 '''
161 161 if type(type_option) is str:
162 162 try:
163 163 if type_option == 'dataset':
164 164 key_replace = ['fq', 'fq_list', 'include_private']
165 165 key_point = ['facet_mincount', 'facet_limit', 'facet_field']
166 166 for key1, value1 in kwargs.items():
167 167 if not key1 in key_replace:
168 168 if key1 in key_point:
169 169 self.dict[key1.replace('_', '.')] = value1
170 170 else:
171 171 self.dict[key1] = value1
172 172
173 173 if query is not None:
174 174 if type(query) is dict:
175 175 self.dict['fq_list'] = []
176 176 #NUM_RESOURCES_MIN / NUM_RESOURCES_MAX
177 177 #----------------------------------------------------#
178 178 if 'dataset_start_date' in query:
179 179 if type(query['dataset_start_date']) is str:
180 180 try:
181 181 datetime.strptime(query['dataset_start_date'], '%Y-%m-%d')
182 182 if len(query['dataset_start_date']) != 10:
183 183 return '"dataset_start_date", must be: <YYYY-MM-DD>'
184 184 self.dict['fq_list'].append('dataset_start_date:"'+query['dataset_start_date']+'"')
185 185 self.list.append('dataset_start_date')
186 186 except:
187 187 return '"dataset_start_date" incorrect: "%s"' % (query['dataset_start_date'])
188 188 else:
189 189 return '"dataset_start_date" must be <str>'
190 190 #----------------------------------------------------#
191 191 if 'dataset_end_date' in query:
192 192 if type(query['dataset_end_date']) is str:
193 193 try:
194 194 datetime.strptime(query['dataset_end_date'], '%Y-%m-%d')
195 195 if len(query['dataset_end_date']) != 10:
196 196 return '"dataset_end_date", must be: <YYYY-MM-DD>'
197 197
198 198 if 'dataset_start_date' in query:
199 199 if query['dataset_start_date'] > query['dataset_end_date']:
200 200 return '"dataset_end_date" must be greater than "dataset_start_date"'
201 201
202 202 self.dict['fq_list'].append('dataset_end_date:"'+query['dataset_end_date']+'"')
203 203 self.list.append('dataset_end_date')
204 204 except:
205 205 return '"dataset_end_date" incorrect: "%s"' % (query['dataset_end_date'])
206 206 else:
207 207 return '"dataset_end_date" must be <str>'
208 208 #----------------------------------------------------#
209 if 'tags' in query:
210 if isinstance(query['tags'], (int, float, str, list)):
211 if type(query['tags']) is list:
212 for u in query['tags']:
213 self.dict['fq_list'].append('tags:"'+str(u)+'"')
214 else:
215 self.dict['fq_list'].append('tags:"'+str(query['tags'])+'"')
216
217 self.list.append('tags')
218 else:
219 return '"tags" must be <list> or <float> or <int> or <str>'
220 #----------------------------------------------------#
209 221 for key, value in query.items():
210 222 if value is not None and not key in self.list:
211 223 self.dict['fq_list'].append(str(key)+':"'+str(value)+'"')
212 224 else:
213 225 return '"query" must be <dict>'
214 226
215 227 return getattr(self.ckan.action, 'package_search')(include_private=True, **self.dict)
216 228
217 229 elif type_option == 'resource':
218 230 for key1, value1 in kwargs.items():
219 231 if key1 != 'fields':
220 232 self.dict[key1] = value1
221 233
222 234 if query is not None:
223 235 if type(query) is dict:
224 236 #----------------------------------------------------#
225 237 if 'file_date_min' in query:
226 238 if type(query['file_date_min']) is str:
227 239 try:
228 240 datetime.strptime(query['file_date_min'], '%Y-%m-%d')
229 241 if len(query['file_date_min']) != 10:
230 242 return '"file_date_min", must be: <YYYY-MM-DD>'
231 243 except:
232 244 return '"file_date_min" incorrect: "%s"' % (query['file_date_min'])
233 245 else:
234 246 return '"file_date_min" must be <str>'
235 247 #----------------------------------------------------#
236 248 if 'file_date_max' in query:
237 249 if type(query['file_date_max']) is str:
238 250 try:
239 251 datetime.strptime(query['file_date_max'], '%Y-%m-%d')
240 252 if len(query['file_date_max']) != 10:
241 253 return '"file_date_max", must be: <YYYY-MM-DD>'
242 254
243 255 if 'file_date_min' in query:
244 256 if query['file_date_min'] > query['file_date_max']:
245 257 return '"file_date_max" must be greater than "file_date_min"'
246 258 except:
247 259 return '"file_date_max" incorrect: "%s"' % (query['file_date_max'])
248 260 else:
249 261 return '"file_date_max" must be <str>'
250 262 #----------------------------------------------------#
251 263 self.dict['query'] = query
252 264 else:
253 265 return '"query" must be <dict>'
254 266 return getattr(self.ckan.action, 'resources_search')(**self.dict)
255 267
256 268 elif type_option == 'tag':
257 269 for key1, value1 in kwargs.items():
258 270 if key1 != 'fields':
259 271 self.dict[key1] = value1
260 272
261 273 if not 'vocabulary_id' in kwargs:
262 274 print('Missing "vocabulary_id" value: tags that don’t belong to any vocabulary')
263 275 else:
264 276 print('Only tags that belong to "{}" vocabulary'.format(kwargs['vocabulary_id']))
265 277
266 278 if query is not None:
267 279 if type(query) is dict:
268 280 if 'search' in query:
269 281 if type(query['search']) is list or type(query['search']) is str:
270 282 self.dict['query'] = query['search']
271 283 else:
272 284 return '"search" must be <list> or <str>'
273 285 else:
274 286 return '"query" must be <dict>'
275 287 return getattr(self.ckan.action, 'tag_search')(**self.dict)
276 288
277 289 else:
278 290 return 'ERROR:: "type_option = %s" is not accepted' % (type_option)
279 291
280 292 except:
281 293 _, exc_value, _ = sys.exc_info()
282 294 return exc_value
283 295 else:
284 296 return 'ERROR:: "type_option" must be <str>'
285 297
286 298 def create(self, type_option, select=None, **kwargs):
287 299 '''
288 300 FINALIDAD:
289 301 Funcion personalizada para crear.
290 302
291 303 PARAMETROS DISPONIBLES:
292 304 CONSULTAR: "GUIA DE SCRIPT.pdf"
293 305
294 306 ESTRUCTURA:
295 307 <access_name>.create(type_option = <class 'str'>, param_1 = <class 'param_1'>, ...)
296 308 '''
297 309 if type(type_option) is str:
298 310 try:
299 311 if type_option == 'dataset':
300 312 return getattr(self.ckan.action, 'package_create')(**kwargs)
301 313 if type_option == 'resource':
302 314 return resource.resource_create(self, **kwargs)
303 315 elif type_option == 'project':
304 316 return getattr(self.ckan.action, 'organization_create')(**kwargs)
305 317 elif type_option == 'member':
306 318 return getattr(self.ckan.action, 'organization_member_create')(**kwargs)
307 319 elif type_option == 'collaborator':
308 320 return getattr(self.ckan.action, 'package_collaborator_create')(**kwargs)
309 321 elif type_option == 'vocabulary':
310 322 return getattr(self.ckan.action, 'vocabulary_create')(**kwargs)
311 323 elif type_option == 'tag':
312 324 return getattr(self.ckan.action, 'tag_create')(**kwargs)
313 325 elif type_option == 'user':
314 326 return getattr(self.ckan.action, 'user_create')(**kwargs)
315 327 elif type_option == 'views':
316 328 if 'resource' == select:
317 329 self.list = ['package']
318 330 for key1, value1 in kwargs.items():
319 331 if not key1 in self.list:
320 332 self.dict[key1] = value1
321 333 return getattr(self.ckan.action, 'resource_create_default_resource_views')(**self.dict)
322 334 elif 'dataset' == select:
323 335 return getattr(self.ckan.action, 'package_create_default_resource_views')(**kwargs)
324 336 else:
325 337 return 'ERROR:: "select = %s" is not accepted' % (select)
326 338 else:
327 339 return 'ERROR:: "type_option = %s" is not accepted' % (type_option)
328 340 except:
329 341 _, exc_value, _ = sys.exc_info()
330 342 return exc_value
331 343 else:
332 344 return 'ERROR:: "type_option" must be <str>'
333 345
334 346 def patch(self, type_option, **kwargs):
335 347 '''
336 348 FINALIDAD:
337 349 Funciones personalizadas para actualizar
338 350
339 351 PARAMETROS DISPONIBLES:
340 352 CONSULTAR: "GUIA DE SCRIPT.pdf"
341 353
342 354 ESTRUCTURA:
343 355 <access_name>.patch(type_option = <class 'str'>, param_1 = <class 'param_1'>, ...)
344 356 '''
345 357 if type(type_option) is str:
346 358 try:
347 359 if type_option == 'dataset':
348 360 #Agregar que solo se debe modificar parámetros del Dataset y que no incluya Resources
349 361 return getattr(self.ckan.action, 'package_patch')(**kwargs)
350 362 elif type_option == 'project':
351 363 return getattr(self.ckan.action, 'organization_patch')(**kwargs)
352 364 elif type_option == 'resource':
353 365 return resource.resource_patch(self, **kwargs)
354 366 elif type_option == 'member':
355 367 return getattr(self.ckan.action, 'organization_member_create')(**kwargs)
356 368 elif type_option == 'collaborator':
357 369 return getattr(self.ckan.action, 'package_collaborator_create')(**kwargs)
358 370 else:
359 371 return 'ERROR:: "type_option = %s" is not accepted' % (type_option)
360 372 except:
361 373 _, exc_value, _ = sys.exc_info()
362 374 return exc_value
363 375 else:
364 376 return 'ERROR:: "type_option" must be <str>'
365 377
366 378 def delete(self, type_option, select=None, **kwargs):
367 379 '''
368 380 FINALIDAD:
369 381 Función personalizada para eliminar y/o purgar.
370 382
371 383 PARAMETROS DISPONIBLES:
372 384 CONSULTAR: "GUIA DE SCRIPT.pdf"
373 385
374 386 ESTRUCTURA:
375 387 <access_name>.delete(type_option = <class 'str'>, param_1 = <class 'param_1'>, ...)
376 388 '''
377 389 if type(type_option) is str:
378 390 try:
379 391 if type_option == 'dataset':
380 392 if select is None:
381 393 return 'ERROR:: "select" must not be "None"'
382 394 else:
383 395 if 'delete' == select:
384 396 return getattr(self.ckan.action, 'package_delete')(**kwargs)
385 397 elif 'purge' == select:
386 398 return getattr(self.ckan.action, 'dataset_purge')(**kwargs)
387 399 else:
388 400 return 'ERROR:: "select = %s" is not accepted' % (select)
389 401 elif type_option == 'project':
390 402 if select is None:
391 403 return 'ERROR:: "select" must not be "None"'
392 404 else:
393 405 if 'delete' == select:
394 406 return getattr(self.ckan.action, 'organization_delete')(**kwargs)
395 407 elif 'purge' == select:
396 408 return getattr(self.ckan.action, 'organization_purge')(**kwargs)
397 409 else:
398 410 return 'ERROR:: "select = %s" is not accepted' % (select)
399 411 elif type_option == 'resource':
400 412 if select is None:
401 413 return 'ERROR:: "select" must not be "None"'
402 414 else:
403 415 return resource.resource_delete(self, select, **kwargs)
404 416 elif type_option == 'vocabulary':
405 417 return getattr(self.ckan.action, 'vocabulary_delete')(**kwargs)
406 418 elif type_option == 'tag':
407 419 return getattr(self.ckan.action, 'tag_delete')(**kwargs)
408 420 elif type_option == 'user':
409 421 return getattr(self.ckan.action, 'user_delete')(**kwargs)
410 422 else:
411 423 return 'ERROR:: "type_option = %s" is not accepted' % (type_option)
412 424 except:
413 425 _, exc_value, _ = sys.exc_info()
414 426 return exc_value
415 427 else:
416 428 return 'ERROR:: "type_option" must be <str>'
417 429
418 430 def download(self, id, processes=1, path=os.path.expanduser("~"), **kwargs):
419 431 '''
420 432 FINALIDAD:
421 433 Funcion personalizada avanzada para la descarga de archivos existentes de un(os) dataset(s).
422 434
423 435 PARAMETROS DISPONIBLES:
424 436 CONSULTAR: "GUIA DE SCRIPT.pdf"
425 437
426 438 ESTRUCTURA:
427 439 <access_name>.download(id = <class 'str' or 'list'>, param_1 = <class 'param_1'>, ...)
428 440 '''
429 441 #------------------ PATH ----------------------#
430 442 if isinstance(path, str):
431 443 if os.path.isdir(path):
432 444 if not path.endswith(os.sep):
433 445 path = path + os.sep
434 446 test_txt = path + datetime.now().strftime("%Y_%m_%d_%H_%M_%S_%f")+'.txt'
435 447 try:
436 448 file_txt = open(test_txt, 'w')
437 449 file_txt.close()
438 450 os.remove(test_txt)
439 451 except:
440 452 return 'ERROR:: Access denied, you are not authorized to write files: "%s"' % (path)
441 453 else:
442 454 return 'ERROR:: "path" does not exist'
443 455 else:
444 456 return 'ERROR:: "path" must be: <class "str">'
445 457
446 458 #------------------ PROCESSES -----------------#
447 459 if not isinstance(processes, int):
448 460 return 'ERROR:: "processes" must be: <class "int">'
449 461
450 462 #------------------ ID OR NAME ----------------#
451 463 if isinstance(id, str):
452 464 id = [id]
453 465 elif isinstance(id, list):
454 466 id = list(map(str, id))
455 467 else:
456 468 return 'ERROR:: dataset "id" must be: <class "str" or "list">'
457 469 #----------------------------------------------#
458 470 arguments = {
459 471 '--apikey': self.Authorization,
460 472 '--ckan-user': None,
461 473 '--config': None,
462 474 '--datapackages': path,
463 475 '--datastore-fields': False,
464 476 '--get-request': False,
465 477 '--insecure': not self.verify,
466 478 '--processes': str(processes),
467 479 '--quiet': False,
468 480 '--remote': self.url,
469 481 '--worker': False,
470 482 #'--log': 'log.txt',
471 483 #'--all': False,
472 484 #'--gzip': False,
473 485 #'--output': None,
474 486 #'--max-records': None,
475 487 #'--output-json': False,
476 488 #'--output-jsonl': False,
477 489 #'--create-only': False,
478 490 #'--help': False,
479 491 #'--input': None,
480 492 #'--input-json': False,
481 493 #'--start-record': '1',
482 494 #'--update-only': False,
483 495 #'--upload-logo': False,
484 496 #'--upload-resources': False,
485 497 #'--version': False,
486 498 'ID_OR_NAME': id,
487 499 'datasets': True,
488 500 'dump': True,
489 501 #'ACTION_NAME': None,
490 502 #'KEY:JSON': [],
491 503 #'KEY=STRING': [],
492 504 #'KEY@FILE': [],
493 505 #'action': False,
494 506 #'delete': False,
495 507 #'groups': False,
496 508 #'load': False,
497 509 #'organizations': False,
498 510 #'related': False,
499 511 #'search': False,
500 512 #'users': False
501 513 }
502 514 return download.dump_things_change(self.ckan, 'datasets', arguments, **kwargs) No newline at end of file
General Comments 0
You need to be logged in to leave comments. Login now