##// END OF EJS Templates
DATABASES » ckanext-jro » api-cliente
RSS
v2.9.2 :: Update files restructuring
eynilupu -
r20:dce75cd83ead
parent child
Show More
Show file before | Show file after | Hide comments
@@ -0,0 +1,1
1 from jrodb.api import Api No newline at end of file
Show file before | Show file after | Hide comments
1 NO CONTENT: modified file, binary diff hidden
Show file before | Show file after | Hide comments
@@ -1,500 +1,502
1 1 from ckanapi import RemoteCKAN
2 2 from datetime import datetime
3 from CKAN_JRO import logic_download
4 from CKAN_JRO import resource
3 from jrodb import download
4 from jrodb import resource
5 5 #from ckanapi.errors import NotAuthorized, NotFound, ValidationError, SearchQueryError, SearchError, CKANAPIError, ServerIncompatibleError
6 6 import sys
7 7 import platform
8 8 import os
9 9 import requests
10 10
11 class JROAPI():
11 class Api():
12 12 """
13 13 FINALIDAD:
14 14 Script para administrar y obtener la data del repositorio por medio de APIs.
15 15
16 16 REQUISITIOS PREVIOS:
17 17 - Paso 1: Tener "pip [Python 2]" o "pip3 [Python 3]" instalado:
18 18 - Paso 2: Instalar los siguientes paquetes:
19 ckanapi==4.7
20 requests
19 En Python 2
20 - pip install -e git+http://intranet.igp.gob.pe:8082/DATABASES/ckanext-jro/api-cliente#egg=jrodb
21 En Python 3
22 - pip3 install -e git+http://intranet.igp.gob.pe:8082/DATABASES/ckanext-jro/api-cliente#egg=jrodb
21 23
22 24 FUNCIONES DISPONIBLES:
23 25 - action
24 26 - show
25 27 - search
26 28 - create
27 29 - patch
28 30 - delete
29 - download_files
31 - download
30 32
31 33 EJEMPLOS:
32 34 #1:
33 with JROAPI('http://demo.example.com', Authorization='#########') as <access_name>:
35 with Api('http://demo.example.com', Authorization='#########') as <access_name>:
34 36 ... some operation(s) ...
35 37 #2:
36 <access_name> = JROAPI('http://example.com', Authorization='#########')
38 <access_name> = Api('http://example.com', Authorization='#########')
37 39 ... some operation(s) ...
38 40 <access_name>.ckan.close()
39 41
40 42 REPORTAR ALGUN PROBLEMA:
41 43 Debe enviar un correo a eynilupu@igp.gob.pe detallando los siguientes pasos:
42 44 1) Correo para contactarlo
43 45 2) Descripcion del problema
44 46 3) ¿En que paso o seccion encontro el problema?
45 47 4) ¿Cual era el resultado que usted esperaba?
46 48 """
47 49 def __init__(self, url, Authorization=None, secure=True):
48 50 #-------- Check Secure -------#
49 51 self.verify = secure
50 52 if not secure and isinstance(secure, bool):
51 53 session = requests.Session()
52 54 session.verify = False
53 55 else:
54 56 session = None
55 57 #------------------------------#
56 58 self.url = url
57 59 ua = 'CKAN_JRO/2.9.2 (+'+str(self.url)+')'
58 60 #ua = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36'
59 61 self.ckan = RemoteCKAN(self.url, apikey=Authorization, user_agent=ua, session=session)
60 62 #self.ckan = RemoteCKAN(self.url, apikey=Authorization)
61 63 self.Authorization = Authorization
62 64 # Change for --> self.separator = os.sep
63 65 if platform.system() == 'Windows':
64 66 self.separator = '\\'
65 67 else:
66 68 self.separator = '/'
67 69
68 70 self.chunk_size = 1024
69 71 self.list = []
70 72 self.dict = {}
71 73 self.str = ''
72 74 self.check = 1
73 75 self.cont = 0
74 76
75 77 def __enter__(self):
76 78 return self
77 79
78 80 def __exit__(self, *args):
79 81 self.ckan.close()
80 82
81 83 def action(self, action, **kwargs):
82 84 """
83 85 FINALIDAD:
84 86 Funcion para llamar a las APIs disponibles
85 87
86 88 APIs DISPONIBLES:
87 89 CONSULTAR: "GUIA DE SCRIPT.pdf"
88 90
89 91 EJEMPLO:
90 92 <access_name>.action(<consuming API>, param_1 = <class 'param_1'>, ...)
91 93 """
92 94 #--------------- CASE: PACKAGE SEARCH ---------------#
93 95 if kwargs is not None:
94 96 if action == 'package_search':
95 97 self.list = ['facet_mincount', 'facet_limit', 'facet_field']
96 98 for facet in self.list:
97 99 if facet in kwargs:
98 100 kwargs[facet.replace('_', '.')] = kwargs[facet]
99 101 kwargs.pop(facet)
100 102 #----------------------------------------------------#
101 103 try:
102 104 return getattr(self.ckan.action, action)(**kwargs)
103 105 except:
104 106 _, exc_value, _ = sys.exc_info()
105 107 return exc_value
106 108
107 109 def show(self, type_option, id, **kwargs):
108 110 '''
109 111 FINALIDAD:
110 112 Funcion personalizada para una busqueda en especifico.
111 113
112 114 PARAMETROS DISPONIBLES:
113 115 CONSULTAR: "GUIA DE SCRIPT.pdf"
114 116
115 117 ESTRUCTURA:
116 118 <access_name>.show(type_option = <class 'str'>, id = <class 'str'>, param_1 = <class 'param_1'>, ...)
117 119 '''
118 120 if type(type_option) is str:
119 121 try:
120 122 if type_option == 'dataset':
121 123 return getattr(self.ckan.action, 'package_show')(id=id, **kwargs)
122 124 elif type_option == 'resource':
123 125 return getattr(self.ckan.action, 'resource_show')(id=id, **kwargs)
124 126 elif type_option == 'project':
125 127 return getattr(self.ckan.action, 'organization_show')(id=id, **kwargs)
126 128 elif type_option == 'collaborator':
127 129 return getattr(self.ckan.action, 'package_collaborator_list_for_user')(id=id, **kwargs)
128 130 elif type_option == 'member':
129 131 return getattr(self.ckan.action, 'organization_list_for_user')(id=id, **kwargs)
130 132 elif type_option == 'vocabulary':
131 133 return getattr(self.ckan.action, 'vocabulary_show')(id=id, **kwargs)
132 134 elif type_option == 'tag':
133 135 if not 'vocabulary_id' in kwargs:
134 136 print('Missing "vocabulary_id" value: assume it is a free tag')
135 137 return getattr(self.ckan.action, 'tag_show')(id=id, **kwargs)
136 138 elif type_option == 'user':
137 139 return getattr(self.ckan.action, 'user_show')(id=id, **kwargs)
138 140 elif type_option == 'job':
139 141 return getattr(self.ckan.action, 'job_show')(id=id, **kwargs)
140 142 else:
141 143 return 'ERROR:: "type_option = %s" is not accepted' % (type_option)
142 144 except:
143 145 _, exc_value, _ = sys.exc_info()
144 146 return exc_value
145 147 else:
146 148 return 'ERROR:: "type_option" must be a str'
147 149
148 150 def search(self, type_option, query=None, **kwargs):
149 151 '''
150 152 FINALIDAD:
151 153 Funcion personalizada para busquedas que satisfagan algun criterio.
152 154
153 155 PARAMETROS DISPONIBLES:
154 156 CONSULTAR: "GUIA DE SCRIPT.pdf"
155 157
156 158 ESTRUCTURA:
157 159 <access_name>.search(type_option = <class 'str'>, query = <class 'dict'>, param_1 = <class 'param_1'>, ...)
158 160 '''
159 161 if type(type_option) is str:
160 162 try:
161 163 if type_option == 'dataset':
162 164 key_replace = ['fq', 'fq_list', 'include_private']
163 165 key_point = ['facet_mincount', 'facet_limit', 'facet_field']
164 166 for key1, value1 in kwargs.items():
165 167 if not key1 in key_replace:
166 168 if key1 in key_point:
167 169 self.dict[key1.replace('_', '.')] = value1
168 170 else:
169 171 self.dict[key1] = value1
170 172
171 173 if query is not None:
172 174 if type(query) is dict:
173 175 self.dict['fq_list'] = []
174 176 #NUM_RESOURCES_MIN / NUM_RESOURCES_MAX
175 177 #----------------------------------------------------#
176 178 if 'dataset_start_date' in query:
177 179 if type(query['dataset_start_date']) is str:
178 180 try:
179 181 datetime.strptime(query['dataset_start_date'], '%Y-%m-%d')
180 182 if len(query['dataset_start_date']) != 10:
181 183 return '"dataset_start_date", must be: <YYYY-MM-DD>'
182 184 self.dict['fq_list'].append('dataset_start_date:"'+query['dataset_start_date']+'"')
183 185 self.list.append('dataset_start_date')
184 186 except:
185 187 return '"dataset_start_date" incorrect: "%s"' % (query['dataset_start_date'])
186 188 else:
187 189 return '"dataset_start_date" must be <str>'
188 190 #----------------------------------------------------#
189 191 if 'dataset_end_date' in query:
190 192 if type(query['dataset_end_date']) is str:
191 193 try:
192 194 datetime.strptime(query['dataset_end_date'], '%Y-%m-%d')
193 195 if len(query['dataset_end_date']) != 10:
194 196 return '"dataset_end_date", must be: <YYYY-MM-DD>'
195 197
196 198 if 'dataset_start_date' in query:
197 199 if query['dataset_start_date'] > query['dataset_end_date']:
198 200 return '"dataset_end_date" must be greater than "dataset_start_date"'
199 201
200 202 self.dict['fq_list'].append('dataset_end_date:"'+query['dataset_end_date']+'"')
201 203 self.list.append('dataset_end_date')
202 204 except:
203 205 return '"dataset_end_date" incorrect: "%s"' % (query['dataset_end_date'])
204 206 else:
205 207 return '"dataset_end_date" must be <str>'
206 208 #----------------------------------------------------#
207 209 for key, value in query.items():
208 210 if value is not None and not key in self.list:
209 211 self.dict['fq_list'].append(str(key)+':"'+str(value)+'"')
210 212 else:
211 213 return '"query" must be <dict>'
212 214
213 215 return getattr(self.ckan.action, 'package_search')(include_private=True, **self.dict)
214 216
215 217 elif type_option == 'resource':
216 218 for key1, value1 in kwargs.items():
217 219 if key1 != 'fields':
218 220 self.dict[key1] = value1
219 221
220 222 if query is not None:
221 223 if type(query) is dict:
222 224 #----------------------------------------------------#
223 225 if 'file_date_min' in query:
224 226 if type(query['file_date_min']) is str:
225 227 try:
226 228 datetime.strptime(query['file_date_min'], '%Y-%m-%d')
227 229 if len(query['file_date_min']) != 10:
228 230 return '"file_date_min", must be: <YYYY-MM-DD>'
229 231 except:
230 232 return '"file_date_min" incorrect: "%s"' % (query['file_date_min'])
231 233 else:
232 234 return '"file_date_min" must be <str>'
233 235 #----------------------------------------------------#
234 236 if 'file_date_max' in query:
235 237 if type(query['file_date_max']) is str:
236 238 try:
237 239 datetime.strptime(query['file_date_max'], '%Y-%m-%d')
238 240 if len(query['file_date_max']) != 10:
239 241 return '"file_date_max", must be: <YYYY-MM-DD>'
240 242
241 243 if 'file_date_min' in query:
242 244 if query['file_date_min'] > query['file_date_max']:
243 245 return '"file_date_max" must be greater than "file_date_min"'
244 246 except:
245 247 return '"file_date_max" incorrect: "%s"' % (query['file_date_max'])
246 248 else:
247 249 return '"file_date_max" must be <str>'
248 250 #----------------------------------------------------#
249 251 self.dict['query'] = query
250 252 else:
251 253 return '"query" must be <dict>'
252 254 return getattr(self.ckan.action, 'resources_search')(**self.dict)
253 255
254 256 elif type_option == 'tag':
255 257 for key1, value1 in kwargs.items():
256 258 if key1 != 'fields':
257 259 self.dict[key1] = value1
258 260
259 261 if not 'vocabulary_id' in kwargs:
260 262 print('Missing "vocabulary_id" value: tags that don’t belong to any vocabulary')
261 263 else:
262 264 print('Only tags that belong to "{}" vocabulary'.format(kwargs['vocabulary_id']))
263 265
264 266 if query is not None:
265 267 if type(query) is dict:
266 268 if 'search' in query:
267 269 if type(query['search']) is list or type(query['search']) is str:
268 270 self.dict['query'] = query['search']
269 271 else:
270 272 return '"search" must be <list> or <str>'
271 273 else:
272 274 return '"query" must be <dict>'
273 275 return getattr(self.ckan.action, 'tag_search')(**self.dict)
274 276
275 277 else:
276 278 return 'ERROR:: "type_option = %s" is not accepted' % (type_option)
277 279
278 280 except:
279 281 _, exc_value, _ = sys.exc_info()
280 282 return exc_value
281 283 else:
282 284 return 'ERROR:: "type_option" must be <str>'
283 285
284 286 def create(self, type_option, select=None, **kwargs):
285 287 '''
286 288 FINALIDAD:
287 289 Funcion personalizada para crear.
288 290
289 291 PARAMETROS DISPONIBLES:
290 292 CONSULTAR: "GUIA DE SCRIPT.pdf"
291 293
292 294 ESTRUCTURA:
293 295 <access_name>.create(type_option = <class 'str'>, param_1 = <class 'param_1'>, ...)
294 296 '''
295 297 if type(type_option) is str:
296 298 try:
297 299 if type_option == 'dataset':
298 300 return getattr(self.ckan.action, 'package_create')(**kwargs)
299 301 if type_option == 'resource':
300 302 return resource.resource_create(self, **kwargs)
301 303 elif type_option == 'project':
302 304 return getattr(self.ckan.action, 'organization_create')(**kwargs)
303 305 elif type_option == 'member':
304 306 return getattr(self.ckan.action, 'organization_member_create')(**kwargs)
305 307 elif type_option == 'collaborator':
306 308 return getattr(self.ckan.action, 'package_collaborator_create')(**kwargs)
307 309 elif type_option == 'vocabulary':
308 310 return getattr(self.ckan.action, 'vocabulary_create')(**kwargs)
309 311 elif type_option == 'tag':
310 312 return getattr(self.ckan.action, 'tag_create')(**kwargs)
311 313 elif type_option == 'user':
312 314 return getattr(self.ckan.action, 'user_create')(**kwargs)
313 315 elif type_option == 'views':
314 316 if 'resource' == select:
315 317 self.list = ['package']
316 318 for key1, value1 in kwargs.items():
317 319 if not key1 in self.list:
318 320 self.dict[key1] = value1
319 321 return getattr(self.ckan.action, 'resource_create_default_resource_views')(**self.dict)
320 322 elif 'dataset' == select:
321 323 return getattr(self.ckan.action, 'package_create_default_resource_views')(**kwargs)
322 324 else:
323 325 return 'ERROR:: "select = %s" is not accepted' % (select)
324 326 else:
325 327 return 'ERROR:: "type_option = %s" is not accepted' % (type_option)
326 328 except:
327 329 _, exc_value, _ = sys.exc_info()
328 330 return exc_value
329 331 else:
330 332 return 'ERROR:: "type_option" must be <str>'
331 333
332 334 def patch(self, type_option, **kwargs):
333 335 '''
334 336 FINALIDAD:
335 337 Funciones personalizadas para actualizar
336 338
337 339 PARAMETROS DISPONIBLES:
338 340 CONSULTAR: "GUIA DE SCRIPT.pdf"
339 341
340 342 ESTRUCTURA:
341 343 <access_name>.patch(type_option = <class 'str'>, param_1 = <class 'param_1'>, ...)
342 344 '''
343 345 if type(type_option) is str:
344 346 try:
345 347 if type_option == 'dataset':
346 348 #Agregar que solo se debe modificar parámetros del Dataset y que no incluya Resources
347 349 return getattr(self.ckan.action, 'package_patch')(**kwargs)
348 350 elif type_option == 'project':
349 351 return getattr(self.ckan.action, 'organization_patch')(**kwargs)
350 352 elif type_option == 'resource':
351 353 return resource.resource_patch(self, **kwargs)
352 354 elif type_option == 'member':
353 355 return getattr(self.ckan.action, 'organization_member_create')(**kwargs)
354 356 elif type_option == 'collaborator':
355 357 return getattr(self.ckan.action, 'package_collaborator_create')(**kwargs)
356 358 else:
357 359 return 'ERROR:: "type_option = %s" is not accepted' % (type_option)
358 360 except:
359 361 _, exc_value, _ = sys.exc_info()
360 362 return exc_value
361 363 else:
362 364 return 'ERROR:: "type_option" must be <str>'
363 365
364 366 def delete(self, type_option, select=None, **kwargs):
365 367 '''
366 368 FINALIDAD:
367 369 Función personalizada para eliminar y/o purgar.
368 370
369 371 PARAMETROS DISPONIBLES:
370 372 CONSULTAR: "GUIA DE SCRIPT.pdf"
371 373
372 374 ESTRUCTURA:
373 375 <access_name>.delete(type_option = <class 'str'>, param_1 = <class 'param_1'>, ...)
374 376 '''
375 377 if type(type_option) is str:
376 378 try:
377 379 if type_option == 'dataset':
378 380 if select is None:
379 381 return 'ERROR:: "select" must not be "None"'
380 382 else:
381 383 if 'delete' == select:
382 384 return getattr(self.ckan.action, 'package_delete')(**kwargs)
383 385 elif 'purge' == select:
384 386 return getattr(self.ckan.action, 'dataset_purge')(**kwargs)
385 387 else:
386 388 return 'ERROR:: "select = %s" is not accepted' % (select)
387 389 elif type_option == 'project':
388 390 if select is None:
389 391 return 'ERROR:: "select" must not be "None"'
390 392 else:
391 393 if 'delete' == select:
392 394 return getattr(self.ckan.action, 'organization_delete')(**kwargs)
393 395 elif 'purge' == select:
394 396 return getattr(self.ckan.action, 'organization_purge')(**kwargs)
395 397 else:
396 398 return 'ERROR:: "select = %s" is not accepted' % (select)
397 399 elif type_option == 'resource':
398 400 if select is None:
399 401 return 'ERROR:: "select" must not be "None"'
400 402 else:
401 403 return resource.resource_delete(self, select, **kwargs)
402 404 elif type_option == 'vocabulary':
403 405 return getattr(self.ckan.action, 'vocabulary_delete')(**kwargs)
404 406 elif type_option == 'tag':
405 407 return getattr(self.ckan.action, 'tag_delete')(**kwargs)
406 408 elif type_option == 'user':
407 409 return getattr(self.ckan.action, 'user_delete')(**kwargs)
408 410 else:
409 411 return 'ERROR:: "type_option = %s" is not accepted' % (type_option)
410 412 except:
411 413 _, exc_value, _ = sys.exc_info()
412 414 return exc_value
413 415 else:
414 416 return 'ERROR:: "type_option" must be <str>'
415 417
416 def download_files(self, id, processes=1, path=os.path.expanduser("~"), **kwargs):
418 def download(self, id, processes=1, path=os.path.expanduser("~"), **kwargs):
417 419 '''
418 420 FINALIDAD:
419 421 Funcion personalizada avanzada para la descarga de archivos existentes de un(os) dataset(s).
420 422
421 423 PARAMETROS DISPONIBLES:
422 424 CONSULTAR: "GUIA DE SCRIPT.pdf"
423 425
424 426 ESTRUCTURA:
425 <access_name>.download_files(id = <class 'str' or 'list'>, param_1 = <class 'param_1'>, ...)
427 <access_name>.download(id = <class 'str' or 'list'>, param_1 = <class 'param_1'>, ...)
426 428 '''
427 429 #------------------ PATH ----------------------#
428 430 if isinstance(path, str):
429 431 if os.path.isdir(path):
430 432 if not path.endswith(os.sep):
431 433 path = path + os.sep
432 434 test_txt = path + datetime.now().strftime("%Y_%m_%d_%H_%M_%S_%f")+'.txt'
433 435 try:
434 436 file_txt = open(test_txt, 'w')
435 437 file_txt.close()
436 438 os.remove(test_txt)
437 439 except:
438 440 return 'ERROR:: Access denied, you are not authorized to write files: "%s"' % (path)
439 441 else:
440 442 return 'ERROR:: "path" does not exist'
441 443 else:
442 444 return 'ERROR:: "path" must be: <class "str">'
443 445
444 446 #------------------ PROCESSES -----------------#
445 447 if not isinstance(processes, int):
446 448 return 'ERROR:: "processes" must be: <class "int">'
447 449
448 450 #------------------ ID OR NAME ----------------#
449 451 if isinstance(id, str):
450 452 id = [id]
451 453 elif isinstance(id, list):
452 454 id = list(map(str, id))
453 455 else:
454 456 return 'ERROR:: dataset "id" must be: <class "str" or "list">'
455 457 #----------------------------------------------#
456 458 arguments = {
457 459 '--apikey': self.Authorization,
458 460 '--ckan-user': None,
459 461 '--config': None,
460 462 '--datapackages': path,
461 463 '--datastore-fields': False,
462 464 '--get-request': False,
463 465 '--insecure': not self.verify,
464 466 '--processes': str(processes),
465 467 '--quiet': False,
466 468 '--remote': self.url,
467 469 '--worker': False,
468 470 #'--log': 'log.txt',
469 471 #'--all': False,
470 472 #'--gzip': False,
471 473 #'--output': None,
472 474 #'--max-records': None,
473 475 #'--output-json': False,
474 476 #'--output-jsonl': False,
475 477 #'--create-only': False,
476 478 #'--help': False,
477 479 #'--input': None,
478 480 #'--input-json': False,
479 481 #'--start-record': '1',
480 482 #'--update-only': False,
481 483 #'--upload-logo': False,
482 484 #'--upload-resources': False,
483 485 #'--version': False,
484 486 'ID_OR_NAME': id,
485 487 'datasets': True,
486 488 'dump': True,
487 489 #'ACTION_NAME': None,
488 490 #'KEY:JSON': [],
489 491 #'KEY=STRING': [],
490 492 #'KEY@FILE': [],
491 493 #'action': False,
492 494 #'delete': False,
493 495 #'groups': False,
494 496 #'load': False,
495 497 #'organizations': False,
496 498 #'related': False,
497 499 #'search': False,
498 500 #'users': False
499 501 }
500 return logic_download.dump_things_change(self.ckan, 'datasets', arguments, **kwargs) No newline at end of file
502 return download.dump_things_change(self.ckan, 'datasets', arguments, **kwargs) No newline at end of file
Show file before | Show file after | Hide comments
1 NO CONTENT: file renamed from script/CKAN_JRO/logic_download.py to jrodb/download.py
Show file before | Show file after | Hide comments
1 NO CONTENT: file renamed from script/CKAN_JRO/resource.py to jrodb/resource.py
Show file before | Show file after | Hide comments
@@ -1,12 +1,16
1 1 # encoding: utf-8
2 2 from setuptools import setup
3 3
4 4 setup(
5 name = "CKAN_JRO",
5 name = "jrodb",
6 6 version = "2.9.2.0",
7 7 description = "Data Repository - JRO",
8 8 author = "Edson Ynilupu Mattos",
9 9 author_email = "eynilupu@igp.gob.pe",
10 10 url = "http://intranet.igp.gob.pe:8082/DATABASES/ckanext-jro/api-cliente",
11 packages = ["CKAN_JRO"]
11 packages = ["jrodb"],
12 install_requires = [
13 "ckanapi==4.7",
14 "requests"
15 ],
12 16 ) No newline at end of file
Show file before | Show file after | Hide comments
1 NO CONTENT: file was removed, binary diff hidden
General Comments 0
You need to be logged in to leave comments. Login now