##// END OF EJS Templates
DATABASES » ckanext-jro » api-cliente
RSS
v2.9.2 :: Add 'resource_views' create function
eynilupu -
r3:ab4b435c96a4
parent child
Show More
Show file before | Show file after | Hide comments
1 NO CONTENT: modified file, binary diff hidden
Show file before | Show file after | Hide comments
@@ -1,813 +1,825
1 1 from ckanapi import RemoteCKAN
2 2 #from ckanapi.errors import NotAuthorized, NotFound, ValidationError, SearchQueryError, SearchError, CKANAPIError, ServerIncompatibleError
3 3 import sys
4 4 import platform
5 5 import os
6 6 import tempfile
7 7 import shutil
8 8 import zipfile
9 9 import concurrent.futures
10 10 import requests
11 11 import json
12 12 import pathlib
13 13 import uuid
14 14 from datetime import datetime
15 15 from tqdm import tqdm
16 16
17 17 class JROAPI():
18 18 """
19 19 FINALIDAD:
20 20 Script para administrar y obtener la data del repositorio por medio de APIs.
21 21
22 22 REQUISITIOS PREVIOS:
23 23 - Paso 1: Tener "pip [Python 2]" o "pip3 [Python 3]" instalado:
24 24 - Paso 2: Instalar lo siguiente como admininstrador:
25 25 En Python 2
26 26 - pip install ckanapi==4.5
27 27 - pip install requests
28 28 - pip install tqdm
29 29 En Python 3
30 30 - pip3 install ckanapi==4.5
31 31 - pip3 install requests
32 32 - pip3 install tqdm
33 33
34 34 FUNCIONES DISPONIBLES:
35 35 - action
36 36 - upload_file
37 37 - upload_multiple_files
38 - upload_multiple_files_advance
38 39 - show
39 40 - search
40 41 - create
41 42 - patch
42 43 - delete
43 44 - download_files
44 45
45 46 EJEMPLOS:
46 47 #1:
47 48 with JROAPI('http://demo.example.com', Authorization='#########') as <access_name>:
48 49 ... some operation(s) ...
49 50 #2:
50 51 <access_name> = JROAPI('http://example.com', Authorization='#########')
51 52 ... some operation(s) ...
52 53 <access_name>.ckan.close()
53 54
54 55 REPORTAR ALGUN PROBLEMA:
55 56 Debe enviar un correo a eynilupu@igp.gob.pe detallando los siguientes pasos:
56 57 1) Identifiquese
57 58 2) Describir el problema
58 59 3) ¿En que funcion esta el problema?
59 60 4) ¿Que esperaba que hiciera la funcion sin el problema?
60 61 """
61 62 def __init__(self, url, Authorization=None):
62 63 ua = 'CKAN_JRO/1.1 (+'+str(url)+')'
63 64 #ua = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36'
64 65 self.ckan = RemoteCKAN(url, apikey=Authorization, user_agent=ua)
65 66 #self.ckan = RemoteCKAN(url, apikey=Authorization)
66 67 self.Authorization = Authorization
67 68 if platform.system() == 'Windows':
68 69 self.separator = '\\'
69 70 else:
70 71 self.separator = '/'
71 72
72 73 self.chunk_size = 1024
73 74 self.list = []
74 75 self.dict = {}
75 76 self.str = ''
76 77 self.check = 1
77 78 self.cont = 0
78 79
79 80 def __enter__(self):
80 81 return self
81 82
82 83 def __exit__(self, *args):
83 84 self.ckan.close()
84 85
85 86 def action(self, action, **kwargs):
86 87 """
87 88 FINALIDAD:
88 89 Funcion para llamar a las APIs disponibles
89 90
90 91 APIs DISPONIBLES:
91 92 CONSULTAR: "GUIA DE SCRIPT.pdf"
92 93
93 94 EJEMPLO:
94 95 <access_name>.action(<consuming API>, param_1 = <class 'param_1'>, ...)
95 96 """
96 97 #--------------- CASE: PACKAGE SEARCH ---------------#
97 98 if kwargs is not None:
98 99 if action == 'package_search':
99 100 self.list = ['facet_mincount', 'facet_limit', 'facet_field']
100 101 for facet in self.list:
101 102 if facet in kwargs:
102 103 kwargs[facet.replace('_', '.')] = kwargs[facet]
103 104 kwargs.pop(facet)
104 105 #----------------------------------------------------#
105 106 try:
106 107 return getattr(self.ckan.action, action)(**kwargs)
107 108 except:
108 109 _, exc_value, _ = sys.exc_info()
109 110 return exc_value
110 111
111 112 def upload_file(self, dataset_id, file_path, file_date, file_type, **kwargs):
112 113 '''
113 114 FINALIDAD:
114 115 Funcion para subir un unico archivo al repositorio del ROJ.
115 116
116 117 PARAMETROS DISPONIBLES:
117 118 CONSULTAR: "GUIA DE SCRIPT.pdf"
118 119
119 120 ESTRUCTURA:
120 121 <access_name>.upload_file(dataset_id = <class 'str'>, file_date = <class 'str'>, file_path = <class 'str'>, file_type = <class 'str'>, param_1 = <class 'param_1'>, ...)
121 122 '''
122 123 self.list = ['package_id', 'upload', 'voc_file_type', 'name'] #file_date
123 124 for key1, value1 in kwargs.items():
124 125 if not key1 in self.list:
125 126 self.dict[key1] = value1
126 127
127 128 #---------------------------#
128 129 if not 'others' in kwargs:
129 130 self.dict['others'] = ''
130 131 else:
131 132 if isinstance(kwargs['others'], list):
132 133 self.dict['others'] = json.dumps(kwargs['others'])
133 134 #---------------------------#
134 135
135 136 if not os.path.isfile(file_path):
136 137 return 'File "%s" not exist' % (file_path)
137 138
138 139 if not 'format' in self.dict:
139 140 self.str = ''.join(pathlib.Path(file_path).suffixes)
140 141 if len(self.str) > 0:
141 142 self.dict['format'] = self.str.upper()[1:]
142 143
143 144 try:
144 145 return getattr(self.ckan.action, 'resource_create')(package_id=dataset_id, file_date=file_date, upload=open(file_path, 'rb'), voc_file_type=file_type, name=pathlib.Path(file_path).name, **self.dict)
145 146 except:
146 147 _, exc_value, _ = sys.exc_info()
147 148 return exc_value
148 149
149 150
150 151 def upload_multiple_files_advance(self, dataset_id, path_files, file_date, file_type, **kwargs):
151 152 '''
152 153 FINALIDAD:
153 154 Funcion para subir multiples archivos al repositorio del ROJ.
154 155
155 156 PARAMETROS DISPONIBLES:
156 157 CONSULTAR: "GUIA DE SCRIPT.pdf"
157 158
158 159 ESTRUCTURA:
159 160 <access_name>.upload_multiple_files_advance(dataset_id = <class 'str'>, path_files = <class 'list of strings'>, file_date = <class 'str'>, file_type = <class 'str'>, param_1 = <class 'param_1'>, ...)
160 161 '''
161 162 self.list = ['package_id', 'upload', 'voc_file_type', 'name']
162 163
163 164 for key1, value1 in kwargs.items():
164 165 if not key1 in self.list:
165 166 self.dict[key1] = value1
166 167
167 168 #---------------------------#
168 169 if not 'others' in kwargs:
169 170 self.dict['others'] = ''
170 171 else:
171 172 if isinstance(kwargs['others'], list):
172 173 self.dict['others'] = json.dumps(kwargs['others'])
173 174 #---------------------------#
174 175
175 176 params_dict = {'upload':[], 'name':[]}
176 177 #---------------CASO : "path" or "path_list"-----------------#
177 178 if type(path_files) is list:
178 179 if len(path_files) != 0:
179 180 path_files.sort()
180 181 for u in path_files:
181 182 if os.path.isfile(u):
182 183 params_dict['upload'].append(open(u, 'rb'))
183 184 params_dict['name'].append(pathlib.Path(u).name)
184 185 else:
185 186 return 'File "%s" does not exist' % (u)
186 187 else:
187 188 return 'ERROR:: "path_list is empty"'
188 189 elif type(path_files) is str:
189 190 if os.path.isdir(path_files):
190 191 path_order = [f for f in os.listdir(path_files) if os.path.isfile(os.path.join(path_files, f))]
191 192 path_order.sort()
192 193 if path_order:
193 194 for name in path_order:
194 195 params_dict['upload'].append(open(os.path.join(path_files, name), 'rb'))
195 196 params_dict['name'].append(name)
196 197 else:
197 198 return "ERROR:: There aren't files in this directory"
198 199 else:
199 200 return 'ERROR:: Directory "%s" does not exist' % (path_files)
200 201 else:
201 202 return 'ERROR:: "path_files" must be a str or list'
202 203 #------------------------------------------------------------#
203 204 resource_extend = []
204 205 files_dict = {}
205 206 for count, name in enumerate(params_dict['name']):
206 207 param_list = {"name": name, "file_date": file_date, "voc_file_type": file_type}
207 208 param_list.update(self.dict)
208 209
209 210 if not 'format' in param_list:
210 211 format = ''.join(pathlib.Path(name).suffixes)
211 212 if len(format) > 0:
212 213 param_list['format'] = format.upper()[1:]
213 214
214 215 resource_extend.append(param_list)
215 216 files_dict['update__resources__-'+ str(len(params_dict['name'])-count) +'__upload'] = (name, params_dict['upload'][count])
216 217
217 218 #------------------------------------------------------------#
218 219 try:
219 220 uuid.UUID(str(dataset_id), version=4)
220 221 package_id_or_name = '"id": "' + str(dataset_id) + '"'
221 222 except ValueError:
222 223 package_id_or_name = '"name": "' + str(dataset_id) + '"'
223 224 #------------------------------------------------------------#
224 225 print('"{}" file(s) found >> uploading'.format(len(params_dict['name'])))
225 226 try:
226 227 result = self.ckan.call_action(
227 228 'package_revise',
228 229 {'match': '{'+ str(package_id_or_name) +'}', 'update__resources__extend': json.dumps(resource_extend)},
229 230 files=files_dict
230 231 )
231 232 print('Uploaded file(s) successfully')
232 233 return result
233 234 except:
234 235 print('ERROR :: Use the "print" option for more information')
235 236 _, exc_value, _ = sys.exc_info()
236 237 return exc_value
237 238
238 239 def upload_multiple_files(self, dataset_id, path_files, date_files, type_files, **kwargs):
239 240 '''
240 241 FINALIDAD:
241 242 Funcion para subir multiples archivos al repositorio del ROJ.
242 243
243 244 PARAMETROS DISPONIBLES:
244 245 CONSULTAR: "GUIA DE SCRIPT.pdf"
245 246
246 247 ESTRUCTURA:
247 248 <access_name>.upload_multiple_files(dataset_id = <class 'str'>, path_files = <class 'str'> or <class 'list of strings'>, date_files = <class 'str'> or <class 'list of strings'>, type_files = <class 'str'> or <class 'list of strings'>, param_1 = <class 'param_1'>, ...)
248 249 '''
249 250
250 251 params_dict = {'upload':[], 'name':[]}
251 252 if not 'format' in kwargs:
252 253 params_dict.update({'format':[]})
253 254 #---------------CASO : "path" or "path_list"-----------------#
254 255 if type(path_files) is list:
255 256 if len(path_files) != 0:
256 257 path_files.sort()
257 258 for u in path_files:
258 259 if os.path.isfile(u):
259 260 params_dict['upload'].append(open(u, 'rb'))
260 261 params_dict['name'].append(pathlib.Path(u).name)
261 262 if not 'format' in kwargs:
262 263 format = ''.join(pathlib.Path(u).suffixes)
263 264 if len(format) > 0:
264 265 params_dict['format'].append(format.upper()[1:])
265 266 else:
266 267 params_dict['format'].append('')
267 268 else:
268 269 return 'File "%s" does not exist' % (u)
269 270 else:
270 271 return 'ERROR:: "path_list is empty"'
271 272 elif type(path_files) is str:
272 273 if os.path.isdir(path_files):
273 274 path_order = [f for f in os.listdir(path_files) if os.path.isfile(os.path.join(path_files, f))]
274 275 path_order.sort()
275 276 if path_order:
276 277 for name in path_order:
277 278 params_dict['upload'].append(open(os.path.join(path_files, name), 'rb'))
278 279 params_dict['name'].append(name)
279 280 if not 'format' in kwargs:
280 281 format = ''.join(pathlib.Path(name).suffixes)
281 282 if len(format) > 0:
282 283 params_dict['format'].append(format.upper()[1:])
283 284 else:
284 285 params_dict['format'].append('')
285 286 else:
286 287 return "ERROR:: There aren't files in this directory"
287 288 else:
288 289 return 'ERROR:: Directory "%s" does not exist' % (path_files)
289 290 else:
290 291 return 'ERROR:: "path_files" must be a str or list'
291 292 #------------------------------------------------------------#
292 293 params_no_dict = {'package_id': dataset_id}
293 294 if type(date_files) is list:
294 295 params_dict['file_date'] = date_files
295 296 else:
296 297 params_no_dict['file_date'] = date_files
297 298
298 299 if type(type_files) is list:
299 300 params_dict['voc_file_type'] = type_files
300 301 else:
301 302 params_no_dict['voc_file_type'] = type_files
302 303
303 304 for key1, value1 in kwargs.items():
304 305 if not key1 in params_dict and not key1 in params_no_dict and key1 != 'others':
305 306 if type(value1) is list:
306 307 params_dict[key1] = value1
307 308 else:
308 309 params_no_dict[key1] = value1
309 310 #------------------------------------------#
310 311 if not 'others' in kwargs:
311 312 params_no_dict['others'] = ''
312 313 else:
313 314 if isinstance(kwargs['others'], tuple):
314 315 params_dict['others'] = [json.dumps(w) for w in kwargs['others']]
315 316 elif isinstance(kwargs['others'], list):
316 317 params_no_dict['others'] = json.dumps(kwargs['others'])
317 318 elif isinstance(kwargs['others'], str):
318 319 params_no_dict['others'] = kwargs['others']
319 320 else:
320 321 return 'ERROR:: "others" must be a tuple, list or str'
321 322 #------------------------------------------#
322 323 len_params_dict = []
323 324 for value2 in params_dict.values():
324 325 len_params_dict.append(len(value2))
325 326
326 327 if len(list(set(len_params_dict))) > 1:
327 328 return 'ERROR:: All lists must be the same length: %s' % (len(params_dict['name']))
328 329 #------------------------------------------------------------#
329 330 print('"{}" file(s) found >> uploading'.format(len(params_dict['name'])))
330 331 for v in range(len(params_dict['name'])):
331 332 try:
332 333 send = {}
333 334 for key_dict, value_dict in params_dict.items():
334 335 send[key_dict] = value_dict[v]
335 336 for key_no_dict, value_no_dict in params_no_dict.items():
336 337 send[key_no_dict] = value_no_dict
337 338
338 339 self.list.append(getattr(self.ckan.action, 'resource_create')(**send))
339 340 print('File #{} :: "{}" was uploaded successfully'.format(v+1, params_dict['name'][v]))
340 341 except:
341 342 _, exc_value, _ = sys.exc_info()
342 343 self.list.append(exc_value)
343 344 print('File #{} :: Error uploading "{}" file'.format(v+1, params_dict['name'][v]))
344 345 return self.list
345 346 #------------------------------------------------------------#
346 347
347 348 def show(self, type_option, id, **kwargs):
348 349 '''
349 350 FINALIDAD:
350 351 Funcion personalizada para una busqueda en especifico.
351 352
352 353 PARAMETROS DISPONIBLES:
353 354 CONSULTAR: "GUIA DE SCRIPT.pdf"
354 355
355 356 ESTRUCTURA:
356 357 <access_name>.show(type_option = <class 'str'>, id = <class 'str'>, param_1 = <class 'param_1'>, ...)
357 358 '''
358 359 if type(type_option) is str:
359 360 try:
360 361 if type_option == 'dataset':
361 362 return getattr(self.ckan.action, 'package_show')(id=id, **kwargs)
362 363 elif type_option == 'resource':
363 364 return getattr(self.ckan.action, 'resource_show')(id=id, **kwargs)
364 365 elif type_option == 'project':
365 366 return getattr(self.ckan.action, 'organization_show')(id=id, **kwargs)
366 367 elif type_option == 'collaborator':
367 368 return getattr(self.ckan.action, 'package_collaborator_list_for_user')(id=id, **kwargs)
368 369 elif type_option == 'member':
369 370 return getattr(self.ckan.action, 'organization_list_for_user')(id=id, **kwargs)
370 371 elif type_option == 'vocabulary':
371 372 return getattr(self.ckan.action, 'vocabulary_show')(id=id, **kwargs)
372 373 elif type_option == 'tag':
373 374 if not 'vocabulary_id' in kwargs:
374 375 print('Missing "vocabulary_id" value: assume it is a free tag')
375 376 return getattr(self.ckan.action, 'tag_show')(id=id, **kwargs)
376 377 elif type_option == 'user':
377 378 return getattr(self.ckan.action, 'user_show')(id=id, **kwargs)
378 379 elif type_option == 'job':
379 380 return getattr(self.ckan.action, 'job_show')(id=id, **kwargs)
380 381 else:
381 return 'ERROR:: "%s" is not accepted' % (type_option)
382 return 'ERROR:: "type_option = %s" is not accepted' % (type_option)
382 383 except:
383 384 _, exc_value, _ = sys.exc_info()
384 385 return exc_value
385 386 else:
386 387 return 'ERROR:: "type_option" must be a str'
387 388
388 389 def search(self, type_option, query=None, **kwargs):
389 390 '''
390 391 FINALIDAD:
391 392 Funcion personalizada para busquedas que satisfagan algun criterio.
392 393
393 394 PARAMETROS DISPONIBLES:
394 395 CONSULTAR: "GUIA DE SCRIPT.pdf"
395 396
396 397 ESTRUCTURA:
397 398 <access_name>.search(type_option = <class 'str'>, query = <class 'dict'>, param_1 = <class 'param_1'>, ...)
398 399 '''
399 400 if type(type_option) is str:
400 401 try:
401 402 if type_option == 'dataset':
402 403 key_replace = ['fq', 'fq_list', 'include_private']
403 404 key_point = ['facet_mincount', 'facet_limit', 'facet_field']
404 405 for key1, value1 in kwargs.items():
405 406 if not key1 in key_replace:
406 407 if key1 in key_point:
407 408 self.dict[key1.replace('_', '.')] = value1
408 409 else:
409 410 self.dict[key1] = value1
410 411
411 412 if query is not None:
412 413 if type(query) is dict:
413 414 self.dict['fq_list'] = []
414 415 #NUM_RESOURCES_MIN / NUM_RESOURCES_MAX
415 416 #----------------------------------------------------#
416 417 if 'dataset_start_date' in query:
417 418 if type(query['dataset_start_date']) is str:
418 419 try:
419 420 datetime.strptime(query['dataset_start_date'], '%Y-%m-%d')
420 421 if len(query['dataset_start_date']) != 10:
421 422 return '"dataset_start_date", must be: <YYYY-MM-DD>'
422 423 self.dict['fq_list'].append('dataset_start_date:"'+query['dataset_start_date']+'"')
423 424 self.list.append('dataset_start_date')
424 425 except:
425 426 return '"dataset_start_date" incorrect: "%s"' % (query['dataset_start_date'])
426 427 else:
427 428 return '"dataset_start_date" must be <str>'
428 429 #----------------------------------------------------#
429 430 if 'dataset_end_date' in query:
430 431 if type(query['dataset_end_date']) is str:
431 432 try:
432 433 datetime.strptime(query['dataset_end_date'], '%Y-%m-%d')
433 434 if len(query['dataset_end_date']) != 10:
434 435 return '"dataset_end_date", must be: <YYYY-MM-DD>'
435 436
436 437 if 'dataset_start_date' in query:
437 438 if query['dataset_start_date'] > query['dataset_end_date']:
438 439 return '"dataset_end_date" must be greater than "dataset_start_date"'
439 440
440 441 self.dict['fq_list'].append('dataset_end_date:"'+query['dataset_end_date']+'"')
441 442 self.list.append('dataset_end_date')
442 443 except:
443 444 return '"dataset_end_date" incorrect: "%s"' % (query['dataset_end_date'])
444 445 else:
445 446 return '"dataset_end_date" must be <str>'
446 447 #----------------------------------------------------#
447 448 for key, value in query.items():
448 449 if value is not None and not key in self.list:
449 450 self.dict['fq_list'].append(str(key)+':"'+str(value)+'"')
450 451 else:
451 452 return '"query" must be <dict>'
452 453
453 454 return getattr(self.ckan.action, 'package_search')(include_private=True, **self.dict)
454 455
455 456 elif type_option == 'resource':
456 457 for key1, value1 in kwargs.items():
457 458 if key1 != 'fields':
458 459 self.dict[key1] = value1
459 460
460 461 if query is not None:
461 462 if type(query) is dict:
462 463 #----------------------------------------------------#
463 464 if 'file_date_min' in query:
464 465 if type(query['file_date_min']) is str:
465 466 try:
466 467 datetime.strptime(query['file_date_min'], '%Y-%m-%d')
467 468 if len(query['file_date_min']) != 10:
468 469 return '"file_date_min", must be: <YYYY-MM-DD>'
469 470 except:
470 471 return '"file_date_min" incorrect: "%s"' % (query['file_date_min'])
471 472 else:
472 473 return '"file_date_min" must be <str>'
473 474 #----------------------------------------------------#
474 475 if 'file_date_max' in query:
475 476 if type(query['file_date_max']) is str:
476 477 try:
477 478 datetime.strptime(query['file_date_max'], '%Y-%m-%d')
478 479 if len(query['file_date_max']) != 10:
479 480 return '"file_date_max", must be: <YYYY-MM-DD>'
480 481
481 482 if 'file_date_min' in query:
482 483 if query['file_date_min'] > query['file_date_max']:
483 484 return '"file_date_max" must be greater than "file_date_min"'
484 485 except:
485 486 return '"file_date_max" incorrect: "%s"' % (query['file_date_max'])
486 487 else:
487 488 return '"file_date_max" must be <str>'
488 489 #----------------------------------------------------#
489 490 self.dict['query'] = query
490 491 else:
491 492 return '"query" must be <dict>'
492 493 return getattr(self.ckan.action, 'resources_search')(**self.dict)
493 494
494 495 elif type_option == 'tag':
495 496 for key1, value1 in kwargs.items():
496 497 if key1 != 'fields':
497 498 self.dict[key1] = value1
498 499
499 500 if not 'vocabulary_id' in kwargs:
500 501 print('Missing "vocabulary_id" value: tags that don’t belong to any vocabulary')
501 502 else:
502 503 print('Only tags that belong to "{}" vocabulary'.format(kwargs['vocabulary_id']))
503 504
504 505 if query is not None:
505 506 if type(query) is dict:
506 507 if 'search' in query:
507 508 if type(query['search']) is list or type(query['search']) is str:
508 509 self.dict['query'] = query['search']
509 510 else:
510 511 return '"search" must be <list> or <str>'
511 512 else:
512 513 return '"query" must be <dict>'
513 514 return getattr(self.ckan.action, 'tag_search')(**self.dict)
514 515
515 516 else:
516 return 'ERROR:: "%s" is not accepted' % (type_option)
517 return 'ERROR:: "type_option = %s" is not accepted' % (type_option)
517 518
518 519 except:
519 520 _, exc_value, _ = sys.exc_info()
520 521 return exc_value
521 522 else:
522 523 return 'ERROR:: "type_option" must be <str>'
523 524
524 def create(self, type_option, **kwargs):
525 def create(self, type_option, select=None, **kwargs):
525 526 '''
526 527 FINALIDAD:
527 528 Funcion personalizada para crear.
528 529
529 530 PARAMETROS DISPONIBLES:
530 531 CONSULTAR: "GUIA DE SCRIPT.pdf"
531 532
532 533 ESTRUCTURA:
533 534 <access_name>.create(type_option = <class 'str'>, param_1 = <class 'param_1'>, ...)
534 535 '''
535 536 if type(type_option) is str:
536 537 try:
537 538 if type_option == 'dataset':
538 539 return getattr(self.ckan.action, 'package_create')(**kwargs)
539 540 elif type_option == 'project':
540 541 return getattr(self.ckan.action, 'organization_create')(**kwargs)
541 542 elif type_option == 'member':
542 543 return getattr(self.ckan.action, 'organization_member_create')(**kwargs)
543 544 elif type_option == 'collaborator':
544 545 return getattr(self.ckan.action, 'package_collaborator_create')(**kwargs)
545 546 elif type_option == 'vocabulary':
546 547 return getattr(self.ckan.action, 'vocabulary_create')(**kwargs)
547 548 elif type_option == 'tag':
548 549 return getattr(self.ckan.action, 'tag_create')(**kwargs)
549 550 elif type_option == 'user':
550 551 return getattr(self.ckan.action, 'user_create')(**kwargs)
552 elif type_option == 'views':
553 if 'resource' == select:
554 self.list = ['package']
555 for key1, value1 in kwargs.items():
556 if not key1 in self.list:
557 self.dict[key1] = value1
558 return getattr(self.ckan.action, 'resource_create_default_resource_views')(**self.dict)
559 elif 'dataset' == select:
560 return getattr(self.ckan.action, 'package_create_default_resource_views')(**kwargs)
561 else:
562 return 'ERROR:: "select = %s" is not accepted' % (select)
551 563 else:
552 return 'ERROR:: "%s" is not accepted' % (type_option)
564 return 'ERROR:: "type_option = %s" is not accepted' % (type_option)
553 565 except:
554 566 _, exc_value, _ = sys.exc_info()
555 567 return exc_value
556 568 else:
557 569 return 'ERROR:: "type_option" must be <str>'
558 570
559 571 def patch(self, type_option, **kwargs):
560 572 '''
561 573 FINALIDAD:
562 574 Funciones personalizadas para actualizar
563 575
564 576 PARAMETROS DISPONIBLES:
565 577 CONSULTAR: "GUIA DE SCRIPT.pdf"
566 578
567 579 ESTRUCTURA:
568 580 <access_name>.patch(type_option = <class 'str'>, param_1 = <class 'param_1'>, ...)
569 581 '''
570 582 if type(type_option) is str:
571 583 try:
572 584 if type_option == 'dataset':
573 585 return getattr(self.ckan.action, 'package_patch')(**kwargs)
574 586 elif type_option == 'project':
575 587 return getattr(self.ckan.action, 'organization_patch')(**kwargs)
576 588 elif type_option == 'resource':
577 589 return getattr(self.ckan.action, 'resource_patch')(**kwargs)
578 590 elif type_option == 'member':
579 591 return getattr(self.ckan.action, 'organization_member_create')(**kwargs)
580 592 elif type_option == 'collaborator':
581 593 return getattr(self.ckan.action, 'package_collaborator_create')(**kwargs)
582 594 else:
583 return 'ERROR:: "%s" is not accepted' % (type_option)
595 return 'ERROR:: "type_option = %s" is not accepted' % (type_option)
584 596 except:
585 597 _, exc_value, _ = sys.exc_info()
586 598 return exc_value
587 599 else:
588 600 return 'ERROR:: "type_option" must be <str>'
589 601
590 602 def delete(self, type_option, select=None, **kwargs):
591 603 '''
592 604 FINALIDAD:
593 605 Función personalizada para eliminar y/o purgar.
594 606
595 607 PARAMETROS DISPONIBLES:
596 608 CONSULTAR: "GUIA DE SCRIPT.pdf"
597 609
598 610 ESTRUCTURA:
599 611 <access_name>.delete(type_option = <class 'str'>, param_1 = <class 'param_1'>, ...)
600 612 '''
601 613 if type(type_option) is str:
602 614 try:
603 615 if type_option == 'dataset':
604 616 if select is None:
605 617 return 'ERROR:: "select" must not be "None"'
606 618 else:
607 619 if 'delete' == select:
608 620 return getattr(self.ckan.action, 'package_delete')(**kwargs)
609 621 elif 'purge' == select:
610 622 return getattr(self.ckan.action, 'dataset_purge')(**kwargs)
611 623 else:
612 return 'ERROR:: "%s" is not accepted' % (select)
624 return 'ERROR:: "select = %s" is not accepted' % (select)
613 625 elif type_option == 'project':
614 626 if select is None:
615 627 return 'ERROR:: "select" must not be "None"'
616 628 else:
617 629 if 'delete' == select:
618 630 return getattr(self.ckan.action, 'organization_delete')(**kwargs)
619 631 elif 'purge' == select:
620 632 return getattr(self.ckan.action, 'organization_purge')(**kwargs)
621 633 else:
622 return 'ERROR:: "%s" is not accepted' % (select)
634 return 'ERROR:: "select = %s" is not accepted' % (select)
623 635 elif type_option == 'resource':
624 636 return getattr(self.ckan.action, 'resource_delete')(**kwargs)
625 637 elif type_option == 'vocabulary':
626 638 return getattr(self.ckan.action, 'vocabulary_delete')(**kwargs)
627 639 elif type_option == 'tag':
628 640 return getattr(self.ckan.action, 'tag_delete')(**kwargs)
629 641 elif type_option == 'user':
630 642 return getattr(self.ckan.action, 'user_delete')(**kwargs)
631 643 else:
632 return 'ERROR:: "%s" is not accepted' % (type_option)
644 return 'ERROR:: "type_option = %s" is not accepted' % (type_option)
633 645 except:
634 646 _, exc_value, _ = sys.exc_info()
635 647 return exc_value
636 648 else:
637 649 return 'ERROR:: "type_option" must be <str>'
638 650
639 651 def f_status_note(self, total, result, path):
640 652 file_txt = open(path+'status_note.txt', 'w')
641 653 file_txt = open(path+'status_note.txt', 'a')
642 654
643 655 file_txt.write('DOWNLOADED FILE(S): "%s"' % (len(result['name'])))
644 656 file_txt.write(''+ os.linesep)
645 657 for u in result['name']:
646 658 file_txt.write(' - '+ u + os.linesep)
647 659 file_txt.write(''+ os.linesep)
648 660
649 661 file_txt.write('FAILED FILE(S): "%s"' % (len(total['name'])-len(result['name'])))
650 662 file_txt.write(''+ os.linesep)
651 663 if len(total['name'])-len(result['name']) != 0:
652 664 for u in total['name']:
653 665 if not u in result['name']:
654 666 file_txt.write(' - '+ u + os.linesep)
655 667 else:
656 668 file_txt.write(' "None"'+ os.linesep)
657 669
658 670 def f_name(self, name_dataset, ext, tempdir):
659 671 while self.check:
660 672 self.str = ''
661 673 if self.cont == 0:
662 674 if os.path.exists(tempdir + name_dataset + ext):
663 675 self.str = name_dataset+'('+str(self.cont+1)+')'+ext
664 676 else:
665 677 self.check = self.check * 0
666 678 self.str = name_dataset + ext
667 679 else:
668 680 if not os.path.exists(tempdir + name_dataset+'('+str(self.cont)+')'+ext):
669 681 self.check = self.check * 0
670 682 self.str = name_dataset+'('+str(self.cont)+')'+ ext
671 683 self.cont = self.cont+1
672 684 return self.str
673 685
674 686 def f_zipdir(self, path, ziph, zip_name):
675 687 for root, _, files in os.walk(path):
676 688 print('.....')
677 689 print('Creating: "{}" >>'.format(zip_name))
678 690 for __file in tqdm(iterable=files, total=len(files)):
679 691 new_dir = os.path.relpath(os.path.join(root, __file), os.path.join(path, '..'))
680 692 ziph.write(os.path.join(root, __file), new_dir)
681 693 print('Created >>')
682 694
683 695 def download_by_step(self, response, tempdir_name):
684 696 try:
685 697 with requests.get(response['url'], stream=True, headers={'Authorization': self.Authorization}) as resp:
686 698 if resp.status_code == 200:
687 699 with open(tempdir_name+response['name'], 'wb') as file:
688 700 for chunk in resp.iter_content(chunk_size = self.chunk_size):
689 701 if chunk:
690 702 file.write(chunk)
691 703 except requests.exceptions.RequestException:
692 704 pass
693 705
694 706 def download_files(self, **kwargs):
695 707 '''
696 708 FINALIDAD:
697 709 Funcion personalizada para la descarga de archivos existentes de un dataset.
698 710
699 711 PARAMETROS DISPONIBLES:
700 712 CONSULTAR: "GUIA DE SCRIPT.pdf"
701 713
702 714 ESTRUCTURA:
703 715 <access_name>.download_files(id = <class 'str'>, param_1 = <class 'param_1'>, ...)
704 716 '''
705 717 dict_local = {}
706 718 #----------------------------------------------#
707 719 if 'zip' in kwargs:
708 720 if type(kwargs['zip']) is not bool:
709 721 return 'ERROR:: "zip" must be: <class "bool">'
710 722 else:
711 723 dict_local['zip'] = kwargs['zip']
712 724 else:
713 725 dict_local['zip'] = False
714 726 #----------------------------------------------#
715 727 if 'status_note' in kwargs:
716 728 if type(kwargs['status_note']) is not bool:
717 729 return 'ERROR:: "status_note" must be: <class "bool">'
718 730 else:
719 731 dict_local['status_note'] = kwargs['status_note']
720 732 else:
721 733 dict_local['status_note'] = False
722 734 #----------------------------------------------#
723 735 if 'path' in kwargs:
724 736 if type(kwargs['path']) is str:
725 737 if os.path.isdir(kwargs['path']) == False:
726 738 return 'ERROR:: "path" does not exist'
727 739 else:
728 740 if kwargs['path'][-1:] != self.separator:
729 741 dict_local['path'] = kwargs['path']+self.separator
730 742 else:
731 743 dict_local['path'] = kwargs['path']
732 744
733 745 txt = dict_local['path']+datetime.now().strftime("%Y_%m_%d_%H_%M_%S_%f")+'.txt'
734 746 if int(platform.python_version()[0]) == 3:
735 747 try:
736 748 file_txt = open(txt, 'w')
737 749 file_txt.close()
738 750 os.remove(txt)
739 751 except PermissionError:
740 752 return 'ERROR:: Access denied, you are not authorized to write files: "%s"' % (dict_local['path'])
741 753 else:
742 754 try:
743 755 file_txt = open(txt, 'w')
744 756 file_txt.close()
745 757 os.remove(txt)
746 758 except:
747 759 return 'ERROR:: Access denied, you are not authorized to write files: "%s"' % (dict_local['path'])
748 760 else:
749 761 return 'ERROR:: "path" must be: <class "str">'
750 762 else:
751 763 dict_local['path'] = ''
752 764 #----------------------------------------------#
753 765 for key, value in kwargs.items():
754 766 if not key in dict_local:
755 767 self.dict[key] = value
756 768 try:
757 769 response = getattr(self.ckan.action, 'url_resources')(**self.dict)
758 770 except:
759 771 _, exc_value, _ = sys.exc_info()
760 772 return exc_value
761 773
762 774 if len(response) != 0:
763 775 #--------------TEMP PATH---------------#
764 776 if dict_local['zip']:
765 777 tempdir = tempfile.mkdtemp(prefix=kwargs['id']+'-')+self.separator
766 778 os.mkdir(tempdir+kwargs['id'])
767 779 dir_name = tempdir + kwargs['id'] + self.separator
768 780 else:
769 781 dir = self.f_name(kwargs['id'], '', dict_local['path'])
770 782 os.mkdir(dict_local['path'] + dir)
771 783 dir_name = dict_local['path'] + dir + self.separator
772 784 #-----------DOWNLOAD FILES-------------#
773 785 print('.....')
774 786 print('Downloading "{}" file(s) >>'.format(len(response)))
775 787 name_total = {'name': []}
776 788 with concurrent.futures.ThreadPoolExecutor() as executor:
777 789 for u in tqdm(iterable=response, total=len(response)):
778 790 name_total['name'].append(u['name'])
779 791 executor.submit(self.download_by_step, u, dir_name)
780 792 name_check = {}
781 793 name_check['name'] = [f for f in os.listdir(dir_name) if os.path.isfile(os.path.join(dir_name, f))]
782 794 print('"{}" downloaded file(s) successfully >>'.format(len(name_check['name'])))
783 795 #--------------------------------------#
784 796 if len(name_check['name']) != 0:
785 797 #----------Status Note---------#
786 798 if dict_local['status_note']:
787 799 print('.....')
788 800 print('Creating: "status_note.txt" >>')
789 801 self.f_status_note(name_total, name_check, dir_name)
790 802 print('Created>>')
791 803 #----------ZIP CREATE----------#
792 804 if dict_local['zip']:
793 805 zip_name = self.f_name(kwargs['id'], '.zip', dict_local['path'])
794 806 ziph = zipfile.ZipFile(dict_local['path'] + zip_name, 'w', zipfile.ZIP_DEFLATED, allowZip64=True)
795 807 self.f_zipdir(dir_name, ziph, zip_name)
796 808 ziph.close()
797 809 #Delete Temporal Path
798 810 if os.path.exists(tempdir[:-1]):
799 811 shutil.rmtree(tempdir[:-1])
800 812 #------------------------------#
801 813 print('.....')
802 814 return 'DOWNLOAD FINISHED'
803 815 else:
804 816 #Delete Temporal Path
805 817 if dict_local['zip']:
806 818 if os.path.exists(tempdir[:-1]):
807 819 shutil.rmtree(tempdir[:-1])
808 820 else:
809 821 if os.path.exists(dir_name[:-1]):
810 822 shutil.rmtree(dir_name[:-1])
811 823 return 'NO FILES WERE DOWNLOADED'
812 824 else:
813 825 return 'FILES NOT FOUND' No newline at end of file
General Comments 0
You need to be logged in to leave comments. Login now