##// END OF EJS Templates
DATABASES » ckanext-jro » api-cliente
RSS
v2.9.2 :: Add 'upload_multiple_files_advance' function
eynilupu -
r2:f8e143a2e63e
parent child
Show More
Show file before | Show file after | Hide comments
1 NO CONTENT: modified file, binary diff hidden
Show file before | Show file after | Hide comments
@@ -1,704 +1,813
1 1 from ckanapi import RemoteCKAN
2 2 #from ckanapi.errors import NotAuthorized, NotFound, ValidationError, SearchQueryError, SearchError, CKANAPIError, ServerIncompatibleError
3 3 import sys
4 4 import platform
5 5 import os
6 6 import tempfile
7 7 import shutil
8 8 import zipfile
9 9 import concurrent.futures
10 10 import requests
11 11 import json
12 import ast
12 import pathlib
13 import uuid
13 14 from datetime import datetime
14 15 from tqdm import tqdm
15 from six import string_types
16 16
17 17 class JROAPI():
18 18 """
19 19 FINALIDAD:
20 20 Script para administrar y obtener la data del repositorio por medio de APIs.
21 21
22 22 REQUISITIOS PREVIOS:
23 23 - Paso 1: Tener "pip [Python 2]" o "pip3 [Python 3]" instalado:
24 24 - Paso 2: Instalar lo siguiente como admininstrador:
25 25 En Python 2
26 26 - pip install ckanapi==4.5
27 27 - pip install requests
28 28 - pip install tqdm
29 29 En Python 3
30 30 - pip3 install ckanapi==4.5
31 31 - pip3 install requests
32 32 - pip3 install tqdm
33 33
34 34 FUNCIONES DISPONIBLES:
35 35 - action
36 36 - upload_file
37 37 - upload_multiple_files
38 38 - show
39 39 - search
40 40 - create
41 41 - patch
42 42 - delete
43 43 - download_files
44 44
45 45 EJEMPLOS:
46 46 #1:
47 47 with JROAPI('http://demo.example.com', Authorization='#########') as <access_name>:
48 48 ... some operation(s) ...
49 49 #2:
50 50 <access_name> = JROAPI('http://example.com', Authorization='#########')
51 51 ... some operation(s) ...
52 52 <access_name>.ckan.close()
53 53
54 54 REPORTAR ALGUN PROBLEMA:
55 55 Debe enviar un correo a eynilupu@igp.gob.pe detallando los siguientes pasos:
56 56 1) Identifiquese
57 57 2) Describir el problema
58 58 3) ¿En que funcion esta el problema?
59 59 4) ¿Que esperaba que hiciera la funcion sin el problema?
60 60 """
61 61 def __init__(self, url, Authorization=None):
62 62 ua = 'CKAN_JRO/1.1 (+'+str(url)+')'
63 63 #ua = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36'
64 64 self.ckan = RemoteCKAN(url, apikey=Authorization, user_agent=ua)
65 65 #self.ckan = RemoteCKAN(url, apikey=Authorization)
66 66 self.Authorization = Authorization
67 67 if platform.system() == 'Windows':
68 68 self.separator = '\\'
69 69 else:
70 70 self.separator = '/'
71 71
72 72 self.chunk_size = 1024
73 73 self.list = []
74 74 self.dict = {}
75 75 self.str = ''
76 76 self.check = 1
77 77 self.cont = 0
78 78
79 79 def __enter__(self):
80 80 return self
81 81
82 82 def __exit__(self, *args):
83 83 self.ckan.close()
84 84
85 85 def action(self, action, **kwargs):
86 86 """
87 87 FINALIDAD:
88 88 Funcion para llamar a las APIs disponibles
89 89
90 90 APIs DISPONIBLES:
91 91 CONSULTAR: "GUIA DE SCRIPT.pdf"
92 92
93 93 EJEMPLO:
94 94 <access_name>.action(<consuming API>, param_1 = <class 'param_1'>, ...)
95 95 """
96 96 #--------------- CASE: PACKAGE SEARCH ---------------#
97 97 if kwargs is not None:
98 98 if action == 'package_search':
99 99 self.list = ['facet_mincount', 'facet_limit', 'facet_field']
100 100 for facet in self.list:
101 101 if facet in kwargs:
102 102 kwargs[facet.replace('_', '.')] = kwargs[facet]
103 103 kwargs.pop(facet)
104 104 #----------------------------------------------------#
105 105 try:
106 106 return getattr(self.ckan.action, action)(**kwargs)
107 107 except:
108 108 _, exc_value, _ = sys.exc_info()
109 109 return exc_value
110 110
111 111 def upload_file(self, dataset_id, file_path, file_date, file_type, **kwargs):
112 112 '''
113 113 FINALIDAD:
114 114 Funcion para subir un unico archivo al repositorio del ROJ.
115 115
116 116 PARAMETROS DISPONIBLES:
117 117 CONSULTAR: "GUIA DE SCRIPT.pdf"
118 118
119 119 ESTRUCTURA:
120 120 <access_name>.upload_file(dataset_id = <class 'str'>, file_date = <class 'str'>, file_path = <class 'str'>, file_type = <class 'str'>, param_1 = <class 'param_1'>, ...)
121 121 '''
122 122 self.list = ['package_id', 'upload', 'voc_file_type', 'name'] #file_date
123 123 for key1, value1 in kwargs.items():
124 124 if not key1 in self.list:
125 125 self.dict[key1] = value1
126 126
127 127 #---------------------------#
128 128 if not 'others' in kwargs:
129 129 self.dict['others'] = ''
130 130 else:
131 131 if isinstance(kwargs['others'], list):
132 132 self.dict['others'] = json.dumps(kwargs['others'])
133 133 #---------------------------#
134 134
135 135 if not os.path.isfile(file_path):
136 136 return 'File "%s" not exist' % (file_path)
137 137
138 if not 'format' in self.dict:
139 self.str = ''.join(pathlib.Path(file_path).suffixes)
140 if len(self.str) > 0:
141 self.dict['format'] = self.str.upper()[1:]
142
143 try:
144 return getattr(self.ckan.action, 'resource_create')(package_id=dataset_id, file_date=file_date, upload=open(file_path, 'rb'), voc_file_type=file_type, name=pathlib.Path(file_path).name, **self.dict)
145 except:
146 _, exc_value, _ = sys.exc_info()
147 return exc_value
148
149
150 def upload_multiple_files_advance(self, dataset_id, path_files, file_date, file_type, **kwargs):
151 '''
152 FINALIDAD:
153 Funcion para subir multiples archivos al repositorio del ROJ.
154
155 PARAMETROS DISPONIBLES:
156 CONSULTAR: "GUIA DE SCRIPT.pdf"
157
158 ESTRUCTURA:
159 <access_name>.upload_multiple_files_advance(dataset_id = <class 'str'>, path_files = <class 'list of strings'>, file_date = <class 'str'>, file_type = <class 'str'>, param_1 = <class 'param_1'>, ...)
160 '''
161 self.list = ['package_id', 'upload', 'voc_file_type', 'name']
162
163 for key1, value1 in kwargs.items():
164 if not key1 in self.list:
165 self.dict[key1] = value1
166
167 #---------------------------#
168 if not 'others' in kwargs:
169 self.dict['others'] = ''
170 else:
171 if isinstance(kwargs['others'], list):
172 self.dict['others'] = json.dumps(kwargs['others'])
173 #---------------------------#
174
175 params_dict = {'upload':[], 'name':[]}
176 #---------------CASO : "path" or "path_list"-----------------#
177 if type(path_files) is list:
178 if len(path_files) != 0:
179 path_files.sort()
180 for u in path_files:
181 if os.path.isfile(u):
182 params_dict['upload'].append(open(u, 'rb'))
183 params_dict['name'].append(pathlib.Path(u).name)
184 else:
185 return 'File "%s" does not exist' % (u)
186 else:
187 return 'ERROR:: "path_list is empty"'
188 elif type(path_files) is str:
189 if os.path.isdir(path_files):
190 path_order = [f for f in os.listdir(path_files) if os.path.isfile(os.path.join(path_files, f))]
191 path_order.sort()
192 if path_order:
193 for name in path_order:
194 params_dict['upload'].append(open(os.path.join(path_files, name), 'rb'))
195 params_dict['name'].append(name)
196 else:
197 return "ERROR:: There aren't files in this directory"
198 else:
199 return 'ERROR:: Directory "%s" does not exist' % (path_files)
200 else:
201 return 'ERROR:: "path_files" must be a str or list'
202 #------------------------------------------------------------#
203 resource_extend = []
204 files_dict = {}
205 for count, name in enumerate(params_dict['name']):
206 param_list = {"name": name, "file_date": file_date, "voc_file_type": file_type}
207 param_list.update(self.dict)
208
209 if not 'format' in param_list:
210 format = ''.join(pathlib.Path(name).suffixes)
211 if len(format) > 0:
212 param_list['format'] = format.upper()[1:]
213
214 resource_extend.append(param_list)
215 files_dict['update__resources__-'+ str(len(params_dict['name'])-count) +'__upload'] = (name, params_dict['upload'][count])
216
217 #------------------------------------------------------------#
218 try:
219 uuid.UUID(str(dataset_id), version=4)
220 package_id_or_name = '"id": "' + str(dataset_id) + '"'
221 except ValueError:
222 package_id_or_name = '"name": "' + str(dataset_id) + '"'
223 #------------------------------------------------------------#
224 print('"{}" file(s) found >> uploading'.format(len(params_dict['name'])))
138 225 try:
139 return getattr(self.ckan.action, 'resource_create')(package_id=dataset_id, file_date=file_date, upload=open(file_path, 'rb'), voc_file_type=file_type, name=file_path.split(self.separator)[-1], **self.dict)
226 result = self.ckan.call_action(
227 'package_revise',
228 {'match': '{'+ str(package_id_or_name) +'}', 'update__resources__extend': json.dumps(resource_extend)},
229 files=files_dict
230 )
231 print('Uploaded file(s) successfully')
232 return result
140 233 except:
234 print('ERROR :: Use the "print" option for more information')
141 235 _, exc_value, _ = sys.exc_info()
142 236 return exc_value
143 237
144 238 def upload_multiple_files(self, dataset_id, path_files, date_files, type_files, **kwargs):
145 239 '''
146 240 FINALIDAD:
147 241 Funcion para subir multiples archivos al repositorio del ROJ.
148 242
149 243 PARAMETROS DISPONIBLES:
150 244 CONSULTAR: "GUIA DE SCRIPT.pdf"
151 245
152 246 ESTRUCTURA:
153 247 <access_name>.upload_multiple_files(dataset_id = <class 'str'>, path_files = <class 'str'> or <class 'list of strings'>, date_files = <class 'str'> or <class 'list of strings'>, type_files = <class 'str'> or <class 'list of strings'>, param_1 = <class 'param_1'>, ...)
154 248 '''
155 params_dict = {'upload':[], 'name':[]} #ADD FORMAT
249
250 params_dict = {'upload':[], 'name':[]}
251 if not 'format' in kwargs:
252 params_dict.update({'format':[]})
156 253 #---------------CASO : "path" or "path_list"-----------------#
157 254 if type(path_files) is list:
158 255 if len(path_files) != 0:
159 256 path_files.sort()
160 257 for u in path_files:
161 258 if os.path.isfile(u):
162 259 params_dict['upload'].append(open(u, 'rb'))
163 params_dict['name'].append(u.split(self.separator)[-1])
260 params_dict['name'].append(pathlib.Path(u).name)
261 if not 'format' in kwargs:
262 format = ''.join(pathlib.Path(u).suffixes)
263 if len(format) > 0:
264 params_dict['format'].append(format.upper()[1:])
164 265 else:
165 return 'File "%s" not exist' % (u)
266 params_dict['format'].append('')
267 else:
268 return 'File "%s" does not exist' % (u)
166 269 else:
167 270 return 'ERROR:: "path_list is empty"'
168 271 elif type(path_files) is str:
169 272 if os.path.isdir(path_files):
170 273 path_order = [f for f in os.listdir(path_files) if os.path.isfile(os.path.join(path_files, f))]
171 274 path_order.sort()
172 275 if path_order:
173 276 for name in path_order:
174 277 params_dict['upload'].append(open(os.path.join(path_files, name), 'rb'))
175 278 params_dict['name'].append(name)
279 if not 'format' in kwargs:
280 format = ''.join(pathlib.Path(name).suffixes)
281 if len(format) > 0:
282 params_dict['format'].append(format.upper()[1:])
283 else:
284 params_dict['format'].append('')
176 285 else:
177 return 'ERROR:: Directory is empty'
286 return "ERROR:: There aren't files in this directory"
178 287 else:
179 return 'ERROR:: Directory "%s" not exist' % (path_files)
288 return 'ERROR:: Directory "%s" does not exist' % (path_files)
180 289 else:
181 290 return 'ERROR:: "path_files" must be a str or list'
182 291 #------------------------------------------------------------#
183 292 params_no_dict = {'package_id': dataset_id}
184 293 if type(date_files) is list:
185 294 params_dict['file_date'] = date_files
186 295 else:
187 296 params_no_dict['file_date'] = date_files
188 297
189 298 if type(type_files) is list:
190 299 params_dict['voc_file_type'] = type_files
191 300 else:
192 301 params_no_dict['voc_file_type'] = type_files
193 302
194 303 for key1, value1 in kwargs.items():
195 304 if not key1 in params_dict and not key1 in params_no_dict and key1 != 'others':
196 305 if type(value1) is list:
197 306 params_dict[key1] = value1
198 307 else:
199 308 params_no_dict[key1] = value1
200 309 #------------------------------------------#
201 310 if not 'others' in kwargs:
202 311 params_no_dict['others'] = ''
203 312 else:
204 313 if isinstance(kwargs['others'], tuple):
205 314 params_dict['others'] = [json.dumps(w) for w in kwargs['others']]
206 315 elif isinstance(kwargs['others'], list):
207 316 params_no_dict['others'] = json.dumps(kwargs['others'])
208 317 elif isinstance(kwargs['others'], str):
209 318 params_no_dict['others'] = kwargs['others']
210 319 else:
211 320 return 'ERROR:: "others" must be a tuple, list or str'
212 321 #------------------------------------------#
213 322 len_params_dict = []
214 323 for value2 in params_dict.values():
215 324 len_params_dict.append(len(value2))
216 325
217 326 if len(list(set(len_params_dict))) > 1:
218 327 return 'ERROR:: All lists must be the same length: %s' % (len(params_dict['name']))
219 328 #------------------------------------------------------------#
220
329 print('"{}" file(s) found >> uploading'.format(len(params_dict['name'])))
221 330 for v in range(len(params_dict['name'])):
222 331 try:
223 332 send = {}
224 333 for key_dict, value_dict in params_dict.items():
225 334 send[key_dict] = value_dict[v]
226 335 for key_no_dict, value_no_dict in params_no_dict.items():
227 336 send[key_no_dict] = value_no_dict
228 337
229 338 self.list.append(getattr(self.ckan.action, 'resource_create')(**send))
230 print('File "{}" was uploaded successfully'.format(params_dict['name'][v]))
339 print('File #{} :: "{}" was uploaded successfully'.format(v+1, params_dict['name'][v]))
231 340 except:
232 341 _, exc_value, _ = sys.exc_info()
233 342 self.list.append(exc_value)
234 print('Error uploading "{}" file'.format(params_dict['name'][v]))
343 print('File #{} :: Error uploading "{}" file'.format(v+1, params_dict['name'][v]))
235 344 return self.list
236 345 #------------------------------------------------------------#
237 346
238 347 def show(self, type_option, id, **kwargs):
239 348 '''
240 349 FINALIDAD:
241 350 Funcion personalizada para una busqueda en especifico.
242 351
243 352 PARAMETROS DISPONIBLES:
244 353 CONSULTAR: "GUIA DE SCRIPT.pdf"
245 354
246 355 ESTRUCTURA:
247 356 <access_name>.show(type_option = <class 'str'>, id = <class 'str'>, param_1 = <class 'param_1'>, ...)
248 357 '''
249 358 if type(type_option) is str:
250 359 try:
251 360 if type_option == 'dataset':
252 361 return getattr(self.ckan.action, 'package_show')(id=id, **kwargs)
253 362 elif type_option == 'resource':
254 363 return getattr(self.ckan.action, 'resource_show')(id=id, **kwargs)
255 364 elif type_option == 'project':
256 365 return getattr(self.ckan.action, 'organization_show')(id=id, **kwargs)
257 366 elif type_option == 'collaborator':
258 367 return getattr(self.ckan.action, 'package_collaborator_list_for_user')(id=id, **kwargs)
259 368 elif type_option == 'member':
260 369 return getattr(self.ckan.action, 'organization_list_for_user')(id=id, **kwargs)
261 370 elif type_option == 'vocabulary':
262 371 return getattr(self.ckan.action, 'vocabulary_show')(id=id, **kwargs)
263 372 elif type_option == 'tag':
264 373 if not 'vocabulary_id' in kwargs:
265 374 print('Missing "vocabulary_id" value: assume it is a free tag')
266 375 return getattr(self.ckan.action, 'tag_show')(id=id, **kwargs)
267 376 elif type_option == 'user':
268 377 return getattr(self.ckan.action, 'user_show')(id=id, **kwargs)
269 378 elif type_option == 'job':
270 379 return getattr(self.ckan.action, 'job_show')(id=id, **kwargs)
271 380 else:
272 381 return 'ERROR:: "%s" is not accepted' % (type_option)
273 382 except:
274 383 _, exc_value, _ = sys.exc_info()
275 384 return exc_value
276 385 else:
277 386 return 'ERROR:: "type_option" must be a str'
278 387
279 388 def search(self, type_option, query=None, **kwargs):
280 389 '''
281 390 FINALIDAD:
282 391 Funcion personalizada para busquedas que satisfagan algun criterio.
283 392
284 393 PARAMETROS DISPONIBLES:
285 394 CONSULTAR: "GUIA DE SCRIPT.pdf"
286 395
287 396 ESTRUCTURA:
288 397 <access_name>.search(type_option = <class 'str'>, query = <class 'dict'>, param_1 = <class 'param_1'>, ...)
289 398 '''
290 399 if type(type_option) is str:
291 400 try:
292 401 if type_option == 'dataset':
293 402 key_replace = ['fq', 'fq_list', 'include_private']
294 403 key_point = ['facet_mincount', 'facet_limit', 'facet_field']
295 404 for key1, value1 in kwargs.items():
296 405 if not key1 in key_replace:
297 406 if key1 in key_point:
298 407 self.dict[key1.replace('_', '.')] = value1
299 408 else:
300 409 self.dict[key1] = value1
301 410
302 411 if query is not None:
303 412 if type(query) is dict:
304 413 self.dict['fq_list'] = []
305 414 #NUM_RESOURCES_MIN / NUM_RESOURCES_MAX
306 415 #----------------------------------------------------#
307 416 if 'dataset_start_date' in query:
308 417 if type(query['dataset_start_date']) is str:
309 418 try:
310 419 datetime.strptime(query['dataset_start_date'], '%Y-%m-%d')
311 420 if len(query['dataset_start_date']) != 10:
312 421 return '"dataset_start_date", must be: <YYYY-MM-DD>'
313 422 self.dict['fq_list'].append('dataset_start_date:"'+query['dataset_start_date']+'"')
314 423 self.list.append('dataset_start_date')
315 424 except:
316 425 return '"dataset_start_date" incorrect: "%s"' % (query['dataset_start_date'])
317 426 else:
318 427 return '"dataset_start_date" must be <str>'
319 428 #----------------------------------------------------#
320 429 if 'dataset_end_date' in query:
321 430 if type(query['dataset_end_date']) is str:
322 431 try:
323 432 datetime.strptime(query['dataset_end_date'], '%Y-%m-%d')
324 433 if len(query['dataset_end_date']) != 10:
325 434 return '"dataset_end_date", must be: <YYYY-MM-DD>'
326 435
327 436 if 'dataset_start_date' in query:
328 437 if query['dataset_start_date'] > query['dataset_end_date']:
329 438 return '"dataset_end_date" must be greater than "dataset_start_date"'
330 439
331 440 self.dict['fq_list'].append('dataset_end_date:"'+query['dataset_end_date']+'"')
332 441 self.list.append('dataset_end_date')
333 442 except:
334 443 return '"dataset_end_date" incorrect: "%s"' % (query['dataset_end_date'])
335 444 else:
336 445 return '"dataset_end_date" must be <str>'
337 446 #----------------------------------------------------#
338 447 for key, value in query.items():
339 448 if value is not None and not key in self.list:
340 449 self.dict['fq_list'].append(str(key)+':"'+str(value)+'"')
341 450 else:
342 451 return '"query" must be <dict>'
343 452
344 453 return getattr(self.ckan.action, 'package_search')(include_private=True, **self.dict)
345 454
346 455 elif type_option == 'resource':
347 456 for key1, value1 in kwargs.items():
348 457 if key1 != 'fields':
349 458 self.dict[key1] = value1
350 459
351 460 if query is not None:
352 461 if type(query) is dict:
353 462 #----------------------------------------------------#
354 463 if 'file_date_min' in query:
355 464 if type(query['file_date_min']) is str:
356 465 try:
357 466 datetime.strptime(query['file_date_min'], '%Y-%m-%d')
358 467 if len(query['file_date_min']) != 10:
359 468 return '"file_date_min", must be: <YYYY-MM-DD>'
360 469 except:
361 470 return '"file_date_min" incorrect: "%s"' % (query['file_date_min'])
362 471 else:
363 472 return '"file_date_min" must be <str>'
364 473 #----------------------------------------------------#
365 474 if 'file_date_max' in query:
366 475 if type(query['file_date_max']) is str:
367 476 try:
368 477 datetime.strptime(query['file_date_max'], '%Y-%m-%d')
369 478 if len(query['file_date_max']) != 10:
370 479 return '"file_date_max", must be: <YYYY-MM-DD>'
371 480
372 481 if 'file_date_min' in query:
373 482 if query['file_date_min'] > query['file_date_max']:
374 483 return '"file_date_max" must be greater than "file_date_min"'
375 484 except:
376 485 return '"file_date_max" incorrect: "%s"' % (query['file_date_max'])
377 486 else:
378 487 return '"file_date_max" must be <str>'
379 488 #----------------------------------------------------#
380 489 self.dict['query'] = query
381 490 else:
382 491 return '"query" must be <dict>'
383 492 return getattr(self.ckan.action, 'resources_search')(**self.dict)
384 493
385 494 elif type_option == 'tag':
386 495 for key1, value1 in kwargs.items():
387 496 if key1 != 'fields':
388 497 self.dict[key1] = value1
389 498
390 499 if not 'vocabulary_id' in kwargs:
391 500 print('Missing "vocabulary_id" value: tags that don’t belong to any vocabulary')
392 501 else:
393 502 print('Only tags that belong to "{}" vocabulary'.format(kwargs['vocabulary_id']))
394 503
395 504 if query is not None:
396 505 if type(query) is dict:
397 506 if 'search' in query:
398 507 if type(query['search']) is list or type(query['search']) is str:
399 508 self.dict['query'] = query['search']
400 509 else:
401 510 return '"search" must be <list> or <str>'
402 511 else:
403 512 return '"query" must be <dict>'
404 513 return getattr(self.ckan.action, 'tag_search')(**self.dict)
405 514
406 515 else:
407 516 return 'ERROR:: "%s" is not accepted' % (type_option)
408 517
409 518 except:
410 519 _, exc_value, _ = sys.exc_info()
411 520 return exc_value
412 521 else:
413 522 return 'ERROR:: "type_option" must be <str>'
414 523
415 524 def create(self, type_option, **kwargs):
416 525 '''
417 526 FINALIDAD:
418 527 Funcion personalizada para crear.
419 528
420 529 PARAMETROS DISPONIBLES:
421 530 CONSULTAR: "GUIA DE SCRIPT.pdf"
422 531
423 532 ESTRUCTURA:
424 533 <access_name>.create(type_option = <class 'str'>, param_1 = <class 'param_1'>, ...)
425 534 '''
426 535 if type(type_option) is str:
427 536 try:
428 537 if type_option == 'dataset':
429 538 return getattr(self.ckan.action, 'package_create')(**kwargs)
430 539 elif type_option == 'project':
431 540 return getattr(self.ckan.action, 'organization_create')(**kwargs)
432 541 elif type_option == 'member':
433 542 return getattr(self.ckan.action, 'organization_member_create')(**kwargs)
434 543 elif type_option == 'collaborator':
435 544 return getattr(self.ckan.action, 'package_collaborator_create')(**kwargs)
436 545 elif type_option == 'vocabulary':
437 546 return getattr(self.ckan.action, 'vocabulary_create')(**kwargs)
438 547 elif type_option == 'tag':
439 548 return getattr(self.ckan.action, 'tag_create')(**kwargs)
440 549 elif type_option == 'user':
441 550 return getattr(self.ckan.action, 'user_create')(**kwargs)
442 551 else:
443 552 return 'ERROR:: "%s" is not accepted' % (type_option)
444 553 except:
445 554 _, exc_value, _ = sys.exc_info()
446 555 return exc_value
447 556 else:
448 557 return 'ERROR:: "type_option" must be <str>'
449 558
450 559 def patch(self, type_option, **kwargs):
451 560 '''
452 561 FINALIDAD:
453 562 Funciones personalizadas para actualizar
454 563
455 564 PARAMETROS DISPONIBLES:
456 565 CONSULTAR: "GUIA DE SCRIPT.pdf"
457 566
458 567 ESTRUCTURA:
459 568 <access_name>.patch(type_option = <class 'str'>, param_1 = <class 'param_1'>, ...)
460 569 '''
461 570 if type(type_option) is str:
462 571 try:
463 572 if type_option == 'dataset':
464 573 return getattr(self.ckan.action, 'package_patch')(**kwargs)
465 574 elif type_option == 'project':
466 575 return getattr(self.ckan.action, 'organization_patch')(**kwargs)
467 576 elif type_option == 'resource':
468 577 return getattr(self.ckan.action, 'resource_patch')(**kwargs)
469 578 elif type_option == 'member':
470 579 return getattr(self.ckan.action, 'organization_member_create')(**kwargs)
471 580 elif type_option == 'collaborator':
472 581 return getattr(self.ckan.action, 'package_collaborator_create')(**kwargs)
473 582 else:
474 583 return 'ERROR:: "%s" is not accepted' % (type_option)
475 584 except:
476 585 _, exc_value, _ = sys.exc_info()
477 586 return exc_value
478 587 else:
479 588 return 'ERROR:: "type_option" must be <str>'
480 589
481 590 def delete(self, type_option, select=None, **kwargs):
482 591 '''
483 592 FINALIDAD:
484 593 Función personalizada para eliminar y/o purgar.
485 594
486 595 PARAMETROS DISPONIBLES:
487 596 CONSULTAR: "GUIA DE SCRIPT.pdf"
488 597
489 598 ESTRUCTURA:
490 599 <access_name>.delete(type_option = <class 'str'>, param_1 = <class 'param_1'>, ...)
491 600 '''
492 601 if type(type_option) is str:
493 602 try:
494 603 if type_option == 'dataset':
495 604 if select is None:
496 605 return 'ERROR:: "select" must not be "None"'
497 606 else:
498 607 if 'delete' == select:
499 608 return getattr(self.ckan.action, 'package_delete')(**kwargs)
500 609 elif 'purge' == select:
501 610 return getattr(self.ckan.action, 'dataset_purge')(**kwargs)
502 611 else:
503 612 return 'ERROR:: "%s" is not accepted' % (select)
504 613 elif type_option == 'project':
505 614 if select is None:
506 615 return 'ERROR:: "select" must not be "None"'
507 616 else:
508 617 if 'delete' == select:
509 618 return getattr(self.ckan.action, 'organization_delete')(**kwargs)
510 619 elif 'purge' == select:
511 620 return getattr(self.ckan.action, 'organization_purge')(**kwargs)
512 621 else:
513 622 return 'ERROR:: "%s" is not accepted' % (select)
514 623 elif type_option == 'resource':
515 624 return getattr(self.ckan.action, 'resource_delete')(**kwargs)
516 625 elif type_option == 'vocabulary':
517 626 return getattr(self.ckan.action, 'vocabulary_delete')(**kwargs)
518 627 elif type_option == 'tag':
519 628 return getattr(self.ckan.action, 'tag_delete')(**kwargs)
520 629 elif type_option == 'user':
521 630 return getattr(self.ckan.action, 'user_delete')(**kwargs)
522 631 else:
523 632 return 'ERROR:: "%s" is not accepted' % (type_option)
524 633 except:
525 634 _, exc_value, _ = sys.exc_info()
526 635 return exc_value
527 636 else:
528 637 return 'ERROR:: "type_option" must be <str>'
529 638
530 639 def f_status_note(self, total, result, path):
531 640 file_txt = open(path+'status_note.txt', 'w')
532 641 file_txt = open(path+'status_note.txt', 'a')
533 642
534 643 file_txt.write('DOWNLOADED FILE(S): "%s"' % (len(result['name'])))
535 644 file_txt.write(''+ os.linesep)
536 645 for u in result['name']:
537 646 file_txt.write(' - '+ u + os.linesep)
538 647 file_txt.write(''+ os.linesep)
539 648
540 649 file_txt.write('FAILED FILE(S): "%s"' % (len(total['name'])-len(result['name'])))
541 650 file_txt.write(''+ os.linesep)
542 651 if len(total['name'])-len(result['name']) != 0:
543 652 for u in total['name']:
544 653 if not u in result['name']:
545 654 file_txt.write(' - '+ u + os.linesep)
546 655 else:
547 656 file_txt.write(' "None"'+ os.linesep)
548 657
549 658 def f_name(self, name_dataset, ext, tempdir):
550 659 while self.check:
551 660 self.str = ''
552 661 if self.cont == 0:
553 662 if os.path.exists(tempdir + name_dataset + ext):
554 663 self.str = name_dataset+'('+str(self.cont+1)+')'+ext
555 664 else:
556 665 self.check = self.check * 0
557 666 self.str = name_dataset + ext
558 667 else:
559 668 if not os.path.exists(tempdir + name_dataset+'('+str(self.cont)+')'+ext):
560 669 self.check = self.check * 0
561 670 self.str = name_dataset+'('+str(self.cont)+')'+ ext
562 671 self.cont = self.cont+1
563 672 return self.str
564 673
565 674 def f_zipdir(self, path, ziph, zip_name):
566 675 for root, _, files in os.walk(path):
567 676 print('.....')
568 677 print('Creating: "{}" >>'.format(zip_name))
569 678 for __file in tqdm(iterable=files, total=len(files)):
570 679 new_dir = os.path.relpath(os.path.join(root, __file), os.path.join(path, '..'))
571 680 ziph.write(os.path.join(root, __file), new_dir)
572 681 print('Created >>')
573 682
574 683 def download_by_step(self, response, tempdir_name):
575 684 try:
576 685 with requests.get(response['url'], stream=True, headers={'Authorization': self.Authorization}) as resp:
577 686 if resp.status_code == 200:
578 687 with open(tempdir_name+response['name'], 'wb') as file:
579 688 for chunk in resp.iter_content(chunk_size = self.chunk_size):
580 689 if chunk:
581 690 file.write(chunk)
582 691 except requests.exceptions.RequestException:
583 692 pass
584 693
585 694 def download_files(self, **kwargs):
586 695 '''
587 696 FINALIDAD:
588 697 Funcion personalizada para la descarga de archivos existentes de un dataset.
589 698
590 699 PARAMETROS DISPONIBLES:
591 700 CONSULTAR: "GUIA DE SCRIPT.pdf"
592 701
593 702 ESTRUCTURA:
594 703 <access_name>.download_files(id = <class 'str'>, param_1 = <class 'param_1'>, ...)
595 704 '''
596 705 dict_local = {}
597 706 #----------------------------------------------#
598 707 if 'zip' in kwargs:
599 708 if type(kwargs['zip']) is not bool:
600 709 return 'ERROR:: "zip" must be: <class "bool">'
601 710 else:
602 711 dict_local['zip'] = kwargs['zip']
603 712 else:
604 713 dict_local['zip'] = False
605 714 #----------------------------------------------#
606 715 if 'status_note' in kwargs:
607 716 if type(kwargs['status_note']) is not bool:
608 717 return 'ERROR:: "status_note" must be: <class "bool">'
609 718 else:
610 719 dict_local['status_note'] = kwargs['status_note']
611 720 else:
612 721 dict_local['status_note'] = False
613 722 #----------------------------------------------#
614 723 if 'path' in kwargs:
615 724 if type(kwargs['path']) is str:
616 725 if os.path.isdir(kwargs['path']) == False:
617 726 return 'ERROR:: "path" does not exist'
618 727 else:
619 728 if kwargs['path'][-1:] != self.separator:
620 729 dict_local['path'] = kwargs['path']+self.separator
621 730 else:
622 731 dict_local['path'] = kwargs['path']
623 732
624 733 txt = dict_local['path']+datetime.now().strftime("%Y_%m_%d_%H_%M_%S_%f")+'.txt'
625 734 if int(platform.python_version()[0]) == 3:
626 735 try:
627 736 file_txt = open(txt, 'w')
628 737 file_txt.close()
629 738 os.remove(txt)
630 739 except PermissionError:
631 740 return 'ERROR:: Access denied, you are not authorized to write files: "%s"' % (dict_local['path'])
632 741 else:
633 742 try:
634 743 file_txt = open(txt, 'w')
635 744 file_txt.close()
636 745 os.remove(txt)
637 746 except:
638 747 return 'ERROR:: Access denied, you are not authorized to write files: "%s"' % (dict_local['path'])
639 748 else:
640 749 return 'ERROR:: "path" must be: <class "str">'
641 750 else:
642 751 dict_local['path'] = ''
643 752 #----------------------------------------------#
644 753 for key, value in kwargs.items():
645 754 if not key in dict_local:
646 755 self.dict[key] = value
647 756 try:
648 757 response = getattr(self.ckan.action, 'url_resources')(**self.dict)
649 758 except:
650 759 _, exc_value, _ = sys.exc_info()
651 760 return exc_value
652 761
653 762 if len(response) != 0:
654 763 #--------------TEMP PATH---------------#
655 764 if dict_local['zip']:
656 765 tempdir = tempfile.mkdtemp(prefix=kwargs['id']+'-')+self.separator
657 766 os.mkdir(tempdir+kwargs['id'])
658 767 dir_name = tempdir + kwargs['id'] + self.separator
659 768 else:
660 769 dir = self.f_name(kwargs['id'], '', dict_local['path'])
661 770 os.mkdir(dict_local['path'] + dir)
662 771 dir_name = dict_local['path'] + dir + self.separator
663 772 #-----------DOWNLOAD FILES-------------#
664 773 print('.....')
665 774 print('Downloading "{}" file(s) >>'.format(len(response)))
666 775 name_total = {'name': []}
667 776 with concurrent.futures.ThreadPoolExecutor() as executor:
668 777 for u in tqdm(iterable=response, total=len(response)):
669 778 name_total['name'].append(u['name'])
670 779 executor.submit(self.download_by_step, u, dir_name)
671 780 name_check = {}
672 781 name_check['name'] = [f for f in os.listdir(dir_name) if os.path.isfile(os.path.join(dir_name, f))]
673 782 print('"{}" downloaded file(s) successfully >>'.format(len(name_check['name'])))
674 783 #--------------------------------------#
675 784 if len(name_check['name']) != 0:
676 785 #----------Status Note---------#
677 786 if dict_local['status_note']:
678 787 print('.....')
679 788 print('Creating: "status_note.txt" >>')
680 789 self.f_status_note(name_total, name_check, dir_name)
681 790 print('Created>>')
682 791 #----------ZIP CREATE----------#
683 792 if dict_local['zip']:
684 793 zip_name = self.f_name(kwargs['id'], '.zip', dict_local['path'])
685 794 ziph = zipfile.ZipFile(dict_local['path'] + zip_name, 'w', zipfile.ZIP_DEFLATED, allowZip64=True)
686 795 self.f_zipdir(dir_name, ziph, zip_name)
687 796 ziph.close()
688 797 #Delete Temporal Path
689 798 if os.path.exists(tempdir[:-1]):
690 799 shutil.rmtree(tempdir[:-1])
691 800 #------------------------------#
692 801 print('.....')
693 802 return 'DOWNLOAD FINISHED'
694 803 else:
695 804 #Delete Temporal Path
696 805 if dict_local['zip']:
697 806 if os.path.exists(tempdir[:-1]):
698 807 shutil.rmtree(tempdir[:-1])
699 808 else:
700 809 if os.path.exists(dir_name[:-1]):
701 810 shutil.rmtree(dir_name[:-1])
702 811 return 'NO FILES WERE DOWNLOADED'
703 812 else:
704 813 return 'FILES NOT FOUND' No newline at end of file
General Comments 0
You need to be logged in to leave comments. Login now