##// END OF EJS Templates
DATABASES » ckanext-jro » api-cliente
RSS
v2.9.2 :: Inicial
eynilupu -
r0:435cf79c941d
parent child
Show More
Show file before | Show file after | Hide comments
1 NO CONTENT: new file 100644, binary diff hidden
NO CONTENT: new file 100644, binary diff hidden
Show file before | Show file after | Hide comments
This diff has been collapsed as it changes many lines, (704 lines changed) Show them Hide them
@@ -0,0 +1,704
1 from ckanapi import RemoteCKAN
2 #from ckanapi.errors import NotAuthorized, NotFound, ValidationError, SearchQueryError, SearchError, CKANAPIError, ServerIncompatibleError
3 import sys
4 import platform
5 import os
6 import tempfile
7 import shutil
8 import zipfile
9 import concurrent.futures
10 import requests
11 import json
12 import ast
13 from datetime import datetime
14 from tqdm import tqdm
15 from six import string_types
16
17 class JROAPI():
18 """
19 FINALIDAD:
20 Script para administrar y obtener la data del repositorio por medio de APIs.
21
22 REQUISITIOS PREVIOS:
23 - Paso 1: Tener "pip [Python 2]" o "pip3 [Python 3]" instalado:
24 - Paso 2: Instalar lo siguiente como admininstrador:
25 En Python 2
26 - pip install ckanapi==4.5
27 - pip install requests
28 - pip install tqdm
29 En Python 3
30 - pip3 install ckanapi==4.5
31 - pip3 install requests
32 - pip3 install tqdm
33
34 FUNCIONES DISPONIBLES:
35 - action
36 - upload_file
37 - upload_multiple_files
38 - show
39 - search
40 - create
41 - patch
42 - delete
43 - download_files
44
45 EJEMPLOS:
46 #1:
47 with JROAPI('http://demo.example.com', Authorization='#########') as <access_name>:
48 ... some operation(s) ...
49 #2:
50 <access_name> = JROAPI('http://example.com', Authorization='#########')
51 ... some operation(s) ...
52 <access_name>.ckan.close()
53
54 REPORTAR ALGUN PROBLEMA:
55 Debe enviar un correo a eynilupu@igp.gob.pe detallando los siguientes pasos:
56 1) Identifiquese
57 2) Describir el problema
58 3) ¿En que funcion esta el problema?
59 4) ¿Que esperaba que hiciera la funcion sin el problema?
60 """
61 def __init__(self, url, Authorization=None):
62 ua = 'CKAN_JRO/1.1 (+'+str(url)+')'
63 #ua = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36'
64 self.ckan = RemoteCKAN(url, apikey=Authorization, user_agent=ua)
65 #self.ckan = RemoteCKAN(url, apikey=Authorization)
66 self.Authorization = Authorization
67 if platform.system() == 'Windows':
68 self.separator = '\\'
69 else:
70 self.separator = '/'
71
72 self.chunk_size = 1024
73 self.list = []
74 self.dict = {}
75 self.str = ''
76 self.check = 1
77 self.cont = 0
78
79 def __enter__(self):
80 return self
81
82 def __exit__(self, *args):
83 self.ckan.close()
84
85 def action(self, action, **kwargs):
86 """
87 FINALIDAD:
88 Funcion para llamar a las APIs disponibles
89
90 APIs DISPONIBLES:
91 CONSULTAR: "GUIA DE SCRIPT.pdf"
92
93 EJEMPLO:
94 <access_name>.action(<consuming API>, param_1 = <class 'param_1'>, ...)
95 """
96 #--------------- CASE: PACKAGE SEARCH ---------------#
97 if kwargs is not None:
98 if action == 'package_search':
99 self.list = ['facet_mincount', 'facet_limit', 'facet_field']
100 for facet in self.list:
101 if facet in kwargs:
102 kwargs[facet.replace('_', '.')] = kwargs[facet]
103 kwargs.pop(facet)
104 #----------------------------------------------------#
105 try:
106 return getattr(self.ckan.action, action)(**kwargs)
107 except:
108 _, exc_value, _ = sys.exc_info()
109 return exc_value
110
111 def upload_file(self, dataset_id, file_path, file_date, file_type, **kwargs):
112 '''
113 FINALIDAD:
114 Funcion para subir un unico archivo al repositorio del ROJ.
115
116 PARAMETROS DISPONIBLES:
117 CONSULTAR: "GUIA DE SCRIPT.pdf"
118
119 ESTRUCTURA:
120 <access_name>.upload_file(dataset_id = <class 'str'>, file_date = <class 'str'>, file_path = <class 'str'>, file_type = <class 'str'>, param_1 = <class 'param_1'>, ...)
121 '''
122 self.list = ['package_id', 'upload', 'voc_file_type', 'name'] #file_date
123 for key1, value1 in kwargs.items():
124 if not key1 in self.list:
125 self.dict[key1] = value1
126
127 #---------------------------#
128 if not 'others' in kwargs:
129 self.dict['others'] = ''
130 else:
131 if isinstance(kwargs['others'], list):
132 self.dict['others'] = json.dumps(kwargs['others'])
133 #---------------------------#
134
135 if not os.path.isfile(file_path):
136 return 'File "%s" not exist' % (file_path)
137
138 try:
139 return getattr(self.ckan.action, 'resource_create')(package_id=dataset_id, file_date=file_date, upload=open(file_path, 'rb'), voc_file_type=file_type, name=file_path.split(self.separator)[-1], **self.dict)
140 except:
141 _, exc_value, _ = sys.exc_info()
142 return exc_value
143
144 def upload_multiple_files(self, dataset_id, path_files, date_files, type_files, **kwargs):
145 '''
146 FINALIDAD:
147 Funcion para subir multiples archivos al repositorio del ROJ.
148
149 PARAMETROS DISPONIBLES:
150 CONSULTAR: "GUIA DE SCRIPT.pdf"
151
152 ESTRUCTURA:
153 <access_name>.upload_multiple_files(dataset_id = <class 'str'>, path_files = <class 'str'> or <class 'list of strings'>, date_files = <class 'str'> or <class 'list of strings'>, type_files = <class 'str'> or <class 'list of strings'>, param_1 = <class 'param_1'>, ...)
154 '''
155 params_dict = {'upload':[], 'name':[]} #ADD FORMAT
156 #---------------CASO : "path" or "path_list"-----------------#
157 if type(path_files) is list:
158 if len(path_files) != 0:
159 path_files.sort()
160 for u in path_files:
161 if os.path.isfile(u):
162 params_dict['upload'].append(open(u, 'rb'))
163 params_dict['name'].append(u.split(self.separator)[-1])
164 else:
165 return 'File "%s" not exist' % (u)
166 else:
167 return 'ERROR:: "path_list is empty"'
168 elif type(path_files) is str:
169 if os.path.isdir(path_files):
170 path_order = [f for f in os.listdir(path_files) if os.path.isfile(os.path.join(path_files, f))]
171 path_order.sort()
172 if path_order:
173 for name in path_order:
174 params_dict['upload'].append(open(os.path.join(path_files, name), 'rb'))
175 params_dict['name'].append(name)
176 else:
177 return 'ERROR:: Directory is empty'
178 else:
179 return 'ERROR:: Directory "%s" not exist' % (path_files)
180 else:
181 return 'ERROR:: "path_files" must be a str or list'
182 #------------------------------------------------------------#
183 params_no_dict = {'package_id': dataset_id}
184 if type(date_files) is list:
185 params_dict['file_date'] = date_files
186 else:
187 params_no_dict['file_date'] = date_files
188
189 if type(type_files) is list:
190 params_dict['voc_file_type'] = type_files
191 else:
192 params_no_dict['voc_file_type'] = type_files
193
194 for key1, value1 in kwargs.items():
195 if not key1 in params_dict and not key1 in params_no_dict and key1 != 'others':
196 if type(value1) is list:
197 params_dict[key1] = value1
198 else:
199 params_no_dict[key1] = value1
200 #------------------------------------------#
201 if not 'others' in kwargs:
202 params_no_dict['others'] = ''
203 else:
204 if isinstance(kwargs['others'], tuple):
205 params_dict['others'] = [json.dumps(w) for w in kwargs['others']]
206 elif isinstance(kwargs['others'], list):
207 params_no_dict['others'] = json.dumps(kwargs['others'])
208 elif isinstance(kwargs['others'], str):
209 params_no_dict['others'] = kwargs['others']
210 else:
211 return 'ERROR:: "others" must be a tuple, list or str'
212 #------------------------------------------#
213 len_params_dict = []
214 for value2 in params_dict.values():
215 len_params_dict.append(len(value2))
216
217 if len(list(set(len_params_dict))) > 1:
218 return 'ERROR:: All lists must be the same length: %s' % (len(params_dict['name']))
219 #------------------------------------------------------------#
220
221 for v in range(len(params_dict['name'])):
222 try:
223 send = {}
224 for key_dict, value_dict in params_dict.items():
225 send[key_dict] = value_dict[v]
226 for key_no_dict, value_no_dict in params_no_dict.items():
227 send[key_no_dict] = value_no_dict
228
229 self.list.append(getattr(self.ckan.action, 'resource_create')(**send))
230 print('File "{}" was uploaded successfully'.format(params_dict['name'][v]))
231 except:
232 _, exc_value, _ = sys.exc_info()
233 self.list.append(exc_value)
234 print('Error uploading "{}" file'.format(params_dict['name'][v]))
235 return self.list
236 #------------------------------------------------------------#
237
238 def show(self, type_option, id, **kwargs):
239 '''
240 FINALIDAD:
241 Funcion personalizada para una busqueda en especifico.
242
243 PARAMETROS DISPONIBLES:
244 CONSULTAR: "GUIA DE SCRIPT.pdf"
245
246 ESTRUCTURA:
247 <access_name>.show(type_option = <class 'str'>, id = <class 'str'>, param_1 = <class 'param_1'>, ...)
248 '''
249 if type(type_option) is str:
250 try:
251 if type_option == 'dataset':
252 return getattr(self.ckan.action, 'package_show')(id=id, **kwargs)
253 elif type_option == 'resource':
254 return getattr(self.ckan.action, 'resource_show')(id=id, **kwargs)
255 elif type_option == 'project':
256 return getattr(self.ckan.action, 'organization_show')(id=id, **kwargs)
257 elif type_option == 'collaborator':
258 return getattr(self.ckan.action, 'package_collaborator_list_for_user')(id=id, **kwargs)
259 elif type_option == 'member':
260 return getattr(self.ckan.action, 'organization_list_for_user')(id=id, **kwargs)
261 elif type_option == 'vocabulary':
262 return getattr(self.ckan.action, 'vocabulary_show')(id=id, **kwargs)
263 elif type_option == 'tag':
264 if not 'vocabulary_id' in kwargs:
265 print('Missing "vocabulary_id" value: assume it is a free tag')
266 return getattr(self.ckan.action, 'tag_show')(id=id, **kwargs)
267 elif type_option == 'user':
268 return getattr(self.ckan.action, 'user_show')(id=id, **kwargs)
269 elif type_option == 'job':
270 return getattr(self.ckan.action, 'job_show')(id=id, **kwargs)
271 else:
272 return 'ERROR:: "%s" is not accepted' % (type_option)
273 except:
274 _, exc_value, _ = sys.exc_info()
275 return exc_value
276 else:
277 return 'ERROR:: "type_option" must be a str'
278
279 def search(self, type_option, query=None, **kwargs):
280 '''
281 FINALIDAD:
282 Funcion personalizada para busquedas que satisfagan algun criterio.
283
284 PARAMETROS DISPONIBLES:
285 CONSULTAR: "GUIA DE SCRIPT.pdf"
286
287 ESTRUCTURA:
288 <access_name>.search(type_option = <class 'str'>, query = <class 'dict'>, param_1 = <class 'param_1'>, ...)
289 '''
290 if type(type_option) is str:
291 try:
292 if type_option == 'dataset':
293 key_replace = ['fq', 'fq_list', 'include_private']
294 key_point = ['facet_mincount', 'facet_limit', 'facet_field']
295 for key1, value1 in kwargs.items():
296 if not key1 in key_replace:
297 if key1 in key_point:
298 self.dict[key1.replace('_', '.')] = value1
299 else:
300 self.dict[key1] = value1
301
302 if query is not None:
303 if type(query) is dict:
304 self.dict['fq_list'] = []
305 #NUM_RESOURCES_MIN / NUM_RESOURCES_MAX
306 #----------------------------------------------------#
307 if 'dataset_start_date' in query:
308 if type(query['dataset_start_date']) is str:
309 try:
310 datetime.strptime(query['dataset_start_date'], '%Y-%m-%d')
311 if len(query['dataset_start_date']) != 10:
312 return '"dataset_start_date", must be: <YYYY-MM-DD>'
313 self.dict['fq_list'].append('dataset_start_date:"'+query['dataset_start_date']+'"')
314 self.list.append('dataset_start_date')
315 except:
316 return '"dataset_start_date" incorrect: "%s"' % (query['dataset_start_date'])
317 else:
318 return '"dataset_start_date" must be <str>'
319 #----------------------------------------------------#
320 if 'dataset_end_date' in query:
321 if type(query['dataset_end_date']) is str:
322 try:
323 datetime.strptime(query['dataset_end_date'], '%Y-%m-%d')
324 if len(query['dataset_end_date']) != 10:
325 return '"dataset_end_date", must be: <YYYY-MM-DD>'
326
327 if 'dataset_start_date' in query:
328 if query['dataset_start_date'] > query['dataset_end_date']:
329 return '"dataset_end_date" must be greater than "dataset_start_date"'
330
331 self.dict['fq_list'].append('dataset_end_date:"'+query['dataset_end_date']+'"')
332 self.list.append('dataset_end_date')
333 except:
334 return '"dataset_end_date" incorrect: "%s"' % (query['dataset_end_date'])
335 else:
336 return '"dataset_end_date" must be <str>'
337 #----------------------------------------------------#
338 for key, value in query.items():
339 if value is not None and not key in self.list:
340 self.dict['fq_list'].append(str(key)+':"'+str(value)+'"')
341 else:
342 return '"query" must be <dict>'
343
344 return getattr(self.ckan.action, 'package_search')(include_private=True, **self.dict)
345
346 elif type_option == 'resource':
347 for key1, value1 in kwargs.items():
348 if key1 != 'fields':
349 self.dict[key1] = value1
350
351 if query is not None:
352 if type(query) is dict:
353 #----------------------------------------------------#
354 if 'file_date_min' in query:
355 if type(query['file_date_min']) is str:
356 try:
357 datetime.strptime(query['file_date_min'], '%Y-%m-%d')
358 if len(query['file_date_min']) != 10:
359 return '"file_date_min", must be: <YYYY-MM-DD>'
360 except:
361 return '"file_date_min" incorrect: "%s"' % (query['file_date_min'])
362 else:
363 return '"file_date_min" must be <str>'
364 #----------------------------------------------------#
365 if 'file_date_max' in query:
366 if type(query['file_date_max']) is str:
367 try:
368 datetime.strptime(query['file_date_max'], '%Y-%m-%d')
369 if len(query['file_date_max']) != 10:
370 return '"file_date_max", must be: <YYYY-MM-DD>'
371
372 if 'file_date_min' in query:
373 if query['file_date_min'] > query['file_date_max']:
374 return '"file_date_max" must be greater than "file_date_min"'
375 except:
376 return '"file_date_max" incorrect: "%s"' % (query['file_date_max'])
377 else:
378 return '"file_date_max" must be <str>'
379 #----------------------------------------------------#
380 self.dict['query'] = query
381 else:
382 return '"query" must be <dict>'
383 return getattr(self.ckan.action, 'resources_search')(**self.dict)
384
385 elif type_option == 'tag':
386 for key1, value1 in kwargs.items():
387 if key1 != 'fields':
388 self.dict[key1] = value1
389
390 if not 'vocabulary_id' in kwargs:
391 print('Missing "vocabulary_id" value: tags that don’t belong to any vocabulary')
392 else:
393 print('Only tags that belong to "{}" vocabulary'.format(kwargs['vocabulary_id']))
394
395 if query is not None:
396 if type(query) is dict:
397 if 'search' in query:
398 if type(query['search']) is list or type(query['search']) is str:
399 self.dict['query'] = query['search']
400 else:
401 return '"search" must be <list> or <str>'
402 else:
403 return '"query" must be <dict>'
404 return getattr(self.ckan.action, 'tag_search')(**self.dict)
405
406 else:
407 return 'ERROR:: "%s" is not accepted' % (type_option)
408
409 except:
410 _, exc_value, _ = sys.exc_info()
411 return exc_value
412 else:
413 return 'ERROR:: "type_option" must be <str>'
414
415 def create(self, type_option, **kwargs):
416 '''
417 FINALIDAD:
418 Funcion personalizada para crear.
419
420 PARAMETROS DISPONIBLES:
421 CONSULTAR: "GUIA DE SCRIPT.pdf"
422
423 ESTRUCTURA:
424 <access_name>.create(type_option = <class 'str'>, param_1 = <class 'param_1'>, ...)
425 '''
426 if type(type_option) is str:
427 try:
428 if type_option == 'dataset':
429 return getattr(self.ckan.action, 'package_create')(**kwargs)
430 elif type_option == 'project':
431 return getattr(self.ckan.action, 'organization_create')(**kwargs)
432 elif type_option == 'member':
433 return getattr(self.ckan.action, 'organization_member_create')(**kwargs)
434 elif type_option == 'collaborator':
435 return getattr(self.ckan.action, 'package_collaborator_create')(**kwargs)
436 elif type_option == 'vocabulary':
437 return getattr(self.ckan.action, 'vocabulary_create')(**kwargs)
438 elif type_option == 'tag':
439 return getattr(self.ckan.action, 'tag_create')(**kwargs)
440 elif type_option == 'user':
441 return getattr(self.ckan.action, 'user_create')(**kwargs)
442 else:
443 return 'ERROR:: "%s" is not accepted' % (type_option)
444 except:
445 _, exc_value, _ = sys.exc_info()
446 return exc_value
447 else:
448 return 'ERROR:: "type_option" must be <str>'
449
450 def patch(self, type_option, **kwargs):
451 '''
452 FINALIDAD:
453 Funciones personalizadas para actualizar
454
455 PARAMETROS DISPONIBLES:
456 CONSULTAR: "GUIA DE SCRIPT.pdf"
457
458 ESTRUCTURA:
459 <access_name>.patch(type_option = <class 'str'>, param_1 = <class 'param_1'>, ...)
460 '''
461 if type(type_option) is str:
462 try:
463 if type_option == 'dataset':
464 return getattr(self.ckan.action, 'package_patch')(**kwargs)
465 elif type_option == 'project':
466 return getattr(self.ckan.action, 'organization_patch')(**kwargs)
467 elif type_option == 'resource':
468 return getattr(self.ckan.action, 'resource_patch')(**kwargs)
469 elif type_option == 'member':
470 return getattr(self.ckan.action, 'organization_member_create')(**kwargs)
471 elif type_option == 'collaborator':
472 return getattr(self.ckan.action, 'package_collaborator_create')(**kwargs)
473 else:
474 return 'ERROR:: "%s" is not accepted' % (type_option)
475 except:
476 _, exc_value, _ = sys.exc_info()
477 return exc_value
478 else:
479 return 'ERROR:: "type_option" must be <str>'
480
481 def delete(self, type_option, select=None, **kwargs):
482 '''
483 FINALIDAD:
484 Función personalizada para eliminar y/o purgar.
485
486 PARAMETROS DISPONIBLES:
487 CONSULTAR: "GUIA DE SCRIPT.pdf"
488
489 ESTRUCTURA:
490 <access_name>.delete(type_option = <class 'str'>, param_1 = <class 'param_1'>, ...)
491 '''
492 if type(type_option) is str:
493 try:
494 if type_option == 'dataset':
495 if select is None:
496 return 'ERROR:: "select" must not be "None"'
497 else:
498 if 'delete' == select:
499 return getattr(self.ckan.action, 'package_delete')(**kwargs)
500 elif 'purge' == select:
501 return getattr(self.ckan.action, 'dataset_purge')(**kwargs)
502 else:
503 return 'ERROR:: "%s" is not accepted' % (select)
504 elif type_option == 'project':
505 if select is None:
506 return 'ERROR:: "select" must not be "None"'
507 else:
508 if 'delete' == select:
509 return getattr(self.ckan.action, 'organization_delete')(**kwargs)
510 elif 'purge' == select:
511 return getattr(self.ckan.action, 'organization_purge')(**kwargs)
512 else:
513 return 'ERROR:: "%s" is not accepted' % (select)
514 elif type_option == 'resource':
515 return getattr(self.ckan.action, 'resource_delete')(**kwargs)
516 elif type_option == 'vocabulary':
517 return getattr(self.ckan.action, 'vocabulary_delete')(**kwargs)
518 elif type_option == 'tag':
519 return getattr(self.ckan.action, 'tag_delete')(**kwargs)
520 elif type_option == 'user':
521 return getattr(self.ckan.action, 'user_delete')(**kwargs)
522 else:
523 return 'ERROR:: "%s" is not accepted' % (type_option)
524 except:
525 _, exc_value, _ = sys.exc_info()
526 return exc_value
527 else:
528 return 'ERROR:: "type_option" must be <str>'
529
530 def f_status_note(self, total, result, path):
531 file_txt = open(path+'status_note.txt', 'w')
532 file_txt = open(path+'status_note.txt', 'a')
533
534 file_txt.write('DOWNLOADED FILE(S): "%s"' % (len(result['name'])))
535 file_txt.write(''+ os.linesep)
536 for u in result['name']:
537 file_txt.write(' - '+ u + os.linesep)
538 file_txt.write(''+ os.linesep)
539
540 file_txt.write('FAILED FILE(S): "%s"' % (len(total['name'])-len(result['name'])))
541 file_txt.write(''+ os.linesep)
542 if len(total['name'])-len(result['name']) != 0:
543 for u in total['name']:
544 if not u in result['name']:
545 file_txt.write(' - '+ u + os.linesep)
546 else:
547 file_txt.write(' "None"'+ os.linesep)
548
549 def f_name(self, name_dataset, ext, tempdir):
550 while self.check:
551 self.str = ''
552 if self.cont == 0:
553 if os.path.exists(tempdir + name_dataset + ext):
554 self.str = name_dataset+'('+str(self.cont+1)+')'+ext
555 else:
556 self.check = self.check * 0
557 self.str = name_dataset + ext
558 else:
559 if not os.path.exists(tempdir + name_dataset+'('+str(self.cont)+')'+ext):
560 self.check = self.check * 0
561 self.str = name_dataset+'('+str(self.cont)+')'+ ext
562 self.cont = self.cont+1
563 return self.str
564
565 def f_zipdir(self, path, ziph, zip_name):
566 for root, _, files in os.walk(path):
567 print('.....')
568 print('Creating: "{}" >>'.format(zip_name))
569 for __file in tqdm(iterable=files, total=len(files)):
570 new_dir = os.path.relpath(os.path.join(root, __file), os.path.join(path, '..'))
571 ziph.write(os.path.join(root, __file), new_dir)
572 print('Created >>')
573
574 def download_by_step(self, response, tempdir_name):
575 try:
576 with requests.get(response['url'], stream=True, headers={'Authorization': self.Authorization}) as resp:
577 if resp.status_code == 200:
578 with open(tempdir_name+response['name'], 'wb') as file:
579 for chunk in resp.iter_content(chunk_size = self.chunk_size):
580 if chunk:
581 file.write(chunk)
582 except requests.exceptions.RequestException:
583 pass
584
585 def download_files(self, **kwargs):
586 '''
587 FINALIDAD:
588 Funcion personalizada para la descarga de archivos existentes de un dataset.
589
590 PARAMETROS DISPONIBLES:
591 CONSULTAR: "GUIA DE SCRIPT.pdf"
592
593 ESTRUCTURA:
594 <access_name>.download_files(id = <class 'str'>, param_1 = <class 'param_1'>, ...)
595 '''
596 dict_local = {}
597 #----------------------------------------------#
598 if 'zip' in kwargs:
599 if type(kwargs['zip']) is not bool:
600 return 'ERROR:: "zip" must be: <class "bool">'
601 else:
602 dict_local['zip'] = kwargs['zip']
603 else:
604 dict_local['zip'] = False
605 #----------------------------------------------#
606 if 'status_note' in kwargs:
607 if type(kwargs['status_note']) is not bool:
608 return 'ERROR:: "status_note" must be: <class "bool">'
609 else:
610 dict_local['status_note'] = kwargs['status_note']
611 else:
612 dict_local['status_note'] = False
613 #----------------------------------------------#
614 if 'path' in kwargs:
615 if type(kwargs['path']) is str:
616 if os.path.isdir(kwargs['path']) == False:
617 return 'ERROR:: "path" does not exist'
618 else:
619 if kwargs['path'][-1:] != self.separator:
620 dict_local['path'] = kwargs['path']+self.separator
621 else:
622 dict_local['path'] = kwargs['path']
623
624 txt = dict_local['path']+datetime.now().strftime("%Y_%m_%d_%H_%M_%S_%f")+'.txt'
625 if int(platform.python_version()[0]) == 3:
626 try:
627 file_txt = open(txt, 'w')
628 file_txt.close()
629 os.remove(txt)
630 except PermissionError:
631 return 'ERROR:: Access denied, you are not authorized to write files: "%s"' % (dict_local['path'])
632 else:
633 try:
634 file_txt = open(txt, 'w')
635 file_txt.close()
636 os.remove(txt)
637 except:
638 return 'ERROR:: Access denied, you are not authorized to write files: "%s"' % (dict_local['path'])
639 else:
640 return 'ERROR:: "path" must be: <class "str">'
641 else:
642 dict_local['path'] = ''
643 #----------------------------------------------#
644 for key, value in kwargs.items():
645 if not key in dict_local:
646 self.dict[key] = value
647 try:
648 response = getattr(self.ckan.action, 'url_resources')(**self.dict)
649 except:
650 _, exc_value, _ = sys.exc_info()
651 return exc_value
652
653 if len(response) != 0:
654 #--------------TEMP PATH---------------#
655 if dict_local['zip']:
656 tempdir = tempfile.mkdtemp(prefix=kwargs['id']+'-')+self.separator
657 os.mkdir(tempdir+kwargs['id'])
658 dir_name = tempdir + kwargs['id'] + self.separator
659 else:
660 dir = self.f_name(kwargs['id'], '', dict_local['path'])
661 os.mkdir(dict_local['path'] + dir)
662 dir_name = dict_local['path'] + dir + self.separator
663 #-----------DOWNLOAD FILES-------------#
664 print('.....')
665 print('Downloading "{}" file(s) >>'.format(len(response)))
666 name_total = {'name': []}
667 with concurrent.futures.ThreadPoolExecutor() as executor:
668 for u in tqdm(iterable=response, total=len(response)):
669 name_total['name'].append(u['name'])
670 executor.submit(self.download_by_step, u, dir_name)
671 name_check = {}
672 name_check['name'] = [f for f in os.listdir(dir_name) if os.path.isfile(os.path.join(dir_name, f))]
673 print('"{}" downloaded file(s) successfully >>'.format(len(name_check['name'])))
674 #--------------------------------------#
675 if len(name_check['name']) != 0:
676 #----------Status Note---------#
677 if dict_local['status_note']:
678 print('.....')
679 print('Creating: "status_note.txt" >>')
680 self.f_status_note(name_total, name_check, dir_name)
681 print('Created>>')
682 #----------ZIP CREATE----------#
683 if dict_local['zip']:
684 zip_name = self.f_name(kwargs['id'], '.zip', dict_local['path'])
685 ziph = zipfile.ZipFile(dict_local['path'] + zip_name, 'w', zipfile.ZIP_DEFLATED, allowZip64=True)
686 self.f_zipdir(dir_name, ziph, zip_name)
687 ziph.close()
688 #Delete Temporal Path
689 if os.path.exists(tempdir[:-1]):
690 shutil.rmtree(tempdir[:-1])
691 #------------------------------#
692 print('.....')
693 return 'DOWNLOAD FINISHED'
694 else:
695 #Delete Temporal Path
696 if dict_local['zip']:
697 if os.path.exists(tempdir[:-1]):
698 shutil.rmtree(tempdir[:-1])
699 else:
700 if os.path.exists(dir_name[:-1]):
701 shutil.rmtree(dir_name[:-1])
702 return 'NO FILES WERE DOWNLOADED'
703 else:
704 return 'FILES NOT FOUND' No newline at end of file
Show file before | Show file after | Hide comments
@@ -0,0 +1,12
1 # encoding: utf-8
2 from setuptools import setup
3
4 setup(
5 name = "CKAN_JRO",
6 version = "master",
7 description = "Data Repository - JRO",
8 author = "Edson Ynilupu Mattos",
9 author_email = "eynilupu@igp.gob.pe",
10 url = "",
11 packages = ["CKAN_JRO"]
12 ) No newline at end of file
General Comments 0
You need to be logged in to leave comments. Login now