##// END OF EJS Templates
DATABASES » ckanext-jro » api-cliente
RSS
v2.9.2 :: Delete 'old functions' create and download resources
eynilupu -
r18:4695b8d73d71
parent child
Show More
Show file before | Show file after | Hide comments
1 NO CONTENT: modified file chmod 100755 => 100644, binary diff hidden
NO CONTENT: modified file chmod 100755 => 100644, binary diff hidden
Show file before | Show file after | Hide comments
1 NO CONTENT: modified file, binary diff hidden
NO CONTENT: modified file, binary diff hidden
Show file before | Show file after | Hide comments
This diff has been collapsed as it changes many lines, (556 lines changed) Show them Hide them
@@ -1,1044 +1,500
1 from ckanapi import RemoteCKAN
1 from ckanapi import RemoteCKAN
2 from datetime import datetime
2 from datetime import datetime
3 from tqdm import tqdm
4 from CKAN_JRO import logic_download
3 from CKAN_JRO import logic_download
5 from CKAN_JRO import resource
4 from CKAN_JRO import resource
6 #from ckanapi.errors import NotAuthorized, NotFound, ValidationError, SearchQueryError, SearchError, CKANAPIError, ServerIncompatibleError
5 #from ckanapi.errors import NotAuthorized, NotFound, ValidationError, SearchQueryError, SearchError, CKANAPIError, ServerIncompatibleError
7 import sys
6 import sys
8 import platform
7 import platform
9 import os
8 import os
10 import tempfile
11 import shutil
12 import zipfile
13 import concurrent.futures
14 import requests
9 import requests
15 import json
16 #import pathlib
17 import uuid
18
19 if sys.version_info.major == 3:
20 from urllib.parse import urlparse
21 else:
22 import urlparse
23
10
24 class JROAPI():
11 class JROAPI():
25 """
12 """
26 FINALIDAD:
13 FINALIDAD:
27 Script para administrar y obtener la data del repositorio por medio de APIs.
14 Script para administrar y obtener la data del repositorio por medio de APIs.
28
15
29 REQUISITIOS PREVIOS:
16 REQUISITIOS PREVIOS:
30 - Paso 1: Tener "pip [Python 2]" o "pip3 [Python 3]" instalado:
17 - Paso 1: Tener "pip [Python 2]" o "pip3 [Python 3]" instalado:
31 - Paso 2: Instalar lo siguiente como admininstrador:
18 - Paso 2: Instalar los siguientes paquetes:
32 En Python 2
19 ckanapi==4.7
33 - pip install ckanapi==4.5
20 requests
34 - pip install requests
35 - pip install futures
36 - pip install tqdm
37 En Python > 3
38 - pip3 install ckanapi==4.5
39 - pip3 install requests
40 - pip3 install tqdm
41
21
42 FUNCIONES DISPONIBLES:
22 FUNCIONES DISPONIBLES:
43 - action
23 - action
44 - upload_file
45 - upload_multiple_files
46 - upload_multiple_files_advance
47 - show
24 - show
48 - search
25 - search
49 - create
26 - create
50 - patch
27 - patch
51 - delete
28 - delete
52 - download_files
29 - download_files
53
30
54 EJEMPLOS:
31 EJEMPLOS:
55 #1:
32 #1:
56 with JROAPI('http://demo.example.com', Authorization='#########') as <access_name>:
33 with JROAPI('http://demo.example.com', Authorization='#########') as <access_name>:
57 ... some operation(s) ...
34 ... some operation(s) ...
58 #2:
35 #2:
59 <access_name> = JROAPI('http://example.com', Authorization='#########')
36 <access_name> = JROAPI('http://example.com', Authorization='#########')
60 ... some operation(s) ...
37 ... some operation(s) ...
61 <access_name>.ckan.close()
38 <access_name>.ckan.close()
62
39
63 REPORTAR ALGUN PROBLEMA:
40 REPORTAR ALGUN PROBLEMA:
64 Debe enviar un correo a eynilupu@igp.gob.pe detallando los siguientes pasos:
41 Debe enviar un correo a eynilupu@igp.gob.pe detallando los siguientes pasos:
65 1) Correo para contactarlo
42 1) Correo para contactarlo
66 2) Descripcion del problema
43 2) Descripcion del problema
67 3) ¿En que paso o seccion encontro el problema?
44 3) ¿En que paso o seccion encontro el problema?
68 4) ¿Cual era el resultado que usted esperaba?
45 4) ¿Cual era el resultado que usted esperaba?
69 """
46 """
70 def __init__(self, url, Authorization=None, secure=True):
47 def __init__(self, url, Authorization=None, secure=True):
71 #-------- Check Secure -------#
48 #-------- Check Secure -------#
72 self.verify = secure
49 self.verify = secure
73 if not secure and isinstance(secure, bool):
50 if not secure and isinstance(secure, bool):
74 session = requests.Session()
51 session = requests.Session()
75 session.verify = False
52 session.verify = False
76 else:
53 else:
77 session = None
54 session = None
78 #------------------------------#
55 #------------------------------#
79 self.url = url
56 self.url = url
80 ua = 'CKAN_JRO/2.9.2 (+'+str(self.url)+')'
57 ua = 'CKAN_JRO/2.9.2 (+'+str(self.url)+')'
81 #ua = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36'
58 #ua = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36'
82 self.ckan = RemoteCKAN(self.url, apikey=Authorization, user_agent=ua, session=session)
59 self.ckan = RemoteCKAN(self.url, apikey=Authorization, user_agent=ua, session=session)
83 #self.ckan = RemoteCKAN(self.url, apikey=Authorization)
60 #self.ckan = RemoteCKAN(self.url, apikey=Authorization)
84 self.Authorization = Authorization
61 self.Authorization = Authorization
85 # Change for --> self.separator = os.sep
62 # Change for --> self.separator = os.sep
86 if platform.system() == 'Windows':
63 if platform.system() == 'Windows':
87 self.separator = '\\'
64 self.separator = '\\'
88 else:
65 else:
89 self.separator = '/'
66 self.separator = '/'
90
67
91 self.chunk_size = 1024
68 self.chunk_size = 1024
92 self.list = []
69 self.list = []
93 self.dict = {}
70 self.dict = {}
94 self.str = ''
71 self.str = ''
95 self.check = 1
72 self.check = 1
96 self.cont = 0
73 self.cont = 0
97
74
98 def __enter__(self):
75 def __enter__(self):
99 return self
76 return self
100
77
101 def __exit__(self, *args):
78 def __exit__(self, *args):
102 self.ckan.close()
79 self.ckan.close()
103
80
104 def action(self, action, **kwargs):
81 def action(self, action, **kwargs):
105 """
82 """
106 FINALIDAD:
83 FINALIDAD:
107 Funcion para llamar a las APIs disponibles
84 Funcion para llamar a las APIs disponibles
108
85
109 APIs DISPONIBLES:
86 APIs DISPONIBLES:
110 CONSULTAR: "GUIA DE SCRIPT.pdf"
87 CONSULTAR: "GUIA DE SCRIPT.pdf"
111
88
112 EJEMPLO:
89 EJEMPLO:
113 <access_name>.action(<consuming API>, param_1 = <class 'param_1'>, ...)
90 <access_name>.action(<consuming API>, param_1 = <class 'param_1'>, ...)
114 """
91 """
115 #--------------- CASE: PACKAGE SEARCH ---------------#
92 #--------------- CASE: PACKAGE SEARCH ---------------#
116 if kwargs is not None:
93 if kwargs is not None:
117 if action == 'package_search':
94 if action == 'package_search':
118 self.list = ['facet_mincount', 'facet_limit', 'facet_field']
95 self.list = ['facet_mincount', 'facet_limit', 'facet_field']
119 for facet in self.list:
96 for facet in self.list:
120 if facet in kwargs:
97 if facet in kwargs:
121 kwargs[facet.replace('_', '.')] = kwargs[facet]
98 kwargs[facet.replace('_', '.')] = kwargs[facet]
122 kwargs.pop(facet)
99 kwargs.pop(facet)
123 #----------------------------------------------------#
100 #----------------------------------------------------#
124 try:
101 try:
125 return getattr(self.ckan.action, action)(**kwargs)
102 return getattr(self.ckan.action, action)(**kwargs)
126 except:
103 except:
127 _, exc_value, _ = sys.exc_info()
104 _, exc_value, _ = sys.exc_info()
128 return exc_value
105 return exc_value
129
106
130 def upload_file(self, dataset_id, file_date, file_type, file_path=False, url_or_path=False, ignore_repetition=False, **kwargs):
131 # Agregar si es interruptido por teclado
132 '''
133 FINALIDAD:
134 Funcion para crear un unico recurso (puede incluir un archivo asociado) al repositorio del ROJ.
135
136 PARAMETROS DISPONIBLES:
137 CONSULTAR: "GUIA DE SCRIPT.pdf"
138
139 ESTRUCTURA:
140 <access_name>.upload_file(dataset_id = <class 'str'>, file_date = <class 'str'>, file_type = <class 'str'>, file_path = <class 'str'>, url_or_path = <class 'str'>, param_1 = <class 'param_1'>, ...)
141 '''
142 #self.list = ['package_id', 'upload', 'voc_file_type', 'name'] #file_date
143 self.list = ['package_id', 'upload', 'voc_file_type'] #file_date
144 for key1, value1 in kwargs.items():
145 if not key1 in self.list:
146 self.dict[key1] = value1
147
148 #---------------------------#
149 if not 'others' in kwargs:
150 self.dict['others'] = ''
151 else:
152 if isinstance(kwargs['others'], list):
153 self.dict['others'] = json.dumps(kwargs['others'])
154 #---------------------------#
155
156 if isinstance(file_path, str) and isinstance(url_or_path, str):
157 return 'ERROR:: Choose one: "file_path" or "url_or_path" parameters'
158
159 if isinstance(file_path, str):
160 if not os.path.isfile(file_path):
161 return 'File "%s" not exist' % (file_path)
162
163 self.dict['upload'] = open(file_path, 'rb')
164 self.dict['name'] = os.path.basename(file_path)
165 elif isinstance(url_or_path, str):
166 self.dict['url'] = url_or_path
167 if not 'name' in self.dict:
168 self.dict['name'] = os.path.basename(url_or_path)
169 else:
170 return 'ERROR: Verify "file_path" or "url_or_path" parameters: <class "str"> or choose one'
171
172 #if not 'format' in self.dict:
173 # self.str = ''.join(pathlib.Path(file_path).suffixes)
174 # if len(self.str) > 0:
175 # self.dict['format'] = self.str.upper()[1:]
176
177 #-------------------------PACKAGE SHOW-----------------------#
178 try:
179 dataset_show = getattr(self.ckan.action, 'package_show')(id=dataset_id)['resources']
180 except:
181 _, exc_value, _ = sys.exc_info()
182 print('ERROR obtaining metadata dataset:: Use the "print" for more information')
183 return exc_value
184
185 resources_name = []
186 for u in dataset_show:
187 resources_name.append(u['name'].lower())
188
189 if self.dict['name'].lower() in resources_name:
190 if not ignore_repetition:
191 return 'ERROR:: "%s" resource already exist in this dataset' % (self.dict['name'])
192 print('WARRING:: "'+ str(self.dict['name']) +'" resource already exist in this dataset')
193 #------------------------------------------------------------#
194 try:
195 return getattr(self.ckan.action, 'resource_create')(package_id=dataset_id, file_date=file_date, voc_file_type=file_type, **self.dict)
196 except:
197 _, exc_value, _ = sys.exc_info()
198 return exc_value
199
200 def upload_multiple_files_advance(self, dataset_id, path_files, file_date, file_type, max_size=100, max_count=500, ignore_repetition=False, **kwargs):
201 # Agregar si es interruptido por teclado
202 '''
203 FINALIDAD:
204 Funcion para subir multiples archivos al repositorio del ROJ.
205
206 PARAMETROS DISPONIBLES:
207 CONSULTAR: "GUIA DE SCRIPT.pdf"
208
209 ESTRUCTURA:
210 <access_name>.upload_multiple_files_advance(dataset_id = <class 'str'>, path_files = <class 'list of strings'>, file_date = <class 'str'>, file_type = <class 'str'>, param_1 = <class 'param_1'>, ...)
211 '''
212 #-------------------------PACKAGE SHOW-----------------------#
213 try:
214 dataset_show = getattr(self.ckan.action, 'package_show')(id=dataset_id)['resources']
215 except:
216 _, exc_value, _ = sys.exc_info()
217 print('ERROR obtaining metadata dataset:: Use the "print" for more information')
218 return exc_value
219 #------------------------------------------------------------#
220 resources_name = []
221 for u in dataset_show:
222 resources_name.append(u['name'].lower())
223 #------------------------------------------------------------#
224 self.list = ['package_id', 'upload', 'voc_file_type', 'name']
225 for key1, value1 in kwargs.items():
226 if not key1 in self.list:
227 self.dict[key1] = value1
228 #------------------------------------------------------------#
229 if not 'others' in kwargs:
230 self.dict['others'] = ''
231 else:
232 if isinstance(kwargs['others'], list):
233 self.dict['others'] = json.dumps(kwargs['others'])
234 #------------------------------------------------------------#
235 total_list = []
236 #---------------CASO : "path" or "path_list"-----------------#
237 if type(path_files) is list:
238 if len(path_files) != 0:
239 path_files.sort()
240 for u in path_files:
241 if os.path.isfile(u):
242 if os.path.basename(u).lower() in resources_name:
243 if not ignore_repetition:
244 return 'ERROR:: "%s" file already exist in this dataset' % (os.path.basename(u))
245 print('WARRING:: "'+ str(os.path.basename(u)) +'" file was ignored because already exist in this dataset')
246 else:
247 total_list.append({'name':os.path.basename(u), 'size': os.stat(u).st_size, 'upload':open(u, 'rb')})
248 else:
249 return 'File "%s" does not exist' % (u)
250 else:
251 return 'ERROR:: "path_list is empty"'
252
253 elif type(path_files) is str:
254 if os.path.isdir(path_files):
255 path_order = [f for f in os.listdir(path_files) if os.path.isfile(os.path.join(path_files, f))]
256 path_order.sort()
257 if path_order:
258 for name in path_order:
259 if name.lower() in resources_name:
260 if not ignore_repetition:
261 return 'ERROR:: "%s" file already exist in this dataset' % (name)
262 print('WARRING:: "'+ name +'" file was ignored because already exist in this dataset')
263 else:
264 total_list.append({'name':name, 'size': os.stat(os.path.join(path_files, name)).st_size, 'upload':open(os.path.join(path_files, name), 'rb')})
265 else:
266 return "ERROR:: There aren't files in this directory"
267 else:
268 return 'ERROR:: Directory "%s" does not exist' % (path_files)
269 else:
270 return 'ERROR:: "path_files" must be a str or list'
271 #------------------------------------------------------------#
272 try:
273 uuid.UUID(str(dataset_id), version=4)
274 package_id_or_name = '"id": "' + str(dataset_id) + '"'
275 except ValueError:
276 package_id_or_name = '"name": "' + str(dataset_id) + '"'
277 #------------------------------------------------------------#
278 blocks = [[]]
279 size_file = 0
280 count_file = 0
281 inter_num = 0
282 for value in total_list:
283 if value['size'] > 1024 * 1024 * float(max_size):
284 return 'ERROR:: The size of the "%s" file is %sMB aprox, please change "max_size" value' % (value['name'], str(round(value['size']/(1024 * 1024), 2)))
285 if not 1 <= int(max_count) <= 999:
286 return 'ERROR:: The count of the number of files must be between 1 and 999, please change "max_count" value'
287
288 size_file = size_file + value['size']
289 count_file = count_file + 1
290 if size_file <= 1024 * 1024 * float(max_size) and count_file <= int(max_count):
291 del value['size']
292 blocks[inter_num].append(value)
293 else:
294 inter_num = inter_num + 1
295 size_file = value['size']
296 count_file = 1
297 blocks.append([])
298 del value['size']
299 blocks[inter_num].append(value)
300 #------------------------------------------------------------#
301 if len(blocks[0]) > 0:
302 print('BLOCK(S) IN TOTAL:: {}'.format(len(blocks)))
303 for count1, block in enumerate(blocks):
304 print('---- BLOCK N°{} ----'.format(count1 + 1))
305 resource_extend = []
306 files_dict = {}
307 for count2, value2 in enumerate(block):
308 value2['file_date'] = file_date
309 value2['voc_file_type'] = file_type
310 value2.update(self.dict)
311
312 #if not 'format' in value2:
313 # format = ''.join(pathlib.Path(value2['name']).suffixes)
314 # if len(format) > 0:
315 # value2['format'] = format.upper()[1:]
316
317 files_dict['update__resources__-'+ str(len(block)-count2) +'__upload'] = (value2['name'], value2['upload'])
318 del value2['upload']
319 resource_extend.append(value2)
320
321 print('BLOCK N°{} :: "{}" file(s) found >> uploading'.format(count1 + 1, len(block)))
322 try:
323 result = self.ckan.call_action(
324 'package_revise',
325 {'match': '{'+ str(package_id_or_name) +'}', 'update__resources__extend': json.dumps(resource_extend)},
326 files=files_dict
327 )
328 print('BLOCK N°{} :: Uploaded file(s) successfully'.format(count1 + 1))
329 if len(blocks) == count1 + 1:
330 return result
331 except:
332 print('ERROR :: Use the "print" for more information')
333 _, exc_value, _ = sys.exc_info()
334 return exc_value
335 else:
336 return "ERROR:: No file(s) found to upload"
337
338 def upload_multiple_files(self, dataset_id, path_files, date_files, type_files, ignore_repetition=False, **kwargs):
339 # Agregar si es interruptido por teclado
340 '''
341 FINALIDAD:
342 Funcion para subir multiples archivos al repositorio del ROJ.
343
344 PARAMETROS DISPONIBLES:
345 CONSULTAR: "GUIA DE SCRIPT.pdf"
346
347 ESTRUCTURA:
348 <access_name>.upload_multiple_files(dataset_id = <class 'str'>, path_files = <class 'str'> or <class 'list of strings'>, date_files = <class 'str'> or <class 'list of strings'>, type_files = <class 'str'> or <class 'list of strings'>, param_1 = <class 'param_1'>, ...)
349 '''
350 #-------------------------PACKAGE SHOW-----------------------#
351 try:
352 dataset_show = getattr(self.ckan.action, 'package_show')(id=dataset_id)['resources']
353 except:
354 _, exc_value, _ = sys.exc_info()
355 print('ERROR obtaining metadata dataset:: Use the "print" for more information')
356 return exc_value
357 #------------------------------------------------------------#
358 resources_name = []
359 for u in dataset_show:
360 resources_name.append(u['name'].lower())
361 #------------------------------------------------------------#
362
363 params_dict = {'upload':[], 'name':[]}
364 #if not 'format' in kwargs:
365 # params_dict.update({'format':[]})
366 #---------------CASO : "path" or "path_list"-----------------#
367 if type(path_files) is list:
368 if len(path_files) != 0:
369 path_files.sort()
370 for u in path_files:
371 if os.path.isfile(u):
372 if os.path.basename(u).lower() in resources_name:
373 if not ignore_repetition:
374 return 'ERROR:: "%s" file already exist in this dataset' % (os.path.basename(u))
375 print('WARRING:: "'+ str(os.path.basename(u)) +'" file was ignored because already exist in this dataset')
376 else:
377 params_dict['upload'].append(open(u, 'rb'))
378 params_dict['name'].append(os.path.basename(u))
379 #if not 'format' in kwargs:
380 # format = ''.join(pathlib.Path(u).suffixes)
381 # if len(format) > 0:
382 # params_dict['format'].append(format.upper()[1:])
383 # else:
384 # params_dict['format'].append('')
385 else:
386 return 'File "%s" does not exist' % (u)
387 else:
388 return 'ERROR:: "path_list is empty"'
389 elif type(path_files) is str:
390 if os.path.isdir(path_files):
391 path_order = [f for f in os.listdir(path_files) if os.path.isfile(os.path.join(path_files, f))]
392 path_order.sort()
393 if path_order:
394 for name in path_order:
395 if name.lower() in resources_name:
396 if not ignore_repetition:
397 return 'ERROR:: "%s" file already exist in this dataset' % (name)
398 print('WARRING:: "'+ str(name) +'" file was ignored because already exist in this dataset')
399 else:
400 params_dict['upload'].append(open(os.path.join(path_files, name), 'rb'))
401 params_dict['name'].append(name)
402 #if not 'format' in kwargs:
403 # format = ''.join(pathlib.Path(name).suffixes)
404 # if len(format) > 0:
405 # params_dict['format'].append(format.upper()[1:])
406 # else:
407 # params_dict['format'].append('')
408 else:
409 return "ERROR:: There aren't files in this directory"
410 else:
411 return 'ERROR:: Directory "%s" does not exist' % (path_files)
412 else:
413 return 'ERROR:: "path_files" must be a str or list'
414 #------------------------------------------------------------#
415 params_no_dict = {'package_id': dataset_id}
416 if type(date_files) is list:
417 params_dict['file_date'] = date_files
418 else:
419 params_no_dict['file_date'] = date_files
420
421 if type(type_files) is list:
422 params_dict['voc_file_type'] = type_files
423 else:
424 params_no_dict['voc_file_type'] = type_files
425
426 for key1, value1 in kwargs.items():
427 if not key1 in params_dict and not key1 in params_no_dict and key1 != 'others':
428 if type(value1) is list:
429 params_dict[key1] = value1
430 else:
431 params_no_dict[key1] = value1
432 #------------------------------------------#
433 if not 'others' in kwargs:
434 params_no_dict['others'] = ''
435 else:
436 if isinstance(kwargs['others'], tuple):
437 params_dict['others'] = [json.dumps(w) for w in kwargs['others']]
438 elif isinstance(kwargs['others'], list):
439 params_no_dict['others'] = json.dumps(kwargs['others'])
440 elif isinstance(kwargs['others'], str):
441 params_no_dict['others'] = kwargs['others']
442 else:
443 return 'ERROR:: "others" must be a tuple, list or str'
444 #------------------------------------------#
445 len_params_dict = []
446 for value2 in params_dict.values():
447 len_params_dict.append(len(value2))
448
449 if len(list(set(len_params_dict))) > 1:
450 return 'ERROR:: All lists must be the same length: %s' % (len(params_dict['name']))
451 #------------------------------------------------------------#
452 print('"{}" file(s) found >> uploading'.format(len(params_dict['name'])))
453 for v in range(len(params_dict['name'])):
454 try:
455 send = {}
456 for key_dict, value_dict in params_dict.items():
457 send[key_dict] = value_dict[v]
458 for key_no_dict, value_no_dict in params_no_dict.items():
459 send[key_no_dict] = value_no_dict
460
461 self.list.append(getattr(self.ckan.action, 'resource_create')(**send))
462 print('File #{} :: "{}" was uploaded successfully'.format(v+1, params_dict['name'][v]))
463 except:
464 _, exc_value, _ = sys.exc_info()
465 self.list.append(exc_value)
466 print('File #{} :: Error uploading "{}" file'.format(v+1, params_dict['name'][v]))
467 return self.list
468 #------------------------------------------------------------#
469
470 def show(self, type_option, id, **kwargs):
107 def show(self, type_option, id, **kwargs):
471 '''
108 '''
472 FINALIDAD:
109 FINALIDAD:
473 Funcion personalizada para una busqueda en especifico.
110 Funcion personalizada para una busqueda en especifico.
474
111
475 PARAMETROS DISPONIBLES:
112 PARAMETROS DISPONIBLES:
476 CONSULTAR: "GUIA DE SCRIPT.pdf"
113 CONSULTAR: "GUIA DE SCRIPT.pdf"
477
114
478 ESTRUCTURA:
115 ESTRUCTURA:
479 <access_name>.show(type_option = <class 'str'>, id = <class 'str'>, param_1 = <class 'param_1'>, ...)
116 <access_name>.show(type_option = <class 'str'>, id = <class 'str'>, param_1 = <class 'param_1'>, ...)
480 '''
117 '''
481 if type(type_option) is str:
118 if type(type_option) is str:
482 try:
119 try:
483 if type_option == 'dataset':
120 if type_option == 'dataset':
484 return getattr(self.ckan.action, 'package_show')(id=id, **kwargs)
121 return getattr(self.ckan.action, 'package_show')(id=id, **kwargs)
485 elif type_option == 'resource':
122 elif type_option == 'resource':
486 return getattr(self.ckan.action, 'resource_show')(id=id, **kwargs)
123 return getattr(self.ckan.action, 'resource_show')(id=id, **kwargs)
487 elif type_option == 'project':
124 elif type_option == 'project':
488 return getattr(self.ckan.action, 'organization_show')(id=id, **kwargs)
125 return getattr(self.ckan.action, 'organization_show')(id=id, **kwargs)
489 elif type_option == 'collaborator':
126 elif type_option == 'collaborator':
490 return getattr(self.ckan.action, 'package_collaborator_list_for_user')(id=id, **kwargs)
127 return getattr(self.ckan.action, 'package_collaborator_list_for_user')(id=id, **kwargs)
491 elif type_option == 'member':
128 elif type_option == 'member':
492 return getattr(self.ckan.action, 'organization_list_for_user')(id=id, **kwargs)
129 return getattr(self.ckan.action, 'organization_list_for_user')(id=id, **kwargs)
493 elif type_option == 'vocabulary':
130 elif type_option == 'vocabulary':
494 return getattr(self.ckan.action, 'vocabulary_show')(id=id, **kwargs)
131 return getattr(self.ckan.action, 'vocabulary_show')(id=id, **kwargs)
495 elif type_option == 'tag':
132 elif type_option == 'tag':
496 if not 'vocabulary_id' in kwargs:
133 if not 'vocabulary_id' in kwargs:
497 print('Missing "vocabulary_id" value: assume it is a free tag')
134 print('Missing "vocabulary_id" value: assume it is a free tag')
498 return getattr(self.ckan.action, 'tag_show')(id=id, **kwargs)
135 return getattr(self.ckan.action, 'tag_show')(id=id, **kwargs)
499 elif type_option == 'user':
136 elif type_option == 'user':
500 return getattr(self.ckan.action, 'user_show')(id=id, **kwargs)
137 return getattr(self.ckan.action, 'user_show')(id=id, **kwargs)
501 elif type_option == 'job':
138 elif type_option == 'job':
502 return getattr(self.ckan.action, 'job_show')(id=id, **kwargs)
139 return getattr(self.ckan.action, 'job_show')(id=id, **kwargs)
503 else:
140 else:
504 return 'ERROR:: "type_option = %s" is not accepted' % (type_option)
141 return 'ERROR:: "type_option = %s" is not accepted' % (type_option)
505 except:
142 except:
506 _, exc_value, _ = sys.exc_info()
143 _, exc_value, _ = sys.exc_info()
507 return exc_value
144 return exc_value
508 else:
145 else:
509 return 'ERROR:: "type_option" must be a str'
146 return 'ERROR:: "type_option" must be a str'
510
147
511 def search(self, type_option, query=None, **kwargs):
148 def search(self, type_option, query=None, **kwargs):
512 '''
149 '''
513 FINALIDAD:
150 FINALIDAD:
514 Funcion personalizada para busquedas que satisfagan algun criterio.
151 Funcion personalizada para busquedas que satisfagan algun criterio.
515
152
516 PARAMETROS DISPONIBLES:
153 PARAMETROS DISPONIBLES:
517 CONSULTAR: "GUIA DE SCRIPT.pdf"
154 CONSULTAR: "GUIA DE SCRIPT.pdf"
518
155
519 ESTRUCTURA:
156 ESTRUCTURA:
520 <access_name>.search(type_option = <class 'str'>, query = <class 'dict'>, param_1 = <class 'param_1'>, ...)
157 <access_name>.search(type_option = <class 'str'>, query = <class 'dict'>, param_1 = <class 'param_1'>, ...)
521 '''
158 '''
522 if type(type_option) is str:
159 if type(type_option) is str:
523 try:
160 try:
524 if type_option == 'dataset':
161 if type_option == 'dataset':
525 key_replace = ['fq', 'fq_list', 'include_private']
162 key_replace = ['fq', 'fq_list', 'include_private']
526 key_point = ['facet_mincount', 'facet_limit', 'facet_field']
163 key_point = ['facet_mincount', 'facet_limit', 'facet_field']
527 for key1, value1 in kwargs.items():
164 for key1, value1 in kwargs.items():
528 if not key1 in key_replace:
165 if not key1 in key_replace:
529 if key1 in key_point:
166 if key1 in key_point:
530 self.dict[key1.replace('_', '.')] = value1
167 self.dict[key1.replace('_', '.')] = value1
531 else:
168 else:
532 self.dict[key1] = value1
169 self.dict[key1] = value1
533
170
534 if query is not None:
171 if query is not None:
535 if type(query) is dict:
172 if type(query) is dict:
536 self.dict['fq_list'] = []
173 self.dict['fq_list'] = []
537 #NUM_RESOURCES_MIN / NUM_RESOURCES_MAX
174 #NUM_RESOURCES_MIN / NUM_RESOURCES_MAX
538 #----------------------------------------------------#
175 #----------------------------------------------------#
539 if 'dataset_start_date' in query:
176 if 'dataset_start_date' in query:
540 if type(query['dataset_start_date']) is str:
177 if type(query['dataset_start_date']) is str:
541 try:
178 try:
542 datetime.strptime(query['dataset_start_date'], '%Y-%m-%d')
179 datetime.strptime(query['dataset_start_date'], '%Y-%m-%d')
543 if len(query['dataset_start_date']) != 10:
180 if len(query['dataset_start_date']) != 10:
544 return '"dataset_start_date", must be: <YYYY-MM-DD>'
181 return '"dataset_start_date", must be: <YYYY-MM-DD>'
545 self.dict['fq_list'].append('dataset_start_date:"'+query['dataset_start_date']+'"')
182 self.dict['fq_list'].append('dataset_start_date:"'+query['dataset_start_date']+'"')
546 self.list.append('dataset_start_date')
183 self.list.append('dataset_start_date')
547 except:
184 except:
548 return '"dataset_start_date" incorrect: "%s"' % (query['dataset_start_date'])
185 return '"dataset_start_date" incorrect: "%s"' % (query['dataset_start_date'])
549 else:
186 else:
550 return '"dataset_start_date" must be <str>'
187 return '"dataset_start_date" must be <str>'
551 #----------------------------------------------------#
188 #----------------------------------------------------#
552 if 'dataset_end_date' in query:
189 if 'dataset_end_date' in query:
553 if type(query['dataset_end_date']) is str:
190 if type(query['dataset_end_date']) is str:
554 try:
191 try:
555 datetime.strptime(query['dataset_end_date'], '%Y-%m-%d')
192 datetime.strptime(query['dataset_end_date'], '%Y-%m-%d')
556 if len(query['dataset_end_date']) != 10:
193 if len(query['dataset_end_date']) != 10:
557 return '"dataset_end_date", must be: <YYYY-MM-DD>'
194 return '"dataset_end_date", must be: <YYYY-MM-DD>'
558
195
559 if 'dataset_start_date' in query:
196 if 'dataset_start_date' in query:
560 if query['dataset_start_date'] > query['dataset_end_date']:
197 if query['dataset_start_date'] > query['dataset_end_date']:
561 return '"dataset_end_date" must be greater than "dataset_start_date"'
198 return '"dataset_end_date" must be greater than "dataset_start_date"'
562
199
563 self.dict['fq_list'].append('dataset_end_date:"'+query['dataset_end_date']+'"')
200 self.dict['fq_list'].append('dataset_end_date:"'+query['dataset_end_date']+'"')
564 self.list.append('dataset_end_date')
201 self.list.append('dataset_end_date')
565 except:
202 except:
566 return '"dataset_end_date" incorrect: "%s"' % (query['dataset_end_date'])
203 return '"dataset_end_date" incorrect: "%s"' % (query['dataset_end_date'])
567 else:
204 else:
568 return '"dataset_end_date" must be <str>'
205 return '"dataset_end_date" must be <str>'
569 #----------------------------------------------------#
206 #----------------------------------------------------#
570 for key, value in query.items():
207 for key, value in query.items():
571 if value is not None and not key in self.list:
208 if value is not None and not key in self.list:
572 self.dict['fq_list'].append(str(key)+':"'+str(value)+'"')
209 self.dict['fq_list'].append(str(key)+':"'+str(value)+'"')
573 else:
210 else:
574 return '"query" must be <dict>'
211 return '"query" must be <dict>'
575
212
576 return getattr(self.ckan.action, 'package_search')(include_private=True, **self.dict)
213 return getattr(self.ckan.action, 'package_search')(include_private=True, **self.dict)
577
214
578 elif type_option == 'resource':
215 elif type_option == 'resource':
579 for key1, value1 in kwargs.items():
216 for key1, value1 in kwargs.items():
580 if key1 != 'fields':
217 if key1 != 'fields':
581 self.dict[key1] = value1
218 self.dict[key1] = value1
582
219
583 if query is not None:
220 if query is not None:
584 if type(query) is dict:
221 if type(query) is dict:
585 #----------------------------------------------------#
222 #----------------------------------------------------#
586 if 'file_date_min' in query:
223 if 'file_date_min' in query:
587 if type(query['file_date_min']) is str:
224 if type(query['file_date_min']) is str:
588 try:
225 try:
589 datetime.strptime(query['file_date_min'], '%Y-%m-%d')
226 datetime.strptime(query['file_date_min'], '%Y-%m-%d')
590 if len(query['file_date_min']) != 10:
227 if len(query['file_date_min']) != 10:
591 return '"file_date_min", must be: <YYYY-MM-DD>'
228 return '"file_date_min", must be: <YYYY-MM-DD>'
592 except:
229 except:
593 return '"file_date_min" incorrect: "%s"' % (query['file_date_min'])
230 return '"file_date_min" incorrect: "%s"' % (query['file_date_min'])
594 else:
231 else:
595 return '"file_date_min" must be <str>'
232 return '"file_date_min" must be <str>'
596 #----------------------------------------------------#
233 #----------------------------------------------------#
597 if 'file_date_max' in query:
234 if 'file_date_max' in query:
598 if type(query['file_date_max']) is str:
235 if type(query['file_date_max']) is str:
599 try:
236 try:
600 datetime.strptime(query['file_date_max'], '%Y-%m-%d')
237 datetime.strptime(query['file_date_max'], '%Y-%m-%d')
601 if len(query['file_date_max']) != 10:
238 if len(query['file_date_max']) != 10:
602 return '"file_date_max", must be: <YYYY-MM-DD>'
239 return '"file_date_max", must be: <YYYY-MM-DD>'
603
240
604 if 'file_date_min' in query:
241 if 'file_date_min' in query:
605 if query['file_date_min'] > query['file_date_max']:
242 if query['file_date_min'] > query['file_date_max']:
606 return '"file_date_max" must be greater than "file_date_min"'
243 return '"file_date_max" must be greater than "file_date_min"'
607 except:
244 except:
608 return '"file_date_max" incorrect: "%s"' % (query['file_date_max'])
245 return '"file_date_max" incorrect: "%s"' % (query['file_date_max'])
609 else:
246 else:
610 return '"file_date_max" must be <str>'
247 return '"file_date_max" must be <str>'
611 #----------------------------------------------------#
248 #----------------------------------------------------#
612 self.dict['query'] = query
249 self.dict['query'] = query
613 else:
250 else:
614 return '"query" must be <dict>'
251 return '"query" must be <dict>'
615 return getattr(self.ckan.action, 'resources_search')(**self.dict)
252 return getattr(self.ckan.action, 'resources_search')(**self.dict)
616
253
617 elif type_option == 'tag':
254 elif type_option == 'tag':
618 for key1, value1 in kwargs.items():
255 for key1, value1 in kwargs.items():
619 if key1 != 'fields':
256 if key1 != 'fields':
620 self.dict[key1] = value1
257 self.dict[key1] = value1
621
258
622 if not 'vocabulary_id' in kwargs:
259 if not 'vocabulary_id' in kwargs:
623 print('Missing "vocabulary_id" value: tags that don’t belong to any vocabulary')
260 print('Missing "vocabulary_id" value: tags that don’t belong to any vocabulary')
624 else:
261 else:
625 print('Only tags that belong to "{}" vocabulary'.format(kwargs['vocabulary_id']))
262 print('Only tags that belong to "{}" vocabulary'.format(kwargs['vocabulary_id']))
626
263
627 if query is not None:
264 if query is not None:
628 if type(query) is dict:
265 if type(query) is dict:
629 if 'search' in query:
266 if 'search' in query:
630 if type(query['search']) is list or type(query['search']) is str:
267 if type(query['search']) is list or type(query['search']) is str:
631 self.dict['query'] = query['search']
268 self.dict['query'] = query['search']
632 else:
269 else:
633 return '"search" must be <list> or <str>'
270 return '"search" must be <list> or <str>'
634 else:
271 else:
635 return '"query" must be <dict>'
272 return '"query" must be <dict>'
636 return getattr(self.ckan.action, 'tag_search')(**self.dict)
273 return getattr(self.ckan.action, 'tag_search')(**self.dict)
637
274
638 else:
275 else:
639 return 'ERROR:: "type_option = %s" is not accepted' % (type_option)
276 return 'ERROR:: "type_option = %s" is not accepted' % (type_option)
640
277
641 except:
278 except:
642 _, exc_value, _ = sys.exc_info()
279 _, exc_value, _ = sys.exc_info()
643 return exc_value
280 return exc_value
644 else:
281 else:
645 return 'ERROR:: "type_option" must be <str>'
282 return 'ERROR:: "type_option" must be <str>'
646
283
647 def create(self, type_option, select=None, **kwargs):
284 def create(self, type_option, select=None, **kwargs):
648 '''
285 '''
649 FINALIDAD:
286 FINALIDAD:
650 Funcion personalizada para crear.
287 Funcion personalizada para crear.
651
288
652 PARAMETROS DISPONIBLES:
289 PARAMETROS DISPONIBLES:
653 CONSULTAR: "GUIA DE SCRIPT.pdf"
290 CONSULTAR: "GUIA DE SCRIPT.pdf"
654
291
655 ESTRUCTURA:
292 ESTRUCTURA:
656 <access_name>.create(type_option = <class 'str'>, param_1 = <class 'param_1'>, ...)
293 <access_name>.create(type_option = <class 'str'>, param_1 = <class 'param_1'>, ...)
657 '''
294 '''
658 if type(type_option) is str:
295 if type(type_option) is str:
659 try:
296 try:
660 if type_option == 'dataset':
297 if type_option == 'dataset':
661 return getattr(self.ckan.action, 'package_create')(**kwargs)
298 return getattr(self.ckan.action, 'package_create')(**kwargs)
662 if type_option == 'resource':
299 if type_option == 'resource':
663 return resource.resource_create(self, **kwargs)
300 return resource.resource_create(self, **kwargs)
664 elif type_option == 'project':
301 elif type_option == 'project':
665 return getattr(self.ckan.action, 'organization_create')(**kwargs)
302 return getattr(self.ckan.action, 'organization_create')(**kwargs)
666 elif type_option == 'member':
303 elif type_option == 'member':
667 return getattr(self.ckan.action, 'organization_member_create')(**kwargs)
304 return getattr(self.ckan.action, 'organization_member_create')(**kwargs)
668 elif type_option == 'collaborator':
305 elif type_option == 'collaborator':
669 return getattr(self.ckan.action, 'package_collaborator_create')(**kwargs)
306 return getattr(self.ckan.action, 'package_collaborator_create')(**kwargs)
670 elif type_option == 'vocabulary':
307 elif type_option == 'vocabulary':
671 return getattr(self.ckan.action, 'vocabulary_create')(**kwargs)
308 return getattr(self.ckan.action, 'vocabulary_create')(**kwargs)
672 elif type_option == 'tag':
309 elif type_option == 'tag':
673 return getattr(self.ckan.action, 'tag_create')(**kwargs)
310 return getattr(self.ckan.action, 'tag_create')(**kwargs)
674 elif type_option == 'user':
311 elif type_option == 'user':
675 return getattr(self.ckan.action, 'user_create')(**kwargs)
312 return getattr(self.ckan.action, 'user_create')(**kwargs)
676 elif type_option == 'views':
313 elif type_option == 'views':
677 if 'resource' == select:
314 if 'resource' == select:
678 self.list = ['package']
315 self.list = ['package']
679 for key1, value1 in kwargs.items():
316 for key1, value1 in kwargs.items():
680 if not key1 in self.list:
317 if not key1 in self.list:
681 self.dict[key1] = value1
318 self.dict[key1] = value1
682 return getattr(self.ckan.action, 'resource_create_default_resource_views')(**self.dict)
319 return getattr(self.ckan.action, 'resource_create_default_resource_views')(**self.dict)
683 elif 'dataset' == select:
320 elif 'dataset' == select:
684 return getattr(self.ckan.action, 'package_create_default_resource_views')(**kwargs)
321 return getattr(self.ckan.action, 'package_create_default_resource_views')(**kwargs)
685 else:
322 else:
686 return 'ERROR:: "select = %s" is not accepted' % (select)
323 return 'ERROR:: "select = %s" is not accepted' % (select)
687 else:
324 else:
688 return 'ERROR:: "type_option = %s" is not accepted' % (type_option)
325 return 'ERROR:: "type_option = %s" is not accepted' % (type_option)
689 except:
326 except:
690 _, exc_value, _ = sys.exc_info()
327 _, exc_value, _ = sys.exc_info()
691 return exc_value
328 return exc_value
692 else:
329 else:
693 return 'ERROR:: "type_option" must be <str>'
330 return 'ERROR:: "type_option" must be <str>'
694
331
695 def patch(self, type_option, **kwargs):
332 def patch(self, type_option, **kwargs):
696 '''
333 '''
697 FINALIDAD:
334 FINALIDAD:
698 Funciones personalizadas para actualizar
335 Funciones personalizadas para actualizar
699
336
700 PARAMETROS DISPONIBLES:
337 PARAMETROS DISPONIBLES:
701 CONSULTAR: "GUIA DE SCRIPT.pdf"
338 CONSULTAR: "GUIA DE SCRIPT.pdf"
702
339
703 ESTRUCTURA:
340 ESTRUCTURA:
704 <access_name>.patch(type_option = <class 'str'>, param_1 = <class 'param_1'>, ...)
341 <access_name>.patch(type_option = <class 'str'>, param_1 = <class 'param_1'>, ...)
705 '''
342 '''
706 if type(type_option) is str:
343 if type(type_option) is str:
707 try:
344 try:
708 if type_option == 'dataset':
345 if type_option == 'dataset':
709 #Agregar que solo se debe modificar parámetros del Dataset y que no incluya Resources
346 #Agregar que solo se debe modificar parámetros del Dataset y que no incluya Resources
710 return getattr(self.ckan.action, 'package_patch')(**kwargs)
347 return getattr(self.ckan.action, 'package_patch')(**kwargs)
711 elif type_option == 'project':
348 elif type_option == 'project':
712 return getattr(self.ckan.action, 'organization_patch')(**kwargs)
349 return getattr(self.ckan.action, 'organization_patch')(**kwargs)
713 elif type_option == 'resource':
350 elif type_option == 'resource':
714 return resource.resource_patch(self, **kwargs)
351 return resource.resource_patch(self, **kwargs)
715 elif type_option == 'member':
352 elif type_option == 'member':
716 return getattr(self.ckan.action, 'organization_member_create')(**kwargs)
353 return getattr(self.ckan.action, 'organization_member_create')(**kwargs)
717 elif type_option == 'collaborator':
354 elif type_option == 'collaborator':
718 return getattr(self.ckan.action, 'package_collaborator_create')(**kwargs)
355 return getattr(self.ckan.action, 'package_collaborator_create')(**kwargs)
719 else:
356 else:
720 return 'ERROR:: "type_option = %s" is not accepted' % (type_option)
357 return 'ERROR:: "type_option = %s" is not accepted' % (type_option)
721 except:
358 except:
722 _, exc_value, _ = sys.exc_info()
359 _, exc_value, _ = sys.exc_info()
723 return exc_value
360 return exc_value
724 else:
361 else:
725 return 'ERROR:: "type_option" must be <str>'
362 return 'ERROR:: "type_option" must be <str>'
726
363
727 def delete(self, type_option, select=None, **kwargs):
364 def delete(self, type_option, select=None, **kwargs):
728 '''
365 '''
729 FINALIDAD:
366 FINALIDAD:
730 Función personalizada para eliminar y/o purgar.
367 Función personalizada para eliminar y/o purgar.
731
368
732 PARAMETROS DISPONIBLES:
369 PARAMETROS DISPONIBLES:
733 CONSULTAR: "GUIA DE SCRIPT.pdf"
370 CONSULTAR: "GUIA DE SCRIPT.pdf"
734
371
735 ESTRUCTURA:
372 ESTRUCTURA:
736 <access_name>.delete(type_option = <class 'str'>, param_1 = <class 'param_1'>, ...)
373 <access_name>.delete(type_option = <class 'str'>, param_1 = <class 'param_1'>, ...)
737 '''
374 '''
738 if type(type_option) is str:
375 if type(type_option) is str:
739 try:
376 try:
740 if type_option == 'dataset':
377 if type_option == 'dataset':
741 if select is None:
378 if select is None:
742 return 'ERROR:: "select" must not be "None"'
379 return 'ERROR:: "select" must not be "None"'
743 else:
380 else:
744 if 'delete' == select:
381 if 'delete' == select:
745 return getattr(self.ckan.action, 'package_delete')(**kwargs)
382 return getattr(self.ckan.action, 'package_delete')(**kwargs)
746 elif 'purge' == select:
383 elif 'purge' == select:
747 return getattr(self.ckan.action, 'dataset_purge')(**kwargs)
384 return getattr(self.ckan.action, 'dataset_purge')(**kwargs)
748 else:
385 else:
749 return 'ERROR:: "select = %s" is not accepted' % (select)
386 return 'ERROR:: "select = %s" is not accepted' % (select)
750 elif type_option == 'project':
387 elif type_option == 'project':
751 if select is None:
388 if select is None:
752 return 'ERROR:: "select" must not be "None"'
389 return 'ERROR:: "select" must not be "None"'
753 else:
390 else:
754 if 'delete' == select:
391 if 'delete' == select:
755 return getattr(self.ckan.action, 'organization_delete')(**kwargs)
392 return getattr(self.ckan.action, 'organization_delete')(**kwargs)
756 elif 'purge' == select:
393 elif 'purge' == select:
757 return getattr(self.ckan.action, 'organization_purge')(**kwargs)
394 return getattr(self.ckan.action, 'organization_purge')(**kwargs)
758 else:
395 else:
759 return 'ERROR:: "select = %s" is not accepted' % (select)
396 return 'ERROR:: "select = %s" is not accepted' % (select)
760 elif type_option == 'resource':
397 elif type_option == 'resource':
761 if select is None:
398 if select is None:
762 return 'ERROR:: "select" must not be "None"'
399 return 'ERROR:: "select" must not be "None"'
763 else:
400 else:
764 return resource.resource_delete(self, select, **kwargs)
401 return resource.resource_delete(self, select, **kwargs)
765 elif type_option == 'vocabulary':
402 elif type_option == 'vocabulary':
766 return getattr(self.ckan.action, 'vocabulary_delete')(**kwargs)
403 return getattr(self.ckan.action, 'vocabulary_delete')(**kwargs)
767 elif type_option == 'tag':
404 elif type_option == 'tag':
768 return getattr(self.ckan.action, 'tag_delete')(**kwargs)
405 return getattr(self.ckan.action, 'tag_delete')(**kwargs)
769 elif type_option == 'user':
406 elif type_option == 'user':
770 return getattr(self.ckan.action, 'user_delete')(**kwargs)
407 return getattr(self.ckan.action, 'user_delete')(**kwargs)
771 else:
408 else:
772 return 'ERROR:: "type_option = %s" is not accepted' % (type_option)
409 return 'ERROR:: "type_option = %s" is not accepted' % (type_option)
773 except:
410 except:
774 _, exc_value, _ = sys.exc_info()
411 _, exc_value, _ = sys.exc_info()
775 return exc_value
412 return exc_value
776 else:
413 else:
777 return 'ERROR:: "type_option" must be <str>'
414 return 'ERROR:: "type_option" must be <str>'
778
779 def f_status_note(self, total, result, path):
780 file_txt = open(path+'status_note.txt', 'w')
781 file_txt = open(path+'status_note.txt', 'a')
782
783 file_txt.write('DOWNLOADED FILE(S): "%s"' % (len(result['name'])))
784 file_txt.write(''+ os.linesep)
785 for u in result['name']:
786 file_txt.write(' - '+ u + os.linesep)
787 file_txt.write(''+ os.linesep)
788
789 file_txt.write('FAILED FILE(S): "%s"' % (len(total['name'])-len(result['name'])))
790 file_txt.write(''+ os.linesep)
791 if len(total['name'])-len(result['name']) != 0:
792 for u in total['name']:
793 if not u in result['name']:
794 file_txt.write(' - '+ u + os.linesep)
795 else:
796 file_txt.write(' "None"'+ os.linesep)
797
798 def f_name(self, name_dataset, ext, tempdir):
799 while self.check:
800 self.str = ''
801 if self.cont == 0:
802 if os.path.exists(tempdir + name_dataset + ext):
803 self.str = name_dataset+'('+str(self.cont+1)+')'+ext
804 else:
805 self.check = self.check * 0
806 self.str = name_dataset + ext
807 else:
808 if not os.path.exists(tempdir + name_dataset+'('+str(self.cont)+')'+ext):
809 self.check = self.check * 0
810 self.str = name_dataset+'('+str(self.cont)+')'+ ext
811 self.cont = self.cont+1
812 return self.str
813
814 def f_zipdir(self, path, ziph, zip_name):
815 for root, _, files in os.walk(path):
816 print('.....')
817 print('Creating: "{}" >>'.format(zip_name))
818 for __file in tqdm(iterable=files, total=len(files)):
819 new_dir = os.path.relpath(os.path.join(root, __file), os.path.join(path, '..'))
820 ziph.write(os.path.join(root, __file), new_dir)
821 print('Created >>')
822
823 def download_by_step(self, response, tempdir_name):
824 try:
825 # ---------- REPLACE URL --------- #
826 if urlparse(self.url).netloc != 'www.igp.gob.pe' and urlparse(response['url']).netloc == 'www.igp.gob.pe':
827 response['url'] = response['url'].replace(urlparse(response['url']).scheme + '://' + urlparse(response['url']).netloc,
828 urlparse(self.url).scheme + '://' + urlparse(self.url).netloc)
829 #----------------------------------#
830 with requests.get(response['url'], stream=True, headers={'Authorization': self.Authorization}, verify=self.verify) as resp:
831 if resp.status_code == 200:
832 with open(tempdir_name+response['name'], 'wb') as file:
833 for chunk in resp.iter_content(chunk_size = self.chunk_size):
834 if chunk:
835 file.write(chunk)
836 except requests.exceptions.RequestException:
837 pass
838
415
839 def download_files(self, **kwargs):
416 def download_files(self, id_or_name, processes=1, path=os.path.expanduser("~"), **kwargs):
840 '''
841 FINALIDAD:
842 Funcion personalizada para la descarga de archivos existentes de un dataset.
843
844 PARAMETROS DISPONIBLES:
845 CONSULTAR: "GUIA DE SCRIPT.pdf"
846
847 ESTRUCTURA:
848 <access_name>.download_files(id = <class 'str'>, param_1 = <class 'param_1'>, ...)
849 '''
850 dict_local = {}
851 #----------------------------------------------#
852 if 'zip' in kwargs:
853 if type(kwargs['zip']) is not bool:
854 return 'ERROR:: "zip" must be: <class "bool">'
855 else:
856 dict_local['zip'] = kwargs['zip']
857 else:
858 dict_local['zip'] = False
859 #----------------------------------------------#
860 if 'status_note' in kwargs:
861 if type(kwargs['status_note']) is not bool:
862 return 'ERROR:: "status_note" must be: <class "bool">'
863 else:
864 dict_local['status_note'] = kwargs['status_note']
865 else:
866 dict_local['status_note'] = False
867 #----------------------------------------------#
868 if 'path' in kwargs:
869 if type(kwargs['path']) is str:
870 if os.path.isdir(kwargs['path']) == False:
871 return 'ERROR:: "path" does not exist'
872 else:
873 if kwargs['path'][-1:] != self.separator:
874 dict_local['path'] = kwargs['path']+self.separator
875 else:
876 dict_local['path'] = kwargs['path']
877
878 txt = dict_local['path']+datetime.now().strftime("%Y_%m_%d_%H_%M_%S_%f")+'.txt'
879 if int(platform.python_version()[0]) == 3:
880 try:
881 file_txt = open(txt, 'w')
882 file_txt.close()
883 os.remove(txt)
884 except PermissionError:
885 return 'ERROR:: Access denied, you are not authorized to write files: "%s"' % (dict_local['path'])
886 else:
887 try:
888 file_txt = open(txt, 'w')
889 file_txt.close()
890 os.remove(txt)
891 except:
892 return 'ERROR:: Access denied, you are not authorized to write files: "%s"' % (dict_local['path'])
893 else:
894 return 'ERROR:: "path" must be: <class "str">'
895 else:
896 dict_local['path'] = ''
897 #----------------------------------------------#
898 for key, value in kwargs.items():
899 if not key in dict_local:
900 self.dict[key] = value
901 try:
902 response = getattr(self.ckan.action, 'url_resources')(**self.dict)
903 except:
904 _, exc_value, _ = sys.exc_info()
905 return exc_value
906
907 if len(response) != 0:
908 #--------------TEMP PATH---------------#
909 if dict_local['zip']:
910 tempdir = tempfile.mkdtemp(prefix=kwargs['id']+'-')+self.separator
911 os.mkdir(tempdir+kwargs['id'])
912 dir_name = tempdir + kwargs['id'] + self.separator
913 else:
914 dir = self.f_name(kwargs['id'], '', dict_local['path'])
915 os.mkdir(dict_local['path'] + dir)
916 dir_name = dict_local['path'] + dir + self.separator
917 #-----------DOWNLOAD FILES-------------#
918 print('.....')
919 print('Downloading "{}" file(s) >>'.format(len(response)))
920 name_total = {'name': []}
921 with concurrent.futures.ThreadPoolExecutor() as executor:
922 for u in tqdm(iterable=response, total=len(response)):
923 name_total['name'].append(u['name'])
924 executor.submit(self.download_by_step, u, dir_name)
925 name_check = {}
926 name_check['name'] = [f for f in os.listdir(dir_name) if os.path.isfile(os.path.join(dir_name, f))]
927 print('"{}" downloaded file(s) successfully >>'.format(len(name_check['name'])))
928 #--------------------------------------#
929 if len(name_check['name']) != 0:
930 #----------Status Note---------#
931 if dict_local['status_note']:
932 print('.....')
933 print('Creating: "status_note.txt" >>')
934 self.f_status_note(name_total, name_check, dir_name)
935 print('Created>>')
936 #----------ZIP CREATE----------#
937 if dict_local['zip']:
938 zip_name = self.f_name(kwargs['id'], '.zip', dict_local['path'])
939 ziph = zipfile.ZipFile(dict_local['path'] + zip_name, 'w', zipfile.ZIP_DEFLATED, allowZip64=True)
940 self.f_zipdir(dir_name, ziph, zip_name)
941 ziph.close()
942 #Delete Temporal Path
943 if os.path.exists(tempdir[:-1]):
944 shutil.rmtree(tempdir[:-1])
945 #------------------------------#
946 print('.....')
947 return 'DOWNLOAD FINISHED'
948 else:
949 #Delete Temporal Path
950 if dict_local['zip']:
951 if os.path.exists(tempdir[:-1]):
952 shutil.rmtree(tempdir[:-1])
953 else:
954 if os.path.exists(dir_name[:-1]):
955 shutil.rmtree(dir_name[:-1])
956 return 'NO FILES WERE DOWNLOADED'
957 else:
958 return 'FILES NOT FOUND'
959
960 def download_files_advance(self, id_or_name, processes=1, path=os.path.expanduser("~"), **kwargs):
961 '''
417 '''
962 FINALIDAD:
418 FINALIDAD:
963 Funcion personalizada avanzada para la descarga de archivos existentes de un(os) dataset(s).
419 Funcion personalizada avanzada para la descarga de archivos existentes de un(os) dataset(s).
964
420
965 PARAMETROS DISPONIBLES:
421 PARAMETROS DISPONIBLES:
966 CONSULTAR: "GUIA DE SCRIPT.pdf"
422 CONSULTAR: "GUIA DE SCRIPT.pdf"
967
423
968 ESTRUCTURA:
424 ESTRUCTURA:
969 <access_name>.download_files_advance(id_or_name= <class 'str' or 'list'>, param_1 = <class 'param_1'>, ...)
425 <access_name>.download_files(id_or_name= <class 'str' or 'list'>, param_1 = <class 'param_1'>, ...)
970 '''
426 '''
971 #------------------ PATH ----------------------#
427 #------------------ PATH ----------------------#
972 if isinstance(path, str):
428 if isinstance(path, str):
973 if os.path.isdir(path):
429 if os.path.isdir(path):
974 if not path.endswith(os.sep):
430 if not path.endswith(os.sep):
975 path = path + os.sep
431 path = path + os.sep
976 test_txt = path + datetime.now().strftime("%Y_%m_%d_%H_%M_%S_%f")+'.txt'
432 test_txt = path + datetime.now().strftime("%Y_%m_%d_%H_%M_%S_%f")+'.txt'
977 try:
433 try:
978 file_txt = open(test_txt, 'w')
434 file_txt = open(test_txt, 'w')
979 file_txt.close()
435 file_txt.close()
980 os.remove(test_txt)
436 os.remove(test_txt)
981 except:
437 except:
982 return 'ERROR:: Access denied, you are not authorized to write files: "%s"' % (path)
438 return 'ERROR:: Access denied, you are not authorized to write files: "%s"' % (path)
983 else:
439 else:
984 return 'ERROR:: "path" does not exist'
440 return 'ERROR:: "path" does not exist'
985 else:
441 else:
986 return 'ERROR:: "path" must be: <class "str">'
442 return 'ERROR:: "path" must be: <class "str">'
987
443
988 #------------------ PROCESSES -----------------#
444 #------------------ PROCESSES -----------------#
989 if not isinstance(processes, int):
445 if not isinstance(processes, int):
990 return 'ERROR:: "processes" must be: <class "int">'
446 return 'ERROR:: "processes" must be: <class "int">'
991
447
992 #------------------ ID OR NAME ----------------#
448 #------------------ ID OR NAME ----------------#
993 if isinstance(id_or_name, str):
449 if isinstance(id_or_name, str):
994 id_or_name = [id_or_name]
450 id_or_name = [id_or_name]
995 elif isinstance(id_or_name, list):
451 elif isinstance(id_or_name, list):
996 id_or_name = list(map(str, id_or_name))
452 id_or_name = list(map(str, id_or_name))
997 else:
453 else:
998 return 'ERROR:: dataset "id_or_name" must be: <class "str" or "list">'
454 return 'ERROR:: dataset "id_or_name" must be: <class "str" or "list">'
999 #----------------------------------------------#
455 #----------------------------------------------#
1000 arguments = {
456 arguments = {
1001 '--apikey': self.Authorization,
457 '--apikey': self.Authorization,
1002 '--ckan-user': None,
458 '--ckan-user': None,
1003 '--config': None,
459 '--config': None,
1004 '--datapackages': path,
460 '--datapackages': path,
1005 '--datastore-fields': False,
461 '--datastore-fields': False,
1006 '--get-request': False,
462 '--get-request': False,
1007 '--insecure': not self.verify,
463 '--insecure': not self.verify,
1008 '--processes': str(processes),
464 '--processes': str(processes),
1009 '--quiet': False,
465 '--quiet': False,
1010 '--remote': self.url,
466 '--remote': self.url,
1011 '--worker': False,
467 '--worker': False,
1012 #'--log': 'log.txt',
468 #'--log': 'log.txt',
1013 #'--all': False,
469 #'--all': False,
1014 #'--gzip': False,
470 #'--gzip': False,
1015 #'--output': None,
471 #'--output': None,
1016 #'--max-records': None,
472 #'--max-records': None,
1017 #'--output-json': False,
473 #'--output-json': False,
1018 #'--output-jsonl': False,
474 #'--output-jsonl': False,
1019 #'--create-only': False,
475 #'--create-only': False,
1020 #'--help': False,
476 #'--help': False,
1021 #'--input': None,
477 #'--input': None,
1022 #'--input-json': False,
478 #'--input-json': False,
1023 #'--start-record': '1',
479 #'--start-record': '1',
1024 #'--update-only': False,
480 #'--update-only': False,
1025 #'--upload-logo': False,
481 #'--upload-logo': False,
1026 #'--upload-resources': False,
482 #'--upload-resources': False,
1027 #'--version': False,
483 #'--version': False,
1028 'ID_OR_NAME': id_or_name,
484 'ID_OR_NAME': id_or_name,
1029 'datasets': True,
485 'datasets': True,
1030 'dump': True,
486 'dump': True,
1031 #'ACTION_NAME': None,
487 #'ACTION_NAME': None,
1032 #'KEY:JSON': [],
488 #'KEY:JSON': [],
1033 #'KEY=STRING': [],
489 #'KEY=STRING': [],
1034 #'KEY@FILE': [],
490 #'KEY@FILE': [],
1035 #'action': False,
491 #'action': False,
1036 #'delete': False,
492 #'delete': False,
1037 #'groups': False,
493 #'groups': False,
1038 #'load': False,
494 #'load': False,
1039 #'organizations': False,
495 #'organizations': False,
1040 #'related': False,
496 #'related': False,
1041 #'search': False,
497 #'search': False,
1042 #'users': False
498 #'users': False
1043 }
499 }
1044 return logic_download.dump_things_change(self.ckan, 'datasets', arguments, **kwargs) No newline at end of file
500 return logic_download.dump_things_change(self.ckan, 'datasets', arguments, **kwargs)
General Comments 0
You need to be logged in to leave comments. Login now