##// END OF EJS Templates
DATABASES » ckanext-jro » api-cliente
RSS
v2.9.2 :: Add 'validator name' upload file and multiple_file
eynilupu -
r8:19d5924fa8b4
parent child
Show More
Show file before | Show file after | Hide comments
1 NO CONTENT: modified file, binary diff hidden
NO CONTENT: modified file, binary diff hidden
Show file before | Show file after | Hide comments
@@ -1,878 +1,916
1 from ckanapi import RemoteCKAN
1 from ckanapi import RemoteCKAN
2 from datetime import datetime
2 from datetime import datetime
3 from tqdm import tqdm
3 from tqdm import tqdm
4 #from ckanapi.errors import NotAuthorized, NotFound, ValidationError, SearchQueryError, SearchError, CKANAPIError, ServerIncompatibleError
4 #from ckanapi.errors import NotAuthorized, NotFound, ValidationError, SearchQueryError, SearchError, CKANAPIError, ServerIncompatibleError
5 import sys
5 import sys
6 import platform
6 import platform
7 import os
7 import os
8 import tempfile
8 import tempfile
9 import shutil
9 import shutil
10 import zipfile
10 import zipfile
11 import concurrent.futures
11 import concurrent.futures
12 import requests
12 import requests
13 import json
13 import json
14 #import pathlib
14 #import pathlib
15 import uuid
15 import uuid
16
16
17 class JROAPI():
17 class JROAPI():
18 """
18 """
19 FINALIDAD:
19 FINALIDAD:
20 Script para administrar y obtener la data del repositorio por medio de APIs.
20 Script para administrar y obtener la data del repositorio por medio de APIs.
21
21
22 REQUISITIOS PREVIOS:
22 REQUISITIOS PREVIOS:
23 - Paso 1: Tener "pip [Python 2]" o "pip3 [Python 3]" instalado:
23 - Paso 1: Tener "pip [Python 2]" o "pip3 [Python 3]" instalado:
24 - Paso 2: Instalar lo siguiente como admininstrador:
24 - Paso 2: Instalar lo siguiente como admininstrador:
25 En Python 2
25 En Python 2
26 - pip install ckanapi==4.5
26 - pip install ckanapi==4.5
27 - pip install requests
27 - pip install requests
28 - pip install futures
28 - pip install futures
29 - pip install tqdm
29 - pip install tqdm
30 En Python > 3
30 En Python > 3
31 - pip3 install ckanapi==4.5
31 - pip3 install ckanapi==4.5
32 - pip3 install requests
32 - pip3 install requests
33 - pip3 install tqdm
33 - pip3 install tqdm
34
34
35 FUNCIONES DISPONIBLES:
35 FUNCIONES DISPONIBLES:
36 - action
36 - action
37 - upload_file
37 - upload_file
38 - upload_multiple_files
38 - upload_multiple_files
39 - upload_multiple_files_advance
39 - upload_multiple_files_advance
40 - show
40 - show
41 - search
41 - search
42 - create
42 - create
43 - patch
43 - patch
44 - delete
44 - delete
45 - download_files
45 - download_files
46
46
47 EJEMPLOS:
47 EJEMPLOS:
48 #1:
48 #1:
49 with JROAPI('http://demo.example.com', Authorization='#########') as <access_name>:
49 with JROAPI('http://demo.example.com', Authorization='#########') as <access_name>:
50 ... some operation(s) ...
50 ... some operation(s) ...
51 #2:
51 #2:
52 <access_name> = JROAPI('http://example.com', Authorization='#########')
52 <access_name> = JROAPI('http://example.com', Authorization='#########')
53 ... some operation(s) ...
53 ... some operation(s) ...
54 <access_name>.ckan.close()
54 <access_name>.ckan.close()
55
55
56 REPORTAR ALGUN PROBLEMA:
56 REPORTAR ALGUN PROBLEMA:
57 Debe enviar un correo a eynilupu@igp.gob.pe detallando los siguientes pasos:
57 Debe enviar un correo a eynilupu@igp.gob.pe detallando los siguientes pasos:
58 1) Correo para contactarlo
58 1) Correo para contactarlo
59 2) Descripcion del problema
59 2) Descripcion del problema
60 3) ¿En que paso o seccion encontro el problema?
60 3) ¿En que paso o seccion encontro el problema?
61 4) ¿Cual era el resultado que usted esperaba?
61 4) ¿Cual era el resultado que usted esperaba?
62 """
62 """
63 def __init__(self, url, Authorization=None):
63 def __init__(self, url, Authorization=None):
64 ua = 'CKAN_JRO/1.1 (+'+str(url)+')'
64 ua = 'CKAN_JRO/1.1 (+'+str(url)+')'
65 #ua = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36'
65 #ua = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36'
66 self.ckan = RemoteCKAN(url, apikey=Authorization, user_agent=ua)
66 self.ckan = RemoteCKAN(url, apikey=Authorization, user_agent=ua)
67 #self.ckan = RemoteCKAN(url, apikey=Authorization)
67 #self.ckan = RemoteCKAN(url, apikey=Authorization)
68 self.Authorization = Authorization
68 self.Authorization = Authorization
69 # Change for --> self.separator = os.sep
69 # Change for --> self.separator = os.sep
70 if platform.system() == 'Windows':
70 if platform.system() == 'Windows':
71 self.separator = '\\'
71 self.separator = '\\'
72 else:
72 else:
73 self.separator = '/'
73 self.separator = '/'
74
74
75 self.chunk_size = 1024
75 self.chunk_size = 1024
76 self.list = []
76 self.list = []
77 self.dict = {}
77 self.dict = {}
78 self.str = ''
78 self.str = ''
79 self.check = 1
79 self.check = 1
80 self.cont = 0
80 self.cont = 0
81
81
82 def __enter__(self):
82 def __enter__(self):
83 return self
83 return self
84
84
85 def __exit__(self, *args):
85 def __exit__(self, *args):
86 self.ckan.close()
86 self.ckan.close()
87
87
88 def action(self, action, **kwargs):
88 def action(self, action, **kwargs):
89 """
89 """
90 FINALIDAD:
90 FINALIDAD:
91 Funcion para llamar a las APIs disponibles
91 Funcion para llamar a las APIs disponibles
92
92
93 APIs DISPONIBLES:
93 APIs DISPONIBLES:
94 CONSULTAR: "GUIA DE SCRIPT.pdf"
94 CONSULTAR: "GUIA DE SCRIPT.pdf"
95
95
96 EJEMPLO:
96 EJEMPLO:
97 <access_name>.action(<consuming API>, param_1 = <class 'param_1'>, ...)
97 <access_name>.action(<consuming API>, param_1 = <class 'param_1'>, ...)
98 """
98 """
99 #--------------- CASE: PACKAGE SEARCH ---------------#
99 #--------------- CASE: PACKAGE SEARCH ---------------#
100 if kwargs is not None:
100 if kwargs is not None:
101 if action == 'package_search':
101 if action == 'package_search':
102 self.list = ['facet_mincount', 'facet_limit', 'facet_field']
102 self.list = ['facet_mincount', 'facet_limit', 'facet_field']
103 for facet in self.list:
103 for facet in self.list:
104 if facet in kwargs:
104 if facet in kwargs:
105 kwargs[facet.replace('_', '.')] = kwargs[facet]
105 kwargs[facet.replace('_', '.')] = kwargs[facet]
106 kwargs.pop(facet)
106 kwargs.pop(facet)
107 #----------------------------------------------------#
107 #----------------------------------------------------#
108 try:
108 try:
109 return getattr(self.ckan.action, action)(**kwargs)
109 return getattr(self.ckan.action, action)(**kwargs)
110 except:
110 except:
111 _, exc_value, _ = sys.exc_info()
111 _, exc_value, _ = sys.exc_info()
112 return exc_value
112 return exc_value
113
113
114 def upload_file(self, dataset_id, file_path, file_date, file_type, **kwargs):
114 def upload_file(self, dataset_id, file_path, file_date, file_type, **kwargs):
115 # Agregar si es interruptido por teclado
115 # Agregar si es interruptido por teclado
116 '''
116 '''
117 FINALIDAD:
117 FINALIDAD:
118 Funcion para subir un unico archivo al repositorio del ROJ.
118 Funcion para subir un unico archivo al repositorio del ROJ.
119
119
120 PARAMETROS DISPONIBLES:
120 PARAMETROS DISPONIBLES:
121 CONSULTAR: "GUIA DE SCRIPT.pdf"
121 CONSULTAR: "GUIA DE SCRIPT.pdf"
122
122
123 ESTRUCTURA:
123 ESTRUCTURA:
124 <access_name>.upload_file(dataset_id = <class 'str'>, file_date = <class 'str'>, file_path = <class 'str'>, file_type = <class 'str'>, param_1 = <class 'param_1'>, ...)
124 <access_name>.upload_file(dataset_id = <class 'str'>, file_date = <class 'str'>, file_path = <class 'str'>, file_type = <class 'str'>, param_1 = <class 'param_1'>, ...)
125 '''
125 '''
126 self.list = ['package_id', 'upload', 'voc_file_type', 'name'] #file_date
126 self.list = ['package_id', 'upload', 'voc_file_type', 'name'] #file_date
127 for key1, value1 in kwargs.items():
127 for key1, value1 in kwargs.items():
128 if not key1 in self.list:
128 if not key1 in self.list:
129 self.dict[key1] = value1
129 self.dict[key1] = value1
130
130
131 #---------------------------#
131 #---------------------------#
132 if not 'others' in kwargs:
132 if not 'others' in kwargs:
133 self.dict['others'] = ''
133 self.dict['others'] = ''
134 else:
134 else:
135 if isinstance(kwargs['others'], list):
135 if isinstance(kwargs['others'], list):
136 self.dict['others'] = json.dumps(kwargs['others'])
136 self.dict['others'] = json.dumps(kwargs['others'])
137 #---------------------------#
137 #---------------------------#
138
138
139 if not os.path.isfile(file_path):
139 if not os.path.isfile(file_path):
140 return 'File "%s" not exist' % (file_path)
140 return 'File "%s" not exist' % (file_path)
141
141
142 #if not 'format' in self.dict:
142 #if not 'format' in self.dict:
143 # self.str = ''.join(pathlib.Path(file_path).suffixes)
143 # self.str = ''.join(pathlib.Path(file_path).suffixes)
144 # if len(self.str) > 0:
144 # if len(self.str) > 0:
145 # self.dict['format'] = self.str.upper()[1:]
145 # self.dict['format'] = self.str.upper()[1:]
146
146
147 #-------------------------PACKAGE SHOW-----------------------#
148 try:
149 dataset_show = getattr(self.ckan.action, 'package_show')(id=dataset_id)['resources']
150 except:
151 _, exc_value, _ = sys.exc_info()
152 print('ERROR obtaining metadata dataset:: Use the "print" for more information')
153 return exc_value
154
155 resources_name = []
156 for u in dataset_show:
157 resources_name.append(u['name'].lower())
158
159 if os.path.basename(file_path).lower() in resources_name:
160 return 'ERROR:: "%s" file already exist in this dataset' % (os.path.basename(file_path))
161 #------------------------------------------------------------#
162
147 try:
163 try:
148 return getattr(self.ckan.action, 'resource_create')(package_id=dataset_id, file_date=file_date, upload=open(file_path, 'rb'), voc_file_type=file_type, name=os.path.basename(file_path), **self.dict)
164 return getattr(self.ckan.action, 'resource_create')(package_id=dataset_id, file_date=file_date, upload=open(file_path, 'rb'), voc_file_type=file_type, name=os.path.basename(file_path), **self.dict)
149 except:
165 except:
150 _, exc_value, _ = sys.exc_info()
166 _, exc_value, _ = sys.exc_info()
151 return exc_value
167 return exc_value
152
168
153 def upload_multiple_files_advance(self, dataset_id, path_files, file_date, file_type, max_size=100, max_count=500, ignore_repetition=False, **kwargs):
169 def upload_multiple_files_advance(self, dataset_id, path_files, file_date, file_type, max_size=100, max_count=500, ignore_repetition=False, **kwargs):
154 # Agregar si es interruptido por teclado
170 # Agregar si es interruptido por teclado
155 '''
171 '''
156 FINALIDAD:
172 FINALIDAD:
157 Funcion para subir multiples archivos al repositorio del ROJ.
173 Funcion para subir multiples archivos al repositorio del ROJ.
158
174
159 PARAMETROS DISPONIBLES:
175 PARAMETROS DISPONIBLES:
160 CONSULTAR: "GUIA DE SCRIPT.pdf"
176 CONSULTAR: "GUIA DE SCRIPT.pdf"
161
177
162 ESTRUCTURA:
178 ESTRUCTURA:
163 <access_name>.upload_multiple_files_advance(dataset_id = <class 'str'>, path_files = <class 'list of strings'>, file_date = <class 'str'>, file_type = <class 'str'>, param_1 = <class 'param_1'>, ...)
179 <access_name>.upload_multiple_files_advance(dataset_id = <class 'str'>, path_files = <class 'list of strings'>, file_date = <class 'str'>, file_type = <class 'str'>, param_1 = <class 'param_1'>, ...)
164 '''
180 '''
165 #-------------------------PACKAGE SHOW-----------------------#
181 #-------------------------PACKAGE SHOW-----------------------#
166 try:
182 try:
167 dataset_show = getattr(self.ckan.action, 'package_show')(id=dataset_id)['resources']
183 dataset_show = getattr(self.ckan.action, 'package_show')(id=dataset_id)['resources']
168 except:
184 except:
169 _, exc_value, _ = sys.exc_info()
185 _, exc_value, _ = sys.exc_info()
170 print('ERROR obtaining metadata dataset:: Use the "print" for more information')
186 print('ERROR obtaining metadata dataset:: Use the "print" for more information')
171 return exc_value
187 return exc_value
172 #------------------------------------------------------------#
188 #------------------------------------------------------------#
173 resources_name = []
189 resources_name = []
174 for u in dataset_show:
190 for u in dataset_show:
175 resources_name.append(u['name'].lower())
191 resources_name.append(u['name'].lower())
176 #------------------------------------------------------------#
192 #------------------------------------------------------------#
177 self.list = ['package_id', 'upload', 'voc_file_type', 'name']
193 self.list = ['package_id', 'upload', 'voc_file_type', 'name']
178 for key1, value1 in kwargs.items():
194 for key1, value1 in kwargs.items():
179 if not key1 in self.list:
195 if not key1 in self.list:
180 self.dict[key1] = value1
196 self.dict[key1] = value1
181 #------------------------------------------------------------#
197 #------------------------------------------------------------#
182 if not 'others' in kwargs:
198 if not 'others' in kwargs:
183 self.dict['others'] = ''
199 self.dict['others'] = ''
184 else:
200 else:
185 if isinstance(kwargs['others'], list):
201 if isinstance(kwargs['others'], list):
186 self.dict['others'] = json.dumps(kwargs['others'])
202 self.dict['others'] = json.dumps(kwargs['others'])
187 #------------------------------------------------------------#
203 #------------------------------------------------------------#
188 total_list = []
204 total_list = []
189 #---------------CASO : "path" or "path_list"-----------------#
205 #---------------CASO : "path" or "path_list"-----------------#
190 if type(path_files) is list:
206 if type(path_files) is list:
191 if len(path_files) != 0:
207 if len(path_files) != 0:
192 path_files.sort()
208 path_files.sort()
193 for u in path_files:
209 for u in path_files:
194 if os.path.isfile(u):
210 if os.path.isfile(u):
195 if os.path.basename(u).lower() in resources_name:
211 if os.path.basename(u).lower() in resources_name:
196 if not ignore_repetition:
212 if not ignore_repetition:
197 return 'ERROR:: "%s" file already exist in this dataset' % (os.path.basename(u))
213 return 'ERROR:: "%s" file already exist in this dataset' % (os.path.basename(u))
198 print('WARRING:: "'+ str(os.path.basename(u)) +'" file was ignored because already exist in this dataset')
214 print('WARRING:: "'+ str(os.path.basename(u)) +'" file was ignored because already exist in this dataset')
199 else:
215 else:
200 total_list.append({'name':os.path.basename(u), 'size': os.stat(u).st_size, 'upload':open(u, 'rb')})
216 total_list.append({'name':os.path.basename(u), 'size': os.stat(u).st_size, 'upload':open(u, 'rb')})
201 else:
217 else:
202 return 'File "%s" does not exist' % (u)
218 return 'File "%s" does not exist' % (u)
203 else:
219 else:
204 return 'ERROR:: "path_list is empty"'
220 return 'ERROR:: "path_list is empty"'
205
221
206 elif type(path_files) is str:
222 elif type(path_files) is str:
207 if os.path.isdir(path_files):
223 if os.path.isdir(path_files):
208 path_order = [f for f in os.listdir(path_files) if os.path.isfile(os.path.join(path_files, f))]
224 path_order = [f for f in os.listdir(path_files) if os.path.isfile(os.path.join(path_files, f))]
209 path_order.sort()
225 path_order.sort()
210 if path_order:
226 if path_order:
211 for name in path_order:
227 for name in path_order:
212 if name.lower() in resources_name:
228 if name.lower() in resources_name:
213 if not ignore_repetition:
229 if not ignore_repetition:
214 return 'ERROR:: "%s" file already exist in this dataset' % (name)
230 return 'ERROR:: "%s" file already exist in this dataset' % (name)
215 print('WARRING:: "'+ name +'" file was ignored because already exist in this dataset')
231 print('WARRING:: "'+ name +'" file was ignored because already exist in this dataset')
216 else:
232 else:
217 total_list.append({'name':name, 'size': os.stat(os.path.join(path_files, name)).st_size, 'upload':open(os.path.join(path_files, name), 'rb')})
233 total_list.append({'name':name, 'size': os.stat(os.path.join(path_files, name)).st_size, 'upload':open(os.path.join(path_files, name), 'rb')})
218 else:
234 else:
219 return "ERROR:: There aren't files in this directory"
235 return "ERROR:: There aren't files in this directory"
220 else:
236 else:
221 return 'ERROR:: Directory "%s" does not exist' % (path_files)
237 return 'ERROR:: Directory "%s" does not exist' % (path_files)
222 else:
238 else:
223 return 'ERROR:: "path_files" must be a str or list'
239 return 'ERROR:: "path_files" must be a str or list'
224 #------------------------------------------------------------#
240 #------------------------------------------------------------#
225 try:
241 try:
226 uuid.UUID(str(dataset_id), version=4)
242 uuid.UUID(str(dataset_id), version=4)
227 package_id_or_name = '"id": "' + str(dataset_id) + '"'
243 package_id_or_name = '"id": "' + str(dataset_id) + '"'
228 except ValueError:
244 except ValueError:
229 package_id_or_name = '"name": "' + str(dataset_id) + '"'
245 package_id_or_name = '"name": "' + str(dataset_id) + '"'
230 #------------------------------------------------------------#
246 #------------------------------------------------------------#
231 blocks = [[]]
247 blocks = [[]]
232 size_file = 0
248 size_file = 0
233 count_file = 0
249 count_file = 0
234 inter_num = 0
250 inter_num = 0
235 for value in total_list:
251 for value in total_list:
236 if value['size'] > 1024 * 1024 * float(max_size):
252 if value['size'] > 1024 * 1024 * float(max_size):
237 return 'ERROR:: The size of the "%s" file is %sMB aprox, please change "max_size" value' % (value['name'], str(round(value['size']/(1024 * 1024), 2)))
253 return 'ERROR:: The size of the "%s" file is %sMB aprox, please change "max_size" value' % (value['name'], str(round(value['size']/(1024 * 1024), 2)))
238 if not 1 <= int(max_count) <= 999:
254 if not 1 <= int(max_count) <= 999:
239 return 'ERROR:: The count of the number of files must be between 1 and 999, please change "max_count" value'
255 return 'ERROR:: The count of the number of files must be between 1 and 999, please change "max_count" value'
240
256
241 size_file = size_file + value['size']
257 size_file = size_file + value['size']
242 count_file = count_file + 1
258 count_file = count_file + 1
243 if size_file <= 1024 * 1024 * float(max_size) and count_file <= int(max_count):
259 if size_file <= 1024 * 1024 * float(max_size) and count_file <= int(max_count):
244 del value['size']
260 del value['size']
245 blocks[inter_num].append(value)
261 blocks[inter_num].append(value)
246 else:
262 else:
247 inter_num = inter_num + 1
263 inter_num = inter_num + 1
248 size_file = value['size']
264 size_file = value['size']
249 count_file = 1
265 count_file = 1
250 blocks.append([])
266 blocks.append([])
251 del value['size']
267 del value['size']
252 blocks[inter_num].append(value)
268 blocks[inter_num].append(value)
253 #------------------------------------------------------------#
269 #------------------------------------------------------------#
254 if len(blocks[0]) > 0:
270 if len(blocks[0]) > 0:
255 print('BLOCK(S) IN TOTAL:: {}'.format(len(blocks)))
271 print('BLOCK(S) IN TOTAL:: {}'.format(len(blocks)))
256 for count1, block in enumerate(blocks):
272 for count1, block in enumerate(blocks):
257 print('---- BLOCK N°{} ----'.format(count1 + 1))
273 print('---- BLOCK N°{} ----'.format(count1 + 1))
258 resource_extend = []
274 resource_extend = []
259 files_dict = {}
275 files_dict = {}
260 for count2, value2 in enumerate(block):
276 for count2, value2 in enumerate(block):
261 value2['file_date'] = file_date
277 value2['file_date'] = file_date
262 value2['voc_file_type'] = file_type
278 value2['voc_file_type'] = file_type
263 value2.update(self.dict)
279 value2.update(self.dict)
264
280
265 #if not 'format' in value2:
281 #if not 'format' in value2:
266 # format = ''.join(pathlib.Path(value2['name']).suffixes)
282 # format = ''.join(pathlib.Path(value2['name']).suffixes)
267 # if len(format) > 0:
283 # if len(format) > 0:
268 # value2['format'] = format.upper()[1:]
284 # value2['format'] = format.upper()[1:]
269
285
270 files_dict['update__resources__-'+ str(len(block)-count2) +'__upload'] = (value2['name'], value2['upload'])
286 files_dict['update__resources__-'+ str(len(block)-count2) +'__upload'] = (value2['name'], value2['upload'])
271 del value2['upload']
287 del value2['upload']
272 resource_extend.append(value2)
288 resource_extend.append(value2)
273
289
274 print('BLOCK N°{} :: "{}" file(s) found >> uploading'.format(count1 + 1, len(block)))
290 print('BLOCK N°{} :: "{}" file(s) found >> uploading'.format(count1 + 1, len(block)))
275 try:
291 try:
276 result = self.ckan.call_action(
292 result = self.ckan.call_action(
277 'package_revise',
293 'package_revise',
278 {'match': '{'+ str(package_id_or_name) +'}', 'update__resources__extend': json.dumps(resource_extend)},
294 {'match': '{'+ str(package_id_or_name) +'}', 'update__resources__extend': json.dumps(resource_extend)},
279 files=files_dict
295 files=files_dict
280 )
296 )
281 print('BLOCK N°{} :: Uploaded file(s) successfully'.format(count1 + 1))
297 print('BLOCK N°{} :: Uploaded file(s) successfully'.format(count1 + 1))
282 if len(blocks) == count1 + 1:
298 if len(blocks) == count1 + 1:
283 return result
299 return result
284 except:
300 except:
285 print('ERROR :: Use the "print" for more information')
301 print('ERROR :: Use the "print" for more information')
286 _, exc_value, _ = sys.exc_info()
302 _, exc_value, _ = sys.exc_info()
287 return exc_value
303 return exc_value
288 else:
304 else:
289 return "ERROR:: No file(s) found to upload"
305 return "ERROR:: No file(s) found to upload"
290
306
291 def upload_multiple_files(self, dataset_id, path_files, date_files, type_files, **kwargs):
307 def upload_multiple_files(self, dataset_id, path_files, date_files, type_files, ignore_repetition=False, **kwargs):
292 # Agregar si es interruptido por teclado
308 # Agregar si es interruptido por teclado
293 '''
309 '''
294 FINALIDAD:
310 FINALIDAD:
295 Funcion para subir multiples archivos al repositorio del ROJ.
311 Funcion para subir multiples archivos al repositorio del ROJ.
296
312
297 PARAMETROS DISPONIBLES:
313 PARAMETROS DISPONIBLES:
298 CONSULTAR: "GUIA DE SCRIPT.pdf"
314 CONSULTAR: "GUIA DE SCRIPT.pdf"
299
315
300 ESTRUCTURA:
316 ESTRUCTURA:
301 <access_name>.upload_multiple_files(dataset_id = <class 'str'>, path_files = <class 'str'> or <class 'list of strings'>, date_files = <class 'str'> or <class 'list of strings'>, type_files = <class 'str'> or <class 'list of strings'>, param_1 = <class 'param_1'>, ...)
317 <access_name>.upload_multiple_files(dataset_id = <class 'str'>, path_files = <class 'str'> or <class 'list of strings'>, date_files = <class 'str'> or <class 'list of strings'>, type_files = <class 'str'> or <class 'list of strings'>, param_1 = <class 'param_1'>, ...)
302 '''
318 '''
319 #-------------------------PACKAGE SHOW-----------------------#
320 try:
321 dataset_show = getattr(self.ckan.action, 'package_show')(id=dataset_id)['resources']
322 except:
323 _, exc_value, _ = sys.exc_info()
324 print('ERROR obtaining metadata dataset:: Use the "print" for more information')
325 return exc_value
326 #------------------------------------------------------------#
327 resources_name = []
328 for u in dataset_show:
329 resources_name.append(u['name'].lower())
330 #------------------------------------------------------------#
303
331
304 params_dict = {'upload':[], 'name':[]}
332 params_dict = {'upload':[], 'name':[]}
305 #if not 'format' in kwargs:
333 #if not 'format' in kwargs:
306 # params_dict.update({'format':[]})
334 # params_dict.update({'format':[]})
307 #---------------CASO : "path" or "path_list"-----------------#
335 #---------------CASO : "path" or "path_list"-----------------#
308 if type(path_files) is list:
336 if type(path_files) is list:
309 if len(path_files) != 0:
337 if len(path_files) != 0:
310 path_files.sort()
338 path_files.sort()
311 for u in path_files:
339 for u in path_files:
312 if os.path.isfile(u):
340 if os.path.isfile(u):
313 params_dict['upload'].append(open(u, 'rb'))
341 if os.path.basename(u).lower() in resources_name:
314 params_dict['name'].append(os.path.basename(u))
342 if not ignore_repetition:
315 #if not 'format' in kwargs:
343 return 'ERROR:: "%s" file already exist in this dataset' % (os.path.basename(u))
316 # format = ''.join(pathlib.Path(u).suffixes)
344 print('WARRING:: "'+ str(os.path.basename(u)) +'" file was ignored because already exist in this dataset')
317 # if len(format) > 0:
345 else:
318 # params_dict['format'].append(format.upper()[1:])
346 params_dict['upload'].append(open(u, 'rb'))
319 # else:
347 params_dict['name'].append(os.path.basename(u))
320 # params_dict['format'].append('')
348 #if not 'format' in kwargs:
349 # format = ''.join(pathlib.Path(u).suffixes)
350 # if len(format) > 0:
351 # params_dict['format'].append(format.upper()[1:])
352 # else:
353 # params_dict['format'].append('')
321 else:
354 else:
322 return 'File "%s" does not exist' % (u)
355 return 'File "%s" does not exist' % (u)
323 else:
356 else:
324 return 'ERROR:: "path_list is empty"'
357 return 'ERROR:: "path_list is empty"'
325 elif type(path_files) is str:
358 elif type(path_files) is str:
326 if os.path.isdir(path_files):
359 if os.path.isdir(path_files):
327 path_order = [f for f in os.listdir(path_files) if os.path.isfile(os.path.join(path_files, f))]
360 path_order = [f for f in os.listdir(path_files) if os.path.isfile(os.path.join(path_files, f))]
328 path_order.sort()
361 path_order.sort()
329 if path_order:
362 if path_order:
330 for name in path_order:
363 for name in path_order:
331 params_dict['upload'].append(open(os.path.join(path_files, name), 'rb'))
364 if name.lower() in resources_name:
332 params_dict['name'].append(name)
365 if not ignore_repetition:
333 #if not 'format' in kwargs:
366 return 'ERROR:: "%s" file already exist in this dataset' % (name)
334 # format = ''.join(pathlib.Path(name).suffixes)
367 print('WARRING:: "'+ str(name) +'" file was ignored because already exist in this dataset')
335 # if len(format) > 0:
368 else:
336 # params_dict['format'].append(format.upper()[1:])
369 params_dict['upload'].append(open(os.path.join(path_files, name), 'rb'))
337 # else:
370 params_dict['name'].append(name)
338 # params_dict['format'].append('')
371 #if not 'format' in kwargs:
372 # format = ''.join(pathlib.Path(name).suffixes)
373 # if len(format) > 0:
374 # params_dict['format'].append(format.upper()[1:])
375 # else:
376 # params_dict['format'].append('')
339 else:
377 else:
340 return "ERROR:: There aren't files in this directory"
378 return "ERROR:: There aren't files in this directory"
341 else:
379 else:
342 return 'ERROR:: Directory "%s" does not exist' % (path_files)
380 return 'ERROR:: Directory "%s" does not exist' % (path_files)
343 else:
381 else:
344 return 'ERROR:: "path_files" must be a str or list'
382 return 'ERROR:: "path_files" must be a str or list'
345 #------------------------------------------------------------#
383 #------------------------------------------------------------#
346 params_no_dict = {'package_id': dataset_id}
384 params_no_dict = {'package_id': dataset_id}
347 if type(date_files) is list:
385 if type(date_files) is list:
348 params_dict['file_date'] = date_files
386 params_dict['file_date'] = date_files
349 else:
387 else:
350 params_no_dict['file_date'] = date_files
388 params_no_dict['file_date'] = date_files
351
389
352 if type(type_files) is list:
390 if type(type_files) is list:
353 params_dict['voc_file_type'] = type_files
391 params_dict['voc_file_type'] = type_files
354 else:
392 else:
355 params_no_dict['voc_file_type'] = type_files
393 params_no_dict['voc_file_type'] = type_files
356
394
357 for key1, value1 in kwargs.items():
395 for key1, value1 in kwargs.items():
358 if not key1 in params_dict and not key1 in params_no_dict and key1 != 'others':
396 if not key1 in params_dict and not key1 in params_no_dict and key1 != 'others':
359 if type(value1) is list:
397 if type(value1) is list:
360 params_dict[key1] = value1
398 params_dict[key1] = value1
361 else:
399 else:
362 params_no_dict[key1] = value1
400 params_no_dict[key1] = value1
363 #------------------------------------------#
401 #------------------------------------------#
364 if not 'others' in kwargs:
402 if not 'others' in kwargs:
365 params_no_dict['others'] = ''
403 params_no_dict['others'] = ''
366 else:
404 else:
367 if isinstance(kwargs['others'], tuple):
405 if isinstance(kwargs['others'], tuple):
368 params_dict['others'] = [json.dumps(w) for w in kwargs['others']]
406 params_dict['others'] = [json.dumps(w) for w in kwargs['others']]
369 elif isinstance(kwargs['others'], list):
407 elif isinstance(kwargs['others'], list):
370 params_no_dict['others'] = json.dumps(kwargs['others'])
408 params_no_dict['others'] = json.dumps(kwargs['others'])
371 elif isinstance(kwargs['others'], str):
409 elif isinstance(kwargs['others'], str):
372 params_no_dict['others'] = kwargs['others']
410 params_no_dict['others'] = kwargs['others']
373 else:
411 else:
374 return 'ERROR:: "others" must be a tuple, list or str'
412 return 'ERROR:: "others" must be a tuple, list or str'
375 #------------------------------------------#
413 #------------------------------------------#
376 len_params_dict = []
414 len_params_dict = []
377 for value2 in params_dict.values():
415 for value2 in params_dict.values():
378 len_params_dict.append(len(value2))
416 len_params_dict.append(len(value2))
379
417
380 if len(list(set(len_params_dict))) > 1:
418 if len(list(set(len_params_dict))) > 1:
381 return 'ERROR:: All lists must be the same length: %s' % (len(params_dict['name']))
419 return 'ERROR:: All lists must be the same length: %s' % (len(params_dict['name']))
382 #------------------------------------------------------------#
420 #------------------------------------------------------------#
383 print('"{}" file(s) found >> uploading'.format(len(params_dict['name'])))
421 print('"{}" file(s) found >> uploading'.format(len(params_dict['name'])))
384 for v in range(len(params_dict['name'])):
422 for v in range(len(params_dict['name'])):
385 try:
423 try:
386 send = {}
424 send = {}
387 for key_dict, value_dict in params_dict.items():
425 for key_dict, value_dict in params_dict.items():
388 send[key_dict] = value_dict[v]
426 send[key_dict] = value_dict[v]
389 for key_no_dict, value_no_dict in params_no_dict.items():
427 for key_no_dict, value_no_dict in params_no_dict.items():
390 send[key_no_dict] = value_no_dict
428 send[key_no_dict] = value_no_dict
391
429
392 self.list.append(getattr(self.ckan.action, 'resource_create')(**send))
430 self.list.append(getattr(self.ckan.action, 'resource_create')(**send))
393 print('File #{} :: "{}" was uploaded successfully'.format(v+1, params_dict['name'][v]))
431 print('File #{} :: "{}" was uploaded successfully'.format(v+1, params_dict['name'][v]))
394 except:
432 except:
395 _, exc_value, _ = sys.exc_info()
433 _, exc_value, _ = sys.exc_info()
396 self.list.append(exc_value)
434 self.list.append(exc_value)
397 print('File #{} :: Error uploading "{}" file'.format(v+1, params_dict['name'][v]))
435 print('File #{} :: Error uploading "{}" file'.format(v+1, params_dict['name'][v]))
398 return self.list
436 return self.list
399 #------------------------------------------------------------#
437 #------------------------------------------------------------#
400
438
401 def show(self, type_option, id, **kwargs):
439 def show(self, type_option, id, **kwargs):
402 '''
440 '''
403 FINALIDAD:
441 FINALIDAD:
404 Funcion personalizada para una busqueda en especifico.
442 Funcion personalizada para una busqueda en especifico.
405
443
406 PARAMETROS DISPONIBLES:
444 PARAMETROS DISPONIBLES:
407 CONSULTAR: "GUIA DE SCRIPT.pdf"
445 CONSULTAR: "GUIA DE SCRIPT.pdf"
408
446
409 ESTRUCTURA:
447 ESTRUCTURA:
410 <access_name>.show(type_option = <class 'str'>, id = <class 'str'>, param_1 = <class 'param_1'>, ...)
448 <access_name>.show(type_option = <class 'str'>, id = <class 'str'>, param_1 = <class 'param_1'>, ...)
411 '''
449 '''
412 if type(type_option) is str:
450 if type(type_option) is str:
413 try:
451 try:
414 if type_option == 'dataset':
452 if type_option == 'dataset':
415 return getattr(self.ckan.action, 'package_show')(id=id, **kwargs)
453 return getattr(self.ckan.action, 'package_show')(id=id, **kwargs)
416 elif type_option == 'resource':
454 elif type_option == 'resource':
417 return getattr(self.ckan.action, 'resource_show')(id=id, **kwargs)
455 return getattr(self.ckan.action, 'resource_show')(id=id, **kwargs)
418 elif type_option == 'project':
456 elif type_option == 'project':
419 return getattr(self.ckan.action, 'organization_show')(id=id, **kwargs)
457 return getattr(self.ckan.action, 'organization_show')(id=id, **kwargs)
420 elif type_option == 'collaborator':
458 elif type_option == 'collaborator':
421 return getattr(self.ckan.action, 'package_collaborator_list_for_user')(id=id, **kwargs)
459 return getattr(self.ckan.action, 'package_collaborator_list_for_user')(id=id, **kwargs)
422 elif type_option == 'member':
460 elif type_option == 'member':
423 return getattr(self.ckan.action, 'organization_list_for_user')(id=id, **kwargs)
461 return getattr(self.ckan.action, 'organization_list_for_user')(id=id, **kwargs)
424 elif type_option == 'vocabulary':
462 elif type_option == 'vocabulary':
425 return getattr(self.ckan.action, 'vocabulary_show')(id=id, **kwargs)
463 return getattr(self.ckan.action, 'vocabulary_show')(id=id, **kwargs)
426 elif type_option == 'tag':
464 elif type_option == 'tag':
427 if not 'vocabulary_id' in kwargs:
465 if not 'vocabulary_id' in kwargs:
428 print('Missing "vocabulary_id" value: assume it is a free tag')
466 print('Missing "vocabulary_id" value: assume it is a free tag')
429 return getattr(self.ckan.action, 'tag_show')(id=id, **kwargs)
467 return getattr(self.ckan.action, 'tag_show')(id=id, **kwargs)
430 elif type_option == 'user':
468 elif type_option == 'user':
431 return getattr(self.ckan.action, 'user_show')(id=id, **kwargs)
469 return getattr(self.ckan.action, 'user_show')(id=id, **kwargs)
432 elif type_option == 'job':
470 elif type_option == 'job':
433 return getattr(self.ckan.action, 'job_show')(id=id, **kwargs)
471 return getattr(self.ckan.action, 'job_show')(id=id, **kwargs)
434 else:
472 else:
435 return 'ERROR:: "type_option = %s" is not accepted' % (type_option)
473 return 'ERROR:: "type_option = %s" is not accepted' % (type_option)
436 except:
474 except:
437 _, exc_value, _ = sys.exc_info()
475 _, exc_value, _ = sys.exc_info()
438 return exc_value
476 return exc_value
439 else:
477 else:
440 return 'ERROR:: "type_option" must be a str'
478 return 'ERROR:: "type_option" must be a str'
441
479
442 def search(self, type_option, query=None, **kwargs):
480 def search(self, type_option, query=None, **kwargs):
443 '''
481 '''
444 FINALIDAD:
482 FINALIDAD:
445 Funcion personalizada para busquedas que satisfagan algun criterio.
483 Funcion personalizada para busquedas que satisfagan algun criterio.
446
484
447 PARAMETROS DISPONIBLES:
485 PARAMETROS DISPONIBLES:
448 CONSULTAR: "GUIA DE SCRIPT.pdf"
486 CONSULTAR: "GUIA DE SCRIPT.pdf"
449
487
450 ESTRUCTURA:
488 ESTRUCTURA:
451 <access_name>.search(type_option = <class 'str'>, query = <class 'dict'>, param_1 = <class 'param_1'>, ...)
489 <access_name>.search(type_option = <class 'str'>, query = <class 'dict'>, param_1 = <class 'param_1'>, ...)
452 '''
490 '''
453 if type(type_option) is str:
491 if type(type_option) is str:
454 try:
492 try:
455 if type_option == 'dataset':
493 if type_option == 'dataset':
456 key_replace = ['fq', 'fq_list', 'include_private']
494 key_replace = ['fq', 'fq_list', 'include_private']
457 key_point = ['facet_mincount', 'facet_limit', 'facet_field']
495 key_point = ['facet_mincount', 'facet_limit', 'facet_field']
458 for key1, value1 in kwargs.items():
496 for key1, value1 in kwargs.items():
459 if not key1 in key_replace:
497 if not key1 in key_replace:
460 if key1 in key_point:
498 if key1 in key_point:
461 self.dict[key1.replace('_', '.')] = value1
499 self.dict[key1.replace('_', '.')] = value1
462 else:
500 else:
463 self.dict[key1] = value1
501 self.dict[key1] = value1
464
502
465 if query is not None:
503 if query is not None:
466 if type(query) is dict:
504 if type(query) is dict:
467 self.dict['fq_list'] = []
505 self.dict['fq_list'] = []
468 #NUM_RESOURCES_MIN / NUM_RESOURCES_MAX
506 #NUM_RESOURCES_MIN / NUM_RESOURCES_MAX
469 #----------------------------------------------------#
507 #----------------------------------------------------#
470 if 'dataset_start_date' in query:
508 if 'dataset_start_date' in query:
471 if type(query['dataset_start_date']) is str:
509 if type(query['dataset_start_date']) is str:
472 try:
510 try:
473 datetime.strptime(query['dataset_start_date'], '%Y-%m-%d')
511 datetime.strptime(query['dataset_start_date'], '%Y-%m-%d')
474 if len(query['dataset_start_date']) != 10:
512 if len(query['dataset_start_date']) != 10:
475 return '"dataset_start_date", must be: <YYYY-MM-DD>'
513 return '"dataset_start_date", must be: <YYYY-MM-DD>'
476 self.dict['fq_list'].append('dataset_start_date:"'+query['dataset_start_date']+'"')
514 self.dict['fq_list'].append('dataset_start_date:"'+query['dataset_start_date']+'"')
477 self.list.append('dataset_start_date')
515 self.list.append('dataset_start_date')
478 except:
516 except:
479 return '"dataset_start_date" incorrect: "%s"' % (query['dataset_start_date'])
517 return '"dataset_start_date" incorrect: "%s"' % (query['dataset_start_date'])
480 else:
518 else:
481 return '"dataset_start_date" must be <str>'
519 return '"dataset_start_date" must be <str>'
482 #----------------------------------------------------#
520 #----------------------------------------------------#
483 if 'dataset_end_date' in query:
521 if 'dataset_end_date' in query:
484 if type(query['dataset_end_date']) is str:
522 if type(query['dataset_end_date']) is str:
485 try:
523 try:
486 datetime.strptime(query['dataset_end_date'], '%Y-%m-%d')
524 datetime.strptime(query['dataset_end_date'], '%Y-%m-%d')
487 if len(query['dataset_end_date']) != 10:
525 if len(query['dataset_end_date']) != 10:
488 return '"dataset_end_date", must be: <YYYY-MM-DD>'
526 return '"dataset_end_date", must be: <YYYY-MM-DD>'
489
527
490 if 'dataset_start_date' in query:
528 if 'dataset_start_date' in query:
491 if query['dataset_start_date'] > query['dataset_end_date']:
529 if query['dataset_start_date'] > query['dataset_end_date']:
492 return '"dataset_end_date" must be greater than "dataset_start_date"'
530 return '"dataset_end_date" must be greater than "dataset_start_date"'
493
531
494 self.dict['fq_list'].append('dataset_end_date:"'+query['dataset_end_date']+'"')
532 self.dict['fq_list'].append('dataset_end_date:"'+query['dataset_end_date']+'"')
495 self.list.append('dataset_end_date')
533 self.list.append('dataset_end_date')
496 except:
534 except:
497 return '"dataset_end_date" incorrect: "%s"' % (query['dataset_end_date'])
535 return '"dataset_end_date" incorrect: "%s"' % (query['dataset_end_date'])
498 else:
536 else:
499 return '"dataset_end_date" must be <str>'
537 return '"dataset_end_date" must be <str>'
500 #----------------------------------------------------#
538 #----------------------------------------------------#
501 for key, value in query.items():
539 for key, value in query.items():
502 if value is not None and not key in self.list:
540 if value is not None and not key in self.list:
503 self.dict['fq_list'].append(str(key)+':"'+str(value)+'"')
541 self.dict['fq_list'].append(str(key)+':"'+str(value)+'"')
504 else:
542 else:
505 return '"query" must be <dict>'
543 return '"query" must be <dict>'
506
544
507 return getattr(self.ckan.action, 'package_search')(include_private=True, **self.dict)
545 return getattr(self.ckan.action, 'package_search')(include_private=True, **self.dict)
508
546
509 elif type_option == 'resource':
547 elif type_option == 'resource':
510 for key1, value1 in kwargs.items():
548 for key1, value1 in kwargs.items():
511 if key1 != 'fields':
549 if key1 != 'fields':
512 self.dict[key1] = value1
550 self.dict[key1] = value1
513
551
514 if query is not None:
552 if query is not None:
515 if type(query) is dict:
553 if type(query) is dict:
516 #----------------------------------------------------#
554 #----------------------------------------------------#
517 if 'file_date_min' in query:
555 if 'file_date_min' in query:
518 if type(query['file_date_min']) is str:
556 if type(query['file_date_min']) is str:
519 try:
557 try:
520 datetime.strptime(query['file_date_min'], '%Y-%m-%d')
558 datetime.strptime(query['file_date_min'], '%Y-%m-%d')
521 if len(query['file_date_min']) != 10:
559 if len(query['file_date_min']) != 10:
522 return '"file_date_min", must be: <YYYY-MM-DD>'
560 return '"file_date_min", must be: <YYYY-MM-DD>'
523 except:
561 except:
524 return '"file_date_min" incorrect: "%s"' % (query['file_date_min'])
562 return '"file_date_min" incorrect: "%s"' % (query['file_date_min'])
525 else:
563 else:
526 return '"file_date_min" must be <str>'
564 return '"file_date_min" must be <str>'
527 #----------------------------------------------------#
565 #----------------------------------------------------#
528 if 'file_date_max' in query:
566 if 'file_date_max' in query:
529 if type(query['file_date_max']) is str:
567 if type(query['file_date_max']) is str:
530 try:
568 try:
531 datetime.strptime(query['file_date_max'], '%Y-%m-%d')
569 datetime.strptime(query['file_date_max'], '%Y-%m-%d')
532 if len(query['file_date_max']) != 10:
570 if len(query['file_date_max']) != 10:
533 return '"file_date_max", must be: <YYYY-MM-DD>'
571 return '"file_date_max", must be: <YYYY-MM-DD>'
534
572
535 if 'file_date_min' in query:
573 if 'file_date_min' in query:
536 if query['file_date_min'] > query['file_date_max']:
574 if query['file_date_min'] > query['file_date_max']:
537 return '"file_date_max" must be greater than "file_date_min"'
575 return '"file_date_max" must be greater than "file_date_min"'
538 except:
576 except:
539 return '"file_date_max" incorrect: "%s"' % (query['file_date_max'])
577 return '"file_date_max" incorrect: "%s"' % (query['file_date_max'])
540 else:
578 else:
541 return '"file_date_max" must be <str>'
579 return '"file_date_max" must be <str>'
542 #----------------------------------------------------#
580 #----------------------------------------------------#
543 self.dict['query'] = query
581 self.dict['query'] = query
544 else:
582 else:
545 return '"query" must be <dict>'
583 return '"query" must be <dict>'
546 return getattr(self.ckan.action, 'resources_search')(**self.dict)
584 return getattr(self.ckan.action, 'resources_search')(**self.dict)
547
585
548 elif type_option == 'tag':
586 elif type_option == 'tag':
549 for key1, value1 in kwargs.items():
587 for key1, value1 in kwargs.items():
550 if key1 != 'fields':
588 if key1 != 'fields':
551 self.dict[key1] = value1
589 self.dict[key1] = value1
552
590
553 if not 'vocabulary_id' in kwargs:
591 if not 'vocabulary_id' in kwargs:
554 print('Missing "vocabulary_id" value: tags that don’t belong to any vocabulary')
592 print('Missing "vocabulary_id" value: tags that don’t belong to any vocabulary')
555 else:
593 else:
556 print('Only tags that belong to "{}" vocabulary'.format(kwargs['vocabulary_id']))
594 print('Only tags that belong to "{}" vocabulary'.format(kwargs['vocabulary_id']))
557
595
558 if query is not None:
596 if query is not None:
559 if type(query) is dict:
597 if type(query) is dict:
560 if 'search' in query:
598 if 'search' in query:
561 if type(query['search']) is list or type(query['search']) is str:
599 if type(query['search']) is list or type(query['search']) is str:
562 self.dict['query'] = query['search']
600 self.dict['query'] = query['search']
563 else:
601 else:
564 return '"search" must be <list> or <str>'
602 return '"search" must be <list> or <str>'
565 else:
603 else:
566 return '"query" must be <dict>'
604 return '"query" must be <dict>'
567 return getattr(self.ckan.action, 'tag_search')(**self.dict)
605 return getattr(self.ckan.action, 'tag_search')(**self.dict)
568
606
569 else:
607 else:
570 return 'ERROR:: "type_option = %s" is not accepted' % (type_option)
608 return 'ERROR:: "type_option = %s" is not accepted' % (type_option)
571
609
572 except:
610 except:
573 _, exc_value, _ = sys.exc_info()
611 _, exc_value, _ = sys.exc_info()
574 return exc_value
612 return exc_value
575 else:
613 else:
576 return 'ERROR:: "type_option" must be <str>'
614 return 'ERROR:: "type_option" must be <str>'
577
615
578 def create(self, type_option, select=None, **kwargs):
616 def create(self, type_option, select=None, **kwargs):
579 '''
617 '''
580 FINALIDAD:
618 FINALIDAD:
581 Funcion personalizada para crear.
619 Funcion personalizada para crear.
582
620
583 PARAMETROS DISPONIBLES:
621 PARAMETROS DISPONIBLES:
584 CONSULTAR: "GUIA DE SCRIPT.pdf"
622 CONSULTAR: "GUIA DE SCRIPT.pdf"
585
623
586 ESTRUCTURA:
624 ESTRUCTURA:
587 <access_name>.create(type_option = <class 'str'>, param_1 = <class 'param_1'>, ...)
625 <access_name>.create(type_option = <class 'str'>, param_1 = <class 'param_1'>, ...)
588 '''
626 '''
589 if type(type_option) is str:
627 if type(type_option) is str:
590 try:
628 try:
591 if type_option == 'dataset':
629 if type_option == 'dataset':
592 return getattr(self.ckan.action, 'package_create')(**kwargs)
630 return getattr(self.ckan.action, 'package_create')(**kwargs)
593 elif type_option == 'project':
631 elif type_option == 'project':
594 return getattr(self.ckan.action, 'organization_create')(**kwargs)
632 return getattr(self.ckan.action, 'organization_create')(**kwargs)
595 elif type_option == 'member':
633 elif type_option == 'member':
596 return getattr(self.ckan.action, 'organization_member_create')(**kwargs)
634 return getattr(self.ckan.action, 'organization_member_create')(**kwargs)
597 elif type_option == 'collaborator':
635 elif type_option == 'collaborator':
598 return getattr(self.ckan.action, 'package_collaborator_create')(**kwargs)
636 return getattr(self.ckan.action, 'package_collaborator_create')(**kwargs)
599 elif type_option == 'vocabulary':
637 elif type_option == 'vocabulary':
600 return getattr(self.ckan.action, 'vocabulary_create')(**kwargs)
638 return getattr(self.ckan.action, 'vocabulary_create')(**kwargs)
601 elif type_option == 'tag':
639 elif type_option == 'tag':
602 return getattr(self.ckan.action, 'tag_create')(**kwargs)
640 return getattr(self.ckan.action, 'tag_create')(**kwargs)
603 elif type_option == 'user':
641 elif type_option == 'user':
604 return getattr(self.ckan.action, 'user_create')(**kwargs)
642 return getattr(self.ckan.action, 'user_create')(**kwargs)
605 elif type_option == 'views':
643 elif type_option == 'views':
606 if 'resource' == select:
644 if 'resource' == select:
607 self.list = ['package']
645 self.list = ['package']
608 for key1, value1 in kwargs.items():
646 for key1, value1 in kwargs.items():
609 if not key1 in self.list:
647 if not key1 in self.list:
610 self.dict[key1] = value1
648 self.dict[key1] = value1
611 return getattr(self.ckan.action, 'resource_create_default_resource_views')(**self.dict)
649 return getattr(self.ckan.action, 'resource_create_default_resource_views')(**self.dict)
612 elif 'dataset' == select:
650 elif 'dataset' == select:
613 return getattr(self.ckan.action, 'package_create_default_resource_views')(**kwargs)
651 return getattr(self.ckan.action, 'package_create_default_resource_views')(**kwargs)
614 else:
652 else:
615 return 'ERROR:: "select = %s" is not accepted' % (select)
653 return 'ERROR:: "select = %s" is not accepted' % (select)
616 else:
654 else:
617 return 'ERROR:: "type_option = %s" is not accepted' % (type_option)
655 return 'ERROR:: "type_option = %s" is not accepted' % (type_option)
618 except:
656 except:
619 _, exc_value, _ = sys.exc_info()
657 _, exc_value, _ = sys.exc_info()
620 return exc_value
658 return exc_value
621 else:
659 else:
622 return 'ERROR:: "type_option" must be <str>'
660 return 'ERROR:: "type_option" must be <str>'
623
661
624 def patch(self, type_option, **kwargs):
662 def patch(self, type_option, **kwargs):
625 '''
663 '''
626 FINALIDAD:
664 FINALIDAD:
627 Funciones personalizadas para actualizar
665 Funciones personalizadas para actualizar
628
666
629 PARAMETROS DISPONIBLES:
667 PARAMETROS DISPONIBLES:
630 CONSULTAR: "GUIA DE SCRIPT.pdf"
668 CONSULTAR: "GUIA DE SCRIPT.pdf"
631
669
632 ESTRUCTURA:
670 ESTRUCTURA:
633 <access_name>.patch(type_option = <class 'str'>, param_1 = <class 'param_1'>, ...)
671 <access_name>.patch(type_option = <class 'str'>, param_1 = <class 'param_1'>, ...)
634 '''
672 '''
635 if type(type_option) is str:
673 if type(type_option) is str:
636 try:
674 try:
637 if type_option == 'dataset':
675 if type_option == 'dataset':
638 return getattr(self.ckan.action, 'package_patch')(**kwargs)
676 return getattr(self.ckan.action, 'package_patch')(**kwargs)
639 elif type_option == 'project':
677 elif type_option == 'project':
640 return getattr(self.ckan.action, 'organization_patch')(**kwargs)
678 return getattr(self.ckan.action, 'organization_patch')(**kwargs)
641 elif type_option == 'resource':
679 elif type_option == 'resource':
642 return getattr(self.ckan.action, 'resource_patch')(**kwargs)
680 return getattr(self.ckan.action, 'resource_patch')(**kwargs)
643 elif type_option == 'member':
681 elif type_option == 'member':
644 return getattr(self.ckan.action, 'organization_member_create')(**kwargs)
682 return getattr(self.ckan.action, 'organization_member_create')(**kwargs)
645 elif type_option == 'collaborator':
683 elif type_option == 'collaborator':
646 return getattr(self.ckan.action, 'package_collaborator_create')(**kwargs)
684 return getattr(self.ckan.action, 'package_collaborator_create')(**kwargs)
647 else:
685 else:
648 return 'ERROR:: "type_option = %s" is not accepted' % (type_option)
686 return 'ERROR:: "type_option = %s" is not accepted' % (type_option)
649 except:
687 except:
650 _, exc_value, _ = sys.exc_info()
688 _, exc_value, _ = sys.exc_info()
651 return exc_value
689 return exc_value
652 else:
690 else:
653 return 'ERROR:: "type_option" must be <str>'
691 return 'ERROR:: "type_option" must be <str>'
654
692
655 def delete(self, type_option, select=None, **kwargs):
693 def delete(self, type_option, select=None, **kwargs):
656 '''
694 '''
657 FINALIDAD:
695 FINALIDAD:
658 Función personalizada para eliminar y/o purgar.
696 Función personalizada para eliminar y/o purgar.
659
697
660 PARAMETROS DISPONIBLES:
698 PARAMETROS DISPONIBLES:
661 CONSULTAR: "GUIA DE SCRIPT.pdf"
699 CONSULTAR: "GUIA DE SCRIPT.pdf"
662
700
663 ESTRUCTURA:
701 ESTRUCTURA:
664 <access_name>.delete(type_option = <class 'str'>, param_1 = <class 'param_1'>, ...)
702 <access_name>.delete(type_option = <class 'str'>, param_1 = <class 'param_1'>, ...)
665 '''
703 '''
666 if type(type_option) is str:
704 if type(type_option) is str:
667 try:
705 try:
668 if type_option == 'dataset':
706 if type_option == 'dataset':
669 if select is None:
707 if select is None:
670 return 'ERROR:: "select" must not be "None"'
708 return 'ERROR:: "select" must not be "None"'
671 else:
709 else:
672 if 'delete' == select:
710 if 'delete' == select:
673 return getattr(self.ckan.action, 'package_delete')(**kwargs)
711 return getattr(self.ckan.action, 'package_delete')(**kwargs)
674 elif 'purge' == select:
712 elif 'purge' == select:
675 return getattr(self.ckan.action, 'dataset_purge')(**kwargs)
713 return getattr(self.ckan.action, 'dataset_purge')(**kwargs)
676 else:
714 else:
677 return 'ERROR:: "select = %s" is not accepted' % (select)
715 return 'ERROR:: "select = %s" is not accepted' % (select)
678 elif type_option == 'project':
716 elif type_option == 'project':
679 if select is None:
717 if select is None:
680 return 'ERROR:: "select" must not be "None"'
718 return 'ERROR:: "select" must not be "None"'
681 else:
719 else:
682 if 'delete' == select:
720 if 'delete' == select:
683 return getattr(self.ckan.action, 'organization_delete')(**kwargs)
721 return getattr(self.ckan.action, 'organization_delete')(**kwargs)
684 elif 'purge' == select:
722 elif 'purge' == select:
685 return getattr(self.ckan.action, 'organization_purge')(**kwargs)
723 return getattr(self.ckan.action, 'organization_purge')(**kwargs)
686 else:
724 else:
687 return 'ERROR:: "select = %s" is not accepted' % (select)
725 return 'ERROR:: "select = %s" is not accepted' % (select)
688 elif type_option == 'resource':
726 elif type_option == 'resource':
689 return getattr(self.ckan.action, 'resource_delete')(**kwargs)
727 return getattr(self.ckan.action, 'resource_delete')(**kwargs)
690 elif type_option == 'vocabulary':
728 elif type_option == 'vocabulary':
691 return getattr(self.ckan.action, 'vocabulary_delete')(**kwargs)
729 return getattr(self.ckan.action, 'vocabulary_delete')(**kwargs)
692 elif type_option == 'tag':
730 elif type_option == 'tag':
693 return getattr(self.ckan.action, 'tag_delete')(**kwargs)
731 return getattr(self.ckan.action, 'tag_delete')(**kwargs)
694 elif type_option == 'user':
732 elif type_option == 'user':
695 return getattr(self.ckan.action, 'user_delete')(**kwargs)
733 return getattr(self.ckan.action, 'user_delete')(**kwargs)
696 else:
734 else:
697 return 'ERROR:: "type_option = %s" is not accepted' % (type_option)
735 return 'ERROR:: "type_option = %s" is not accepted' % (type_option)
698 except:
736 except:
699 _, exc_value, _ = sys.exc_info()
737 _, exc_value, _ = sys.exc_info()
700 return exc_value
738 return exc_value
701 else:
739 else:
702 return 'ERROR:: "type_option" must be <str>'
740 return 'ERROR:: "type_option" must be <str>'
703
741
704 def f_status_note(self, total, result, path):
742 def f_status_note(self, total, result, path):
705 file_txt = open(path+'status_note.txt', 'w')
743 file_txt = open(path+'status_note.txt', 'w')
706 file_txt = open(path+'status_note.txt', 'a')
744 file_txt = open(path+'status_note.txt', 'a')
707
745
708 file_txt.write('DOWNLOADED FILE(S): "%s"' % (len(result['name'])))
746 file_txt.write('DOWNLOADED FILE(S): "%s"' % (len(result['name'])))
709 file_txt.write(''+ os.linesep)
747 file_txt.write(''+ os.linesep)
710 for u in result['name']:
748 for u in result['name']:
711 file_txt.write(' - '+ u + os.linesep)
749 file_txt.write(' - '+ u + os.linesep)
712 file_txt.write(''+ os.linesep)
750 file_txt.write(''+ os.linesep)
713
751
714 file_txt.write('FAILED FILE(S): "%s"' % (len(total['name'])-len(result['name'])))
752 file_txt.write('FAILED FILE(S): "%s"' % (len(total['name'])-len(result['name'])))
715 file_txt.write(''+ os.linesep)
753 file_txt.write(''+ os.linesep)
716 if len(total['name'])-len(result['name']) != 0:
754 if len(total['name'])-len(result['name']) != 0:
717 for u in total['name']:
755 for u in total['name']:
718 if not u in result['name']:
756 if not u in result['name']:
719 file_txt.write(' - '+ u + os.linesep)
757 file_txt.write(' - '+ u + os.linesep)
720 else:
758 else:
721 file_txt.write(' "None"'+ os.linesep)
759 file_txt.write(' "None"'+ os.linesep)
722
760
723 def f_name(self, name_dataset, ext, tempdir):
761 def f_name(self, name_dataset, ext, tempdir):
724 while self.check:
762 while self.check:
725 self.str = ''
763 self.str = ''
726 if self.cont == 0:
764 if self.cont == 0:
727 if os.path.exists(tempdir + name_dataset + ext):
765 if os.path.exists(tempdir + name_dataset + ext):
728 self.str = name_dataset+'('+str(self.cont+1)+')'+ext
766 self.str = name_dataset+'('+str(self.cont+1)+')'+ext
729 else:
767 else:
730 self.check = self.check * 0
768 self.check = self.check * 0
731 self.str = name_dataset + ext
769 self.str = name_dataset + ext
732 else:
770 else:
733 if not os.path.exists(tempdir + name_dataset+'('+str(self.cont)+')'+ext):
771 if not os.path.exists(tempdir + name_dataset+'('+str(self.cont)+')'+ext):
734 self.check = self.check * 0
772 self.check = self.check * 0
735 self.str = name_dataset+'('+str(self.cont)+')'+ ext
773 self.str = name_dataset+'('+str(self.cont)+')'+ ext
736 self.cont = self.cont+1
774 self.cont = self.cont+1
737 return self.str
775 return self.str
738
776
739 def f_zipdir(self, path, ziph, zip_name):
777 def f_zipdir(self, path, ziph, zip_name):
740 for root, _, files in os.walk(path):
778 for root, _, files in os.walk(path):
741 print('.....')
779 print('.....')
742 print('Creating: "{}" >>'.format(zip_name))
780 print('Creating: "{}" >>'.format(zip_name))
743 for __file in tqdm(iterable=files, total=len(files)):
781 for __file in tqdm(iterable=files, total=len(files)):
744 new_dir = os.path.relpath(os.path.join(root, __file), os.path.join(path, '..'))
782 new_dir = os.path.relpath(os.path.join(root, __file), os.path.join(path, '..'))
745 ziph.write(os.path.join(root, __file), new_dir)
783 ziph.write(os.path.join(root, __file), new_dir)
746 print('Created >>')
784 print('Created >>')
747
785
748 def download_by_step(self, response, tempdir_name):
786 def download_by_step(self, response, tempdir_name):
749 try:
787 try:
750 with requests.get(response['url'], stream=True, headers={'Authorization': self.Authorization}) as resp:
788 with requests.get(response['url'], stream=True, headers={'Authorization': self.Authorization}) as resp:
751 if resp.status_code == 200:
789 if resp.status_code == 200:
752 with open(tempdir_name+response['name'], 'wb') as file:
790 with open(tempdir_name+response['name'], 'wb') as file:
753 for chunk in resp.iter_content(chunk_size = self.chunk_size):
791 for chunk in resp.iter_content(chunk_size = self.chunk_size):
754 if chunk:
792 if chunk:
755 file.write(chunk)
793 file.write(chunk)
756 except requests.exceptions.RequestException:
794 except requests.exceptions.RequestException:
757 pass
795 pass
758
796
759 def download_files(self, **kwargs):
797 def download_files(self, **kwargs):
760 '''
798 '''
761 FINALIDAD:
799 FINALIDAD:
762 Funcion personalizada para la descarga de archivos existentes de un dataset.
800 Funcion personalizada para la descarga de archivos existentes de un dataset.
763
801
764 PARAMETROS DISPONIBLES:
802 PARAMETROS DISPONIBLES:
765 CONSULTAR: "GUIA DE SCRIPT.pdf"
803 CONSULTAR: "GUIA DE SCRIPT.pdf"
766
804
767 ESTRUCTURA:
805 ESTRUCTURA:
768 <access_name>.download_files(id = <class 'str'>, param_1 = <class 'param_1'>, ...)
806 <access_name>.download_files(id = <class 'str'>, param_1 = <class 'param_1'>, ...)
769 '''
807 '''
770 dict_local = {}
808 dict_local = {}
771 #----------------------------------------------#
809 #----------------------------------------------#
772 if 'zip' in kwargs:
810 if 'zip' in kwargs:
773 if type(kwargs['zip']) is not bool:
811 if type(kwargs['zip']) is not bool:
774 return 'ERROR:: "zip" must be: <class "bool">'
812 return 'ERROR:: "zip" must be: <class "bool">'
775 else:
813 else:
776 dict_local['zip'] = kwargs['zip']
814 dict_local['zip'] = kwargs['zip']
777 else:
815 else:
778 dict_local['zip'] = False
816 dict_local['zip'] = False
779 #----------------------------------------------#
817 #----------------------------------------------#
780 if 'status_note' in kwargs:
818 if 'status_note' in kwargs:
781 if type(kwargs['status_note']) is not bool:
819 if type(kwargs['status_note']) is not bool:
782 return 'ERROR:: "status_note" must be: <class "bool">'
820 return 'ERROR:: "status_note" must be: <class "bool">'
783 else:
821 else:
784 dict_local['status_note'] = kwargs['status_note']
822 dict_local['status_note'] = kwargs['status_note']
785 else:
823 else:
786 dict_local['status_note'] = False
824 dict_local['status_note'] = False
787 #----------------------------------------------#
825 #----------------------------------------------#
788 if 'path' in kwargs:
826 if 'path' in kwargs:
789 if type(kwargs['path']) is str:
827 if type(kwargs['path']) is str:
790 if os.path.isdir(kwargs['path']) == False:
828 if os.path.isdir(kwargs['path']) == False:
791 return 'ERROR:: "path" does not exist'
829 return 'ERROR:: "path" does not exist'
792 else:
830 else:
793 if kwargs['path'][-1:] != self.separator:
831 if kwargs['path'][-1:] != self.separator:
794 dict_local['path'] = kwargs['path']+self.separator
832 dict_local['path'] = kwargs['path']+self.separator
795 else:
833 else:
796 dict_local['path'] = kwargs['path']
834 dict_local['path'] = kwargs['path']
797
835
798 txt = dict_local['path']+datetime.now().strftime("%Y_%m_%d_%H_%M_%S_%f")+'.txt'
836 txt = dict_local['path']+datetime.now().strftime("%Y_%m_%d_%H_%M_%S_%f")+'.txt'
799 if int(platform.python_version()[0]) == 3:
837 if int(platform.python_version()[0]) == 3:
800 try:
838 try:
801 file_txt = open(txt, 'w')
839 file_txt = open(txt, 'w')
802 file_txt.close()
840 file_txt.close()
803 os.remove(txt)
841 os.remove(txt)
804 except PermissionError:
842 except PermissionError:
805 return 'ERROR:: Access denied, you are not authorized to write files: "%s"' % (dict_local['path'])
843 return 'ERROR:: Access denied, you are not authorized to write files: "%s"' % (dict_local['path'])
806 else:
844 else:
807 try:
845 try:
808 file_txt = open(txt, 'w')
846 file_txt = open(txt, 'w')
809 file_txt.close()
847 file_txt.close()
810 os.remove(txt)
848 os.remove(txt)
811 except:
849 except:
812 return 'ERROR:: Access denied, you are not authorized to write files: "%s"' % (dict_local['path'])
850 return 'ERROR:: Access denied, you are not authorized to write files: "%s"' % (dict_local['path'])
813 else:
851 else:
814 return 'ERROR:: "path" must be: <class "str">'
852 return 'ERROR:: "path" must be: <class "str">'
815 else:
853 else:
816 dict_local['path'] = ''
854 dict_local['path'] = ''
817 #----------------------------------------------#
855 #----------------------------------------------#
818 for key, value in kwargs.items():
856 for key, value in kwargs.items():
819 if not key in dict_local:
857 if not key in dict_local:
820 self.dict[key] = value
858 self.dict[key] = value
821 try:
859 try:
822 response = getattr(self.ckan.action, 'url_resources')(**self.dict)
860 response = getattr(self.ckan.action, 'url_resources')(**self.dict)
823 except:
861 except:
824 _, exc_value, _ = sys.exc_info()
862 _, exc_value, _ = sys.exc_info()
825 return exc_value
863 return exc_value
826
864
827 if len(response) != 0:
865 if len(response) != 0:
828 #--------------TEMP PATH---------------#
866 #--------------TEMP PATH---------------#
829 if dict_local['zip']:
867 if dict_local['zip']:
830 tempdir = tempfile.mkdtemp(prefix=kwargs['id']+'-')+self.separator
868 tempdir = tempfile.mkdtemp(prefix=kwargs['id']+'-')+self.separator
831 os.mkdir(tempdir+kwargs['id'])
869 os.mkdir(tempdir+kwargs['id'])
832 dir_name = tempdir + kwargs['id'] + self.separator
870 dir_name = tempdir + kwargs['id'] + self.separator
833 else:
871 else:
834 dir = self.f_name(kwargs['id'], '', dict_local['path'])
872 dir = self.f_name(kwargs['id'], '', dict_local['path'])
835 os.mkdir(dict_local['path'] + dir)
873 os.mkdir(dict_local['path'] + dir)
836 dir_name = dict_local['path'] + dir + self.separator
874 dir_name = dict_local['path'] + dir + self.separator
837 #-----------DOWNLOAD FILES-------------#
875 #-----------DOWNLOAD FILES-------------#
838 print('.....')
876 print('.....')
839 print('Downloading "{}" file(s) >>'.format(len(response)))
877 print('Downloading "{}" file(s) >>'.format(len(response)))
840 name_total = {'name': []}
878 name_total = {'name': []}
841 with concurrent.futures.ThreadPoolExecutor() as executor:
879 with concurrent.futures.ThreadPoolExecutor() as executor:
842 for u in tqdm(iterable=response, total=len(response)):
880 for u in tqdm(iterable=response, total=len(response)):
843 name_total['name'].append(u['name'])
881 name_total['name'].append(u['name'])
844 executor.submit(self.download_by_step, u, dir_name)
882 executor.submit(self.download_by_step, u, dir_name)
845 name_check = {}
883 name_check = {}
846 name_check['name'] = [f for f in os.listdir(dir_name) if os.path.isfile(os.path.join(dir_name, f))]
884 name_check['name'] = [f for f in os.listdir(dir_name) if os.path.isfile(os.path.join(dir_name, f))]
847 print('"{}" downloaded file(s) successfully >>'.format(len(name_check['name'])))
885 print('"{}" downloaded file(s) successfully >>'.format(len(name_check['name'])))
848 #--------------------------------------#
886 #--------------------------------------#
849 if len(name_check['name']) != 0:
887 if len(name_check['name']) != 0:
850 #----------Status Note---------#
888 #----------Status Note---------#
851 if dict_local['status_note']:
889 if dict_local['status_note']:
852 print('.....')
890 print('.....')
853 print('Creating: "status_note.txt" >>')
891 print('Creating: "status_note.txt" >>')
854 self.f_status_note(name_total, name_check, dir_name)
892 self.f_status_note(name_total, name_check, dir_name)
855 print('Created>>')
893 print('Created>>')
856 #----------ZIP CREATE----------#
894 #----------ZIP CREATE----------#
857 if dict_local['zip']:
895 if dict_local['zip']:
858 zip_name = self.f_name(kwargs['id'], '.zip', dict_local['path'])
896 zip_name = self.f_name(kwargs['id'], '.zip', dict_local['path'])
859 ziph = zipfile.ZipFile(dict_local['path'] + zip_name, 'w', zipfile.ZIP_DEFLATED, allowZip64=True)
897 ziph = zipfile.ZipFile(dict_local['path'] + zip_name, 'w', zipfile.ZIP_DEFLATED, allowZip64=True)
860 self.f_zipdir(dir_name, ziph, zip_name)
898 self.f_zipdir(dir_name, ziph, zip_name)
861 ziph.close()
899 ziph.close()
862 #Delete Temporal Path
900 #Delete Temporal Path
863 if os.path.exists(tempdir[:-1]):
901 if os.path.exists(tempdir[:-1]):
864 shutil.rmtree(tempdir[:-1])
902 shutil.rmtree(tempdir[:-1])
865 #------------------------------#
903 #------------------------------#
866 print('.....')
904 print('.....')
867 return 'DOWNLOAD FINISHED'
905 return 'DOWNLOAD FINISHED'
868 else:
906 else:
869 #Delete Temporal Path
907 #Delete Temporal Path
870 if dict_local['zip']:
908 if dict_local['zip']:
871 if os.path.exists(tempdir[:-1]):
909 if os.path.exists(tempdir[:-1]):
872 shutil.rmtree(tempdir[:-1])
910 shutil.rmtree(tempdir[:-1])
873 else:
911 else:
874 if os.path.exists(dir_name[:-1]):
912 if os.path.exists(dir_name[:-1]):
875 shutil.rmtree(dir_name[:-1])
913 shutil.rmtree(dir_name[:-1])
876 return 'NO FILES WERE DOWNLOADED'
914 return 'NO FILES WERE DOWNLOADED'
877 else:
915 else:
878 return 'FILES NOT FOUND' No newline at end of file
916 return 'FILES NOT FOUND'
General Comments 0
You need to be logged in to leave comments. Login now