##// END OF EJS Templates
DATABASES » ckanext-jro » api-cliente
RSS
v2.9.2 :: Add 'max_count' option in upload_multiple_files_advance function
eynilupu -
r6:9f9e218fed1d
parent child
Show More
Show file before | Show file after | Hide comments
1 NO CONTENT: modified file, binary diff hidden
NO CONTENT: modified file, binary diff hidden
Show file before | Show file after | Hide comments
@@ -1,871 +1,878
1 from ckanapi import RemoteCKAN
1 from ckanapi import RemoteCKAN
2 from datetime import datetime
2 from datetime import datetime
3 from tqdm import tqdm
3 from tqdm import tqdm
4 #from ckanapi.errors import NotAuthorized, NotFound, ValidationError, SearchQueryError, SearchError, CKANAPIError, ServerIncompatibleError
4 #from ckanapi.errors import NotAuthorized, NotFound, ValidationError, SearchQueryError, SearchError, CKANAPIError, ServerIncompatibleError
5 import sys
5 import sys
6 import platform
6 import platform
7 import os
7 import os
8 import tempfile
8 import tempfile
9 import shutil
9 import shutil
10 import zipfile
10 import zipfile
11 import concurrent.futures
11 import concurrent.futures
12 import requests
12 import requests
13 import json
13 import json
14 import pathlib
14 import pathlib
15 import uuid
15 import uuid
16
16
17 class JROAPI():
17 class JROAPI():
18 """
18 """
19 FINALIDAD:
19 FINALIDAD:
20 Script para administrar y obtener la data del repositorio por medio de APIs.
20 Script para administrar y obtener la data del repositorio por medio de APIs.
21
21
22 REQUISITIOS PREVIOS:
22 REQUISITIOS PREVIOS:
23 - Paso 1: Tener "pip [Python 2]" o "pip3 [Python 3]" instalado:
23 - Paso 1: Tener "pip [Python 2]" o "pip3 [Python 3]" instalado:
24 - Paso 2: Instalar lo siguiente como admininstrador:
24 - Paso 2: Instalar lo siguiente como admininstrador:
25 En Python 2
25 En Python 2
26 - pip install ckanapi==4.5
26 - pip install ckanapi==4.5
27 - pip install requests
27 - pip install requests
28 - pip install pathlib
28 - pip install pathlib
29 - pip install futures
29 - pip install futures
30 - pip install tqdm
30 - pip install tqdm
31 En Python > 3
31 En Python > 3
32 - pip3 install ckanapi==4.5
32 - pip3 install ckanapi==4.5
33 - pip3 install requests
33 - pip3 install requests
34 - pip3 install tqdm
34 - pip3 install tqdm
35
35
36 FUNCIONES DISPONIBLES:
36 FUNCIONES DISPONIBLES:
37 - action
37 - action
38 - upload_file
38 - upload_file
39 - upload_multiple_files
39 - upload_multiple_files
40 - upload_multiple_files_advance
40 - upload_multiple_files_advance
41 - show
41 - show
42 - search
42 - search
43 - create
43 - create
44 - patch
44 - patch
45 - delete
45 - delete
46 - download_files
46 - download_files
47
47
48 EJEMPLOS:
48 EJEMPLOS:
49 #1:
49 #1:
50 with JROAPI('http://demo.example.com', Authorization='#########') as <access_name>:
50 with JROAPI('http://demo.example.com', Authorization='#########') as <access_name>:
51 ... some operation(s) ...
51 ... some operation(s) ...
52 #2:
52 #2:
53 <access_name> = JROAPI('http://example.com', Authorization='#########')
53 <access_name> = JROAPI('http://example.com', Authorization='#########')
54 ... some operation(s) ...
54 ... some operation(s) ...
55 <access_name>.ckan.close()
55 <access_name>.ckan.close()
56
56
57 REPORTAR ALGUN PROBLEMA:
57 REPORTAR ALGUN PROBLEMA:
58 Debe enviar un correo a eynilupu@igp.gob.pe detallando los siguientes pasos:
58 Debe enviar un correo a eynilupu@igp.gob.pe detallando los siguientes pasos:
59 1) Correo para contactarlo
59 1) Correo para contactarlo
60 2) Descripcion del problema
60 2) Descripcion del problema
61 3) ¿En que paso o seccion encontro el problema?
61 3) ¿En que paso o seccion encontro el problema?
62 4) ¿Cual era el resultado que usted esperaba?
62 4) ¿Cual era el resultado que usted esperaba?
63 """
63 """
64 def __init__(self, url, Authorization=None):
64 def __init__(self, url, Authorization=None):
65 ua = 'CKAN_JRO/1.1 (+'+str(url)+')'
65 ua = 'CKAN_JRO/1.1 (+'+str(url)+')'
66 #ua = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36'
66 #ua = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36'
67 self.ckan = RemoteCKAN(url, apikey=Authorization, user_agent=ua)
67 self.ckan = RemoteCKAN(url, apikey=Authorization, user_agent=ua)
68 #self.ckan = RemoteCKAN(url, apikey=Authorization)
68 #self.ckan = RemoteCKAN(url, apikey=Authorization)
69 self.Authorization = Authorization
69 self.Authorization = Authorization
70 if platform.system() == 'Windows':
70 if platform.system() == 'Windows':
71 self.separator = '\\'
71 self.separator = '\\'
72 else:
72 else:
73 self.separator = '/'
73 self.separator = '/'
74
74
75 self.chunk_size = 1024
75 self.chunk_size = 1024
76 self.list = []
76 self.list = []
77 self.dict = {}
77 self.dict = {}
78 self.str = ''
78 self.str = ''
79 self.check = 1
79 self.check = 1
80 self.cont = 0
80 self.cont = 0
81
81
82 def __enter__(self):
82 def __enter__(self):
83 return self
83 return self
84
84
85 def __exit__(self, *args):
85 def __exit__(self, *args):
86 self.ckan.close()
86 self.ckan.close()
87
87
88 def action(self, action, **kwargs):
88 def action(self, action, **kwargs):
89 """
89 """
90 FINALIDAD:
90 FINALIDAD:
91 Funcion para llamar a las APIs disponibles
91 Funcion para llamar a las APIs disponibles
92
92
93 APIs DISPONIBLES:
93 APIs DISPONIBLES:
94 CONSULTAR: "GUIA DE SCRIPT.pdf"
94 CONSULTAR: "GUIA DE SCRIPT.pdf"
95
95
96 EJEMPLO:
96 EJEMPLO:
97 <access_name>.action(<consuming API>, param_1 = <class 'param_1'>, ...)
97 <access_name>.action(<consuming API>, param_1 = <class 'param_1'>, ...)
98 """
98 """
99 #--------------- CASE: PACKAGE SEARCH ---------------#
99 #--------------- CASE: PACKAGE SEARCH ---------------#
100 if kwargs is not None:
100 if kwargs is not None:
101 if action == 'package_search':
101 if action == 'package_search':
102 self.list = ['facet_mincount', 'facet_limit', 'facet_field']
102 self.list = ['facet_mincount', 'facet_limit', 'facet_field']
103 for facet in self.list:
103 for facet in self.list:
104 if facet in kwargs:
104 if facet in kwargs:
105 kwargs[facet.replace('_', '.')] = kwargs[facet]
105 kwargs[facet.replace('_', '.')] = kwargs[facet]
106 kwargs.pop(facet)
106 kwargs.pop(facet)
107 #----------------------------------------------------#
107 #----------------------------------------------------#
108 try:
108 try:
109 return getattr(self.ckan.action, action)(**kwargs)
109 return getattr(self.ckan.action, action)(**kwargs)
110 except:
110 except:
111 _, exc_value, _ = sys.exc_info()
111 _, exc_value, _ = sys.exc_info()
112 return exc_value
112 return exc_value
113
113
114 def upload_file(self, dataset_id, file_path, file_date, file_type, **kwargs):
114 def upload_file(self, dataset_id, file_path, file_date, file_type, **kwargs):
115 # Agregar si es interruptido por teclado
115 # Agregar si es interruptido por teclado
116 '''
116 '''
117 FINALIDAD:
117 FINALIDAD:
118 Funcion para subir un unico archivo al repositorio del ROJ.
118 Funcion para subir un unico archivo al repositorio del ROJ.
119
119
120 PARAMETROS DISPONIBLES:
120 PARAMETROS DISPONIBLES:
121 CONSULTAR: "GUIA DE SCRIPT.pdf"
121 CONSULTAR: "GUIA DE SCRIPT.pdf"
122
122
123 ESTRUCTURA:
123 ESTRUCTURA:
124 <access_name>.upload_file(dataset_id = <class 'str'>, file_date = <class 'str'>, file_path = <class 'str'>, file_type = <class 'str'>, param_1 = <class 'param_1'>, ...)
124 <access_name>.upload_file(dataset_id = <class 'str'>, file_date = <class 'str'>, file_path = <class 'str'>, file_type = <class 'str'>, param_1 = <class 'param_1'>, ...)
125 '''
125 '''
126 self.list = ['package_id', 'upload', 'voc_file_type', 'name'] #file_date
126 self.list = ['package_id', 'upload', 'voc_file_type', 'name'] #file_date
127 for key1, value1 in kwargs.items():
127 for key1, value1 in kwargs.items():
128 if not key1 in self.list:
128 if not key1 in self.list:
129 self.dict[key1] = value1
129 self.dict[key1] = value1
130
130
131 #---------------------------#
131 #---------------------------#
132 if not 'others' in kwargs:
132 if not 'others' in kwargs:
133 self.dict['others'] = ''
133 self.dict['others'] = ''
134 else:
134 else:
135 if isinstance(kwargs['others'], list):
135 if isinstance(kwargs['others'], list):
136 self.dict['others'] = json.dumps(kwargs['others'])
136 self.dict['others'] = json.dumps(kwargs['others'])
137 #---------------------------#
137 #---------------------------#
138
138
139 if not os.path.isfile(file_path):
139 if not os.path.isfile(file_path):
140 return 'File "%s" not exist' % (file_path)
140 return 'File "%s" not exist' % (file_path)
141
141
142 if not 'format' in self.dict:
142 if not 'format' in self.dict:
143 self.str = ''.join(pathlib.Path(file_path).suffixes)
143 self.str = ''.join(pathlib.Path(file_path).suffixes)
144 if len(self.str) > 0:
144 if len(self.str) > 0:
145 self.dict['format'] = self.str.upper()[1:]
145 self.dict['format'] = self.str.upper()[1:]
146
146
147 try:
147 try:
148 return getattr(self.ckan.action, 'resource_create')(package_id=dataset_id, file_date=file_date, upload=open(file_path, 'rb'), voc_file_type=file_type, name=pathlib.Path(file_path).name, **self.dict)
148 return getattr(self.ckan.action, 'resource_create')(package_id=dataset_id, file_date=file_date, upload=open(file_path, 'rb'), voc_file_type=file_type, name=pathlib.Path(file_path).name, **self.dict)
149 except:
149 except:
150 _, exc_value, _ = sys.exc_info()
150 _, exc_value, _ = sys.exc_info()
151 return exc_value
151 return exc_value
152
152
153 def upload_multiple_files_advance(self, dataset_id, path_files, file_date, file_type, max_size=100, ignore_repetition=False, **kwargs):
153 def upload_multiple_files_advance(self, dataset_id, path_files, file_date, file_type, max_size=100, max_count=500, ignore_repetition=False, **kwargs):
154 # Agregar si es interruptido por teclado
154 # Agregar si es interruptido por teclado
155 '''
155 '''
156 FINALIDAD:
156 FINALIDAD:
157 Funcion para subir multiples archivos al repositorio del ROJ.
157 Funcion para subir multiples archivos al repositorio del ROJ.
158
158
159 PARAMETROS DISPONIBLES:
159 PARAMETROS DISPONIBLES:
160 CONSULTAR: "GUIA DE SCRIPT.pdf"
160 CONSULTAR: "GUIA DE SCRIPT.pdf"
161
161
162 ESTRUCTURA:
162 ESTRUCTURA:
163 <access_name>.upload_multiple_files_advance(dataset_id = <class 'str'>, path_files = <class 'list of strings'>, file_date = <class 'str'>, file_type = <class 'str'>, param_1 = <class 'param_1'>, ...)
163 <access_name>.upload_multiple_files_advance(dataset_id = <class 'str'>, path_files = <class 'list of strings'>, file_date = <class 'str'>, file_type = <class 'str'>, param_1 = <class 'param_1'>, ...)
164 '''
164 '''
165 #-------------------------PACKAGE SHOW-----------------------#
165 #-------------------------PACKAGE SHOW-----------------------#
166 try:
166 try:
167 dataset_show = getattr(self.ckan.action, 'package_show')(id=dataset_id)['resources']
167 dataset_show = getattr(self.ckan.action, 'package_show')(id=dataset_id)['resources']
168 except:
168 except:
169 _, exc_value, _ = sys.exc_info()
169 _, exc_value, _ = sys.exc_info()
170 print('ERROR obtaining metadata dataset:: Use the "print" for more information')
170 print('ERROR obtaining metadata dataset:: Use the "print" for more information')
171 return exc_value
171 return exc_value
172 #------------------------------------------------------------#
172 #------------------------------------------------------------#
173 resources_name = []
173 resources_name = []
174 for u in dataset_show:
174 for u in dataset_show:
175 resources_name.append(u['name'].lower())
175 resources_name.append(u['name'].lower())
176 #------------------------------------------------------------#
176 #------------------------------------------------------------#
177 self.list = ['package_id', 'upload', 'voc_file_type', 'name']
177 self.list = ['package_id', 'upload', 'voc_file_type', 'name']
178 for key1, value1 in kwargs.items():
178 for key1, value1 in kwargs.items():
179 if not key1 in self.list:
179 if not key1 in self.list:
180 self.dict[key1] = value1
180 self.dict[key1] = value1
181 #------------------------------------------------------------#
181 #------------------------------------------------------------#
182 if not 'others' in kwargs:
182 if not 'others' in kwargs:
183 self.dict['others'] = ''
183 self.dict['others'] = ''
184 else:
184 else:
185 if isinstance(kwargs['others'], list):
185 if isinstance(kwargs['others'], list):
186 self.dict['others'] = json.dumps(kwargs['others'])
186 self.dict['others'] = json.dumps(kwargs['others'])
187 #------------------------------------------------------------#
187 #------------------------------------------------------------#
188 total_list = []
188 total_list = []
189 #---------------CASO : "path" or "path_list"-----------------#
189 #---------------CASO : "path" or "path_list"-----------------#
190 if type(path_files) is list:
190 if type(path_files) is list:
191 if len(path_files) != 0:
191 if len(path_files) != 0:
192 path_files.sort()
192 path_files.sort()
193 for u in path_files:
193 for u in path_files:
194 if os.path.isfile(u):
194 if os.path.isfile(u):
195 if pathlib.Path(u).name.lower() in resources_name:
195 if pathlib.Path(u).name.lower() in resources_name:
196 if not ignore_repetition:
196 if not ignore_repetition:
197 return 'ERROR:: "%s" file already exist in this dataset' % (pathlib.Path(u).name)
197 return 'ERROR:: "%s" file already exist in this dataset' % (pathlib.Path(u).name)
198 print('WARRING:: "'+ str(pathlib.Path(u).name) +'" file was ignored because already exist in this dataset')
198 print('WARRING:: "'+ str(pathlib.Path(u).name) +'" file was ignored because already exist in this dataset')
199 else:
199 else:
200 total_list.append({'name':pathlib.Path(u).name, 'size': os.stat(u).st_size, 'upload':open(u, 'rb')})
200 total_list.append({'name':pathlib.Path(u).name, 'size': os.stat(u).st_size, 'upload':open(u, 'rb')})
201 else:
201 else:
202 return 'File "%s" does not exist' % (u)
202 return 'File "%s" does not exist' % (u)
203 else:
203 else:
204 return 'ERROR:: "path_list is empty"'
204 return 'ERROR:: "path_list is empty"'
205
205
206 elif type(path_files) is str:
206 elif type(path_files) is str:
207 if os.path.isdir(path_files):
207 if os.path.isdir(path_files):
208 path_order = [f for f in os.listdir(path_files) if os.path.isfile(os.path.join(path_files, f))]
208 path_order = [f for f in os.listdir(path_files) if os.path.isfile(os.path.join(path_files, f))]
209 path_order.sort()
209 path_order.sort()
210 if path_order:
210 if path_order:
211 for name in path_order:
211 for name in path_order:
212 if name.lower() in resources_name:
212 if name.lower() in resources_name:
213 if not ignore_repetition:
213 if not ignore_repetition:
214 return 'ERROR:: "%s" file already exist in this dataset' % (name)
214 return 'ERROR:: "%s" file already exist in this dataset' % (name)
215 print('WARRING:: "'+ name +'" file was ignored because already exist in this dataset')
215 print('WARRING:: "'+ name +'" file was ignored because already exist in this dataset')
216 else:
216 else:
217 total_list.append({'name':name, 'size': os.stat(os.path.join(path_files, name)).st_size, 'upload':open(os.path.join(path_files, name), 'rb')})
217 total_list.append({'name':name, 'size': os.stat(os.path.join(path_files, name)).st_size, 'upload':open(os.path.join(path_files, name), 'rb')})
218 else:
218 else:
219 return "ERROR:: There aren't files in this directory"
219 return "ERROR:: There aren't files in this directory"
220 else:
220 else:
221 return 'ERROR:: Directory "%s" does not exist' % (path_files)
221 return 'ERROR:: Directory "%s" does not exist' % (path_files)
222 else:
222 else:
223 return 'ERROR:: "path_files" must be a str or list'
223 return 'ERROR:: "path_files" must be a str or list'
224 #------------------------------------------------------------#
224 #------------------------------------------------------------#
225 try:
225 try:
226 uuid.UUID(str(dataset_id), version=4)
226 uuid.UUID(str(dataset_id), version=4)
227 package_id_or_name = '"id": "' + str(dataset_id) + '"'
227 package_id_or_name = '"id": "' + str(dataset_id) + '"'
228 except ValueError:
228 except ValueError:
229 package_id_or_name = '"name": "' + str(dataset_id) + '"'
229 package_id_or_name = '"name": "' + str(dataset_id) + '"'
230 #------------------------------------------------------------#
230 #------------------------------------------------------------#
231 blocks = [[]]
231 blocks = [[]]
232 size_file = 0
232 size_file = 0
233 count_file = 0
233 inter_num = 0
234 inter_num = 0
234 for value in total_list:
235 for value in total_list:
235 if value['size'] > 1048576 * float(max_size):
236 if value['size'] > 1024 * 1024 * float(max_size):
236 return 'ERROR:: The size of the "%s" file is %sMB, please change "max_size" value' % (value['name'], str(round(value['size']/1048576, 1)))
237 return 'ERROR:: The size of the "%s" file is %sMB aprox, please change "max_size" value' % (value['name'], str(round(value['size']/(1024 * 1024), 2)))
238 if not 1 <= int(max_count) <= 999:
239 return 'ERROR:: The count of the number of files must be between 1 and 999, please change "max_count" value'
240
237 size_file = size_file + value['size']
241 size_file = size_file + value['size']
238 if size_file <= 1048576 * float(max_size):
242 count_file = count_file + 1
243 if size_file <= 1024 * 1024 * float(max_size) and count_file <= int(max_count):
239 del value['size']
244 del value['size']
240 blocks[inter_num].append(value)
245 blocks[inter_num].append(value)
241 else:
246 else:
242 inter_num = inter_num + 1
247 inter_num = inter_num + 1
248 size_file = value['size']
249 count_file = 1
243 blocks.append([])
250 blocks.append([])
244 del value['size']
251 del value['size']
245 blocks[inter_num].append(value)
252 blocks[inter_num].append(value)
246 #------------------------------------------------------------#
253 #------------------------------------------------------------#
247 if len(blocks[0]) > 0:
254 if len(blocks[0]) > 0:
248 print('BLOCK(S) IN TOTAL:: {}'.format(len(blocks)))
255 print('BLOCK(S) IN TOTAL:: {}'.format(len(blocks)))
249 for count1, block in enumerate(blocks):
256 for count1, block in enumerate(blocks):
250 print('---- BLOCK N°{} ----'.format(count1 + 1))
257 print('---- BLOCK N°{} ----'.format(count1 + 1))
251 resource_extend = []
258 resource_extend = []
252 files_dict = {}
259 files_dict = {}
253 for count2, value2 in enumerate(block):
260 for count2, value2 in enumerate(block):
254 value2['file_date'] = file_date
261 value2['file_date'] = file_date
255 value2['voc_file_type'] = file_type
262 value2['voc_file_type'] = file_type
256 value2.update(self.dict)
263 value2.update(self.dict)
257
264
258 if not 'format' in value2:
265 if not 'format' in value2:
259 format = ''.join(pathlib.Path(value2['name']).suffixes)
266 format = ''.join(pathlib.Path(value2['name']).suffixes)
260 if len(format) > 0:
267 if len(format) > 0:
261 value2['format'] = format.upper()[1:]
268 value2['format'] = format.upper()[1:]
262
269
263 files_dict['update__resources__-'+ str(len(block)-count2) +'__upload'] = (value2['name'], value2['upload'])
270 files_dict['update__resources__-'+ str(len(block)-count2) +'__upload'] = (value2['name'], value2['upload'])
264 del value2['upload']
271 del value2['upload']
265 resource_extend.append(value2)
272 resource_extend.append(value2)
266
273
267 print('BLOCK N°{} :: "{}" file(s) found >> uploading'.format(count1 + 1, len(block)))
274 print('BLOCK N°{} :: "{}" file(s) found >> uploading'.format(count1 + 1, len(block)))
268 try:
275 try:
269 result = self.ckan.call_action(
276 result = self.ckan.call_action(
270 'package_revise',
277 'package_revise',
271 {'match': '{'+ str(package_id_or_name) +'}', 'update__resources__extend': json.dumps(resource_extend)},
278 {'match': '{'+ str(package_id_or_name) +'}', 'update__resources__extend': json.dumps(resource_extend)},
272 files=files_dict
279 files=files_dict
273 )
280 )
274 print('BLOCK N°{} :: Uploaded file(s) successfully'.format(count1 + 1))
281 print('BLOCK N°{} :: Uploaded file(s) successfully'.format(count1 + 1))
275 if len(blocks) == count1 + 1:
282 if len(blocks) == count1 + 1:
276 return result
283 return result
277 except:
284 except:
278 print('ERROR :: Use the "print" for more information')
285 print('ERROR :: Use the "print" for more information')
279 _, exc_value, _ = sys.exc_info()
286 _, exc_value, _ = sys.exc_info()
280 return exc_value
287 return exc_value
281 else:
288 else:
282 return "ERROR:: No file(s) found to upload"
289 return "ERROR:: No file(s) found to upload"
283
290
284 def upload_multiple_files(self, dataset_id, path_files, date_files, type_files, **kwargs):
291 def upload_multiple_files(self, dataset_id, path_files, date_files, type_files, **kwargs):
285 # Agregar si es interruptido por teclado
292 # Agregar si es interruptido por teclado
286 '''
293 '''
287 FINALIDAD:
294 FINALIDAD:
288 Funcion para subir multiples archivos al repositorio del ROJ.
295 Funcion para subir multiples archivos al repositorio del ROJ.
289
296
290 PARAMETROS DISPONIBLES:
297 PARAMETROS DISPONIBLES:
291 CONSULTAR: "GUIA DE SCRIPT.pdf"
298 CONSULTAR: "GUIA DE SCRIPT.pdf"
292
299
293 ESTRUCTURA:
300 ESTRUCTURA:
294 <access_name>.upload_multiple_files(dataset_id = <class 'str'>, path_files = <class 'str'> or <class 'list of strings'>, date_files = <class 'str'> or <class 'list of strings'>, type_files = <class 'str'> or <class 'list of strings'>, param_1 = <class 'param_1'>, ...)
301 <access_name>.upload_multiple_files(dataset_id = <class 'str'>, path_files = <class 'str'> or <class 'list of strings'>, date_files = <class 'str'> or <class 'list of strings'>, type_files = <class 'str'> or <class 'list of strings'>, param_1 = <class 'param_1'>, ...)
295 '''
302 '''
296
303
297 params_dict = {'upload':[], 'name':[]}
304 params_dict = {'upload':[], 'name':[]}
298 if not 'format' in kwargs:
305 if not 'format' in kwargs:
299 params_dict.update({'format':[]})
306 params_dict.update({'format':[]})
300 #---------------CASO : "path" or "path_list"-----------------#
307 #---------------CASO : "path" or "path_list"-----------------#
301 if type(path_files) is list:
308 if type(path_files) is list:
302 if len(path_files) != 0:
309 if len(path_files) != 0:
303 path_files.sort()
310 path_files.sort()
304 for u in path_files:
311 for u in path_files:
305 if os.path.isfile(u):
312 if os.path.isfile(u):
306 params_dict['upload'].append(open(u, 'rb'))
313 params_dict['upload'].append(open(u, 'rb'))
307 params_dict['name'].append(pathlib.Path(u).name)
314 params_dict['name'].append(pathlib.Path(u).name)
308 if not 'format' in kwargs:
315 if not 'format' in kwargs:
309 format = ''.join(pathlib.Path(u).suffixes)
316 format = ''.join(pathlib.Path(u).suffixes)
310 if len(format) > 0:
317 if len(format) > 0:
311 params_dict['format'].append(format.upper()[1:])
318 params_dict['format'].append(format.upper()[1:])
312 else:
319 else:
313 params_dict['format'].append('')
320 params_dict['format'].append('')
314 else:
321 else:
315 return 'File "%s" does not exist' % (u)
322 return 'File "%s" does not exist' % (u)
316 else:
323 else:
317 return 'ERROR:: "path_list is empty"'
324 return 'ERROR:: "path_list is empty"'
318 elif type(path_files) is str:
325 elif type(path_files) is str:
319 if os.path.isdir(path_files):
326 if os.path.isdir(path_files):
320 path_order = [f for f in os.listdir(path_files) if os.path.isfile(os.path.join(path_files, f))]
327 path_order = [f for f in os.listdir(path_files) if os.path.isfile(os.path.join(path_files, f))]
321 path_order.sort()
328 path_order.sort()
322 if path_order:
329 if path_order:
323 for name in path_order:
330 for name in path_order:
324 params_dict['upload'].append(open(os.path.join(path_files, name), 'rb'))
331 params_dict['upload'].append(open(os.path.join(path_files, name), 'rb'))
325 params_dict['name'].append(name)
332 params_dict['name'].append(name)
326 if not 'format' in kwargs:
333 if not 'format' in kwargs:
327 format = ''.join(pathlib.Path(name).suffixes)
334 format = ''.join(pathlib.Path(name).suffixes)
328 if len(format) > 0:
335 if len(format) > 0:
329 params_dict['format'].append(format.upper()[1:])
336 params_dict['format'].append(format.upper()[1:])
330 else:
337 else:
331 params_dict['format'].append('')
338 params_dict['format'].append('')
332 else:
339 else:
333 return "ERROR:: There aren't files in this directory"
340 return "ERROR:: There aren't files in this directory"
334 else:
341 else:
335 return 'ERROR:: Directory "%s" does not exist' % (path_files)
342 return 'ERROR:: Directory "%s" does not exist' % (path_files)
336 else:
343 else:
337 return 'ERROR:: "path_files" must be a str or list'
344 return 'ERROR:: "path_files" must be a str or list'
338 #------------------------------------------------------------#
345 #------------------------------------------------------------#
339 params_no_dict = {'package_id': dataset_id}
346 params_no_dict = {'package_id': dataset_id}
340 if type(date_files) is list:
347 if type(date_files) is list:
341 params_dict['file_date'] = date_files
348 params_dict['file_date'] = date_files
342 else:
349 else:
343 params_no_dict['file_date'] = date_files
350 params_no_dict['file_date'] = date_files
344
351
345 if type(type_files) is list:
352 if type(type_files) is list:
346 params_dict['voc_file_type'] = type_files
353 params_dict['voc_file_type'] = type_files
347 else:
354 else:
348 params_no_dict['voc_file_type'] = type_files
355 params_no_dict['voc_file_type'] = type_files
349
356
350 for key1, value1 in kwargs.items():
357 for key1, value1 in kwargs.items():
351 if not key1 in params_dict and not key1 in params_no_dict and key1 != 'others':
358 if not key1 in params_dict and not key1 in params_no_dict and key1 != 'others':
352 if type(value1) is list:
359 if type(value1) is list:
353 params_dict[key1] = value1
360 params_dict[key1] = value1
354 else:
361 else:
355 params_no_dict[key1] = value1
362 params_no_dict[key1] = value1
356 #------------------------------------------#
363 #------------------------------------------#
357 if not 'others' in kwargs:
364 if not 'others' in kwargs:
358 params_no_dict['others'] = ''
365 params_no_dict['others'] = ''
359 else:
366 else:
360 if isinstance(kwargs['others'], tuple):
367 if isinstance(kwargs['others'], tuple):
361 params_dict['others'] = [json.dumps(w) for w in kwargs['others']]
368 params_dict['others'] = [json.dumps(w) for w in kwargs['others']]
362 elif isinstance(kwargs['others'], list):
369 elif isinstance(kwargs['others'], list):
363 params_no_dict['others'] = json.dumps(kwargs['others'])
370 params_no_dict['others'] = json.dumps(kwargs['others'])
364 elif isinstance(kwargs['others'], str):
371 elif isinstance(kwargs['others'], str):
365 params_no_dict['others'] = kwargs['others']
372 params_no_dict['others'] = kwargs['others']
366 else:
373 else:
367 return 'ERROR:: "others" must be a tuple, list or str'
374 return 'ERROR:: "others" must be a tuple, list or str'
368 #------------------------------------------#
375 #------------------------------------------#
369 len_params_dict = []
376 len_params_dict = []
370 for value2 in params_dict.values():
377 for value2 in params_dict.values():
371 len_params_dict.append(len(value2))
378 len_params_dict.append(len(value2))
372
379
373 if len(list(set(len_params_dict))) > 1:
380 if len(list(set(len_params_dict))) > 1:
374 return 'ERROR:: All lists must be the same length: %s' % (len(params_dict['name']))
381 return 'ERROR:: All lists must be the same length: %s' % (len(params_dict['name']))
375 #------------------------------------------------------------#
382 #------------------------------------------------------------#
376 print('"{}" file(s) found >> uploading'.format(len(params_dict['name'])))
383 print('"{}" file(s) found >> uploading'.format(len(params_dict['name'])))
377 for v in range(len(params_dict['name'])):
384 for v in range(len(params_dict['name'])):
378 try:
385 try:
379 send = {}
386 send = {}
380 for key_dict, value_dict in params_dict.items():
387 for key_dict, value_dict in params_dict.items():
381 send[key_dict] = value_dict[v]
388 send[key_dict] = value_dict[v]
382 for key_no_dict, value_no_dict in params_no_dict.items():
389 for key_no_dict, value_no_dict in params_no_dict.items():
383 send[key_no_dict] = value_no_dict
390 send[key_no_dict] = value_no_dict
384
391
385 self.list.append(getattr(self.ckan.action, 'resource_create')(**send))
392 self.list.append(getattr(self.ckan.action, 'resource_create')(**send))
386 print('File #{} :: "{}" was uploaded successfully'.format(v+1, params_dict['name'][v]))
393 print('File #{} :: "{}" was uploaded successfully'.format(v+1, params_dict['name'][v]))
387 except:
394 except:
388 _, exc_value, _ = sys.exc_info()
395 _, exc_value, _ = sys.exc_info()
389 self.list.append(exc_value)
396 self.list.append(exc_value)
390 print('File #{} :: Error uploading "{}" file'.format(v+1, params_dict['name'][v]))
397 print('File #{} :: Error uploading "{}" file'.format(v+1, params_dict['name'][v]))
391 return self.list
398 return self.list
392 #------------------------------------------------------------#
399 #------------------------------------------------------------#
393
400
394 def show(self, type_option, id, **kwargs):
401 def show(self, type_option, id, **kwargs):
395 '''
402 '''
396 FINALIDAD:
403 FINALIDAD:
397 Funcion personalizada para una busqueda en especifico.
404 Funcion personalizada para una busqueda en especifico.
398
405
399 PARAMETROS DISPONIBLES:
406 PARAMETROS DISPONIBLES:
400 CONSULTAR: "GUIA DE SCRIPT.pdf"
407 CONSULTAR: "GUIA DE SCRIPT.pdf"
401
408
402 ESTRUCTURA:
409 ESTRUCTURA:
403 <access_name>.show(type_option = <class 'str'>, id = <class 'str'>, param_1 = <class 'param_1'>, ...)
410 <access_name>.show(type_option = <class 'str'>, id = <class 'str'>, param_1 = <class 'param_1'>, ...)
404 '''
411 '''
405 if type(type_option) is str:
412 if type(type_option) is str:
406 try:
413 try:
407 if type_option == 'dataset':
414 if type_option == 'dataset':
408 return getattr(self.ckan.action, 'package_show')(id=id, **kwargs)
415 return getattr(self.ckan.action, 'package_show')(id=id, **kwargs)
409 elif type_option == 'resource':
416 elif type_option == 'resource':
410 return getattr(self.ckan.action, 'resource_show')(id=id, **kwargs)
417 return getattr(self.ckan.action, 'resource_show')(id=id, **kwargs)
411 elif type_option == 'project':
418 elif type_option == 'project':
412 return getattr(self.ckan.action, 'organization_show')(id=id, **kwargs)
419 return getattr(self.ckan.action, 'organization_show')(id=id, **kwargs)
413 elif type_option == 'collaborator':
420 elif type_option == 'collaborator':
414 return getattr(self.ckan.action, 'package_collaborator_list_for_user')(id=id, **kwargs)
421 return getattr(self.ckan.action, 'package_collaborator_list_for_user')(id=id, **kwargs)
415 elif type_option == 'member':
422 elif type_option == 'member':
416 return getattr(self.ckan.action, 'organization_list_for_user')(id=id, **kwargs)
423 return getattr(self.ckan.action, 'organization_list_for_user')(id=id, **kwargs)
417 elif type_option == 'vocabulary':
424 elif type_option == 'vocabulary':
418 return getattr(self.ckan.action, 'vocabulary_show')(id=id, **kwargs)
425 return getattr(self.ckan.action, 'vocabulary_show')(id=id, **kwargs)
419 elif type_option == 'tag':
426 elif type_option == 'tag':
420 if not 'vocabulary_id' in kwargs:
427 if not 'vocabulary_id' in kwargs:
421 print('Missing "vocabulary_id" value: assume it is a free tag')
428 print('Missing "vocabulary_id" value: assume it is a free tag')
422 return getattr(self.ckan.action, 'tag_show')(id=id, **kwargs)
429 return getattr(self.ckan.action, 'tag_show')(id=id, **kwargs)
423 elif type_option == 'user':
430 elif type_option == 'user':
424 return getattr(self.ckan.action, 'user_show')(id=id, **kwargs)
431 return getattr(self.ckan.action, 'user_show')(id=id, **kwargs)
425 elif type_option == 'job':
432 elif type_option == 'job':
426 return getattr(self.ckan.action, 'job_show')(id=id, **kwargs)
433 return getattr(self.ckan.action, 'job_show')(id=id, **kwargs)
427 else:
434 else:
428 return 'ERROR:: "type_option = %s" is not accepted' % (type_option)
435 return 'ERROR:: "type_option = %s" is not accepted' % (type_option)
429 except:
436 except:
430 _, exc_value, _ = sys.exc_info()
437 _, exc_value, _ = sys.exc_info()
431 return exc_value
438 return exc_value
432 else:
439 else:
433 return 'ERROR:: "type_option" must be a str'
440 return 'ERROR:: "type_option" must be a str'
434
441
435 def search(self, type_option, query=None, **kwargs):
442 def search(self, type_option, query=None, **kwargs):
436 '''
443 '''
437 FINALIDAD:
444 FINALIDAD:
438 Funcion personalizada para busquedas que satisfagan algun criterio.
445 Funcion personalizada para busquedas que satisfagan algun criterio.
439
446
440 PARAMETROS DISPONIBLES:
447 PARAMETROS DISPONIBLES:
441 CONSULTAR: "GUIA DE SCRIPT.pdf"
448 CONSULTAR: "GUIA DE SCRIPT.pdf"
442
449
443 ESTRUCTURA:
450 ESTRUCTURA:
444 <access_name>.search(type_option = <class 'str'>, query = <class 'dict'>, param_1 = <class 'param_1'>, ...)
451 <access_name>.search(type_option = <class 'str'>, query = <class 'dict'>, param_1 = <class 'param_1'>, ...)
445 '''
452 '''
446 if type(type_option) is str:
453 if type(type_option) is str:
447 try:
454 try:
448 if type_option == 'dataset':
455 if type_option == 'dataset':
449 key_replace = ['fq', 'fq_list', 'include_private']
456 key_replace = ['fq', 'fq_list', 'include_private']
450 key_point = ['facet_mincount', 'facet_limit', 'facet_field']
457 key_point = ['facet_mincount', 'facet_limit', 'facet_field']
451 for key1, value1 in kwargs.items():
458 for key1, value1 in kwargs.items():
452 if not key1 in key_replace:
459 if not key1 in key_replace:
453 if key1 in key_point:
460 if key1 in key_point:
454 self.dict[key1.replace('_', '.')] = value1
461 self.dict[key1.replace('_', '.')] = value1
455 else:
462 else:
456 self.dict[key1] = value1
463 self.dict[key1] = value1
457
464
458 if query is not None:
465 if query is not None:
459 if type(query) is dict:
466 if type(query) is dict:
460 self.dict['fq_list'] = []
467 self.dict['fq_list'] = []
461 #NUM_RESOURCES_MIN / NUM_RESOURCES_MAX
468 #NUM_RESOURCES_MIN / NUM_RESOURCES_MAX
462 #----------------------------------------------------#
469 #----------------------------------------------------#
463 if 'dataset_start_date' in query:
470 if 'dataset_start_date' in query:
464 if type(query['dataset_start_date']) is str:
471 if type(query['dataset_start_date']) is str:
465 try:
472 try:
466 datetime.strptime(query['dataset_start_date'], '%Y-%m-%d')
473 datetime.strptime(query['dataset_start_date'], '%Y-%m-%d')
467 if len(query['dataset_start_date']) != 10:
474 if len(query['dataset_start_date']) != 10:
468 return '"dataset_start_date", must be: <YYYY-MM-DD>'
475 return '"dataset_start_date", must be: <YYYY-MM-DD>'
469 self.dict['fq_list'].append('dataset_start_date:"'+query['dataset_start_date']+'"')
476 self.dict['fq_list'].append('dataset_start_date:"'+query['dataset_start_date']+'"')
470 self.list.append('dataset_start_date')
477 self.list.append('dataset_start_date')
471 except:
478 except:
472 return '"dataset_start_date" incorrect: "%s"' % (query['dataset_start_date'])
479 return '"dataset_start_date" incorrect: "%s"' % (query['dataset_start_date'])
473 else:
480 else:
474 return '"dataset_start_date" must be <str>'
481 return '"dataset_start_date" must be <str>'
475 #----------------------------------------------------#
482 #----------------------------------------------------#
476 if 'dataset_end_date' in query:
483 if 'dataset_end_date' in query:
477 if type(query['dataset_end_date']) is str:
484 if type(query['dataset_end_date']) is str:
478 try:
485 try:
479 datetime.strptime(query['dataset_end_date'], '%Y-%m-%d')
486 datetime.strptime(query['dataset_end_date'], '%Y-%m-%d')
480 if len(query['dataset_end_date']) != 10:
487 if len(query['dataset_end_date']) != 10:
481 return '"dataset_end_date", must be: <YYYY-MM-DD>'
488 return '"dataset_end_date", must be: <YYYY-MM-DD>'
482
489
483 if 'dataset_start_date' in query:
490 if 'dataset_start_date' in query:
484 if query['dataset_start_date'] > query['dataset_end_date']:
491 if query['dataset_start_date'] > query['dataset_end_date']:
485 return '"dataset_end_date" must be greater than "dataset_start_date"'
492 return '"dataset_end_date" must be greater than "dataset_start_date"'
486
493
487 self.dict['fq_list'].append('dataset_end_date:"'+query['dataset_end_date']+'"')
494 self.dict['fq_list'].append('dataset_end_date:"'+query['dataset_end_date']+'"')
488 self.list.append('dataset_end_date')
495 self.list.append('dataset_end_date')
489 except:
496 except:
490 return '"dataset_end_date" incorrect: "%s"' % (query['dataset_end_date'])
497 return '"dataset_end_date" incorrect: "%s"' % (query['dataset_end_date'])
491 else:
498 else:
492 return '"dataset_end_date" must be <str>'
499 return '"dataset_end_date" must be <str>'
493 #----------------------------------------------------#
500 #----------------------------------------------------#
494 for key, value in query.items():
501 for key, value in query.items():
495 if value is not None and not key in self.list:
502 if value is not None and not key in self.list:
496 self.dict['fq_list'].append(str(key)+':"'+str(value)+'"')
503 self.dict['fq_list'].append(str(key)+':"'+str(value)+'"')
497 else:
504 else:
498 return '"query" must be <dict>'
505 return '"query" must be <dict>'
499
506
500 return getattr(self.ckan.action, 'package_search')(include_private=True, **self.dict)
507 return getattr(self.ckan.action, 'package_search')(include_private=True, **self.dict)
501
508
502 elif type_option == 'resource':
509 elif type_option == 'resource':
503 for key1, value1 in kwargs.items():
510 for key1, value1 in kwargs.items():
504 if key1 != 'fields':
511 if key1 != 'fields':
505 self.dict[key1] = value1
512 self.dict[key1] = value1
506
513
507 if query is not None:
514 if query is not None:
508 if type(query) is dict:
515 if type(query) is dict:
509 #----------------------------------------------------#
516 #----------------------------------------------------#
510 if 'file_date_min' in query:
517 if 'file_date_min' in query:
511 if type(query['file_date_min']) is str:
518 if type(query['file_date_min']) is str:
512 try:
519 try:
513 datetime.strptime(query['file_date_min'], '%Y-%m-%d')
520 datetime.strptime(query['file_date_min'], '%Y-%m-%d')
514 if len(query['file_date_min']) != 10:
521 if len(query['file_date_min']) != 10:
515 return '"file_date_min", must be: <YYYY-MM-DD>'
522 return '"file_date_min", must be: <YYYY-MM-DD>'
516 except:
523 except:
517 return '"file_date_min" incorrect: "%s"' % (query['file_date_min'])
524 return '"file_date_min" incorrect: "%s"' % (query['file_date_min'])
518 else:
525 else:
519 return '"file_date_min" must be <str>'
526 return '"file_date_min" must be <str>'
520 #----------------------------------------------------#
527 #----------------------------------------------------#
521 if 'file_date_max' in query:
528 if 'file_date_max' in query:
522 if type(query['file_date_max']) is str:
529 if type(query['file_date_max']) is str:
523 try:
530 try:
524 datetime.strptime(query['file_date_max'], '%Y-%m-%d')
531 datetime.strptime(query['file_date_max'], '%Y-%m-%d')
525 if len(query['file_date_max']) != 10:
532 if len(query['file_date_max']) != 10:
526 return '"file_date_max", must be: <YYYY-MM-DD>'
533 return '"file_date_max", must be: <YYYY-MM-DD>'
527
534
528 if 'file_date_min' in query:
535 if 'file_date_min' in query:
529 if query['file_date_min'] > query['file_date_max']:
536 if query['file_date_min'] > query['file_date_max']:
530 return '"file_date_max" must be greater than "file_date_min"'
537 return '"file_date_max" must be greater than "file_date_min"'
531 except:
538 except:
532 return '"file_date_max" incorrect: "%s"' % (query['file_date_max'])
539 return '"file_date_max" incorrect: "%s"' % (query['file_date_max'])
533 else:
540 else:
534 return '"file_date_max" must be <str>'
541 return '"file_date_max" must be <str>'
535 #----------------------------------------------------#
542 #----------------------------------------------------#
536 self.dict['query'] = query
543 self.dict['query'] = query
537 else:
544 else:
538 return '"query" must be <dict>'
545 return '"query" must be <dict>'
539 return getattr(self.ckan.action, 'resources_search')(**self.dict)
546 return getattr(self.ckan.action, 'resources_search')(**self.dict)
540
547
541 elif type_option == 'tag':
548 elif type_option == 'tag':
542 for key1, value1 in kwargs.items():
549 for key1, value1 in kwargs.items():
543 if key1 != 'fields':
550 if key1 != 'fields':
544 self.dict[key1] = value1
551 self.dict[key1] = value1
545
552
546 if not 'vocabulary_id' in kwargs:
553 if not 'vocabulary_id' in kwargs:
547 print('Missing "vocabulary_id" value: tags that don’t belong to any vocabulary')
554 print('Missing "vocabulary_id" value: tags that don’t belong to any vocabulary')
548 else:
555 else:
549 print('Only tags that belong to "{}" vocabulary'.format(kwargs['vocabulary_id']))
556 print('Only tags that belong to "{}" vocabulary'.format(kwargs['vocabulary_id']))
550
557
551 if query is not None:
558 if query is not None:
552 if type(query) is dict:
559 if type(query) is dict:
553 if 'search' in query:
560 if 'search' in query:
554 if type(query['search']) is list or type(query['search']) is str:
561 if type(query['search']) is list or type(query['search']) is str:
555 self.dict['query'] = query['search']
562 self.dict['query'] = query['search']
556 else:
563 else:
557 return '"search" must be <list> or <str>'
564 return '"search" must be <list> or <str>'
558 else:
565 else:
559 return '"query" must be <dict>'
566 return '"query" must be <dict>'
560 return getattr(self.ckan.action, 'tag_search')(**self.dict)
567 return getattr(self.ckan.action, 'tag_search')(**self.dict)
561
568
562 else:
569 else:
563 return 'ERROR:: "type_option = %s" is not accepted' % (type_option)
570 return 'ERROR:: "type_option = %s" is not accepted' % (type_option)
564
571
565 except:
572 except:
566 _, exc_value, _ = sys.exc_info()
573 _, exc_value, _ = sys.exc_info()
567 return exc_value
574 return exc_value
568 else:
575 else:
569 return 'ERROR:: "type_option" must be <str>'
576 return 'ERROR:: "type_option" must be <str>'
570
577
571 def create(self, type_option, select=None, **kwargs):
578 def create(self, type_option, select=None, **kwargs):
572 '''
579 '''
573 FINALIDAD:
580 FINALIDAD:
574 Funcion personalizada para crear.
581 Funcion personalizada para crear.
575
582
576 PARAMETROS DISPONIBLES:
583 PARAMETROS DISPONIBLES:
577 CONSULTAR: "GUIA DE SCRIPT.pdf"
584 CONSULTAR: "GUIA DE SCRIPT.pdf"
578
585
579 ESTRUCTURA:
586 ESTRUCTURA:
580 <access_name>.create(type_option = <class 'str'>, param_1 = <class 'param_1'>, ...)
587 <access_name>.create(type_option = <class 'str'>, param_1 = <class 'param_1'>, ...)
581 '''
588 '''
582 if type(type_option) is str:
589 if type(type_option) is str:
583 try:
590 try:
584 if type_option == 'dataset':
591 if type_option == 'dataset':
585 return getattr(self.ckan.action, 'package_create')(**kwargs)
592 return getattr(self.ckan.action, 'package_create')(**kwargs)
586 elif type_option == 'project':
593 elif type_option == 'project':
587 return getattr(self.ckan.action, 'organization_create')(**kwargs)
594 return getattr(self.ckan.action, 'organization_create')(**kwargs)
588 elif type_option == 'member':
595 elif type_option == 'member':
589 return getattr(self.ckan.action, 'organization_member_create')(**kwargs)
596 return getattr(self.ckan.action, 'organization_member_create')(**kwargs)
590 elif type_option == 'collaborator':
597 elif type_option == 'collaborator':
591 return getattr(self.ckan.action, 'package_collaborator_create')(**kwargs)
598 return getattr(self.ckan.action, 'package_collaborator_create')(**kwargs)
592 elif type_option == 'vocabulary':
599 elif type_option == 'vocabulary':
593 return getattr(self.ckan.action, 'vocabulary_create')(**kwargs)
600 return getattr(self.ckan.action, 'vocabulary_create')(**kwargs)
594 elif type_option == 'tag':
601 elif type_option == 'tag':
595 return getattr(self.ckan.action, 'tag_create')(**kwargs)
602 return getattr(self.ckan.action, 'tag_create')(**kwargs)
596 elif type_option == 'user':
603 elif type_option == 'user':
597 return getattr(self.ckan.action, 'user_create')(**kwargs)
604 return getattr(self.ckan.action, 'user_create')(**kwargs)
598 elif type_option == 'views':
605 elif type_option == 'views':
599 if 'resource' == select:
606 if 'resource' == select:
600 self.list = ['package']
607 self.list = ['package']
601 for key1, value1 in kwargs.items():
608 for key1, value1 in kwargs.items():
602 if not key1 in self.list:
609 if not key1 in self.list:
603 self.dict[key1] = value1
610 self.dict[key1] = value1
604 return getattr(self.ckan.action, 'resource_create_default_resource_views')(**self.dict)
611 return getattr(self.ckan.action, 'resource_create_default_resource_views')(**self.dict)
605 elif 'dataset' == select:
612 elif 'dataset' == select:
606 return getattr(self.ckan.action, 'package_create_default_resource_views')(**kwargs)
613 return getattr(self.ckan.action, 'package_create_default_resource_views')(**kwargs)
607 else:
614 else:
608 return 'ERROR:: "select = %s" is not accepted' % (select)
615 return 'ERROR:: "select = %s" is not accepted' % (select)
609 else:
616 else:
610 return 'ERROR:: "type_option = %s" is not accepted' % (type_option)
617 return 'ERROR:: "type_option = %s" is not accepted' % (type_option)
611 except:
618 except:
612 _, exc_value, _ = sys.exc_info()
619 _, exc_value, _ = sys.exc_info()
613 return exc_value
620 return exc_value
614 else:
621 else:
615 return 'ERROR:: "type_option" must be <str>'
622 return 'ERROR:: "type_option" must be <str>'
616
623
617 def patch(self, type_option, **kwargs):
624 def patch(self, type_option, **kwargs):
618 '''
625 '''
619 FINALIDAD:
626 FINALIDAD:
620 Funciones personalizadas para actualizar
627 Funciones personalizadas para actualizar
621
628
622 PARAMETROS DISPONIBLES:
629 PARAMETROS DISPONIBLES:
623 CONSULTAR: "GUIA DE SCRIPT.pdf"
630 CONSULTAR: "GUIA DE SCRIPT.pdf"
624
631
625 ESTRUCTURA:
632 ESTRUCTURA:
626 <access_name>.patch(type_option = <class 'str'>, param_1 = <class 'param_1'>, ...)
633 <access_name>.patch(type_option = <class 'str'>, param_1 = <class 'param_1'>, ...)
627 '''
634 '''
628 if type(type_option) is str:
635 if type(type_option) is str:
629 try:
636 try:
630 if type_option == 'dataset':
637 if type_option == 'dataset':
631 return getattr(self.ckan.action, 'package_patch')(**kwargs)
638 return getattr(self.ckan.action, 'package_patch')(**kwargs)
632 elif type_option == 'project':
639 elif type_option == 'project':
633 return getattr(self.ckan.action, 'organization_patch')(**kwargs)
640 return getattr(self.ckan.action, 'organization_patch')(**kwargs)
634 elif type_option == 'resource':
641 elif type_option == 'resource':
635 return getattr(self.ckan.action, 'resource_patch')(**kwargs)
642 return getattr(self.ckan.action, 'resource_patch')(**kwargs)
636 elif type_option == 'member':
643 elif type_option == 'member':
637 return getattr(self.ckan.action, 'organization_member_create')(**kwargs)
644 return getattr(self.ckan.action, 'organization_member_create')(**kwargs)
638 elif type_option == 'collaborator':
645 elif type_option == 'collaborator':
639 return getattr(self.ckan.action, 'package_collaborator_create')(**kwargs)
646 return getattr(self.ckan.action, 'package_collaborator_create')(**kwargs)
640 else:
647 else:
641 return 'ERROR:: "type_option = %s" is not accepted' % (type_option)
648 return 'ERROR:: "type_option = %s" is not accepted' % (type_option)
642 except:
649 except:
643 _, exc_value, _ = sys.exc_info()
650 _, exc_value, _ = sys.exc_info()
644 return exc_value
651 return exc_value
645 else:
652 else:
646 return 'ERROR:: "type_option" must be <str>'
653 return 'ERROR:: "type_option" must be <str>'
647
654
648 def delete(self, type_option, select=None, **kwargs):
655 def delete(self, type_option, select=None, **kwargs):
649 '''
656 '''
650 FINALIDAD:
657 FINALIDAD:
651 Función personalizada para eliminar y/o purgar.
658 Función personalizada para eliminar y/o purgar.
652
659
653 PARAMETROS DISPONIBLES:
660 PARAMETROS DISPONIBLES:
654 CONSULTAR: "GUIA DE SCRIPT.pdf"
661 CONSULTAR: "GUIA DE SCRIPT.pdf"
655
662
656 ESTRUCTURA:
663 ESTRUCTURA:
657 <access_name>.delete(type_option = <class 'str'>, param_1 = <class 'param_1'>, ...)
664 <access_name>.delete(type_option = <class 'str'>, param_1 = <class 'param_1'>, ...)
658 '''
665 '''
659 if type(type_option) is str:
666 if type(type_option) is str:
660 try:
667 try:
661 if type_option == 'dataset':
668 if type_option == 'dataset':
662 if select is None:
669 if select is None:
663 return 'ERROR:: "select" must not be "None"'
670 return 'ERROR:: "select" must not be "None"'
664 else:
671 else:
665 if 'delete' == select:
672 if 'delete' == select:
666 return getattr(self.ckan.action, 'package_delete')(**kwargs)
673 return getattr(self.ckan.action, 'package_delete')(**kwargs)
667 elif 'purge' == select:
674 elif 'purge' == select:
668 return getattr(self.ckan.action, 'dataset_purge')(**kwargs)
675 return getattr(self.ckan.action, 'dataset_purge')(**kwargs)
669 else:
676 else:
670 return 'ERROR:: "select = %s" is not accepted' % (select)
677 return 'ERROR:: "select = %s" is not accepted' % (select)
671 elif type_option == 'project':
678 elif type_option == 'project':
672 if select is None:
679 if select is None:
673 return 'ERROR:: "select" must not be "None"'
680 return 'ERROR:: "select" must not be "None"'
674 else:
681 else:
675 if 'delete' == select:
682 if 'delete' == select:
676 return getattr(self.ckan.action, 'organization_delete')(**kwargs)
683 return getattr(self.ckan.action, 'organization_delete')(**kwargs)
677 elif 'purge' == select:
684 elif 'purge' == select:
678 return getattr(self.ckan.action, 'organization_purge')(**kwargs)
685 return getattr(self.ckan.action, 'organization_purge')(**kwargs)
679 else:
686 else:
680 return 'ERROR:: "select = %s" is not accepted' % (select)
687 return 'ERROR:: "select = %s" is not accepted' % (select)
681 elif type_option == 'resource':
688 elif type_option == 'resource':
682 return getattr(self.ckan.action, 'resource_delete')(**kwargs)
689 return getattr(self.ckan.action, 'resource_delete')(**kwargs)
683 elif type_option == 'vocabulary':
690 elif type_option == 'vocabulary':
684 return getattr(self.ckan.action, 'vocabulary_delete')(**kwargs)
691 return getattr(self.ckan.action, 'vocabulary_delete')(**kwargs)
685 elif type_option == 'tag':
692 elif type_option == 'tag':
686 return getattr(self.ckan.action, 'tag_delete')(**kwargs)
693 return getattr(self.ckan.action, 'tag_delete')(**kwargs)
687 elif type_option == 'user':
694 elif type_option == 'user':
688 return getattr(self.ckan.action, 'user_delete')(**kwargs)
695 return getattr(self.ckan.action, 'user_delete')(**kwargs)
689 else:
696 else:
690 return 'ERROR:: "type_option = %s" is not accepted' % (type_option)
697 return 'ERROR:: "type_option = %s" is not accepted' % (type_option)
691 except:
698 except:
692 _, exc_value, _ = sys.exc_info()
699 _, exc_value, _ = sys.exc_info()
693 return exc_value
700 return exc_value
694 else:
701 else:
695 return 'ERROR:: "type_option" must be <str>'
702 return 'ERROR:: "type_option" must be <str>'
696
703
697 def f_status_note(self, total, result, path):
704 def f_status_note(self, total, result, path):
698 file_txt = open(path+'status_note.txt', 'w')
705 file_txt = open(path+'status_note.txt', 'w')
699 file_txt = open(path+'status_note.txt', 'a')
706 file_txt = open(path+'status_note.txt', 'a')
700
707
701 file_txt.write('DOWNLOADED FILE(S): "%s"' % (len(result['name'])))
708 file_txt.write('DOWNLOADED FILE(S): "%s"' % (len(result['name'])))
702 file_txt.write(''+ os.linesep)
709 file_txt.write(''+ os.linesep)
703 for u in result['name']:
710 for u in result['name']:
704 file_txt.write(' - '+ u + os.linesep)
711 file_txt.write(' - '+ u + os.linesep)
705 file_txt.write(''+ os.linesep)
712 file_txt.write(''+ os.linesep)
706
713
707 file_txt.write('FAILED FILE(S): "%s"' % (len(total['name'])-len(result['name'])))
714 file_txt.write('FAILED FILE(S): "%s"' % (len(total['name'])-len(result['name'])))
708 file_txt.write(''+ os.linesep)
715 file_txt.write(''+ os.linesep)
709 if len(total['name'])-len(result['name']) != 0:
716 if len(total['name'])-len(result['name']) != 0:
710 for u in total['name']:
717 for u in total['name']:
711 if not u in result['name']:
718 if not u in result['name']:
712 file_txt.write(' - '+ u + os.linesep)
719 file_txt.write(' - '+ u + os.linesep)
713 else:
720 else:
714 file_txt.write(' "None"'+ os.linesep)
721 file_txt.write(' "None"'+ os.linesep)
715
722
716 def f_name(self, name_dataset, ext, tempdir):
723 def f_name(self, name_dataset, ext, tempdir):
717 while self.check:
724 while self.check:
718 self.str = ''
725 self.str = ''
719 if self.cont == 0:
726 if self.cont == 0:
720 if os.path.exists(tempdir + name_dataset + ext):
727 if os.path.exists(tempdir + name_dataset + ext):
721 self.str = name_dataset+'('+str(self.cont+1)+')'+ext
728 self.str = name_dataset+'('+str(self.cont+1)+')'+ext
722 else:
729 else:
723 self.check = self.check * 0
730 self.check = self.check * 0
724 self.str = name_dataset + ext
731 self.str = name_dataset + ext
725 else:
732 else:
726 if not os.path.exists(tempdir + name_dataset+'('+str(self.cont)+')'+ext):
733 if not os.path.exists(tempdir + name_dataset+'('+str(self.cont)+')'+ext):
727 self.check = self.check * 0
734 self.check = self.check * 0
728 self.str = name_dataset+'('+str(self.cont)+')'+ ext
735 self.str = name_dataset+'('+str(self.cont)+')'+ ext
729 self.cont = self.cont+1
736 self.cont = self.cont+1
730 return self.str
737 return self.str
731
738
732 def f_zipdir(self, path, ziph, zip_name):
739 def f_zipdir(self, path, ziph, zip_name):
733 for root, _, files in os.walk(path):
740 for root, _, files in os.walk(path):
734 print('.....')
741 print('.....')
735 print('Creating: "{}" >>'.format(zip_name))
742 print('Creating: "{}" >>'.format(zip_name))
736 for __file in tqdm(iterable=files, total=len(files)):
743 for __file in tqdm(iterable=files, total=len(files)):
737 new_dir = os.path.relpath(os.path.join(root, __file), os.path.join(path, '..'))
744 new_dir = os.path.relpath(os.path.join(root, __file), os.path.join(path, '..'))
738 ziph.write(os.path.join(root, __file), new_dir)
745 ziph.write(os.path.join(root, __file), new_dir)
739 print('Created >>')
746 print('Created >>')
740
747
741 def download_by_step(self, response, tempdir_name):
748 def download_by_step(self, response, tempdir_name):
742 try:
749 try:
743 with requests.get(response['url'], stream=True, headers={'Authorization': self.Authorization}) as resp:
750 with requests.get(response['url'], stream=True, headers={'Authorization': self.Authorization}) as resp:
744 if resp.status_code == 200:
751 if resp.status_code == 200:
745 with open(tempdir_name+response['name'], 'wb') as file:
752 with open(tempdir_name+response['name'], 'wb') as file:
746 for chunk in resp.iter_content(chunk_size = self.chunk_size):
753 for chunk in resp.iter_content(chunk_size = self.chunk_size):
747 if chunk:
754 if chunk:
748 file.write(chunk)
755 file.write(chunk)
749 except requests.exceptions.RequestException:
756 except requests.exceptions.RequestException:
750 pass
757 pass
751
758
752 def download_files(self, **kwargs):
759 def download_files(self, **kwargs):
753 '''
760 '''
754 FINALIDAD:
761 FINALIDAD:
755 Funcion personalizada para la descarga de archivos existentes de un dataset.
762 Funcion personalizada para la descarga de archivos existentes de un dataset.
756
763
757 PARAMETROS DISPONIBLES:
764 PARAMETROS DISPONIBLES:
758 CONSULTAR: "GUIA DE SCRIPT.pdf"
765 CONSULTAR: "GUIA DE SCRIPT.pdf"
759
766
760 ESTRUCTURA:
767 ESTRUCTURA:
761 <access_name>.download_files(id = <class 'str'>, param_1 = <class 'param_1'>, ...)
768 <access_name>.download_files(id = <class 'str'>, param_1 = <class 'param_1'>, ...)
762 '''
769 '''
763 dict_local = {}
770 dict_local = {}
764 #----------------------------------------------#
771 #----------------------------------------------#
765 if 'zip' in kwargs:
772 if 'zip' in kwargs:
766 if type(kwargs['zip']) is not bool:
773 if type(kwargs['zip']) is not bool:
767 return 'ERROR:: "zip" must be: <class "bool">'
774 return 'ERROR:: "zip" must be: <class "bool">'
768 else:
775 else:
769 dict_local['zip'] = kwargs['zip']
776 dict_local['zip'] = kwargs['zip']
770 else:
777 else:
771 dict_local['zip'] = False
778 dict_local['zip'] = False
772 #----------------------------------------------#
779 #----------------------------------------------#
773 if 'status_note' in kwargs:
780 if 'status_note' in kwargs:
774 if type(kwargs['status_note']) is not bool:
781 if type(kwargs['status_note']) is not bool:
775 return 'ERROR:: "status_note" must be: <class "bool">'
782 return 'ERROR:: "status_note" must be: <class "bool">'
776 else:
783 else:
777 dict_local['status_note'] = kwargs['status_note']
784 dict_local['status_note'] = kwargs['status_note']
778 else:
785 else:
779 dict_local['status_note'] = False
786 dict_local['status_note'] = False
780 #----------------------------------------------#
787 #----------------------------------------------#
781 if 'path' in kwargs:
788 if 'path' in kwargs:
782 if type(kwargs['path']) is str:
789 if type(kwargs['path']) is str:
783 if os.path.isdir(kwargs['path']) == False:
790 if os.path.isdir(kwargs['path']) == False:
784 return 'ERROR:: "path" does not exist'
791 return 'ERROR:: "path" does not exist'
785 else:
792 else:
786 if kwargs['path'][-1:] != self.separator:
793 if kwargs['path'][-1:] != self.separator:
787 dict_local['path'] = kwargs['path']+self.separator
794 dict_local['path'] = kwargs['path']+self.separator
788 else:
795 else:
789 dict_local['path'] = kwargs['path']
796 dict_local['path'] = kwargs['path']
790
797
791 txt = dict_local['path']+datetime.now().strftime("%Y_%m_%d_%H_%M_%S_%f")+'.txt'
798 txt = dict_local['path']+datetime.now().strftime("%Y_%m_%d_%H_%M_%S_%f")+'.txt'
792 if int(platform.python_version()[0]) == 3:
799 if int(platform.python_version()[0]) == 3:
793 try:
800 try:
794 file_txt = open(txt, 'w')
801 file_txt = open(txt, 'w')
795 file_txt.close()
802 file_txt.close()
796 os.remove(txt)
803 os.remove(txt)
797 except PermissionError:
804 except PermissionError:
798 return 'ERROR:: Access denied, you are not authorized to write files: "%s"' % (dict_local['path'])
805 return 'ERROR:: Access denied, you are not authorized to write files: "%s"' % (dict_local['path'])
799 else:
806 else:
800 try:
807 try:
801 file_txt = open(txt, 'w')
808 file_txt = open(txt, 'w')
802 file_txt.close()
809 file_txt.close()
803 os.remove(txt)
810 os.remove(txt)
804 except:
811 except:
805 return 'ERROR:: Access denied, you are not authorized to write files: "%s"' % (dict_local['path'])
812 return 'ERROR:: Access denied, you are not authorized to write files: "%s"' % (dict_local['path'])
806 else:
813 else:
807 return 'ERROR:: "path" must be: <class "str">'
814 return 'ERROR:: "path" must be: <class "str">'
808 else:
815 else:
809 dict_local['path'] = ''
816 dict_local['path'] = ''
810 #----------------------------------------------#
817 #----------------------------------------------#
811 for key, value in kwargs.items():
818 for key, value in kwargs.items():
812 if not key in dict_local:
819 if not key in dict_local:
813 self.dict[key] = value
820 self.dict[key] = value
814 try:
821 try:
815 response = getattr(self.ckan.action, 'url_resources')(**self.dict)
822 response = getattr(self.ckan.action, 'url_resources')(**self.dict)
816 except:
823 except:
817 _, exc_value, _ = sys.exc_info()
824 _, exc_value, _ = sys.exc_info()
818 return exc_value
825 return exc_value
819
826
820 if len(response) != 0:
827 if len(response) != 0:
821 #--------------TEMP PATH---------------#
828 #--------------TEMP PATH---------------#
822 if dict_local['zip']:
829 if dict_local['zip']:
823 tempdir = tempfile.mkdtemp(prefix=kwargs['id']+'-')+self.separator
830 tempdir = tempfile.mkdtemp(prefix=kwargs['id']+'-')+self.separator
824 os.mkdir(tempdir+kwargs['id'])
831 os.mkdir(tempdir+kwargs['id'])
825 dir_name = tempdir + kwargs['id'] + self.separator
832 dir_name = tempdir + kwargs['id'] + self.separator
826 else:
833 else:
827 dir = self.f_name(kwargs['id'], '', dict_local['path'])
834 dir = self.f_name(kwargs['id'], '', dict_local['path'])
828 os.mkdir(dict_local['path'] + dir)
835 os.mkdir(dict_local['path'] + dir)
829 dir_name = dict_local['path'] + dir + self.separator
836 dir_name = dict_local['path'] + dir + self.separator
830 #-----------DOWNLOAD FILES-------------#
837 #-----------DOWNLOAD FILES-------------#
831 print('.....')
838 print('.....')
832 print('Downloading "{}" file(s) >>'.format(len(response)))
839 print('Downloading "{}" file(s) >>'.format(len(response)))
833 name_total = {'name': []}
840 name_total = {'name': []}
834 with concurrent.futures.ThreadPoolExecutor() as executor:
841 with concurrent.futures.ThreadPoolExecutor() as executor:
835 for u in tqdm(iterable=response, total=len(response)):
842 for u in tqdm(iterable=response, total=len(response)):
836 name_total['name'].append(u['name'])
843 name_total['name'].append(u['name'])
837 executor.submit(self.download_by_step, u, dir_name)
844 executor.submit(self.download_by_step, u, dir_name)
838 name_check = {}
845 name_check = {}
839 name_check['name'] = [f for f in os.listdir(dir_name) if os.path.isfile(os.path.join(dir_name, f))]
846 name_check['name'] = [f for f in os.listdir(dir_name) if os.path.isfile(os.path.join(dir_name, f))]
840 print('"{}" downloaded file(s) successfully >>'.format(len(name_check['name'])))
847 print('"{}" downloaded file(s) successfully >>'.format(len(name_check['name'])))
841 #--------------------------------------#
848 #--------------------------------------#
842 if len(name_check['name']) != 0:
849 if len(name_check['name']) != 0:
843 #----------Status Note---------#
850 #----------Status Note---------#
844 if dict_local['status_note']:
851 if dict_local['status_note']:
845 print('.....')
852 print('.....')
846 print('Creating: "status_note.txt" >>')
853 print('Creating: "status_note.txt" >>')
847 self.f_status_note(name_total, name_check, dir_name)
854 self.f_status_note(name_total, name_check, dir_name)
848 print('Created>>')
855 print('Created>>')
849 #----------ZIP CREATE----------#
856 #----------ZIP CREATE----------#
850 if dict_local['zip']:
857 if dict_local['zip']:
851 zip_name = self.f_name(kwargs['id'], '.zip', dict_local['path'])
858 zip_name = self.f_name(kwargs['id'], '.zip', dict_local['path'])
852 ziph = zipfile.ZipFile(dict_local['path'] + zip_name, 'w', zipfile.ZIP_DEFLATED, allowZip64=True)
859 ziph = zipfile.ZipFile(dict_local['path'] + zip_name, 'w', zipfile.ZIP_DEFLATED, allowZip64=True)
853 self.f_zipdir(dir_name, ziph, zip_name)
860 self.f_zipdir(dir_name, ziph, zip_name)
854 ziph.close()
861 ziph.close()
855 #Delete Temporal Path
862 #Delete Temporal Path
856 if os.path.exists(tempdir[:-1]):
863 if os.path.exists(tempdir[:-1]):
857 shutil.rmtree(tempdir[:-1])
864 shutil.rmtree(tempdir[:-1])
858 #------------------------------#
865 #------------------------------#
859 print('.....')
866 print('.....')
860 return 'DOWNLOAD FINISHED'
867 return 'DOWNLOAD FINISHED'
861 else:
868 else:
862 #Delete Temporal Path
869 #Delete Temporal Path
863 if dict_local['zip']:
870 if dict_local['zip']:
864 if os.path.exists(tempdir[:-1]):
871 if os.path.exists(tempdir[:-1]):
865 shutil.rmtree(tempdir[:-1])
872 shutil.rmtree(tempdir[:-1])
866 else:
873 else:
867 if os.path.exists(dir_name[:-1]):
874 if os.path.exists(dir_name[:-1]):
868 shutil.rmtree(dir_name[:-1])
875 shutil.rmtree(dir_name[:-1])
869 return 'NO FILES WERE DOWNLOADED'
876 return 'NO FILES WERE DOWNLOADED'
870 else:
877 else:
871 return 'FILES NOT FOUND' No newline at end of file
878 return 'FILES NOT FOUND'
General Comments 0
You need to be logged in to leave comments. Login now