##// END OF EJS Templates
DATABASES » ckanext-jro » api-cliente
RSS
v2.9.2 :: Delete 'format' automatically
eynilupu -
r7:9ab1ac032168
parent child
Show More
Show file before | Show file after | Hide comments
1 NO CONTENT: modified file, binary diff hidden
NO CONTENT: modified file, binary diff hidden
Show file before | Show file after | Hide comments
@@ -1,878 +1,878
1 from ckanapi import RemoteCKAN
1 from ckanapi import RemoteCKAN
2 from datetime import datetime
2 from datetime import datetime
3 from tqdm import tqdm
3 from tqdm import tqdm
4 #from ckanapi.errors import NotAuthorized, NotFound, ValidationError, SearchQueryError, SearchError, CKANAPIError, ServerIncompatibleError
4 #from ckanapi.errors import NotAuthorized, NotFound, ValidationError, SearchQueryError, SearchError, CKANAPIError, ServerIncompatibleError
5 import sys
5 import sys
6 import platform
6 import platform
7 import os
7 import os
8 import tempfile
8 import tempfile
9 import shutil
9 import shutil
10 import zipfile
10 import zipfile
11 import concurrent.futures
11 import concurrent.futures
12 import requests
12 import requests
13 import json
13 import json
14 import pathlib
14 #import pathlib
15 import uuid
15 import uuid
16
16
17 class JROAPI():
17 class JROAPI():
18 """
18 """
19 FINALIDAD:
19 FINALIDAD:
20 Script para administrar y obtener la data del repositorio por medio de APIs.
20 Script para administrar y obtener la data del repositorio por medio de APIs.
21
21
22 REQUISITIOS PREVIOS:
22 REQUISITIOS PREVIOS:
23 - Paso 1: Tener "pip [Python 2]" o "pip3 [Python 3]" instalado:
23 - Paso 1: Tener "pip [Python 2]" o "pip3 [Python 3]" instalado:
24 - Paso 2: Instalar lo siguiente como admininstrador:
24 - Paso 2: Instalar lo siguiente como admininstrador:
25 En Python 2
25 En Python 2
26 - pip install ckanapi==4.5
26 - pip install ckanapi==4.5
27 - pip install requests
27 - pip install requests
28 - pip install pathlib
29 - pip install futures
28 - pip install futures
30 - pip install tqdm
29 - pip install tqdm
31 En Python > 3
30 En Python > 3
32 - pip3 install ckanapi==4.5
31 - pip3 install ckanapi==4.5
33 - pip3 install requests
32 - pip3 install requests
34 - pip3 install tqdm
33 - pip3 install tqdm
35
34
36 FUNCIONES DISPONIBLES:
35 FUNCIONES DISPONIBLES:
37 - action
36 - action
38 - upload_file
37 - upload_file
39 - upload_multiple_files
38 - upload_multiple_files
40 - upload_multiple_files_advance
39 - upload_multiple_files_advance
41 - show
40 - show
42 - search
41 - search
43 - create
42 - create
44 - patch
43 - patch
45 - delete
44 - delete
46 - download_files
45 - download_files
47
46
48 EJEMPLOS:
47 EJEMPLOS:
49 #1:
48 #1:
50 with JROAPI('http://demo.example.com', Authorization='#########') as <access_name>:
49 with JROAPI('http://demo.example.com', Authorization='#########') as <access_name>:
51 ... some operation(s) ...
50 ... some operation(s) ...
52 #2:
51 #2:
53 <access_name> = JROAPI('http://example.com', Authorization='#########')
52 <access_name> = JROAPI('http://example.com', Authorization='#########')
54 ... some operation(s) ...
53 ... some operation(s) ...
55 <access_name>.ckan.close()
54 <access_name>.ckan.close()
56
55
57 REPORTAR ALGUN PROBLEMA:
56 REPORTAR ALGUN PROBLEMA:
58 Debe enviar un correo a eynilupu@igp.gob.pe detallando los siguientes pasos:
57 Debe enviar un correo a eynilupu@igp.gob.pe detallando los siguientes pasos:
59 1) Correo para contactarlo
58 1) Correo para contactarlo
60 2) Descripcion del problema
59 2) Descripcion del problema
61 3) ¿En que paso o seccion encontro el problema?
60 3) ¿En que paso o seccion encontro el problema?
62 4) ¿Cual era el resultado que usted esperaba?
61 4) ¿Cual era el resultado que usted esperaba?
63 """
62 """
64 def __init__(self, url, Authorization=None):
63 def __init__(self, url, Authorization=None):
65 ua = 'CKAN_JRO/1.1 (+'+str(url)+')'
64 ua = 'CKAN_JRO/1.1 (+'+str(url)+')'
66 #ua = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36'
65 #ua = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36'
67 self.ckan = RemoteCKAN(url, apikey=Authorization, user_agent=ua)
66 self.ckan = RemoteCKAN(url, apikey=Authorization, user_agent=ua)
68 #self.ckan = RemoteCKAN(url, apikey=Authorization)
67 #self.ckan = RemoteCKAN(url, apikey=Authorization)
69 self.Authorization = Authorization
68 self.Authorization = Authorization
69 # Change for --> self.separator = os.sep
70 if platform.system() == 'Windows':
70 if platform.system() == 'Windows':
71 self.separator = '\\'
71 self.separator = '\\'
72 else:
72 else:
73 self.separator = '/'
73 self.separator = '/'
74
74
75 self.chunk_size = 1024
75 self.chunk_size = 1024
76 self.list = []
76 self.list = []
77 self.dict = {}
77 self.dict = {}
78 self.str = ''
78 self.str = ''
79 self.check = 1
79 self.check = 1
80 self.cont = 0
80 self.cont = 0
81
81
82 def __enter__(self):
82 def __enter__(self):
83 return self
83 return self
84
84
85 def __exit__(self, *args):
85 def __exit__(self, *args):
86 self.ckan.close()
86 self.ckan.close()
87
87
88 def action(self, action, **kwargs):
88 def action(self, action, **kwargs):
89 """
89 """
90 FINALIDAD:
90 FINALIDAD:
91 Funcion para llamar a las APIs disponibles
91 Funcion para llamar a las APIs disponibles
92
92
93 APIs DISPONIBLES:
93 APIs DISPONIBLES:
94 CONSULTAR: "GUIA DE SCRIPT.pdf"
94 CONSULTAR: "GUIA DE SCRIPT.pdf"
95
95
96 EJEMPLO:
96 EJEMPLO:
97 <access_name>.action(<consuming API>, param_1 = <class 'param_1'>, ...)
97 <access_name>.action(<consuming API>, param_1 = <class 'param_1'>, ...)
98 """
98 """
99 #--------------- CASE: PACKAGE SEARCH ---------------#
99 #--------------- CASE: PACKAGE SEARCH ---------------#
100 if kwargs is not None:
100 if kwargs is not None:
101 if action == 'package_search':
101 if action == 'package_search':
102 self.list = ['facet_mincount', 'facet_limit', 'facet_field']
102 self.list = ['facet_mincount', 'facet_limit', 'facet_field']
103 for facet in self.list:
103 for facet in self.list:
104 if facet in kwargs:
104 if facet in kwargs:
105 kwargs[facet.replace('_', '.')] = kwargs[facet]
105 kwargs[facet.replace('_', '.')] = kwargs[facet]
106 kwargs.pop(facet)
106 kwargs.pop(facet)
107 #----------------------------------------------------#
107 #----------------------------------------------------#
108 try:
108 try:
109 return getattr(self.ckan.action, action)(**kwargs)
109 return getattr(self.ckan.action, action)(**kwargs)
110 except:
110 except:
111 _, exc_value, _ = sys.exc_info()
111 _, exc_value, _ = sys.exc_info()
112 return exc_value
112 return exc_value
113
113
114 def upload_file(self, dataset_id, file_path, file_date, file_type, **kwargs):
114 def upload_file(self, dataset_id, file_path, file_date, file_type, **kwargs):
115 # Agregar si es interruptido por teclado
115 # Agregar si es interruptido por teclado
116 '''
116 '''
117 FINALIDAD:
117 FINALIDAD:
118 Funcion para subir un unico archivo al repositorio del ROJ.
118 Funcion para subir un unico archivo al repositorio del ROJ.
119
119
120 PARAMETROS DISPONIBLES:
120 PARAMETROS DISPONIBLES:
121 CONSULTAR: "GUIA DE SCRIPT.pdf"
121 CONSULTAR: "GUIA DE SCRIPT.pdf"
122
122
123 ESTRUCTURA:
123 ESTRUCTURA:
124 <access_name>.upload_file(dataset_id = <class 'str'>, file_date = <class 'str'>, file_path = <class 'str'>, file_type = <class 'str'>, param_1 = <class 'param_1'>, ...)
124 <access_name>.upload_file(dataset_id = <class 'str'>, file_date = <class 'str'>, file_path = <class 'str'>, file_type = <class 'str'>, param_1 = <class 'param_1'>, ...)
125 '''
125 '''
126 self.list = ['package_id', 'upload', 'voc_file_type', 'name'] #file_date
126 self.list = ['package_id', 'upload', 'voc_file_type', 'name'] #file_date
127 for key1, value1 in kwargs.items():
127 for key1, value1 in kwargs.items():
128 if not key1 in self.list:
128 if not key1 in self.list:
129 self.dict[key1] = value1
129 self.dict[key1] = value1
130
130
131 #---------------------------#
131 #---------------------------#
132 if not 'others' in kwargs:
132 if not 'others' in kwargs:
133 self.dict['others'] = ''
133 self.dict['others'] = ''
134 else:
134 else:
135 if isinstance(kwargs['others'], list):
135 if isinstance(kwargs['others'], list):
136 self.dict['others'] = json.dumps(kwargs['others'])
136 self.dict['others'] = json.dumps(kwargs['others'])
137 #---------------------------#
137 #---------------------------#
138
138
139 if not os.path.isfile(file_path):
139 if not os.path.isfile(file_path):
140 return 'File "%s" not exist' % (file_path)
140 return 'File "%s" not exist' % (file_path)
141
141
142 if not 'format' in self.dict:
142 #if not 'format' in self.dict:
143 self.str = ''.join(pathlib.Path(file_path).suffixes)
143 # self.str = ''.join(pathlib.Path(file_path).suffixes)
144 if len(self.str) > 0:
144 # if len(self.str) > 0:
145 self.dict['format'] = self.str.upper()[1:]
145 # self.dict['format'] = self.str.upper()[1:]
146
146
147 try:
147 try:
148 return getattr(self.ckan.action, 'resource_create')(package_id=dataset_id, file_date=file_date, upload=open(file_path, 'rb'), voc_file_type=file_type, name=pathlib.Path(file_path).name, **self.dict)
148 return getattr(self.ckan.action, 'resource_create')(package_id=dataset_id, file_date=file_date, upload=open(file_path, 'rb'), voc_file_type=file_type, name=os.path.basename(file_path), **self.dict)
149 except:
149 except:
150 _, exc_value, _ = sys.exc_info()
150 _, exc_value, _ = sys.exc_info()
151 return exc_value
151 return exc_value
152
152
153 def upload_multiple_files_advance(self, dataset_id, path_files, file_date, file_type, max_size=100, max_count=500, ignore_repetition=False, **kwargs):
153 def upload_multiple_files_advance(self, dataset_id, path_files, file_date, file_type, max_size=100, max_count=500, ignore_repetition=False, **kwargs):
154 # Agregar si es interruptido por teclado
154 # Agregar si es interruptido por teclado
155 '''
155 '''
156 FINALIDAD:
156 FINALIDAD:
157 Funcion para subir multiples archivos al repositorio del ROJ.
157 Funcion para subir multiples archivos al repositorio del ROJ.
158
158
159 PARAMETROS DISPONIBLES:
159 PARAMETROS DISPONIBLES:
160 CONSULTAR: "GUIA DE SCRIPT.pdf"
160 CONSULTAR: "GUIA DE SCRIPT.pdf"
161
161
162 ESTRUCTURA:
162 ESTRUCTURA:
163 <access_name>.upload_multiple_files_advance(dataset_id = <class 'str'>, path_files = <class 'list of strings'>, file_date = <class 'str'>, file_type = <class 'str'>, param_1 = <class 'param_1'>, ...)
163 <access_name>.upload_multiple_files_advance(dataset_id = <class 'str'>, path_files = <class 'list of strings'>, file_date = <class 'str'>, file_type = <class 'str'>, param_1 = <class 'param_1'>, ...)
164 '''
164 '''
165 #-------------------------PACKAGE SHOW-----------------------#
165 #-------------------------PACKAGE SHOW-----------------------#
166 try:
166 try:
167 dataset_show = getattr(self.ckan.action, 'package_show')(id=dataset_id)['resources']
167 dataset_show = getattr(self.ckan.action, 'package_show')(id=dataset_id)['resources']
168 except:
168 except:
169 _, exc_value, _ = sys.exc_info()
169 _, exc_value, _ = sys.exc_info()
170 print('ERROR obtaining metadata dataset:: Use the "print" for more information')
170 print('ERROR obtaining metadata dataset:: Use the "print" for more information')
171 return exc_value
171 return exc_value
172 #------------------------------------------------------------#
172 #------------------------------------------------------------#
173 resources_name = []
173 resources_name = []
174 for u in dataset_show:
174 for u in dataset_show:
175 resources_name.append(u['name'].lower())
175 resources_name.append(u['name'].lower())
176 #------------------------------------------------------------#
176 #------------------------------------------------------------#
177 self.list = ['package_id', 'upload', 'voc_file_type', 'name']
177 self.list = ['package_id', 'upload', 'voc_file_type', 'name']
178 for key1, value1 in kwargs.items():
178 for key1, value1 in kwargs.items():
179 if not key1 in self.list:
179 if not key1 in self.list:
180 self.dict[key1] = value1
180 self.dict[key1] = value1
181 #------------------------------------------------------------#
181 #------------------------------------------------------------#
182 if not 'others' in kwargs:
182 if not 'others' in kwargs:
183 self.dict['others'] = ''
183 self.dict['others'] = ''
184 else:
184 else:
185 if isinstance(kwargs['others'], list):
185 if isinstance(kwargs['others'], list):
186 self.dict['others'] = json.dumps(kwargs['others'])
186 self.dict['others'] = json.dumps(kwargs['others'])
187 #------------------------------------------------------------#
187 #------------------------------------------------------------#
188 total_list = []
188 total_list = []
189 #---------------CASO : "path" or "path_list"-----------------#
189 #---------------CASO : "path" or "path_list"-----------------#
190 if type(path_files) is list:
190 if type(path_files) is list:
191 if len(path_files) != 0:
191 if len(path_files) != 0:
192 path_files.sort()
192 path_files.sort()
193 for u in path_files:
193 for u in path_files:
194 if os.path.isfile(u):
194 if os.path.isfile(u):
195 if pathlib.Path(u).name.lower() in resources_name:
195 if os.path.basename(u).lower() in resources_name:
196 if not ignore_repetition:
196 if not ignore_repetition:
197 return 'ERROR:: "%s" file already exist in this dataset' % (pathlib.Path(u).name)
197 return 'ERROR:: "%s" file already exist in this dataset' % (os.path.basename(u))
198 print('WARRING:: "'+ str(pathlib.Path(u).name) +'" file was ignored because already exist in this dataset')
198 print('WARRING:: "'+ str(os.path.basename(u)) +'" file was ignored because already exist in this dataset')
199 else:
199 else:
200 total_list.append({'name':pathlib.Path(u).name, 'size': os.stat(u).st_size, 'upload':open(u, 'rb')})
200 total_list.append({'name':os.path.basename(u), 'size': os.stat(u).st_size, 'upload':open(u, 'rb')})
201 else:
201 else:
202 return 'File "%s" does not exist' % (u)
202 return 'File "%s" does not exist' % (u)
203 else:
203 else:
204 return 'ERROR:: "path_list is empty"'
204 return 'ERROR:: "path_list is empty"'
205
205
206 elif type(path_files) is str:
206 elif type(path_files) is str:
207 if os.path.isdir(path_files):
207 if os.path.isdir(path_files):
208 path_order = [f for f in os.listdir(path_files) if os.path.isfile(os.path.join(path_files, f))]
208 path_order = [f for f in os.listdir(path_files) if os.path.isfile(os.path.join(path_files, f))]
209 path_order.sort()
209 path_order.sort()
210 if path_order:
210 if path_order:
211 for name in path_order:
211 for name in path_order:
212 if name.lower() in resources_name:
212 if name.lower() in resources_name:
213 if not ignore_repetition:
213 if not ignore_repetition:
214 return 'ERROR:: "%s" file already exist in this dataset' % (name)
214 return 'ERROR:: "%s" file already exist in this dataset' % (name)
215 print('WARRING:: "'+ name +'" file was ignored because already exist in this dataset')
215 print('WARRING:: "'+ name +'" file was ignored because already exist in this dataset')
216 else:
216 else:
217 total_list.append({'name':name, 'size': os.stat(os.path.join(path_files, name)).st_size, 'upload':open(os.path.join(path_files, name), 'rb')})
217 total_list.append({'name':name, 'size': os.stat(os.path.join(path_files, name)).st_size, 'upload':open(os.path.join(path_files, name), 'rb')})
218 else:
218 else:
219 return "ERROR:: There aren't files in this directory"
219 return "ERROR:: There aren't files in this directory"
220 else:
220 else:
221 return 'ERROR:: Directory "%s" does not exist' % (path_files)
221 return 'ERROR:: Directory "%s" does not exist' % (path_files)
222 else:
222 else:
223 return 'ERROR:: "path_files" must be a str or list'
223 return 'ERROR:: "path_files" must be a str or list'
224 #------------------------------------------------------------#
224 #------------------------------------------------------------#
225 try:
225 try:
226 uuid.UUID(str(dataset_id), version=4)
226 uuid.UUID(str(dataset_id), version=4)
227 package_id_or_name = '"id": "' + str(dataset_id) + '"'
227 package_id_or_name = '"id": "' + str(dataset_id) + '"'
228 except ValueError:
228 except ValueError:
229 package_id_or_name = '"name": "' + str(dataset_id) + '"'
229 package_id_or_name = '"name": "' + str(dataset_id) + '"'
230 #------------------------------------------------------------#
230 #------------------------------------------------------------#
231 blocks = [[]]
231 blocks = [[]]
232 size_file = 0
232 size_file = 0
233 count_file = 0
233 count_file = 0
234 inter_num = 0
234 inter_num = 0
235 for value in total_list:
235 for value in total_list:
236 if value['size'] > 1024 * 1024 * float(max_size):
236 if value['size'] > 1024 * 1024 * float(max_size):
237 return 'ERROR:: The size of the "%s" file is %sMB aprox, please change "max_size" value' % (value['name'], str(round(value['size']/(1024 * 1024), 2)))
237 return 'ERROR:: The size of the "%s" file is %sMB aprox, please change "max_size" value' % (value['name'], str(round(value['size']/(1024 * 1024), 2)))
238 if not 1 <= int(max_count) <= 999:
238 if not 1 <= int(max_count) <= 999:
239 return 'ERROR:: The count of the number of files must be between 1 and 999, please change "max_count" value'
239 return 'ERROR:: The count of the number of files must be between 1 and 999, please change "max_count" value'
240
240
241 size_file = size_file + value['size']
241 size_file = size_file + value['size']
242 count_file = count_file + 1
242 count_file = count_file + 1
243 if size_file <= 1024 * 1024 * float(max_size) and count_file <= int(max_count):
243 if size_file <= 1024 * 1024 * float(max_size) and count_file <= int(max_count):
244 del value['size']
244 del value['size']
245 blocks[inter_num].append(value)
245 blocks[inter_num].append(value)
246 else:
246 else:
247 inter_num = inter_num + 1
247 inter_num = inter_num + 1
248 size_file = value['size']
248 size_file = value['size']
249 count_file = 1
249 count_file = 1
250 blocks.append([])
250 blocks.append([])
251 del value['size']
251 del value['size']
252 blocks[inter_num].append(value)
252 blocks[inter_num].append(value)
253 #------------------------------------------------------------#
253 #------------------------------------------------------------#
254 if len(blocks[0]) > 0:
254 if len(blocks[0]) > 0:
255 print('BLOCK(S) IN TOTAL:: {}'.format(len(blocks)))
255 print('BLOCK(S) IN TOTAL:: {}'.format(len(blocks)))
256 for count1, block in enumerate(blocks):
256 for count1, block in enumerate(blocks):
257 print('---- BLOCK N°{} ----'.format(count1 + 1))
257 print('---- BLOCK N°{} ----'.format(count1 + 1))
258 resource_extend = []
258 resource_extend = []
259 files_dict = {}
259 files_dict = {}
260 for count2, value2 in enumerate(block):
260 for count2, value2 in enumerate(block):
261 value2['file_date'] = file_date
261 value2['file_date'] = file_date
262 value2['voc_file_type'] = file_type
262 value2['voc_file_type'] = file_type
263 value2.update(self.dict)
263 value2.update(self.dict)
264
264
265 if not 'format' in value2:
265 #if not 'format' in value2:
266 format = ''.join(pathlib.Path(value2['name']).suffixes)
266 # format = ''.join(pathlib.Path(value2['name']).suffixes)
267 if len(format) > 0:
267 # if len(format) > 0:
268 value2['format'] = format.upper()[1:]
268 # value2['format'] = format.upper()[1:]
269
269
270 files_dict['update__resources__-'+ str(len(block)-count2) +'__upload'] = (value2['name'], value2['upload'])
270 files_dict['update__resources__-'+ str(len(block)-count2) +'__upload'] = (value2['name'], value2['upload'])
271 del value2['upload']
271 del value2['upload']
272 resource_extend.append(value2)
272 resource_extend.append(value2)
273
273
274 print('BLOCK N°{} :: "{}" file(s) found >> uploading'.format(count1 + 1, len(block)))
274 print('BLOCK N°{} :: "{}" file(s) found >> uploading'.format(count1 + 1, len(block)))
275 try:
275 try:
276 result = self.ckan.call_action(
276 result = self.ckan.call_action(
277 'package_revise',
277 'package_revise',
278 {'match': '{'+ str(package_id_or_name) +'}', 'update__resources__extend': json.dumps(resource_extend)},
278 {'match': '{'+ str(package_id_or_name) +'}', 'update__resources__extend': json.dumps(resource_extend)},
279 files=files_dict
279 files=files_dict
280 )
280 )
281 print('BLOCK N°{} :: Uploaded file(s) successfully'.format(count1 + 1))
281 print('BLOCK N°{} :: Uploaded file(s) successfully'.format(count1 + 1))
282 if len(blocks) == count1 + 1:
282 if len(blocks) == count1 + 1:
283 return result
283 return result
284 except:
284 except:
285 print('ERROR :: Use the "print" for more information')
285 print('ERROR :: Use the "print" for more information')
286 _, exc_value, _ = sys.exc_info()
286 _, exc_value, _ = sys.exc_info()
287 return exc_value
287 return exc_value
288 else:
288 else:
289 return "ERROR:: No file(s) found to upload"
289 return "ERROR:: No file(s) found to upload"
290
290
291 def upload_multiple_files(self, dataset_id, path_files, date_files, type_files, **kwargs):
291 def upload_multiple_files(self, dataset_id, path_files, date_files, type_files, **kwargs):
292 # Agregar si es interruptido por teclado
292 # Agregar si es interruptido por teclado
293 '''
293 '''
294 FINALIDAD:
294 FINALIDAD:
295 Funcion para subir multiples archivos al repositorio del ROJ.
295 Funcion para subir multiples archivos al repositorio del ROJ.
296
296
297 PARAMETROS DISPONIBLES:
297 PARAMETROS DISPONIBLES:
298 CONSULTAR: "GUIA DE SCRIPT.pdf"
298 CONSULTAR: "GUIA DE SCRIPT.pdf"
299
299
300 ESTRUCTURA:
300 ESTRUCTURA:
301 <access_name>.upload_multiple_files(dataset_id = <class 'str'>, path_files = <class 'str'> or <class 'list of strings'>, date_files = <class 'str'> or <class 'list of strings'>, type_files = <class 'str'> or <class 'list of strings'>, param_1 = <class 'param_1'>, ...)
301 <access_name>.upload_multiple_files(dataset_id = <class 'str'>, path_files = <class 'str'> or <class 'list of strings'>, date_files = <class 'str'> or <class 'list of strings'>, type_files = <class 'str'> or <class 'list of strings'>, param_1 = <class 'param_1'>, ...)
302 '''
302 '''
303
303
304 params_dict = {'upload':[], 'name':[]}
304 params_dict = {'upload':[], 'name':[]}
305 if not 'format' in kwargs:
305 #if not 'format' in kwargs:
306 params_dict.update({'format':[]})
306 # params_dict.update({'format':[]})
307 #---------------CASO : "path" or "path_list"-----------------#
307 #---------------CASO : "path" or "path_list"-----------------#
308 if type(path_files) is list:
308 if type(path_files) is list:
309 if len(path_files) != 0:
309 if len(path_files) != 0:
310 path_files.sort()
310 path_files.sort()
311 for u in path_files:
311 for u in path_files:
312 if os.path.isfile(u):
312 if os.path.isfile(u):
313 params_dict['upload'].append(open(u, 'rb'))
313 params_dict['upload'].append(open(u, 'rb'))
314 params_dict['name'].append(pathlib.Path(u).name)
314 params_dict['name'].append(os.path.basename(u))
315 if not 'format' in kwargs:
315 #if not 'format' in kwargs:
316 format = ''.join(pathlib.Path(u).suffixes)
316 # format = ''.join(pathlib.Path(u).suffixes)
317 if len(format) > 0:
317 # if len(format) > 0:
318 params_dict['format'].append(format.upper()[1:])
318 # params_dict['format'].append(format.upper()[1:])
319 else:
319 # else:
320 params_dict['format'].append('')
320 # params_dict['format'].append('')
321 else:
321 else:
322 return 'File "%s" does not exist' % (u)
322 return 'File "%s" does not exist' % (u)
323 else:
323 else:
324 return 'ERROR:: "path_list is empty"'
324 return 'ERROR:: "path_list is empty"'
325 elif type(path_files) is str:
325 elif type(path_files) is str:
326 if os.path.isdir(path_files):
326 if os.path.isdir(path_files):
327 path_order = [f for f in os.listdir(path_files) if os.path.isfile(os.path.join(path_files, f))]
327 path_order = [f for f in os.listdir(path_files) if os.path.isfile(os.path.join(path_files, f))]
328 path_order.sort()
328 path_order.sort()
329 if path_order:
329 if path_order:
330 for name in path_order:
330 for name in path_order:
331 params_dict['upload'].append(open(os.path.join(path_files, name), 'rb'))
331 params_dict['upload'].append(open(os.path.join(path_files, name), 'rb'))
332 params_dict['name'].append(name)
332 params_dict['name'].append(name)
333 if not 'format' in kwargs:
333 #if not 'format' in kwargs:
334 format = ''.join(pathlib.Path(name).suffixes)
334 # format = ''.join(pathlib.Path(name).suffixes)
335 if len(format) > 0:
335 # if len(format) > 0:
336 params_dict['format'].append(format.upper()[1:])
336 # params_dict['format'].append(format.upper()[1:])
337 else:
337 # else:
338 params_dict['format'].append('')
338 # params_dict['format'].append('')
339 else:
339 else:
340 return "ERROR:: There aren't files in this directory"
340 return "ERROR:: There aren't files in this directory"
341 else:
341 else:
342 return 'ERROR:: Directory "%s" does not exist' % (path_files)
342 return 'ERROR:: Directory "%s" does not exist' % (path_files)
343 else:
343 else:
344 return 'ERROR:: "path_files" must be a str or list'
344 return 'ERROR:: "path_files" must be a str or list'
345 #------------------------------------------------------------#
345 #------------------------------------------------------------#
346 params_no_dict = {'package_id': dataset_id}
346 params_no_dict = {'package_id': dataset_id}
347 if type(date_files) is list:
347 if type(date_files) is list:
348 params_dict['file_date'] = date_files
348 params_dict['file_date'] = date_files
349 else:
349 else:
350 params_no_dict['file_date'] = date_files
350 params_no_dict['file_date'] = date_files
351
351
352 if type(type_files) is list:
352 if type(type_files) is list:
353 params_dict['voc_file_type'] = type_files
353 params_dict['voc_file_type'] = type_files
354 else:
354 else:
355 params_no_dict['voc_file_type'] = type_files
355 params_no_dict['voc_file_type'] = type_files
356
356
357 for key1, value1 in kwargs.items():
357 for key1, value1 in kwargs.items():
358 if not key1 in params_dict and not key1 in params_no_dict and key1 != 'others':
358 if not key1 in params_dict and not key1 in params_no_dict and key1 != 'others':
359 if type(value1) is list:
359 if type(value1) is list:
360 params_dict[key1] = value1
360 params_dict[key1] = value1
361 else:
361 else:
362 params_no_dict[key1] = value1
362 params_no_dict[key1] = value1
363 #------------------------------------------#
363 #------------------------------------------#
364 if not 'others' in kwargs:
364 if not 'others' in kwargs:
365 params_no_dict['others'] = ''
365 params_no_dict['others'] = ''
366 else:
366 else:
367 if isinstance(kwargs['others'], tuple):
367 if isinstance(kwargs['others'], tuple):
368 params_dict['others'] = [json.dumps(w) for w in kwargs['others']]
368 params_dict['others'] = [json.dumps(w) for w in kwargs['others']]
369 elif isinstance(kwargs['others'], list):
369 elif isinstance(kwargs['others'], list):
370 params_no_dict['others'] = json.dumps(kwargs['others'])
370 params_no_dict['others'] = json.dumps(kwargs['others'])
371 elif isinstance(kwargs['others'], str):
371 elif isinstance(kwargs['others'], str):
372 params_no_dict['others'] = kwargs['others']
372 params_no_dict['others'] = kwargs['others']
373 else:
373 else:
374 return 'ERROR:: "others" must be a tuple, list or str'
374 return 'ERROR:: "others" must be a tuple, list or str'
375 #------------------------------------------#
375 #------------------------------------------#
376 len_params_dict = []
376 len_params_dict = []
377 for value2 in params_dict.values():
377 for value2 in params_dict.values():
378 len_params_dict.append(len(value2))
378 len_params_dict.append(len(value2))
379
379
380 if len(list(set(len_params_dict))) > 1:
380 if len(list(set(len_params_dict))) > 1:
381 return 'ERROR:: All lists must be the same length: %s' % (len(params_dict['name']))
381 return 'ERROR:: All lists must be the same length: %s' % (len(params_dict['name']))
382 #------------------------------------------------------------#
382 #------------------------------------------------------------#
383 print('"{}" file(s) found >> uploading'.format(len(params_dict['name'])))
383 print('"{}" file(s) found >> uploading'.format(len(params_dict['name'])))
384 for v in range(len(params_dict['name'])):
384 for v in range(len(params_dict['name'])):
385 try:
385 try:
386 send = {}
386 send = {}
387 for key_dict, value_dict in params_dict.items():
387 for key_dict, value_dict in params_dict.items():
388 send[key_dict] = value_dict[v]
388 send[key_dict] = value_dict[v]
389 for key_no_dict, value_no_dict in params_no_dict.items():
389 for key_no_dict, value_no_dict in params_no_dict.items():
390 send[key_no_dict] = value_no_dict
390 send[key_no_dict] = value_no_dict
391
391
392 self.list.append(getattr(self.ckan.action, 'resource_create')(**send))
392 self.list.append(getattr(self.ckan.action, 'resource_create')(**send))
393 print('File #{} :: "{}" was uploaded successfully'.format(v+1, params_dict['name'][v]))
393 print('File #{} :: "{}" was uploaded successfully'.format(v+1, params_dict['name'][v]))
394 except:
394 except:
395 _, exc_value, _ = sys.exc_info()
395 _, exc_value, _ = sys.exc_info()
396 self.list.append(exc_value)
396 self.list.append(exc_value)
397 print('File #{} :: Error uploading "{}" file'.format(v+1, params_dict['name'][v]))
397 print('File #{} :: Error uploading "{}" file'.format(v+1, params_dict['name'][v]))
398 return self.list
398 return self.list
399 #------------------------------------------------------------#
399 #------------------------------------------------------------#
400
400
401 def show(self, type_option, id, **kwargs):
401 def show(self, type_option, id, **kwargs):
402 '''
402 '''
403 FINALIDAD:
403 FINALIDAD:
404 Funcion personalizada para una busqueda en especifico.
404 Funcion personalizada para una busqueda en especifico.
405
405
406 PARAMETROS DISPONIBLES:
406 PARAMETROS DISPONIBLES:
407 CONSULTAR: "GUIA DE SCRIPT.pdf"
407 CONSULTAR: "GUIA DE SCRIPT.pdf"
408
408
409 ESTRUCTURA:
409 ESTRUCTURA:
410 <access_name>.show(type_option = <class 'str'>, id = <class 'str'>, param_1 = <class 'param_1'>, ...)
410 <access_name>.show(type_option = <class 'str'>, id = <class 'str'>, param_1 = <class 'param_1'>, ...)
411 '''
411 '''
412 if type(type_option) is str:
412 if type(type_option) is str:
413 try:
413 try:
414 if type_option == 'dataset':
414 if type_option == 'dataset':
415 return getattr(self.ckan.action, 'package_show')(id=id, **kwargs)
415 return getattr(self.ckan.action, 'package_show')(id=id, **kwargs)
416 elif type_option == 'resource':
416 elif type_option == 'resource':
417 return getattr(self.ckan.action, 'resource_show')(id=id, **kwargs)
417 return getattr(self.ckan.action, 'resource_show')(id=id, **kwargs)
418 elif type_option == 'project':
418 elif type_option == 'project':
419 return getattr(self.ckan.action, 'organization_show')(id=id, **kwargs)
419 return getattr(self.ckan.action, 'organization_show')(id=id, **kwargs)
420 elif type_option == 'collaborator':
420 elif type_option == 'collaborator':
421 return getattr(self.ckan.action, 'package_collaborator_list_for_user')(id=id, **kwargs)
421 return getattr(self.ckan.action, 'package_collaborator_list_for_user')(id=id, **kwargs)
422 elif type_option == 'member':
422 elif type_option == 'member':
423 return getattr(self.ckan.action, 'organization_list_for_user')(id=id, **kwargs)
423 return getattr(self.ckan.action, 'organization_list_for_user')(id=id, **kwargs)
424 elif type_option == 'vocabulary':
424 elif type_option == 'vocabulary':
425 return getattr(self.ckan.action, 'vocabulary_show')(id=id, **kwargs)
425 return getattr(self.ckan.action, 'vocabulary_show')(id=id, **kwargs)
426 elif type_option == 'tag':
426 elif type_option == 'tag':
427 if not 'vocabulary_id' in kwargs:
427 if not 'vocabulary_id' in kwargs:
428 print('Missing "vocabulary_id" value: assume it is a free tag')
428 print('Missing "vocabulary_id" value: assume it is a free tag')
429 return getattr(self.ckan.action, 'tag_show')(id=id, **kwargs)
429 return getattr(self.ckan.action, 'tag_show')(id=id, **kwargs)
430 elif type_option == 'user':
430 elif type_option == 'user':
431 return getattr(self.ckan.action, 'user_show')(id=id, **kwargs)
431 return getattr(self.ckan.action, 'user_show')(id=id, **kwargs)
432 elif type_option == 'job':
432 elif type_option == 'job':
433 return getattr(self.ckan.action, 'job_show')(id=id, **kwargs)
433 return getattr(self.ckan.action, 'job_show')(id=id, **kwargs)
434 else:
434 else:
435 return 'ERROR:: "type_option = %s" is not accepted' % (type_option)
435 return 'ERROR:: "type_option = %s" is not accepted' % (type_option)
436 except:
436 except:
437 _, exc_value, _ = sys.exc_info()
437 _, exc_value, _ = sys.exc_info()
438 return exc_value
438 return exc_value
439 else:
439 else:
440 return 'ERROR:: "type_option" must be a str'
440 return 'ERROR:: "type_option" must be a str'
441
441
442 def search(self, type_option, query=None, **kwargs):
442 def search(self, type_option, query=None, **kwargs):
443 '''
443 '''
444 FINALIDAD:
444 FINALIDAD:
445 Funcion personalizada para busquedas que satisfagan algun criterio.
445 Funcion personalizada para busquedas que satisfagan algun criterio.
446
446
447 PARAMETROS DISPONIBLES:
447 PARAMETROS DISPONIBLES:
448 CONSULTAR: "GUIA DE SCRIPT.pdf"
448 CONSULTAR: "GUIA DE SCRIPT.pdf"
449
449
450 ESTRUCTURA:
450 ESTRUCTURA:
451 <access_name>.search(type_option = <class 'str'>, query = <class 'dict'>, param_1 = <class 'param_1'>, ...)
451 <access_name>.search(type_option = <class 'str'>, query = <class 'dict'>, param_1 = <class 'param_1'>, ...)
452 '''
452 '''
453 if type(type_option) is str:
453 if type(type_option) is str:
454 try:
454 try:
455 if type_option == 'dataset':
455 if type_option == 'dataset':
456 key_replace = ['fq', 'fq_list', 'include_private']
456 key_replace = ['fq', 'fq_list', 'include_private']
457 key_point = ['facet_mincount', 'facet_limit', 'facet_field']
457 key_point = ['facet_mincount', 'facet_limit', 'facet_field']
458 for key1, value1 in kwargs.items():
458 for key1, value1 in kwargs.items():
459 if not key1 in key_replace:
459 if not key1 in key_replace:
460 if key1 in key_point:
460 if key1 in key_point:
461 self.dict[key1.replace('_', '.')] = value1
461 self.dict[key1.replace('_', '.')] = value1
462 else:
462 else:
463 self.dict[key1] = value1
463 self.dict[key1] = value1
464
464
465 if query is not None:
465 if query is not None:
466 if type(query) is dict:
466 if type(query) is dict:
467 self.dict['fq_list'] = []
467 self.dict['fq_list'] = []
468 #NUM_RESOURCES_MIN / NUM_RESOURCES_MAX
468 #NUM_RESOURCES_MIN / NUM_RESOURCES_MAX
469 #----------------------------------------------------#
469 #----------------------------------------------------#
470 if 'dataset_start_date' in query:
470 if 'dataset_start_date' in query:
471 if type(query['dataset_start_date']) is str:
471 if type(query['dataset_start_date']) is str:
472 try:
472 try:
473 datetime.strptime(query['dataset_start_date'], '%Y-%m-%d')
473 datetime.strptime(query['dataset_start_date'], '%Y-%m-%d')
474 if len(query['dataset_start_date']) != 10:
474 if len(query['dataset_start_date']) != 10:
475 return '"dataset_start_date", must be: <YYYY-MM-DD>'
475 return '"dataset_start_date", must be: <YYYY-MM-DD>'
476 self.dict['fq_list'].append('dataset_start_date:"'+query['dataset_start_date']+'"')
476 self.dict['fq_list'].append('dataset_start_date:"'+query['dataset_start_date']+'"')
477 self.list.append('dataset_start_date')
477 self.list.append('dataset_start_date')
478 except:
478 except:
479 return '"dataset_start_date" incorrect: "%s"' % (query['dataset_start_date'])
479 return '"dataset_start_date" incorrect: "%s"' % (query['dataset_start_date'])
480 else:
480 else:
481 return '"dataset_start_date" must be <str>'
481 return '"dataset_start_date" must be <str>'
482 #----------------------------------------------------#
482 #----------------------------------------------------#
483 if 'dataset_end_date' in query:
483 if 'dataset_end_date' in query:
484 if type(query['dataset_end_date']) is str:
484 if type(query['dataset_end_date']) is str:
485 try:
485 try:
486 datetime.strptime(query['dataset_end_date'], '%Y-%m-%d')
486 datetime.strptime(query['dataset_end_date'], '%Y-%m-%d')
487 if len(query['dataset_end_date']) != 10:
487 if len(query['dataset_end_date']) != 10:
488 return '"dataset_end_date", must be: <YYYY-MM-DD>'
488 return '"dataset_end_date", must be: <YYYY-MM-DD>'
489
489
490 if 'dataset_start_date' in query:
490 if 'dataset_start_date' in query:
491 if query['dataset_start_date'] > query['dataset_end_date']:
491 if query['dataset_start_date'] > query['dataset_end_date']:
492 return '"dataset_end_date" must be greater than "dataset_start_date"'
492 return '"dataset_end_date" must be greater than "dataset_start_date"'
493
493
494 self.dict['fq_list'].append('dataset_end_date:"'+query['dataset_end_date']+'"')
494 self.dict['fq_list'].append('dataset_end_date:"'+query['dataset_end_date']+'"')
495 self.list.append('dataset_end_date')
495 self.list.append('dataset_end_date')
496 except:
496 except:
497 return '"dataset_end_date" incorrect: "%s"' % (query['dataset_end_date'])
497 return '"dataset_end_date" incorrect: "%s"' % (query['dataset_end_date'])
498 else:
498 else:
499 return '"dataset_end_date" must be <str>'
499 return '"dataset_end_date" must be <str>'
500 #----------------------------------------------------#
500 #----------------------------------------------------#
501 for key, value in query.items():
501 for key, value in query.items():
502 if value is not None and not key in self.list:
502 if value is not None and not key in self.list:
503 self.dict['fq_list'].append(str(key)+':"'+str(value)+'"')
503 self.dict['fq_list'].append(str(key)+':"'+str(value)+'"')
504 else:
504 else:
505 return '"query" must be <dict>'
505 return '"query" must be <dict>'
506
506
507 return getattr(self.ckan.action, 'package_search')(include_private=True, **self.dict)
507 return getattr(self.ckan.action, 'package_search')(include_private=True, **self.dict)
508
508
509 elif type_option == 'resource':
509 elif type_option == 'resource':
510 for key1, value1 in kwargs.items():
510 for key1, value1 in kwargs.items():
511 if key1 != 'fields':
511 if key1 != 'fields':
512 self.dict[key1] = value1
512 self.dict[key1] = value1
513
513
514 if query is not None:
514 if query is not None:
515 if type(query) is dict:
515 if type(query) is dict:
516 #----------------------------------------------------#
516 #----------------------------------------------------#
517 if 'file_date_min' in query:
517 if 'file_date_min' in query:
518 if type(query['file_date_min']) is str:
518 if type(query['file_date_min']) is str:
519 try:
519 try:
520 datetime.strptime(query['file_date_min'], '%Y-%m-%d')
520 datetime.strptime(query['file_date_min'], '%Y-%m-%d')
521 if len(query['file_date_min']) != 10:
521 if len(query['file_date_min']) != 10:
522 return '"file_date_min", must be: <YYYY-MM-DD>'
522 return '"file_date_min", must be: <YYYY-MM-DD>'
523 except:
523 except:
524 return '"file_date_min" incorrect: "%s"' % (query['file_date_min'])
524 return '"file_date_min" incorrect: "%s"' % (query['file_date_min'])
525 else:
525 else:
526 return '"file_date_min" must be <str>'
526 return '"file_date_min" must be <str>'
527 #----------------------------------------------------#
527 #----------------------------------------------------#
528 if 'file_date_max' in query:
528 if 'file_date_max' in query:
529 if type(query['file_date_max']) is str:
529 if type(query['file_date_max']) is str:
530 try:
530 try:
531 datetime.strptime(query['file_date_max'], '%Y-%m-%d')
531 datetime.strptime(query['file_date_max'], '%Y-%m-%d')
532 if len(query['file_date_max']) != 10:
532 if len(query['file_date_max']) != 10:
533 return '"file_date_max", must be: <YYYY-MM-DD>'
533 return '"file_date_max", must be: <YYYY-MM-DD>'
534
534
535 if 'file_date_min' in query:
535 if 'file_date_min' in query:
536 if query['file_date_min'] > query['file_date_max']:
536 if query['file_date_min'] > query['file_date_max']:
537 return '"file_date_max" must be greater than "file_date_min"'
537 return '"file_date_max" must be greater than "file_date_min"'
538 except:
538 except:
539 return '"file_date_max" incorrect: "%s"' % (query['file_date_max'])
539 return '"file_date_max" incorrect: "%s"' % (query['file_date_max'])
540 else:
540 else:
541 return '"file_date_max" must be <str>'
541 return '"file_date_max" must be <str>'
542 #----------------------------------------------------#
542 #----------------------------------------------------#
543 self.dict['query'] = query
543 self.dict['query'] = query
544 else:
544 else:
545 return '"query" must be <dict>'
545 return '"query" must be <dict>'
546 return getattr(self.ckan.action, 'resources_search')(**self.dict)
546 return getattr(self.ckan.action, 'resources_search')(**self.dict)
547
547
548 elif type_option == 'tag':
548 elif type_option == 'tag':
549 for key1, value1 in kwargs.items():
549 for key1, value1 in kwargs.items():
550 if key1 != 'fields':
550 if key1 != 'fields':
551 self.dict[key1] = value1
551 self.dict[key1] = value1
552
552
553 if not 'vocabulary_id' in kwargs:
553 if not 'vocabulary_id' in kwargs:
554 print('Missing "vocabulary_id" value: tags that don’t belong to any vocabulary')
554 print('Missing "vocabulary_id" value: tags that don’t belong to any vocabulary')
555 else:
555 else:
556 print('Only tags that belong to "{}" vocabulary'.format(kwargs['vocabulary_id']))
556 print('Only tags that belong to "{}" vocabulary'.format(kwargs['vocabulary_id']))
557
557
558 if query is not None:
558 if query is not None:
559 if type(query) is dict:
559 if type(query) is dict:
560 if 'search' in query:
560 if 'search' in query:
561 if type(query['search']) is list or type(query['search']) is str:
561 if type(query['search']) is list or type(query['search']) is str:
562 self.dict['query'] = query['search']
562 self.dict['query'] = query['search']
563 else:
563 else:
564 return '"search" must be <list> or <str>'
564 return '"search" must be <list> or <str>'
565 else:
565 else:
566 return '"query" must be <dict>'
566 return '"query" must be <dict>'
567 return getattr(self.ckan.action, 'tag_search')(**self.dict)
567 return getattr(self.ckan.action, 'tag_search')(**self.dict)
568
568
569 else:
569 else:
570 return 'ERROR:: "type_option = %s" is not accepted' % (type_option)
570 return 'ERROR:: "type_option = %s" is not accepted' % (type_option)
571
571
572 except:
572 except:
573 _, exc_value, _ = sys.exc_info()
573 _, exc_value, _ = sys.exc_info()
574 return exc_value
574 return exc_value
575 else:
575 else:
576 return 'ERROR:: "type_option" must be <str>'
576 return 'ERROR:: "type_option" must be <str>'
577
577
578 def create(self, type_option, select=None, **kwargs):
578 def create(self, type_option, select=None, **kwargs):
579 '''
579 '''
580 FINALIDAD:
580 FINALIDAD:
581 Funcion personalizada para crear.
581 Funcion personalizada para crear.
582
582
583 PARAMETROS DISPONIBLES:
583 PARAMETROS DISPONIBLES:
584 CONSULTAR: "GUIA DE SCRIPT.pdf"
584 CONSULTAR: "GUIA DE SCRIPT.pdf"
585
585
586 ESTRUCTURA:
586 ESTRUCTURA:
587 <access_name>.create(type_option = <class 'str'>, param_1 = <class 'param_1'>, ...)
587 <access_name>.create(type_option = <class 'str'>, param_1 = <class 'param_1'>, ...)
588 '''
588 '''
589 if type(type_option) is str:
589 if type(type_option) is str:
590 try:
590 try:
591 if type_option == 'dataset':
591 if type_option == 'dataset':
592 return getattr(self.ckan.action, 'package_create')(**kwargs)
592 return getattr(self.ckan.action, 'package_create')(**kwargs)
593 elif type_option == 'project':
593 elif type_option == 'project':
594 return getattr(self.ckan.action, 'organization_create')(**kwargs)
594 return getattr(self.ckan.action, 'organization_create')(**kwargs)
595 elif type_option == 'member':
595 elif type_option == 'member':
596 return getattr(self.ckan.action, 'organization_member_create')(**kwargs)
596 return getattr(self.ckan.action, 'organization_member_create')(**kwargs)
597 elif type_option == 'collaborator':
597 elif type_option == 'collaborator':
598 return getattr(self.ckan.action, 'package_collaborator_create')(**kwargs)
598 return getattr(self.ckan.action, 'package_collaborator_create')(**kwargs)
599 elif type_option == 'vocabulary':
599 elif type_option == 'vocabulary':
600 return getattr(self.ckan.action, 'vocabulary_create')(**kwargs)
600 return getattr(self.ckan.action, 'vocabulary_create')(**kwargs)
601 elif type_option == 'tag':
601 elif type_option == 'tag':
602 return getattr(self.ckan.action, 'tag_create')(**kwargs)
602 return getattr(self.ckan.action, 'tag_create')(**kwargs)
603 elif type_option == 'user':
603 elif type_option == 'user':
604 return getattr(self.ckan.action, 'user_create')(**kwargs)
604 return getattr(self.ckan.action, 'user_create')(**kwargs)
605 elif type_option == 'views':
605 elif type_option == 'views':
606 if 'resource' == select:
606 if 'resource' == select:
607 self.list = ['package']
607 self.list = ['package']
608 for key1, value1 in kwargs.items():
608 for key1, value1 in kwargs.items():
609 if not key1 in self.list:
609 if not key1 in self.list:
610 self.dict[key1] = value1
610 self.dict[key1] = value1
611 return getattr(self.ckan.action, 'resource_create_default_resource_views')(**self.dict)
611 return getattr(self.ckan.action, 'resource_create_default_resource_views')(**self.dict)
612 elif 'dataset' == select:
612 elif 'dataset' == select:
613 return getattr(self.ckan.action, 'package_create_default_resource_views')(**kwargs)
613 return getattr(self.ckan.action, 'package_create_default_resource_views')(**kwargs)
614 else:
614 else:
615 return 'ERROR:: "select = %s" is not accepted' % (select)
615 return 'ERROR:: "select = %s" is not accepted' % (select)
616 else:
616 else:
617 return 'ERROR:: "type_option = %s" is not accepted' % (type_option)
617 return 'ERROR:: "type_option = %s" is not accepted' % (type_option)
618 except:
618 except:
619 _, exc_value, _ = sys.exc_info()
619 _, exc_value, _ = sys.exc_info()
620 return exc_value
620 return exc_value
621 else:
621 else:
622 return 'ERROR:: "type_option" must be <str>'
622 return 'ERROR:: "type_option" must be <str>'
623
623
624 def patch(self, type_option, **kwargs):
624 def patch(self, type_option, **kwargs):
625 '''
625 '''
626 FINALIDAD:
626 FINALIDAD:
627 Funciones personalizadas para actualizar
627 Funciones personalizadas para actualizar
628
628
629 PARAMETROS DISPONIBLES:
629 PARAMETROS DISPONIBLES:
630 CONSULTAR: "GUIA DE SCRIPT.pdf"
630 CONSULTAR: "GUIA DE SCRIPT.pdf"
631
631
632 ESTRUCTURA:
632 ESTRUCTURA:
633 <access_name>.patch(type_option = <class 'str'>, param_1 = <class 'param_1'>, ...)
633 <access_name>.patch(type_option = <class 'str'>, param_1 = <class 'param_1'>, ...)
634 '''
634 '''
635 if type(type_option) is str:
635 if type(type_option) is str:
636 try:
636 try:
637 if type_option == 'dataset':
637 if type_option == 'dataset':
638 return getattr(self.ckan.action, 'package_patch')(**kwargs)
638 return getattr(self.ckan.action, 'package_patch')(**kwargs)
639 elif type_option == 'project':
639 elif type_option == 'project':
640 return getattr(self.ckan.action, 'organization_patch')(**kwargs)
640 return getattr(self.ckan.action, 'organization_patch')(**kwargs)
641 elif type_option == 'resource':
641 elif type_option == 'resource':
642 return getattr(self.ckan.action, 'resource_patch')(**kwargs)
642 return getattr(self.ckan.action, 'resource_patch')(**kwargs)
643 elif type_option == 'member':
643 elif type_option == 'member':
644 return getattr(self.ckan.action, 'organization_member_create')(**kwargs)
644 return getattr(self.ckan.action, 'organization_member_create')(**kwargs)
645 elif type_option == 'collaborator':
645 elif type_option == 'collaborator':
646 return getattr(self.ckan.action, 'package_collaborator_create')(**kwargs)
646 return getattr(self.ckan.action, 'package_collaborator_create')(**kwargs)
647 else:
647 else:
648 return 'ERROR:: "type_option = %s" is not accepted' % (type_option)
648 return 'ERROR:: "type_option = %s" is not accepted' % (type_option)
649 except:
649 except:
650 _, exc_value, _ = sys.exc_info()
650 _, exc_value, _ = sys.exc_info()
651 return exc_value
651 return exc_value
652 else:
652 else:
653 return 'ERROR:: "type_option" must be <str>'
653 return 'ERROR:: "type_option" must be <str>'
654
654
655 def delete(self, type_option, select=None, **kwargs):
655 def delete(self, type_option, select=None, **kwargs):
656 '''
656 '''
657 FINALIDAD:
657 FINALIDAD:
658 Función personalizada para eliminar y/o purgar.
658 Función personalizada para eliminar y/o purgar.
659
659
660 PARAMETROS DISPONIBLES:
660 PARAMETROS DISPONIBLES:
661 CONSULTAR: "GUIA DE SCRIPT.pdf"
661 CONSULTAR: "GUIA DE SCRIPT.pdf"
662
662
663 ESTRUCTURA:
663 ESTRUCTURA:
664 <access_name>.delete(type_option = <class 'str'>, param_1 = <class 'param_1'>, ...)
664 <access_name>.delete(type_option = <class 'str'>, param_1 = <class 'param_1'>, ...)
665 '''
665 '''
666 if type(type_option) is str:
666 if type(type_option) is str:
667 try:
667 try:
668 if type_option == 'dataset':
668 if type_option == 'dataset':
669 if select is None:
669 if select is None:
670 return 'ERROR:: "select" must not be "None"'
670 return 'ERROR:: "select" must not be "None"'
671 else:
671 else:
672 if 'delete' == select:
672 if 'delete' == select:
673 return getattr(self.ckan.action, 'package_delete')(**kwargs)
673 return getattr(self.ckan.action, 'package_delete')(**kwargs)
674 elif 'purge' == select:
674 elif 'purge' == select:
675 return getattr(self.ckan.action, 'dataset_purge')(**kwargs)
675 return getattr(self.ckan.action, 'dataset_purge')(**kwargs)
676 else:
676 else:
677 return 'ERROR:: "select = %s" is not accepted' % (select)
677 return 'ERROR:: "select = %s" is not accepted' % (select)
678 elif type_option == 'project':
678 elif type_option == 'project':
679 if select is None:
679 if select is None:
680 return 'ERROR:: "select" must not be "None"'
680 return 'ERROR:: "select" must not be "None"'
681 else:
681 else:
682 if 'delete' == select:
682 if 'delete' == select:
683 return getattr(self.ckan.action, 'organization_delete')(**kwargs)
683 return getattr(self.ckan.action, 'organization_delete')(**kwargs)
684 elif 'purge' == select:
684 elif 'purge' == select:
685 return getattr(self.ckan.action, 'organization_purge')(**kwargs)
685 return getattr(self.ckan.action, 'organization_purge')(**kwargs)
686 else:
686 else:
687 return 'ERROR:: "select = %s" is not accepted' % (select)
687 return 'ERROR:: "select = %s" is not accepted' % (select)
688 elif type_option == 'resource':
688 elif type_option == 'resource':
689 return getattr(self.ckan.action, 'resource_delete')(**kwargs)
689 return getattr(self.ckan.action, 'resource_delete')(**kwargs)
690 elif type_option == 'vocabulary':
690 elif type_option == 'vocabulary':
691 return getattr(self.ckan.action, 'vocabulary_delete')(**kwargs)
691 return getattr(self.ckan.action, 'vocabulary_delete')(**kwargs)
692 elif type_option == 'tag':
692 elif type_option == 'tag':
693 return getattr(self.ckan.action, 'tag_delete')(**kwargs)
693 return getattr(self.ckan.action, 'tag_delete')(**kwargs)
694 elif type_option == 'user':
694 elif type_option == 'user':
695 return getattr(self.ckan.action, 'user_delete')(**kwargs)
695 return getattr(self.ckan.action, 'user_delete')(**kwargs)
696 else:
696 else:
697 return 'ERROR:: "type_option = %s" is not accepted' % (type_option)
697 return 'ERROR:: "type_option = %s" is not accepted' % (type_option)
698 except:
698 except:
699 _, exc_value, _ = sys.exc_info()
699 _, exc_value, _ = sys.exc_info()
700 return exc_value
700 return exc_value
701 else:
701 else:
702 return 'ERROR:: "type_option" must be <str>'
702 return 'ERROR:: "type_option" must be <str>'
703
703
704 def f_status_note(self, total, result, path):
704 def f_status_note(self, total, result, path):
705 file_txt = open(path+'status_note.txt', 'w')
705 file_txt = open(path+'status_note.txt', 'w')
706 file_txt = open(path+'status_note.txt', 'a')
706 file_txt = open(path+'status_note.txt', 'a')
707
707
708 file_txt.write('DOWNLOADED FILE(S): "%s"' % (len(result['name'])))
708 file_txt.write('DOWNLOADED FILE(S): "%s"' % (len(result['name'])))
709 file_txt.write(''+ os.linesep)
709 file_txt.write(''+ os.linesep)
710 for u in result['name']:
710 for u in result['name']:
711 file_txt.write(' - '+ u + os.linesep)
711 file_txt.write(' - '+ u + os.linesep)
712 file_txt.write(''+ os.linesep)
712 file_txt.write(''+ os.linesep)
713
713
714 file_txt.write('FAILED FILE(S): "%s"' % (len(total['name'])-len(result['name'])))
714 file_txt.write('FAILED FILE(S): "%s"' % (len(total['name'])-len(result['name'])))
715 file_txt.write(''+ os.linesep)
715 file_txt.write(''+ os.linesep)
716 if len(total['name'])-len(result['name']) != 0:
716 if len(total['name'])-len(result['name']) != 0:
717 for u in total['name']:
717 for u in total['name']:
718 if not u in result['name']:
718 if not u in result['name']:
719 file_txt.write(' - '+ u + os.linesep)
719 file_txt.write(' - '+ u + os.linesep)
720 else:
720 else:
721 file_txt.write(' "None"'+ os.linesep)
721 file_txt.write(' "None"'+ os.linesep)
722
722
723 def f_name(self, name_dataset, ext, tempdir):
723 def f_name(self, name_dataset, ext, tempdir):
724 while self.check:
724 while self.check:
725 self.str = ''
725 self.str = ''
726 if self.cont == 0:
726 if self.cont == 0:
727 if os.path.exists(tempdir + name_dataset + ext):
727 if os.path.exists(tempdir + name_dataset + ext):
728 self.str = name_dataset+'('+str(self.cont+1)+')'+ext
728 self.str = name_dataset+'('+str(self.cont+1)+')'+ext
729 else:
729 else:
730 self.check = self.check * 0
730 self.check = self.check * 0
731 self.str = name_dataset + ext
731 self.str = name_dataset + ext
732 else:
732 else:
733 if not os.path.exists(tempdir + name_dataset+'('+str(self.cont)+')'+ext):
733 if not os.path.exists(tempdir + name_dataset+'('+str(self.cont)+')'+ext):
734 self.check = self.check * 0
734 self.check = self.check * 0
735 self.str = name_dataset+'('+str(self.cont)+')'+ ext
735 self.str = name_dataset+'('+str(self.cont)+')'+ ext
736 self.cont = self.cont+1
736 self.cont = self.cont+1
737 return self.str
737 return self.str
738
738
739 def f_zipdir(self, path, ziph, zip_name):
739 def f_zipdir(self, path, ziph, zip_name):
740 for root, _, files in os.walk(path):
740 for root, _, files in os.walk(path):
741 print('.....')
741 print('.....')
742 print('Creating: "{}" >>'.format(zip_name))
742 print('Creating: "{}" >>'.format(zip_name))
743 for __file in tqdm(iterable=files, total=len(files)):
743 for __file in tqdm(iterable=files, total=len(files)):
744 new_dir = os.path.relpath(os.path.join(root, __file), os.path.join(path, '..'))
744 new_dir = os.path.relpath(os.path.join(root, __file), os.path.join(path, '..'))
745 ziph.write(os.path.join(root, __file), new_dir)
745 ziph.write(os.path.join(root, __file), new_dir)
746 print('Created >>')
746 print('Created >>')
747
747
748 def download_by_step(self, response, tempdir_name):
748 def download_by_step(self, response, tempdir_name):
749 try:
749 try:
750 with requests.get(response['url'], stream=True, headers={'Authorization': self.Authorization}) as resp:
750 with requests.get(response['url'], stream=True, headers={'Authorization': self.Authorization}) as resp:
751 if resp.status_code == 200:
751 if resp.status_code == 200:
752 with open(tempdir_name+response['name'], 'wb') as file:
752 with open(tempdir_name+response['name'], 'wb') as file:
753 for chunk in resp.iter_content(chunk_size = self.chunk_size):
753 for chunk in resp.iter_content(chunk_size = self.chunk_size):
754 if chunk:
754 if chunk:
755 file.write(chunk)
755 file.write(chunk)
756 except requests.exceptions.RequestException:
756 except requests.exceptions.RequestException:
757 pass
757 pass
758
758
759 def download_files(self, **kwargs):
759 def download_files(self, **kwargs):
760 '''
760 '''
761 FINALIDAD:
761 FINALIDAD:
762 Funcion personalizada para la descarga de archivos existentes de un dataset.
762 Funcion personalizada para la descarga de archivos existentes de un dataset.
763
763
764 PARAMETROS DISPONIBLES:
764 PARAMETROS DISPONIBLES:
765 CONSULTAR: "GUIA DE SCRIPT.pdf"
765 CONSULTAR: "GUIA DE SCRIPT.pdf"
766
766
767 ESTRUCTURA:
767 ESTRUCTURA:
768 <access_name>.download_files(id = <class 'str'>, param_1 = <class 'param_1'>, ...)
768 <access_name>.download_files(id = <class 'str'>, param_1 = <class 'param_1'>, ...)
769 '''
769 '''
770 dict_local = {}
770 dict_local = {}
771 #----------------------------------------------#
771 #----------------------------------------------#
772 if 'zip' in kwargs:
772 if 'zip' in kwargs:
773 if type(kwargs['zip']) is not bool:
773 if type(kwargs['zip']) is not bool:
774 return 'ERROR:: "zip" must be: <class "bool">'
774 return 'ERROR:: "zip" must be: <class "bool">'
775 else:
775 else:
776 dict_local['zip'] = kwargs['zip']
776 dict_local['zip'] = kwargs['zip']
777 else:
777 else:
778 dict_local['zip'] = False
778 dict_local['zip'] = False
779 #----------------------------------------------#
779 #----------------------------------------------#
780 if 'status_note' in kwargs:
780 if 'status_note' in kwargs:
781 if type(kwargs['status_note']) is not bool:
781 if type(kwargs['status_note']) is not bool:
782 return 'ERROR:: "status_note" must be: <class "bool">'
782 return 'ERROR:: "status_note" must be: <class "bool">'
783 else:
783 else:
784 dict_local['status_note'] = kwargs['status_note']
784 dict_local['status_note'] = kwargs['status_note']
785 else:
785 else:
786 dict_local['status_note'] = False
786 dict_local['status_note'] = False
787 #----------------------------------------------#
787 #----------------------------------------------#
788 if 'path' in kwargs:
788 if 'path' in kwargs:
789 if type(kwargs['path']) is str:
789 if type(kwargs['path']) is str:
790 if os.path.isdir(kwargs['path']) == False:
790 if os.path.isdir(kwargs['path']) == False:
791 return 'ERROR:: "path" does not exist'
791 return 'ERROR:: "path" does not exist'
792 else:
792 else:
793 if kwargs['path'][-1:] != self.separator:
793 if kwargs['path'][-1:] != self.separator:
794 dict_local['path'] = kwargs['path']+self.separator
794 dict_local['path'] = kwargs['path']+self.separator
795 else:
795 else:
796 dict_local['path'] = kwargs['path']
796 dict_local['path'] = kwargs['path']
797
797
798 txt = dict_local['path']+datetime.now().strftime("%Y_%m_%d_%H_%M_%S_%f")+'.txt'
798 txt = dict_local['path']+datetime.now().strftime("%Y_%m_%d_%H_%M_%S_%f")+'.txt'
799 if int(platform.python_version()[0]) == 3:
799 if int(platform.python_version()[0]) == 3:
800 try:
800 try:
801 file_txt = open(txt, 'w')
801 file_txt = open(txt, 'w')
802 file_txt.close()
802 file_txt.close()
803 os.remove(txt)
803 os.remove(txt)
804 except PermissionError:
804 except PermissionError:
805 return 'ERROR:: Access denied, you are not authorized to write files: "%s"' % (dict_local['path'])
805 return 'ERROR:: Access denied, you are not authorized to write files: "%s"' % (dict_local['path'])
806 else:
806 else:
807 try:
807 try:
808 file_txt = open(txt, 'w')
808 file_txt = open(txt, 'w')
809 file_txt.close()
809 file_txt.close()
810 os.remove(txt)
810 os.remove(txt)
811 except:
811 except:
812 return 'ERROR:: Access denied, you are not authorized to write files: "%s"' % (dict_local['path'])
812 return 'ERROR:: Access denied, you are not authorized to write files: "%s"' % (dict_local['path'])
813 else:
813 else:
814 return 'ERROR:: "path" must be: <class "str">'
814 return 'ERROR:: "path" must be: <class "str">'
815 else:
815 else:
816 dict_local['path'] = ''
816 dict_local['path'] = ''
817 #----------------------------------------------#
817 #----------------------------------------------#
818 for key, value in kwargs.items():
818 for key, value in kwargs.items():
819 if not key in dict_local:
819 if not key in dict_local:
820 self.dict[key] = value
820 self.dict[key] = value
821 try:
821 try:
822 response = getattr(self.ckan.action, 'url_resources')(**self.dict)
822 response = getattr(self.ckan.action, 'url_resources')(**self.dict)
823 except:
823 except:
824 _, exc_value, _ = sys.exc_info()
824 _, exc_value, _ = sys.exc_info()
825 return exc_value
825 return exc_value
826
826
827 if len(response) != 0:
827 if len(response) != 0:
828 #--------------TEMP PATH---------------#
828 #--------------TEMP PATH---------------#
829 if dict_local['zip']:
829 if dict_local['zip']:
830 tempdir = tempfile.mkdtemp(prefix=kwargs['id']+'-')+self.separator
830 tempdir = tempfile.mkdtemp(prefix=kwargs['id']+'-')+self.separator
831 os.mkdir(tempdir+kwargs['id'])
831 os.mkdir(tempdir+kwargs['id'])
832 dir_name = tempdir + kwargs['id'] + self.separator
832 dir_name = tempdir + kwargs['id'] + self.separator
833 else:
833 else:
834 dir = self.f_name(kwargs['id'], '', dict_local['path'])
834 dir = self.f_name(kwargs['id'], '', dict_local['path'])
835 os.mkdir(dict_local['path'] + dir)
835 os.mkdir(dict_local['path'] + dir)
836 dir_name = dict_local['path'] + dir + self.separator
836 dir_name = dict_local['path'] + dir + self.separator
837 #-----------DOWNLOAD FILES-------------#
837 #-----------DOWNLOAD FILES-------------#
838 print('.....')
838 print('.....')
839 print('Downloading "{}" file(s) >>'.format(len(response)))
839 print('Downloading "{}" file(s) >>'.format(len(response)))
840 name_total = {'name': []}
840 name_total = {'name': []}
841 with concurrent.futures.ThreadPoolExecutor() as executor:
841 with concurrent.futures.ThreadPoolExecutor() as executor:
842 for u in tqdm(iterable=response, total=len(response)):
842 for u in tqdm(iterable=response, total=len(response)):
843 name_total['name'].append(u['name'])
843 name_total['name'].append(u['name'])
844 executor.submit(self.download_by_step, u, dir_name)
844 executor.submit(self.download_by_step, u, dir_name)
845 name_check = {}
845 name_check = {}
846 name_check['name'] = [f for f in os.listdir(dir_name) if os.path.isfile(os.path.join(dir_name, f))]
846 name_check['name'] = [f for f in os.listdir(dir_name) if os.path.isfile(os.path.join(dir_name, f))]
847 print('"{}" downloaded file(s) successfully >>'.format(len(name_check['name'])))
847 print('"{}" downloaded file(s) successfully >>'.format(len(name_check['name'])))
848 #--------------------------------------#
848 #--------------------------------------#
849 if len(name_check['name']) != 0:
849 if len(name_check['name']) != 0:
850 #----------Status Note---------#
850 #----------Status Note---------#
851 if dict_local['status_note']:
851 if dict_local['status_note']:
852 print('.....')
852 print('.....')
853 print('Creating: "status_note.txt" >>')
853 print('Creating: "status_note.txt" >>')
854 self.f_status_note(name_total, name_check, dir_name)
854 self.f_status_note(name_total, name_check, dir_name)
855 print('Created>>')
855 print('Created>>')
856 #----------ZIP CREATE----------#
856 #----------ZIP CREATE----------#
857 if dict_local['zip']:
857 if dict_local['zip']:
858 zip_name = self.f_name(kwargs['id'], '.zip', dict_local['path'])
858 zip_name = self.f_name(kwargs['id'], '.zip', dict_local['path'])
859 ziph = zipfile.ZipFile(dict_local['path'] + zip_name, 'w', zipfile.ZIP_DEFLATED, allowZip64=True)
859 ziph = zipfile.ZipFile(dict_local['path'] + zip_name, 'w', zipfile.ZIP_DEFLATED, allowZip64=True)
860 self.f_zipdir(dir_name, ziph, zip_name)
860 self.f_zipdir(dir_name, ziph, zip_name)
861 ziph.close()
861 ziph.close()
862 #Delete Temporal Path
862 #Delete Temporal Path
863 if os.path.exists(tempdir[:-1]):
863 if os.path.exists(tempdir[:-1]):
864 shutil.rmtree(tempdir[:-1])
864 shutil.rmtree(tempdir[:-1])
865 #------------------------------#
865 #------------------------------#
866 print('.....')
866 print('.....')
867 return 'DOWNLOAD FINISHED'
867 return 'DOWNLOAD FINISHED'
868 else:
868 else:
869 #Delete Temporal Path
869 #Delete Temporal Path
870 if dict_local['zip']:
870 if dict_local['zip']:
871 if os.path.exists(tempdir[:-1]):
871 if os.path.exists(tempdir[:-1]):
872 shutil.rmtree(tempdir[:-1])
872 shutil.rmtree(tempdir[:-1])
873 else:
873 else:
874 if os.path.exists(dir_name[:-1]):
874 if os.path.exists(dir_name[:-1]):
875 shutil.rmtree(dir_name[:-1])
875 shutil.rmtree(dir_name[:-1])
876 return 'NO FILES WERE DOWNLOADED'
876 return 'NO FILES WERE DOWNLOADED'
877 else:
877 else:
878 return 'FILES NOT FOUND' No newline at end of file
878 return 'FILES NOT FOUND'
General Comments 0
You need to be logged in to leave comments. Login now