|
|
1 | NO CONTENT: modified file, binary diff hidden |
| @@ -9,10 +9,10 import zipfile | |||
|
|
9 | 9 | import concurrent.futures |
|
|
10 | 10 | import requests |
|
|
11 | 11 | import json |
|
|
12 |
import a |
|
|
|
12 | import pathlib | |
|
|
13 | import uuid | |
|
|
13 | 14 | from datetime import datetime |
|
|
14 | 15 | from tqdm import tqdm |
|
|
15 | from six import string_types | |
|
|
16 | 16 | |
|
|
17 | 17 | class JROAPI(): |
|
|
18 | 18 | """ |
| @@ -135,9 +135,103 class JROAPI(): | |||
|
|
135 | 135 | if not os.path.isfile(file_path): |
|
|
136 | 136 | return 'File "%s" not exist' % (file_path) |
|
|
137 | 137 | |
|
|
138 | if not 'format' in self.dict: | |
|
|
139 | self.str = ''.join(pathlib.Path(file_path).suffixes) | |
|
|
140 | if len(self.str) > 0: | |
|
|
141 | self.dict['format'] = self.str.upper()[1:] | |
|
|
142 | ||
|
|
143 | try: | |
|
|
144 | return getattr(self.ckan.action, 'resource_create')(package_id=dataset_id, file_date=file_date, upload=open(file_path, 'rb'), voc_file_type=file_type, name=pathlib.Path(file_path).name, **self.dict) | |
|
|
145 | except: | |
|
|
146 | _, exc_value, _ = sys.exc_info() | |
|
|
147 | return exc_value | |
|
|
148 | ||
|
|
149 | ||
|
|
150 | def upload_multiple_files_advance(self, dataset_id, path_files, file_date, file_type, **kwargs): | |
|
|
151 | ''' | |
|
|
152 | FINALIDAD: | |
|
|
153 | Funcion para subir multiples archivos al repositorio del ROJ. | |
|
|
154 | ||
|
|
155 | PARAMETROS DISPONIBLES: | |
|
|
156 | CONSULTAR: "GUIA DE SCRIPT.pdf" | |
|
|
157 | ||
|
|
158 | ESTRUCTURA: | |
|
|
159 | <access_name>.upload_multiple_files_advance(dataset_id = <class 'str'>, path_files = <class 'list of strings'>, file_date = <class 'str'>, file_type = <class 'str'>, param_1 = <class 'param_1'>, ...) | |
|
|
160 | ''' | |
|
|
161 | self.list = ['package_id', 'upload', 'voc_file_type', 'name'] | |
|
|
162 | ||
|
|
163 | for key1, value1 in kwargs.items(): | |
|
|
164 | if not key1 in self.list: | |
|
|
165 | self.dict[key1] = value1 | |
|
|
166 | ||
|
|
167 | #---------------------------# | |
|
|
168 | if not 'others' in kwargs: | |
|
|
169 | self.dict['others'] = '' | |
|
|
170 | else: | |
|
|
171 | if isinstance(kwargs['others'], list): | |
|
|
172 | self.dict['others'] = json.dumps(kwargs['others']) | |
|
|
173 | #---------------------------# | |
|
|
174 | ||
|
|
175 | params_dict = {'upload':[], 'name':[]} | |
|
|
176 | #---------------CASO : "path" or "path_list"-----------------# | |
|
|
177 | if type(path_files) is list: | |
|
|
178 | if len(path_files) != 0: | |
|
|
179 | path_files.sort() | |
|
|
180 | for u in path_files: | |
|
|
181 | if os.path.isfile(u): | |
|
|
182 | params_dict['upload'].append(open(u, 'rb')) | |
|
|
183 | params_dict['name'].append(pathlib.Path(u).name) | |
|
|
184 | else: | |
|
|
185 | return 'File "%s" does not exist' % (u) | |
|
|
186 | else: | |
|
|
187 | return 'ERROR:: "path_list is empty"' | |
|
|
188 | elif type(path_files) is str: | |
|
|
189 | if os.path.isdir(path_files): | |
|
|
190 | path_order = [f for f in os.listdir(path_files) if os.path.isfile(os.path.join(path_files, f))] | |
|
|
191 | path_order.sort() | |
|
|
192 | if path_order: | |
|
|
193 | for name in path_order: | |
|
|
194 | params_dict['upload'].append(open(os.path.join(path_files, name), 'rb')) | |
|
|
195 | params_dict['name'].append(name) | |
|
|
196 | else: | |
|
|
197 | return "ERROR:: There aren't files in this directory" | |
|
|
198 | else: | |
|
|
199 | return 'ERROR:: Directory "%s" does not exist' % (path_files) | |
|
|
200 | else: | |
|
|
201 | return 'ERROR:: "path_files" must be a str or list' | |
|
|
202 | #------------------------------------------------------------# | |
|
|
203 | resource_extend = [] | |
|
|
204 | files_dict = {} | |
|
|
205 | for count, name in enumerate(params_dict['name']): | |
|
|
206 | param_list = {"name": name, "file_date": file_date, "voc_file_type": file_type} | |
|
|
207 | param_list.update(self.dict) | |
|
|
208 | ||
|
|
209 | if not 'format' in param_list: | |
|
|
210 | format = ''.join(pathlib.Path(name).suffixes) | |
|
|
211 | if len(format) > 0: | |
|
|
212 | param_list['format'] = format.upper()[1:] | |
|
|
213 | ||
|
|
214 | resource_extend.append(param_list) | |
|
|
215 | files_dict['update__resources__-'+ str(len(params_dict['name'])-count) +'__upload'] = (name, params_dict['upload'][count]) | |
|
|
216 | ||
|
|
217 | #------------------------------------------------------------# | |
|
|
218 | try: | |
|
|
219 | uuid.UUID(str(dataset_id), version=4) | |
|
|
220 | package_id_or_name = '"id": "' + str(dataset_id) + '"' | |
|
|
221 | except ValueError: | |
|
|
222 | package_id_or_name = '"name": "' + str(dataset_id) + '"' | |
|
|
223 | #------------------------------------------------------------# | |
|
|
224 | print('"{}" file(s) found >> uploading'.format(len(params_dict['name']))) | |
|
|
138 | 225 | try: |
|
|
139 | return getattr(self.ckan.action, 'resource_create')(package_id=dataset_id, file_date=file_date, upload=open(file_path, 'rb'), voc_file_type=file_type, name=file_path.split(self.separator)[-1], **self.dict) | |
|
|
226 | result = self.ckan.call_action( | |
|
|
227 | 'package_revise', | |
|
|
228 | {'match': '{'+ str(package_id_or_name) +'}', 'update__resources__extend': json.dumps(resource_extend)}, | |
|
|
229 | files=files_dict | |
|
|
230 | ) | |
|
|
231 | print('Uploaded file(s) successfully') | |
|
|
232 | return result | |
|
|
140 | 233 | except: |
|
|
234 | print('ERROR :: Use the "print" option for more information') | |
|
|
141 | 235 | _, exc_value, _ = sys.exc_info() |
|
|
142 | 236 | return exc_value |
|
|
143 | 237 | |
| @@ -152,7 +246,10 class JROAPI(): | |||
|
|
152 | 246 | ESTRUCTURA: |
|
|
153 | 247 | <access_name>.upload_multiple_files(dataset_id = <class 'str'>, path_files = <class 'str'> or <class 'list of strings'>, date_files = <class 'str'> or <class 'list of strings'>, type_files = <class 'str'> or <class 'list of strings'>, param_1 = <class 'param_1'>, ...) |
|
|
154 | 248 | ''' |
|
|
155 | params_dict = {'upload':[], 'name':[]} #ADD FORMAT | |
|
|
249 | ||
|
|
250 | params_dict = {'upload':[], 'name':[]} | |
|
|
251 | if not 'format' in kwargs: | |
|
|
252 | params_dict.update({'format':[]}) | |
|
|
156 | 253 | #---------------CASO : "path" or "path_list"-----------------# |
|
|
157 | 254 | if type(path_files) is list: |
|
|
158 | 255 | if len(path_files) != 0: |
| @@ -160,9 +257,15 class JROAPI(): | |||
|
|
160 | 257 | for u in path_files: |
|
|
161 | 258 | if os.path.isfile(u): |
|
|
162 | 259 | params_dict['upload'].append(open(u, 'rb')) |
|
|
163 |
params_dict['name'].append(u. |
|
|
|
260 | params_dict['name'].append(pathlib.Path(u).name) | |
|
|
261 | if not 'format' in kwargs: | |
|
|
262 | format = ''.join(pathlib.Path(u).suffixes) | |
|
|
263 | if len(format) > 0: | |
|
|
264 | params_dict['format'].append(format.upper()[1:]) | |
|
|
164 | 265 | else: |
|
|
165 | return 'File "%s" not exist' % (u) | |
|
|
266 | params_dict['format'].append('') | |
|
|
267 | else: | |
|
|
268 | return 'File "%s" does not exist' % (u) | |
|
|
166 | 269 | else: |
|
|
167 | 270 | return 'ERROR:: "path_list is empty"' |
|
|
168 | 271 | elif type(path_files) is str: |
| @@ -173,10 +276,16 class JROAPI(): | |||
|
|
173 | 276 | for name in path_order: |
|
|
174 | 277 | params_dict['upload'].append(open(os.path.join(path_files, name), 'rb')) |
|
|
175 | 278 | params_dict['name'].append(name) |
|
|
279 | if not 'format' in kwargs: | |
|
|
280 | format = ''.join(pathlib.Path(name).suffixes) | |
|
|
281 | if len(format) > 0: | |
|
|
282 | params_dict['format'].append(format.upper()[1:]) | |
|
|
283 | else: | |
|
|
284 | params_dict['format'].append('') | |
|
|
176 | 285 | else: |
|
|
177 |
return ' |
|
|
|
286 | return "ERROR:: There aren't files in this directory" | |
|
|
178 | 287 | else: |
|
|
179 | return 'ERROR:: Directory "%s" not exist' % (path_files) | |
|
|
288 | return 'ERROR:: Directory "%s" does not exist' % (path_files) | |
|
|
180 | 289 | else: |
|
|
181 | 290 | return 'ERROR:: "path_files" must be a str or list' |
|
|
182 | 291 | #------------------------------------------------------------# |
| @@ -217,7 +326,7 class JROAPI(): | |||
|
|
217 | 326 | if len(list(set(len_params_dict))) > 1: |
|
|
218 | 327 | return 'ERROR:: All lists must be the same length: %s' % (len(params_dict['name'])) |
|
|
219 | 328 | #------------------------------------------------------------# |
|
|
220 | ||
|
|
329 | print('"{}" file(s) found >> uploading'.format(len(params_dict['name']))) | |
|
|
221 | 330 | for v in range(len(params_dict['name'])): |
|
|
222 | 331 | try: |
|
|
223 | 332 | send = {} |
| @@ -227,11 +336,11 class JROAPI(): | |||
|
|
227 | 336 | send[key_no_dict] = value_no_dict |
|
|
228 | 337 | |
|
|
229 | 338 | self.list.append(getattr(self.ckan.action, 'resource_create')(**send)) |
|
|
230 | print('File "{}" was uploaded successfully'.format(params_dict['name'][v])) | |
|
|
339 | print('File #{} :: "{}" was uploaded successfully'.format(v+1, params_dict['name'][v])) | |
|
|
231 | 340 | except: |
|
|
232 | 341 | _, exc_value, _ = sys.exc_info() |
|
|
233 | 342 | self.list.append(exc_value) |
|
|
234 | print('Error uploading "{}" file'.format(params_dict['name'][v])) | |
|
|
343 | print('File #{} :: Error uploading "{}" file'.format(v+1, params_dict['name'][v])) | |
|
|
235 | 344 | return self.list |
|
|
236 | 345 | #------------------------------------------------------------# |
|
|
237 | 346 | |
General Comments 0
You need to be logged in to leave comments.
Login now