|
|
1 | NO CONTENT: modified file, binary diff hidden |
| @@ -1,4 +1,6 | |||
|
|
1 | 1 | from ckanapi import RemoteCKAN |
|
|
2 | from datetime import datetime | |
|
|
3 | from tqdm import tqdm | |
|
|
2 | 4 | #from ckanapi.errors import NotAuthorized, NotFound, ValidationError, SearchQueryError, SearchError, CKANAPIError, ServerIncompatibleError |
|
|
3 | 5 | import sys |
|
|
4 | 6 | import platform |
| @@ -11,8 +13,6 import requests | |||
|
|
11 | 13 | import json |
|
|
12 | 14 | import pathlib |
|
|
13 | 15 | import uuid |
|
|
14 | from datetime import datetime | |
|
|
15 | from tqdm import tqdm | |
|
|
16 | 16 | |
|
|
17 | 17 | class JROAPI(): |
|
|
18 | 18 | """ |
| @@ -25,8 +25,10 class JROAPI(): | |||
|
|
25 | 25 | En Python 2 |
|
|
26 | 26 | - pip install ckanapi==4.5 |
|
|
27 | 27 | - pip install requests |
|
|
28 | - pip install pathlib | |
|
|
29 | - pip install futures | |
|
|
28 | 30 | - pip install tqdm |
|
|
29 | En Python 3 | |
|
|
31 | En Python > 3 | |
|
|
30 | 32 | - pip3 install ckanapi==4.5 |
|
|
31 | 33 | - pip3 install requests |
|
|
32 | 34 | - pip3 install tqdm |
| @@ -54,10 +56,10 class JROAPI(): | |||
|
|
54 | 56 | |
|
|
55 | 57 | REPORTAR ALGUN PROBLEMA: |
|
|
56 | 58 | Debe enviar un correo a eynilupu@igp.gob.pe detallando los siguientes pasos: |
|
|
57 | 1) Identifiquese | |
|
|
58 |
2) Descri |
|
|
|
59 |
3) ¿En que |
|
|
|
60 | 4) ¿Que esperaba que hiciera la funcion sin el problema? | |
|
|
59 | 1) Correo para contactarlo | |
|
|
60 | 2) Descripcion del problema | |
|
|
61 | 3) ¿En que paso o seccion encontro el problema? | |
|
|
62 | 4) ¿Cual era el resultado que usted esperaba? | |
|
|
61 | 63 | """ |
|
|
62 | 64 | def __init__(self, url, Authorization=None): |
|
|
63 | 65 | ua = 'CKAN_JRO/1.1 (+'+str(url)+')' |
| @@ -110,6 +112,7 class JROAPI(): | |||
|
|
110 | 112 | return exc_value |
|
|
111 | 113 | |
|
|
112 | 114 | def upload_file(self, dataset_id, file_path, file_date, file_type, **kwargs): |
|
|
115 | # Agregar si es interruptido por teclado | |
|
|
113 | 116 | ''' |
|
|
114 | 117 | FINALIDAD: |
|
|
115 | 118 | Funcion para subir un unico archivo al repositorio del ROJ. |
| @@ -147,8 +150,8 class JROAPI(): | |||
|
|
147 | 150 | _, exc_value, _ = sys.exc_info() |
|
|
148 | 151 | return exc_value |
|
|
149 | 152 | |
|
|
150 | ||
|
|
151 | def upload_multiple_files_advance(self, dataset_id, path_files, file_date, file_type, **kwargs): | |
|
|
153 | def upload_multiple_files_advance(self, dataset_id, path_files, file_date, file_type, max_size=100, ignore_repetition=False, **kwargs): | |
|
|
154 | # Agregar si es interruptido por teclado | |
|
|
152 | 155 | ''' |
|
|
153 | 156 | FINALIDAD: |
|
|
154 | 157 | Funcion para subir multiples archivos al repositorio del ROJ. |
| @@ -159,41 +162,59 class JROAPI(): | |||
|
|
159 | 162 | ESTRUCTURA: |
|
|
160 | 163 | <access_name>.upload_multiple_files_advance(dataset_id = <class 'str'>, path_files = <class 'list of strings'>, file_date = <class 'str'>, file_type = <class 'str'>, param_1 = <class 'param_1'>, ...) |
|
|
161 | 164 | ''' |
|
|
165 | #-------------------------PACKAGE SHOW-----------------------# | |
|
|
166 | try: | |
|
|
167 | dataset_show = getattr(self.ckan.action, 'package_show')(id=dataset_id)['resources'] | |
|
|
168 | except: | |
|
|
169 | _, exc_value, _ = sys.exc_info() | |
|
|
170 | print('ERROR obtaining metadata dataset:: Use the "print" for more information') | |
|
|
171 | return exc_value | |
|
|
172 | #------------------------------------------------------------# | |
|
|
173 | resources_name = [] | |
|
|
174 | for u in dataset_show: | |
|
|
175 | resources_name.append(u['name'].lower()) | |
|
|
176 | #------------------------------------------------------------# | |
|
|
162 | 177 | self.list = ['package_id', 'upload', 'voc_file_type', 'name'] |
|
|
163 | ||
|
|
164 | 178 | for key1, value1 in kwargs.items(): |
|
|
165 | 179 | if not key1 in self.list: |
|
|
166 | 180 | self.dict[key1] = value1 |
|
|
167 | ||
|
|
168 | #---------------------------# | |
|
|
181 | #------------------------------------------------------------# | |
|
|
169 | 182 | if not 'others' in kwargs: |
|
|
170 | 183 | self.dict['others'] = '' |
|
|
171 | 184 | else: |
|
|
172 | 185 | if isinstance(kwargs['others'], list): |
|
|
173 | 186 | self.dict['others'] = json.dumps(kwargs['others']) |
|
|
174 | #---------------------------# | |
|
|
175 | ||
|
|
176 | params_dict = {'upload':[], 'name':[]} | |
|
|
187 | #------------------------------------------------------------# | |
|
|
188 | total_list = [] | |
|
|
177 | 189 | #---------------CASO : "path" or "path_list"-----------------# |
|
|
178 | 190 | if type(path_files) is list: |
|
|
179 | 191 | if len(path_files) != 0: |
|
|
180 | 192 | path_files.sort() |
|
|
181 | 193 | for u in path_files: |
|
|
182 | 194 | if os.path.isfile(u): |
|
|
183 | params_dict['upload'].append(open(u, 'rb')) | |
|
|
184 | params_dict['name'].append(pathlib.Path(u).name) | |
|
|
195 | if pathlib.Path(u).name.lower() in resources_name: | |
|
|
196 | if not ignore_repetition: | |
|
|
197 | return 'ERROR:: "%s" file already exist in this dataset' % (pathlib.Path(u).name) | |
|
|
198 | print('WARRING:: "'+ str(pathlib.Path(u).name) +'" file was ignored because already exist in this dataset') | |
|
|
199 | else: | |
|
|
200 | total_list.append({'name':pathlib.Path(u).name, 'size': os.stat(u).st_size, 'upload':open(u, 'rb')}) | |
|
|
185 | 201 | else: |
|
|
186 | 202 | return 'File "%s" does not exist' % (u) |
|
|
187 | 203 | else: |
|
|
188 | 204 | return 'ERROR:: "path_list is empty"' |
|
|
205 | ||
|
|
189 | 206 | elif type(path_files) is str: |
|
|
190 | 207 | if os.path.isdir(path_files): |
|
|
191 | 208 | path_order = [f for f in os.listdir(path_files) if os.path.isfile(os.path.join(path_files, f))] |
|
|
192 | 209 | path_order.sort() |
|
|
193 | 210 | if path_order: |
|
|
194 | 211 | for name in path_order: |
|
|
195 | params_dict['upload'].append(open(os.path.join(path_files, name), 'rb')) | |
|
|
196 | params_dict['name'].append(name) | |
|
|
212 | if name.lower() in resources_name: | |
|
|
213 | if not ignore_repetition: | |
|
|
214 | return 'ERROR:: "%s" file already exist in this dataset' % (name) | |
|
|
215 | print('WARRING:: "'+ name +'" file was ignored because already exist in this dataset') | |
|
|
216 | else: | |
|
|
217 | total_list.append({'name':name, 'size': os.stat(os.path.join(path_files, name)).st_size, 'upload':open(os.path.join(path_files, name), 'rb')}) | |
|
|
197 | 218 | else: |
|
|
198 | 219 | return "ERROR:: There aren't files in this directory" |
|
|
199 | 220 | else: |
| @@ -201,42 +222,67 class JROAPI(): | |||
|
|
201 | 222 | else: |
|
|
202 | 223 | return 'ERROR:: "path_files" must be a str or list' |
|
|
203 | 224 | #------------------------------------------------------------# |
|
|
204 | resource_extend = [] | |
|
|
205 | files_dict = {} | |
|
|
206 | for count, name in enumerate(params_dict['name']): | |
|
|
207 | param_list = {"name": name, "file_date": file_date, "voc_file_type": file_type} | |
|
|
208 | param_list.update(self.dict) | |
|
|
209 | ||
|
|
210 | if not 'format' in param_list: | |
|
|
211 | format = ''.join(pathlib.Path(name).suffixes) | |
|
|
212 | if len(format) > 0: | |
|
|
213 | param_list['format'] = format.upper()[1:] | |
|
|
214 | ||
|
|
215 | resource_extend.append(param_list) | |
|
|
216 | files_dict['update__resources__-'+ str(len(params_dict['name'])-count) +'__upload'] = (name, params_dict['upload'][count]) | |
|
|
217 | ||
|
|
218 | #------------------------------------------------------------# | |
|
|
219 | 225 | try: |
|
|
220 | 226 | uuid.UUID(str(dataset_id), version=4) |
|
|
221 | 227 | package_id_or_name = '"id": "' + str(dataset_id) + '"' |
|
|
222 | 228 | except ValueError: |
|
|
223 | 229 | package_id_or_name = '"name": "' + str(dataset_id) + '"' |
|
|
224 | 230 | #------------------------------------------------------------# |
|
|
225 | print('"{}" file(s) found >> uploading'.format(len(params_dict['name']))) | |
|
|
231 | blocks = [[]] | |
|
|
232 | size_file = 0 | |
|
|
233 | inter_num = 0 | |
|
|
234 | for value in total_list: | |
|
|
235 | if value['size'] > 1048576 * float(max_size): | |
|
|
236 | return 'ERROR:: The size of the "%s" file is %sMB, please change "max_size" value' % (value['name'], str(round(value['size']/1048576, 1))) | |
|
|
237 | size_file = size_file + value['size'] | |
|
|
238 | if size_file <= 1048576 * float(max_size): | |
|
|
239 | del value['size'] | |
|
|
240 | blocks[inter_num].append(value) | |
|
|
241 | else: | |
|
|
242 | inter_num = inter_num + 1 | |
|
|
243 | blocks.append([]) | |
|
|
244 | del value['size'] | |
|
|
245 | blocks[inter_num].append(value) | |
|
|
246 | #------------------------------------------------------------# | |
|
|
247 | if len(blocks[0]) > 0: | |
|
|
248 | print('BLOCK(S) IN TOTAL:: {}'.format(len(blocks))) | |
|
|
249 | for count1, block in enumerate(blocks): | |
|
|
250 | print('---- BLOCK N°{} ----'.format(count1 + 1)) | |
|
|
251 | resource_extend = [] | |
|
|
252 | files_dict = {} | |
|
|
253 | for count2, value2 in enumerate(block): | |
|
|
254 | value2['file_date'] = file_date | |
|
|
255 | value2['voc_file_type'] = file_type | |
|
|
256 | value2.update(self.dict) | |
|
|
257 | ||
|
|
258 | if not 'format' in value2: | |
|
|
259 | format = ''.join(pathlib.Path(value2['name']).suffixes) | |
|
|
260 | if len(format) > 0: | |
|
|
261 | value2['format'] = format.upper()[1:] | |
|
|
262 | ||
|
|
263 | files_dict['update__resources__-'+ str(len(block)-count2) +'__upload'] = (value2['name'], value2['upload']) | |
|
|
264 | del value2['upload'] | |
|
|
265 | resource_extend.append(value2) | |
|
|
266 | ||
|
|
267 | print('BLOCK N°{} :: "{}" file(s) found >> uploading'.format(count1 + 1, len(block))) | |
|
|
226 | 268 | try: |
|
|
227 | 269 | result = self.ckan.call_action( |
|
|
228 | 270 | 'package_revise', |
|
|
229 | 271 | {'match': '{'+ str(package_id_or_name) +'}', 'update__resources__extend': json.dumps(resource_extend)}, |
|
|
230 | 272 | files=files_dict |
|
|
231 | 273 | ) |
|
|
232 | print('Uploaded file(s) successfully') | |
|
|
274 | print('BLOCK N°{} :: Uploaded file(s) successfully'.format(count1 + 1)) | |
|
|
275 | if len(blocks) == count1 + 1: | |
|
|
233 | 276 | return result |
|
|
234 | 277 | except: |
|
|
235 |
print('ERROR :: Use the "print" |
|
|
|
278 | print('ERROR :: Use the "print" for more information') | |
|
|
236 | 279 | _, exc_value, _ = sys.exc_info() |
|
|
237 | 280 | return exc_value |
|
|
281 | else: | |
|
|
282 | return "ERROR:: No file(s) found to upload" | |
|
|
238 | 283 | |
|
|
239 | 284 | def upload_multiple_files(self, dataset_id, path_files, date_files, type_files, **kwargs): |
|
|
285 | # Agregar si es interruptido por teclado | |
|
|
240 | 286 | ''' |
|
|
241 | 287 | FINALIDAD: |
|
|
242 | 288 | Funcion para subir multiples archivos al repositorio del ROJ. |
General Comments 0
You need to be logged in to leave comments.
Login now