##// END OF EJS Templates
DATABASES » ckanext-jro » api-cliente
RSS
v2.9.2 :: Add 'create, patch and delete' resource
eynilupu -
r15:7e5d53ddbf91
parent child
Show More
Show file before | Show file after | Hide comments
@@ -0,0 +1,343
1 import json
2 import uuid
3 import os
4 import sys
5
6 def resource_create(self, package_id, file_type, others='', max_size=100, max_count=500, ignore_repetition=False, **kwargs):
7 #---------------------------------------------------------------------#
8 kwargs['voc_file_type'] = file_type
9 kwargs['others'] = others
10
11 if 'clear_upload' in kwargs:
12 del kwargs['clear_upload']
13 #---------------------------------------------------------------------#
14
15 url_upload = {}
16 if 'upload' in kwargs and 'url' in kwargs:
17 return 'ERROR:: Choose one: "upload" or "url" parameters'
18 elif 'upload' in kwargs:
19 url_upload['upload'] = kwargs['upload']
20 elif 'url' in kwargs:
21 url_upload['url'] = kwargs['url']
22 else:
23 return 'ERROR:: Missing value: "upload" or "url" parameters'
24
25 value_u = list(url_upload.values())[0]
26 key_u = list(url_upload.keys())[0]
27
28 if not isinstance(value_u, list) and not isinstance(value_u, str):
29 return 'ERROR:: "%s" must be <str> or <list>' % (key_u)
30 #---------------------------------------------------------------------#
31
32 if isinstance(value_u, str):
33 if len(value_u) != 0:
34 if key_u == 'upload':
35 if os.path.isdir(value_u):
36 path_order = [f for f in os.listdir(value_u) if os.path.isfile(os.path.join(value_u, f))]
37 path_order.sort()
38 if path_order:
39 kwargs['upload'] = []
40 for name in path_order:
41 kwargs['upload'].append(os.path.join(value_u, name))
42 else:
43 return "ERROR:: There aren't files in this directory"
44 else:
45 return 'ERROR:: Directory or File does not exist'
46 else:
47 return 'ERROR:: "path_list is empty"'
48 #---------------------------------------------------------------------#
49
50 if not isinstance(kwargs[key_u], list):
51 kwargs[key_u] = [kwargs[key_u]]
52
53 if kwargs.get('upload', None):
54 if len(kwargs['upload']) != len(set(kwargs['upload'])):
55 return 'Duplicate files found in "upload" parameter'
56 #---------------------------------------------------------------------#
57
58 change_kwargs = {}
59 for key1, value1 in kwargs.items():
60 if key1 == 'others':
61 if isinstance(value1, tuple):
62 if len(value1) != len(kwargs[key_u]):
63 return 'ERROR:: "%s" value(s) must be same length as "%s" value(s)' % (key1, key_u)
64 else:
65 change_kwargs[key1] = value1
66 else:
67 change_kwargs[key1] = (value1,) * len(kwargs[key_u])
68 else:
69 if isinstance(value1, list):
70 if len(value1) != len(kwargs[key_u]):
71 return 'ERROR:: "%s" value(s) must be same length as "%s" value(s)' % (key1, key_u)
72 else:
73 change_kwargs[key1] = value1
74 else:
75 change_kwargs[key1] = [value1] * len(kwargs[key_u])
76 #---------------------------------------------------------------------#
77
78 try:
79 dataset_show = getattr(self.ckan.action, 'package_show')(id=package_id)['resources']
80 resources_name = []
81 for u in dataset_show:
82 resources_name.append(u['name'].lower())
83 except:
84 _, exc_value, _ = sys.exc_info()
85 print('ERROR obtaining metadata dataset:: Use the "print" for more information')
86 return exc_value
87 #---------------------------------------------------------------------#
88
89 for c1 in range(len(kwargs[key_u])):
90 new_kwargs = {}
91 for k2, v2 in change_kwargs.items():
92 #-------------------------------------------------------------#
93 if k2 == 'upload':
94 if not os.path.isfile(v2[c1]):
95 return 'File "%s" does not exist' % (v2[c1])
96
97 if not kwargs.get('size', None):
98 new_kwargs['size'] = os.stat(v2[c1]).st_size
99 #-------------------------------------------------------------#
100 new_kwargs[k2] = v2[c1]
101 #-----------------------------------------------------------------#
102
103 if not kwargs.get('name', None):
104 new_kwargs['name'] = os.path.basename(new_kwargs[key_u])
105
106 if new_kwargs['name'].lower() in resources_name:
107 if not ignore_repetition:
108 return 'ERROR:: "%s" resource already exist in this dataset' % (new_kwargs['name'])
109 print('WARRING:: "'+ new_kwargs['name'] +'" resource was ignored because already exist in this dataset')
110 else:
111 self.list.append(new_kwargs)
112 #---------------------------------------------------------------------#
113
114 try:
115 uuid.UUID(package_id, version=4)
116 self.dict['match'] = json.dumps({'id': package_id})
117 except ValueError:
118 self.dict['match'] = json.dumps({'name': package_id})
119 #---------------------------------------------------------------------#
120
121 if kwargs.get('upload', None):
122 blocks = [[]]
123 size_file = 0
124 count_file = 0
125 inter_num = 0
126 for value in self.list:
127 if value['size'] > 1024 * 1024 * float(max_size):
128 return 'ERROR:: The size of the "%s" file is %sMB aprox, please change "max_size" value' % (value['name'], str(round(value['size']/(1024 * 1024), 2)))
129 if not 1 <= int(max_count) <= 999:
130 return 'ERROR:: The count of the number of files must be between 1 and 999, please change "max_count" value'
131
132 size_file = size_file + value['size']
133 count_file = count_file + 1
134 if size_file > 1024 * 1024 * float(max_size) or count_file > int(max_count):
135 inter_num = inter_num + 1
136 size_file = value['size']
137 count_file = 1
138 blocks.append([])
139
140 del value['size']
141 blocks[inter_num].append(value)
142 #------------------------------------------------------------#
143
144 if len(blocks[0]) > 0:
145 print('BLOCK(S) IN TOTAL:: {}'.format(len(blocks)))
146
147 for count1, block in enumerate(blocks):
148 upload_files = {}
149 for count2, value2 in enumerate(block):
150 upload_files['update__resources__-'+ str(len(block)-count2) +'__upload'] = (value2['name'], open(value2['upload'], 'rb'))
151 del value2['upload']
152
153 print('---- BLOCK N°{} ----'.format(count1 + 1))
154 print('BLOCK N°{} :: "{}" file(s) found >> uploading'.format(count1 + 1, len(block)))
155
156 try:
157 result = self.ckan.call_action('package_revise', {'match': self.dict['match'], 'update__resources__extend': json.dumps(block)}, files=upload_files)
158 print('BLOCK N°{} :: Uploaded file(s) successfully'.format(count1 + 1))
159 if len(blocks) == count1 + 1:
160 return result
161 except:
162 print('ERROR :: Use the "print" for more information')
163 _, exc_value, _ = sys.exc_info()
164 return exc_value
165 else:
166 return "ERROR:: No file(s) found to upload"
167 else:
168 if len(self.list) > 0:
169 return self.ckan.call_action('package_revise', {'match': self.dict['match'], 'update__resources__extend': json.dumps(self.list)})
170 else:
171 return "ERROR:: No resource(s) found to create"
172
173
174 def resource_patch(self, id, package_id, max_size=100, max_count=500, **kwargs):
175 #Cambiar el nombre al actualizar con un nuevo archivo
176
177 if 'file_type' in kwargs:
178 kwargs['voc_file_type'] = kwargs['file_type']
179 del kwargs['file_type']
180
181 if 'upload' in kwargs and 'url' in kwargs:
182 return 'ERROR:: Choose one: "upload" or "url" parameters'
183 #---------------------------------------------------------------------#
184
185 if not isinstance(id, list) and not isinstance(id, str):
186 return 'ERROR:: "id" must be <str> or <list>'
187
188 if isinstance(id, str):
189 id = [id]
190
191 change_kwargs = {}
192 for key1, value1 in kwargs.items():
193 if key1 == 'others':
194 if isinstance(value1, tuple):
195 if len(value1) != len(id):
196 return 'ERROR:: "%s" value(s) must be same length as "id" value(s)' % (key1)
197 else:
198 change_kwargs[key1] = value1
199 else:
200 change_kwargs[key1] = (value1,) * len(id)
201 else:
202 if isinstance(value1, list):
203 if len(value1) != len(id):
204 return 'ERROR:: "%s" value(s) must be same length as "id" value(s)' % (key1)
205 else:
206 change_kwargs[key1] = value1
207 else:
208 change_kwargs[key1] = [value1] * len(id)
209 #---------------------------------------------------------------------#
210
211 for c1, v1 in enumerate(id):
212 new_kwargs = {}
213 for k2, v2 in change_kwargs.items():
214 #-------------------------------------------------------------#
215 if k2 == 'upload':
216 if not os.path.isfile(v2[c1]):
217 return 'File "%s" does not exist' % (v2[c1])
218
219 new_kwargs['size'] = os.stat(v2[c1]).st_size
220
221 if k2 == 'url':
222 new_kwargs['clear_upload'] = True
223 new_kwargs['size'] = 0
224 new_kwargs['mimetype'] = None
225 #-------------------------------------------------------------#
226 new_kwargs[k2] = v2[c1]
227
228 if new_kwargs.get('upload', None):
229 #-------------------------------------------------------------#
230 if new_kwargs.get('clear_upload', None):
231 del new_kwargs['clear_upload']
232 #-------------------------------------------------------------#
233 self.dict['update__resources__'+v1[:6]] = new_kwargs
234 else:
235 self.dict['update__resources__'+v1[:6]] = json.dumps(new_kwargs)
236 #---------------------------------------------------------------------#
237
238 try:
239 uuid.UUID(package_id, version=4)
240 package_id = json.dumps({'id': package_id})
241 except ValueError:
242 package_id = json.dumps({'name': package_id})
243 #---------------------------------------------------------------------#
244
245 if kwargs.get('upload', None):
246 blocks = [{}]
247 upload_files = [{}]
248 size_file = 0
249 count_file = 0
250 inter_num = 0
251
252 for dict_key, dict_value in self.dict.items():
253 if dict_value['size'] > 1024 * 1024 * float(max_size):
254 return 'ERROR:: "%s" size out of limit' % (dict_value['upload'])
255
256 if not 1 <= int(max_count) <= 999:
257 return 'ERROR:: The count of the number of files must be between 1 and 999, please change "max_count" value'
258
259 size_file = size_file + dict_value['size']
260 count_file = count_file + 1
261 if size_file <= 1024 * 1024 * float(max_size) and count_file <= int(max_count):
262 upload_files[inter_num][dict_key+'__upload'] = (os.path.basename(dict_value['upload']), open(dict_value['upload'], 'rb'))
263
264 else:
265 inter_num = inter_num + 1
266 size_file = dict_value['size']
267 count_file = 1
268 upload_files.append({dict_key+'__upload': (os.path.basename(dict_value['upload']), open(dict_value['upload'], 'rb'))})
269 blocks.append({})
270
271 del dict_value['upload']
272 del dict_value['size']
273 blocks[inter_num]['match'] = package_id
274 blocks[inter_num][dict_key] = json.dumps(dict_value)
275
276 #------------------------------------------------------------#
277 if len(blocks[0]) > 0:
278 print('BLOCK(S) IN TOTAL:: {}'.format(len(blocks)))
279 for count1, block in enumerate(blocks):
280 print('---- BLOCK N°{} ----'.format(count1 + 1))
281 print('BLOCK N°{} :: "{}" file(s) found >> uploading'.format(count1 + 1, len(block)-1))
282 try:
283 result = self.ckan.call_action('package_revise', block, files=upload_files[count1])
284
285 print('BLOCK N°{} :: Uploaded file(s) successfully'.format(count1 + 1))
286 if len(blocks) == count1 + 1:
287 return result
288 except:
289 print('ERROR :: Use the "print" for more information')
290 _, exc_value, _ = sys.exc_info()
291 return exc_value
292 else:
293 return "ERROR:: No file(s) found to upload"
294 else:
295 self.dict['match'] = package_id
296 return self.ckan.call_action('package_revise', self.dict)
297
298
299 def resource_delete(self, select, id, **kwargs):
300
301 if not isinstance(id, list) and not isinstance(id, str):
302 return 'ERROR:: "id" must be <str> or <list>'
303
304 if isinstance(id, list):
305 if not 'package_id' in kwargs:
306 return "ERROR:: 'package_id' parameter is empty"
307 #---------------------------------------------------------------------#
308
309 if 'delete' == select:
310 if kwargs.get('package_id', None):
311 pkg_dict = getattr(self.ckan.action, 'package_show')(id=kwargs['package_id'])
312
313 if pkg_dict.get('resources'):
314 pkg_dict['resources'] = [res for res in pkg_dict['resources'] if not
315 res['id'] in id]
316 if pkg_dict['num_resources'] - len(pkg_dict['resources']) == len(id):
317 return getattr(self.ckan.action, 'package_update')(**pkg_dict)
318 else:
319 return "ERROR:: No changes have been applied"
320 else:
321 return getattr(self.ckan.action, 'resource_delete')(id=id)
322
323 elif 'purge' == select:
324 if kwargs.get('package_id', None):
325 pkg_dict = getattr(self.ckan.action, 'package_show')(id=kwargs['package_id'])
326 if pkg_dict.get('resources'):
327 pkg_dict['resources'] = [res for res in pkg_dict['resources'] if not
328 res['id'] in id]
329 if pkg_dict['num_resources'] - len(pkg_dict['resources']) == len(id):
330 print('[DELETING FILES]')
331 resource_patch(self=self, id=id, package_id=kwargs['package_id'], clear_upload=True)
332 print('[DELETING RESOURCES]')
333 return getattr(self.ckan.action, 'package_update')(**pkg_dict)
334 else:
335 return "ERROR:: No changes have been applied, please check 'id' parameter"
336
337 else:
338 print('[DELETING FILE]')
339 getattr(self.ckan.action, 'resource_patch')(id=id, clear_upload=True)
340 print('[DELETING RESOURCE]')
341 return getattr(self.ckan.action, 'resource_delete')(id=id)
342 else:
343 return 'ERROR:: "select = %s" is not accepted' % (select) No newline at end of file
Show file before | Show file after | Hide comments
1 NO CONTENT: modified file, binary diff hidden
Show file before | Show file after | Hide comments
@@ -2,6 +2,7 from ckanapi import RemoteCKAN
2 2 from datetime import datetime
3 3 from tqdm import tqdm
4 4 from CKAN_JRO import logic_download
5 from CKAN_JRO import resource
5 6 #from ckanapi.errors import NotAuthorized, NotFound, ValidationError, SearchQueryError, SearchError, CKANAPIError, ServerIncompatibleError
6 7 import sys
7 8 import platform
@@ -318,19 +319,21 class JROAPI():
318 319 resource_extend.append(value2)
319 320
320 321 print('BLOCK N°{} :: "{}" file(s) found >> uploading'.format(count1 + 1, len(block)))
321 try:
322 result = self.ckan.call_action(
323 'package_revise',
324 {'match': '{'+ str(package_id_or_name) +'}', 'update__resources__extend': json.dumps(resource_extend)},
325 files=files_dict
326 )
327 print('BLOCK N°{} :: Uploaded file(s) successfully'.format(count1 + 1))
328 if len(blocks) == count1 + 1:
329 return result
330 except:
331 print('ERROR :: Use the "print" for more information')
332 _, exc_value, _ = sys.exc_info()
333 return exc_value
322 print(resource_extend)
323 print(files_dict)
324 #try:
325 # result = self.ckan.call_action(
326 # 'package_revise',
327 # {'match': '{'+ str(package_id_or_name) +'}', 'update__resources__extend': json.dumps(resource_extend)},
328 # files=files_dict
329 # )
330 # print('BLOCK N°{} :: Uploaded file(s) successfully'.format(count1 + 1))
331 # if len(blocks) == count1 + 1:
332 # return result
333 #except:
334 # print('ERROR :: Use the "print" for more information')
335 # _, exc_value, _ = sys.exc_info()
336 # return exc_value
334 337 else:
335 338 return "ERROR:: No file(s) found to upload"
336 339
@@ -658,6 +661,8 class JROAPI():
658 661 try:
659 662 if type_option == 'dataset':
660 663 return getattr(self.ckan.action, 'package_create')(**kwargs)
664 if type_option == 'resource':
665 return resource.resource_create(self, **kwargs)
661 666 elif type_option == 'project':
662 667 return getattr(self.ckan.action, 'organization_create')(**kwargs)
663 668 elif type_option == 'member':
@@ -703,11 +708,12 class JROAPI():
703 708 if type(type_option) is str:
704 709 try:
705 710 if type_option == 'dataset':
711 #Agregar que solo se debe modificar parámetros del Dataset y que no incluya Resources
706 712 return getattr(self.ckan.action, 'package_patch')(**kwargs)
707 713 elif type_option == 'project':
708 714 return getattr(self.ckan.action, 'organization_patch')(**kwargs)
709 715 elif type_option == 'resource':
710 return getattr(self.ckan.action, 'resource_patch')(**kwargs)
716 return resource.resource_patch(self, **kwargs)
711 717 elif type_option == 'member':
712 718 return getattr(self.ckan.action, 'organization_member_create')(**kwargs)
713 719 elif type_option == 'collaborator':
@@ -754,7 +760,10 class JROAPI():
754 760 else:
755 761 return 'ERROR:: "select = %s" is not accepted' % (select)
756 762 elif type_option == 'resource':
757 return getattr(self.ckan.action, 'resource_delete')(**kwargs)
763 if select is None:
764 return 'ERROR:: "select" must not be "None"'
765 else:
766 return resource.resource_delete(self, select, **kwargs)
758 767 elif type_option == 'vocabulary':
759 768 return getattr(self.ckan.action, 'vocabulary_delete')(**kwargs)
760 769 elif type_option == 'tag':
General Comments 0
You need to be logged in to leave comments. Login now