resource.py
344 lines
| 15.0 KiB
| text/x-python
|
PythonLexer
| r15 | import json | ||
| import uuid | |||
| import os | |||
| import sys | |||
| def resource_create(self, package_id, file_type, others='', max_size=100, max_count=500, ignore_repetition=False, **kwargs): | |||
| #---------------------------------------------------------------------# | |||
| kwargs['voc_file_type'] = file_type | |||
| kwargs['others'] = others | |||
| if 'clear_upload' in kwargs: | |||
| del kwargs['clear_upload'] | |||
| #---------------------------------------------------------------------# | |||
| url_upload = {} | |||
| if 'upload' in kwargs and 'url' in kwargs: | |||
| return 'ERROR:: Choose one: "upload" or "url" parameters' | |||
| elif 'upload' in kwargs: | |||
| url_upload['upload'] = kwargs['upload'] | |||
| elif 'url' in kwargs: | |||
| url_upload['url'] = kwargs['url'] | |||
| else: | |||
| return 'ERROR:: Missing value: "upload" or "url" parameters' | |||
| value_u = list(url_upload.values())[0] | |||
| key_u = list(url_upload.keys())[0] | |||
| if not isinstance(value_u, list) and not isinstance(value_u, str): | |||
| return 'ERROR:: "%s" must be <str> or <list>' % (key_u) | |||
| #---------------------------------------------------------------------# | |||
| if isinstance(value_u, str): | |||
| if len(value_u) != 0: | |||
| if key_u == 'upload': | |||
| if os.path.isdir(value_u): | |||
| path_order = [f for f in os.listdir(value_u) if os.path.isfile(os.path.join(value_u, f))] | |||
| path_order.sort() | |||
| if path_order: | |||
| kwargs['upload'] = [] | |||
| for name in path_order: | |||
| kwargs['upload'].append(os.path.join(value_u, name)) | |||
| else: | |||
| return "ERROR:: There aren't files in this directory" | |||
| r17 | elif os.path.isfile(value_u): | ||
| pass | |||
| r15 | else: | ||
| return 'ERROR:: Directory or File does not exist' | |||
| else: | |||
| return 'ERROR:: "path_list is empty"' | |||
| #---------------------------------------------------------------------# | |||
| if not isinstance(kwargs[key_u], list): | |||
| kwargs[key_u] = [kwargs[key_u]] | |||
| if kwargs.get('upload', None): | |||
| if len(kwargs['upload']) != len(set(kwargs['upload'])): | |||
| return 'Duplicate files found in "upload" parameter' | |||
| #---------------------------------------------------------------------# | |||
| change_kwargs = {} | |||
| for key1, value1 in kwargs.items(): | |||
| if key1 == 'others': | |||
| if isinstance(value1, tuple): | |||
| if len(value1) != len(kwargs[key_u]): | |||
| return 'ERROR:: "%s" value(s) must be same length as "%s" value(s)' % (key1, key_u) | |||
| else: | |||
| change_kwargs[key1] = value1 | |||
| else: | |||
| change_kwargs[key1] = (value1,) * len(kwargs[key_u]) | |||
| else: | |||
| if isinstance(value1, list): | |||
| if len(value1) != len(kwargs[key_u]): | |||
| return 'ERROR:: "%s" value(s) must be same length as "%s" value(s)' % (key1, key_u) | |||
| else: | |||
| change_kwargs[key1] = value1 | |||
| else: | |||
| change_kwargs[key1] = [value1] * len(kwargs[key_u]) | |||
| #---------------------------------------------------------------------# | |||
| try: | |||
| dataset_show = getattr(self.ckan.action, 'package_show')(id=package_id)['resources'] | |||
| resources_name = [] | |||
| for u in dataset_show: | |||
| resources_name.append(u['name'].lower()) | |||
| except: | |||
| _, exc_value, _ = sys.exc_info() | |||
| print('ERROR obtaining metadata dataset:: Use the "print" for more information') | |||
| return exc_value | |||
| #---------------------------------------------------------------------# | |||
| for c1 in range(len(kwargs[key_u])): | |||
| new_kwargs = {} | |||
| for k2, v2 in change_kwargs.items(): | |||
| #-------------------------------------------------------------# | |||
| if k2 == 'upload': | |||
| if not os.path.isfile(v2[c1]): | |||
| return 'File "%s" does not exist' % (v2[c1]) | |||
| if not kwargs.get('size', None): | |||
| new_kwargs['size'] = os.stat(v2[c1]).st_size | |||
| #-------------------------------------------------------------# | |||
| new_kwargs[k2] = v2[c1] | |||
| #-----------------------------------------------------------------# | |||
| if not kwargs.get('name', None): | |||
| new_kwargs['name'] = os.path.basename(new_kwargs[key_u]) | |||
| if new_kwargs['name'].lower() in resources_name: | |||
| if not ignore_repetition: | |||
| return 'ERROR:: "%s" resource already exist in this dataset' % (new_kwargs['name']) | |||
| print('WARRING:: "'+ new_kwargs['name'] +'" resource was ignored because already exist in this dataset') | |||
| else: | |||
| self.list.append(new_kwargs) | |||
| #---------------------------------------------------------------------# | |||
| try: | |||
| uuid.UUID(package_id, version=4) | |||
| self.dict['match'] = json.dumps({'id': package_id}) | |||
| except ValueError: | |||
| self.dict['match'] = json.dumps({'name': package_id}) | |||
| #---------------------------------------------------------------------# | |||
| if kwargs.get('upload', None): | |||
| blocks = [[]] | |||
| size_file = 0 | |||
| count_file = 0 | |||
| inter_num = 0 | |||
| for value in self.list: | |||
| if value['size'] > 1024 * 1024 * float(max_size): | |||
| return 'ERROR:: The size of the "%s" file is %sMB aprox, please change "max_size" value' % (value['name'], str(round(value['size']/(1024 * 1024), 2))) | |||
| if not 1 <= int(max_count) <= 999: | |||
| return 'ERROR:: The count of the number of files must be between 1 and 999, please change "max_count" value' | |||
| size_file = size_file + value['size'] | |||
| count_file = count_file + 1 | |||
| if size_file > 1024 * 1024 * float(max_size) or count_file > int(max_count): | |||
| inter_num = inter_num + 1 | |||
| size_file = value['size'] | |||
| count_file = 1 | |||
| blocks.append([]) | |||
| del value['size'] | |||
| blocks[inter_num].append(value) | |||
| #------------------------------------------------------------# | |||
| if len(blocks[0]) > 0: | |||
| print('BLOCK(S) IN TOTAL:: {}'.format(len(blocks))) | |||
| for count1, block in enumerate(blocks): | |||
| upload_files = {} | |||
| for count2, value2 in enumerate(block): | |||
| upload_files['update__resources__-'+ str(len(block)-count2) +'__upload'] = (value2['name'], open(value2['upload'], 'rb')) | |||
| del value2['upload'] | |||
| print('---- BLOCK N°{} ----'.format(count1 + 1)) | |||
| print('BLOCK N°{} :: "{}" file(s) found >> uploading'.format(count1 + 1, len(block))) | |||
| try: | |||
| result = self.ckan.call_action('package_revise', {'match': self.dict['match'], 'update__resources__extend': json.dumps(block)}, files=upload_files) | |||
| print('BLOCK N°{} :: Uploaded file(s) successfully'.format(count1 + 1)) | |||
| if len(blocks) == count1 + 1: | |||
| return result | |||
| except: | |||
| print('ERROR :: Use the "print" for more information') | |||
| _, exc_value, _ = sys.exc_info() | |||
| return exc_value | |||
| else: | |||
| return "ERROR:: No file(s) found to upload" | |||
| else: | |||
| if len(self.list) > 0: | |||
| return self.ckan.call_action('package_revise', {'match': self.dict['match'], 'update__resources__extend': json.dumps(self.list)}) | |||
| else: | |||
| return "ERROR:: No resource(s) found to create" | |||
| def resource_patch(self, id, package_id, max_size=100, max_count=500, **kwargs): | |||
| #Cambiar el nombre al actualizar con un nuevo archivo | |||
| if 'file_type' in kwargs: | |||
| kwargs['voc_file_type'] = kwargs['file_type'] | |||
| del kwargs['file_type'] | |||
| if 'upload' in kwargs and 'url' in kwargs: | |||
| return 'ERROR:: Choose one: "upload" or "url" parameters' | |||
| #---------------------------------------------------------------------# | |||
| if not isinstance(id, list) and not isinstance(id, str): | |||
| return 'ERROR:: "id" must be <str> or <list>' | |||
| if isinstance(id, str): | |||
| id = [id] | |||
| change_kwargs = {} | |||
| for key1, value1 in kwargs.items(): | |||
| if key1 == 'others': | |||
| if isinstance(value1, tuple): | |||
| if len(value1) != len(id): | |||
| return 'ERROR:: "%s" value(s) must be same length as "id" value(s)' % (key1) | |||
| else: | |||
| change_kwargs[key1] = value1 | |||
| else: | |||
| change_kwargs[key1] = (value1,) * len(id) | |||
| else: | |||
| if isinstance(value1, list): | |||
| if len(value1) != len(id): | |||
| return 'ERROR:: "%s" value(s) must be same length as "id" value(s)' % (key1) | |||
| else: | |||
| change_kwargs[key1] = value1 | |||
| else: | |||
| change_kwargs[key1] = [value1] * len(id) | |||
| #---------------------------------------------------------------------# | |||
| for c1, v1 in enumerate(id): | |||
| new_kwargs = {} | |||
| for k2, v2 in change_kwargs.items(): | |||
| #-------------------------------------------------------------# | |||
| if k2 == 'upload': | |||
| if not os.path.isfile(v2[c1]): | |||
| return 'File "%s" does not exist' % (v2[c1]) | |||
| new_kwargs['size'] = os.stat(v2[c1]).st_size | |||
| if k2 == 'url': | |||
| new_kwargs['clear_upload'] = True | |||
| new_kwargs['size'] = 0 | |||
| new_kwargs['mimetype'] = None | |||
| #-------------------------------------------------------------# | |||
| new_kwargs[k2] = v2[c1] | |||
| if new_kwargs.get('upload', None): | |||
| #-------------------------------------------------------------# | |||
| if new_kwargs.get('clear_upload', None): | |||
| del new_kwargs['clear_upload'] | |||
| #-------------------------------------------------------------# | |||
| self.dict['update__resources__'+v1[:6]] = new_kwargs | |||
| else: | |||
| self.dict['update__resources__'+v1[:6]] = json.dumps(new_kwargs) | |||
| #---------------------------------------------------------------------# | |||
| try: | |||
| uuid.UUID(package_id, version=4) | |||
| package_id = json.dumps({'id': package_id}) | |||
| except ValueError: | |||
| package_id = json.dumps({'name': package_id}) | |||
| #---------------------------------------------------------------------# | |||
| if kwargs.get('upload', None): | |||
| blocks = [{}] | |||
| upload_files = [{}] | |||
| size_file = 0 | |||
| count_file = 0 | |||
| inter_num = 0 | |||
| for dict_key, dict_value in self.dict.items(): | |||
| if dict_value['size'] > 1024 * 1024 * float(max_size): | |||
| return 'ERROR:: "%s" size out of limit' % (dict_value['upload']) | |||
| if not 1 <= int(max_count) <= 999: | |||
| return 'ERROR:: The count of the number of files must be between 1 and 999, please change "max_count" value' | |||
| size_file = size_file + dict_value['size'] | |||
| count_file = count_file + 1 | |||
| if size_file <= 1024 * 1024 * float(max_size) and count_file <= int(max_count): | |||
| upload_files[inter_num][dict_key+'__upload'] = (os.path.basename(dict_value['upload']), open(dict_value['upload'], 'rb')) | |||
| else: | |||
| inter_num = inter_num + 1 | |||
| size_file = dict_value['size'] | |||
| count_file = 1 | |||
| upload_files.append({dict_key+'__upload': (os.path.basename(dict_value['upload']), open(dict_value['upload'], 'rb'))}) | |||
| blocks.append({}) | |||
| del dict_value['upload'] | |||
| del dict_value['size'] | |||
| blocks[inter_num]['match'] = package_id | |||
| blocks[inter_num][dict_key] = json.dumps(dict_value) | |||
| #------------------------------------------------------------# | |||
| if len(blocks[0]) > 0: | |||
| print('BLOCK(S) IN TOTAL:: {}'.format(len(blocks))) | |||
| for count1, block in enumerate(blocks): | |||
| print('---- BLOCK N°{} ----'.format(count1 + 1)) | |||
| print('BLOCK N°{} :: "{}" file(s) found >> uploading'.format(count1 + 1, len(block)-1)) | |||
| try: | |||
| result = self.ckan.call_action('package_revise', block, files=upload_files[count1]) | |||
| print('BLOCK N°{} :: Uploaded file(s) successfully'.format(count1 + 1)) | |||
| if len(blocks) == count1 + 1: | |||
| return result | |||
| except: | |||
| print('ERROR :: Use the "print" for more information') | |||
| _, exc_value, _ = sys.exc_info() | |||
| return exc_value | |||
| else: | |||
| return "ERROR:: No file(s) found to upload" | |||
| else: | |||
| self.dict['match'] = package_id | |||
| return self.ckan.call_action('package_revise', self.dict) | |||
| def resource_delete(self, select, id, **kwargs): | |||
| if not isinstance(id, list) and not isinstance(id, str): | |||
| return 'ERROR:: "id" must be <str> or <list>' | |||
| if isinstance(id, list): | |||
| if not 'package_id' in kwargs: | |||
| return "ERROR:: 'package_id' parameter is empty" | |||
| #---------------------------------------------------------------------# | |||
| if 'delete' == select: | |||
| if kwargs.get('package_id', None): | |||
| pkg_dict = getattr(self.ckan.action, 'package_show')(id=kwargs['package_id']) | |||
| if pkg_dict.get('resources'): | |||
| pkg_dict['resources'] = [res for res in pkg_dict['resources'] if not | |||
| res['id'] in id] | |||
| if pkg_dict['num_resources'] - len(pkg_dict['resources']) == len(id): | |||
| return getattr(self.ckan.action, 'package_update')(**pkg_dict) | |||
| else: | |||
| return "ERROR:: No changes have been applied" | |||
| else: | |||
| return getattr(self.ckan.action, 'resource_delete')(id=id) | |||
| elif 'purge' == select: | |||
| if kwargs.get('package_id', None): | |||
| pkg_dict = getattr(self.ckan.action, 'package_show')(id=kwargs['package_id']) | |||
| if pkg_dict.get('resources'): | |||
| pkg_dict['resources'] = [res for res in pkg_dict['resources'] if not | |||
| res['id'] in id] | |||
| if pkg_dict['num_resources'] - len(pkg_dict['resources']) == len(id): | |||
| print('[DELETING FILES]') | |||
| resource_patch(self=self, id=id, package_id=kwargs['package_id'], clear_upload=True) | |||
| print('[DELETING RESOURCES]') | |||
| return getattr(self.ckan.action, 'package_update')(**pkg_dict) | |||
| else: | |||
| return "ERROR:: No changes have been applied, please check 'id' parameter" | |||
| else: | |||
| print('[DELETING FILE]') | |||
| getattr(self.ckan.action, 'resource_patch')(id=id, clear_upload=True) | |||
| print('[DELETING RESOURCE]') | |||
| return getattr(self.ckan.action, 'resource_delete')(id=id) | |||
| else: | |||
| return 'ERROR:: "select = %s" is not accepted' % (select) |
