import json import uuid import os import sys def resource_create(self, package_id, file_type, others='', max_size=100, max_count=500, ignore_repetition=False, **kwargs): #---------------------------------------------------------------------# kwargs['voc_file_type'] = file_type kwargs['others'] = others if 'clear_upload' in kwargs: del kwargs['clear_upload'] #---------------------------------------------------------------------# url_upload = {} if 'upload' in kwargs and 'url' in kwargs: return 'ERROR:: Choose one: "upload" or "url" parameters' elif 'upload' in kwargs: url_upload['upload'] = kwargs['upload'] elif 'url' in kwargs: url_upload['url'] = kwargs['url'] else: return 'ERROR:: Missing value: "upload" or "url" parameters' value_u = list(url_upload.values())[0] key_u = list(url_upload.keys())[0] if not isinstance(value_u, list) and not isinstance(value_u, str): return 'ERROR:: "%s" must be or ' % (key_u) #---------------------------------------------------------------------# if isinstance(value_u, str): if len(value_u) != 0: if key_u == 'upload': if os.path.isdir(value_u): path_order = [f for f in os.listdir(value_u) if os.path.isfile(os.path.join(value_u, f))] path_order.sort() if path_order: kwargs['upload'] = [] for name in path_order: kwargs['upload'].append(os.path.join(value_u, name)) else: return "ERROR:: There aren't files in this directory" elif os.path.isfile(value_u): pass else: return 'ERROR:: Directory or File does not exist' else: return 'ERROR:: "path_list is empty"' #---------------------------------------------------------------------# if not isinstance(kwargs[key_u], list): kwargs[key_u] = [kwargs[key_u]] if kwargs.get('upload', None): if len(kwargs['upload']) != len(set(kwargs['upload'])): return 'Duplicate files found in "upload" parameter' #---------------------------------------------------------------------# change_kwargs = {} for key1, value1 in kwargs.items(): if key1 == 'others': if isinstance(value1, tuple): if len(value1) != len(kwargs[key_u]): return 'ERROR:: "%s" value(s) must be same length as "%s" value(s)' % (key1, key_u) else: change_kwargs[key1] = value1 else: change_kwargs[key1] = (value1,) * len(kwargs[key_u]) else: if isinstance(value1, list): if len(value1) != len(kwargs[key_u]): return 'ERROR:: "%s" value(s) must be same length as "%s" value(s)' % (key1, key_u) else: change_kwargs[key1] = value1 else: change_kwargs[key1] = [value1] * len(kwargs[key_u]) #---------------------------------------------------------------------# try: dataset_show = getattr(self.ckan.action, 'package_show')(id=package_id)['resources'] resources_name = [] for u in dataset_show: resources_name.append(u['name'].lower()) except: _, exc_value, _ = sys.exc_info() print('ERROR obtaining metadata dataset:: Use the "print" for more information') return exc_value #---------------------------------------------------------------------# for c1 in range(len(kwargs[key_u])): new_kwargs = {} for k2, v2 in change_kwargs.items(): #-------------------------------------------------------------# if k2 == 'upload': if not os.path.isfile(v2[c1]): return 'File "%s" does not exist' % (v2[c1]) if not kwargs.get('size', None): new_kwargs['size'] = os.stat(v2[c1]).st_size #-------------------------------------------------------------# new_kwargs[k2] = v2[c1] #-----------------------------------------------------------------# if not kwargs.get('name', None): new_kwargs['name'] = os.path.basename(new_kwargs[key_u]) if new_kwargs['name'].lower() in resources_name: if not ignore_repetition: return 'ERROR:: "%s" resource already exist in this dataset' % (new_kwargs['name']) print('WARRING:: "'+ new_kwargs['name'] +'" resource was ignored because already exist in this dataset') else: self.list.append(new_kwargs) #---------------------------------------------------------------------# try: uuid.UUID(package_id, version=4) self.dict['match'] = json.dumps({'id': package_id}) except ValueError: self.dict['match'] = json.dumps({'name': package_id}) #---------------------------------------------------------------------# if kwargs.get('upload', None): blocks = [[]] size_file = 0 count_file = 0 inter_num = 0 for value in self.list: if value['size'] > 1024 * 1024 * float(max_size): return 'ERROR:: The size of the "%s" file is %sMB aprox, please change "max_size" value' % (value['name'], str(round(value['size']/(1024 * 1024), 2))) if not 1 <= int(max_count) <= 999: return 'ERROR:: The count of the number of files must be between 1 and 999, please change "max_count" value' size_file = size_file + value['size'] count_file = count_file + 1 if size_file > 1024 * 1024 * float(max_size) or count_file > int(max_count): inter_num = inter_num + 1 size_file = value['size'] count_file = 1 blocks.append([]) del value['size'] blocks[inter_num].append(value) #------------------------------------------------------------# if len(blocks[0]) > 0: print('BLOCK(S) IN TOTAL:: {}'.format(len(blocks))) for count1, block in enumerate(blocks): upload_files = {} for count2, value2 in enumerate(block): upload_files['update__resources__-'+ str(len(block)-count2) +'__upload'] = (value2['name'], open(value2['upload'], 'rb')) del value2['upload'] print('---- BLOCK N°{} ----'.format(count1 + 1)) print('BLOCK N°{} :: "{}" file(s) found >> uploading'.format(count1 + 1, len(block))) try: result = self.ckan.call_action('package_revise', {'match': self.dict['match'], 'update__resources__extend': json.dumps(block)}, files=upload_files) print('BLOCK N°{} :: Uploaded file(s) successfully'.format(count1 + 1)) if len(blocks) == count1 + 1: return result except: print('ERROR :: Use the "print" for more information') _, exc_value, _ = sys.exc_info() return exc_value else: return "ERROR:: No file(s) found to upload" else: if len(self.list) > 0: return self.ckan.call_action('package_revise', {'match': self.dict['match'], 'update__resources__extend': json.dumps(self.list)}) else: return "ERROR:: No resource(s) found to create" def resource_patch(self, id, package_id, max_size=100, max_count=500, **kwargs): #Cambiar el nombre al actualizar con un nuevo archivo if 'file_type' in kwargs: kwargs['voc_file_type'] = kwargs['file_type'] del kwargs['file_type'] if 'upload' in kwargs and 'url' in kwargs: return 'ERROR:: Choose one: "upload" or "url" parameters' #---------------------------------------------------------------------# if not isinstance(id, list) and not isinstance(id, str): return 'ERROR:: "id" must be or ' if isinstance(id, str): id = [id] change_kwargs = {} for key1, value1 in kwargs.items(): if key1 == 'others': if isinstance(value1, tuple): if len(value1) != len(id): return 'ERROR:: "%s" value(s) must be same length as "id" value(s)' % (key1) else: change_kwargs[key1] = value1 else: change_kwargs[key1] = (value1,) * len(id) else: if isinstance(value1, list): if len(value1) != len(id): return 'ERROR:: "%s" value(s) must be same length as "id" value(s)' % (key1) else: change_kwargs[key1] = value1 else: change_kwargs[key1] = [value1] * len(id) #---------------------------------------------------------------------# for c1, v1 in enumerate(id): new_kwargs = {} for k2, v2 in change_kwargs.items(): #-------------------------------------------------------------# if k2 == 'upload': if not os.path.isfile(v2[c1]): return 'File "%s" does not exist' % (v2[c1]) new_kwargs['size'] = os.stat(v2[c1]).st_size if k2 == 'url': new_kwargs['clear_upload'] = True new_kwargs['size'] = 0 new_kwargs['mimetype'] = None #-------------------------------------------------------------# new_kwargs[k2] = v2[c1] if new_kwargs.get('upload', None): #-------------------------------------------------------------# if new_kwargs.get('clear_upload', None): del new_kwargs['clear_upload'] #-------------------------------------------------------------# self.dict['update__resources__'+v1[:6]] = new_kwargs else: self.dict['update__resources__'+v1[:6]] = json.dumps(new_kwargs) #---------------------------------------------------------------------# try: uuid.UUID(package_id, version=4) package_id = json.dumps({'id': package_id}) except ValueError: package_id = json.dumps({'name': package_id}) #---------------------------------------------------------------------# if kwargs.get('upload', None): blocks = [{}] upload_files = [{}] size_file = 0 count_file = 0 inter_num = 0 for dict_key, dict_value in self.dict.items(): if dict_value['size'] > 1024 * 1024 * float(max_size): return 'ERROR:: "%s" size out of limit' % (dict_value['upload']) if not 1 <= int(max_count) <= 999: return 'ERROR:: The count of the number of files must be between 1 and 999, please change "max_count" value' size_file = size_file + dict_value['size'] count_file = count_file + 1 if size_file <= 1024 * 1024 * float(max_size) and count_file <= int(max_count): upload_files[inter_num][dict_key+'__upload'] = (os.path.basename(dict_value['upload']), open(dict_value['upload'], 'rb')) else: inter_num = inter_num + 1 size_file = dict_value['size'] count_file = 1 upload_files.append({dict_key+'__upload': (os.path.basename(dict_value['upload']), open(dict_value['upload'], 'rb'))}) blocks.append({}) del dict_value['upload'] del dict_value['size'] blocks[inter_num]['match'] = package_id blocks[inter_num][dict_key] = json.dumps(dict_value) #------------------------------------------------------------# if len(blocks[0]) > 0: print('BLOCK(S) IN TOTAL:: {}'.format(len(blocks))) for count1, block in enumerate(blocks): print('---- BLOCK N°{} ----'.format(count1 + 1)) print('BLOCK N°{} :: "{}" file(s) found >> uploading'.format(count1 + 1, len(block)-1)) try: result = self.ckan.call_action('package_revise', block, files=upload_files[count1]) print('BLOCK N°{} :: Uploaded file(s) successfully'.format(count1 + 1)) if len(blocks) == count1 + 1: return result except: print('ERROR :: Use the "print" for more information') _, exc_value, _ = sys.exc_info() return exc_value else: return "ERROR:: No file(s) found to upload" else: self.dict['match'] = package_id return self.ckan.call_action('package_revise', self.dict) def resource_delete(self, select, id, **kwargs): if not isinstance(id, list) and not isinstance(id, str): return 'ERROR:: "id" must be or ' if isinstance(id, list): if not 'package_id' in kwargs: return "ERROR:: 'package_id' parameter is empty" #---------------------------------------------------------------------# if 'delete' == select: if kwargs.get('package_id', None): pkg_dict = getattr(self.ckan.action, 'package_show')(id=kwargs['package_id']) if pkg_dict.get('resources'): pkg_dict['resources'] = [res for res in pkg_dict['resources'] if not res['id'] in id] if pkg_dict['num_resources'] - len(pkg_dict['resources']) == len(id): return getattr(self.ckan.action, 'package_update')(**pkg_dict) else: return "ERROR:: No changes have been applied" else: return getattr(self.ckan.action, 'resource_delete')(id=id) elif 'purge' == select: if kwargs.get('package_id', None): pkg_dict = getattr(self.ckan.action, 'package_show')(id=kwargs['package_id']) if pkg_dict.get('resources'): pkg_dict['resources'] = [res for res in pkg_dict['resources'] if not res['id'] in id] if pkg_dict['num_resources'] - len(pkg_dict['resources']) == len(id): print('[DELETING FILES]') resource_patch(self=self, id=id, package_id=kwargs['package_id'], clear_upload=True) print('[DELETING RESOURCES]') return getattr(self.ckan.action, 'package_update')(**pkg_dict) else: return "ERROR:: No changes have been applied, please check 'id' parameter" else: print('[DELETING FILE]') getattr(self.ckan.action, 'resource_patch')(id=id, clear_upload=True) print('[DELETING RESOURCE]') return getattr(self.ckan.action, 'resource_delete')(id=id) else: return 'ERROR:: "select = %s" is not accepted' % (select)