|
|
import json
|
|
|
import uuid
|
|
|
import os
|
|
|
import sys
|
|
|
|
|
|
def resource_create(self, package_id, file_type, others='', max_size=100, max_count=500, ignore_repetition=False, **kwargs):
|
|
|
#---------------------------------------------------------------------#
|
|
|
kwargs['voc_file_type'] = file_type
|
|
|
kwargs['others'] = others
|
|
|
|
|
|
if 'clear_upload' in kwargs:
|
|
|
del kwargs['clear_upload']
|
|
|
#---------------------------------------------------------------------#
|
|
|
|
|
|
url_upload = {}
|
|
|
if 'upload' in kwargs and 'url' in kwargs:
|
|
|
return 'ERROR:: Choose one: "upload" or "url" parameters'
|
|
|
elif 'upload' in kwargs:
|
|
|
url_upload['upload'] = kwargs['upload']
|
|
|
elif 'url' in kwargs:
|
|
|
url_upload['url'] = kwargs['url']
|
|
|
else:
|
|
|
return 'ERROR:: Missing value: "upload" or "url" parameters'
|
|
|
|
|
|
value_u = list(url_upload.values())[0]
|
|
|
key_u = list(url_upload.keys())[0]
|
|
|
|
|
|
if not isinstance(value_u, list) and not isinstance(value_u, str):
|
|
|
return 'ERROR:: "%s" must be <str> or <list>' % (key_u)
|
|
|
#---------------------------------------------------------------------#
|
|
|
|
|
|
if isinstance(value_u, str):
|
|
|
if len(value_u) != 0:
|
|
|
if key_u == 'upload':
|
|
|
if os.path.isdir(value_u):
|
|
|
path_order = [f for f in os.listdir(value_u) if os.path.isfile(os.path.join(value_u, f))]
|
|
|
path_order.sort()
|
|
|
if path_order:
|
|
|
kwargs['upload'] = []
|
|
|
for name in path_order:
|
|
|
kwargs['upload'].append(os.path.join(value_u, name))
|
|
|
else:
|
|
|
return "ERROR:: There aren't files in this directory"
|
|
|
elif os.path.isfile(value_u):
|
|
|
pass
|
|
|
else:
|
|
|
return 'ERROR:: Directory or File does not exist'
|
|
|
else:
|
|
|
return 'ERROR:: "path_list is empty"'
|
|
|
#---------------------------------------------------------------------#
|
|
|
|
|
|
if not isinstance(kwargs[key_u], list):
|
|
|
kwargs[key_u] = [kwargs[key_u]]
|
|
|
|
|
|
if kwargs.get('upload', None):
|
|
|
if len(kwargs['upload']) != len(set(kwargs['upload'])):
|
|
|
return 'Duplicate files found in "upload" parameter'
|
|
|
#---------------------------------------------------------------------#
|
|
|
|
|
|
change_kwargs = {}
|
|
|
for key1, value1 in kwargs.items():
|
|
|
if key1 == 'others':
|
|
|
if isinstance(value1, tuple):
|
|
|
if len(value1) != len(kwargs[key_u]):
|
|
|
return 'ERROR:: "%s" value(s) must be same length as "%s" value(s)' % (key1, key_u)
|
|
|
else:
|
|
|
change_kwargs[key1] = value1
|
|
|
else:
|
|
|
change_kwargs[key1] = (value1,) * len(kwargs[key_u])
|
|
|
else:
|
|
|
if isinstance(value1, list):
|
|
|
if len(value1) != len(kwargs[key_u]):
|
|
|
return 'ERROR:: "%s" value(s) must be same length as "%s" value(s)' % (key1, key_u)
|
|
|
else:
|
|
|
change_kwargs[key1] = value1
|
|
|
else:
|
|
|
change_kwargs[key1] = [value1] * len(kwargs[key_u])
|
|
|
#---------------------------------------------------------------------#
|
|
|
|
|
|
try:
|
|
|
dataset_show = getattr(self.ckan.action, 'package_show')(id=package_id)['resources']
|
|
|
resources_name = []
|
|
|
for u in dataset_show:
|
|
|
resources_name.append(u['name'].lower())
|
|
|
except:
|
|
|
_, exc_value, _ = sys.exc_info()
|
|
|
print('ERROR obtaining metadata dataset:: Use the "print" for more information')
|
|
|
return exc_value
|
|
|
#---------------------------------------------------------------------#
|
|
|
|
|
|
for c1 in range(len(kwargs[key_u])):
|
|
|
new_kwargs = {}
|
|
|
for k2, v2 in change_kwargs.items():
|
|
|
#-------------------------------------------------------------#
|
|
|
if k2 == 'upload':
|
|
|
if not os.path.isfile(v2[c1]):
|
|
|
return 'File "%s" does not exist' % (v2[c1])
|
|
|
|
|
|
if not kwargs.get('size', None):
|
|
|
new_kwargs['size'] = os.stat(v2[c1]).st_size
|
|
|
#-------------------------------------------------------------#
|
|
|
new_kwargs[k2] = v2[c1]
|
|
|
#-----------------------------------------------------------------#
|
|
|
|
|
|
if not kwargs.get('name', None):
|
|
|
new_kwargs['name'] = os.path.basename(new_kwargs[key_u])
|
|
|
|
|
|
if new_kwargs['name'].lower() in resources_name:
|
|
|
if not ignore_repetition:
|
|
|
return 'ERROR:: "%s" resource already exist in this dataset' % (new_kwargs['name'])
|
|
|
print('WARRING:: "'+ new_kwargs['name'] +'" resource was ignored because already exist in this dataset')
|
|
|
else:
|
|
|
self.list.append(new_kwargs)
|
|
|
#---------------------------------------------------------------------#
|
|
|
|
|
|
try:
|
|
|
uuid.UUID(package_id, version=4)
|
|
|
self.dict['match'] = json.dumps({'id': package_id})
|
|
|
except ValueError:
|
|
|
self.dict['match'] = json.dumps({'name': package_id})
|
|
|
#---------------------------------------------------------------------#
|
|
|
|
|
|
if kwargs.get('upload', None):
|
|
|
blocks = [[]]
|
|
|
size_file = 0
|
|
|
count_file = 0
|
|
|
inter_num = 0
|
|
|
for value in self.list:
|
|
|
if value['size'] > 1024 * 1024 * float(max_size):
|
|
|
return 'ERROR:: The size of the "%s" file is %sMB aprox, please change "max_size" value' % (value['name'], str(round(value['size']/(1024 * 1024), 2)))
|
|
|
if not 1 <= int(max_count) <= 999:
|
|
|
return 'ERROR:: The count of the number of files must be between 1 and 999, please change "max_count" value'
|
|
|
|
|
|
size_file = size_file + value['size']
|
|
|
count_file = count_file + 1
|
|
|
if size_file > 1024 * 1024 * float(max_size) or count_file > int(max_count):
|
|
|
inter_num = inter_num + 1
|
|
|
size_file = value['size']
|
|
|
count_file = 1
|
|
|
blocks.append([])
|
|
|
|
|
|
del value['size']
|
|
|
blocks[inter_num].append(value)
|
|
|
#------------------------------------------------------------#
|
|
|
|
|
|
if len(blocks[0]) > 0:
|
|
|
print('BLOCK(S) IN TOTAL:: {}'.format(len(blocks)))
|
|
|
|
|
|
for count1, block in enumerate(blocks):
|
|
|
upload_files = {}
|
|
|
for count2, value2 in enumerate(block):
|
|
|
upload_files['update__resources__-'+ str(len(block)-count2) +'__upload'] = (value2['name'], open(value2['upload'], 'rb'))
|
|
|
del value2['upload']
|
|
|
|
|
|
print('---- BLOCK N°{} ----'.format(count1 + 1))
|
|
|
print('BLOCK N°{} :: "{}" file(s) found >> uploading'.format(count1 + 1, len(block)))
|
|
|
|
|
|
try:
|
|
|
result = self.ckan.call_action('package_revise', {'match': self.dict['match'], 'update__resources__extend': json.dumps(block)}, files=upload_files)
|
|
|
print('BLOCK N°{} :: Uploaded file(s) successfully'.format(count1 + 1))
|
|
|
if len(blocks) == count1 + 1:
|
|
|
return result
|
|
|
except:
|
|
|
print('ERROR :: Use the "print" for more information')
|
|
|
_, exc_value, _ = sys.exc_info()
|
|
|
return exc_value
|
|
|
else:
|
|
|
return "ERROR:: No file(s) found to upload"
|
|
|
else:
|
|
|
if len(self.list) > 0:
|
|
|
return self.ckan.call_action('package_revise', {'match': self.dict['match'], 'update__resources__extend': json.dumps(self.list)})
|
|
|
else:
|
|
|
return "ERROR:: No resource(s) found to create"
|
|
|
|
|
|
|
|
|
def resource_patch(self, id, package_id, max_size=100, max_count=500, **kwargs):
|
|
|
#Cambiar el nombre al actualizar con un nuevo archivo
|
|
|
|
|
|
if 'file_type' in kwargs:
|
|
|
kwargs['voc_file_type'] = kwargs['file_type']
|
|
|
del kwargs['file_type']
|
|
|
|
|
|
if 'upload' in kwargs and 'url' in kwargs:
|
|
|
return 'ERROR:: Choose one: "upload" or "url" parameters'
|
|
|
#---------------------------------------------------------------------#
|
|
|
|
|
|
if not isinstance(id, list) and not isinstance(id, str):
|
|
|
return 'ERROR:: "id" must be <str> or <list>'
|
|
|
|
|
|
if isinstance(id, str):
|
|
|
id = [id]
|
|
|
|
|
|
change_kwargs = {}
|
|
|
for key1, value1 in kwargs.items():
|
|
|
if key1 == 'others':
|
|
|
if isinstance(value1, tuple):
|
|
|
if len(value1) != len(id):
|
|
|
return 'ERROR:: "%s" value(s) must be same length as "id" value(s)' % (key1)
|
|
|
else:
|
|
|
change_kwargs[key1] = value1
|
|
|
else:
|
|
|
change_kwargs[key1] = (value1,) * len(id)
|
|
|
else:
|
|
|
if isinstance(value1, list):
|
|
|
if len(value1) != len(id):
|
|
|
return 'ERROR:: "%s" value(s) must be same length as "id" value(s)' % (key1)
|
|
|
else:
|
|
|
change_kwargs[key1] = value1
|
|
|
else:
|
|
|
change_kwargs[key1] = [value1] * len(id)
|
|
|
#---------------------------------------------------------------------#
|
|
|
|
|
|
for c1, v1 in enumerate(id):
|
|
|
new_kwargs = {}
|
|
|
for k2, v2 in change_kwargs.items():
|
|
|
#-------------------------------------------------------------#
|
|
|
if k2 == 'upload':
|
|
|
if not os.path.isfile(v2[c1]):
|
|
|
return 'File "%s" does not exist' % (v2[c1])
|
|
|
|
|
|
new_kwargs['size'] = os.stat(v2[c1]).st_size
|
|
|
|
|
|
if k2 == 'url':
|
|
|
new_kwargs['clear_upload'] = True
|
|
|
new_kwargs['size'] = 0
|
|
|
new_kwargs['mimetype'] = None
|
|
|
#-------------------------------------------------------------#
|
|
|
new_kwargs[k2] = v2[c1]
|
|
|
|
|
|
if new_kwargs.get('upload', None):
|
|
|
#-------------------------------------------------------------#
|
|
|
if new_kwargs.get('clear_upload', None):
|
|
|
del new_kwargs['clear_upload']
|
|
|
#-------------------------------------------------------------#
|
|
|
self.dict['update__resources__'+v1[:6]] = new_kwargs
|
|
|
else:
|
|
|
self.dict['update__resources__'+v1[:6]] = json.dumps(new_kwargs)
|
|
|
#---------------------------------------------------------------------#
|
|
|
|
|
|
try:
|
|
|
uuid.UUID(package_id, version=4)
|
|
|
package_id = json.dumps({'id': package_id})
|
|
|
except ValueError:
|
|
|
package_id = json.dumps({'name': package_id})
|
|
|
#---------------------------------------------------------------------#
|
|
|
|
|
|
if kwargs.get('upload', None):
|
|
|
blocks = [{}]
|
|
|
upload_files = [{}]
|
|
|
size_file = 0
|
|
|
count_file = 0
|
|
|
inter_num = 0
|
|
|
|
|
|
for dict_key, dict_value in self.dict.items():
|
|
|
if dict_value['size'] > 1024 * 1024 * float(max_size):
|
|
|
return 'ERROR:: "%s" size out of limit' % (dict_value['upload'])
|
|
|
|
|
|
if not 1 <= int(max_count) <= 999:
|
|
|
return 'ERROR:: The count of the number of files must be between 1 and 999, please change "max_count" value'
|
|
|
|
|
|
size_file = size_file + dict_value['size']
|
|
|
count_file = count_file + 1
|
|
|
if size_file <= 1024 * 1024 * float(max_size) and count_file <= int(max_count):
|
|
|
upload_files[inter_num][dict_key+'__upload'] = (os.path.basename(dict_value['upload']), open(dict_value['upload'], 'rb'))
|
|
|
|
|
|
else:
|
|
|
inter_num = inter_num + 1
|
|
|
size_file = dict_value['size']
|
|
|
count_file = 1
|
|
|
upload_files.append({dict_key+'__upload': (os.path.basename(dict_value['upload']), open(dict_value['upload'], 'rb'))})
|
|
|
blocks.append({})
|
|
|
|
|
|
del dict_value['upload']
|
|
|
del dict_value['size']
|
|
|
blocks[inter_num]['match'] = package_id
|
|
|
blocks[inter_num][dict_key] = json.dumps(dict_value)
|
|
|
|
|
|
#------------------------------------------------------------#
|
|
|
if len(blocks[0]) > 0:
|
|
|
print('BLOCK(S) IN TOTAL:: {}'.format(len(blocks)))
|
|
|
for count1, block in enumerate(blocks):
|
|
|
print('---- BLOCK N°{} ----'.format(count1 + 1))
|
|
|
print('BLOCK N°{} :: "{}" file(s) found >> uploading'.format(count1 + 1, len(block)-1))
|
|
|
try:
|
|
|
result = self.ckan.call_action('package_revise', block, files=upload_files[count1])
|
|
|
|
|
|
print('BLOCK N°{} :: Uploaded file(s) successfully'.format(count1 + 1))
|
|
|
if len(blocks) == count1 + 1:
|
|
|
return result
|
|
|
except:
|
|
|
print('ERROR :: Use the "print" for more information')
|
|
|
_, exc_value, _ = sys.exc_info()
|
|
|
return exc_value
|
|
|
else:
|
|
|
return "ERROR:: No file(s) found to upload"
|
|
|
else:
|
|
|
self.dict['match'] = package_id
|
|
|
return self.ckan.call_action('package_revise', self.dict)
|
|
|
|
|
|
|
|
|
def resource_delete(self, select, id, **kwargs):
|
|
|
|
|
|
if not isinstance(id, list) and not isinstance(id, str):
|
|
|
return 'ERROR:: "id" must be <str> or <list>'
|
|
|
|
|
|
if isinstance(id, list):
|
|
|
if not 'package_id' in kwargs:
|
|
|
return "ERROR:: 'package_id' parameter is empty"
|
|
|
#---------------------------------------------------------------------#
|
|
|
|
|
|
if 'delete' == select:
|
|
|
if kwargs.get('package_id', None):
|
|
|
pkg_dict = getattr(self.ckan.action, 'package_show')(id=kwargs['package_id'])
|
|
|
|
|
|
if pkg_dict.get('resources'):
|
|
|
pkg_dict['resources'] = [res for res in pkg_dict['resources'] if not
|
|
|
res['id'] in id]
|
|
|
if pkg_dict['num_resources'] - len(pkg_dict['resources']) == len(id):
|
|
|
return getattr(self.ckan.action, 'package_update')(**pkg_dict)
|
|
|
else:
|
|
|
return "ERROR:: No changes have been applied"
|
|
|
else:
|
|
|
return getattr(self.ckan.action, 'resource_delete')(id=id)
|
|
|
|
|
|
elif 'purge' == select:
|
|
|
if kwargs.get('package_id', None):
|
|
|
pkg_dict = getattr(self.ckan.action, 'package_show')(id=kwargs['package_id'])
|
|
|
if pkg_dict.get('resources'):
|
|
|
pkg_dict['resources'] = [res for res in pkg_dict['resources'] if not
|
|
|
res['id'] in id]
|
|
|
if pkg_dict['num_resources'] - len(pkg_dict['resources']) == len(id):
|
|
|
print('[DELETING FILES]')
|
|
|
resource_patch(self=self, id=id, package_id=kwargs['package_id'], clear_upload=True)
|
|
|
print('[DELETING RESOURCES]')
|
|
|
return getattr(self.ckan.action, 'package_update')(**pkg_dict)
|
|
|
else:
|
|
|
return "ERROR:: No changes have been applied, please check 'id' parameter"
|
|
|
|
|
|
else:
|
|
|
print('[DELETING FILE]')
|
|
|
getattr(self.ckan.action, 'resource_patch')(id=id, clear_upload=True)
|
|
|
print('[DELETING RESOURCE]')
|
|
|
return getattr(self.ckan.action, 'resource_delete')(id=id)
|
|
|
else:
|
|
|
return 'ERROR:: "select = %s" is not accepted' % (select)
|
