DATABASES/ckanext-jro/api-cliente Commit - r4:27e924dd52b0

v2.9.2 :: Update 'upload_multiple_files_advance' api and Add views resource

eynilupu -

r4:27e924dd52b0

parent child

Context file:

r4:27e924dd52b0

Collapse all files

API_CLIENTE.pdf binary modified 0 0

NO CONTENT: modified file, binary diff hidden

script/CKAN_JRO/jro_api.py +93 -47

             from ckanapi import RemoteCKAN
+            from datetime import datetime
+            from tqdm import tqdm
             #from ckanapi.errors import NotAuthorized, NotFound, ValidationError, SearchQueryError, SearchError, CKANAPIError, ServerIncompatibleError
             import sys
             import platform
             import json
             import pathlib
             import uuid
-            from datetime import datetime
-            from tqdm import tqdm
             class JROAPI():
                 """
                         En Python 2
                             - pip install ckanapi==4.5
                             - pip install requests
+                            - pip install pathlib
+                            - pip install futures
                             - pip install tqdm
-                        En Python 3
+                        En Python > 3
                             - pip3 install ckanapi==4.5
                             - pip3 install requests
                             - pip3 install tqdm
                 REPORTAR ALGUN PROBLEMA:
                     Debe enviar un correo a eynilupu@igp.gob.pe detallando los siguientes pasos:
-) Identifiquese
+) Correo para contactarlo
-) Describir el problema
+) Descripcion del problema
-) ¿En que funcion esta el problema?
+) ¿En que paso o seccion encontro el problema?
-) ¿Que esperaba que hiciera la funcion sin el problema?
+) ¿Cual era el resultado que usted esperaba?
                 """
                 def __init__(self, url, Authorization=None):
                     ua = 'CKAN_JRO/1.1 (+'+str(url)+')'
                         return exc_value
                 def upload_file(self, dataset_id, file_path, file_date, file_type, **kwargs):
+                    # Agregar si es interruptido por teclado
                     '''
                     FINALIDAD:
                         Funcion para subir un unico archivo al repositorio del ROJ.
                         _, exc_value, _ = sys.exc_info()
                         return exc_value
+                def upload_multiple_files_advance(self, dataset_id, path_files, file_date, file_type, max_size=100, ignore_repetition=False, **kwargs):
-                def upload_multiple_files_advance(self, dataset_id, path_files, file_date, file_type, **kwargs):
+                    # Agregar si es interruptido por teclado
                     '''
                     FINALIDAD:
                         Funcion para subir multiples archivos al repositorio del ROJ.
                     ESTRUCTURA:
                         <access_name>.upload_multiple_files_advance(dataset_id = <class 'str'>, path_files = <class 'list of strings'>, file_date = <class 'str'>, file_type = <class 'str'>, param_1 = <class 'param_1'>, ...)
                     '''
+                    #-------------------------PACKAGE SHOW-----------------------#
+                    try:
+                        dataset_show = getattr(self.ckan.action, 'package_show')(id=dataset_id)['resources']
+                    except:
+                        _, exc_value, _ = sys.exc_info()
+                        print('ERROR obtaining metadata dataset:: Use the "print" for more information')
+                        return exc_value
+                    #------------------------------------------------------------#
+                    resources_name = []
+                    for u in dataset_show:
+                        resources_name.append(u['name'].lower())
+                    #------------------------------------------------------------#
                     self.list = ['package_id', 'upload', 'voc_file_type', 'name']
                     for key1, value1 in kwargs.items():
                         if not key1 in self.list:
                             self.dict[key1] = value1
+                    #------------------------------------------------------------#
-                    #---------------------------#
                     if not 'others' in kwargs:
                         self.dict['others'] = ''
                     else:
                         if isinstance(kwargs['others'], list):
                             self.dict['others'] = json.dumps(kwargs['others'])
-                    #---------------------------#
+                    #------------------------------------------------------------#
+                    total_list = []
-                    params_dict = {'upload':[], 'name':[]}
                     #---------------CASO : "path" or "path_list"-----------------#
                     if type(path_files) is list:
                         if len(path_files) != 0:
                             path_files.sort()
                             for u in path_files:
                                 if os.path.isfile(u):
-                                    params_dict['upload'].append(open(u, 'rb'))
+                                    if pathlib.Path(u).name.lower() in resources_name:
-                                    params_dict['name'].append(pathlib.Path(u).name)
+                                        if not ignore_repetition:
+                                            return 'ERROR:: "%s" file already exist in this dataset' % (pathlib.Path(u).name)
+                                        print('WARRING:: "'+ str(pathlib.Path(u).name) +'" file was ignored because already exist in this dataset')
+                                    else:
+                                        total_list.append({'name':pathlib.Path(u).name, 'size': os.stat(u).st_size, 'upload':open(u, 'rb')})
                                 else:
                                     return 'File "%s" does not exist' % (u)
                         else:
                             return 'ERROR:: "path_list is empty"'
                     elif type(path_files) is str:
                         if os.path.isdir(path_files):
                             path_order = [f for f in os.listdir(path_files) if os.path.isfile(os.path.join(path_files, f))]
                             path_order.sort()
                             if path_order:
                                 for name in path_order:
-                                    params_dict['upload'].append(open(os.path.join(path_files, name), 'rb'))
+                                    if name.lower() in resources_name:
-                                    params_dict['name'].append(name)
+                                        if not ignore_repetition:
+                                            return 'ERROR:: "%s" file already exist in this dataset' % (name)
+                                        print('WARRING:: "'+ name +'" file was ignored because already exist in this dataset')
+                                    else:
+                                        total_list.append({'name':name, 'size': os.stat(os.path.join(path_files, name)).st_size, 'upload':open(os.path.join(path_files, name), 'rb')})
                             else:
                                 return "ERROR:: There aren't files in this directory"
                         else:
                     else:
                         return 'ERROR:: "path_files" must be a str or list'
                     #------------------------------------------------------------#
-                    resource_extend = []
-                    files_dict = {}
-                    for count, name in enumerate(params_dict['name']):
-                        param_list = {"name": name, "file_date": file_date, "voc_file_type": file_type}
-                        param_list.update(self.dict)
-                        if not 'format' in param_list:
-                            format = ''.join(pathlib.Path(name).suffixes)
-                            if len(format) > 0:
-                                param_list['format'] = format.upper()[1:]
-                        resource_extend.append(param_list)
-                        files_dict['update__resources__-'+ str(len(params_dict['name'])-count) +'__upload'] = (name, params_dict['upload'][count])
-                    #------------------------------------------------------------#
                     try:
                         uuid.UUID(str(dataset_id), version=4)
                         package_id_or_name = '"id": "' + str(dataset_id) + '"'
                     except ValueError:
                         package_id_or_name = '"name": "' + str(dataset_id) + '"'
                     #------------------------------------------------------------#
-                    print('"{}" file(s) found >> uploading'.format(len(params_dict['name'])))
+                    blocks = [[]]
-                    try:
+                    size_file = 0
-                        result = self.ckan.call_action(
+                    inter_num = 0
-                            'package_revise',
+                    for value in total_list:
-                            {'match': '{'+ str(package_id_or_name) +'}', 'update__resources__extend': json.dumps(resource_extend)},
+                        if value['size'] > 1048576 * float(max_size):
-                            files=files_dict
+                            return 'ERROR:: The size of the "%s" file is %sMB, please change "max_size" value' % (value['name'], str(round(value['size']/1048576, 1)))
+                        size_file = size_file + value['size']
-                        print('Uploaded file(s) successfully')
+                        if size_file <= 1048576 * float(max_size):
-                        return result
+                            del value['size']
-                    except:
+                            blocks[inter_num].append(value)
-                        print('ERROR :: Use the "print" option for more information')
+                        else:
-                        _, exc_value, _ = sys.exc_info()
+                            inter_num = inter_num + 1
-                        return exc_value
+                            blocks.append([])
+                            del value['size']
+                            blocks[inter_num].append(value)
+                    #------------------------------------------------------------#
+                    if len(blocks[0]) > 0:
+                        print('BLOCK(S) IN TOTAL:: {}'.format(len(blocks)))
+                        for count1, block in enumerate(blocks):
+                            print('---- BLOCK N°{} ----'.format(count1 + 1))
+                            resource_extend = []
+                            files_dict = {}
+                            for count2, value2 in enumerate(block):
+                                value2['file_date'] = file_date
+                                value2['voc_file_type'] = file_type
+                                value2.update(self.dict)
+                                if not 'format' in value2:
+                                    format = ''.join(pathlib.Path(value2['name']).suffixes)
+                                    if len(format) > 0:
+                                        value2['format'] = format.upper()[1:]
+                                files_dict['update__resources__-'+ str(len(block)-count2) +'__upload'] = (value2['name'], value2['upload'])
+                                del value2['upload']
+                                resource_extend.append(value2)
+                            print('BLOCK N°{} :: "{}" file(s) found >> uploading'.format(count1 + 1, len(block)))
+                            try:
+                                result = self.ckan.call_action(
+                                    'package_revise',
+                                    {'match': '{'+ str(package_id_or_name) +'}', 'update__resources__extend': json.dumps(resource_extend)},
+                                    files=files_dict
+                                    )
+                                print('BLOCK N°{} :: Uploaded file(s) successfully'.format(count1 + 1))
+                                if len(blocks) == count1 + 1:
+                                    return result
+                            except:
+                                print('ERROR :: Use the "print" for more information')
+                                _, exc_value, _ = sys.exc_info()
+                                return exc_value
+                    else:
+                        return "ERROR:: No file(s) found to upload"
                 def upload_multiple_files(self, dataset_id, path_files, date_files, type_files, **kwargs):
+                    # Agregar si es interruptido por teclado
                     '''
                     FINALIDAD:
                         Funcion para subir multiples archivos al repositorio del ROJ.

General Comments 0

Write
Preview

You need to be logged in to leave comments. Login now

No TODOs yet

	Site-wide shortcuts
/	Use quick search box
g h	Goto home page
g g	Goto my private gists page
g G	Goto my public gists page
g 0-9	Goto bookmarked items from 0-9
n r	New repository page
n g	New gist page

	Repositories
g s	Goto summary page
g c	Goto changelog page
g f	Goto files page
g F	Goto files page with file search activated
g p	Goto pull requests page
g o	Goto repository settings
g O	Goto repository access permissions settings
t s	Toggle sidebar on some pages