| 1 | NO CONTENT: modified file, binary diff hidden |
|
NO CONTENT: modified file, binary diff hidden |
| @@ -1,1046 +1,1044 | |||||
| 1 | from ckanapi import RemoteCKAN |
|
1 | from ckanapi import RemoteCKAN | |
| 2 | from datetime import datetime |
|
2 | from datetime import datetime | |
| 3 | from tqdm import tqdm |
|
3 | from tqdm import tqdm | |
| 4 | from CKAN_JRO import logic_download |
|
4 | from CKAN_JRO import logic_download | |
| 5 | from CKAN_JRO import resource |
|
5 | from CKAN_JRO import resource | |
| 6 | #from ckanapi.errors import NotAuthorized, NotFound, ValidationError, SearchQueryError, SearchError, CKANAPIError, ServerIncompatibleError |
|
6 | #from ckanapi.errors import NotAuthorized, NotFound, ValidationError, SearchQueryError, SearchError, CKANAPIError, ServerIncompatibleError | |
| 7 | import sys |
|
7 | import sys | |
| 8 | import platform |
|
8 | import platform | |
| 9 | import os |
|
9 | import os | |
| 10 | import tempfile |
|
10 | import tempfile | |
| 11 | import shutil |
|
11 | import shutil | |
| 12 | import zipfile |
|
12 | import zipfile | |
| 13 | import concurrent.futures |
|
13 | import concurrent.futures | |
| 14 | import requests |
|
14 | import requests | |
| 15 | import json |
|
15 | import json | |
| 16 | #import pathlib |
|
16 | #import pathlib | |
| 17 | import uuid |
|
17 | import uuid | |
| 18 |
|
18 | |||
| 19 | if sys.version_info.major == 3: |
|
19 | if sys.version_info.major == 3: | |
| 20 | from urllib.parse import urlparse |
|
20 | from urllib.parse import urlparse | |
| 21 | else: |
|
21 | else: | |
| 22 | import urlparse |
|
22 | import urlparse | |
| 23 |
|
23 | |||
| 24 | class JROAPI(): |
|
24 | class JROAPI(): | |
| 25 | """ |
|
25 | """ | |
| 26 | FINALIDAD: |
|
26 | FINALIDAD: | |
| 27 | Script para administrar y obtener la data del repositorio por medio de APIs. |
|
27 | Script para administrar y obtener la data del repositorio por medio de APIs. | |
| 28 |
|
28 | |||
| 29 | REQUISITIOS PREVIOS: |
|
29 | REQUISITIOS PREVIOS: | |
| 30 | - Paso 1: Tener "pip [Python 2]" o "pip3 [Python 3]" instalado: |
|
30 | - Paso 1: Tener "pip [Python 2]" o "pip3 [Python 3]" instalado: | |
| 31 | - Paso 2: Instalar lo siguiente como admininstrador: |
|
31 | - Paso 2: Instalar lo siguiente como admininstrador: | |
| 32 | En Python 2 |
|
32 | En Python 2 | |
| 33 | - pip install ckanapi==4.5 |
|
33 | - pip install ckanapi==4.5 | |
| 34 | - pip install requests |
|
34 | - pip install requests | |
| 35 | - pip install futures |
|
35 | - pip install futures | |
| 36 | - pip install tqdm |
|
36 | - pip install tqdm | |
| 37 | En Python > 3 |
|
37 | En Python > 3 | |
| 38 | - pip3 install ckanapi==4.5 |
|
38 | - pip3 install ckanapi==4.5 | |
| 39 | - pip3 install requests |
|
39 | - pip3 install requests | |
| 40 | - pip3 install tqdm |
|
40 | - pip3 install tqdm | |
| 41 |
|
41 | |||
| 42 | FUNCIONES DISPONIBLES: |
|
42 | FUNCIONES DISPONIBLES: | |
| 43 | - action |
|
43 | - action | |
| 44 | - upload_file |
|
44 | - upload_file | |
| 45 | - upload_multiple_files |
|
45 | - upload_multiple_files | |
| 46 | - upload_multiple_files_advance |
|
46 | - upload_multiple_files_advance | |
| 47 | - show |
|
47 | - show | |
| 48 | - search |
|
48 | - search | |
| 49 | - create |
|
49 | - create | |
| 50 | - patch |
|
50 | - patch | |
| 51 | - delete |
|
51 | - delete | |
| 52 | - download_files |
|
52 | - download_files | |
| 53 |
|
53 | |||
| 54 | EJEMPLOS: |
|
54 | EJEMPLOS: | |
| 55 | #1: |
|
55 | #1: | |
| 56 | with JROAPI('http://demo.example.com', Authorization='#########') as <access_name>: |
|
56 | with JROAPI('http://demo.example.com', Authorization='#########') as <access_name>: | |
| 57 | ... some operation(s) ... |
|
57 | ... some operation(s) ... | |
| 58 | #2: |
|
58 | #2: | |
| 59 | <access_name> = JROAPI('http://example.com', Authorization='#########') |
|
59 | <access_name> = JROAPI('http://example.com', Authorization='#########') | |
| 60 | ... some operation(s) ... |
|
60 | ... some operation(s) ... | |
| 61 | <access_name>.ckan.close() |
|
61 | <access_name>.ckan.close() | |
| 62 |
|
62 | |||
| 63 | REPORTAR ALGUN PROBLEMA: |
|
63 | REPORTAR ALGUN PROBLEMA: | |
| 64 | Debe enviar un correo a eynilupu@igp.gob.pe detallando los siguientes pasos: |
|
64 | Debe enviar un correo a eynilupu@igp.gob.pe detallando los siguientes pasos: | |
| 65 | 1) Correo para contactarlo |
|
65 | 1) Correo para contactarlo | |
| 66 | 2) Descripcion del problema |
|
66 | 2) Descripcion del problema | |
| 67 | 3) ¿En que paso o seccion encontro el problema? |
|
67 | 3) ¿En que paso o seccion encontro el problema? | |
| 68 | 4) ¿Cual era el resultado que usted esperaba? |
|
68 | 4) ¿Cual era el resultado que usted esperaba? | |
| 69 | """ |
|
69 | """ | |
| 70 | def __init__(self, url, Authorization=None, secure=True): |
|
70 | def __init__(self, url, Authorization=None, secure=True): | |
| 71 | #-------- Check Secure -------# |
|
71 | #-------- Check Secure -------# | |
| 72 | self.verify = secure |
|
72 | self.verify = secure | |
| 73 | if not secure and isinstance(secure, bool): |
|
73 | if not secure and isinstance(secure, bool): | |
| 74 | session = requests.Session() |
|
74 | session = requests.Session() | |
| 75 | session.verify = False |
|
75 | session.verify = False | |
| 76 | else: |
|
76 | else: | |
| 77 | session = None |
|
77 | session = None | |
| 78 | #------------------------------# |
|
78 | #------------------------------# | |
| 79 | self.url = url |
|
79 | self.url = url | |
| 80 | ua = 'CKAN_JRO/2.9.2 (+'+str(self.url)+')' |
|
80 | ua = 'CKAN_JRO/2.9.2 (+'+str(self.url)+')' | |
| 81 | #ua = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36' |
|
81 | #ua = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36' | |
| 82 | self.ckan = RemoteCKAN(self.url, apikey=Authorization, user_agent=ua, session=session) |
|
82 | self.ckan = RemoteCKAN(self.url, apikey=Authorization, user_agent=ua, session=session) | |
| 83 | #self.ckan = RemoteCKAN(self.url, apikey=Authorization) |
|
83 | #self.ckan = RemoteCKAN(self.url, apikey=Authorization) | |
| 84 | self.Authorization = Authorization |
|
84 | self.Authorization = Authorization | |
| 85 | # Change for --> self.separator = os.sep |
|
85 | # Change for --> self.separator = os.sep | |
| 86 | if platform.system() == 'Windows': |
|
86 | if platform.system() == 'Windows': | |
| 87 | self.separator = '\\' |
|
87 | self.separator = '\\' | |
| 88 | else: |
|
88 | else: | |
| 89 | self.separator = '/' |
|
89 | self.separator = '/' | |
| 90 |
|
90 | |||
| 91 | self.chunk_size = 1024 |
|
91 | self.chunk_size = 1024 | |
| 92 | self.list = [] |
|
92 | self.list = [] | |
| 93 | self.dict = {} |
|
93 | self.dict = {} | |
| 94 | self.str = '' |
|
94 | self.str = '' | |
| 95 | self.check = 1 |
|
95 | self.check = 1 | |
| 96 | self.cont = 0 |
|
96 | self.cont = 0 | |
| 97 |
|
97 | |||
| 98 | def __enter__(self): |
|
98 | def __enter__(self): | |
| 99 | return self |
|
99 | return self | |
| 100 |
|
100 | |||
| 101 | def __exit__(self, *args): |
|
101 | def __exit__(self, *args): | |
| 102 | self.ckan.close() |
|
102 | self.ckan.close() | |
| 103 |
|
103 | |||
| 104 | def action(self, action, **kwargs): |
|
104 | def action(self, action, **kwargs): | |
| 105 | """ |
|
105 | """ | |
| 106 | FINALIDAD: |
|
106 | FINALIDAD: | |
| 107 | Funcion para llamar a las APIs disponibles |
|
107 | Funcion para llamar a las APIs disponibles | |
| 108 |
|
108 | |||
| 109 | APIs DISPONIBLES: |
|
109 | APIs DISPONIBLES: | |
| 110 | CONSULTAR: "GUIA DE SCRIPT.pdf" |
|
110 | CONSULTAR: "GUIA DE SCRIPT.pdf" | |
| 111 |
|
111 | |||
| 112 | EJEMPLO: |
|
112 | EJEMPLO: | |
| 113 | <access_name>.action(<consuming API>, param_1 = <class 'param_1'>, ...) |
|
113 | <access_name>.action(<consuming API>, param_1 = <class 'param_1'>, ...) | |
| 114 | """ |
|
114 | """ | |
| 115 | #--------------- CASE: PACKAGE SEARCH ---------------# |
|
115 | #--------------- CASE: PACKAGE SEARCH ---------------# | |
| 116 | if kwargs is not None: |
|
116 | if kwargs is not None: | |
| 117 | if action == 'package_search': |
|
117 | if action == 'package_search': | |
| 118 | self.list = ['facet_mincount', 'facet_limit', 'facet_field'] |
|
118 | self.list = ['facet_mincount', 'facet_limit', 'facet_field'] | |
| 119 | for facet in self.list: |
|
119 | for facet in self.list: | |
| 120 | if facet in kwargs: |
|
120 | if facet in kwargs: | |
| 121 | kwargs[facet.replace('_', '.')] = kwargs[facet] |
|
121 | kwargs[facet.replace('_', '.')] = kwargs[facet] | |
| 122 | kwargs.pop(facet) |
|
122 | kwargs.pop(facet) | |
| 123 | #----------------------------------------------------# |
|
123 | #----------------------------------------------------# | |
| 124 | try: |
|
124 | try: | |
| 125 | return getattr(self.ckan.action, action)(**kwargs) |
|
125 | return getattr(self.ckan.action, action)(**kwargs) | |
| 126 | except: |
|
126 | except: | |
| 127 | _, exc_value, _ = sys.exc_info() |
|
127 | _, exc_value, _ = sys.exc_info() | |
| 128 | return exc_value |
|
128 | return exc_value | |
| 129 |
|
129 | |||
| 130 | def upload_file(self, dataset_id, file_date, file_type, file_path=False, url_or_path=False, ignore_repetition=False, **kwargs): |
|
130 | def upload_file(self, dataset_id, file_date, file_type, file_path=False, url_or_path=False, ignore_repetition=False, **kwargs): | |
| 131 | # Agregar si es interruptido por teclado |
|
131 | # Agregar si es interruptido por teclado | |
| 132 | ''' |
|
132 | ''' | |
| 133 | FINALIDAD: |
|
133 | FINALIDAD: | |
| 134 | Funcion para crear un unico recurso (puede incluir un archivo asociado) al repositorio del ROJ. |
|
134 | Funcion para crear un unico recurso (puede incluir un archivo asociado) al repositorio del ROJ. | |
| 135 |
|
135 | |||
| 136 | PARAMETROS DISPONIBLES: |
|
136 | PARAMETROS DISPONIBLES: | |
| 137 | CONSULTAR: "GUIA DE SCRIPT.pdf" |
|
137 | CONSULTAR: "GUIA DE SCRIPT.pdf" | |
| 138 |
|
138 | |||
| 139 | ESTRUCTURA: |
|
139 | ESTRUCTURA: | |
| 140 | <access_name>.upload_file(dataset_id = <class 'str'>, file_date = <class 'str'>, file_type = <class 'str'>, file_path = <class 'str'>, url_or_path = <class 'str'>, param_1 = <class 'param_1'>, ...) |
|
140 | <access_name>.upload_file(dataset_id = <class 'str'>, file_date = <class 'str'>, file_type = <class 'str'>, file_path = <class 'str'>, url_or_path = <class 'str'>, param_1 = <class 'param_1'>, ...) | |
| 141 | ''' |
|
141 | ''' | |
| 142 | #self.list = ['package_id', 'upload', 'voc_file_type', 'name'] #file_date |
|
142 | #self.list = ['package_id', 'upload', 'voc_file_type', 'name'] #file_date | |
| 143 | self.list = ['package_id', 'upload', 'voc_file_type'] #file_date |
|
143 | self.list = ['package_id', 'upload', 'voc_file_type'] #file_date | |
| 144 | for key1, value1 in kwargs.items(): |
|
144 | for key1, value1 in kwargs.items(): | |
| 145 | if not key1 in self.list: |
|
145 | if not key1 in self.list: | |
| 146 | self.dict[key1] = value1 |
|
146 | self.dict[key1] = value1 | |
| 147 |
|
147 | |||
| 148 | #---------------------------# |
|
148 | #---------------------------# | |
| 149 | if not 'others' in kwargs: |
|
149 | if not 'others' in kwargs: | |
| 150 | self.dict['others'] = '' |
|
150 | self.dict['others'] = '' | |
| 151 | else: |
|
151 | else: | |
| 152 | if isinstance(kwargs['others'], list): |
|
152 | if isinstance(kwargs['others'], list): | |
| 153 | self.dict['others'] = json.dumps(kwargs['others']) |
|
153 | self.dict['others'] = json.dumps(kwargs['others']) | |
| 154 | #---------------------------# |
|
154 | #---------------------------# | |
| 155 |
|
155 | |||
| 156 | if isinstance(file_path, str) and isinstance(url_or_path, str): |
|
156 | if isinstance(file_path, str) and isinstance(url_or_path, str): | |
| 157 | return 'ERROR:: Choose one: "file_path" or "url_or_path" parameters' |
|
157 | return 'ERROR:: Choose one: "file_path" or "url_or_path" parameters' | |
| 158 |
|
158 | |||
| 159 | if isinstance(file_path, str): |
|
159 | if isinstance(file_path, str): | |
| 160 | if not os.path.isfile(file_path): |
|
160 | if not os.path.isfile(file_path): | |
| 161 | return 'File "%s" not exist' % (file_path) |
|
161 | return 'File "%s" not exist' % (file_path) | |
| 162 |
|
162 | |||
| 163 | self.dict['upload'] = open(file_path, 'rb') |
|
163 | self.dict['upload'] = open(file_path, 'rb') | |
| 164 | self.dict['name'] = os.path.basename(file_path) |
|
164 | self.dict['name'] = os.path.basename(file_path) | |
| 165 | elif isinstance(url_or_path, str): |
|
165 | elif isinstance(url_or_path, str): | |
| 166 | self.dict['url'] = url_or_path |
|
166 | self.dict['url'] = url_or_path | |
| 167 | if not 'name' in self.dict: |
|
167 | if not 'name' in self.dict: | |
| 168 | self.dict['name'] = os.path.basename(url_or_path) |
|
168 | self.dict['name'] = os.path.basename(url_or_path) | |
| 169 | else: |
|
169 | else: | |
| 170 | return 'ERROR: Verify "file_path" or "url_or_path" parameters: <class "str"> or choose one' |
|
170 | return 'ERROR: Verify "file_path" or "url_or_path" parameters: <class "str"> or choose one' | |
| 171 |
|
171 | |||
| 172 | #if not 'format' in self.dict: |
|
172 | #if not 'format' in self.dict: | |
| 173 | # self.str = ''.join(pathlib.Path(file_path).suffixes) |
|
173 | # self.str = ''.join(pathlib.Path(file_path).suffixes) | |
| 174 | # if len(self.str) > 0: |
|
174 | # if len(self.str) > 0: | |
| 175 | # self.dict['format'] = self.str.upper()[1:] |
|
175 | # self.dict['format'] = self.str.upper()[1:] | |
| 176 |
|
176 | |||
| 177 | #-------------------------PACKAGE SHOW-----------------------# |
|
177 | #-------------------------PACKAGE SHOW-----------------------# | |
| 178 | try: |
|
178 | try: | |
| 179 | dataset_show = getattr(self.ckan.action, 'package_show')(id=dataset_id)['resources'] |
|
179 | dataset_show = getattr(self.ckan.action, 'package_show')(id=dataset_id)['resources'] | |
| 180 | except: |
|
180 | except: | |
| 181 | _, exc_value, _ = sys.exc_info() |
|
181 | _, exc_value, _ = sys.exc_info() | |
| 182 | print('ERROR obtaining metadata dataset:: Use the "print" for more information') |
|
182 | print('ERROR obtaining metadata dataset:: Use the "print" for more information') | |
| 183 | return exc_value |
|
183 | return exc_value | |
| 184 |
|
184 | |||
| 185 | resources_name = [] |
|
185 | resources_name = [] | |
| 186 | for u in dataset_show: |
|
186 | for u in dataset_show: | |
| 187 | resources_name.append(u['name'].lower()) |
|
187 | resources_name.append(u['name'].lower()) | |
| 188 |
|
188 | |||
| 189 | if self.dict['name'].lower() in resources_name: |
|
189 | if self.dict['name'].lower() in resources_name: | |
| 190 | if not ignore_repetition: |
|
190 | if not ignore_repetition: | |
| 191 | return 'ERROR:: "%s" resource already exist in this dataset' % (self.dict['name']) |
|
191 | return 'ERROR:: "%s" resource already exist in this dataset' % (self.dict['name']) | |
| 192 | print('WARRING:: "'+ str(self.dict['name']) +'" resource already exist in this dataset') |
|
192 | print('WARRING:: "'+ str(self.dict['name']) +'" resource already exist in this dataset') | |
| 193 | #------------------------------------------------------------# |
|
193 | #------------------------------------------------------------# | |
| 194 | try: |
|
194 | try: | |
| 195 | return getattr(self.ckan.action, 'resource_create')(package_id=dataset_id, file_date=file_date, voc_file_type=file_type, **self.dict) |
|
195 | return getattr(self.ckan.action, 'resource_create')(package_id=dataset_id, file_date=file_date, voc_file_type=file_type, **self.dict) | |
| 196 | except: |
|
196 | except: | |
| 197 | _, exc_value, _ = sys.exc_info() |
|
197 | _, exc_value, _ = sys.exc_info() | |
| 198 | return exc_value |
|
198 | return exc_value | |
| 199 |
|
199 | |||
| 200 | def upload_multiple_files_advance(self, dataset_id, path_files, file_date, file_type, max_size=100, max_count=500, ignore_repetition=False, **kwargs): |
|
200 | def upload_multiple_files_advance(self, dataset_id, path_files, file_date, file_type, max_size=100, max_count=500, ignore_repetition=False, **kwargs): | |
| 201 | # Agregar si es interruptido por teclado |
|
201 | # Agregar si es interruptido por teclado | |
| 202 | ''' |
|
202 | ''' | |
| 203 | FINALIDAD: |
|
203 | FINALIDAD: | |
| 204 | Funcion para subir multiples archivos al repositorio del ROJ. |
|
204 | Funcion para subir multiples archivos al repositorio del ROJ. | |
| 205 |
|
205 | |||
| 206 | PARAMETROS DISPONIBLES: |
|
206 | PARAMETROS DISPONIBLES: | |
| 207 | CONSULTAR: "GUIA DE SCRIPT.pdf" |
|
207 | CONSULTAR: "GUIA DE SCRIPT.pdf" | |
| 208 |
|
208 | |||
| 209 | ESTRUCTURA: |
|
209 | ESTRUCTURA: | |
| 210 | <access_name>.upload_multiple_files_advance(dataset_id = <class 'str'>, path_files = <class 'list of strings'>, file_date = <class 'str'>, file_type = <class 'str'>, param_1 = <class 'param_1'>, ...) |
|
210 | <access_name>.upload_multiple_files_advance(dataset_id = <class 'str'>, path_files = <class 'list of strings'>, file_date = <class 'str'>, file_type = <class 'str'>, param_1 = <class 'param_1'>, ...) | |
| 211 | ''' |
|
211 | ''' | |
| 212 | #-------------------------PACKAGE SHOW-----------------------# |
|
212 | #-------------------------PACKAGE SHOW-----------------------# | |
| 213 | try: |
|
213 | try: | |
| 214 | dataset_show = getattr(self.ckan.action, 'package_show')(id=dataset_id)['resources'] |
|
214 | dataset_show = getattr(self.ckan.action, 'package_show')(id=dataset_id)['resources'] | |
| 215 | except: |
|
215 | except: | |
| 216 | _, exc_value, _ = sys.exc_info() |
|
216 | _, exc_value, _ = sys.exc_info() | |
| 217 | print('ERROR obtaining metadata dataset:: Use the "print" for more information') |
|
217 | print('ERROR obtaining metadata dataset:: Use the "print" for more information') | |
| 218 | return exc_value |
|
218 | return exc_value | |
| 219 | #------------------------------------------------------------# |
|
219 | #------------------------------------------------------------# | |
| 220 | resources_name = [] |
|
220 | resources_name = [] | |
| 221 | for u in dataset_show: |
|
221 | for u in dataset_show: | |
| 222 | resources_name.append(u['name'].lower()) |
|
222 | resources_name.append(u['name'].lower()) | |
| 223 | #------------------------------------------------------------# |
|
223 | #------------------------------------------------------------# | |
| 224 | self.list = ['package_id', 'upload', 'voc_file_type', 'name'] |
|
224 | self.list = ['package_id', 'upload', 'voc_file_type', 'name'] | |
| 225 | for key1, value1 in kwargs.items(): |
|
225 | for key1, value1 in kwargs.items(): | |
| 226 | if not key1 in self.list: |
|
226 | if not key1 in self.list: | |
| 227 | self.dict[key1] = value1 |
|
227 | self.dict[key1] = value1 | |
| 228 | #------------------------------------------------------------# |
|
228 | #------------------------------------------------------------# | |
| 229 | if not 'others' in kwargs: |
|
229 | if not 'others' in kwargs: | |
| 230 | self.dict['others'] = '' |
|
230 | self.dict['others'] = '' | |
| 231 | else: |
|
231 | else: | |
| 232 | if isinstance(kwargs['others'], list): |
|
232 | if isinstance(kwargs['others'], list): | |
| 233 | self.dict['others'] = json.dumps(kwargs['others']) |
|
233 | self.dict['others'] = json.dumps(kwargs['others']) | |
| 234 | #------------------------------------------------------------# |
|
234 | #------------------------------------------------------------# | |
| 235 | total_list = [] |
|
235 | total_list = [] | |
| 236 | #---------------CASO : "path" or "path_list"-----------------# |
|
236 | #---------------CASO : "path" or "path_list"-----------------# | |
| 237 | if type(path_files) is list: |
|
237 | if type(path_files) is list: | |
| 238 | if len(path_files) != 0: |
|
238 | if len(path_files) != 0: | |
| 239 | path_files.sort() |
|
239 | path_files.sort() | |
| 240 | for u in path_files: |
|
240 | for u in path_files: | |
| 241 | if os.path.isfile(u): |
|
241 | if os.path.isfile(u): | |
| 242 | if os.path.basename(u).lower() in resources_name: |
|
242 | if os.path.basename(u).lower() in resources_name: | |
| 243 | if not ignore_repetition: |
|
243 | if not ignore_repetition: | |
| 244 | return 'ERROR:: "%s" file already exist in this dataset' % (os.path.basename(u)) |
|
244 | return 'ERROR:: "%s" file already exist in this dataset' % (os.path.basename(u)) | |
| 245 | print('WARRING:: "'+ str(os.path.basename(u)) +'" file was ignored because already exist in this dataset') |
|
245 | print('WARRING:: "'+ str(os.path.basename(u)) +'" file was ignored because already exist in this dataset') | |
| 246 | else: |
|
246 | else: | |
| 247 | total_list.append({'name':os.path.basename(u), 'size': os.stat(u).st_size, 'upload':open(u, 'rb')}) |
|
247 | total_list.append({'name':os.path.basename(u), 'size': os.stat(u).st_size, 'upload':open(u, 'rb')}) | |
| 248 | else: |
|
248 | else: | |
| 249 | return 'File "%s" does not exist' % (u) |
|
249 | return 'File "%s" does not exist' % (u) | |
| 250 | else: |
|
250 | else: | |
| 251 | return 'ERROR:: "path_list is empty"' |
|
251 | return 'ERROR:: "path_list is empty"' | |
| 252 |
|
252 | |||
| 253 | elif type(path_files) is str: |
|
253 | elif type(path_files) is str: | |
| 254 | if os.path.isdir(path_files): |
|
254 | if os.path.isdir(path_files): | |
| 255 | path_order = [f for f in os.listdir(path_files) if os.path.isfile(os.path.join(path_files, f))] |
|
255 | path_order = [f for f in os.listdir(path_files) if os.path.isfile(os.path.join(path_files, f))] | |
| 256 | path_order.sort() |
|
256 | path_order.sort() | |
| 257 | if path_order: |
|
257 | if path_order: | |
| 258 | for name in path_order: |
|
258 | for name in path_order: | |
| 259 | if name.lower() in resources_name: |
|
259 | if name.lower() in resources_name: | |
| 260 | if not ignore_repetition: |
|
260 | if not ignore_repetition: | |
| 261 | return 'ERROR:: "%s" file already exist in this dataset' % (name) |
|
261 | return 'ERROR:: "%s" file already exist in this dataset' % (name) | |
| 262 | print('WARRING:: "'+ name +'" file was ignored because already exist in this dataset') |
|
262 | print('WARRING:: "'+ name +'" file was ignored because already exist in this dataset') | |
| 263 | else: |
|
263 | else: | |
| 264 | total_list.append({'name':name, 'size': os.stat(os.path.join(path_files, name)).st_size, 'upload':open(os.path.join(path_files, name), 'rb')}) |
|
264 | total_list.append({'name':name, 'size': os.stat(os.path.join(path_files, name)).st_size, 'upload':open(os.path.join(path_files, name), 'rb')}) | |
| 265 | else: |
|
265 | else: | |
| 266 | return "ERROR:: There aren't files in this directory" |
|
266 | return "ERROR:: There aren't files in this directory" | |
| 267 | else: |
|
267 | else: | |
| 268 | return 'ERROR:: Directory "%s" does not exist' % (path_files) |
|
268 | return 'ERROR:: Directory "%s" does not exist' % (path_files) | |
| 269 | else: |
|
269 | else: | |
| 270 | return 'ERROR:: "path_files" must be a str or list' |
|
270 | return 'ERROR:: "path_files" must be a str or list' | |
| 271 | #------------------------------------------------------------# |
|
271 | #------------------------------------------------------------# | |
| 272 | try: |
|
272 | try: | |
| 273 | uuid.UUID(str(dataset_id), version=4) |
|
273 | uuid.UUID(str(dataset_id), version=4) | |
| 274 | package_id_or_name = '"id": "' + str(dataset_id) + '"' |
|
274 | package_id_or_name = '"id": "' + str(dataset_id) + '"' | |
| 275 | except ValueError: |
|
275 | except ValueError: | |
| 276 | package_id_or_name = '"name": "' + str(dataset_id) + '"' |
|
276 | package_id_or_name = '"name": "' + str(dataset_id) + '"' | |
| 277 | #------------------------------------------------------------# |
|
277 | #------------------------------------------------------------# | |
| 278 | blocks = [[]] |
|
278 | blocks = [[]] | |
| 279 | size_file = 0 |
|
279 | size_file = 0 | |
| 280 | count_file = 0 |
|
280 | count_file = 0 | |
| 281 | inter_num = 0 |
|
281 | inter_num = 0 | |
| 282 | for value in total_list: |
|
282 | for value in total_list: | |
| 283 | if value['size'] > 1024 * 1024 * float(max_size): |
|
283 | if value['size'] > 1024 * 1024 * float(max_size): | |
| 284 | return 'ERROR:: The size of the "%s" file is %sMB aprox, please change "max_size" value' % (value['name'], str(round(value['size']/(1024 * 1024), 2))) |
|
284 | return 'ERROR:: The size of the "%s" file is %sMB aprox, please change "max_size" value' % (value['name'], str(round(value['size']/(1024 * 1024), 2))) | |
| 285 | if not 1 <= int(max_count) <= 999: |
|
285 | if not 1 <= int(max_count) <= 999: | |
| 286 | return 'ERROR:: The count of the number of files must be between 1 and 999, please change "max_count" value' |
|
286 | return 'ERROR:: The count of the number of files must be between 1 and 999, please change "max_count" value' | |
| 287 |
|
287 | |||
| 288 | size_file = size_file + value['size'] |
|
288 | size_file = size_file + value['size'] | |
| 289 | count_file = count_file + 1 |
|
289 | count_file = count_file + 1 | |
| 290 | if size_file <= 1024 * 1024 * float(max_size) and count_file <= int(max_count): |
|
290 | if size_file <= 1024 * 1024 * float(max_size) and count_file <= int(max_count): | |
| 291 | del value['size'] |
|
291 | del value['size'] | |
| 292 | blocks[inter_num].append(value) |
|
292 | blocks[inter_num].append(value) | |
| 293 | else: |
|
293 | else: | |
| 294 | inter_num = inter_num + 1 |
|
294 | inter_num = inter_num + 1 | |
| 295 | size_file = value['size'] |
|
295 | size_file = value['size'] | |
| 296 | count_file = 1 |
|
296 | count_file = 1 | |
| 297 | blocks.append([]) |
|
297 | blocks.append([]) | |
| 298 | del value['size'] |
|
298 | del value['size'] | |
| 299 | blocks[inter_num].append(value) |
|
299 | blocks[inter_num].append(value) | |
| 300 | #------------------------------------------------------------# |
|
300 | #------------------------------------------------------------# | |
| 301 | if len(blocks[0]) > 0: |
|
301 | if len(blocks[0]) > 0: | |
| 302 | print('BLOCK(S) IN TOTAL:: {}'.format(len(blocks))) |
|
302 | print('BLOCK(S) IN TOTAL:: {}'.format(len(blocks))) | |
| 303 | for count1, block in enumerate(blocks): |
|
303 | for count1, block in enumerate(blocks): | |
| 304 | print('---- BLOCK N°{} ----'.format(count1 + 1)) |
|
304 | print('---- BLOCK N°{} ----'.format(count1 + 1)) | |
| 305 | resource_extend = [] |
|
305 | resource_extend = [] | |
| 306 | files_dict = {} |
|
306 | files_dict = {} | |
| 307 | for count2, value2 in enumerate(block): |
|
307 | for count2, value2 in enumerate(block): | |
| 308 | value2['file_date'] = file_date |
|
308 | value2['file_date'] = file_date | |
| 309 | value2['voc_file_type'] = file_type |
|
309 | value2['voc_file_type'] = file_type | |
| 310 | value2.update(self.dict) |
|
310 | value2.update(self.dict) | |
| 311 |
|
311 | |||
| 312 | #if not 'format' in value2: |
|
312 | #if not 'format' in value2: | |
| 313 | # format = ''.join(pathlib.Path(value2['name']).suffixes) |
|
313 | # format = ''.join(pathlib.Path(value2['name']).suffixes) | |
| 314 | # if len(format) > 0: |
|
314 | # if len(format) > 0: | |
| 315 | # value2['format'] = format.upper()[1:] |
|
315 | # value2['format'] = format.upper()[1:] | |
| 316 |
|
316 | |||
| 317 | files_dict['update__resources__-'+ str(len(block)-count2) +'__upload'] = (value2['name'], value2['upload']) |
|
317 | files_dict['update__resources__-'+ str(len(block)-count2) +'__upload'] = (value2['name'], value2['upload']) | |
| 318 | del value2['upload'] |
|
318 | del value2['upload'] | |
| 319 | resource_extend.append(value2) |
|
319 | resource_extend.append(value2) | |
| 320 |
|
320 | |||
| 321 | print('BLOCK N°{} :: "{}" file(s) found >> uploading'.format(count1 + 1, len(block))) |
|
321 | print('BLOCK N°{} :: "{}" file(s) found >> uploading'.format(count1 + 1, len(block))) | |
| 322 | print(resource_extend) |
|
322 | try: | |
| 323 | print(files_dict) |
|
323 | result = self.ckan.call_action( | |
| 324 | #try: |
|
324 | 'package_revise', | |
| 325 | # result = self.ckan.call_action( |
|
325 | {'match': '{'+ str(package_id_or_name) +'}', 'update__resources__extend': json.dumps(resource_extend)}, | |
| 326 |
|
|
326 | files=files_dict | |
| 327 | # {'match': '{'+ str(package_id_or_name) +'}', 'update__resources__extend': json.dumps(resource_extend)}, |
|
327 | ) | |
| 328 | # files=files_dict |
|
328 | print('BLOCK N°{} :: Uploaded file(s) successfully'.format(count1 + 1)) | |
| 329 |
|
|
329 | if len(blocks) == count1 + 1: | |
| 330 | # print('BLOCK N°{} :: Uploaded file(s) successfully'.format(count1 + 1)) |
|
330 | return result | |
| 331 | # if len(blocks) == count1 + 1: |
|
331 | except: | |
| 332 | # return result |
|
332 | print('ERROR :: Use the "print" for more information') | |
| 333 | #except: |
|
333 | _, exc_value, _ = sys.exc_info() | |
| 334 | # print('ERROR :: Use the "print" for more information') |
|
334 | return exc_value | |
| 335 | # _, exc_value, _ = sys.exc_info() |
|
|||
| 336 | # return exc_value |
|
|||
| 337 | else: |
|
335 | else: | |
| 338 | return "ERROR:: No file(s) found to upload" |
|
336 | return "ERROR:: No file(s) found to upload" | |
| 339 |
|
337 | |||
| 340 | def upload_multiple_files(self, dataset_id, path_files, date_files, type_files, ignore_repetition=False, **kwargs): |
|
338 | def upload_multiple_files(self, dataset_id, path_files, date_files, type_files, ignore_repetition=False, **kwargs): | |
| 341 | # Agregar si es interruptido por teclado |
|
339 | # Agregar si es interruptido por teclado | |
| 342 | ''' |
|
340 | ''' | |
| 343 | FINALIDAD: |
|
341 | FINALIDAD: | |
| 344 | Funcion para subir multiples archivos al repositorio del ROJ. |
|
342 | Funcion para subir multiples archivos al repositorio del ROJ. | |
| 345 |
|
343 | |||
| 346 | PARAMETROS DISPONIBLES: |
|
344 | PARAMETROS DISPONIBLES: | |
| 347 | CONSULTAR: "GUIA DE SCRIPT.pdf" |
|
345 | CONSULTAR: "GUIA DE SCRIPT.pdf" | |
| 348 |
|
346 | |||
| 349 | ESTRUCTURA: |
|
347 | ESTRUCTURA: | |
| 350 | <access_name>.upload_multiple_files(dataset_id = <class 'str'>, path_files = <class 'str'> or <class 'list of strings'>, date_files = <class 'str'> or <class 'list of strings'>, type_files = <class 'str'> or <class 'list of strings'>, param_1 = <class 'param_1'>, ...) |
|
348 | <access_name>.upload_multiple_files(dataset_id = <class 'str'>, path_files = <class 'str'> or <class 'list of strings'>, date_files = <class 'str'> or <class 'list of strings'>, type_files = <class 'str'> or <class 'list of strings'>, param_1 = <class 'param_1'>, ...) | |
| 351 | ''' |
|
349 | ''' | |
| 352 | #-------------------------PACKAGE SHOW-----------------------# |
|
350 | #-------------------------PACKAGE SHOW-----------------------# | |
| 353 | try: |
|
351 | try: | |
| 354 | dataset_show = getattr(self.ckan.action, 'package_show')(id=dataset_id)['resources'] |
|
352 | dataset_show = getattr(self.ckan.action, 'package_show')(id=dataset_id)['resources'] | |
| 355 | except: |
|
353 | except: | |
| 356 | _, exc_value, _ = sys.exc_info() |
|
354 | _, exc_value, _ = sys.exc_info() | |
| 357 | print('ERROR obtaining metadata dataset:: Use the "print" for more information') |
|
355 | print('ERROR obtaining metadata dataset:: Use the "print" for more information') | |
| 358 | return exc_value |
|
356 | return exc_value | |
| 359 | #------------------------------------------------------------# |
|
357 | #------------------------------------------------------------# | |
| 360 | resources_name = [] |
|
358 | resources_name = [] | |
| 361 | for u in dataset_show: |
|
359 | for u in dataset_show: | |
| 362 | resources_name.append(u['name'].lower()) |
|
360 | resources_name.append(u['name'].lower()) | |
| 363 | #------------------------------------------------------------# |
|
361 | #------------------------------------------------------------# | |
| 364 |
|
362 | |||
| 365 | params_dict = {'upload':[], 'name':[]} |
|
363 | params_dict = {'upload':[], 'name':[]} | |
| 366 | #if not 'format' in kwargs: |
|
364 | #if not 'format' in kwargs: | |
| 367 | # params_dict.update({'format':[]}) |
|
365 | # params_dict.update({'format':[]}) | |
| 368 | #---------------CASO : "path" or "path_list"-----------------# |
|
366 | #---------------CASO : "path" or "path_list"-----------------# | |
| 369 | if type(path_files) is list: |
|
367 | if type(path_files) is list: | |
| 370 | if len(path_files) != 0: |
|
368 | if len(path_files) != 0: | |
| 371 | path_files.sort() |
|
369 | path_files.sort() | |
| 372 | for u in path_files: |
|
370 | for u in path_files: | |
| 373 | if os.path.isfile(u): |
|
371 | if os.path.isfile(u): | |
| 374 | if os.path.basename(u).lower() in resources_name: |
|
372 | if os.path.basename(u).lower() in resources_name: | |
| 375 | if not ignore_repetition: |
|
373 | if not ignore_repetition: | |
| 376 | return 'ERROR:: "%s" file already exist in this dataset' % (os.path.basename(u)) |
|
374 | return 'ERROR:: "%s" file already exist in this dataset' % (os.path.basename(u)) | |
| 377 | print('WARRING:: "'+ str(os.path.basename(u)) +'" file was ignored because already exist in this dataset') |
|
375 | print('WARRING:: "'+ str(os.path.basename(u)) +'" file was ignored because already exist in this dataset') | |
| 378 | else: |
|
376 | else: | |
| 379 | params_dict['upload'].append(open(u, 'rb')) |
|
377 | params_dict['upload'].append(open(u, 'rb')) | |
| 380 | params_dict['name'].append(os.path.basename(u)) |
|
378 | params_dict['name'].append(os.path.basename(u)) | |
| 381 | #if not 'format' in kwargs: |
|
379 | #if not 'format' in kwargs: | |
| 382 | # format = ''.join(pathlib.Path(u).suffixes) |
|
380 | # format = ''.join(pathlib.Path(u).suffixes) | |
| 383 | # if len(format) > 0: |
|
381 | # if len(format) > 0: | |
| 384 | # params_dict['format'].append(format.upper()[1:]) |
|
382 | # params_dict['format'].append(format.upper()[1:]) | |
| 385 | # else: |
|
383 | # else: | |
| 386 | # params_dict['format'].append('') |
|
384 | # params_dict['format'].append('') | |
| 387 | else: |
|
385 | else: | |
| 388 | return 'File "%s" does not exist' % (u) |
|
386 | return 'File "%s" does not exist' % (u) | |
| 389 | else: |
|
387 | else: | |
| 390 | return 'ERROR:: "path_list is empty"' |
|
388 | return 'ERROR:: "path_list is empty"' | |
| 391 | elif type(path_files) is str: |
|
389 | elif type(path_files) is str: | |
| 392 | if os.path.isdir(path_files): |
|
390 | if os.path.isdir(path_files): | |
| 393 | path_order = [f for f in os.listdir(path_files) if os.path.isfile(os.path.join(path_files, f))] |
|
391 | path_order = [f for f in os.listdir(path_files) if os.path.isfile(os.path.join(path_files, f))] | |
| 394 | path_order.sort() |
|
392 | path_order.sort() | |
| 395 | if path_order: |
|
393 | if path_order: | |
| 396 | for name in path_order: |
|
394 | for name in path_order: | |
| 397 | if name.lower() in resources_name: |
|
395 | if name.lower() in resources_name: | |
| 398 | if not ignore_repetition: |
|
396 | if not ignore_repetition: | |
| 399 | return 'ERROR:: "%s" file already exist in this dataset' % (name) |
|
397 | return 'ERROR:: "%s" file already exist in this dataset' % (name) | |
| 400 | print('WARRING:: "'+ str(name) +'" file was ignored because already exist in this dataset') |
|
398 | print('WARRING:: "'+ str(name) +'" file was ignored because already exist in this dataset') | |
| 401 | else: |
|
399 | else: | |
| 402 | params_dict['upload'].append(open(os.path.join(path_files, name), 'rb')) |
|
400 | params_dict['upload'].append(open(os.path.join(path_files, name), 'rb')) | |
| 403 | params_dict['name'].append(name) |
|
401 | params_dict['name'].append(name) | |
| 404 | #if not 'format' in kwargs: |
|
402 | #if not 'format' in kwargs: | |
| 405 | # format = ''.join(pathlib.Path(name).suffixes) |
|
403 | # format = ''.join(pathlib.Path(name).suffixes) | |
| 406 | # if len(format) > 0: |
|
404 | # if len(format) > 0: | |
| 407 | # params_dict['format'].append(format.upper()[1:]) |
|
405 | # params_dict['format'].append(format.upper()[1:]) | |
| 408 | # else: |
|
406 | # else: | |
| 409 | # params_dict['format'].append('') |
|
407 | # params_dict['format'].append('') | |
| 410 | else: |
|
408 | else: | |
| 411 | return "ERROR:: There aren't files in this directory" |
|
409 | return "ERROR:: There aren't files in this directory" | |
| 412 | else: |
|
410 | else: | |
| 413 | return 'ERROR:: Directory "%s" does not exist' % (path_files) |
|
411 | return 'ERROR:: Directory "%s" does not exist' % (path_files) | |
| 414 | else: |
|
412 | else: | |
| 415 | return 'ERROR:: "path_files" must be a str or list' |
|
413 | return 'ERROR:: "path_files" must be a str or list' | |
| 416 | #------------------------------------------------------------# |
|
414 | #------------------------------------------------------------# | |
| 417 | params_no_dict = {'package_id': dataset_id} |
|
415 | params_no_dict = {'package_id': dataset_id} | |
| 418 | if type(date_files) is list: |
|
416 | if type(date_files) is list: | |
| 419 | params_dict['file_date'] = date_files |
|
417 | params_dict['file_date'] = date_files | |
| 420 | else: |
|
418 | else: | |
| 421 | params_no_dict['file_date'] = date_files |
|
419 | params_no_dict['file_date'] = date_files | |
| 422 |
|
420 | |||
| 423 | if type(type_files) is list: |
|
421 | if type(type_files) is list: | |
| 424 | params_dict['voc_file_type'] = type_files |
|
422 | params_dict['voc_file_type'] = type_files | |
| 425 | else: |
|
423 | else: | |
| 426 | params_no_dict['voc_file_type'] = type_files |
|
424 | params_no_dict['voc_file_type'] = type_files | |
| 427 |
|
425 | |||
| 428 | for key1, value1 in kwargs.items(): |
|
426 | for key1, value1 in kwargs.items(): | |
| 429 | if not key1 in params_dict and not key1 in params_no_dict and key1 != 'others': |
|
427 | if not key1 in params_dict and not key1 in params_no_dict and key1 != 'others': | |
| 430 | if type(value1) is list: |
|
428 | if type(value1) is list: | |
| 431 | params_dict[key1] = value1 |
|
429 | params_dict[key1] = value1 | |
| 432 | else: |
|
430 | else: | |
| 433 | params_no_dict[key1] = value1 |
|
431 | params_no_dict[key1] = value1 | |
| 434 | #------------------------------------------# |
|
432 | #------------------------------------------# | |
| 435 | if not 'others' in kwargs: |
|
433 | if not 'others' in kwargs: | |
| 436 | params_no_dict['others'] = '' |
|
434 | params_no_dict['others'] = '' | |
| 437 | else: |
|
435 | else: | |
| 438 | if isinstance(kwargs['others'], tuple): |
|
436 | if isinstance(kwargs['others'], tuple): | |
| 439 | params_dict['others'] = [json.dumps(w) for w in kwargs['others']] |
|
437 | params_dict['others'] = [json.dumps(w) for w in kwargs['others']] | |
| 440 | elif isinstance(kwargs['others'], list): |
|
438 | elif isinstance(kwargs['others'], list): | |
| 441 | params_no_dict['others'] = json.dumps(kwargs['others']) |
|
439 | params_no_dict['others'] = json.dumps(kwargs['others']) | |
| 442 | elif isinstance(kwargs['others'], str): |
|
440 | elif isinstance(kwargs['others'], str): | |
| 443 | params_no_dict['others'] = kwargs['others'] |
|
441 | params_no_dict['others'] = kwargs['others'] | |
| 444 | else: |
|
442 | else: | |
| 445 | return 'ERROR:: "others" must be a tuple, list or str' |
|
443 | return 'ERROR:: "others" must be a tuple, list or str' | |
| 446 | #------------------------------------------# |
|
444 | #------------------------------------------# | |
| 447 | len_params_dict = [] |
|
445 | len_params_dict = [] | |
| 448 | for value2 in params_dict.values(): |
|
446 | for value2 in params_dict.values(): | |
| 449 | len_params_dict.append(len(value2)) |
|
447 | len_params_dict.append(len(value2)) | |
| 450 |
|
448 | |||
| 451 | if len(list(set(len_params_dict))) > 1: |
|
449 | if len(list(set(len_params_dict))) > 1: | |
| 452 | return 'ERROR:: All lists must be the same length: %s' % (len(params_dict['name'])) |
|
450 | return 'ERROR:: All lists must be the same length: %s' % (len(params_dict['name'])) | |
| 453 | #------------------------------------------------------------# |
|
451 | #------------------------------------------------------------# | |
| 454 | print('"{}" file(s) found >> uploading'.format(len(params_dict['name']))) |
|
452 | print('"{}" file(s) found >> uploading'.format(len(params_dict['name']))) | |
| 455 | for v in range(len(params_dict['name'])): |
|
453 | for v in range(len(params_dict['name'])): | |
| 456 | try: |
|
454 | try: | |
| 457 | send = {} |
|
455 | send = {} | |
| 458 | for key_dict, value_dict in params_dict.items(): |
|
456 | for key_dict, value_dict in params_dict.items(): | |
| 459 | send[key_dict] = value_dict[v] |
|
457 | send[key_dict] = value_dict[v] | |
| 460 | for key_no_dict, value_no_dict in params_no_dict.items(): |
|
458 | for key_no_dict, value_no_dict in params_no_dict.items(): | |
| 461 | send[key_no_dict] = value_no_dict |
|
459 | send[key_no_dict] = value_no_dict | |
| 462 |
|
460 | |||
| 463 | self.list.append(getattr(self.ckan.action, 'resource_create')(**send)) |
|
461 | self.list.append(getattr(self.ckan.action, 'resource_create')(**send)) | |
| 464 | print('File #{} :: "{}" was uploaded successfully'.format(v+1, params_dict['name'][v])) |
|
462 | print('File #{} :: "{}" was uploaded successfully'.format(v+1, params_dict['name'][v])) | |
| 465 | except: |
|
463 | except: | |
| 466 | _, exc_value, _ = sys.exc_info() |
|
464 | _, exc_value, _ = sys.exc_info() | |
| 467 | self.list.append(exc_value) |
|
465 | self.list.append(exc_value) | |
| 468 | print('File #{} :: Error uploading "{}" file'.format(v+1, params_dict['name'][v])) |
|
466 | print('File #{} :: Error uploading "{}" file'.format(v+1, params_dict['name'][v])) | |
| 469 | return self.list |
|
467 | return self.list | |
| 470 | #------------------------------------------------------------# |
|
468 | #------------------------------------------------------------# | |
| 471 |
|
469 | |||
| 472 | def show(self, type_option, id, **kwargs): |
|
470 | def show(self, type_option, id, **kwargs): | |
| 473 | ''' |
|
471 | ''' | |
| 474 | FINALIDAD: |
|
472 | FINALIDAD: | |
| 475 | Funcion personalizada para una busqueda en especifico. |
|
473 | Funcion personalizada para una busqueda en especifico. | |
| 476 |
|
474 | |||
| 477 | PARAMETROS DISPONIBLES: |
|
475 | PARAMETROS DISPONIBLES: | |
| 478 | CONSULTAR: "GUIA DE SCRIPT.pdf" |
|
476 | CONSULTAR: "GUIA DE SCRIPT.pdf" | |
| 479 |
|
477 | |||
| 480 | ESTRUCTURA: |
|
478 | ESTRUCTURA: | |
| 481 | <access_name>.show(type_option = <class 'str'>, id = <class 'str'>, param_1 = <class 'param_1'>, ...) |
|
479 | <access_name>.show(type_option = <class 'str'>, id = <class 'str'>, param_1 = <class 'param_1'>, ...) | |
| 482 | ''' |
|
480 | ''' | |
| 483 | if type(type_option) is str: |
|
481 | if type(type_option) is str: | |
| 484 | try: |
|
482 | try: | |
| 485 | if type_option == 'dataset': |
|
483 | if type_option == 'dataset': | |
| 486 | return getattr(self.ckan.action, 'package_show')(id=id, **kwargs) |
|
484 | return getattr(self.ckan.action, 'package_show')(id=id, **kwargs) | |
| 487 | elif type_option == 'resource': |
|
485 | elif type_option == 'resource': | |
| 488 | return getattr(self.ckan.action, 'resource_show')(id=id, **kwargs) |
|
486 | return getattr(self.ckan.action, 'resource_show')(id=id, **kwargs) | |
| 489 | elif type_option == 'project': |
|
487 | elif type_option == 'project': | |
| 490 | return getattr(self.ckan.action, 'organization_show')(id=id, **kwargs) |
|
488 | return getattr(self.ckan.action, 'organization_show')(id=id, **kwargs) | |
| 491 | elif type_option == 'collaborator': |
|
489 | elif type_option == 'collaborator': | |
| 492 | return getattr(self.ckan.action, 'package_collaborator_list_for_user')(id=id, **kwargs) |
|
490 | return getattr(self.ckan.action, 'package_collaborator_list_for_user')(id=id, **kwargs) | |
| 493 | elif type_option == 'member': |
|
491 | elif type_option == 'member': | |
| 494 | return getattr(self.ckan.action, 'organization_list_for_user')(id=id, **kwargs) |
|
492 | return getattr(self.ckan.action, 'organization_list_for_user')(id=id, **kwargs) | |
| 495 | elif type_option == 'vocabulary': |
|
493 | elif type_option == 'vocabulary': | |
| 496 | return getattr(self.ckan.action, 'vocabulary_show')(id=id, **kwargs) |
|
494 | return getattr(self.ckan.action, 'vocabulary_show')(id=id, **kwargs) | |
| 497 | elif type_option == 'tag': |
|
495 | elif type_option == 'tag': | |
| 498 | if not 'vocabulary_id' in kwargs: |
|
496 | if not 'vocabulary_id' in kwargs: | |
| 499 | print('Missing "vocabulary_id" value: assume it is a free tag') |
|
497 | print('Missing "vocabulary_id" value: assume it is a free tag') | |
| 500 | return getattr(self.ckan.action, 'tag_show')(id=id, **kwargs) |
|
498 | return getattr(self.ckan.action, 'tag_show')(id=id, **kwargs) | |
| 501 | elif type_option == 'user': |
|
499 | elif type_option == 'user': | |
| 502 | return getattr(self.ckan.action, 'user_show')(id=id, **kwargs) |
|
500 | return getattr(self.ckan.action, 'user_show')(id=id, **kwargs) | |
| 503 | elif type_option == 'job': |
|
501 | elif type_option == 'job': | |
| 504 | return getattr(self.ckan.action, 'job_show')(id=id, **kwargs) |
|
502 | return getattr(self.ckan.action, 'job_show')(id=id, **kwargs) | |
| 505 | else: |
|
503 | else: | |
| 506 | return 'ERROR:: "type_option = %s" is not accepted' % (type_option) |
|
504 | return 'ERROR:: "type_option = %s" is not accepted' % (type_option) | |
| 507 | except: |
|
505 | except: | |
| 508 | _, exc_value, _ = sys.exc_info() |
|
506 | _, exc_value, _ = sys.exc_info() | |
| 509 | return exc_value |
|
507 | return exc_value | |
| 510 | else: |
|
508 | else: | |
| 511 | return 'ERROR:: "type_option" must be a str' |
|
509 | return 'ERROR:: "type_option" must be a str' | |
| 512 |
|
510 | |||
| 513 | def search(self, type_option, query=None, **kwargs): |
|
511 | def search(self, type_option, query=None, **kwargs): | |
| 514 | ''' |
|
512 | ''' | |
| 515 | FINALIDAD: |
|
513 | FINALIDAD: | |
| 516 | Funcion personalizada para busquedas que satisfagan algun criterio. |
|
514 | Funcion personalizada para busquedas que satisfagan algun criterio. | |
| 517 |
|
515 | |||
| 518 | PARAMETROS DISPONIBLES: |
|
516 | PARAMETROS DISPONIBLES: | |
| 519 | CONSULTAR: "GUIA DE SCRIPT.pdf" |
|
517 | CONSULTAR: "GUIA DE SCRIPT.pdf" | |
| 520 |
|
518 | |||
| 521 | ESTRUCTURA: |
|
519 | ESTRUCTURA: | |
| 522 | <access_name>.search(type_option = <class 'str'>, query = <class 'dict'>, param_1 = <class 'param_1'>, ...) |
|
520 | <access_name>.search(type_option = <class 'str'>, query = <class 'dict'>, param_1 = <class 'param_1'>, ...) | |
| 523 | ''' |
|
521 | ''' | |
| 524 | if type(type_option) is str: |
|
522 | if type(type_option) is str: | |
| 525 | try: |
|
523 | try: | |
| 526 | if type_option == 'dataset': |
|
524 | if type_option == 'dataset': | |
| 527 | key_replace = ['fq', 'fq_list', 'include_private'] |
|
525 | key_replace = ['fq', 'fq_list', 'include_private'] | |
| 528 | key_point = ['facet_mincount', 'facet_limit', 'facet_field'] |
|
526 | key_point = ['facet_mincount', 'facet_limit', 'facet_field'] | |
| 529 | for key1, value1 in kwargs.items(): |
|
527 | for key1, value1 in kwargs.items(): | |
| 530 | if not key1 in key_replace: |
|
528 | if not key1 in key_replace: | |
| 531 | if key1 in key_point: |
|
529 | if key1 in key_point: | |
| 532 | self.dict[key1.replace('_', '.')] = value1 |
|
530 | self.dict[key1.replace('_', '.')] = value1 | |
| 533 | else: |
|
531 | else: | |
| 534 | self.dict[key1] = value1 |
|
532 | self.dict[key1] = value1 | |
| 535 |
|
533 | |||
| 536 | if query is not None: |
|
534 | if query is not None: | |
| 537 | if type(query) is dict: |
|
535 | if type(query) is dict: | |
| 538 | self.dict['fq_list'] = [] |
|
536 | self.dict['fq_list'] = [] | |
| 539 | #NUM_RESOURCES_MIN / NUM_RESOURCES_MAX |
|
537 | #NUM_RESOURCES_MIN / NUM_RESOURCES_MAX | |
| 540 | #----------------------------------------------------# |
|
538 | #----------------------------------------------------# | |
| 541 | if 'dataset_start_date' in query: |
|
539 | if 'dataset_start_date' in query: | |
| 542 | if type(query['dataset_start_date']) is str: |
|
540 | if type(query['dataset_start_date']) is str: | |
| 543 | try: |
|
541 | try: | |
| 544 | datetime.strptime(query['dataset_start_date'], '%Y-%m-%d') |
|
542 | datetime.strptime(query['dataset_start_date'], '%Y-%m-%d') | |
| 545 | if len(query['dataset_start_date']) != 10: |
|
543 | if len(query['dataset_start_date']) != 10: | |
| 546 | return '"dataset_start_date", must be: <YYYY-MM-DD>' |
|
544 | return '"dataset_start_date", must be: <YYYY-MM-DD>' | |
| 547 | self.dict['fq_list'].append('dataset_start_date:"'+query['dataset_start_date']+'"') |
|
545 | self.dict['fq_list'].append('dataset_start_date:"'+query['dataset_start_date']+'"') | |
| 548 | self.list.append('dataset_start_date') |
|
546 | self.list.append('dataset_start_date') | |
| 549 | except: |
|
547 | except: | |
| 550 | return '"dataset_start_date" incorrect: "%s"' % (query['dataset_start_date']) |
|
548 | return '"dataset_start_date" incorrect: "%s"' % (query['dataset_start_date']) | |
| 551 | else: |
|
549 | else: | |
| 552 | return '"dataset_start_date" must be <str>' |
|
550 | return '"dataset_start_date" must be <str>' | |
| 553 | #----------------------------------------------------# |
|
551 | #----------------------------------------------------# | |
| 554 | if 'dataset_end_date' in query: |
|
552 | if 'dataset_end_date' in query: | |
| 555 | if type(query['dataset_end_date']) is str: |
|
553 | if type(query['dataset_end_date']) is str: | |
| 556 | try: |
|
554 | try: | |
| 557 | datetime.strptime(query['dataset_end_date'], '%Y-%m-%d') |
|
555 | datetime.strptime(query['dataset_end_date'], '%Y-%m-%d') | |
| 558 | if len(query['dataset_end_date']) != 10: |
|
556 | if len(query['dataset_end_date']) != 10: | |
| 559 | return '"dataset_end_date", must be: <YYYY-MM-DD>' |
|
557 | return '"dataset_end_date", must be: <YYYY-MM-DD>' | |
| 560 |
|
558 | |||
| 561 | if 'dataset_start_date' in query: |
|
559 | if 'dataset_start_date' in query: | |
| 562 | if query['dataset_start_date'] > query['dataset_end_date']: |
|
560 | if query['dataset_start_date'] > query['dataset_end_date']: | |
| 563 | return '"dataset_end_date" must be greater than "dataset_start_date"' |
|
561 | return '"dataset_end_date" must be greater than "dataset_start_date"' | |
| 564 |
|
562 | |||
| 565 | self.dict['fq_list'].append('dataset_end_date:"'+query['dataset_end_date']+'"') |
|
563 | self.dict['fq_list'].append('dataset_end_date:"'+query['dataset_end_date']+'"') | |
| 566 | self.list.append('dataset_end_date') |
|
564 | self.list.append('dataset_end_date') | |
| 567 | except: |
|
565 | except: | |
| 568 | return '"dataset_end_date" incorrect: "%s"' % (query['dataset_end_date']) |
|
566 | return '"dataset_end_date" incorrect: "%s"' % (query['dataset_end_date']) | |
| 569 | else: |
|
567 | else: | |
| 570 | return '"dataset_end_date" must be <str>' |
|
568 | return '"dataset_end_date" must be <str>' | |
| 571 | #----------------------------------------------------# |
|
569 | #----------------------------------------------------# | |
| 572 | for key, value in query.items(): |
|
570 | for key, value in query.items(): | |
| 573 | if value is not None and not key in self.list: |
|
571 | if value is not None and not key in self.list: | |
| 574 | self.dict['fq_list'].append(str(key)+':"'+str(value)+'"') |
|
572 | self.dict['fq_list'].append(str(key)+':"'+str(value)+'"') | |
| 575 | else: |
|
573 | else: | |
| 576 | return '"query" must be <dict>' |
|
574 | return '"query" must be <dict>' | |
| 577 |
|
575 | |||
| 578 | return getattr(self.ckan.action, 'package_search')(include_private=True, **self.dict) |
|
576 | return getattr(self.ckan.action, 'package_search')(include_private=True, **self.dict) | |
| 579 |
|
577 | |||
| 580 | elif type_option == 'resource': |
|
578 | elif type_option == 'resource': | |
| 581 | for key1, value1 in kwargs.items(): |
|
579 | for key1, value1 in kwargs.items(): | |
| 582 | if key1 != 'fields': |
|
580 | if key1 != 'fields': | |
| 583 | self.dict[key1] = value1 |
|
581 | self.dict[key1] = value1 | |
| 584 |
|
582 | |||
| 585 | if query is not None: |
|
583 | if query is not None: | |
| 586 | if type(query) is dict: |
|
584 | if type(query) is dict: | |
| 587 | #----------------------------------------------------# |
|
585 | #----------------------------------------------------# | |
| 588 | if 'file_date_min' in query: |
|
586 | if 'file_date_min' in query: | |
| 589 | if type(query['file_date_min']) is str: |
|
587 | if type(query['file_date_min']) is str: | |
| 590 | try: |
|
588 | try: | |
| 591 | datetime.strptime(query['file_date_min'], '%Y-%m-%d') |
|
589 | datetime.strptime(query['file_date_min'], '%Y-%m-%d') | |
| 592 | if len(query['file_date_min']) != 10: |
|
590 | if len(query['file_date_min']) != 10: | |
| 593 | return '"file_date_min", must be: <YYYY-MM-DD>' |
|
591 | return '"file_date_min", must be: <YYYY-MM-DD>' | |
| 594 | except: |
|
592 | except: | |
| 595 | return '"file_date_min" incorrect: "%s"' % (query['file_date_min']) |
|
593 | return '"file_date_min" incorrect: "%s"' % (query['file_date_min']) | |
| 596 | else: |
|
594 | else: | |
| 597 | return '"file_date_min" must be <str>' |
|
595 | return '"file_date_min" must be <str>' | |
| 598 | #----------------------------------------------------# |
|
596 | #----------------------------------------------------# | |
| 599 | if 'file_date_max' in query: |
|
597 | if 'file_date_max' in query: | |
| 600 | if type(query['file_date_max']) is str: |
|
598 | if type(query['file_date_max']) is str: | |
| 601 | try: |
|
599 | try: | |
| 602 | datetime.strptime(query['file_date_max'], '%Y-%m-%d') |
|
600 | datetime.strptime(query['file_date_max'], '%Y-%m-%d') | |
| 603 | if len(query['file_date_max']) != 10: |
|
601 | if len(query['file_date_max']) != 10: | |
| 604 | return '"file_date_max", must be: <YYYY-MM-DD>' |
|
602 | return '"file_date_max", must be: <YYYY-MM-DD>' | |
| 605 |
|
603 | |||
| 606 | if 'file_date_min' in query: |
|
604 | if 'file_date_min' in query: | |
| 607 | if query['file_date_min'] > query['file_date_max']: |
|
605 | if query['file_date_min'] > query['file_date_max']: | |
| 608 | return '"file_date_max" must be greater than "file_date_min"' |
|
606 | return '"file_date_max" must be greater than "file_date_min"' | |
| 609 | except: |
|
607 | except: | |
| 610 | return '"file_date_max" incorrect: "%s"' % (query['file_date_max']) |
|
608 | return '"file_date_max" incorrect: "%s"' % (query['file_date_max']) | |
| 611 | else: |
|
609 | else: | |
| 612 | return '"file_date_max" must be <str>' |
|
610 | return '"file_date_max" must be <str>' | |
| 613 | #----------------------------------------------------# |
|
611 | #----------------------------------------------------# | |
| 614 | self.dict['query'] = query |
|
612 | self.dict['query'] = query | |
| 615 | else: |
|
613 | else: | |
| 616 | return '"query" must be <dict>' |
|
614 | return '"query" must be <dict>' | |
| 617 | return getattr(self.ckan.action, 'resources_search')(**self.dict) |
|
615 | return getattr(self.ckan.action, 'resources_search')(**self.dict) | |
| 618 |
|
616 | |||
| 619 | elif type_option == 'tag': |
|
617 | elif type_option == 'tag': | |
| 620 | for key1, value1 in kwargs.items(): |
|
618 | for key1, value1 in kwargs.items(): | |
| 621 | if key1 != 'fields': |
|
619 | if key1 != 'fields': | |
| 622 | self.dict[key1] = value1 |
|
620 | self.dict[key1] = value1 | |
| 623 |
|
621 | |||
| 624 | if not 'vocabulary_id' in kwargs: |
|
622 | if not 'vocabulary_id' in kwargs: | |
| 625 | print('Missing "vocabulary_id" value: tags that don’t belong to any vocabulary') |
|
623 | print('Missing "vocabulary_id" value: tags that don’t belong to any vocabulary') | |
| 626 | else: |
|
624 | else: | |
| 627 | print('Only tags that belong to "{}" vocabulary'.format(kwargs['vocabulary_id'])) |
|
625 | print('Only tags that belong to "{}" vocabulary'.format(kwargs['vocabulary_id'])) | |
| 628 |
|
626 | |||
| 629 | if query is not None: |
|
627 | if query is not None: | |
| 630 | if type(query) is dict: |
|
628 | if type(query) is dict: | |
| 631 | if 'search' in query: |
|
629 | if 'search' in query: | |
| 632 | if type(query['search']) is list or type(query['search']) is str: |
|
630 | if type(query['search']) is list or type(query['search']) is str: | |
| 633 | self.dict['query'] = query['search'] |
|
631 | self.dict['query'] = query['search'] | |
| 634 | else: |
|
632 | else: | |
| 635 | return '"search" must be <list> or <str>' |
|
633 | return '"search" must be <list> or <str>' | |
| 636 | else: |
|
634 | else: | |
| 637 | return '"query" must be <dict>' |
|
635 | return '"query" must be <dict>' | |
| 638 | return getattr(self.ckan.action, 'tag_search')(**self.dict) |
|
636 | return getattr(self.ckan.action, 'tag_search')(**self.dict) | |
| 639 |
|
637 | |||
| 640 | else: |
|
638 | else: | |
| 641 | return 'ERROR:: "type_option = %s" is not accepted' % (type_option) |
|
639 | return 'ERROR:: "type_option = %s" is not accepted' % (type_option) | |
| 642 |
|
640 | |||
| 643 | except: |
|
641 | except: | |
| 644 | _, exc_value, _ = sys.exc_info() |
|
642 | _, exc_value, _ = sys.exc_info() | |
| 645 | return exc_value |
|
643 | return exc_value | |
| 646 | else: |
|
644 | else: | |
| 647 | return 'ERROR:: "type_option" must be <str>' |
|
645 | return 'ERROR:: "type_option" must be <str>' | |
| 648 |
|
646 | |||
| 649 | def create(self, type_option, select=None, **kwargs): |
|
647 | def create(self, type_option, select=None, **kwargs): | |
| 650 | ''' |
|
648 | ''' | |
| 651 | FINALIDAD: |
|
649 | FINALIDAD: | |
| 652 | Funcion personalizada para crear. |
|
650 | Funcion personalizada para crear. | |
| 653 |
|
651 | |||
| 654 | PARAMETROS DISPONIBLES: |
|
652 | PARAMETROS DISPONIBLES: | |
| 655 | CONSULTAR: "GUIA DE SCRIPT.pdf" |
|
653 | CONSULTAR: "GUIA DE SCRIPT.pdf" | |
| 656 |
|
654 | |||
| 657 | ESTRUCTURA: |
|
655 | ESTRUCTURA: | |
| 658 | <access_name>.create(type_option = <class 'str'>, param_1 = <class 'param_1'>, ...) |
|
656 | <access_name>.create(type_option = <class 'str'>, param_1 = <class 'param_1'>, ...) | |
| 659 | ''' |
|
657 | ''' | |
| 660 | if type(type_option) is str: |
|
658 | if type(type_option) is str: | |
| 661 | try: |
|
659 | try: | |
| 662 | if type_option == 'dataset': |
|
660 | if type_option == 'dataset': | |
| 663 | return getattr(self.ckan.action, 'package_create')(**kwargs) |
|
661 | return getattr(self.ckan.action, 'package_create')(**kwargs) | |
| 664 | if type_option == 'resource': |
|
662 | if type_option == 'resource': | |
| 665 | return resource.resource_create(self, **kwargs) |
|
663 | return resource.resource_create(self, **kwargs) | |
| 666 | elif type_option == 'project': |
|
664 | elif type_option == 'project': | |
| 667 | return getattr(self.ckan.action, 'organization_create')(**kwargs) |
|
665 | return getattr(self.ckan.action, 'organization_create')(**kwargs) | |
| 668 | elif type_option == 'member': |
|
666 | elif type_option == 'member': | |
| 669 | return getattr(self.ckan.action, 'organization_member_create')(**kwargs) |
|
667 | return getattr(self.ckan.action, 'organization_member_create')(**kwargs) | |
| 670 | elif type_option == 'collaborator': |
|
668 | elif type_option == 'collaborator': | |
| 671 | return getattr(self.ckan.action, 'package_collaborator_create')(**kwargs) |
|
669 | return getattr(self.ckan.action, 'package_collaborator_create')(**kwargs) | |
| 672 | elif type_option == 'vocabulary': |
|
670 | elif type_option == 'vocabulary': | |
| 673 | return getattr(self.ckan.action, 'vocabulary_create')(**kwargs) |
|
671 | return getattr(self.ckan.action, 'vocabulary_create')(**kwargs) | |
| 674 | elif type_option == 'tag': |
|
672 | elif type_option == 'tag': | |
| 675 | return getattr(self.ckan.action, 'tag_create')(**kwargs) |
|
673 | return getattr(self.ckan.action, 'tag_create')(**kwargs) | |
| 676 | elif type_option == 'user': |
|
674 | elif type_option == 'user': | |
| 677 | return getattr(self.ckan.action, 'user_create')(**kwargs) |
|
675 | return getattr(self.ckan.action, 'user_create')(**kwargs) | |
| 678 | elif type_option == 'views': |
|
676 | elif type_option == 'views': | |
| 679 | if 'resource' == select: |
|
677 | if 'resource' == select: | |
| 680 | self.list = ['package'] |
|
678 | self.list = ['package'] | |
| 681 | for key1, value1 in kwargs.items(): |
|
679 | for key1, value1 in kwargs.items(): | |
| 682 | if not key1 in self.list: |
|
680 | if not key1 in self.list: | |
| 683 | self.dict[key1] = value1 |
|
681 | self.dict[key1] = value1 | |
| 684 | return getattr(self.ckan.action, 'resource_create_default_resource_views')(**self.dict) |
|
682 | return getattr(self.ckan.action, 'resource_create_default_resource_views')(**self.dict) | |
| 685 | elif 'dataset' == select: |
|
683 | elif 'dataset' == select: | |
| 686 | return getattr(self.ckan.action, 'package_create_default_resource_views')(**kwargs) |
|
684 | return getattr(self.ckan.action, 'package_create_default_resource_views')(**kwargs) | |
| 687 | else: |
|
685 | else: | |
| 688 | return 'ERROR:: "select = %s" is not accepted' % (select) |
|
686 | return 'ERROR:: "select = %s" is not accepted' % (select) | |
| 689 | else: |
|
687 | else: | |
| 690 | return 'ERROR:: "type_option = %s" is not accepted' % (type_option) |
|
688 | return 'ERROR:: "type_option = %s" is not accepted' % (type_option) | |
| 691 | except: |
|
689 | except: | |
| 692 | _, exc_value, _ = sys.exc_info() |
|
690 | _, exc_value, _ = sys.exc_info() | |
| 693 | return exc_value |
|
691 | return exc_value | |
| 694 | else: |
|
692 | else: | |
| 695 | return 'ERROR:: "type_option" must be <str>' |
|
693 | return 'ERROR:: "type_option" must be <str>' | |
| 696 |
|
694 | |||
| 697 | def patch(self, type_option, **kwargs): |
|
695 | def patch(self, type_option, **kwargs): | |
| 698 | ''' |
|
696 | ''' | |
| 699 | FINALIDAD: |
|
697 | FINALIDAD: | |
| 700 | Funciones personalizadas para actualizar |
|
698 | Funciones personalizadas para actualizar | |
| 701 |
|
699 | |||
| 702 | PARAMETROS DISPONIBLES: |
|
700 | PARAMETROS DISPONIBLES: | |
| 703 | CONSULTAR: "GUIA DE SCRIPT.pdf" |
|
701 | CONSULTAR: "GUIA DE SCRIPT.pdf" | |
| 704 |
|
702 | |||
| 705 | ESTRUCTURA: |
|
703 | ESTRUCTURA: | |
| 706 | <access_name>.patch(type_option = <class 'str'>, param_1 = <class 'param_1'>, ...) |
|
704 | <access_name>.patch(type_option = <class 'str'>, param_1 = <class 'param_1'>, ...) | |
| 707 | ''' |
|
705 | ''' | |
| 708 | if type(type_option) is str: |
|
706 | if type(type_option) is str: | |
| 709 | try: |
|
707 | try: | |
| 710 | if type_option == 'dataset': |
|
708 | if type_option == 'dataset': | |
| 711 | #Agregar que solo se debe modificar parámetros del Dataset y que no incluya Resources |
|
709 | #Agregar que solo se debe modificar parámetros del Dataset y que no incluya Resources | |
| 712 | return getattr(self.ckan.action, 'package_patch')(**kwargs) |
|
710 | return getattr(self.ckan.action, 'package_patch')(**kwargs) | |
| 713 | elif type_option == 'project': |
|
711 | elif type_option == 'project': | |
| 714 | return getattr(self.ckan.action, 'organization_patch')(**kwargs) |
|
712 | return getattr(self.ckan.action, 'organization_patch')(**kwargs) | |
| 715 | elif type_option == 'resource': |
|
713 | elif type_option == 'resource': | |
| 716 | return resource.resource_patch(self, **kwargs) |
|
714 | return resource.resource_patch(self, **kwargs) | |
| 717 | elif type_option == 'member': |
|
715 | elif type_option == 'member': | |
| 718 | return getattr(self.ckan.action, 'organization_member_create')(**kwargs) |
|
716 | return getattr(self.ckan.action, 'organization_member_create')(**kwargs) | |
| 719 | elif type_option == 'collaborator': |
|
717 | elif type_option == 'collaborator': | |
| 720 | return getattr(self.ckan.action, 'package_collaborator_create')(**kwargs) |
|
718 | return getattr(self.ckan.action, 'package_collaborator_create')(**kwargs) | |
| 721 | else: |
|
719 | else: | |
| 722 | return 'ERROR:: "type_option = %s" is not accepted' % (type_option) |
|
720 | return 'ERROR:: "type_option = %s" is not accepted' % (type_option) | |
| 723 | except: |
|
721 | except: | |
| 724 | _, exc_value, _ = sys.exc_info() |
|
722 | _, exc_value, _ = sys.exc_info() | |
| 725 | return exc_value |
|
723 | return exc_value | |
| 726 | else: |
|
724 | else: | |
| 727 | return 'ERROR:: "type_option" must be <str>' |
|
725 | return 'ERROR:: "type_option" must be <str>' | |
| 728 |
|
726 | |||
| 729 | def delete(self, type_option, select=None, **kwargs): |
|
727 | def delete(self, type_option, select=None, **kwargs): | |
| 730 | ''' |
|
728 | ''' | |
| 731 | FINALIDAD: |
|
729 | FINALIDAD: | |
| 732 | Función personalizada para eliminar y/o purgar. |
|
730 | Función personalizada para eliminar y/o purgar. | |
| 733 |
|
731 | |||
| 734 | PARAMETROS DISPONIBLES: |
|
732 | PARAMETROS DISPONIBLES: | |
| 735 | CONSULTAR: "GUIA DE SCRIPT.pdf" |
|
733 | CONSULTAR: "GUIA DE SCRIPT.pdf" | |
| 736 |
|
734 | |||
| 737 | ESTRUCTURA: |
|
735 | ESTRUCTURA: | |
| 738 | <access_name>.delete(type_option = <class 'str'>, param_1 = <class 'param_1'>, ...) |
|
736 | <access_name>.delete(type_option = <class 'str'>, param_1 = <class 'param_1'>, ...) | |
| 739 | ''' |
|
737 | ''' | |
| 740 | if type(type_option) is str: |
|
738 | if type(type_option) is str: | |
| 741 | try: |
|
739 | try: | |
| 742 | if type_option == 'dataset': |
|
740 | if type_option == 'dataset': | |
| 743 | if select is None: |
|
741 | if select is None: | |
| 744 | return 'ERROR:: "select" must not be "None"' |
|
742 | return 'ERROR:: "select" must not be "None"' | |
| 745 | else: |
|
743 | else: | |
| 746 | if 'delete' == select: |
|
744 | if 'delete' == select: | |
| 747 | return getattr(self.ckan.action, 'package_delete')(**kwargs) |
|
745 | return getattr(self.ckan.action, 'package_delete')(**kwargs) | |
| 748 | elif 'purge' == select: |
|
746 | elif 'purge' == select: | |
| 749 | return getattr(self.ckan.action, 'dataset_purge')(**kwargs) |
|
747 | return getattr(self.ckan.action, 'dataset_purge')(**kwargs) | |
| 750 | else: |
|
748 | else: | |
| 751 | return 'ERROR:: "select = %s" is not accepted' % (select) |
|
749 | return 'ERROR:: "select = %s" is not accepted' % (select) | |
| 752 | elif type_option == 'project': |
|
750 | elif type_option == 'project': | |
| 753 | if select is None: |
|
751 | if select is None: | |
| 754 | return 'ERROR:: "select" must not be "None"' |
|
752 | return 'ERROR:: "select" must not be "None"' | |
| 755 | else: |
|
753 | else: | |
| 756 | if 'delete' == select: |
|
754 | if 'delete' == select: | |
| 757 | return getattr(self.ckan.action, 'organization_delete')(**kwargs) |
|
755 | return getattr(self.ckan.action, 'organization_delete')(**kwargs) | |
| 758 | elif 'purge' == select: |
|
756 | elif 'purge' == select: | |
| 759 | return getattr(self.ckan.action, 'organization_purge')(**kwargs) |
|
757 | return getattr(self.ckan.action, 'organization_purge')(**kwargs) | |
| 760 | else: |
|
758 | else: | |
| 761 | return 'ERROR:: "select = %s" is not accepted' % (select) |
|
759 | return 'ERROR:: "select = %s" is not accepted' % (select) | |
| 762 | elif type_option == 'resource': |
|
760 | elif type_option == 'resource': | |
| 763 | if select is None: |
|
761 | if select is None: | |
| 764 | return 'ERROR:: "select" must not be "None"' |
|
762 | return 'ERROR:: "select" must not be "None"' | |
| 765 | else: |
|
763 | else: | |
| 766 | return resource.resource_delete(self, select, **kwargs) |
|
764 | return resource.resource_delete(self, select, **kwargs) | |
| 767 | elif type_option == 'vocabulary': |
|
765 | elif type_option == 'vocabulary': | |
| 768 | return getattr(self.ckan.action, 'vocabulary_delete')(**kwargs) |
|
766 | return getattr(self.ckan.action, 'vocabulary_delete')(**kwargs) | |
| 769 | elif type_option == 'tag': |
|
767 | elif type_option == 'tag': | |
| 770 | return getattr(self.ckan.action, 'tag_delete')(**kwargs) |
|
768 | return getattr(self.ckan.action, 'tag_delete')(**kwargs) | |
| 771 | elif type_option == 'user': |
|
769 | elif type_option == 'user': | |
| 772 | return getattr(self.ckan.action, 'user_delete')(**kwargs) |
|
770 | return getattr(self.ckan.action, 'user_delete')(**kwargs) | |
| 773 | else: |
|
771 | else: | |
| 774 | return 'ERROR:: "type_option = %s" is not accepted' % (type_option) |
|
772 | return 'ERROR:: "type_option = %s" is not accepted' % (type_option) | |
| 775 | except: |
|
773 | except: | |
| 776 | _, exc_value, _ = sys.exc_info() |
|
774 | _, exc_value, _ = sys.exc_info() | |
| 777 | return exc_value |
|
775 | return exc_value | |
| 778 | else: |
|
776 | else: | |
| 779 | return 'ERROR:: "type_option" must be <str>' |
|
777 | return 'ERROR:: "type_option" must be <str>' | |
| 780 |
|
778 | |||
| 781 | def f_status_note(self, total, result, path): |
|
779 | def f_status_note(self, total, result, path): | |
| 782 | file_txt = open(path+'status_note.txt', 'w') |
|
780 | file_txt = open(path+'status_note.txt', 'w') | |
| 783 | file_txt = open(path+'status_note.txt', 'a') |
|
781 | file_txt = open(path+'status_note.txt', 'a') | |
| 784 |
|
782 | |||
| 785 | file_txt.write('DOWNLOADED FILE(S): "%s"' % (len(result['name']))) |
|
783 | file_txt.write('DOWNLOADED FILE(S): "%s"' % (len(result['name']))) | |
| 786 | file_txt.write(''+ os.linesep) |
|
784 | file_txt.write(''+ os.linesep) | |
| 787 | for u in result['name']: |
|
785 | for u in result['name']: | |
| 788 | file_txt.write(' - '+ u + os.linesep) |
|
786 | file_txt.write(' - '+ u + os.linesep) | |
| 789 | file_txt.write(''+ os.linesep) |
|
787 | file_txt.write(''+ os.linesep) | |
| 790 |
|
788 | |||
| 791 | file_txt.write('FAILED FILE(S): "%s"' % (len(total['name'])-len(result['name']))) |
|
789 | file_txt.write('FAILED FILE(S): "%s"' % (len(total['name'])-len(result['name']))) | |
| 792 | file_txt.write(''+ os.linesep) |
|
790 | file_txt.write(''+ os.linesep) | |
| 793 | if len(total['name'])-len(result['name']) != 0: |
|
791 | if len(total['name'])-len(result['name']) != 0: | |
| 794 | for u in total['name']: |
|
792 | for u in total['name']: | |
| 795 | if not u in result['name']: |
|
793 | if not u in result['name']: | |
| 796 | file_txt.write(' - '+ u + os.linesep) |
|
794 | file_txt.write(' - '+ u + os.linesep) | |
| 797 | else: |
|
795 | else: | |
| 798 | file_txt.write(' "None"'+ os.linesep) |
|
796 | file_txt.write(' "None"'+ os.linesep) | |
| 799 |
|
797 | |||
| 800 | def f_name(self, name_dataset, ext, tempdir): |
|
798 | def f_name(self, name_dataset, ext, tempdir): | |
| 801 | while self.check: |
|
799 | while self.check: | |
| 802 | self.str = '' |
|
800 | self.str = '' | |
| 803 | if self.cont == 0: |
|
801 | if self.cont == 0: | |
| 804 | if os.path.exists(tempdir + name_dataset + ext): |
|
802 | if os.path.exists(tempdir + name_dataset + ext): | |
| 805 | self.str = name_dataset+'('+str(self.cont+1)+')'+ext |
|
803 | self.str = name_dataset+'('+str(self.cont+1)+')'+ext | |
| 806 | else: |
|
804 | else: | |
| 807 | self.check = self.check * 0 |
|
805 | self.check = self.check * 0 | |
| 808 | self.str = name_dataset + ext |
|
806 | self.str = name_dataset + ext | |
| 809 | else: |
|
807 | else: | |
| 810 | if not os.path.exists(tempdir + name_dataset+'('+str(self.cont)+')'+ext): |
|
808 | if not os.path.exists(tempdir + name_dataset+'('+str(self.cont)+')'+ext): | |
| 811 | self.check = self.check * 0 |
|
809 | self.check = self.check * 0 | |
| 812 | self.str = name_dataset+'('+str(self.cont)+')'+ ext |
|
810 | self.str = name_dataset+'('+str(self.cont)+')'+ ext | |
| 813 | self.cont = self.cont+1 |
|
811 | self.cont = self.cont+1 | |
| 814 | return self.str |
|
812 | return self.str | |
| 815 |
|
813 | |||
| 816 | def f_zipdir(self, path, ziph, zip_name): |
|
814 | def f_zipdir(self, path, ziph, zip_name): | |
| 817 | for root, _, files in os.walk(path): |
|
815 | for root, _, files in os.walk(path): | |
| 818 | print('.....') |
|
816 | print('.....') | |
| 819 | print('Creating: "{}" >>'.format(zip_name)) |
|
817 | print('Creating: "{}" >>'.format(zip_name)) | |
| 820 | for __file in tqdm(iterable=files, total=len(files)): |
|
818 | for __file in tqdm(iterable=files, total=len(files)): | |
| 821 | new_dir = os.path.relpath(os.path.join(root, __file), os.path.join(path, '..')) |
|
819 | new_dir = os.path.relpath(os.path.join(root, __file), os.path.join(path, '..')) | |
| 822 | ziph.write(os.path.join(root, __file), new_dir) |
|
820 | ziph.write(os.path.join(root, __file), new_dir) | |
| 823 | print('Created >>') |
|
821 | print('Created >>') | |
| 824 |
|
822 | |||
| 825 | def download_by_step(self, response, tempdir_name): |
|
823 | def download_by_step(self, response, tempdir_name): | |
| 826 | try: |
|
824 | try: | |
| 827 | # ---------- REPLACE URL --------- # |
|
825 | # ---------- REPLACE URL --------- # | |
| 828 | if urlparse(self.url).netloc != 'www.igp.gob.pe' and urlparse(response['url']).netloc == 'www.igp.gob.pe': |
|
826 | if urlparse(self.url).netloc != 'www.igp.gob.pe' and urlparse(response['url']).netloc == 'www.igp.gob.pe': | |
| 829 | response['url'] = response['url'].replace(urlparse(response['url']).scheme + '://' + urlparse(response['url']).netloc, |
|
827 | response['url'] = response['url'].replace(urlparse(response['url']).scheme + '://' + urlparse(response['url']).netloc, | |
| 830 | urlparse(self.url).scheme + '://' + urlparse(self.url).netloc) |
|
828 | urlparse(self.url).scheme + '://' + urlparse(self.url).netloc) | |
| 831 | #----------------------------------# |
|
829 | #----------------------------------# | |
| 832 | with requests.get(response['url'], stream=True, headers={'Authorization': self.Authorization}, verify=self.verify) as resp: |
|
830 | with requests.get(response['url'], stream=True, headers={'Authorization': self.Authorization}, verify=self.verify) as resp: | |
| 833 | if resp.status_code == 200: |
|
831 | if resp.status_code == 200: | |
| 834 | with open(tempdir_name+response['name'], 'wb') as file: |
|
832 | with open(tempdir_name+response['name'], 'wb') as file: | |
| 835 | for chunk in resp.iter_content(chunk_size = self.chunk_size): |
|
833 | for chunk in resp.iter_content(chunk_size = self.chunk_size): | |
| 836 | if chunk: |
|
834 | if chunk: | |
| 837 | file.write(chunk) |
|
835 | file.write(chunk) | |
| 838 | except requests.exceptions.RequestException: |
|
836 | except requests.exceptions.RequestException: | |
| 839 | pass |
|
837 | pass | |
| 840 |
|
838 | |||
| 841 | def download_files(self, **kwargs): |
|
839 | def download_files(self, **kwargs): | |
| 842 | ''' |
|
840 | ''' | |
| 843 | FINALIDAD: |
|
841 | FINALIDAD: | |
| 844 | Funcion personalizada para la descarga de archivos existentes de un dataset. |
|
842 | Funcion personalizada para la descarga de archivos existentes de un dataset. | |
| 845 |
|
843 | |||
| 846 | PARAMETROS DISPONIBLES: |
|
844 | PARAMETROS DISPONIBLES: | |
| 847 | CONSULTAR: "GUIA DE SCRIPT.pdf" |
|
845 | CONSULTAR: "GUIA DE SCRIPT.pdf" | |
| 848 |
|
846 | |||
| 849 | ESTRUCTURA: |
|
847 | ESTRUCTURA: | |
| 850 | <access_name>.download_files(id = <class 'str'>, param_1 = <class 'param_1'>, ...) |
|
848 | <access_name>.download_files(id = <class 'str'>, param_1 = <class 'param_1'>, ...) | |
| 851 | ''' |
|
849 | ''' | |
| 852 | dict_local = {} |
|
850 | dict_local = {} | |
| 853 | #----------------------------------------------# |
|
851 | #----------------------------------------------# | |
| 854 | if 'zip' in kwargs: |
|
852 | if 'zip' in kwargs: | |
| 855 | if type(kwargs['zip']) is not bool: |
|
853 | if type(kwargs['zip']) is not bool: | |
| 856 | return 'ERROR:: "zip" must be: <class "bool">' |
|
854 | return 'ERROR:: "zip" must be: <class "bool">' | |
| 857 | else: |
|
855 | else: | |
| 858 | dict_local['zip'] = kwargs['zip'] |
|
856 | dict_local['zip'] = kwargs['zip'] | |
| 859 | else: |
|
857 | else: | |
| 860 | dict_local['zip'] = False |
|
858 | dict_local['zip'] = False | |
| 861 | #----------------------------------------------# |
|
859 | #----------------------------------------------# | |
| 862 | if 'status_note' in kwargs: |
|
860 | if 'status_note' in kwargs: | |
| 863 | if type(kwargs['status_note']) is not bool: |
|
861 | if type(kwargs['status_note']) is not bool: | |
| 864 | return 'ERROR:: "status_note" must be: <class "bool">' |
|
862 | return 'ERROR:: "status_note" must be: <class "bool">' | |
| 865 | else: |
|
863 | else: | |
| 866 | dict_local['status_note'] = kwargs['status_note'] |
|
864 | dict_local['status_note'] = kwargs['status_note'] | |
| 867 | else: |
|
865 | else: | |
| 868 | dict_local['status_note'] = False |
|
866 | dict_local['status_note'] = False | |
| 869 | #----------------------------------------------# |
|
867 | #----------------------------------------------# | |
| 870 | if 'path' in kwargs: |
|
868 | if 'path' in kwargs: | |
| 871 | if type(kwargs['path']) is str: |
|
869 | if type(kwargs['path']) is str: | |
| 872 | if os.path.isdir(kwargs['path']) == False: |
|
870 | if os.path.isdir(kwargs['path']) == False: | |
| 873 | return 'ERROR:: "path" does not exist' |
|
871 | return 'ERROR:: "path" does not exist' | |
| 874 | else: |
|
872 | else: | |
| 875 | if kwargs['path'][-1:] != self.separator: |
|
873 | if kwargs['path'][-1:] != self.separator: | |
| 876 | dict_local['path'] = kwargs['path']+self.separator |
|
874 | dict_local['path'] = kwargs['path']+self.separator | |
| 877 | else: |
|
875 | else: | |
| 878 | dict_local['path'] = kwargs['path'] |
|
876 | dict_local['path'] = kwargs['path'] | |
| 879 |
|
877 | |||
| 880 | txt = dict_local['path']+datetime.now().strftime("%Y_%m_%d_%H_%M_%S_%f")+'.txt' |
|
878 | txt = dict_local['path']+datetime.now().strftime("%Y_%m_%d_%H_%M_%S_%f")+'.txt' | |
| 881 | if int(platform.python_version()[0]) == 3: |
|
879 | if int(platform.python_version()[0]) == 3: | |
| 882 | try: |
|
880 | try: | |
| 883 | file_txt = open(txt, 'w') |
|
881 | file_txt = open(txt, 'w') | |
| 884 | file_txt.close() |
|
882 | file_txt.close() | |
| 885 | os.remove(txt) |
|
883 | os.remove(txt) | |
| 886 | except PermissionError: |
|
884 | except PermissionError: | |
| 887 | return 'ERROR:: Access denied, you are not authorized to write files: "%s"' % (dict_local['path']) |
|
885 | return 'ERROR:: Access denied, you are not authorized to write files: "%s"' % (dict_local['path']) | |
| 888 | else: |
|
886 | else: | |
| 889 | try: |
|
887 | try: | |
| 890 | file_txt = open(txt, 'w') |
|
888 | file_txt = open(txt, 'w') | |
| 891 | file_txt.close() |
|
889 | file_txt.close() | |
| 892 | os.remove(txt) |
|
890 | os.remove(txt) | |
| 893 | except: |
|
891 | except: | |
| 894 | return 'ERROR:: Access denied, you are not authorized to write files: "%s"' % (dict_local['path']) |
|
892 | return 'ERROR:: Access denied, you are not authorized to write files: "%s"' % (dict_local['path']) | |
| 895 | else: |
|
893 | else: | |
| 896 | return 'ERROR:: "path" must be: <class "str">' |
|
894 | return 'ERROR:: "path" must be: <class "str">' | |
| 897 | else: |
|
895 | else: | |
| 898 | dict_local['path'] = '' |
|
896 | dict_local['path'] = '' | |
| 899 | #----------------------------------------------# |
|
897 | #----------------------------------------------# | |
| 900 | for key, value in kwargs.items(): |
|
898 | for key, value in kwargs.items(): | |
| 901 | if not key in dict_local: |
|
899 | if not key in dict_local: | |
| 902 | self.dict[key] = value |
|
900 | self.dict[key] = value | |
| 903 | try: |
|
901 | try: | |
| 904 | response = getattr(self.ckan.action, 'url_resources')(**self.dict) |
|
902 | response = getattr(self.ckan.action, 'url_resources')(**self.dict) | |
| 905 | except: |
|
903 | except: | |
| 906 | _, exc_value, _ = sys.exc_info() |
|
904 | _, exc_value, _ = sys.exc_info() | |
| 907 | return exc_value |
|
905 | return exc_value | |
| 908 |
|
906 | |||
| 909 | if len(response) != 0: |
|
907 | if len(response) != 0: | |
| 910 | #--------------TEMP PATH---------------# |
|
908 | #--------------TEMP PATH---------------# | |
| 911 | if dict_local['zip']: |
|
909 | if dict_local['zip']: | |
| 912 | tempdir = tempfile.mkdtemp(prefix=kwargs['id']+'-')+self.separator |
|
910 | tempdir = tempfile.mkdtemp(prefix=kwargs['id']+'-')+self.separator | |
| 913 | os.mkdir(tempdir+kwargs['id']) |
|
911 | os.mkdir(tempdir+kwargs['id']) | |
| 914 | dir_name = tempdir + kwargs['id'] + self.separator |
|
912 | dir_name = tempdir + kwargs['id'] + self.separator | |
| 915 | else: |
|
913 | else: | |
| 916 | dir = self.f_name(kwargs['id'], '', dict_local['path']) |
|
914 | dir = self.f_name(kwargs['id'], '', dict_local['path']) | |
| 917 | os.mkdir(dict_local['path'] + dir) |
|
915 | os.mkdir(dict_local['path'] + dir) | |
| 918 | dir_name = dict_local['path'] + dir + self.separator |
|
916 | dir_name = dict_local['path'] + dir + self.separator | |
| 919 | #-----------DOWNLOAD FILES-------------# |
|
917 | #-----------DOWNLOAD FILES-------------# | |
| 920 | print('.....') |
|
918 | print('.....') | |
| 921 | print('Downloading "{}" file(s) >>'.format(len(response))) |
|
919 | print('Downloading "{}" file(s) >>'.format(len(response))) | |
| 922 | name_total = {'name': []} |
|
920 | name_total = {'name': []} | |
| 923 | with concurrent.futures.ThreadPoolExecutor() as executor: |
|
921 | with concurrent.futures.ThreadPoolExecutor() as executor: | |
| 924 | for u in tqdm(iterable=response, total=len(response)): |
|
922 | for u in tqdm(iterable=response, total=len(response)): | |
| 925 | name_total['name'].append(u['name']) |
|
923 | name_total['name'].append(u['name']) | |
| 926 | executor.submit(self.download_by_step, u, dir_name) |
|
924 | executor.submit(self.download_by_step, u, dir_name) | |
| 927 | name_check = {} |
|
925 | name_check = {} | |
| 928 | name_check['name'] = [f for f in os.listdir(dir_name) if os.path.isfile(os.path.join(dir_name, f))] |
|
926 | name_check['name'] = [f for f in os.listdir(dir_name) if os.path.isfile(os.path.join(dir_name, f))] | |
| 929 | print('"{}" downloaded file(s) successfully >>'.format(len(name_check['name']))) |
|
927 | print('"{}" downloaded file(s) successfully >>'.format(len(name_check['name']))) | |
| 930 | #--------------------------------------# |
|
928 | #--------------------------------------# | |
| 931 | if len(name_check['name']) != 0: |
|
929 | if len(name_check['name']) != 0: | |
| 932 | #----------Status Note---------# |
|
930 | #----------Status Note---------# | |
| 933 | if dict_local['status_note']: |
|
931 | if dict_local['status_note']: | |
| 934 | print('.....') |
|
932 | print('.....') | |
| 935 | print('Creating: "status_note.txt" >>') |
|
933 | print('Creating: "status_note.txt" >>') | |
| 936 | self.f_status_note(name_total, name_check, dir_name) |
|
934 | self.f_status_note(name_total, name_check, dir_name) | |
| 937 | print('Created>>') |
|
935 | print('Created>>') | |
| 938 | #----------ZIP CREATE----------# |
|
936 | #----------ZIP CREATE----------# | |
| 939 | if dict_local['zip']: |
|
937 | if dict_local['zip']: | |
| 940 | zip_name = self.f_name(kwargs['id'], '.zip', dict_local['path']) |
|
938 | zip_name = self.f_name(kwargs['id'], '.zip', dict_local['path']) | |
| 941 | ziph = zipfile.ZipFile(dict_local['path'] + zip_name, 'w', zipfile.ZIP_DEFLATED, allowZip64=True) |
|
939 | ziph = zipfile.ZipFile(dict_local['path'] + zip_name, 'w', zipfile.ZIP_DEFLATED, allowZip64=True) | |
| 942 | self.f_zipdir(dir_name, ziph, zip_name) |
|
940 | self.f_zipdir(dir_name, ziph, zip_name) | |
| 943 | ziph.close() |
|
941 | ziph.close() | |
| 944 | #Delete Temporal Path |
|
942 | #Delete Temporal Path | |
| 945 | if os.path.exists(tempdir[:-1]): |
|
943 | if os.path.exists(tempdir[:-1]): | |
| 946 | shutil.rmtree(tempdir[:-1]) |
|
944 | shutil.rmtree(tempdir[:-1]) | |
| 947 | #------------------------------# |
|
945 | #------------------------------# | |
| 948 | print('.....') |
|
946 | print('.....') | |
| 949 | return 'DOWNLOAD FINISHED' |
|
947 | return 'DOWNLOAD FINISHED' | |
| 950 | else: |
|
948 | else: | |
| 951 | #Delete Temporal Path |
|
949 | #Delete Temporal Path | |
| 952 | if dict_local['zip']: |
|
950 | if dict_local['zip']: | |
| 953 | if os.path.exists(tempdir[:-1]): |
|
951 | if os.path.exists(tempdir[:-1]): | |
| 954 | shutil.rmtree(tempdir[:-1]) |
|
952 | shutil.rmtree(tempdir[:-1]) | |
| 955 | else: |
|
953 | else: | |
| 956 | if os.path.exists(dir_name[:-1]): |
|
954 | if os.path.exists(dir_name[:-1]): | |
| 957 | shutil.rmtree(dir_name[:-1]) |
|
955 | shutil.rmtree(dir_name[:-1]) | |
| 958 | return 'NO FILES WERE DOWNLOADED' |
|
956 | return 'NO FILES WERE DOWNLOADED' | |
| 959 | else: |
|
957 | else: | |
| 960 | return 'FILES NOT FOUND' |
|
958 | return 'FILES NOT FOUND' | |
| 961 |
|
959 | |||
| 962 | def download_files_advance(self, id_or_name, processes=1, path=os.path.expanduser("~"), **kwargs): |
|
960 | def download_files_advance(self, id_or_name, processes=1, path=os.path.expanduser("~"), **kwargs): | |
| 963 | ''' |
|
961 | ''' | |
| 964 | FINALIDAD: |
|
962 | FINALIDAD: | |
| 965 | Funcion personalizada avanzada para la descarga de archivos existentes de un(os) dataset(s). |
|
963 | Funcion personalizada avanzada para la descarga de archivos existentes de un(os) dataset(s). | |
| 966 |
|
964 | |||
| 967 | PARAMETROS DISPONIBLES: |
|
965 | PARAMETROS DISPONIBLES: | |
| 968 | CONSULTAR: "GUIA DE SCRIPT.pdf" |
|
966 | CONSULTAR: "GUIA DE SCRIPT.pdf" | |
| 969 |
|
967 | |||
| 970 | ESTRUCTURA: |
|
968 | ESTRUCTURA: | |
| 971 | <access_name>.download_files_advance(id_or_name= <class 'str' or 'list'>, param_1 = <class 'param_1'>, ...) |
|
969 | <access_name>.download_files_advance(id_or_name= <class 'str' or 'list'>, param_1 = <class 'param_1'>, ...) | |
| 972 | ''' |
|
970 | ''' | |
| 973 | #------------------ PATH ----------------------# |
|
971 | #------------------ PATH ----------------------# | |
| 974 | if isinstance(path, str): |
|
972 | if isinstance(path, str): | |
| 975 | if os.path.isdir(path): |
|
973 | if os.path.isdir(path): | |
| 976 | if not path.endswith(os.sep): |
|
974 | if not path.endswith(os.sep): | |
| 977 | path = path + os.sep |
|
975 | path = path + os.sep | |
| 978 | test_txt = path + datetime.now().strftime("%Y_%m_%d_%H_%M_%S_%f")+'.txt' |
|
976 | test_txt = path + datetime.now().strftime("%Y_%m_%d_%H_%M_%S_%f")+'.txt' | |
| 979 | try: |
|
977 | try: | |
| 980 | file_txt = open(test_txt, 'w') |
|
978 | file_txt = open(test_txt, 'w') | |
| 981 | file_txt.close() |
|
979 | file_txt.close() | |
| 982 | os.remove(test_txt) |
|
980 | os.remove(test_txt) | |
| 983 | except: |
|
981 | except: | |
| 984 | return 'ERROR:: Access denied, you are not authorized to write files: "%s"' % (path) |
|
982 | return 'ERROR:: Access denied, you are not authorized to write files: "%s"' % (path) | |
| 985 | else: |
|
983 | else: | |
| 986 | return 'ERROR:: "path" does not exist' |
|
984 | return 'ERROR:: "path" does not exist' | |
| 987 | else: |
|
985 | else: | |
| 988 | return 'ERROR:: "path" must be: <class "str">' |
|
986 | return 'ERROR:: "path" must be: <class "str">' | |
| 989 |
|
987 | |||
| 990 | #------------------ PROCESSES -----------------# |
|
988 | #------------------ PROCESSES -----------------# | |
| 991 | if not isinstance(processes, int): |
|
989 | if not isinstance(processes, int): | |
| 992 | return 'ERROR:: "processes" must be: <class "int">' |
|
990 | return 'ERROR:: "processes" must be: <class "int">' | |
| 993 |
|
991 | |||
| 994 | #------------------ ID OR NAME ----------------# |
|
992 | #------------------ ID OR NAME ----------------# | |
| 995 | if isinstance(id_or_name, str): |
|
993 | if isinstance(id_or_name, str): | |
| 996 | id_or_name = [id_or_name] |
|
994 | id_or_name = [id_or_name] | |
| 997 | elif isinstance(id_or_name, list): |
|
995 | elif isinstance(id_or_name, list): | |
| 998 | id_or_name = list(map(str, id_or_name)) |
|
996 | id_or_name = list(map(str, id_or_name)) | |
| 999 | else: |
|
997 | else: | |
| 1000 | return 'ERROR:: dataset "id_or_name" must be: <class "str" or "list">' |
|
998 | return 'ERROR:: dataset "id_or_name" must be: <class "str" or "list">' | |
| 1001 | #----------------------------------------------# |
|
999 | #----------------------------------------------# | |
| 1002 | arguments = { |
|
1000 | arguments = { | |
| 1003 | '--apikey': self.Authorization, |
|
1001 | '--apikey': self.Authorization, | |
| 1004 | '--ckan-user': None, |
|
1002 | '--ckan-user': None, | |
| 1005 | '--config': None, |
|
1003 | '--config': None, | |
| 1006 | '--datapackages': path, |
|
1004 | '--datapackages': path, | |
| 1007 | '--datastore-fields': False, |
|
1005 | '--datastore-fields': False, | |
| 1008 | '--get-request': False, |
|
1006 | '--get-request': False, | |
| 1009 | '--insecure': not self.verify, |
|
1007 | '--insecure': not self.verify, | |
| 1010 | '--log': '/home/soporte/DUMP/download.txt', |
|
1008 | '--log': '/home/soporte/DUMP/download.txt', | |
| 1011 | '--processes': str(processes), |
|
1009 | '--processes': str(processes), | |
| 1012 | '--quiet': False, |
|
1010 | '--quiet': False, | |
| 1013 | '--remote': self.url, |
|
1011 | '--remote': self.url, | |
| 1014 | '--worker': False, |
|
1012 | '--worker': False, | |
| 1015 | #'--all': False, |
|
1013 | #'--all': False, | |
| 1016 | #'--gzip': False, |
|
1014 | #'--gzip': False, | |
| 1017 | #'--output': None, |
|
1015 | #'--output': None, | |
| 1018 | #'--max-records': None, |
|
1016 | #'--max-records': None, | |
| 1019 | #'--output-json': False, |
|
1017 | #'--output-json': False, | |
| 1020 | #'--output-jsonl': False, |
|
1018 | #'--output-jsonl': False, | |
| 1021 | #'--create-only': False, |
|
1019 | #'--create-only': False, | |
| 1022 | #'--help': False, |
|
1020 | #'--help': False, | |
| 1023 | #'--input': None, |
|
1021 | #'--input': None, | |
| 1024 | #'--input-json': False, |
|
1022 | #'--input-json': False, | |
| 1025 | #'--start-record': '1', |
|
1023 | #'--start-record': '1', | |
| 1026 | #'--update-only': False, |
|
1024 | #'--update-only': False, | |
| 1027 | #'--upload-logo': False, |
|
1025 | #'--upload-logo': False, | |
| 1028 | #'--upload-resources': False, |
|
1026 | #'--upload-resources': False, | |
| 1029 | #'--version': False, |
|
1027 | #'--version': False, | |
| 1030 | 'ID_OR_NAME': id_or_name, |
|
1028 | 'ID_OR_NAME': id_or_name, | |
| 1031 | 'datasets': True, |
|
1029 | 'datasets': True, | |
| 1032 | 'dump': True, |
|
1030 | 'dump': True, | |
| 1033 | #'ACTION_NAME': None, |
|
1031 | #'ACTION_NAME': None, | |
| 1034 | #'KEY:JSON': [], |
|
1032 | #'KEY:JSON': [], | |
| 1035 | #'KEY=STRING': [], |
|
1033 | #'KEY=STRING': [], | |
| 1036 | #'KEY@FILE': [], |
|
1034 | #'KEY@FILE': [], | |
| 1037 | #'action': False, |
|
1035 | #'action': False, | |
| 1038 | #'delete': False, |
|
1036 | #'delete': False, | |
| 1039 | #'groups': False, |
|
1037 | #'groups': False, | |
| 1040 | #'load': False, |
|
1038 | #'load': False, | |
| 1041 | #'organizations': False, |
|
1039 | #'organizations': False, | |
| 1042 | #'related': False, |
|
1040 | #'related': False, | |
| 1043 | #'search': False, |
|
1041 | #'search': False, | |
| 1044 | #'users': False |
|
1042 | #'users': False | |
| 1045 | } |
|
1043 | } | |
| 1046 | return logic_download.dump_things_change(self.ckan, 'datasets', arguments, **kwargs) No newline at end of file |
|
1044 | return logic_download.dump_things_change(self.ckan, 'datasets', arguments, **kwargs) | |
General Comments 0
You need to be logged in to leave comments.
Login now