##// END OF EJS Templates
DATABASES » ckanext-jro » api-cliente
RSS
v2.9.2 :: Update 'upload_file' function - Add empty resource
eynilupu -
r14:a8d46058b456
parent child
Show More
Show file before | Show file after | Hide comments
1 NO CONTENT: modified file, binary diff hidden
NO CONTENT: modified file, binary diff hidden
Show file before | Show file after | Hide comments
@@ -1,1022 +1,1037
1 from ckanapi import RemoteCKAN
1 from ckanapi import RemoteCKAN
2 from datetime import datetime
2 from datetime import datetime
3 from tqdm import tqdm
3 from tqdm import tqdm
4 from CKAN_JRO import logic_download
4 from CKAN_JRO import logic_download
5 #from ckanapi.errors import NotAuthorized, NotFound, ValidationError, SearchQueryError, SearchError, CKANAPIError, ServerIncompatibleError
5 #from ckanapi.errors import NotAuthorized, NotFound, ValidationError, SearchQueryError, SearchError, CKANAPIError, ServerIncompatibleError
6 import sys
6 import sys
7 import platform
7 import platform
8 import os
8 import os
9 import tempfile
9 import tempfile
10 import shutil
10 import shutil
11 import zipfile
11 import zipfile
12 import concurrent.futures
12 import concurrent.futures
13 import requests
13 import requests
14 import json
14 import json
15 #import pathlib
15 #import pathlib
16 import uuid
16 import uuid
17
17
18 if sys.version_info.major == 3:
18 if sys.version_info.major == 3:
19 from urllib.parse import urlparse
19 from urllib.parse import urlparse
20 else:
20 else:
21 import urlparse
21 import urlparse
22
22
23 class JROAPI():
23 class JROAPI():
24 """
24 """
25 FINALIDAD:
25 FINALIDAD:
26 Script para administrar y obtener la data del repositorio por medio de APIs.
26 Script para administrar y obtener la data del repositorio por medio de APIs.
27
27
28 REQUISITIOS PREVIOS:
28 REQUISITIOS PREVIOS:
29 - Paso 1: Tener "pip [Python 2]" o "pip3 [Python 3]" instalado:
29 - Paso 1: Tener "pip [Python 2]" o "pip3 [Python 3]" instalado:
30 - Paso 2: Instalar lo siguiente como admininstrador:
30 - Paso 2: Instalar lo siguiente como admininstrador:
31 En Python 2
31 En Python 2
32 - pip install ckanapi==4.5
32 - pip install ckanapi==4.5
33 - pip install requests
33 - pip install requests
34 - pip install futures
34 - pip install futures
35 - pip install tqdm
35 - pip install tqdm
36 En Python > 3
36 En Python > 3
37 - pip3 install ckanapi==4.5
37 - pip3 install ckanapi==4.5
38 - pip3 install requests
38 - pip3 install requests
39 - pip3 install tqdm
39 - pip3 install tqdm
40
40
41 FUNCIONES DISPONIBLES:
41 FUNCIONES DISPONIBLES:
42 - action
42 - action
43 - upload_file
43 - upload_file
44 - upload_multiple_files
44 - upload_multiple_files
45 - upload_multiple_files_advance
45 - upload_multiple_files_advance
46 - show
46 - show
47 - search
47 - search
48 - create
48 - create
49 - patch
49 - patch
50 - delete
50 - delete
51 - download_files
51 - download_files
52
52
53 EJEMPLOS:
53 EJEMPLOS:
54 #1:
54 #1:
55 with JROAPI('http://demo.example.com', Authorization='#########') as <access_name>:
55 with JROAPI('http://demo.example.com', Authorization='#########') as <access_name>:
56 ... some operation(s) ...
56 ... some operation(s) ...
57 #2:
57 #2:
58 <access_name> = JROAPI('http://example.com', Authorization='#########')
58 <access_name> = JROAPI('http://example.com', Authorization='#########')
59 ... some operation(s) ...
59 ... some operation(s) ...
60 <access_name>.ckan.close()
60 <access_name>.ckan.close()
61
61
62 REPORTAR ALGUN PROBLEMA:
62 REPORTAR ALGUN PROBLEMA:
63 Debe enviar un correo a eynilupu@igp.gob.pe detallando los siguientes pasos:
63 Debe enviar un correo a eynilupu@igp.gob.pe detallando los siguientes pasos:
64 1) Correo para contactarlo
64 1) Correo para contactarlo
65 2) Descripcion del problema
65 2) Descripcion del problema
66 3) ¿En que paso o seccion encontro el problema?
66 3) ¿En que paso o seccion encontro el problema?
67 4) ¿Cual era el resultado que usted esperaba?
67 4) ¿Cual era el resultado que usted esperaba?
68 """
68 """
69 def __init__(self, url, Authorization=None, secure=True):
69 def __init__(self, url, Authorization=None, secure=True):
70 #-------- Check Secure -------#
70 #-------- Check Secure -------#
71 self.verify = secure
71 self.verify = secure
72 if not secure and isinstance(secure, bool):
72 if not secure and isinstance(secure, bool):
73 session = requests.Session()
73 session = requests.Session()
74 session.verify = False
74 session.verify = False
75 else:
75 else:
76 session = None
76 session = None
77 #------------------------------#
77 #------------------------------#
78 self.url = url
78 self.url = url
79 ua = 'CKAN_JRO/2.9.2 (+'+str(self.url)+')'
79 ua = 'CKAN_JRO/2.9.2 (+'+str(self.url)+')'
80 #ua = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36'
80 #ua = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36'
81 self.ckan = RemoteCKAN(self.url, apikey=Authorization, user_agent=ua, session=session)
81 self.ckan = RemoteCKAN(self.url, apikey=Authorization, user_agent=ua, session=session)
82 #self.ckan = RemoteCKAN(self.url, apikey=Authorization)
82 #self.ckan = RemoteCKAN(self.url, apikey=Authorization)
83 self.Authorization = Authorization
83 self.Authorization = Authorization
84 # Change for --> self.separator = os.sep
84 # Change for --> self.separator = os.sep
85 if platform.system() == 'Windows':
85 if platform.system() == 'Windows':
86 self.separator = '\\'
86 self.separator = '\\'
87 else:
87 else:
88 self.separator = '/'
88 self.separator = '/'
89
89
90 self.chunk_size = 1024
90 self.chunk_size = 1024
91 self.list = []
91 self.list = []
92 self.dict = {}
92 self.dict = {}
93 self.str = ''
93 self.str = ''
94 self.check = 1
94 self.check = 1
95 self.cont = 0
95 self.cont = 0
96
96
97 def __enter__(self):
97 def __enter__(self):
98 return self
98 return self
99
99
100 def __exit__(self, *args):
100 def __exit__(self, *args):
101 self.ckan.close()
101 self.ckan.close()
102
102
103 def action(self, action, **kwargs):
103 def action(self, action, **kwargs):
104 """
104 """
105 FINALIDAD:
105 FINALIDAD:
106 Funcion para llamar a las APIs disponibles
106 Funcion para llamar a las APIs disponibles
107
107
108 APIs DISPONIBLES:
108 APIs DISPONIBLES:
109 CONSULTAR: "GUIA DE SCRIPT.pdf"
109 CONSULTAR: "GUIA DE SCRIPT.pdf"
110
110
111 EJEMPLO:
111 EJEMPLO:
112 <access_name>.action(<consuming API>, param_1 = <class 'param_1'>, ...)
112 <access_name>.action(<consuming API>, param_1 = <class 'param_1'>, ...)
113 """
113 """
114 #--------------- CASE: PACKAGE SEARCH ---------------#
114 #--------------- CASE: PACKAGE SEARCH ---------------#
115 if kwargs is not None:
115 if kwargs is not None:
116 if action == 'package_search':
116 if action == 'package_search':
117 self.list = ['facet_mincount', 'facet_limit', 'facet_field']
117 self.list = ['facet_mincount', 'facet_limit', 'facet_field']
118 for facet in self.list:
118 for facet in self.list:
119 if facet in kwargs:
119 if facet in kwargs:
120 kwargs[facet.replace('_', '.')] = kwargs[facet]
120 kwargs[facet.replace('_', '.')] = kwargs[facet]
121 kwargs.pop(facet)
121 kwargs.pop(facet)
122 #----------------------------------------------------#
122 #----------------------------------------------------#
123 try:
123 try:
124 return getattr(self.ckan.action, action)(**kwargs)
124 return getattr(self.ckan.action, action)(**kwargs)
125 except:
125 except:
126 _, exc_value, _ = sys.exc_info()
126 _, exc_value, _ = sys.exc_info()
127 return exc_value
127 return exc_value
128
128
129 def upload_file(self, dataset_id, file_path, file_date, file_type, **kwargs):
129 def upload_file(self, dataset_id, file_date, file_type, file_path=False, url_or_path=False, ignore_repetition=False, **kwargs):
130 # Agregar si es interruptido por teclado
130 # Agregar si es interruptido por teclado
131 '''
131 '''
132 FINALIDAD:
132 FINALIDAD:
133 Funcion para subir un unico archivo al repositorio del ROJ.
133 Funcion para crear un unico recurso (puede incluir un archivo asociado) al repositorio del ROJ.
134
134
135 PARAMETROS DISPONIBLES:
135 PARAMETROS DISPONIBLES:
136 CONSULTAR: "GUIA DE SCRIPT.pdf"
136 CONSULTAR: "GUIA DE SCRIPT.pdf"
137
137
138 ESTRUCTURA:
138 ESTRUCTURA:
139 <access_name>.upload_file(dataset_id = <class 'str'>, file_date = <class 'str'>, file_path = <class 'str'>, file_type = <class 'str'>, param_1 = <class 'param_1'>, ...)
139 <access_name>.upload_file(dataset_id = <class 'str'>, file_date = <class 'str'>, file_type = <class 'str'>, file_path = <class 'str'>, url_or_path = <class 'str'>, param_1 = <class 'param_1'>, ...)
140 '''
140 '''
141 self.list = ['package_id', 'upload', 'voc_file_type', 'name'] #file_date
141 #self.list = ['package_id', 'upload', 'voc_file_type', 'name'] #file_date
142 self.list = ['package_id', 'upload', 'voc_file_type'] #file_date
142 for key1, value1 in kwargs.items():
143 for key1, value1 in kwargs.items():
143 if not key1 in self.list:
144 if not key1 in self.list:
144 self.dict[key1] = value1
145 self.dict[key1] = value1
145
146
146 #---------------------------#
147 #---------------------------#
147 if not 'others' in kwargs:
148 if not 'others' in kwargs:
148 self.dict['others'] = ''
149 self.dict['others'] = ''
149 else:
150 else:
150 if isinstance(kwargs['others'], list):
151 if isinstance(kwargs['others'], list):
151 self.dict['others'] = json.dumps(kwargs['others'])
152 self.dict['others'] = json.dumps(kwargs['others'])
152 #---------------------------#
153 #---------------------------#
153
154
154 if not os.path.isfile(file_path):
155 if isinstance(file_path, str) and isinstance(url_or_path, str):
155 return 'File "%s" not exist' % (file_path)
156 return 'ERROR:: Choose one: "file_path" or "url_or_path" parameters'
156
157
158 if isinstance(file_path, str):
159 if not os.path.isfile(file_path):
160 return 'File "%s" not exist' % (file_path)
161
162 self.dict['upload'] = open(file_path, 'rb')
163 self.dict['name'] = os.path.basename(file_path)
164 elif isinstance(url_or_path, str):
165 self.dict['url'] = url_or_path
166 if not 'name' in self.dict:
167 self.dict['name'] = os.path.basename(url_or_path)
168 else:
169 return 'ERROR: Verify "file_path" or "url_or_path" parameters: <class "str"> or choose one'
170
157 #if not 'format' in self.dict:
171 #if not 'format' in self.dict:
158 # self.str = ''.join(pathlib.Path(file_path).suffixes)
172 # self.str = ''.join(pathlib.Path(file_path).suffixes)
159 # if len(self.str) > 0:
173 # if len(self.str) > 0:
160 # self.dict['format'] = self.str.upper()[1:]
174 # self.dict['format'] = self.str.upper()[1:]
161
175
162 #-------------------------PACKAGE SHOW-----------------------#
176 #-------------------------PACKAGE SHOW-----------------------#
163 try:
177 try:
164 dataset_show = getattr(self.ckan.action, 'package_show')(id=dataset_id)['resources']
178 dataset_show = getattr(self.ckan.action, 'package_show')(id=dataset_id)['resources']
165 except:
179 except:
166 _, exc_value, _ = sys.exc_info()
180 _, exc_value, _ = sys.exc_info()
167 print('ERROR obtaining metadata dataset:: Use the "print" for more information')
181 print('ERROR obtaining metadata dataset:: Use the "print" for more information')
168 return exc_value
182 return exc_value
169
183
170 resources_name = []
184 resources_name = []
171 for u in dataset_show:
185 for u in dataset_show:
172 resources_name.append(u['name'].lower())
186 resources_name.append(u['name'].lower())
173
187
174 if os.path.basename(file_path).lower() in resources_name:
188 if self.dict['name'].lower() in resources_name:
175 return 'ERROR:: "%s" file already exist in this dataset' % (os.path.basename(file_path))
189 if not ignore_repetition:
190 return 'ERROR:: "%s" resource already exist in this dataset' % (self.dict['name'])
191 print('WARRING:: "'+ str(self.dict['name']) +'" resource already exist in this dataset')
176 #------------------------------------------------------------#
192 #------------------------------------------------------------#
177
178 try:
193 try:
179 return getattr(self.ckan.action, 'resource_create')(package_id=dataset_id, file_date=file_date, upload=open(file_path, 'rb'), voc_file_type=file_type, name=os.path.basename(file_path), **self.dict)
194 return getattr(self.ckan.action, 'resource_create')(package_id=dataset_id, file_date=file_date, voc_file_type=file_type, **self.dict)
180 except:
195 except:
181 _, exc_value, _ = sys.exc_info()
196 _, exc_value, _ = sys.exc_info()
182 return exc_value
197 return exc_value
183
198
184 def upload_multiple_files_advance(self, dataset_id, path_files, file_date, file_type, max_size=100, max_count=500, ignore_repetition=False, **kwargs):
199 def upload_multiple_files_advance(self, dataset_id, path_files, file_date, file_type, max_size=100, max_count=500, ignore_repetition=False, **kwargs):
185 # Agregar si es interruptido por teclado
200 # Agregar si es interruptido por teclado
186 '''
201 '''
187 FINALIDAD:
202 FINALIDAD:
188 Funcion para subir multiples archivos al repositorio del ROJ.
203 Funcion para subir multiples archivos al repositorio del ROJ.
189
204
190 PARAMETROS DISPONIBLES:
205 PARAMETROS DISPONIBLES:
191 CONSULTAR: "GUIA DE SCRIPT.pdf"
206 CONSULTAR: "GUIA DE SCRIPT.pdf"
192
207
193 ESTRUCTURA:
208 ESTRUCTURA:
194 <access_name>.upload_multiple_files_advance(dataset_id = <class 'str'>, path_files = <class 'list of strings'>, file_date = <class 'str'>, file_type = <class 'str'>, param_1 = <class 'param_1'>, ...)
209 <access_name>.upload_multiple_files_advance(dataset_id = <class 'str'>, path_files = <class 'list of strings'>, file_date = <class 'str'>, file_type = <class 'str'>, param_1 = <class 'param_1'>, ...)
195 '''
210 '''
196 #-------------------------PACKAGE SHOW-----------------------#
211 #-------------------------PACKAGE SHOW-----------------------#
197 try:
212 try:
198 dataset_show = getattr(self.ckan.action, 'package_show')(id=dataset_id)['resources']
213 dataset_show = getattr(self.ckan.action, 'package_show')(id=dataset_id)['resources']
199 except:
214 except:
200 _, exc_value, _ = sys.exc_info()
215 _, exc_value, _ = sys.exc_info()
201 print('ERROR obtaining metadata dataset:: Use the "print" for more information')
216 print('ERROR obtaining metadata dataset:: Use the "print" for more information')
202 return exc_value
217 return exc_value
203 #------------------------------------------------------------#
218 #------------------------------------------------------------#
204 resources_name = []
219 resources_name = []
205 for u in dataset_show:
220 for u in dataset_show:
206 resources_name.append(u['name'].lower())
221 resources_name.append(u['name'].lower())
207 #------------------------------------------------------------#
222 #------------------------------------------------------------#
208 self.list = ['package_id', 'upload', 'voc_file_type', 'name']
223 self.list = ['package_id', 'upload', 'voc_file_type', 'name']
209 for key1, value1 in kwargs.items():
224 for key1, value1 in kwargs.items():
210 if not key1 in self.list:
225 if not key1 in self.list:
211 self.dict[key1] = value1
226 self.dict[key1] = value1
212 #------------------------------------------------------------#
227 #------------------------------------------------------------#
213 if not 'others' in kwargs:
228 if not 'others' in kwargs:
214 self.dict['others'] = ''
229 self.dict['others'] = ''
215 else:
230 else:
216 if isinstance(kwargs['others'], list):
231 if isinstance(kwargs['others'], list):
217 self.dict['others'] = json.dumps(kwargs['others'])
232 self.dict['others'] = json.dumps(kwargs['others'])
218 #------------------------------------------------------------#
233 #------------------------------------------------------------#
219 total_list = []
234 total_list = []
220 #---------------CASO : "path" or "path_list"-----------------#
235 #---------------CASO : "path" or "path_list"-----------------#
221 if type(path_files) is list:
236 if type(path_files) is list:
222 if len(path_files) != 0:
237 if len(path_files) != 0:
223 path_files.sort()
238 path_files.sort()
224 for u in path_files:
239 for u in path_files:
225 if os.path.isfile(u):
240 if os.path.isfile(u):
226 if os.path.basename(u).lower() in resources_name:
241 if os.path.basename(u).lower() in resources_name:
227 if not ignore_repetition:
242 if not ignore_repetition:
228 return 'ERROR:: "%s" file already exist in this dataset' % (os.path.basename(u))
243 return 'ERROR:: "%s" file already exist in this dataset' % (os.path.basename(u))
229 print('WARRING:: "'+ str(os.path.basename(u)) +'" file was ignored because already exist in this dataset')
244 print('WARRING:: "'+ str(os.path.basename(u)) +'" file was ignored because already exist in this dataset')
230 else:
245 else:
231 total_list.append({'name':os.path.basename(u), 'size': os.stat(u).st_size, 'upload':open(u, 'rb')})
246 total_list.append({'name':os.path.basename(u), 'size': os.stat(u).st_size, 'upload':open(u, 'rb')})
232 else:
247 else:
233 return 'File "%s" does not exist' % (u)
248 return 'File "%s" does not exist' % (u)
234 else:
249 else:
235 return 'ERROR:: "path_list is empty"'
250 return 'ERROR:: "path_list is empty"'
236
251
237 elif type(path_files) is str:
252 elif type(path_files) is str:
238 if os.path.isdir(path_files):
253 if os.path.isdir(path_files):
239 path_order = [f for f in os.listdir(path_files) if os.path.isfile(os.path.join(path_files, f))]
254 path_order = [f for f in os.listdir(path_files) if os.path.isfile(os.path.join(path_files, f))]
240 path_order.sort()
255 path_order.sort()
241 if path_order:
256 if path_order:
242 for name in path_order:
257 for name in path_order:
243 if name.lower() in resources_name:
258 if name.lower() in resources_name:
244 if not ignore_repetition:
259 if not ignore_repetition:
245 return 'ERROR:: "%s" file already exist in this dataset' % (name)
260 return 'ERROR:: "%s" file already exist in this dataset' % (name)
246 print('WARRING:: "'+ name +'" file was ignored because already exist in this dataset')
261 print('WARRING:: "'+ name +'" file was ignored because already exist in this dataset')
247 else:
262 else:
248 total_list.append({'name':name, 'size': os.stat(os.path.join(path_files, name)).st_size, 'upload':open(os.path.join(path_files, name), 'rb')})
263 total_list.append({'name':name, 'size': os.stat(os.path.join(path_files, name)).st_size, 'upload':open(os.path.join(path_files, name), 'rb')})
249 else:
264 else:
250 return "ERROR:: There aren't files in this directory"
265 return "ERROR:: There aren't files in this directory"
251 else:
266 else:
252 return 'ERROR:: Directory "%s" does not exist' % (path_files)
267 return 'ERROR:: Directory "%s" does not exist' % (path_files)
253 else:
268 else:
254 return 'ERROR:: "path_files" must be a str or list'
269 return 'ERROR:: "path_files" must be a str or list'
255 #------------------------------------------------------------#
270 #------------------------------------------------------------#
256 try:
271 try:
257 uuid.UUID(str(dataset_id), version=4)
272 uuid.UUID(str(dataset_id), version=4)
258 package_id_or_name = '"id": "' + str(dataset_id) + '"'
273 package_id_or_name = '"id": "' + str(dataset_id) + '"'
259 except ValueError:
274 except ValueError:
260 package_id_or_name = '"name": "' + str(dataset_id) + '"'
275 package_id_or_name = '"name": "' + str(dataset_id) + '"'
261 #------------------------------------------------------------#
276 #------------------------------------------------------------#
262 blocks = [[]]
277 blocks = [[]]
263 size_file = 0
278 size_file = 0
264 count_file = 0
279 count_file = 0
265 inter_num = 0
280 inter_num = 0
266 for value in total_list:
281 for value in total_list:
267 if value['size'] > 1024 * 1024 * float(max_size):
282 if value['size'] > 1024 * 1024 * float(max_size):
268 return 'ERROR:: The size of the "%s" file is %sMB aprox, please change "max_size" value' % (value['name'], str(round(value['size']/(1024 * 1024), 2)))
283 return 'ERROR:: The size of the "%s" file is %sMB aprox, please change "max_size" value' % (value['name'], str(round(value['size']/(1024 * 1024), 2)))
269 if not 1 <= int(max_count) <= 999:
284 if not 1 <= int(max_count) <= 999:
270 return 'ERROR:: The count of the number of files must be between 1 and 999, please change "max_count" value'
285 return 'ERROR:: The count of the number of files must be between 1 and 999, please change "max_count" value'
271
286
272 size_file = size_file + value['size']
287 size_file = size_file + value['size']
273 count_file = count_file + 1
288 count_file = count_file + 1
274 if size_file <= 1024 * 1024 * float(max_size) and count_file <= int(max_count):
289 if size_file <= 1024 * 1024 * float(max_size) and count_file <= int(max_count):
275 del value['size']
290 del value['size']
276 blocks[inter_num].append(value)
291 blocks[inter_num].append(value)
277 else:
292 else:
278 inter_num = inter_num + 1
293 inter_num = inter_num + 1
279 size_file = value['size']
294 size_file = value['size']
280 count_file = 1
295 count_file = 1
281 blocks.append([])
296 blocks.append([])
282 del value['size']
297 del value['size']
283 blocks[inter_num].append(value)
298 blocks[inter_num].append(value)
284 #------------------------------------------------------------#
299 #------------------------------------------------------------#
285 if len(blocks[0]) > 0:
300 if len(blocks[0]) > 0:
286 print('BLOCK(S) IN TOTAL:: {}'.format(len(blocks)))
301 print('BLOCK(S) IN TOTAL:: {}'.format(len(blocks)))
287 for count1, block in enumerate(blocks):
302 for count1, block in enumerate(blocks):
288 print('---- BLOCK N°{} ----'.format(count1 + 1))
303 print('---- BLOCK N°{} ----'.format(count1 + 1))
289 resource_extend = []
304 resource_extend = []
290 files_dict = {}
305 files_dict = {}
291 for count2, value2 in enumerate(block):
306 for count2, value2 in enumerate(block):
292 value2['file_date'] = file_date
307 value2['file_date'] = file_date
293 value2['voc_file_type'] = file_type
308 value2['voc_file_type'] = file_type
294 value2.update(self.dict)
309 value2.update(self.dict)
295
310
296 #if not 'format' in value2:
311 #if not 'format' in value2:
297 # format = ''.join(pathlib.Path(value2['name']).suffixes)
312 # format = ''.join(pathlib.Path(value2['name']).suffixes)
298 # if len(format) > 0:
313 # if len(format) > 0:
299 # value2['format'] = format.upper()[1:]
314 # value2['format'] = format.upper()[1:]
300
315
301 files_dict['update__resources__-'+ str(len(block)-count2) +'__upload'] = (value2['name'], value2['upload'])
316 files_dict['update__resources__-'+ str(len(block)-count2) +'__upload'] = (value2['name'], value2['upload'])
302 del value2['upload']
317 del value2['upload']
303 resource_extend.append(value2)
318 resource_extend.append(value2)
304
319
305 print('BLOCK N°{} :: "{}" file(s) found >> uploading'.format(count1 + 1, len(block)))
320 print('BLOCK N°{} :: "{}" file(s) found >> uploading'.format(count1 + 1, len(block)))
306 try:
321 try:
307 result = self.ckan.call_action(
322 result = self.ckan.call_action(
308 'package_revise',
323 'package_revise',
309 {'match': '{'+ str(package_id_or_name) +'}', 'update__resources__extend': json.dumps(resource_extend)},
324 {'match': '{'+ str(package_id_or_name) +'}', 'update__resources__extend': json.dumps(resource_extend)},
310 files=files_dict
325 files=files_dict
311 )
326 )
312 print('BLOCK N°{} :: Uploaded file(s) successfully'.format(count1 + 1))
327 print('BLOCK N°{} :: Uploaded file(s) successfully'.format(count1 + 1))
313 if len(blocks) == count1 + 1:
328 if len(blocks) == count1 + 1:
314 return result
329 return result
315 except:
330 except:
316 print('ERROR :: Use the "print" for more information')
331 print('ERROR :: Use the "print" for more information')
317 _, exc_value, _ = sys.exc_info()
332 _, exc_value, _ = sys.exc_info()
318 return exc_value
333 return exc_value
319 else:
334 else:
320 return "ERROR:: No file(s) found to upload"
335 return "ERROR:: No file(s) found to upload"
321
336
322 def upload_multiple_files(self, dataset_id, path_files, date_files, type_files, ignore_repetition=False, **kwargs):
337 def upload_multiple_files(self, dataset_id, path_files, date_files, type_files, ignore_repetition=False, **kwargs):
323 # Agregar si es interruptido por teclado
338 # Agregar si es interruptido por teclado
324 '''
339 '''
325 FINALIDAD:
340 FINALIDAD:
326 Funcion para subir multiples archivos al repositorio del ROJ.
341 Funcion para subir multiples archivos al repositorio del ROJ.
327
342
328 PARAMETROS DISPONIBLES:
343 PARAMETROS DISPONIBLES:
329 CONSULTAR: "GUIA DE SCRIPT.pdf"
344 CONSULTAR: "GUIA DE SCRIPT.pdf"
330
345
331 ESTRUCTURA:
346 ESTRUCTURA:
332 <access_name>.upload_multiple_files(dataset_id = <class 'str'>, path_files = <class 'str'> or <class 'list of strings'>, date_files = <class 'str'> or <class 'list of strings'>, type_files = <class 'str'> or <class 'list of strings'>, param_1 = <class 'param_1'>, ...)
347 <access_name>.upload_multiple_files(dataset_id = <class 'str'>, path_files = <class 'str'> or <class 'list of strings'>, date_files = <class 'str'> or <class 'list of strings'>, type_files = <class 'str'> or <class 'list of strings'>, param_1 = <class 'param_1'>, ...)
333 '''
348 '''
334 #-------------------------PACKAGE SHOW-----------------------#
349 #-------------------------PACKAGE SHOW-----------------------#
335 try:
350 try:
336 dataset_show = getattr(self.ckan.action, 'package_show')(id=dataset_id)['resources']
351 dataset_show = getattr(self.ckan.action, 'package_show')(id=dataset_id)['resources']
337 except:
352 except:
338 _, exc_value, _ = sys.exc_info()
353 _, exc_value, _ = sys.exc_info()
339 print('ERROR obtaining metadata dataset:: Use the "print" for more information')
354 print('ERROR obtaining metadata dataset:: Use the "print" for more information')
340 return exc_value
355 return exc_value
341 #------------------------------------------------------------#
356 #------------------------------------------------------------#
342 resources_name = []
357 resources_name = []
343 for u in dataset_show:
358 for u in dataset_show:
344 resources_name.append(u['name'].lower())
359 resources_name.append(u['name'].lower())
345 #------------------------------------------------------------#
360 #------------------------------------------------------------#
346
361
347 params_dict = {'upload':[], 'name':[]}
362 params_dict = {'upload':[], 'name':[]}
348 #if not 'format' in kwargs:
363 #if not 'format' in kwargs:
349 # params_dict.update({'format':[]})
364 # params_dict.update({'format':[]})
350 #---------------CASO : "path" or "path_list"-----------------#
365 #---------------CASO : "path" or "path_list"-----------------#
351 if type(path_files) is list:
366 if type(path_files) is list:
352 if len(path_files) != 0:
367 if len(path_files) != 0:
353 path_files.sort()
368 path_files.sort()
354 for u in path_files:
369 for u in path_files:
355 if os.path.isfile(u):
370 if os.path.isfile(u):
356 if os.path.basename(u).lower() in resources_name:
371 if os.path.basename(u).lower() in resources_name:
357 if not ignore_repetition:
372 if not ignore_repetition:
358 return 'ERROR:: "%s" file already exist in this dataset' % (os.path.basename(u))
373 return 'ERROR:: "%s" file already exist in this dataset' % (os.path.basename(u))
359 print('WARRING:: "'+ str(os.path.basename(u)) +'" file was ignored because already exist in this dataset')
374 print('WARRING:: "'+ str(os.path.basename(u)) +'" file was ignored because already exist in this dataset')
360 else:
375 else:
361 params_dict['upload'].append(open(u, 'rb'))
376 params_dict['upload'].append(open(u, 'rb'))
362 params_dict['name'].append(os.path.basename(u))
377 params_dict['name'].append(os.path.basename(u))
363 #if not 'format' in kwargs:
378 #if not 'format' in kwargs:
364 # format = ''.join(pathlib.Path(u).suffixes)
379 # format = ''.join(pathlib.Path(u).suffixes)
365 # if len(format) > 0:
380 # if len(format) > 0:
366 # params_dict['format'].append(format.upper()[1:])
381 # params_dict['format'].append(format.upper()[1:])
367 # else:
382 # else:
368 # params_dict['format'].append('')
383 # params_dict['format'].append('')
369 else:
384 else:
370 return 'File "%s" does not exist' % (u)
385 return 'File "%s" does not exist' % (u)
371 else:
386 else:
372 return 'ERROR:: "path_list is empty"'
387 return 'ERROR:: "path_list is empty"'
373 elif type(path_files) is str:
388 elif type(path_files) is str:
374 if os.path.isdir(path_files):
389 if os.path.isdir(path_files):
375 path_order = [f for f in os.listdir(path_files) if os.path.isfile(os.path.join(path_files, f))]
390 path_order = [f for f in os.listdir(path_files) if os.path.isfile(os.path.join(path_files, f))]
376 path_order.sort()
391 path_order.sort()
377 if path_order:
392 if path_order:
378 for name in path_order:
393 for name in path_order:
379 if name.lower() in resources_name:
394 if name.lower() in resources_name:
380 if not ignore_repetition:
395 if not ignore_repetition:
381 return 'ERROR:: "%s" file already exist in this dataset' % (name)
396 return 'ERROR:: "%s" file already exist in this dataset' % (name)
382 print('WARRING:: "'+ str(name) +'" file was ignored because already exist in this dataset')
397 print('WARRING:: "'+ str(name) +'" file was ignored because already exist in this dataset')
383 else:
398 else:
384 params_dict['upload'].append(open(os.path.join(path_files, name), 'rb'))
399 params_dict['upload'].append(open(os.path.join(path_files, name), 'rb'))
385 params_dict['name'].append(name)
400 params_dict['name'].append(name)
386 #if not 'format' in kwargs:
401 #if not 'format' in kwargs:
387 # format = ''.join(pathlib.Path(name).suffixes)
402 # format = ''.join(pathlib.Path(name).suffixes)
388 # if len(format) > 0:
403 # if len(format) > 0:
389 # params_dict['format'].append(format.upper()[1:])
404 # params_dict['format'].append(format.upper()[1:])
390 # else:
405 # else:
391 # params_dict['format'].append('')
406 # params_dict['format'].append('')
392 else:
407 else:
393 return "ERROR:: There aren't files in this directory"
408 return "ERROR:: There aren't files in this directory"
394 else:
409 else:
395 return 'ERROR:: Directory "%s" does not exist' % (path_files)
410 return 'ERROR:: Directory "%s" does not exist' % (path_files)
396 else:
411 else:
397 return 'ERROR:: "path_files" must be a str or list'
412 return 'ERROR:: "path_files" must be a str or list'
398 #------------------------------------------------------------#
413 #------------------------------------------------------------#
399 params_no_dict = {'package_id': dataset_id}
414 params_no_dict = {'package_id': dataset_id}
400 if type(date_files) is list:
415 if type(date_files) is list:
401 params_dict['file_date'] = date_files
416 params_dict['file_date'] = date_files
402 else:
417 else:
403 params_no_dict['file_date'] = date_files
418 params_no_dict['file_date'] = date_files
404
419
405 if type(type_files) is list:
420 if type(type_files) is list:
406 params_dict['voc_file_type'] = type_files
421 params_dict['voc_file_type'] = type_files
407 else:
422 else:
408 params_no_dict['voc_file_type'] = type_files
423 params_no_dict['voc_file_type'] = type_files
409
424
410 for key1, value1 in kwargs.items():
425 for key1, value1 in kwargs.items():
411 if not key1 in params_dict and not key1 in params_no_dict and key1 != 'others':
426 if not key1 in params_dict and not key1 in params_no_dict and key1 != 'others':
412 if type(value1) is list:
427 if type(value1) is list:
413 params_dict[key1] = value1
428 params_dict[key1] = value1
414 else:
429 else:
415 params_no_dict[key1] = value1
430 params_no_dict[key1] = value1
416 #------------------------------------------#
431 #------------------------------------------#
417 if not 'others' in kwargs:
432 if not 'others' in kwargs:
418 params_no_dict['others'] = ''
433 params_no_dict['others'] = ''
419 else:
434 else:
420 if isinstance(kwargs['others'], tuple):
435 if isinstance(kwargs['others'], tuple):
421 params_dict['others'] = [json.dumps(w) for w in kwargs['others']]
436 params_dict['others'] = [json.dumps(w) for w in kwargs['others']]
422 elif isinstance(kwargs['others'], list):
437 elif isinstance(kwargs['others'], list):
423 params_no_dict['others'] = json.dumps(kwargs['others'])
438 params_no_dict['others'] = json.dumps(kwargs['others'])
424 elif isinstance(kwargs['others'], str):
439 elif isinstance(kwargs['others'], str):
425 params_no_dict['others'] = kwargs['others']
440 params_no_dict['others'] = kwargs['others']
426 else:
441 else:
427 return 'ERROR:: "others" must be a tuple, list or str'
442 return 'ERROR:: "others" must be a tuple, list or str'
428 #------------------------------------------#
443 #------------------------------------------#
429 len_params_dict = []
444 len_params_dict = []
430 for value2 in params_dict.values():
445 for value2 in params_dict.values():
431 len_params_dict.append(len(value2))
446 len_params_dict.append(len(value2))
432
447
433 if len(list(set(len_params_dict))) > 1:
448 if len(list(set(len_params_dict))) > 1:
434 return 'ERROR:: All lists must be the same length: %s' % (len(params_dict['name']))
449 return 'ERROR:: All lists must be the same length: %s' % (len(params_dict['name']))
435 #------------------------------------------------------------#
450 #------------------------------------------------------------#
436 print('"{}" file(s) found >> uploading'.format(len(params_dict['name'])))
451 print('"{}" file(s) found >> uploading'.format(len(params_dict['name'])))
437 for v in range(len(params_dict['name'])):
452 for v in range(len(params_dict['name'])):
438 try:
453 try:
439 send = {}
454 send = {}
440 for key_dict, value_dict in params_dict.items():
455 for key_dict, value_dict in params_dict.items():
441 send[key_dict] = value_dict[v]
456 send[key_dict] = value_dict[v]
442 for key_no_dict, value_no_dict in params_no_dict.items():
457 for key_no_dict, value_no_dict in params_no_dict.items():
443 send[key_no_dict] = value_no_dict
458 send[key_no_dict] = value_no_dict
444
459
445 self.list.append(getattr(self.ckan.action, 'resource_create')(**send))
460 self.list.append(getattr(self.ckan.action, 'resource_create')(**send))
446 print('File #{} :: "{}" was uploaded successfully'.format(v+1, params_dict['name'][v]))
461 print('File #{} :: "{}" was uploaded successfully'.format(v+1, params_dict['name'][v]))
447 except:
462 except:
448 _, exc_value, _ = sys.exc_info()
463 _, exc_value, _ = sys.exc_info()
449 self.list.append(exc_value)
464 self.list.append(exc_value)
450 print('File #{} :: Error uploading "{}" file'.format(v+1, params_dict['name'][v]))
465 print('File #{} :: Error uploading "{}" file'.format(v+1, params_dict['name'][v]))
451 return self.list
466 return self.list
452 #------------------------------------------------------------#
467 #------------------------------------------------------------#
453
468
454 def show(self, type_option, id, **kwargs):
469 def show(self, type_option, id, **kwargs):
455 '''
470 '''
456 FINALIDAD:
471 FINALIDAD:
457 Funcion personalizada para una busqueda en especifico.
472 Funcion personalizada para una busqueda en especifico.
458
473
459 PARAMETROS DISPONIBLES:
474 PARAMETROS DISPONIBLES:
460 CONSULTAR: "GUIA DE SCRIPT.pdf"
475 CONSULTAR: "GUIA DE SCRIPT.pdf"
461
476
462 ESTRUCTURA:
477 ESTRUCTURA:
463 <access_name>.show(type_option = <class 'str'>, id = <class 'str'>, param_1 = <class 'param_1'>, ...)
478 <access_name>.show(type_option = <class 'str'>, id = <class 'str'>, param_1 = <class 'param_1'>, ...)
464 '''
479 '''
465 if type(type_option) is str:
480 if type(type_option) is str:
466 try:
481 try:
467 if type_option == 'dataset':
482 if type_option == 'dataset':
468 return getattr(self.ckan.action, 'package_show')(id=id, **kwargs)
483 return getattr(self.ckan.action, 'package_show')(id=id, **kwargs)
469 elif type_option == 'resource':
484 elif type_option == 'resource':
470 return getattr(self.ckan.action, 'resource_show')(id=id, **kwargs)
485 return getattr(self.ckan.action, 'resource_show')(id=id, **kwargs)
471 elif type_option == 'project':
486 elif type_option == 'project':
472 return getattr(self.ckan.action, 'organization_show')(id=id, **kwargs)
487 return getattr(self.ckan.action, 'organization_show')(id=id, **kwargs)
473 elif type_option == 'collaborator':
488 elif type_option == 'collaborator':
474 return getattr(self.ckan.action, 'package_collaborator_list_for_user')(id=id, **kwargs)
489 return getattr(self.ckan.action, 'package_collaborator_list_for_user')(id=id, **kwargs)
475 elif type_option == 'member':
490 elif type_option == 'member':
476 return getattr(self.ckan.action, 'organization_list_for_user')(id=id, **kwargs)
491 return getattr(self.ckan.action, 'organization_list_for_user')(id=id, **kwargs)
477 elif type_option == 'vocabulary':
492 elif type_option == 'vocabulary':
478 return getattr(self.ckan.action, 'vocabulary_show')(id=id, **kwargs)
493 return getattr(self.ckan.action, 'vocabulary_show')(id=id, **kwargs)
479 elif type_option == 'tag':
494 elif type_option == 'tag':
480 if not 'vocabulary_id' in kwargs:
495 if not 'vocabulary_id' in kwargs:
481 print('Missing "vocabulary_id" value: assume it is a free tag')
496 print('Missing "vocabulary_id" value: assume it is a free tag')
482 return getattr(self.ckan.action, 'tag_show')(id=id, **kwargs)
497 return getattr(self.ckan.action, 'tag_show')(id=id, **kwargs)
483 elif type_option == 'user':
498 elif type_option == 'user':
484 return getattr(self.ckan.action, 'user_show')(id=id, **kwargs)
499 return getattr(self.ckan.action, 'user_show')(id=id, **kwargs)
485 elif type_option == 'job':
500 elif type_option == 'job':
486 return getattr(self.ckan.action, 'job_show')(id=id, **kwargs)
501 return getattr(self.ckan.action, 'job_show')(id=id, **kwargs)
487 else:
502 else:
488 return 'ERROR:: "type_option = %s" is not accepted' % (type_option)
503 return 'ERROR:: "type_option = %s" is not accepted' % (type_option)
489 except:
504 except:
490 _, exc_value, _ = sys.exc_info()
505 _, exc_value, _ = sys.exc_info()
491 return exc_value
506 return exc_value
492 else:
507 else:
493 return 'ERROR:: "type_option" must be a str'
508 return 'ERROR:: "type_option" must be a str'
494
509
495 def search(self, type_option, query=None, **kwargs):
510 def search(self, type_option, query=None, **kwargs):
496 '''
511 '''
497 FINALIDAD:
512 FINALIDAD:
498 Funcion personalizada para busquedas que satisfagan algun criterio.
513 Funcion personalizada para busquedas que satisfagan algun criterio.
499
514
500 PARAMETROS DISPONIBLES:
515 PARAMETROS DISPONIBLES:
501 CONSULTAR: "GUIA DE SCRIPT.pdf"
516 CONSULTAR: "GUIA DE SCRIPT.pdf"
502
517
503 ESTRUCTURA:
518 ESTRUCTURA:
504 <access_name>.search(type_option = <class 'str'>, query = <class 'dict'>, param_1 = <class 'param_1'>, ...)
519 <access_name>.search(type_option = <class 'str'>, query = <class 'dict'>, param_1 = <class 'param_1'>, ...)
505 '''
520 '''
506 if type(type_option) is str:
521 if type(type_option) is str:
507 try:
522 try:
508 if type_option == 'dataset':
523 if type_option == 'dataset':
509 key_replace = ['fq', 'fq_list', 'include_private']
524 key_replace = ['fq', 'fq_list', 'include_private']
510 key_point = ['facet_mincount', 'facet_limit', 'facet_field']
525 key_point = ['facet_mincount', 'facet_limit', 'facet_field']
511 for key1, value1 in kwargs.items():
526 for key1, value1 in kwargs.items():
512 if not key1 in key_replace:
527 if not key1 in key_replace:
513 if key1 in key_point:
528 if key1 in key_point:
514 self.dict[key1.replace('_', '.')] = value1
529 self.dict[key1.replace('_', '.')] = value1
515 else:
530 else:
516 self.dict[key1] = value1
531 self.dict[key1] = value1
517
532
518 if query is not None:
533 if query is not None:
519 if type(query) is dict:
534 if type(query) is dict:
520 self.dict['fq_list'] = []
535 self.dict['fq_list'] = []
521 #NUM_RESOURCES_MIN / NUM_RESOURCES_MAX
536 #NUM_RESOURCES_MIN / NUM_RESOURCES_MAX
522 #----------------------------------------------------#
537 #----------------------------------------------------#
523 if 'dataset_start_date' in query:
538 if 'dataset_start_date' in query:
524 if type(query['dataset_start_date']) is str:
539 if type(query['dataset_start_date']) is str:
525 try:
540 try:
526 datetime.strptime(query['dataset_start_date'], '%Y-%m-%d')
541 datetime.strptime(query['dataset_start_date'], '%Y-%m-%d')
527 if len(query['dataset_start_date']) != 10:
542 if len(query['dataset_start_date']) != 10:
528 return '"dataset_start_date", must be: <YYYY-MM-DD>'
543 return '"dataset_start_date", must be: <YYYY-MM-DD>'
529 self.dict['fq_list'].append('dataset_start_date:"'+query['dataset_start_date']+'"')
544 self.dict['fq_list'].append('dataset_start_date:"'+query['dataset_start_date']+'"')
530 self.list.append('dataset_start_date')
545 self.list.append('dataset_start_date')
531 except:
546 except:
532 return '"dataset_start_date" incorrect: "%s"' % (query['dataset_start_date'])
547 return '"dataset_start_date" incorrect: "%s"' % (query['dataset_start_date'])
533 else:
548 else:
534 return '"dataset_start_date" must be <str>'
549 return '"dataset_start_date" must be <str>'
535 #----------------------------------------------------#
550 #----------------------------------------------------#
536 if 'dataset_end_date' in query:
551 if 'dataset_end_date' in query:
537 if type(query['dataset_end_date']) is str:
552 if type(query['dataset_end_date']) is str:
538 try:
553 try:
539 datetime.strptime(query['dataset_end_date'], '%Y-%m-%d')
554 datetime.strptime(query['dataset_end_date'], '%Y-%m-%d')
540 if len(query['dataset_end_date']) != 10:
555 if len(query['dataset_end_date']) != 10:
541 return '"dataset_end_date", must be: <YYYY-MM-DD>'
556 return '"dataset_end_date", must be: <YYYY-MM-DD>'
542
557
543 if 'dataset_start_date' in query:
558 if 'dataset_start_date' in query:
544 if query['dataset_start_date'] > query['dataset_end_date']:
559 if query['dataset_start_date'] > query['dataset_end_date']:
545 return '"dataset_end_date" must be greater than "dataset_start_date"'
560 return '"dataset_end_date" must be greater than "dataset_start_date"'
546
561
547 self.dict['fq_list'].append('dataset_end_date:"'+query['dataset_end_date']+'"')
562 self.dict['fq_list'].append('dataset_end_date:"'+query['dataset_end_date']+'"')
548 self.list.append('dataset_end_date')
563 self.list.append('dataset_end_date')
549 except:
564 except:
550 return '"dataset_end_date" incorrect: "%s"' % (query['dataset_end_date'])
565 return '"dataset_end_date" incorrect: "%s"' % (query['dataset_end_date'])
551 else:
566 else:
552 return '"dataset_end_date" must be <str>'
567 return '"dataset_end_date" must be <str>'
553 #----------------------------------------------------#
568 #----------------------------------------------------#
554 for key, value in query.items():
569 for key, value in query.items():
555 if value is not None and not key in self.list:
570 if value is not None and not key in self.list:
556 self.dict['fq_list'].append(str(key)+':"'+str(value)+'"')
571 self.dict['fq_list'].append(str(key)+':"'+str(value)+'"')
557 else:
572 else:
558 return '"query" must be <dict>'
573 return '"query" must be <dict>'
559
574
560 return getattr(self.ckan.action, 'package_search')(include_private=True, **self.dict)
575 return getattr(self.ckan.action, 'package_search')(include_private=True, **self.dict)
561
576
562 elif type_option == 'resource':
577 elif type_option == 'resource':
563 for key1, value1 in kwargs.items():
578 for key1, value1 in kwargs.items():
564 if key1 != 'fields':
579 if key1 != 'fields':
565 self.dict[key1] = value1
580 self.dict[key1] = value1
566
581
567 if query is not None:
582 if query is not None:
568 if type(query) is dict:
583 if type(query) is dict:
569 #----------------------------------------------------#
584 #----------------------------------------------------#
570 if 'file_date_min' in query:
585 if 'file_date_min' in query:
571 if type(query['file_date_min']) is str:
586 if type(query['file_date_min']) is str:
572 try:
587 try:
573 datetime.strptime(query['file_date_min'], '%Y-%m-%d')
588 datetime.strptime(query['file_date_min'], '%Y-%m-%d')
574 if len(query['file_date_min']) != 10:
589 if len(query['file_date_min']) != 10:
575 return '"file_date_min", must be: <YYYY-MM-DD>'
590 return '"file_date_min", must be: <YYYY-MM-DD>'
576 except:
591 except:
577 return '"file_date_min" incorrect: "%s"' % (query['file_date_min'])
592 return '"file_date_min" incorrect: "%s"' % (query['file_date_min'])
578 else:
593 else:
579 return '"file_date_min" must be <str>'
594 return '"file_date_min" must be <str>'
580 #----------------------------------------------------#
595 #----------------------------------------------------#
581 if 'file_date_max' in query:
596 if 'file_date_max' in query:
582 if type(query['file_date_max']) is str:
597 if type(query['file_date_max']) is str:
583 try:
598 try:
584 datetime.strptime(query['file_date_max'], '%Y-%m-%d')
599 datetime.strptime(query['file_date_max'], '%Y-%m-%d')
585 if len(query['file_date_max']) != 10:
600 if len(query['file_date_max']) != 10:
586 return '"file_date_max", must be: <YYYY-MM-DD>'
601 return '"file_date_max", must be: <YYYY-MM-DD>'
587
602
588 if 'file_date_min' in query:
603 if 'file_date_min' in query:
589 if query['file_date_min'] > query['file_date_max']:
604 if query['file_date_min'] > query['file_date_max']:
590 return '"file_date_max" must be greater than "file_date_min"'
605 return '"file_date_max" must be greater than "file_date_min"'
591 except:
606 except:
592 return '"file_date_max" incorrect: "%s"' % (query['file_date_max'])
607 return '"file_date_max" incorrect: "%s"' % (query['file_date_max'])
593 else:
608 else:
594 return '"file_date_max" must be <str>'
609 return '"file_date_max" must be <str>'
595 #----------------------------------------------------#
610 #----------------------------------------------------#
596 self.dict['query'] = query
611 self.dict['query'] = query
597 else:
612 else:
598 return '"query" must be <dict>'
613 return '"query" must be <dict>'
599 return getattr(self.ckan.action, 'resources_search')(**self.dict)
614 return getattr(self.ckan.action, 'resources_search')(**self.dict)
600
615
601 elif type_option == 'tag':
616 elif type_option == 'tag':
602 for key1, value1 in kwargs.items():
617 for key1, value1 in kwargs.items():
603 if key1 != 'fields':
618 if key1 != 'fields':
604 self.dict[key1] = value1
619 self.dict[key1] = value1
605
620
606 if not 'vocabulary_id' in kwargs:
621 if not 'vocabulary_id' in kwargs:
607 print('Missing "vocabulary_id" value: tags that don’t belong to any vocabulary')
622 print('Missing "vocabulary_id" value: tags that don’t belong to any vocabulary')
608 else:
623 else:
609 print('Only tags that belong to "{}" vocabulary'.format(kwargs['vocabulary_id']))
624 print('Only tags that belong to "{}" vocabulary'.format(kwargs['vocabulary_id']))
610
625
611 if query is not None:
626 if query is not None:
612 if type(query) is dict:
627 if type(query) is dict:
613 if 'search' in query:
628 if 'search' in query:
614 if type(query['search']) is list or type(query['search']) is str:
629 if type(query['search']) is list or type(query['search']) is str:
615 self.dict['query'] = query['search']
630 self.dict['query'] = query['search']
616 else:
631 else:
617 return '"search" must be <list> or <str>'
632 return '"search" must be <list> or <str>'
618 else:
633 else:
619 return '"query" must be <dict>'
634 return '"query" must be <dict>'
620 return getattr(self.ckan.action, 'tag_search')(**self.dict)
635 return getattr(self.ckan.action, 'tag_search')(**self.dict)
621
636
622 else:
637 else:
623 return 'ERROR:: "type_option = %s" is not accepted' % (type_option)
638 return 'ERROR:: "type_option = %s" is not accepted' % (type_option)
624
639
625 except:
640 except:
626 _, exc_value, _ = sys.exc_info()
641 _, exc_value, _ = sys.exc_info()
627 return exc_value
642 return exc_value
628 else:
643 else:
629 return 'ERROR:: "type_option" must be <str>'
644 return 'ERROR:: "type_option" must be <str>'
630
645
631 def create(self, type_option, select=None, **kwargs):
646 def create(self, type_option, select=None, **kwargs):
632 '''
647 '''
633 FINALIDAD:
648 FINALIDAD:
634 Funcion personalizada para crear.
649 Funcion personalizada para crear.
635
650
636 PARAMETROS DISPONIBLES:
651 PARAMETROS DISPONIBLES:
637 CONSULTAR: "GUIA DE SCRIPT.pdf"
652 CONSULTAR: "GUIA DE SCRIPT.pdf"
638
653
639 ESTRUCTURA:
654 ESTRUCTURA:
640 <access_name>.create(type_option = <class 'str'>, param_1 = <class 'param_1'>, ...)
655 <access_name>.create(type_option = <class 'str'>, param_1 = <class 'param_1'>, ...)
641 '''
656 '''
642 if type(type_option) is str:
657 if type(type_option) is str:
643 try:
658 try:
644 if type_option == 'dataset':
659 if type_option == 'dataset':
645 return getattr(self.ckan.action, 'package_create')(**kwargs)
660 return getattr(self.ckan.action, 'package_create')(**kwargs)
646 elif type_option == 'project':
661 elif type_option == 'project':
647 return getattr(self.ckan.action, 'organization_create')(**kwargs)
662 return getattr(self.ckan.action, 'organization_create')(**kwargs)
648 elif type_option == 'member':
663 elif type_option == 'member':
649 return getattr(self.ckan.action, 'organization_member_create')(**kwargs)
664 return getattr(self.ckan.action, 'organization_member_create')(**kwargs)
650 elif type_option == 'collaborator':
665 elif type_option == 'collaborator':
651 return getattr(self.ckan.action, 'package_collaborator_create')(**kwargs)
666 return getattr(self.ckan.action, 'package_collaborator_create')(**kwargs)
652 elif type_option == 'vocabulary':
667 elif type_option == 'vocabulary':
653 return getattr(self.ckan.action, 'vocabulary_create')(**kwargs)
668 return getattr(self.ckan.action, 'vocabulary_create')(**kwargs)
654 elif type_option == 'tag':
669 elif type_option == 'tag':
655 return getattr(self.ckan.action, 'tag_create')(**kwargs)
670 return getattr(self.ckan.action, 'tag_create')(**kwargs)
656 elif type_option == 'user':
671 elif type_option == 'user':
657 return getattr(self.ckan.action, 'user_create')(**kwargs)
672 return getattr(self.ckan.action, 'user_create')(**kwargs)
658 elif type_option == 'views':
673 elif type_option == 'views':
659 if 'resource' == select:
674 if 'resource' == select:
660 self.list = ['package']
675 self.list = ['package']
661 for key1, value1 in kwargs.items():
676 for key1, value1 in kwargs.items():
662 if not key1 in self.list:
677 if not key1 in self.list:
663 self.dict[key1] = value1
678 self.dict[key1] = value1
664 return getattr(self.ckan.action, 'resource_create_default_resource_views')(**self.dict)
679 return getattr(self.ckan.action, 'resource_create_default_resource_views')(**self.dict)
665 elif 'dataset' == select:
680 elif 'dataset' == select:
666 return getattr(self.ckan.action, 'package_create_default_resource_views')(**kwargs)
681 return getattr(self.ckan.action, 'package_create_default_resource_views')(**kwargs)
667 else:
682 else:
668 return 'ERROR:: "select = %s" is not accepted' % (select)
683 return 'ERROR:: "select = %s" is not accepted' % (select)
669 else:
684 else:
670 return 'ERROR:: "type_option = %s" is not accepted' % (type_option)
685 return 'ERROR:: "type_option = %s" is not accepted' % (type_option)
671 except:
686 except:
672 _, exc_value, _ = sys.exc_info()
687 _, exc_value, _ = sys.exc_info()
673 return exc_value
688 return exc_value
674 else:
689 else:
675 return 'ERROR:: "type_option" must be <str>'
690 return 'ERROR:: "type_option" must be <str>'
676
691
677 def patch(self, type_option, **kwargs):
692 def patch(self, type_option, **kwargs):
678 '''
693 '''
679 FINALIDAD:
694 FINALIDAD:
680 Funciones personalizadas para actualizar
695 Funciones personalizadas para actualizar
681
696
682 PARAMETROS DISPONIBLES:
697 PARAMETROS DISPONIBLES:
683 CONSULTAR: "GUIA DE SCRIPT.pdf"
698 CONSULTAR: "GUIA DE SCRIPT.pdf"
684
699
685 ESTRUCTURA:
700 ESTRUCTURA:
686 <access_name>.patch(type_option = <class 'str'>, param_1 = <class 'param_1'>, ...)
701 <access_name>.patch(type_option = <class 'str'>, param_1 = <class 'param_1'>, ...)
687 '''
702 '''
688 if type(type_option) is str:
703 if type(type_option) is str:
689 try:
704 try:
690 if type_option == 'dataset':
705 if type_option == 'dataset':
691 return getattr(self.ckan.action, 'package_patch')(**kwargs)
706 return getattr(self.ckan.action, 'package_patch')(**kwargs)
692 elif type_option == 'project':
707 elif type_option == 'project':
693 return getattr(self.ckan.action, 'organization_patch')(**kwargs)
708 return getattr(self.ckan.action, 'organization_patch')(**kwargs)
694 elif type_option == 'resource':
709 elif type_option == 'resource':
695 return getattr(self.ckan.action, 'resource_patch')(**kwargs)
710 return getattr(self.ckan.action, 'resource_patch')(**kwargs)
696 elif type_option == 'member':
711 elif type_option == 'member':
697 return getattr(self.ckan.action, 'organization_member_create')(**kwargs)
712 return getattr(self.ckan.action, 'organization_member_create')(**kwargs)
698 elif type_option == 'collaborator':
713 elif type_option == 'collaborator':
699 return getattr(self.ckan.action, 'package_collaborator_create')(**kwargs)
714 return getattr(self.ckan.action, 'package_collaborator_create')(**kwargs)
700 else:
715 else:
701 return 'ERROR:: "type_option = %s" is not accepted' % (type_option)
716 return 'ERROR:: "type_option = %s" is not accepted' % (type_option)
702 except:
717 except:
703 _, exc_value, _ = sys.exc_info()
718 _, exc_value, _ = sys.exc_info()
704 return exc_value
719 return exc_value
705 else:
720 else:
706 return 'ERROR:: "type_option" must be <str>'
721 return 'ERROR:: "type_option" must be <str>'
707
722
708 def delete(self, type_option, select=None, **kwargs):
723 def delete(self, type_option, select=None, **kwargs):
709 '''
724 '''
710 FINALIDAD:
725 FINALIDAD:
711 Función personalizada para eliminar y/o purgar.
726 Función personalizada para eliminar y/o purgar.
712
727
713 PARAMETROS DISPONIBLES:
728 PARAMETROS DISPONIBLES:
714 CONSULTAR: "GUIA DE SCRIPT.pdf"
729 CONSULTAR: "GUIA DE SCRIPT.pdf"
715
730
716 ESTRUCTURA:
731 ESTRUCTURA:
717 <access_name>.delete(type_option = <class 'str'>, param_1 = <class 'param_1'>, ...)
732 <access_name>.delete(type_option = <class 'str'>, param_1 = <class 'param_1'>, ...)
718 '''
733 '''
719 if type(type_option) is str:
734 if type(type_option) is str:
720 try:
735 try:
721 if type_option == 'dataset':
736 if type_option == 'dataset':
722 if select is None:
737 if select is None:
723 return 'ERROR:: "select" must not be "None"'
738 return 'ERROR:: "select" must not be "None"'
724 else:
739 else:
725 if 'delete' == select:
740 if 'delete' == select:
726 return getattr(self.ckan.action, 'package_delete')(**kwargs)
741 return getattr(self.ckan.action, 'package_delete')(**kwargs)
727 elif 'purge' == select:
742 elif 'purge' == select:
728 return getattr(self.ckan.action, 'dataset_purge')(**kwargs)
743 return getattr(self.ckan.action, 'dataset_purge')(**kwargs)
729 else:
744 else:
730 return 'ERROR:: "select = %s" is not accepted' % (select)
745 return 'ERROR:: "select = %s" is not accepted' % (select)
731 elif type_option == 'project':
746 elif type_option == 'project':
732 if select is None:
747 if select is None:
733 return 'ERROR:: "select" must not be "None"'
748 return 'ERROR:: "select" must not be "None"'
734 else:
749 else:
735 if 'delete' == select:
750 if 'delete' == select:
736 return getattr(self.ckan.action, 'organization_delete')(**kwargs)
751 return getattr(self.ckan.action, 'organization_delete')(**kwargs)
737 elif 'purge' == select:
752 elif 'purge' == select:
738 return getattr(self.ckan.action, 'organization_purge')(**kwargs)
753 return getattr(self.ckan.action, 'organization_purge')(**kwargs)
739 else:
754 else:
740 return 'ERROR:: "select = %s" is not accepted' % (select)
755 return 'ERROR:: "select = %s" is not accepted' % (select)
741 elif type_option == 'resource':
756 elif type_option == 'resource':
742 return getattr(self.ckan.action, 'resource_delete')(**kwargs)
757 return getattr(self.ckan.action, 'resource_delete')(**kwargs)
743 elif type_option == 'vocabulary':
758 elif type_option == 'vocabulary':
744 return getattr(self.ckan.action, 'vocabulary_delete')(**kwargs)
759 return getattr(self.ckan.action, 'vocabulary_delete')(**kwargs)
745 elif type_option == 'tag':
760 elif type_option == 'tag':
746 return getattr(self.ckan.action, 'tag_delete')(**kwargs)
761 return getattr(self.ckan.action, 'tag_delete')(**kwargs)
747 elif type_option == 'user':
762 elif type_option == 'user':
748 return getattr(self.ckan.action, 'user_delete')(**kwargs)
763 return getattr(self.ckan.action, 'user_delete')(**kwargs)
749 else:
764 else:
750 return 'ERROR:: "type_option = %s" is not accepted' % (type_option)
765 return 'ERROR:: "type_option = %s" is not accepted' % (type_option)
751 except:
766 except:
752 _, exc_value, _ = sys.exc_info()
767 _, exc_value, _ = sys.exc_info()
753 return exc_value
768 return exc_value
754 else:
769 else:
755 return 'ERROR:: "type_option" must be <str>'
770 return 'ERROR:: "type_option" must be <str>'
756
771
757 def f_status_note(self, total, result, path):
772 def f_status_note(self, total, result, path):
758 file_txt = open(path+'status_note.txt', 'w')
773 file_txt = open(path+'status_note.txt', 'w')
759 file_txt = open(path+'status_note.txt', 'a')
774 file_txt = open(path+'status_note.txt', 'a')
760
775
761 file_txt.write('DOWNLOADED FILE(S): "%s"' % (len(result['name'])))
776 file_txt.write('DOWNLOADED FILE(S): "%s"' % (len(result['name'])))
762 file_txt.write(''+ os.linesep)
777 file_txt.write(''+ os.linesep)
763 for u in result['name']:
778 for u in result['name']:
764 file_txt.write(' - '+ u + os.linesep)
779 file_txt.write(' - '+ u + os.linesep)
765 file_txt.write(''+ os.linesep)
780 file_txt.write(''+ os.linesep)
766
781
767 file_txt.write('FAILED FILE(S): "%s"' % (len(total['name'])-len(result['name'])))
782 file_txt.write('FAILED FILE(S): "%s"' % (len(total['name'])-len(result['name'])))
768 file_txt.write(''+ os.linesep)
783 file_txt.write(''+ os.linesep)
769 if len(total['name'])-len(result['name']) != 0:
784 if len(total['name'])-len(result['name']) != 0:
770 for u in total['name']:
785 for u in total['name']:
771 if not u in result['name']:
786 if not u in result['name']:
772 file_txt.write(' - '+ u + os.linesep)
787 file_txt.write(' - '+ u + os.linesep)
773 else:
788 else:
774 file_txt.write(' "None"'+ os.linesep)
789 file_txt.write(' "None"'+ os.linesep)
775
790
776 def f_name(self, name_dataset, ext, tempdir):
791 def f_name(self, name_dataset, ext, tempdir):
777 while self.check:
792 while self.check:
778 self.str = ''
793 self.str = ''
779 if self.cont == 0:
794 if self.cont == 0:
780 if os.path.exists(tempdir + name_dataset + ext):
795 if os.path.exists(tempdir + name_dataset + ext):
781 self.str = name_dataset+'('+str(self.cont+1)+')'+ext
796 self.str = name_dataset+'('+str(self.cont+1)+')'+ext
782 else:
797 else:
783 self.check = self.check * 0
798 self.check = self.check * 0
784 self.str = name_dataset + ext
799 self.str = name_dataset + ext
785 else:
800 else:
786 if not os.path.exists(tempdir + name_dataset+'('+str(self.cont)+')'+ext):
801 if not os.path.exists(tempdir + name_dataset+'('+str(self.cont)+')'+ext):
787 self.check = self.check * 0
802 self.check = self.check * 0
788 self.str = name_dataset+'('+str(self.cont)+')'+ ext
803 self.str = name_dataset+'('+str(self.cont)+')'+ ext
789 self.cont = self.cont+1
804 self.cont = self.cont+1
790 return self.str
805 return self.str
791
806
792 def f_zipdir(self, path, ziph, zip_name):
807 def f_zipdir(self, path, ziph, zip_name):
793 for root, _, files in os.walk(path):
808 for root, _, files in os.walk(path):
794 print('.....')
809 print('.....')
795 print('Creating: "{}" >>'.format(zip_name))
810 print('Creating: "{}" >>'.format(zip_name))
796 for __file in tqdm(iterable=files, total=len(files)):
811 for __file in tqdm(iterable=files, total=len(files)):
797 new_dir = os.path.relpath(os.path.join(root, __file), os.path.join(path, '..'))
812 new_dir = os.path.relpath(os.path.join(root, __file), os.path.join(path, '..'))
798 ziph.write(os.path.join(root, __file), new_dir)
813 ziph.write(os.path.join(root, __file), new_dir)
799 print('Created >>')
814 print('Created >>')
800
815
801 def download_by_step(self, response, tempdir_name):
816 def download_by_step(self, response, tempdir_name):
802 try:
817 try:
803 # ---------- REPLACE URL --------- #
818 # ---------- REPLACE URL --------- #
804 if urlparse(self.url).netloc != 'www.igp.gob.pe' and urlparse(response['url']).netloc == 'www.igp.gob.pe':
819 if urlparse(self.url).netloc != 'www.igp.gob.pe' and urlparse(response['url']).netloc == 'www.igp.gob.pe':
805 response['url'] = response['url'].replace(urlparse(response['url']).scheme + '://' + urlparse(response['url']).netloc,
820 response['url'] = response['url'].replace(urlparse(response['url']).scheme + '://' + urlparse(response['url']).netloc,
806 urlparse(self.url).scheme + '://' + urlparse(self.url).netloc)
821 urlparse(self.url).scheme + '://' + urlparse(self.url).netloc)
807 #----------------------------------#
822 #----------------------------------#
808 with requests.get(response['url'], stream=True, headers={'Authorization': self.Authorization}, verify=self.verify) as resp:
823 with requests.get(response['url'], stream=True, headers={'Authorization': self.Authorization}, verify=self.verify) as resp:
809 if resp.status_code == 200:
824 if resp.status_code == 200:
810 with open(tempdir_name+response['name'], 'wb') as file:
825 with open(tempdir_name+response['name'], 'wb') as file:
811 for chunk in resp.iter_content(chunk_size = self.chunk_size):
826 for chunk in resp.iter_content(chunk_size = self.chunk_size):
812 if chunk:
827 if chunk:
813 file.write(chunk)
828 file.write(chunk)
814 except requests.exceptions.RequestException:
829 except requests.exceptions.RequestException:
815 pass
830 pass
816
831
817 def download_files(self, **kwargs):
832 def download_files(self, **kwargs):
818 '''
833 '''
819 FINALIDAD:
834 FINALIDAD:
820 Funcion personalizada para la descarga de archivos existentes de un dataset.
835 Funcion personalizada para la descarga de archivos existentes de un dataset.
821
836
822 PARAMETROS DISPONIBLES:
837 PARAMETROS DISPONIBLES:
823 CONSULTAR: "GUIA DE SCRIPT.pdf"
838 CONSULTAR: "GUIA DE SCRIPT.pdf"
824
839
825 ESTRUCTURA:
840 ESTRUCTURA:
826 <access_name>.download_files(id = <class 'str'>, param_1 = <class 'param_1'>, ...)
841 <access_name>.download_files(id = <class 'str'>, param_1 = <class 'param_1'>, ...)
827 '''
842 '''
828 dict_local = {}
843 dict_local = {}
829 #----------------------------------------------#
844 #----------------------------------------------#
830 if 'zip' in kwargs:
845 if 'zip' in kwargs:
831 if type(kwargs['zip']) is not bool:
846 if type(kwargs['zip']) is not bool:
832 return 'ERROR:: "zip" must be: <class "bool">'
847 return 'ERROR:: "zip" must be: <class "bool">'
833 else:
848 else:
834 dict_local['zip'] = kwargs['zip']
849 dict_local['zip'] = kwargs['zip']
835 else:
850 else:
836 dict_local['zip'] = False
851 dict_local['zip'] = False
837 #----------------------------------------------#
852 #----------------------------------------------#
838 if 'status_note' in kwargs:
853 if 'status_note' in kwargs:
839 if type(kwargs['status_note']) is not bool:
854 if type(kwargs['status_note']) is not bool:
840 return 'ERROR:: "status_note" must be: <class "bool">'
855 return 'ERROR:: "status_note" must be: <class "bool">'
841 else:
856 else:
842 dict_local['status_note'] = kwargs['status_note']
857 dict_local['status_note'] = kwargs['status_note']
843 else:
858 else:
844 dict_local['status_note'] = False
859 dict_local['status_note'] = False
845 #----------------------------------------------#
860 #----------------------------------------------#
846 if 'path' in kwargs:
861 if 'path' in kwargs:
847 if type(kwargs['path']) is str:
862 if type(kwargs['path']) is str:
848 if os.path.isdir(kwargs['path']) == False:
863 if os.path.isdir(kwargs['path']) == False:
849 return 'ERROR:: "path" does not exist'
864 return 'ERROR:: "path" does not exist'
850 else:
865 else:
851 if kwargs['path'][-1:] != self.separator:
866 if kwargs['path'][-1:] != self.separator:
852 dict_local['path'] = kwargs['path']+self.separator
867 dict_local['path'] = kwargs['path']+self.separator
853 else:
868 else:
854 dict_local['path'] = kwargs['path']
869 dict_local['path'] = kwargs['path']
855
870
856 txt = dict_local['path']+datetime.now().strftime("%Y_%m_%d_%H_%M_%S_%f")+'.txt'
871 txt = dict_local['path']+datetime.now().strftime("%Y_%m_%d_%H_%M_%S_%f")+'.txt'
857 if int(platform.python_version()[0]) == 3:
872 if int(platform.python_version()[0]) == 3:
858 try:
873 try:
859 file_txt = open(txt, 'w')
874 file_txt = open(txt, 'w')
860 file_txt.close()
875 file_txt.close()
861 os.remove(txt)
876 os.remove(txt)
862 except PermissionError:
877 except PermissionError:
863 return 'ERROR:: Access denied, you are not authorized to write files: "%s"' % (dict_local['path'])
878 return 'ERROR:: Access denied, you are not authorized to write files: "%s"' % (dict_local['path'])
864 else:
879 else:
865 try:
880 try:
866 file_txt = open(txt, 'w')
881 file_txt = open(txt, 'w')
867 file_txt.close()
882 file_txt.close()
868 os.remove(txt)
883 os.remove(txt)
869 except:
884 except:
870 return 'ERROR:: Access denied, you are not authorized to write files: "%s"' % (dict_local['path'])
885 return 'ERROR:: Access denied, you are not authorized to write files: "%s"' % (dict_local['path'])
871 else:
886 else:
872 return 'ERROR:: "path" must be: <class "str">'
887 return 'ERROR:: "path" must be: <class "str">'
873 else:
888 else:
874 dict_local['path'] = ''
889 dict_local['path'] = ''
875 #----------------------------------------------#
890 #----------------------------------------------#
876 for key, value in kwargs.items():
891 for key, value in kwargs.items():
877 if not key in dict_local:
892 if not key in dict_local:
878 self.dict[key] = value
893 self.dict[key] = value
879 try:
894 try:
880 response = getattr(self.ckan.action, 'url_resources')(**self.dict)
895 response = getattr(self.ckan.action, 'url_resources')(**self.dict)
881 except:
896 except:
882 _, exc_value, _ = sys.exc_info()
897 _, exc_value, _ = sys.exc_info()
883 return exc_value
898 return exc_value
884
899
885 if len(response) != 0:
900 if len(response) != 0:
886 #--------------TEMP PATH---------------#
901 #--------------TEMP PATH---------------#
887 if dict_local['zip']:
902 if dict_local['zip']:
888 tempdir = tempfile.mkdtemp(prefix=kwargs['id']+'-')+self.separator
903 tempdir = tempfile.mkdtemp(prefix=kwargs['id']+'-')+self.separator
889 os.mkdir(tempdir+kwargs['id'])
904 os.mkdir(tempdir+kwargs['id'])
890 dir_name = tempdir + kwargs['id'] + self.separator
905 dir_name = tempdir + kwargs['id'] + self.separator
891 else:
906 else:
892 dir = self.f_name(kwargs['id'], '', dict_local['path'])
907 dir = self.f_name(kwargs['id'], '', dict_local['path'])
893 os.mkdir(dict_local['path'] + dir)
908 os.mkdir(dict_local['path'] + dir)
894 dir_name = dict_local['path'] + dir + self.separator
909 dir_name = dict_local['path'] + dir + self.separator
895 #-----------DOWNLOAD FILES-------------#
910 #-----------DOWNLOAD FILES-------------#
896 print('.....')
911 print('.....')
897 print('Downloading "{}" file(s) >>'.format(len(response)))
912 print('Downloading "{}" file(s) >>'.format(len(response)))
898 name_total = {'name': []}
913 name_total = {'name': []}
899 with concurrent.futures.ThreadPoolExecutor() as executor:
914 with concurrent.futures.ThreadPoolExecutor() as executor:
900 for u in tqdm(iterable=response, total=len(response)):
915 for u in tqdm(iterable=response, total=len(response)):
901 name_total['name'].append(u['name'])
916 name_total['name'].append(u['name'])
902 executor.submit(self.download_by_step, u, dir_name)
917 executor.submit(self.download_by_step, u, dir_name)
903 name_check = {}
918 name_check = {}
904 name_check['name'] = [f for f in os.listdir(dir_name) if os.path.isfile(os.path.join(dir_name, f))]
919 name_check['name'] = [f for f in os.listdir(dir_name) if os.path.isfile(os.path.join(dir_name, f))]
905 print('"{}" downloaded file(s) successfully >>'.format(len(name_check['name'])))
920 print('"{}" downloaded file(s) successfully >>'.format(len(name_check['name'])))
906 #--------------------------------------#
921 #--------------------------------------#
907 if len(name_check['name']) != 0:
922 if len(name_check['name']) != 0:
908 #----------Status Note---------#
923 #----------Status Note---------#
909 if dict_local['status_note']:
924 if dict_local['status_note']:
910 print('.....')
925 print('.....')
911 print('Creating: "status_note.txt" >>')
926 print('Creating: "status_note.txt" >>')
912 self.f_status_note(name_total, name_check, dir_name)
927 self.f_status_note(name_total, name_check, dir_name)
913 print('Created>>')
928 print('Created>>')
914 #----------ZIP CREATE----------#
929 #----------ZIP CREATE----------#
915 if dict_local['zip']:
930 if dict_local['zip']:
916 zip_name = self.f_name(kwargs['id'], '.zip', dict_local['path'])
931 zip_name = self.f_name(kwargs['id'], '.zip', dict_local['path'])
917 ziph = zipfile.ZipFile(dict_local['path'] + zip_name, 'w', zipfile.ZIP_DEFLATED, allowZip64=True)
932 ziph = zipfile.ZipFile(dict_local['path'] + zip_name, 'w', zipfile.ZIP_DEFLATED, allowZip64=True)
918 self.f_zipdir(dir_name, ziph, zip_name)
933 self.f_zipdir(dir_name, ziph, zip_name)
919 ziph.close()
934 ziph.close()
920 #Delete Temporal Path
935 #Delete Temporal Path
921 if os.path.exists(tempdir[:-1]):
936 if os.path.exists(tempdir[:-1]):
922 shutil.rmtree(tempdir[:-1])
937 shutil.rmtree(tempdir[:-1])
923 #------------------------------#
938 #------------------------------#
924 print('.....')
939 print('.....')
925 return 'DOWNLOAD FINISHED'
940 return 'DOWNLOAD FINISHED'
926 else:
941 else:
927 #Delete Temporal Path
942 #Delete Temporal Path
928 if dict_local['zip']:
943 if dict_local['zip']:
929 if os.path.exists(tempdir[:-1]):
944 if os.path.exists(tempdir[:-1]):
930 shutil.rmtree(tempdir[:-1])
945 shutil.rmtree(tempdir[:-1])
931 else:
946 else:
932 if os.path.exists(dir_name[:-1]):
947 if os.path.exists(dir_name[:-1]):
933 shutil.rmtree(dir_name[:-1])
948 shutil.rmtree(dir_name[:-1])
934 return 'NO FILES WERE DOWNLOADED'
949 return 'NO FILES WERE DOWNLOADED'
935 else:
950 else:
936 return 'FILES NOT FOUND'
951 return 'FILES NOT FOUND'
937
952
938 def download_files_advance(self, id_or_name, processes=1, path=os.path.expanduser("~"), **kwargs):
953 def download_files_advance(self, id_or_name, processes=1, path=os.path.expanduser("~"), **kwargs):
939 '''
954 '''
940 FINALIDAD:
955 FINALIDAD:
941 Funcion personalizada avanzada para la descarga de archivos existentes de un(os) dataset(s).
956 Funcion personalizada avanzada para la descarga de archivos existentes de un(os) dataset(s).
942
957
943 PARAMETROS DISPONIBLES:
958 PARAMETROS DISPONIBLES:
944 CONSULTAR: "GUIA DE SCRIPT.pdf"
959 CONSULTAR: "GUIA DE SCRIPT.pdf"
945
960
946 ESTRUCTURA:
961 ESTRUCTURA:
947 <access_name>.download_files_advance(id_or_name= <class 'str' or 'list'>, param_1 = <class 'param_1'>, ...)
962 <access_name>.download_files_advance(id_or_name= <class 'str' or 'list'>, param_1 = <class 'param_1'>, ...)
948 '''
963 '''
949 #------------------ PATH ----------------------#
964 #------------------ PATH ----------------------#
950 if isinstance(path, str):
965 if isinstance(path, str):
951 if os.path.isdir(path):
966 if os.path.isdir(path):
952 if not path.endswith(os.sep):
967 if not path.endswith(os.sep):
953 path = path + os.sep
968 path = path + os.sep
954 test_txt = path + datetime.now().strftime("%Y_%m_%d_%H_%M_%S_%f")+'.txt'
969 test_txt = path + datetime.now().strftime("%Y_%m_%d_%H_%M_%S_%f")+'.txt'
955 try:
970 try:
956 file_txt = open(test_txt, 'w')
971 file_txt = open(test_txt, 'w')
957 file_txt.close()
972 file_txt.close()
958 os.remove(test_txt)
973 os.remove(test_txt)
959 except:
974 except:
960 return 'ERROR:: Access denied, you are not authorized to write files: "%s"' % (path)
975 return 'ERROR:: Access denied, you are not authorized to write files: "%s"' % (path)
961 else:
976 else:
962 return 'ERROR:: "path" does not exist'
977 return 'ERROR:: "path" does not exist'
963 else:
978 else:
964 return 'ERROR:: "path" must be: <class "str">'
979 return 'ERROR:: "path" must be: <class "str">'
965
980
966 #------------------ PROCESSES -----------------#
981 #------------------ PROCESSES -----------------#
967 if not isinstance(processes, int):
982 if not isinstance(processes, int):
968 return 'ERROR:: "processes" must be: <class "int">'
983 return 'ERROR:: "processes" must be: <class "int">'
969
984
970 #------------------ ID OR NAME ----------------#
985 #------------------ ID OR NAME ----------------#
971 if isinstance(id_or_name, str):
986 if isinstance(id_or_name, str):
972 id_or_name = [id_or_name]
987 id_or_name = [id_or_name]
973 elif isinstance(id_or_name, list):
988 elif isinstance(id_or_name, list):
974 id_or_name = list(map(str, id_or_name))
989 id_or_name = list(map(str, id_or_name))
975 else:
990 else:
976 return 'ERROR:: dataset "id_or_name" must be: <class "str" or "list">'
991 return 'ERROR:: dataset "id_or_name" must be: <class "str" or "list">'
977 #----------------------------------------------#
992 #----------------------------------------------#
978 arguments = {
993 arguments = {
979 '--apikey': self.Authorization,
994 '--apikey': self.Authorization,
980 '--ckan-user': None,
995 '--ckan-user': None,
981 '--config': None,
996 '--config': None,
982 '--datapackages': path,
997 '--datapackages': path,
983 '--datastore-fields': False,
998 '--datastore-fields': False,
984 '--get-request': False,
999 '--get-request': False,
985 '--insecure': not self.verify,
1000 '--insecure': not self.verify,
986 '--log': '/home/soporte/DUMP/download.txt',
1001 '--log': '/home/soporte/DUMP/download.txt',
987 '--processes': str(processes),
1002 '--processes': str(processes),
988 '--quiet': False,
1003 '--quiet': False,
989 '--remote': self.url,
1004 '--remote': self.url,
990 '--worker': False,
1005 '--worker': False,
991 #'--all': False,
1006 #'--all': False,
992 #'--gzip': False,
1007 #'--gzip': False,
993 #'--output': None,
1008 #'--output': None,
994 #'--max-records': None,
1009 #'--max-records': None,
995 #'--output-json': False,
1010 #'--output-json': False,
996 #'--output-jsonl': False,
1011 #'--output-jsonl': False,
997 #'--create-only': False,
1012 #'--create-only': False,
998 #'--help': False,
1013 #'--help': False,
999 #'--input': None,
1014 #'--input': None,
1000 #'--input-json': False,
1015 #'--input-json': False,
1001 #'--start-record': '1',
1016 #'--start-record': '1',
1002 #'--update-only': False,
1017 #'--update-only': False,
1003 #'--upload-logo': False,
1018 #'--upload-logo': False,
1004 #'--upload-resources': False,
1019 #'--upload-resources': False,
1005 #'--version': False,
1020 #'--version': False,
1006 'ID_OR_NAME': id_or_name,
1021 'ID_OR_NAME': id_or_name,
1007 'datasets': True,
1022 'datasets': True,
1008 'dump': True,
1023 'dump': True,
1009 #'ACTION_NAME': None,
1024 #'ACTION_NAME': None,
1010 #'KEY:JSON': [],
1025 #'KEY:JSON': [],
1011 #'KEY=STRING': [],
1026 #'KEY=STRING': [],
1012 #'KEY@FILE': [],
1027 #'KEY@FILE': [],
1013 #'action': False,
1028 #'action': False,
1014 #'delete': False,
1029 #'delete': False,
1015 #'groups': False,
1030 #'groups': False,
1016 #'load': False,
1031 #'load': False,
1017 #'organizations': False,
1032 #'organizations': False,
1018 #'related': False,
1033 #'related': False,
1019 #'search': False,
1034 #'search': False,
1020 #'users': False
1035 #'users': False
1021 }
1036 }
1022 return logic_download.dump_things_change(self.ckan, 'datasets', arguments, **kwargs) No newline at end of file
1037 return logic_download.dump_things_change(self.ckan, 'datasets', arguments, **kwargs)
Show file before | Show file after | Hide comments
@@ -1,228 +1,226
1 #from ckanapi.datapackage import populate_schema_from_datastore
1 #from ckanapi.datapackage import populate_schema_from_datastore
2 from ckanapi.cli import workers, dump
2 from ckanapi.cli import workers, dump
3 from ckanapi.cli.utils import pretty_json, completion_stats, compact_json, quiet_int_pipe
3 from ckanapi.cli.utils import pretty_json, completion_stats, compact_json, quiet_int_pipe
4 from datetime import datetime
4 from datetime import datetime
5 import sys
5 import sys
6 import json
6 import json
7 import os
7 import os
8 import requests
8 import requests
9 import six
9 import six
10
10
11 if sys.version_info.major == 3:
11 if sys.version_info.major == 3:
12 from urllib.parse import urlparse
12 from urllib.parse import urlparse
13 else:
13 else:
14 import urlparse
14 import urlparse
15
15
16 DL_CHUNK_SIZE = 100 * 1024
16 DL_CHUNK_SIZE = 100 * 1024
17
17
18 print()
19
20 def dump_things_change(ckan, thing, arguments, worker_pool=None, stdout=None, stderr=None, **kwargs):
18 def dump_things_change(ckan, thing, arguments, worker_pool=None, stdout=None, stderr=None, **kwargs):
21 if worker_pool is None:
19 if worker_pool is None:
22 worker_pool = workers.worker_pool
20 worker_pool = workers.worker_pool
23 if stdout is None:
21 if stdout is None:
24 stdout = getattr(sys.__stdout__, 'buffer', sys.__stdout__)
22 stdout = getattr(sys.__stdout__, 'buffer', sys.__stdout__)
25 if stderr is None:
23 if stderr is None:
26 stderr = getattr(sys.stderr, 'buffer', sys.stderr)
24 stderr = getattr(sys.stderr, 'buffer', sys.stderr)
27
25
28 if arguments['--worker']:
26 if arguments['--worker']:
29 return dump.dump_things_worker(ckan, thing, arguments)
27 return dump.dump_things_worker(ckan, thing, arguments)
30 '''
28 '''
31 log = None
29 log = None
32 if arguments['--log']:
30 if arguments['--log']:
33 log = open(arguments['--log'], 'a')
31 log = open(arguments['--log'], 'a')
34 '''
32 '''
35 jsonl_output = stdout
33 jsonl_output = stdout
36 if arguments['--datapackages']:
34 if arguments['--datapackages']:
37 jsonl_output = open(os.devnull, 'wb')
35 jsonl_output = open(os.devnull, 'wb')
38
36
39 names = arguments['ID_OR_NAME']
37 names = arguments['ID_OR_NAME']
40
38
41 if names and isinstance(names[0], dict):
39 if names and isinstance(names[0], dict):
42 names = [rec.get('name',rec.get('id')) for rec in names]
40 names = [rec.get('name',rec.get('id')) for rec in names]
43 '''
41 '''
44 if arguments['--datapackages']:
42 if arguments['--datapackages']:
45 arguments['--datastore-fields'] = True
43 arguments['--datastore-fields'] = True
46 '''
44 '''
47 #----------------------------#
45 #----------------------------#
48 filtered_urls = {}
46 filtered_urls = {}
49 for name in names:
47 for name in names:
50 try:
48 try:
51 response = getattr(ckan.action, 'url_resources')(id=name, **kwargs)
49 response = getattr(ckan.action, 'url_resources')(id=name, **kwargs)
52 except:
50 except:
53 _, exc_value, _ = sys.exc_info()
51 _, exc_value, _ = sys.exc_info()
54 return exc_value
52 return exc_value
55 filtered_urls[name] = response
53 filtered_urls[name] = response
56 #----------------------------#
54 #----------------------------#
57
55
58 cmd = dump._worker_command_line(thing, arguments)
56 cmd = dump._worker_command_line(thing, arguments)
59 processes = int(arguments['--processes'])
57 processes = int(arguments['--processes'])
60 if hasattr(ckan, 'parallel_limit'):
58 if hasattr(ckan, 'parallel_limit'):
61 processes = min(processes, ckan.parallel_limit)
59 processes = min(processes, ckan.parallel_limit)
62 stats = completion_stats(processes)
60 stats = completion_stats(processes)
63 pool = worker_pool(cmd, processes, enumerate(compact_json(n) + b'\n' for n in names))
61 pool = worker_pool(cmd, processes, enumerate(compact_json(n) + b'\n' for n in names))
64
62
65 results = {}
63 results = {}
66 expecting_number = 0
64 expecting_number = 0
67 with quiet_int_pipe() as errors:
65 with quiet_int_pipe() as errors:
68 for job_ids, finished, result in pool:
66 for job_ids, finished, result in pool:
69 if not result:
67 if not result:
70 return 1
68 return 1
71 timestamp, error, record = json.loads(result.decode('utf-8'))
69 timestamp, error, record = json.loads(result.decode('utf-8'))
72 results[finished] = record
70 results[finished] = record
73
71
74 if not arguments['--quiet']:
72 if not arguments['--quiet']:
75 stderr.write('** Finished: {0} | Job IDs: {1} | Next Report: {2} | Error: {3} | Dataset Name: {4}\n'.format(
73 stderr.write('** Finished: {0} | Job IDs: {1} | Next Report: {2} | Error: {3} | Dataset Name: {4}\n'.format(
76 finished,
74 finished,
77 job_ids,
75 job_ids,
78 next(stats),
76 next(stats),
79 error,
77 error,
80 record.get('name', '') if record else '',
78 record.get('name', '') if record else '',
81 ).encode('utf-8'))
79 ).encode('utf-8'))
82 '''
80 '''
83 if log:
81 if log:
84 log.write(compact_json([
82 log.write(compact_json([
85 timestamp,
83 timestamp,
86 finished,
84 finished,
87 error,
85 error,
88 record.get('name', '') if record else None,
86 record.get('name', '') if record else None,
89 ]) + b'\n')
87 ]) + b'\n')
90 '''
88 '''
91 datapackages_path = arguments['--datapackages']
89 datapackages_path = arguments['--datapackages']
92 if datapackages_path:
90 if datapackages_path:
93 create_datapackage_change(record, filtered_urls[record.get('name', '')], datapackages_path, stderr, arguments['--apikey'], arguments['--remote'], arguments['--insecure'])
91 create_datapackage_change(record, filtered_urls[record.get('name', '')], datapackages_path, stderr, arguments['--apikey'], arguments['--remote'], arguments['--insecure'])
94 while expecting_number in results:
92 while expecting_number in results:
95 record = results.pop(expecting_number)
93 record = results.pop(expecting_number)
96 if record:
94 if record:
97 jsonl_output.write(compact_json(record, sort_keys=True) + b'\n')
95 jsonl_output.write(compact_json(record, sort_keys=True) + b'\n')
98 expecting_number += 1
96 expecting_number += 1
99 if 'pipe' in errors:
97 if 'pipe' in errors:
100 return 1
98 return 1
101 if 'interrupt' in errors:
99 if 'interrupt' in errors:
102 return 2
100 return 2
103
101
104 def create_datapackage_change(record, filtered_url, base_path, stderr, apikey, host_url, insecure):
102 def create_datapackage_change(record, filtered_url, base_path, stderr, apikey, host_url, insecure):
105 resource_formats_to_ignore = ['API', 'api']
103 resource_formats_to_ignore = ['API', 'api']
106 #----------------------------------------#
104 #----------------------------------------#
107 datapackage_dir = name_no_repetition(record.get('name', ''), base_path)
105 datapackage_dir = name_no_repetition(record.get('name', ''), base_path)
108 #----------------------------------------#
106 #----------------------------------------#
109 os.makedirs(os.path.join(datapackage_dir, 'data'))
107 os.makedirs(os.path.join(datapackage_dir, 'data'))
110 record['path'] = datapackage_dir
108 record['path'] = datapackage_dir
111
109
112 ckan_resources = []
110 ckan_resources = []
113 for resource in record.get('resources', []):
111 for resource in record.get('resources', []):
114 if resource['format'] in resource_formats_to_ignore:
112 if resource['format'] in resource_formats_to_ignore:
115 continue
113 continue
116
114
117 if not {'name': resource['name'], 'url': resource['url']} in filtered_url:
115 if not {'name': resource['name'], 'url': resource['url']} in filtered_url:
118 continue
116 continue
119
117
120 if len(resource['url']) == 0:
118 if len(resource['url']) == 0:
121 continue
119 continue
122
120
123 filename = name_no_repetition(resource['name'], os.path.join(datapackage_dir, 'data'), 'resource')
121 filename = name_no_repetition(resource['name'], os.path.join(datapackage_dir, 'data'), 'resource')
124 resource['path'] = os.path.join(datapackage_dir, 'data', filename)
122 resource['path'] = os.path.join(datapackage_dir, 'data', filename)
125
123
126 cres = create_resource_change(resource, stderr, apikey, host_url, insecure)
124 cres = create_resource_change(resource, stderr, apikey, host_url, insecure)
127 if not cres:
125 if not cres:
128 continue
126 continue
129 '''
127 '''
130 #----------------------------------------#
128 #----------------------------------------#
131 dres = {'path': os.path.join('data', filename),
129 dres = {'path': os.path.join('data', filename),
132 'description': cres.get('description', ''),
130 'description': cres.get('description', ''),
133 'format': cres.get('format', ''),
131 'format': cres.get('format', ''),
134 'name': cres.get('name', ''),
132 'name': cres.get('name', ''),
135 'title': cres.get('name', '').title()}
133 'title': cres.get('name', '').title()}
136 #----------------------------------------#
134 #----------------------------------------#
137 populate_schema_from_datastore(cres, dres)
135 populate_schema_from_datastore(cres, dres)
138 '''
136 '''
139 ckan_resources.append(resource)
137 ckan_resources.append(resource)
140
138
141 dataset = dict(record, resources=ckan_resources)
139 dataset = dict(record, resources=ckan_resources)
142 datapackage = dataset_to_datapackage_change(dataset)
140 datapackage = dataset_to_datapackage_change(dataset)
143
141
144 json_path = os.path.join(datapackage_dir, 'datapackage.json')
142 json_path = os.path.join(datapackage_dir, 'datapackage.json')
145 with open(json_path, 'wb') as out:
143 with open(json_path, 'wb') as out:
146 out.write(pretty_json(datapackage))
144 out.write(pretty_json(datapackage))
147
145
148 return datapackage_dir, datapackage, json_path
146 return datapackage_dir, datapackage, json_path
149
147
150 def create_resource_change(resource, stderr, apikey, host_url, insecure):
148 def create_resource_change(resource, stderr, apikey, host_url, insecure):
151 # ---------- REPLACE URL --------- #
149 # ---------- REPLACE URL --------- #
152 if urlparse(host_url).netloc != 'www.igp.gob.pe' and urlparse(resource['url']).netloc == 'www.igp.gob.pe':
150 if urlparse(host_url).netloc != 'www.igp.gob.pe' and urlparse(resource['url']).netloc == 'www.igp.gob.pe':
153 resource['url'] = resource['url'].replace(urlparse(resource['url']).scheme + '://' + urlparse(resource['url']).netloc,
151 resource['url'] = resource['url'].replace(urlparse(resource['url']).scheme + '://' + urlparse(resource['url']).netloc,
154 urlparse(host_url).scheme + '://' + urlparse(host_url).netloc)
152 urlparse(host_url).scheme + '://' + urlparse(host_url).netloc)
155 #----------------------------------#
153 #----------------------------------#
156 try:
154 try:
157 r = requests.get(resource['url'], headers={'Authorization': apikey}, stream=True, verify=not insecure)
155 r = requests.get(resource['url'], headers={'Authorization': apikey}, stream=True, verify=not insecure)
158 #---------------------------------------#
156 #---------------------------------------#
159 try:
157 try:
160 r.raise_for_status()
158 r.raise_for_status()
161 except requests.exceptions.HTTPError as e:
159 except requests.exceptions.HTTPError as e:
162 return False
160 return False
163 #---------------------------------------#
161 #---------------------------------------#
164 with open(resource['path'], 'wb') as f:
162 with open(resource['path'], 'wb') as f:
165 for chunk in r.iter_content(chunk_size=DL_CHUNK_SIZE):
163 for chunk in r.iter_content(chunk_size=DL_CHUNK_SIZE):
166 if chunk:
164 if chunk:
167 f.write(chunk)
165 f.write(chunk)
168
166
169 except requests.ConnectionError:
167 except requests.ConnectionError:
170 stderr.write('URL {0} refused connection. The resource will not be downloaded\n'.format(resource['url']).encode('utf-8'))
168 stderr.write('URL {0} refused connection. The resource will not be downloaded\n'.format(resource['url']).encode('utf-8'))
171 except requests.exceptions.RequestException as e:
169 except requests.exceptions.RequestException as e:
172 stderr.write('{0}\n'.format(str(e.args[0]) if len(e.args) > 0 else '').encode('utf-8'))
170 stderr.write('{0}\n'.format(str(e.args[0]) if len(e.args) > 0 else '').encode('utf-8'))
173 except Exception as e:
171 except Exception as e:
174 stderr.write('{0}'.format(str(e.args[0]) if len(e.args) > 0 else '').encode('utf-8'))
172 stderr.write('{0}'.format(str(e.args[0]) if len(e.args) > 0 else '').encode('utf-8'))
175 return resource
173 return resource
176
174
177 def dataset_to_datapackage_change(dataset_dict):
175 def dataset_to_datapackage_change(dataset_dict):
178 dp = {'name': dataset_dict['name'],
176 dp = {'name': dataset_dict['name'],
179 'id': dataset_dict['id'],
177 'id': dataset_dict['id'],
180 'path': dataset_dict['path'],
178 'path': dataset_dict['path'],
181 'last_update': datetime.strptime(dataset_dict['metadata_modified'], "%Y-%m-%dT%H:%M:%S.%f").strftime("%d-%b-%Y %I.%M %p")}
179 'last_update': datetime.strptime(dataset_dict['metadata_modified'], "%Y-%m-%dT%H:%M:%S.%f").strftime("%d-%b-%Y %I.%M %p")}
182
180
183 resources = dataset_dict.get('resources')
181 resources = dataset_dict.get('resources')
184 if resources:
182 if resources:
185 dp['resources'] = [convert_to_datapackage_resource_change(r)
183 dp['resources'] = [convert_to_datapackage_resource_change(r)
186 for r in resources]
184 for r in resources]
187 return dp
185 return dp
188
186
189 def convert_to_datapackage_resource_change(resource_dict):
187 def convert_to_datapackage_resource_change(resource_dict):
190 resource = {}
188 resource = {}
191
189
192 if resource_dict.get('id'):
190 if resource_dict.get('id'):
193 resource['id'] = resource_dict['id']
191 resource['id'] = resource_dict['id']
194
192
195 if resource_dict.get('name'):
193 if resource_dict.get('name'):
196 resource['name'] = resource_dict['name']
194 resource['name'] = resource_dict['name']
197
195
198 if resource_dict.get('path'):
196 if resource_dict.get('path'):
199 resource['path'] = resource_dict['path']
197 resource['path'] = resource_dict['path']
200
198
201 schema = resource_dict.get('schema')
199 schema = resource_dict.get('schema')
202 if isinstance(schema, six.string_types):
200 if isinstance(schema, six.string_types):
203 try:
201 try:
204 resource['schema'] = json.loads(schema)
202 resource['schema'] = json.loads(schema)
205 except ValueError:
203 except ValueError:
206 resource['schema'] = schema
204 resource['schema'] = schema
207 elif isinstance(schema, dict):
205 elif isinstance(schema, dict):
208 resource['schema'] = schema
206 resource['schema'] = schema
209
207
210 return resource
208 return resource
211
209
212 def name_no_repetition(name, dir, option=''):
210 def name_no_repetition(name, dir, option=''):
213 count = 0
211 count = 0
214 while True:
212 while True:
215 count = count + 1
213 count = count + 1
216 if not os.path.exists(os.path.join(dir, name)):
214 if not os.path.exists(os.path.join(dir, name)):
217 if option == 'resource':
215 if option == 'resource':
218 return name
216 return name
219 else:
217 else:
220 return os.path.join(dir, name)
218 return os.path.join(dir, name)
221
219
222 elif not os.path.exists(os.path.join(dir, '('+str(count)+')'+name)):
220 elif not os.path.exists(os.path.join(dir, '('+str(count)+')'+name)):
223 if option == 'resource':
221 if option == 'resource':
224 return '('+str(count)+')'+name
222 return '('+str(count)+')'+name
225 else:
223 else:
226 return os.path.join(dir, '('+str(count)+')'+name)
224 return os.path.join(dir, '('+str(count)+')'+name)
227 else:
225 else:
228 pass No newline at end of file
226 pass
General Comments 0
You need to be logged in to leave comments. Login now