diff --git a/jrodb/api.py b/jrodb/api.py index 647f468..4546872 100755 --- a/jrodb/api.py +++ b/jrodb/api.py @@ -56,9 +56,10 @@ class Api(): session = None #------------------------------# self.url = url - ua = 'CKAN_JRO/2.9.2 (+'+str(self.url)+')' + #ua = 'CKAN_JRO/2.9.2 (+'+str(self.url)+')' + self.ua = 'CKAN_JRO/2.9.2 (+http://intranet.igp.gob.pe:8082/DATABASES/ckanext-jro/api-cliente)' #ua = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36' - self.ckan = RemoteCKAN(self.url, apikey=Authorization, user_agent=ua, session=session) + self.ckan = RemoteCKAN(self.url, apikey=Authorization, user_agent=self.ua, session=session) #self.ckan = RemoteCKAN(self.url, apikey=Authorization) self.Authorization = Authorization # Change for --> self.separator = os.sep @@ -511,4 +512,4 @@ class Api(): #'search': False, #'users': False } - return download.dump_things_change(self.ckan, 'datasets', arguments, **kwargs) \ No newline at end of file + return download.dump_things_change(self.ckan, 'datasets', arguments, self.ua, **kwargs) \ No newline at end of file diff --git a/jrodb/download.py b/jrodb/download.py index a6859be..feb152f 100644 --- a/jrodb/download.py +++ b/jrodb/download.py @@ -16,7 +16,7 @@ else: DL_CHUNK_SIZE = 100 * 1024 -def dump_things_change(ckan, thing, arguments, worker_pool=None, stdout=None, stderr=None, **kwargs): +def dump_things_change(ckan, thing, arguments, ua, worker_pool=None, stdout=None, stderr=None, **kwargs): if worker_pool is None: worker_pool = workers.worker_pool if stdout is None: @@ -96,7 +96,7 @@ def dump_things_change(ckan, thing, arguments, worker_pool=None, stdout=None, st filter_url = filtered_urls[record.get('name', '')] except: filter_url = filtered_urls[record.get('id', '')] - create_datapackage_change(record, filter_url, datapackage_dir, stderr, arguments['--apikey'], arguments['--remote'], arguments['--insecure']) + create_datapackage_change(record, filter_url, datapackage_dir, stderr, arguments['--apikey'], arguments['--remote'], arguments['--insecure'], ua) while expecting_number in results: record = results.pop(expecting_number) @@ -108,7 +108,7 @@ def dump_things_change(ckan, thing, arguments, worker_pool=None, stdout=None, st if 'interrupt' in errors: return 2 -def create_datapackage_change(record, filtered_url, datapackage_dir, stderr, apikey, host_url, insecure): +def create_datapackage_change(record, filtered_url, datapackage_dir, stderr, apikey, host_url, insecure, ua): resource_formats_to_ignore = ['API', 'api'] os.makedirs(os.path.join(datapackage_dir, 'data')) @@ -129,7 +129,7 @@ def create_datapackage_change(record, filtered_url, datapackage_dir, stderr, api filename = name_no_repetition(resource['name'], os.path.join(datapackage_dir, 'data'), 'resource') resource['path'] = os.path.join(datapackage_dir, 'data', filename) - cres = create_resource_change(resource, stderr, apikey, host_url, insecure) + cres = create_resource_change(resource, stderr, apikey, host_url, insecure, ua) if not cres: continue ''' @@ -153,14 +153,14 @@ def create_datapackage_change(record, filtered_url, datapackage_dir, stderr, api return datapackage_dir, datapackage, json_path -def create_resource_change(resource, stderr, apikey, host_url, insecure): +def create_resource_change(resource, stderr, apikey, host_url, insecure, ua): # ---------- REPLACE URL --------- # if urlparse(host_url).netloc != 'www.igp.gob.pe' and urlparse(resource['url']).netloc == 'www.igp.gob.pe': resource['url'] = resource['url'].replace(urlparse(resource['url']).scheme + '://' + urlparse(resource['url']).netloc, urlparse(host_url).scheme + '://' + urlparse(host_url).netloc) #----------------------------------# try: - r = requests.get(resource['url'], headers={'Authorization': apikey}, stream=True, verify=not insecure) + r = requests.get(resource['url'], headers={'Authorization': apikey, 'User-Agent': ua}, stream=True, verify=not insecure) #---------------------------------------# try: r.raise_for_status() diff --git a/setup.py b/setup.py index 150527d..6a71a0b 100755 --- a/setup.py +++ b/setup.py @@ -3,7 +3,7 @@ from setuptools import setup setup( name = "jrodb", - version = "2.9.2.0", + version = "2.9.2.1", description = "Data Repository - JRO", author = "Edson Ynilupu Mattos", author_email = "eynilupu@igp.gob.pe",