DATABASES/ckanext-jro/api-cliente Commit - r23:7969fa062c2f

v2.9.2 :: Update 'User-Agent' in URL parameters - download

eynilupu -

r23:7969fa062c2f master

parent child

Context file:

r23:7969fa062c2f

jrodb/api.py +4 -3

                          session = None
                      #------------------------------#
                      self.url = url
-                     ua = 'CKAN_JRO/2.9.2 (+'+str(self.url)+')'
+                     #ua = 'CKAN_JRO/2.9.2 (+'+str(self.url)+')'
+                     self.ua = 'CKAN_JRO/2.9.2 (+http://intranet.igp.gob.pe:8082/DATABASES/ckanext-jro/api-cliente)'
                      #ua = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36'
-                     self.ckan = RemoteCKAN(self.url, apikey=Authorization, user_agent=ua, session=session)
+                     self.ckan = RemoteCKAN(self.url, apikey=Authorization, user_agent=self.ua, session=session)
                      #self.ckan = RemoteCKAN(self.url, apikey=Authorization)
                      self.Authorization = Authorization
                      # Change for --> self.separator = os.sep
                              #'search': False,
                              #'users': False
                              }
-                     return download.dump_things_change(self.ckan, 'datasets', arguments, **kwargs)
  No newline at end of file
+                     return download.dump_things_change(self.ckan, 'datasets', arguments, self.ua, **kwargs)
  No newline at end of file

jrodb/download.py +6 -6

              DL_CHUNK_SIZE = 100 * 1024
-             def dump_things_change(ckan, thing, arguments, worker_pool=None, stdout=None, stderr=None, **kwargs):
+             def dump_things_change(ckan, thing, arguments, ua, worker_pool=None, stdout=None, stderr=None, **kwargs):
                  if worker_pool is None:
                      worker_pool = workers.worker_pool
                  if stdout is None:
                                  filter_url = filtered_urls[record.get('name', '')]
                              except:
                                  filter_url = filtered_urls[record.get('id', '')]
-                             create_datapackage_change(record, filter_url, datapackage_dir, stderr, arguments['--apikey'], arguments['--remote'], arguments['--insecure'])
+                             create_datapackage_change(record, filter_url, datapackage_dir, stderr, arguments['--apikey'], arguments['--remote'], arguments['--insecure'], ua)
                          while expecting_number in results:
                              record = results.pop(expecting_number)
                  if 'interrupt' in errors:
                      return 2
-             def create_datapackage_change(record, filtered_url, datapackage_dir, stderr, apikey, host_url, insecure):
+             def create_datapackage_change(record, filtered_url, datapackage_dir, stderr, apikey, host_url, insecure, ua):
                  resource_formats_to_ignore = ['API', 'api']
                  os.makedirs(os.path.join(datapackage_dir, 'data'))
                      filename = name_no_repetition(resource['name'], os.path.join(datapackage_dir, 'data'), 'resource')
                      resource['path'] = os.path.join(datapackage_dir, 'data', filename)
-                     cres = create_resource_change(resource, stderr, apikey, host_url, insecure)
+                     cres = create_resource_change(resource, stderr, apikey, host_url, insecure, ua)
                      if not cres:
                          continue
                      '''
                  return datapackage_dir, datapackage, json_path
-             def create_resource_change(resource, stderr, apikey, host_url, insecure):
+             def create_resource_change(resource, stderr, apikey, host_url, insecure, ua):
                  # ---------- REPLACE URL --------- #
                  if urlparse(host_url).netloc != 'www.igp.gob.pe' and urlparse(resource['url']).netloc == 'www.igp.gob.pe':
                      resource['url'] = resource['url'].replace(urlparse(resource['url']).scheme + '://' + urlparse(resource['url']).netloc,
                                                              urlparse(host_url).scheme + '://' + urlparse(host_url).netloc)
                  #----------------------------------#
                  try:
-                     r = requests.get(resource['url'], headers={'Authorization': apikey}, stream=True, verify=not insecure)
+                     r = requests.get(resource['url'], headers={'Authorization': apikey, 'User-Agent': ua}, stream=True, verify=not insecure)
                      #---------------------------------------#
                      try:
                          r.raise_for_status()

setup.py +1 -1

              setup(
                  name = "jrodb",
-                 version = "2.9.2.0",
+                 version = "2.9.2.1",
                  description = "Data Repository - JRO",
                  author = "Edson Ynilupu Mattos",
                  author_email = "eynilupu@igp.gob.pe",

General Comments 0

Write
Preview

You need to be logged in to leave comments. Login now

No TODOs yet

	Site-wide shortcuts
/	Use quick search box
g h	Goto home page
g g	Goto my private gists page
g G	Goto my public gists page
g 0-9	Goto bookmarked items from 0-9
n r	New repository page
n g	New gist page

	Repositories
g s	Goto summary page
g c	Goto changelog page
g f	Goto files page
g F	Goto files page with file search activated
g p	Goto pull requests page
g o	Goto repository settings
g O	Goto repository access permissions settings
t s	Toggle sidebar on some pages