|
|
@@
-16,7
+16,7
else:
|
|
|
16
|
|
|
16
|
|
|
|
17
|
DL_CHUNK_SIZE = 100 * 1024
|
|
17
|
DL_CHUNK_SIZE = 100 * 1024
|
|
|
18
|
|
|
18
|
|
|
|
19
|
def dump_things_change(ckan, thing, arguments, worker_pool=None, stdout=None, stderr=None, **kwargs):
|
|
19
|
def dump_things_change(ckan, thing, arguments, ua, worker_pool=None, stdout=None, stderr=None, **kwargs):
|
|
|
20
|
if worker_pool is None:
|
|
20
|
if worker_pool is None:
|
|
|
21
|
worker_pool = workers.worker_pool
|
|
21
|
worker_pool = workers.worker_pool
|
|
|
22
|
if stdout is None:
|
|
22
|
if stdout is None:
|
|
|
@@
-96,7
+96,7
def dump_things_change(ckan, thing, arguments, worker_pool=None, stdout=None, st
|
|
|
96
|
filter_url = filtered_urls[record.get('name', '')]
|
|
96
|
filter_url = filtered_urls[record.get('name', '')]
|
|
|
97
|
except:
|
|
97
|
except:
|
|
|
98
|
filter_url = filtered_urls[record.get('id', '')]
|
|
98
|
filter_url = filtered_urls[record.get('id', '')]
|
|
|
99
|
create_datapackage_change(record, filter_url, datapackage_dir, stderr, arguments['--apikey'], arguments['--remote'], arguments['--insecure'])
|
|
99
|
create_datapackage_change(record, filter_url, datapackage_dir, stderr, arguments['--apikey'], arguments['--remote'], arguments['--insecure'], ua)
|
|
|
100
|
|
|
100
|
|
|
|
101
|
while expecting_number in results:
|
|
101
|
while expecting_number in results:
|
|
|
102
|
record = results.pop(expecting_number)
|
|
102
|
record = results.pop(expecting_number)
|
|
|
@@
-108,7
+108,7
def dump_things_change(ckan, thing, arguments, worker_pool=None, stdout=None, st
|
|
|
108
|
if 'interrupt' in errors:
|
|
108
|
if 'interrupt' in errors:
|
|
|
109
|
return 2
|
|
109
|
return 2
|
|
|
110
|
|
|
110
|
|
|
|
111
|
def create_datapackage_change(record, filtered_url, datapackage_dir, stderr, apikey, host_url, insecure):
|
|
111
|
def create_datapackage_change(record, filtered_url, datapackage_dir, stderr, apikey, host_url, insecure, ua):
|
|
|
112
|
resource_formats_to_ignore = ['API', 'api']
|
|
112
|
resource_formats_to_ignore = ['API', 'api']
|
|
|
113
|
|
|
113
|
|
|
|
114
|
os.makedirs(os.path.join(datapackage_dir, 'data'))
|
|
114
|
os.makedirs(os.path.join(datapackage_dir, 'data'))
|
|
|
@@
-129,7
+129,7
def create_datapackage_change(record, filtered_url, datapackage_dir, stderr, api
|
|
|
129
|
filename = name_no_repetition(resource['name'], os.path.join(datapackage_dir, 'data'), 'resource')
|
|
129
|
filename = name_no_repetition(resource['name'], os.path.join(datapackage_dir, 'data'), 'resource')
|
|
|
130
|
resource['path'] = os.path.join(datapackage_dir, 'data', filename)
|
|
130
|
resource['path'] = os.path.join(datapackage_dir, 'data', filename)
|
|
|
131
|
|
|
131
|
|
|
|
132
|
cres = create_resource_change(resource, stderr, apikey, host_url, insecure)
|
|
132
|
cres = create_resource_change(resource, stderr, apikey, host_url, insecure, ua)
|
|
|
133
|
if not cres:
|
|
133
|
if not cres:
|
|
|
134
|
continue
|
|
134
|
continue
|
|
|
135
|
'''
|
|
135
|
'''
|
|
|
@@
-153,14
+153,14
def create_datapackage_change(record, filtered_url, datapackage_dir, stderr, api
|
|
|
153
|
|
|
153
|
|
|
|
154
|
return datapackage_dir, datapackage, json_path
|
|
154
|
return datapackage_dir, datapackage, json_path
|
|
|
155
|
|
|
155
|
|
|
|
156
|
def create_resource_change(resource, stderr, apikey, host_url, insecure):
|
|
156
|
def create_resource_change(resource, stderr, apikey, host_url, insecure, ua):
|
|
|
157
|
# ---------- REPLACE URL --------- #
|
|
157
|
# ---------- REPLACE URL --------- #
|
|
|
158
|
if urlparse(host_url).netloc != 'www.igp.gob.pe' and urlparse(resource['url']).netloc == 'www.igp.gob.pe':
|
|
158
|
if urlparse(host_url).netloc != 'www.igp.gob.pe' and urlparse(resource['url']).netloc == 'www.igp.gob.pe':
|
|
|
159
|
resource['url'] = resource['url'].replace(urlparse(resource['url']).scheme + '://' + urlparse(resource['url']).netloc,
|
|
159
|
resource['url'] = resource['url'].replace(urlparse(resource['url']).scheme + '://' + urlparse(resource['url']).netloc,
|
|
|
160
|
urlparse(host_url).scheme + '://' + urlparse(host_url).netloc)
|
|
160
|
urlparse(host_url).scheme + '://' + urlparse(host_url).netloc)
|
|
|
161
|
#----------------------------------#
|
|
161
|
#----------------------------------#
|
|
|
162
|
try:
|
|
162
|
try:
|
|
|
163
|
r = requests.get(resource['url'], headers={'Authorization': apikey}, stream=True, verify=not insecure)
|
|
163
|
r = requests.get(resource['url'], headers={'Authorization': apikey, 'User-Agent': ua}, stream=True, verify=not insecure)
|
|
|
164
|
#---------------------------------------#
|
|
164
|
#---------------------------------------#
|
|
|
165
|
try:
|
|
165
|
try:
|
|
|
166
|
r.raise_for_status()
|
|
166
|
r.raise_for_status()
|
