|
|
@@
-44,13
+44,12
def dump_things_change(ckan, thing, arguments, worker_pool=None, stdout=None, st
|
|
|
44
|
'''
|
|
44
|
'''
|
|
|
45
|
#----------------------------#
|
|
45
|
#----------------------------#
|
|
|
46
|
filtered_urls = {}
|
|
46
|
filtered_urls = {}
|
|
|
47
|
for name in names:
|
|
47
|
for val in names:
|
|
|
48
|
try:
|
|
48
|
try:
|
|
|
49
|
response = getattr(ckan.action, 'url_resources')(id=name, **kwargs)
|
|
49
|
filtered_urls[val] = getattr(ckan.action, 'url_resources')(id=val, **kwargs)
|
|
|
50
|
except:
|
|
50
|
except:
|
|
|
51
|
_, exc_value, _ = sys.exc_info()
|
|
51
|
_, exc_value, _ = sys.exc_info()
|
|
|
52
|
return exc_value
|
|
52
|
return exc_value
|
|
|
53
|
filtered_urls[name] = response
|
|
|
|
|
|
54
|
#----------------------------#
|
|
53
|
#----------------------------#
|
|
|
55
|
|
|
54
|
|
|
|
56
|
cmd = dump._worker_command_line(thing, arguments)
|
|
55
|
cmd = dump._worker_command_line(thing, arguments)
|
|
|
@@
-69,12
+68,17
def dump_things_change(ckan, thing, arguments, worker_pool=None, stdout=None, st
|
|
|
69
|
timestamp, error, record = json.loads(result.decode('utf-8'))
|
|
68
|
timestamp, error, record = json.loads(result.decode('utf-8'))
|
|
|
70
|
results[finished] = record
|
|
69
|
results[finished] = record
|
|
|
71
|
|
|
70
|
|
|
|
|
|
|
71
|
#----------------------------------------#
|
|
|
|
|
|
72
|
datapackages_path = arguments['--datapackages']
|
|
|
|
|
|
73
|
datapackage_dir = name_no_repetition(record.get('name', ''), datapackages_path)
|
|
|
|
|
|
74
|
#----------------------------------------#
|
|
|
72
|
if not arguments['--quiet']:
|
|
75
|
if not arguments['--quiet']:
|
|
|
73
|
stderr.write('** Finished: {0} | Job IDs: {1} | Next Report: {2} | Error: {3} | Dataset Name: {4}\n'.format(
|
|
76
|
stderr.write('** Finished: {0} | Job IDs: {1} | Next Report: {2} | Error: {3} | Path: {4} | Dataset Name: {5}\n'.format(
|
|
|
74
|
finished,
|
|
77
|
finished,
|
|
|
75
|
job_ids,
|
|
78
|
job_ids,
|
|
|
76
|
next(stats),
|
|
79
|
next(stats),
|
|
|
77
|
error,
|
|
80
|
error,
|
|
|
|
|
|
81
|
datapackage_dir,
|
|
|
78
|
record.get('name', '') if record else '',
|
|
82
|
record.get('name', '') if record else '',
|
|
|
79
|
).encode('utf-8'))
|
|
83
|
).encode('utf-8'))
|
|
|
80
|
'''
|
|
84
|
'''
|
|
|
@@
-86,9
+90,13
def dump_things_change(ckan, thing, arguments, worker_pool=None, stdout=None, st
|
|
|
86
|
record.get('name', '') if record else None,
|
|
90
|
record.get('name', '') if record else None,
|
|
|
87
|
]) + b'\n')
|
|
91
|
]) + b'\n')
|
|
|
88
|
'''
|
|
92
|
'''
|
|
|
89
|
datapackages_path = arguments['--datapackages']
|
|
|
|
|
|
90
|
if datapackages_path:
|
|
93
|
if datapackages_path:
|
|
|
91
|
create_datapackage_change(record, filtered_urls[record.get('name', '')], datapackages_path, stderr, arguments['--apikey'], arguments['--remote'], arguments['--insecure'])
|
|
94
|
try:
|
|
|
|
|
|
95
|
filter_url = filtered_urls[record.get('name', '')]
|
|
|
|
|
|
96
|
except:
|
|
|
|
|
|
97
|
filter_url = filtered_urls[record.get('id', '')]
|
|
|
|
|
|
98
|
create_datapackage_change(record, filter_url, datapackage_dir, stderr, arguments['--apikey'], arguments['--remote'], arguments['--insecure'])
|
|
|
|
|
|
99
|
|
|
|
92
|
while expecting_number in results:
|
|
100
|
while expecting_number in results:
|
|
|
93
|
record = results.pop(expecting_number)
|
|
101
|
record = results.pop(expecting_number)
|
|
|
94
|
if record:
|
|
102
|
if record:
|
|
|
@@
-99,11
+107,9
def dump_things_change(ckan, thing, arguments, worker_pool=None, stdout=None, st
|
|
|
99
|
if 'interrupt' in errors:
|
|
107
|
if 'interrupt' in errors:
|
|
|
100
|
return 2
|
|
108
|
return 2
|
|
|
101
|
|
|
109
|
|
|
|
102
|
def create_datapackage_change(record, filtered_url, base_path, stderr, apikey, host_url, insecure):
|
|
110
|
def create_datapackage_change(record, filtered_url, datapackage_dir, stderr, apikey, host_url, insecure):
|
|
|
103
|
resource_formats_to_ignore = ['API', 'api']
|
|
111
|
resource_formats_to_ignore = ['API', 'api']
|
|
|
104
|
#----------------------------------------#
|
|
112
|
|
|
|
105
|
datapackage_dir = name_no_repetition(record.get('name', ''), base_path)
|
|
|
|
|
|
106
|
#----------------------------------------#
|
|
|
|
|
|
107
|
os.makedirs(os.path.join(datapackage_dir, 'data'))
|
|
113
|
os.makedirs(os.path.join(datapackage_dir, 'data'))
|
|
|
108
|
record['path'] = datapackage_dir
|
|
114
|
record['path'] = datapackage_dir
|
|
|
109
|
|
|
115
|
|
|
|
@@
-194,7
+200,10
def convert_to_datapackage_resource_change(resource_dict):
|
|
|
194
|
resource['name'] = resource_dict['name']
|
|
200
|
resource['name'] = resource_dict['name']
|
|
|
195
|
|
|
201
|
|
|
|
196
|
if resource_dict.get('path'):
|
|
202
|
if resource_dict.get('path'):
|
|
|
197
|
resource['path'] = resource_dict['path']
|
|
203
|
if os.path.isfile(resource_dict['path']):
|
|
|
|
|
|
204
|
resource['path'] = resource_dict['path']
|
|
|
|
|
|
205
|
else:
|
|
|
|
|
|
206
|
resource['url'] = resource_dict['url']
|
|
|
198
|
|
|
207
|
|
|
|
199
|
schema = resource_dict.get('schema')
|
|
208
|
schema = resource_dict.get('schema')
|
|
|
200
|
if isinstance(schema, six.string_types):
|
|
209
|
if isinstance(schema, six.string_types):
|
|
|
@@
-204,7
+213,6
def convert_to_datapackage_resource_change(resource_dict):
|
|
|
204
|
resource['schema'] = schema
|
|
213
|
resource['schema'] = schema
|
|
|
205
|
elif isinstance(schema, dict):
|
|
214
|
elif isinstance(schema, dict):
|
|
|
206
|
resource['schema'] = schema
|
|
215
|
resource['schema'] = schema
|
|
|
207
|
|
|
|
|
|
|
208
|
return resource
|
|
216
|
return resource
|
|
|
209
|
|
|
217
|
|
|
|
210
|
def name_no_repetition(name, dir, option=''):
|
|
218
|
def name_no_repetition(name, dir, option=''):
|
