tasks: normalize the url argument name of most lister

Since all the listing tasks accepts an url as first argument (whatever the
argument name is), it makes sense to use a simple common argument name for
this. I've chosen 'url' instead of api_baseurl/forge_url/url.

Also kill now useless `new_lister()` functions.
This commit is contained in:
David Douard 2019-09-04 11:19:16 +02:00
parent 631b8e7668
commit b810876ef8
19 changed files with 60 additions and 91 deletions

View file

@ -12,20 +12,16 @@ from swh.lister.github.lister import GitHubLister
GROUP_SPLIT = 10000
def new_lister(api_baseurl='https://api.github.com', **kw):
return GitHubLister(api_baseurl=api_baseurl, **kw)
@app.task(name=__name__ + '.IncrementalGitHubLister')
def list_github_incremental(**lister_args):
'Incremental update of GitHub'
lister = new_lister(**lister_args)
lister = GitHubLister(**lister_args)
lister.run(min_bound=lister.db_last_index(), max_bound=None)
@app.task(name=__name__ + '.RangeGitHubLister')
def _range_github_lister(start, end, **lister_args):
lister = new_lister(**lister_args)
lister = GitHubLister(**lister_args)
lister.run(min_bound=start, max_bound=end)
@ -36,7 +32,7 @@ def list_github_full(self, split=None, **lister_args):
It's not to be called for an initial listing.
"""
lister = new_lister(**lister_args)
lister = GitHubLister(**lister_args)
ranges = lister.db_partition_indices(split or GROUP_SPLIT)
if not ranges:
self.log.info('Nothing to list')

View file

@ -26,7 +26,7 @@ def test_incremental(lister, swh_app, celery_session_worker):
res.wait()
assert res.successful()
lister.assert_called_once_with(api_baseurl='https://api.github.com')
lister.assert_called_once_with()
lister.db_last_index.assert_called_once_with()
lister.run.assert_called_once_with(min_bound=42, max_bound=None)
@ -44,7 +44,7 @@ def test_range(lister, swh_app, celery_session_worker):
res.wait()
assert res.successful()
lister.assert_called_once_with(api_baseurl='https://api.github.com')
lister.assert_called_once_with()
lister.db_last_index.assert_not_called()
lister.run.assert_called_once_with(min_bound=12, max_bound=42)
@ -74,7 +74,7 @@ def test_relister(lister, swh_app, celery_session_worker):
break
sleep(1)
lister.assert_called_with(api_baseurl='https://api.github.com')
lister.assert_called_with()
# one by the FullGitHubRelister task
# + 5 for the RangeGitHubLister subtasks