tasks: normalize the url argument name of most lister
Since all the listing tasks accepts an url as first argument (whatever the argument name is), it makes sense to use a simple common argument name for this. I've chosen 'url' instead of api_baseurl/forge_url/url. Also kill now useless `new_lister()` functions.
This commit is contained in:
parent
631b8e7668
commit
b810876ef8
19 changed files with 60 additions and 91 deletions
|
@ -12,20 +12,16 @@ from swh.lister.github.lister import GitHubLister
|
|||
GROUP_SPLIT = 10000
|
||||
|
||||
|
||||
def new_lister(api_baseurl='https://api.github.com', **kw):
|
||||
return GitHubLister(api_baseurl=api_baseurl, **kw)
|
||||
|
||||
|
||||
@app.task(name=__name__ + '.IncrementalGitHubLister')
|
||||
def list_github_incremental(**lister_args):
|
||||
'Incremental update of GitHub'
|
||||
lister = new_lister(**lister_args)
|
||||
lister = GitHubLister(**lister_args)
|
||||
lister.run(min_bound=lister.db_last_index(), max_bound=None)
|
||||
|
||||
|
||||
@app.task(name=__name__ + '.RangeGitHubLister')
|
||||
def _range_github_lister(start, end, **lister_args):
|
||||
lister = new_lister(**lister_args)
|
||||
lister = GitHubLister(**lister_args)
|
||||
lister.run(min_bound=start, max_bound=end)
|
||||
|
||||
|
||||
|
@ -36,7 +32,7 @@ def list_github_full(self, split=None, **lister_args):
|
|||
It's not to be called for an initial listing.
|
||||
|
||||
"""
|
||||
lister = new_lister(**lister_args)
|
||||
lister = GitHubLister(**lister_args)
|
||||
ranges = lister.db_partition_indices(split or GROUP_SPLIT)
|
||||
if not ranges:
|
||||
self.log.info('Nothing to list')
|
||||
|
|
|
@ -26,7 +26,7 @@ def test_incremental(lister, swh_app, celery_session_worker):
|
|||
res.wait()
|
||||
assert res.successful()
|
||||
|
||||
lister.assert_called_once_with(api_baseurl='https://api.github.com')
|
||||
lister.assert_called_once_with()
|
||||
lister.db_last_index.assert_called_once_with()
|
||||
lister.run.assert_called_once_with(min_bound=42, max_bound=None)
|
||||
|
||||
|
@ -44,7 +44,7 @@ def test_range(lister, swh_app, celery_session_worker):
|
|||
res.wait()
|
||||
assert res.successful()
|
||||
|
||||
lister.assert_called_once_with(api_baseurl='https://api.github.com')
|
||||
lister.assert_called_once_with()
|
||||
lister.db_last_index.assert_not_called()
|
||||
lister.run.assert_called_once_with(min_bound=12, max_bound=42)
|
||||
|
||||
|
@ -74,7 +74,7 @@ def test_relister(lister, swh_app, celery_session_worker):
|
|||
break
|
||||
sleep(1)
|
||||
|
||||
lister.assert_called_with(api_baseurl='https://api.github.com')
|
||||
lister.assert_called_with()
|
||||
|
||||
# one by the FullGitHubRelister task
|
||||
# + 5 for the RangeGitHubLister subtasks
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue