tasks: normalize the url argument name of most lister

Since all the listing tasks accepts an url as first argument (whatever the
argument name is), it makes sense to use a simple common argument name for
this. I've chosen 'url' instead of api_baseurl/forge_url/url.

Also kill now useless `new_lister()` functions.
This commit is contained in:
David Douard 2019-09-04 11:19:16 +02:00
parent 631b8e7668
commit b810876ef8
19 changed files with 60 additions and 91 deletions

View file

@ -16,10 +16,9 @@ class NpmListerBase(IndexingHttpLister):
LISTER_NAME = 'npm'
instance = 'npm'
def __init__(self, api_baseurl='https://replicate.npmjs.com',
def __init__(self, url='https://replicate.npmjs.com',
per_page=1000, override_config=None):
super().__init__(api_baseurl=api_baseurl,
override_config=override_config)
super().__init__(url=url, override_config=override_config)
self.per_page = per_page + 1
self.PATH_TEMPLATE += '&limit=%s' % self.per_page
@ -76,7 +75,7 @@ class NpmListerBase(IndexingHttpLister):
'https://www.npmjs.com/package/%s' % repo_name,
# package metadata url needs to be escaped otherwise some requests
# may fail (for instance when a package name contains '/')
'%s/%s' % (self.api_baseurl, quote(repo_name, safe=''))
'%s/%s' % (self.url, quote(repo_name, safe=''))
)
def string_pattern_check(self, inner, lower, upper=None):

View file

@ -14,7 +14,7 @@ from swh.lister.npm.models import NpmVisitModel
@contextmanager
def save_registry_state(lister):
params = {'headers': lister.request_headers()}
registry_state = lister.session.get(lister.api_baseurl, **params)
registry_state = lister.session.get(lister.url, **params)
registry_state = registry_state.json()
keys = ('doc_count', 'doc_del_count', 'update_seq', 'purge_seq',
'disk_size', 'data_size', 'committed_update_seq',