reonokiy/swh-lister
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

swh.lister: Make LISTER_NAME a class attribute

Browse source 
swh.lister.gitlab: make the 'instance' a constructor parameter
This commit is contained in:
Antoine R. Dumont (@ardumont) 2018-07-11 15:10:36 +02:00
parent 581028cfc5
commit 4c4aa0ead2
No known key found for this signature in database
GPG key ID: 52E2E9840D10C3B8
12 changed files with 36 additions and 52 deletions
Download patch file Download diff file Expand all files Collapse all files

1
swh/lister/bitbucket/lister.py
View file

@ -11,6 +11,7 @@ from swh.lister.core.indexing_lister import SWHIndexingHttpLister
class BitBucketLister(SWHIndexingHttpLister):
PATH_TEMPLATE = '/repositories?after=%s'
MODEL = BitBucketModel
LISTER_NAME = 'bitbucket.com'
def get_model_from_repo(self, repo):
return {

3
swh/lister/bitbucket/tasks.py
View file

@ -11,8 +11,7 @@ from .lister import BitBucketLister
class BitBucketListerTask(ListerTaskBase):
def new_lister(self):
return BitBucketLister(lister_name='bitbucket.com',
api_baseurl='https://api.bitbucket.org/2.0')
return BitBucketLister(api_baseurl='https://api.bitbucket.org/2.0')
class IncrementalBitBucketLister(BitBucketListerTask,

9
swh/lister/cli.py
View file

@ -29,21 +29,18 @@ def cli(db_url, lister, create_tables, drop_tables):
from .github import models
from .github.lister import GitHubLister
_lister = GitHubLister(lister_name='github.com',
api_baseurl='https://api.github.com',
_lister = GitHubLister(api_baseurl='https://api.github.com',
override_config=override_conf)
elif lister == 'bitbucket':
from .bitbucket import models
from .bitbucket.lister import BitBucketLister
_lister = BitBucketLister(lister_name='bitbucket.com',
api_baseurl='https://api.bitbucket.org/2.0',
_lister = BitBucketLister(api_baseurl='https://api.bitbucket.org/2.0',
override_config=override_conf)
elif lister == 'gitlab':
from .gitlab import models
from .gitlab.lister import GitLabLister
_lister = GitLabLister(lister_name='gitlab.com',
api_baseurl='https://gitlab.com/api/v4/',
_lister = GitLabLister(api_baseurl='https://gitlab.com/api/v4/',
override_config=override_conf)
else:
raise ValueError('Only supported listers are %s' % supported_listers)

6
swh/lister/core/indexing_lister.py
View file

@ -205,8 +205,6 @@ class SWHIndexingLister(SWHListerBase):
class SWHIndexingHttpLister(SWHListerHttpTransport, SWHIndexingLister):
"""Convenience class for ensuring right lookup and init order
when combining SWHIndexingLister and SWHListerHttpTransport."""
def __init__(self, lister_name=None, api_baseurl=None,
override_config=None):
def __init__(self, api_baseurl=None, override_config=None):
SWHListerHttpTransport.__init__(self, api_baseurl=api_baseurl)
SWHIndexingLister.__init__(self, lister_name=lister_name,
override_config=override_config)
SWHIndexingLister.__init__(self, override_config=override_config)

12
swh/lister/core/lister_base.py
View file

@ -64,6 +64,7 @@ class SWHListerBase(abc.ABC, config.SWHConfig):
MODEL = AbstractAttribute('Subclass type (not instance)'
' of swh.lister.core.models.ModelBase'
' customized for a specific service.')
LISTER_NAME = AbstractAttribute("Lister's name")
@abc.abstractmethod
def transport_request(self, identifier):
@ -199,30 +200,27 @@ class SWHListerBase(abc.ABC, config.SWHConfig):
@property
def CONFIG_BASE_FILENAME(self): # noqa: N802
return 'lister-%s' % self.lister_name
return 'lister-%s' % self.LISTER_NAME
@property
def ADDITIONAL_CONFIG(self): # noqa: N802
return {
'lister_db_url':
('str', 'postgresql:///lister-%s' % self.lister_name),
('str', 'postgresql:///lister-%s' % self.LISTER_NAME),
'credentials':
('list[dict]', []),
'cache_responses':
('bool', False),
'cache_dir':
('str', '~/.cache/swh/lister/%s' % self.lister_name),
('str', '~/.cache/swh/lister/%s' % self.LISTER_NAME),
}
INITIAL_BACKOFF = 10
MAX_RETRIES = 7
CONN_SLEEP = 10
def __init__(self, lister_name=None, override_config=None):
def __init__(self, override_config=None):
self.backoff = self.INITIAL_BACKOFF
if lister_name is None:
raise NameError("Every lister must be assigned a lister_name.")
self.lister_name = lister_name # 'github?', 'bitbucket?', 'foo.com?'
self.config = self.parse_config_file(
base_filename=self.CONFIG_BASE_FILENAME,
additional_configs=[self.ADDITIONAL_CONFIG]

6
swh/lister/core/paging_lister.py
View file

@ -131,8 +131,6 @@ class SWHPagingHttpLister(SWHListerHttpTransport, SWHPagingLister):
combining SWHPagingLister and SWHListerHttpTransport.
"""
def __init__(self, lister_name=None, api_baseurl=None,
override_config=None):
def __init__(self, api_baseurl=None, override_config=None):
SWHListerHttpTransport.__init__(self, api_baseurl=api_baseurl)
SWHPagingLister.__init__(self, lister_name=lister_name,
override_config=override_config)
SWHPagingLister.__init__(self, override_config=override_config)

4
swh/lister/core/tests/test_lister.py
View file

@ -39,6 +39,7 @@ class HttpListerTesterBase(abc.ABC):
first_index = AbstractAttribute('First index in good_api_response')
last_index = AbstractAttribute('Last index in good_api_response')
entries_per_page = AbstractAttribute('Number of results in good response')
LISTER_NAME = 'fake-lister'
# May need to override this if the headers are used for something
def response_headers(self, request):
@ -109,8 +110,7 @@ class HttpListerTesterBase(abc.ABC):
with patch(
'swh.scheduler.backend.SchedulerBackend.reconnect', noop
):
self.fl = self.Lister(lister_name='fakelister',
api_baseurl='https://fakeurl',
self.fl = self.Lister(api_baseurl='https://fakeurl',
override_config=override_config)
self.fl.INITIAL_BACKOFF = 1

4
swh/lister/debian/lister.py
View file

@ -31,11 +31,11 @@ decompressors = {
class DebianLister(SWHListerHttpTransport, SWHListerBase):
MODEL = Package
PATH_TEMPLATE = None
LISTER_NAME = 'debian'
def __init__(self, override_config=None):
SWHListerHttpTransport.__init__(self, api_baseurl="bogus")
SWHListerBase.__init__(self, lister_name='debian',
override_config=override_config)
SWHListerBase.__init__(self, override_config=override_config)
def transport_request(self, identifier):
"""Subvert SWHListerHttpTransport.transport_request, to try several

1
swh/lister/github/lister.py
View file

@ -13,6 +13,7 @@ class GitHubLister(SWHIndexingHttpLister):
PATH_TEMPLATE = '/repositories?since=%d'
MODEL = GitHubModel
API_URL_INDEX_RE = re.compile(r'^.*/repositories\?since=(\d+)')
LISTER_NAME = 'github.com'
def get_model_from_repo(self, repo):
return {

3
swh/lister/github/tasks.py
View file

@ -11,8 +11,7 @@ from .lister import GitHubLister
class GitHubListerTask(ListerTaskBase):
def new_lister(self):
return GitHubLister(lister_name='github.com',
api_baseurl='https://api.github.com')
return GitHubLister(api_baseurl='https://api.github.com')
class IncrementalGitHubLister(GitHubListerTask, IndexingDiscoveryListerTask):

32
swh/lister/gitlab/lister.py
View file

@ -15,14 +15,14 @@ class GitLabLister(SWHPagingHttpLister):
PATH_TEMPLATE = '/projects?page=%d&order_by=id&sort=asc&simple=true'
API_URL_INDEX_RE = re.compile(r'^.*/projects.*page=(\d+).*')
MODEL = GitLabModel
LISTER_NAME = 'gitlab'
@property
def CONFIG_BASE_FILENAME(self):
"""One gitlab lister for all instances. We discriminate between the
origin on a per instance basis in the table.
"""
return 'lister-gitlab'
def __init__(self, api_baseurl=None, instance=None,
override_config=None, sort='asc'):
super().__init__(api_baseurl=api_baseurl,
override_config=override_config)
self.instance = instance
self.PATH_TEMPLATE = '%s&sort=%s' % (self.PATH_TEMPLATE, sort)
@property
def ADDITIONAL_CONFIG(self):
@ -32,16 +32,10 @@ class GitLabLister(SWHPagingHttpLister):
cf. request_params method below
"""
return {
'lister_db_url':
('str', 'postgresql:///lister-gitlab'),
'credentials': # credentials is a dict
('dict', {}),
'cache_responses':
('bool', False),
'cache_dir':
('str', '~/.cache/swh/lister/%s' % self.lister_name),
}
default_config = super().ADDITIONAL_CONFIG
# 'credentials' is a dict of (instance, {username, password}) dict
default_config['credentials'] = ('dict', {})
return default_config
def request_params(self, identifier):
"""Get the full parameters passed to requests given the
@ -67,7 +61,7 @@ class GitLabLister(SWHPagingHttpLister):
# Retrieve the credentials per instance
creds = self.config['credentials']
if creds:
creds_lister = creds[self.lister_name]
creds_lister = creds[self.instance]
auth = random.choice(creds_lister) if creds else None
if auth:
params['auth'] = (auth['username'], auth['password'])
@ -75,7 +69,7 @@ class GitLabLister(SWHPagingHttpLister):
def get_model_from_repo(self, repo):
return {
'instance': self.lister_name,
'instance': self.instance,
'uid': repo['id'],
'indexable': repo['id'],
'name': repo['name'],

7
swh/lister/gitlab/tasks.py
View file

@ -11,10 +11,9 @@ from .lister import GitLabLister
class GitLabListerTask(ListerTaskBase):
def new_lister(self, lister_name='gitlab.com',
api_baseurl='https://gitlab.com/api/v4'):
return GitLabLister(
lister_name=lister_name, api_baseurl=api_baseurl)
def new_lister(self, api_baseurl='https://gitlab.com/api/v4',
instance='gitlab.com'):
return GitLabLister(api_baseurl=api_baseurl, instance=instance)
class RangeGitLabLister(GitLabListerTask, RangeListerTask):