reonokiy/swh-lister
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Drop SWH prefix in classes everywhere

Browse source 
It's redundant with the swh modules in itself.
This commit is contained in:
Antoine R. Dumont (@ardumont) 2019-06-20 19:08:46 +02:00
parent 8d1b5d2d2d
commit b3463ecddc
No known key found for this signature in database
GPG key ID: 52E2E9840D10C3B8
12 changed files with 82 additions and 84 deletions
Download patch file Download diff file Expand all files Collapse all files

6
swh/lister/bitbucket/lister.py
View file

@ -7,13 +7,13 @@ import logging
import iso8601
from swh.lister.bitbucket.models import BitBucketModel
from swh.lister.core.indexing_lister import SWHIndexingHttpLister
from swh.lister.core.indexing_lister import IndexingHttpLister
logger = logging.getLogger(__name__)
class BitBucketLister(SWHIndexingHttpLister):
class BitBucketLister(IndexingHttpLister):
PATH_TEMPLATE = '/repositories?after=%s'
MODEL = BitBucketModel
LISTER_NAME = 'bitbucket'
@ -34,8 +34,6 @@ class BitBucketLister(SWHIndexingHttpLister):
body = response.json()
if 'next' in body:
return parse.unquote(body['next'].split('after=')[1])
else:
return None
def transport_response_simplified(self, response):
repos = response.json()['values']

18
swh/lister/core/indexing_lister.py
View file

@ -9,13 +9,13 @@ from itertools import count
import dateutil
from sqlalchemy import func
from .lister_transports import SWHListerHttpTransport
from .lister_base import SWHListerBase
from .lister_transports import ListerHttpTransport
from .lister_base import ListerBase
logger = logging.getLogger(__name__)
class SWHIndexingLister(SWHListerBase):
class IndexingLister(ListerBase):
"""Lister* intermediate class for any service that follows the pattern:
- The service must report at least one stable unique identifier, known
@ -32,7 +32,7 @@ class SWHIndexingLister(SWHListerBase):
necessary/available, some indication of the URL or index for fetching the
next series of repository data.
See :class:`swh.lister.core.lister_base.SWHListerBase` for more details.
See :class:`swh.lister.core.lister_base.ListerBase` for more details.
This class cannot be instantiated. To create a new Lister for a source
code listing service that follows the model described above, you must
@ -64,7 +64,7 @@ class SWHIndexingLister(SWHListerBase):
# You probably don't need to override anything below this line.
def filter_before_inject(self, models_list):
"""Overrides SWHListerBase.filter_before_inject
"""Overrides ListerBase.filter_before_inject
Bounds query results by this Lister's set max_index.
"""
@ -234,9 +234,9 @@ class SWHIndexingLister(SWHListerBase):
self.db_session = self.mk_session()
class SWHIndexingHttpLister(SWHListerHttpTransport, SWHIndexingLister):
class IndexingHttpLister(ListerHttpTransport, IndexingLister):
"""Convenience class for ensuring right lookup and init order
when combining SWHIndexingLister and SWHListerHttpTransport."""
when combining IndexingLister and ListerHttpTransport."""
def __init__(self, api_baseurl=None, override_config=None):
SWHListerHttpTransport.__init__(self, api_baseurl=api_baseurl)
SWHIndexingLister.__init__(self, override_config=override_config)
ListerHttpTransport.__init__(self, api_baseurl=api_baseurl)
IndexingLister.__init__(self, override_config=override_config)

8
swh/lister/core/lister_base.py
View file

@ -36,7 +36,7 @@ class FetchError(RuntimeError):
return repr(self.response)
class SWHListerBase(abc.ABC, config.SWHConfig):
class ListerBase(abc.ABC, config.SWHConfig):
"""Lister core base class.
Generally a source code hosting service provides an API endpoint
for listing the set of stored repositories. A Lister is the discovery
@ -46,11 +46,11 @@ class SWHListerBase(abc.ABC, config.SWHConfig):
The core method in this class is ingest_data. Any subclasses should be
calling this method one or more times to fetch and ingest data from API
endpoints. See swh.lister.core.lister_base.SWHIndexingLister for
endpoints. See swh.lister.core.lister_base.IndexingLister for
example usage.
This class cannot be instantiated. Any instantiable Lister descending
from SWHListerBase must provide at least the required overrides.
from ListerBase must provide at least the required overrides.
(see member docstrings for details):
Required Overrides:
@ -172,7 +172,7 @@ class SWHListerBase(abc.ABC, config.SWHConfig):
MAY BE OVERRIDDEN, for example if the server indexable* key is
technically sortable but not automatically so.
* - ( see: swh.lister.core.indexing_lister.SWHIndexingLister )
* - ( see: swh.lister.core.indexing_lister.IndexingLister )
Args:
inner (sortable type): the value being checked

14
swh/lister/core/lister_transports.py
View file

@ -24,10 +24,10 @@ from .lister_base import FetchError
logger = logging.getLogger(__name__)
class SWHListerHttpTransport(abc.ABC):
class ListerHttpTransport(abc.ABC):
"""Use the Requests library for making Lister endpoint requests.
To be used in conjunction with SWHListerBase or a subclass of it.
To be used in conjunction with ListerBase or a subclass of it.
"""
PATH_TEMPLATE = AbstractAttribute('string containing a python string'
@ -117,7 +117,7 @@ class SWHListerHttpTransport(abc.ABC):
return params
def transport_quota_check(self, response):
"""Implements SWHListerBase.transport_quota_check with standard 429
"""Implements ListerBase.transport_quota_check with standard 429
code check for HTTP with Requests library.
MAY BE OVERRIDDEN if the server notifies about rate limits in a
@ -174,7 +174,7 @@ class SWHListerHttpTransport(abc.ABC):
return self._transport_action(identifier, method='head')
def transport_request(self, identifier):
"""Implements SWHListerBase.transport_request for HTTP using Requests.
"""Implements ListerBase.transport_request for HTTP using Requests.
Retrieve get information on api.
@ -182,7 +182,7 @@ class SWHListerHttpTransport(abc.ABC):
return self._transport_action(identifier)
def transport_response_to_string(self, response):
"""Implements SWHListerBase.transport_response_to_string for HTTP given
"""Implements ListerBase.transport_response_to_string for HTTP given
Requests responses.
"""
s = pformat(response.request.path_url)
@ -200,11 +200,11 @@ class SWHListerHttpTransport(abc.ABC):
return s
class ListerOnePageApiTransport(SWHListerHttpTransport):
class ListerOnePageApiTransport(ListerHttpTransport):
"""Leverage requests library to retrieve basic html page and parse
result.
To be used in conjunction with SWHListerBase or a subclass of it.
To be used in conjunction with ListerBase or a subclass of it.
"""
PAGE = AbstractAttribute("The server api's unique page to retrieve and "

14
swh/lister/core/page_by_page_lister.py
View file

@ -5,11 +5,11 @@
import abc
import logging
from .lister_transports import SWHListerHttpTransport
from .lister_base import SWHListerBase
from .lister_transports import ListerHttpTransport
from .lister_base import ListerBase
class PageByPageLister(SWHListerBase):
class PageByPageLister(ListerBase):
"""Lister* intermediate class for any service that follows the simple
pagination page pattern.
@ -22,7 +22,7 @@ class PageByPageLister(SWHListerBase):
of the next page index for fetching the remaining repository
data.
See :class:`swh.lister.core.lister_base.SWHListerBase` for more
See :class:`swh.lister.core.lister_base.ListerBase` for more
details.
This class cannot be instantiated. To create a new Lister for a
@ -150,11 +150,11 @@ class PageByPageLister(SWHListerBase):
self.db_session = self.mk_session()
class PageByPageHttpLister(SWHListerHttpTransport, PageByPageLister):
class PageByPageHttpLister(ListerHttpTransport, PageByPageLister):
"""Convenience class for ensuring right lookup and init order when
combining PageByPageLister and SWHListerHttpTransport.
combining PageByPageLister and ListerHttpTransport.
"""
def __init__(self, api_baseurl=None, override_config=None):
SWHListerHttpTransport.__init__(self, api_baseurl=api_baseurl)
ListerHttpTransport.__init__(self, api_baseurl=api_baseurl)
PageByPageLister.__init__(self, override_config=override_config)

4
swh/lister/core/simple_lister.py
View file

@ -6,10 +6,10 @@ import logging
from swh.core import utils
from .lister_base import SWHListerBase
from .lister_base import ListerBase
class SimpleLister(SWHListerBase):
class SimpleLister(ListerBase):
"""Lister* intermediate class for any service that follows the simple,
'list in oneshot information' pattern.

2
swh/lister/core/tests/test_lister.py
View file

@ -22,7 +22,7 @@ def noop(*args, **kwargs):
class HttpListerTesterBase(abc.ABC):
"""Base testing class for subclasses of
swh.lister.core.indexing_lister.SWHIndexingHttpLister.
swh.lister.core.indexing_lister.IndexingHttpLister.
swh.lister.core.page_by_page_lister.PageByPageHttpLister
See swh.lister.github.tests.test_gh_lister for an example of how

16
swh/lister/debian/lister.py
View file

@ -18,8 +18,8 @@ from swh.storage.schemata.distribution import (
TempPackage,
)
from swh.lister.core.lister_base import SWHListerBase, FetchError
from swh.lister.core.lister_transports import SWHListerHttpTransport
from swh.lister.core.lister_base import ListerBase, FetchError
from swh.lister.core.lister_transports import ListerHttpTransport
decompressors = {
'gz': lambda f: gzip.GzipFile(fileobj=f),
@ -28,18 +28,18 @@ decompressors = {
}
class DebianLister(SWHListerHttpTransport, SWHListerBase):
class DebianLister(ListerHttpTransport, ListerBase):
MODEL = Package
PATH_TEMPLATE = None
LISTER_NAME = 'debian'
instance = 'debian'
def __init__(self, override_config=None):
SWHListerHttpTransport.__init__(self, api_baseurl="bogus")
SWHListerBase.__init__(self, override_config=override_config)
ListerHttpTransport.__init__(self, api_baseurl="bogus")
ListerBase.__init__(self, override_config=override_config)
def transport_request(self, identifier):
"""Subvert SWHListerHttpTransport.transport_request, to try several
"""Subvert ListerHttpTransport.transport_request, to try several
index URIs in turn.
The Debian repository format supports several compression algorithms
@ -70,7 +70,7 @@ class DebianLister(SWHListerHttpTransport, SWHListerBase):
def request_uri(self, identifier):
# In the overridden transport_request, we pass
# SWHListerBase.transport_request() the full URI as identifier, so we
# ListerBase.transport_request() the full URI as identifier, so we
# need to return it here.
return identifier
@ -118,7 +118,7 @@ class DebianLister(SWHListerHttpTransport, SWHListerBase):
def inject_repo_data_into_db(self, models_list):
"""Generate the Package entries that didn't previously exist.
Contrary to SWHListerBase, we don't actually insert the data in
Contrary to ListerBase, we don't actually insert the data in
database. `schedule_missing_tasks` does it once we have the
origin and task identifiers.
"""

4
swh/lister/github/lister.py
View file

@ -5,11 +5,11 @@
import re
import time
from swh.lister.core.indexing_lister import SWHIndexingHttpLister
from swh.lister.core.indexing_lister import IndexingHttpLister
from swh.lister.github.models import GitHubModel
class GitHubLister(SWHIndexingHttpLister):
class GitHubLister(IndexingHttpLister):
PATH_TEMPLATE = '/repositories?since=%d'
MODEL = GitHubModel
API_URL_INDEX_RE = re.compile(r'^.*/repositories\?since=(\d+)')

4
swh/lister/npm/lister.py
View file

@ -4,12 +4,12 @@
from urllib.parse import quote
from swh.lister.core.indexing_lister import SWHIndexingHttpLister
from swh.lister.core.indexing_lister import IndexingHttpLister
from swh.lister.npm.models import NpmModel
from swh.scheduler.utils import create_task_dict
class NpmListerBase(SWHIndexingHttpLister):
class NpmListerBase(IndexingHttpLister):
"""List packages available in the npm registry in a paginated way
"""
MODEL = NpmModel

6
swh/lister/phabricator/lister.py
View file

@ -6,14 +6,14 @@ import logging
import urllib.parse
from swh.lister.core.indexing_lister import SWHIndexingHttpLister
from swh.lister.core.indexing_lister import IndexingHttpLister
from swh.lister.phabricator.models import PhabricatorModel
from collections import defaultdict
logger = logging.getLogger(__name__)
class PhabricatorLister(SWHIndexingHttpLister):
class PhabricatorLister(IndexingHttpLister):
PATH_TEMPLATE = '?order=oldest&attachments[uris]=1&after=%s'
MODEL = PhabricatorModel
LISTER_NAME = 'phabricator'
@ -104,7 +104,7 @@ class PhabricatorLister(SWHIndexingHttpLister):
def filter_before_inject(self, models_list):
"""
(Overrides) SWHIndexingLister.filter_before_inject
(Overrides) IndexingLister.filter_before_inject
Bounds query results by this Lister's set max_index.
"""
models_list = [m for m in models_list if m is not None]