Make debian lister constructors compatible with credentials

In effect, it just allows to add credentials to cgit, cran and pypi listers.

This fixes instances of error [1]

[1] https://sentry.softwareheritage.org/share/issue/a5fb50f8e43e4b328c4917771576c6b0/

Related to T2998
This commit is contained in:
Antoine R. Dumont (@ardumont) 2021-01-28 18:45:36 +01:00
parent e8725eb247
commit 130ad7d73e
No known key found for this signature in database
GPG key ID: 52E2E9840D10C3B8
2 changed files with 38 additions and 3 deletions

View file

@ -11,7 +11,7 @@ import gzip
from itertools import product
import logging
import lzma
from typing import Any, Callable, Dict, Iterator, List, Set, Tuple
from typing import Any, Callable, Dict, Iterator, List, Optional, Set, Tuple
from urllib.parse import urljoin
from debian.deb822 import Sources
@ -21,7 +21,7 @@ from swh.scheduler.interface import SchedulerInterface
from swh.scheduler.model import ListedOrigin
from .. import USER_AGENT
from ..pattern import Lister
from ..pattern import CredentialsType, Lister
logger = logging.getLogger(__name__)
@ -76,9 +76,13 @@ class DebianLister(Lister[DebianListerState, DebianPageType]):
mirror_url: str = "http://deb.debian.org/debian/",
suites: List[Suite] = ["stretch", "buster", "bullseye"],
components: List[Component] = ["main", "contrib", "non-free"],
credentials: Optional[CredentialsType] = None,
):
super().__init__(
scheduler=scheduler, url=mirror_url, instance=distribution,
scheduler=scheduler,
url=mirror_url,
instance=distribution,
credentials=credentials,
)
# to ensure urljoin will produce valid Sources URL

View file

@ -199,3 +199,34 @@ def test_lister_debian_updated_packages(
assert stats.origins == 0
else:
assert stats.origins != 0
@pytest.mark.parametrize(
"credentials, expected_credentials",
[
(None, []),
({"key": "value"}, []),
(
{"debian": {"Debian": [{"username": "user", "password": "pass"}]}},
[{"username": "user", "password": "pass"}],
),
],
)
def test_lister_debian_instantiation_with_credentials(
credentials, expected_credentials, swh_scheduler
):
lister = DebianLister(swh_scheduler, credentials=credentials)
# Credentials are allowed in constructor
assert lister.credentials == expected_credentials
def test_lister_debian_from_configfile(swh_scheduler_config, mocker):
load_from_envvar = mocker.patch("swh.lister.pattern.load_from_envvar")
load_from_envvar.return_value = {
"scheduler": {"cls": "local", **swh_scheduler_config},
"credentials": {},
}
lister = DebianLister.from_configfile()
assert lister.scheduler is not None
assert lister.credentials is not None