Make stateless lister constructors compatible with credentials

In effect, it just allows to add credentials to cgit, cran and pypi listers.

This fixes instances of error [1]

[1] https://sentry.softwareheritage.org/share/issue/2c35a9f129cf4982a2dd003a232d507a/

Related to T2998

swh/lister/cgit/lister.py

@@ -13,7 +13,7 @@ import requests
 from requests.exceptions import HTTPError
 
 from swh.lister import USER_AGENT
-from swh.lister.pattern import StatelessLister
+from swh.lister.pattern import CredentialsType, StatelessLister
 from swh.scheduler.interface import SchedulerInterface
 from swh.scheduler.model import ListedOrigin
 
@@ -39,7 +39,11 @@ class CGitLister(StatelessLister[Repositories]):
     LISTER_NAME = "cgit"
 
     def __init__(
-        self, scheduler: SchedulerInterface, url: str, instance: Optional[str] = None
+        self,
+        scheduler: SchedulerInterface,
+        url: str,
+        instance: Optional[str] = None,
+        credentials: Optional[CredentialsType] = None,
     ):
         """Lister class for CGit repositories.
 
@@ -55,7 +59,7 @@ class CGitLister(StatelessLister[Repositories]):
         assert instance is not None  # Make mypy happy
 
         super().__init__(
-            scheduler=scheduler, credentials=None, url=url, instance=instance,
+            scheduler=scheduler, url=url, instance=instance, credentials=credentials,
         )
 
         self.session = requests.Session()

swh/lister/cgit/tests/test_lister.py

@@ -160,3 +160,39 @@ def test_lister_cgit_get_origin_from_repo_failing(
 
     expected_nb_origins = 15
     assert stats == ListerStats(pages=3, origins=expected_nb_origins)
+
+
+@pytest.mark.parametrize(
+    "credentials, expected_credentials",
+    [
+        (None, []),
+        ({"key": "value"}, []),
+        (
+            {"cgit": {"tizen": [{"username": "user", "password": "pass"}]}},
+            [{"username": "user", "password": "pass"}],
+        ),
+    ],
+)
+def test_lister_cgit_instantiation_with_credentials(
+    credentials, expected_credentials, swh_scheduler
+):
+    url = "https://git.tizen/cgit/"
+    lister = CGitLister(
+        swh_scheduler, url=url, instance="tizen", credentials=credentials
+    )
+
+    # Credentials are allowed in constructor
+    assert lister.credentials == expected_credentials
+
+
+def test_lister_cgit_from_configfile(swh_scheduler_config, mocker):
+    load_from_envvar = mocker.patch("swh.lister.pattern.load_from_envvar")
+    load_from_envvar.return_value = {
+        "scheduler": {"cls": "local", **swh_scheduler_config},
+        "url": "https://git.tizen/cgit/",
+        "instance": "tizen",
+        "credentials": {},
+    }
+    lister = CGitLister.from_configfile()
+    assert lister.scheduler is not None
+    assert lister.credentials is not None

swh/lister/cgit/tests/test_tasks.py

@@ -3,8 +3,6 @@
 # License: GNU General Public License version 3, or any later version
 # See top-level LICENSE file for more information
 
-from unittest.mock import patch
-
 from swh.lister.pattern import ListerStats
 
 
@@ -16,11 +14,11 @@ def test_cgit_ping(swh_scheduler_celery_app, swh_scheduler_celery_worker):
     assert res.result == "OK"
 
 
-@patch("swh.lister.cgit.tasks.CGitLister")
 def test_cgit_lister_task(
-    lister, swh_scheduler_celery_app, swh_scheduler_celery_worker
+    swh_scheduler_celery_app, swh_scheduler_celery_worker, mocker
 ):
     # setup the mocked CGitLister
+    lister = mocker.patch("swh.lister.cgit.tasks.CGitLister")
     lister.from_configfile.return_value = lister
     lister.run.return_value = ListerStats(pages=10, origins=500)
 

swh/lister/cran/lister.py

@@ -10,7 +10,7 @@ from typing import Dict, Iterator, List, Optional, Tuple
 
 import pkg_resources
 
-from swh.lister.pattern import StatelessLister
+from swh.lister.pattern import CredentialsType, StatelessLister
 from swh.scheduler.interface import SchedulerInterface
 from swh.scheduler.model import ListedOrigin
 
@@ -29,9 +29,13 @@ class CRANLister(StatelessLister[PageType]):
     LISTER_NAME = "CRAN"
 
     def __init__(
-        self, scheduler: SchedulerInterface,
+        self,
+        scheduler: SchedulerInterface,
+        credentials: Optional[CredentialsType] = None,
     ):
-        super().__init__(scheduler, url=CRAN_MIRROR, instance="cran")
+        super().__init__(
+            scheduler, url=CRAN_MIRROR, instance="cran", credentials=credentials
+        )
 
     def get_pages(self) -> Iterator[PageType]:
         """

swh/lister/cran/tests/test_lister.py

@@ -89,3 +89,34 @@ def test_cran_lister_cran(datadir, swh_scheduler, mocker):
         }
 
         filtered_origins[0].last_update == parse_packaged_date(package_info)
+
+
+@pytest.mark.parametrize(
+    "credentials, expected_credentials",
+    [
+        (None, []),
+        ({"key": "value"}, []),
+        (
+            {"CRAN": {"cran": [{"username": "user", "password": "pass"}]}},
+            [{"username": "user", "password": "pass"}],
+        ),
+    ],
+)
+def test_lister_cran_instantiation_with_credentials(
+    credentials, expected_credentials, swh_scheduler
+):
+    lister = CRANLister(swh_scheduler, credentials=credentials)
+
+    # Credentials are allowed in constructor
+    assert lister.credentials == expected_credentials
+
+
+def test_lister_cran_from_configfile(swh_scheduler_config, mocker):
+    load_from_envvar = mocker.patch("swh.lister.pattern.load_from_envvar")
+    load_from_envvar.return_value = {
+        "scheduler": {"cls": "local", **swh_scheduler_config},
+        "credentials": {},
+    }
+    lister = CRANLister.from_configfile()
+    assert lister.scheduler is not None
+    assert lister.credentials is not None

swh/lister/pypi/lister.py

@@ -1,10 +1,10 @@
-# Copyright (C) 2018-2019 The Software Heritage developers
+# Copyright (C) 2018-2021 The Software Heritage developers
 # See the AUTHORS file at the top-level directory of this distribution
 # License: GNU General Public License version 3, or any later version
 # See top-level LICENSE file for more information
 
 import logging
-from typing import Iterator, List
+from typing import Iterator, List, Optional
 
 import requests
 import xmltodict
@@ -13,7 +13,7 @@ from swh.scheduler.interface import SchedulerInterface
 from swh.scheduler.model import ListedOrigin
 
 from .. import USER_AGENT
-from ..pattern import StatelessLister
+from ..pattern import CredentialsType, StatelessLister
 
 logger = logging.getLogger(__name__)
 
@@ -31,12 +31,16 @@ class PyPILister(StatelessLister[PackageListPage]):
     PACKAGE_LIST_URL = "https://pypi.org/simple/"
     PACKAGE_URL = "https://pypi.org/project/{package_name}/"
 
-    def __init__(self, scheduler: SchedulerInterface):
+    def __init__(
+        self,
+        scheduler: SchedulerInterface,
+        credentials: Optional[CredentialsType] = None,
+    ):
         super().__init__(
             scheduler=scheduler,
-            credentials=None,
             url=self.PACKAGE_LIST_URL,
             instance=self.INSTANCE,
+            credentials=credentials,
         )
 
         self.session = requests.Session()

swh/lister/pypi/tests/test_lister.py

@@ -78,3 +78,34 @@ def test_pypi_list_http_error(swh_scheduler, requests_mock, mocker, http_code):
 
     scheduler_origins = swh_scheduler.get_listed_origins(lister.lister_obj.id).results
     assert len(scheduler_origins) == 0
+
+
+@pytest.mark.parametrize(
+    "credentials, expected_credentials",
+    [
+        (None, []),
+        ({"key": "value"}, []),
+        (
+            {"pypi": {"pypi": [{"username": "user", "password": "pass"}]}},
+            [{"username": "user", "password": "pass"}],
+        ),
+    ],
+)
+def test_lister_pypi_instantiation_with_credentials(
+    credentials, expected_credentials, swh_scheduler
+):
+    lister = PyPILister(swh_scheduler, credentials=credentials)
+
+    # Credentials are allowed in constructor
+    assert lister.credentials == expected_credentials
+
+
+def test_lister_pypi_from_configfile(swh_scheduler_config, mocker):
+    load_from_envvar = mocker.patch("swh.lister.pattern.load_from_envvar")
+    load_from_envvar.return_value = {
+        "scheduler": {"cls": "local", **swh_scheduler_config},
+        "credentials": {},
+    }
+    lister = PyPILister.from_configfile()
+    assert lister.scheduler is not None
+    assert lister.credentials is not None