Harmonize listers parameters and add test to check mandatory ones

Ensure that all lister classes have the same set of mandatory parameters
in their constructors, notably: scheduler, url, instance and credentials.

Add a new test checking listers classes have mandatory parameters declared
in their constructors. The purpose is to avoid deployment issues on staging
or production environment as celery tasks can fail to be executed if mandatory
parameters are not handled by listers.

Reated to swh/infra/sysadm-environment#5030.

swh/lister/debian/lister.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2017-2022 The Software Heritage developers
+# Copyright (C) 2017-2023 The Software Heritage developers
 # See the AUTHORS file at the top-level directory of this distribution
 # License: GNU General Public License version 3, or any later version
 # See top-level LICENSE file for more information
@@ -68,12 +68,14 @@ class DebianLister(Lister[DebianListerState, DebianPageType]):
     """
 
     LISTER_NAME = "debian"
+    MIRROR_URL = "http://deb.debian.org/debian/"
+    INSTANCE = "Debian"
 
     def __init__(
         self,
         scheduler: SchedulerInterface,
-        distribution: str = "Debian",
-        mirror_url: str = "http://deb.debian.org/debian/",
+        url: str = MIRROR_URL,
+        instance: str = INSTANCE,
         suites: List[Suite] = ["stretch", "buster", "bullseye"],
         components: List[Component] = ["main", "contrib", "non-free"],
         credentials: Optional[CredentialsType] = None,
@@ -83,8 +85,8 @@ class DebianLister(Lister[DebianListerState, DebianPageType]):
     ):
         super().__init__(
             scheduler=scheduler,
-            url=mirror_url,
-            instance=distribution,
+            url=url,
+            instance=instance,
             credentials=credentials,
             max_origins_per_page=max_origins_per_page,
             max_pages=max_pages,
@@ -95,7 +97,7 @@ class DebianLister(Lister[DebianListerState, DebianPageType]):
         if not self.url.endswith("/"):
             self.url += "/"
 
-        self.distribution = distribution
+        self.distribution = instance
         self.suites = suites
         self.components = components
 

swh/lister/debian/tasks.py

@@ -10,6 +10,11 @@ from .lister import DebianLister
 @shared_task(name=__name__ + ".DebianListerTask")
 def list_debian_distribution(**lister_args):
     """List a Debian distribution"""
+    # for backward compatibility with previous parameter names
+    if "mirror_url" in lister_args:
+        lister_args["url"] = lister_args.pop("mirror_url")
+    if "distribution" in lister_args:
+        lister_args["instance"] = lister_args.pop("distribution")
     return DebianLister.from_configfile(**lister_args).run().dict()
 
 

swh/lister/debian/tests/test_lister.py

@@ -65,7 +65,7 @@ def _init_test(
 
     lister = DebianLister(
         scheduler=swh_scheduler,
-        mirror_url=_mirror_url,
+        url=_mirror_url,
         suites=list(debian_sources.keys()),
         components=_components,
     )

swh/lister/debian/tests/test_tasks.py

@@ -24,8 +24,8 @@ def test_lister(lister, swh_scheduler_celery_app, swh_scheduler_celery_worker):
     lister.run.return_value = stats
 
     kwargs = dict(
-        mirror_url="http://www-ftp.lip6.fr/pub/linux/distributions/Ubuntu/archive/",
-        distribution="Ubuntu",
+        url="http://www-ftp.lip6.fr/pub/linux/distributions/Ubuntu/archive/",
+        instance="Ubuntu",
         suites=["xenial", "bionic", "focal"],
         components=["main", "multiverse", "restricted", "universe"],
     )
@@ -41,3 +41,35 @@ def test_lister(lister, swh_scheduler_celery_app, swh_scheduler_celery_worker):
     lister.run.assert_called_once_with()
 
     assert res.result == stats.dict()
+
+
+@patch("swh.lister.debian.tasks.DebianLister")
+def test_lister_old_parameter_names(
+    lister, swh_scheduler_celery_app, swh_scheduler_celery_worker
+):
+    # setup the mocked DebianLister
+    lister.from_configfile.return_value = lister
+    stats = ListerStats(pages=12, origins=35618)
+    lister.run.return_value = stats
+
+    kwargs = dict(
+        mirror_url="http://www-ftp.lip6.fr/pub/linux/distributions/Ubuntu/archive/",
+        distribution="Ubuntu",
+        suites=["xenial", "bionic", "focal"],
+        components=["main", "multiverse", "restricted", "universe"],
+    )
+
+    res = swh_scheduler_celery_app.send_task(
+        "swh.lister.debian.tasks.DebianListerTask", kwargs=kwargs
+    )
+    assert res
+    res.wait()
+    assert res.successful()
+
+    kwargs["url"] = kwargs.pop("mirror_url")
+    kwargs["instance"] = kwargs.pop("distribution")
+
+    lister.from_configfile.assert_called_once_with(**kwargs)
+    lister.run.assert_called_once_with()
+
+    assert res.result == stats.dict()