lister*: Migrate away from SWHConfig mixin

Related to T1532 T1410 D3965

swh/lister/core/lister_base.py

@@ -37,6 +37,14 @@ class FetchError(RuntimeError):
         return repr(self.response)
 
 
+DEFAULT_CONFIG = {
+    "scheduler": {"cls": "memory"},
+    "lister": {"cls": "local", "args": {"db": "postgresql:///lister",},},
+    "credentials": {},
+    "cache_responses": False,
+}
+
+
 class ListerBase(abc.ABC, config.SWHConfig):
     """Lister core base class.
         Generally a source code hosting service provides an API endpoint
@@ -214,39 +222,18 @@ class ListerBase(abc.ABC, config.SWHConfig):
 
     # You probably don't need to override anything below this line.
 
-    DEFAULT_CONFIG = {
-        "scheduler": (
-            "dict",
-            {"cls": "remote", "args": {"url": "http://localhost:5008/"},},
-        ),
-        "lister": ("dict", {"cls": "local", "args": {"db": "postgresql:///lister",},}),
-    }
-
-    @property
-    def CONFIG_BASE_FILENAME(self):  # noqa: N802
-        return "lister_%s" % self.LISTER_NAME
-
-    @property
-    def ADDITIONAL_CONFIG(self):  # noqa: N802
-        return {
-            "credentials": ("dict", {}),
-            "cache_responses": ("bool", False),
-            "cache_dir": ("str", "~/.cache/swh/lister/%s" % self.LISTER_NAME),
-        }
-
     INITIAL_BACKOFF = 10
     MAX_RETRIES = 7
     CONN_SLEEP = 10
 
     def __init__(self, override_config=None):
         self.backoff = self.INITIAL_BACKOFF
-        logger.debug("Loading config from %s" % self.CONFIG_BASE_FILENAME)
-        self.config = self.parse_config_file(
-            base_filename=self.CONFIG_BASE_FILENAME,
-            additional_configs=[self.ADDITIONAL_CONFIG],
-        )
-        self.config["cache_dir"] = os.path.expanduser(self.config["cache_dir"])
+        self.config = config.load_from_envvar(DEFAULT_CONFIG)
         if self.config["cache_responses"]:
+            cache_dir = self.config.get(
+                "cache_dir", f"~/.cache/swh/lister/{self.LISTER_NAME}"
+            )
+            self.config["cache_dir"] = os.path.expanduser(cache_dir)
             config.prepare_folders(self.config, "cache_dir")
 
         if override_config:

swh/lister/npm/lister.py

@@ -6,10 +6,15 @@ from typing import Any, Dict, List, Optional
 
 from requests import Response
 
+from swh.core import config
 from swh.lister.core.indexing_lister import IndexingHttpLister
 from swh.lister.npm.models import NpmModel
 from swh.scheduler.utils import create_task_dict
 
+DEFAULT_CONFIG = {
+    "loading_task_policy": "recurring",
+}
+
 
 class NpmListerBase(IndexingHttpLister):
     """List packages available in the npm registry in a paginated way
@@ -24,18 +29,10 @@ class NpmListerBase(IndexingHttpLister):
         self, url="https://replicate.npmjs.com", per_page=1000, override_config=None
     ):
         super().__init__(url=url, override_config=override_config)
+        self.config = config.merge_configs(DEFAULT_CONFIG, self.config)
         self.per_page = per_page + 1
         self.PATH_TEMPLATE += "&limit=%s" % self.per_page
 
-    @property
-    def ADDITIONAL_CONFIG(self) -> Dict[str, Any]:
-        """(Override) Add extra configuration
-
-        """
-        default_config = super().ADDITIONAL_CONFIG
-        default_config["loading_task_policy"] = ("str", "recurring")
-        return default_config
-
     def get_model_from_repo(self, repo_name: str) -> Dict[str, str]:
         """(Override) Transform from npm package name to model
 

swh/lister/pytest_plugin.py

@@ -4,9 +4,11 @@
 # See top-level LICENSE file for more information
 
 import logging
+import os
 
 import pytest
 from sqlalchemy import create_engine
+import yaml
 
 from swh.lister import SUPPORTED_LISTERS, get_lister
 from swh.lister.core.models import initialize
@@ -33,7 +35,28 @@ def lister_under_test():
 
 
 @pytest.fixture
-def swh_lister(mock_get_scheduler, lister_db_url, swh_scheduler, lister_under_test):
+def swh_lister_config(lister_db_url, swh_scheduler_config):
+    return {
+        "scheduler": {"cls": "local", "args": {"db": swh_scheduler_config}["db"]},
+        "lister": {"cls": "local", "args": {"db": lister_db_url},},
+        "credentials": {},
+        "cache_responses": False,
+    }
+
+
+@pytest.fixture(autouse=True)
+def swh_config(swh_lister_config, monkeypatch, tmp_path):
+    conf_path = os.path.join(str(tmp_path), "lister.yml")
+    with open(conf_path, "w") as f:
+        f.write(yaml.dump(swh_lister_config))
+    monkeypatch.setenv("SWH_CONFIG_FILENAME", conf_path)
+    return conf_path
+
+
+@pytest.fixture
+def swh_lister(
+    mock_get_scheduler, lister_db_url, swh_scheduler, lister_under_test, swh_config
+):
     assert lister_under_test in SUPPORTED_LISTERS
     lister = get_lister(lister_under_test, db_url=lister_db_url)
     initialize(create_engine(lister_db_url), drop_tables=True)