tests: Separate lister instantiations

Prior to this commit, all listers were instantiated at the same time even if
only one was needed. This commit separates those instantiations.

The only drawback to this is the db model initialization which now happens at
each lister instantiation. This can be dealt with if needed at another time
though.

swh/lister/bitbucket/tests/test_lister.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2017-2019 The Software Heritage developers
+# Copyright (C) 2017-2020 The Software Heritage developers
 # See the AUTHORS file at the top-level directory of this distribution
 # License: GNU General Public License version 3, or any later version
 # See top-level LICENSE file for more information
@@ -10,12 +10,18 @@ from datetime import timedelta
 from urllib.parse import unquote
 
 import iso8601
+import pytest
 import requests_mock
 
 from swh.lister.bitbucket.lister import BitBucketLister
 from swh.lister.core.tests.test_lister import HttpListerTester
 
 
+@pytest.fixture
+def lister_under_test():
+    return "bitbucket"
+
+
 def _convert_type(req_index):
     """Convert the req_index to its right type according to the model's
        "indexable" column.
@@ -80,11 +86,11 @@ class BitBucketListerTester(HttpListerTester, unittest.TestCase):
         )
 
 
-def test_lister_bitbucket(swh_listers, requests_mock_datadir):
+def test_lister_bitbucket(swh_lister, requests_mock_datadir):
     """Simple bitbucket listing should create scheduled tasks (git, hg)
 
     """
-    lister = swh_listers["bitbucket"]
+    lister = swh_lister
 
     lister.run()
 

swh/lister/cgit/tests/test_lister.py

@@ -1,13 +1,19 @@
-# Copyright (C) 2019 the Software Heritage developers
+# Copyright (C) 2019-2020 the Software Heritage developers
 # License: GNU General Public License version 3, or any later version
 # See top-level LICENSE file for more information
 
+import pytest
 
 from swh.lister import __version__
 
 
-def test_lister_no_page(requests_mock_datadir, swh_listers):
-    lister = swh_listers["cgit"]
+@pytest.fixture
+def lister_under_test():
+    return "cgit"
+
+
+def test_lister_no_page(requests_mock_datadir, swh_lister):
+    lister = swh_lister
 
     assert lister.url == "https://git.savannah.gnu.org/cgit/"
 
@@ -21,8 +27,8 @@ def test_lister_no_page(requests_mock_datadir, swh_listers):
     assert repos[-2] == "http://example.org/cgit/xstarcastle.git/"
 
 
-def test_lister_model(requests_mock_datadir, swh_listers):
-    lister = swh_listers["cgit"]
+def test_lister_model(requests_mock_datadir, swh_lister):
+    lister = swh_lister
 
     repo = next(lister.get_repos())
 
@@ -36,8 +42,8 @@ def test_lister_model(requests_mock_datadir, swh_listers):
     }
 
 
-def test_lister_with_pages(requests_mock_datadir, swh_listers):
-    lister = swh_listers["cgit"]
+def test_lister_with_pages(requests_mock_datadir, swh_lister):
+    lister = swh_lister
     lister.url = "https://git.tizen/cgit/"
 
     repos = list(lister.get_repos())
@@ -45,8 +51,8 @@ def test_lister_with_pages(requests_mock_datadir, swh_listers):
     assert len(repos) == 16
 
 
-def test_lister_run(requests_mock_datadir, swh_listers):
-    lister = swh_listers["cgit"]
+def test_lister_run(requests_mock_datadir, swh_lister):
+    lister = swh_lister
     lister.url = "https://git.tizen/cgit/"
     lister.run()
 
@@ -69,8 +75,8 @@ def test_lister_run(requests_mock_datadir, swh_listers):
         assert row["priority"] is None
 
 
-def test_lister_requests(requests_mock_datadir, swh_listers):
-    lister = swh_listers["cgit"]
+def test_lister_requests(requests_mock_datadir, swh_lister):
+    lister = swh_lister
     lister.url = "https://git.tizen/cgit/"
     lister.run()
 

swh/lister/cran/tests/conftest.py

@@ -7,11 +7,14 @@ import pytest
 
 
 @pytest.fixture
-def lister_cran(swh_listers):
-    lister = swh_listers["cran"]
+def lister_under_test():
+    return "cran"
 
-    # Add the load-deb-package in the scheduler backend
-    lister.scheduler.create_task_type(
+
+@pytest.fixture
+def lister_cran(swh_lister):
+    # Add the load-cran in the scheduler backend
+    swh_lister.scheduler.create_task_type(
         {
             "type": "load-cran",
             "description": "Load a CRAN package",
@@ -20,4 +23,4 @@ def lister_cran(swh_listers):
         }
     )
 
-    return lister
+    return swh_lister

swh/lister/debian/tests/conftest.py

@@ -14,14 +14,19 @@ from swh.lister.debian import debian_init
 
 
 @pytest.fixture
-def lister_debian(swh_listers):
-    lister = swh_listers["debian"]
+def lister_under_test():
+    return "debian"
 
+
+@pytest.fixture
+def lister_debian(swh_lister):
     # Initialize the debian data model
-    debian_init(lister.db_engine, suites=["stretch"], components=["main", "contrib"])
+    debian_init(
+        swh_lister.db_engine, suites=["stretch"], components=["main", "contrib"]
+    )
 
     # Add the load-deb-package in the scheduler backend
-    lister.scheduler.create_task_type(
+    swh_lister.scheduler.create_task_type(
         {
             "type": "load-deb-package",
             "description": "Load a Debian package",
@@ -30,7 +35,7 @@ def lister_debian(swh_listers):
         }
     )
 
-    return lister
+    return swh_lister
 
 
 @pytest.fixture

swh/lister/gitea/tests/test_lister.py

@@ -7,6 +7,8 @@ import logging
 import re
 import unittest
 
+import pytest
+
 from swh.lister.core.tests.test_lister import HttpListerTesterBase
 from swh.lister.gitea.lister import GiteaLister
 
@@ -38,8 +40,13 @@ class GiteaListerTester(HttpListerTesterBase, unittest.TestCase):
         return headers
 
 
-def test_lister_gitea(swh_listers, requests_mock_datadir):
-    lister: GiteaLister = swh_listers["gitea"]
+@pytest.fixture
+def lister_under_test():
+    return "gitea"
+
+
+def test_lister_gitea(swh_lister, requests_mock_datadir):
+    lister: GiteaLister = swh_lister
 
     lister.run()
     r = lister.scheduler.search_tasks(task_type="load-git")

swh/lister/github/tests/test_lister.py

@@ -6,6 +6,7 @@
 import re
 import unittest
 
+import pytest
 import requests_mock
 
 from swh.lister.core.tests.test_lister import HttpListerTester
@@ -57,15 +58,18 @@ class GitHubListerTester(HttpListerTester, unittest.TestCase):
         )
 
 
-def test_lister_github(swh_listers, requests_mock_datadir):
+@pytest.fixture
+def lister_under_test():
+    return "github"
+
+
+def test_lister_github(swh_lister, requests_mock_datadir):
     """Simple github listing should create scheduled tasks
 
     """
-    lister = swh_listers["github"]
+    swh_lister.run()
 
-    lister.run()
-
-    r = lister.scheduler.search_tasks(task_type="load-git")
+    r = swh_lister.scheduler.search_tasks(task_type="load-git")
     assert len(r) == 100
 
     for row in r:

swh/lister/gitlab/tests/test_lister.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2017-2019 The Software Heritage developers
+# Copyright (C) 2017-2020 The Software Heritage developers
 # See the AUTHORS file at the top-level directory of this distribution
 # License: GNU General Public License version 3, or any later version
 # See top-level LICENSE file for more information
@@ -6,8 +6,11 @@
 import logging
 import re
 import unittest
+
 from datetime import datetime, timedelta
 
+import pytest
+
 from swh.lister.core.tests.test_lister import HttpListerTesterBase
 from swh.lister.gitlab.lister import GitLabLister
 
@@ -43,12 +46,15 @@ class GitLabListerTester(HttpListerTesterBase, unittest.TestCase):
         return '{"error":"dummy"}'
 
 
-def test_lister_gitlab(swh_listers, requests_mock_datadir):
-    lister = swh_listers["gitlab"]
+@pytest.fixture
+def lister_under_test():
+    return "gitlab"
 
-    lister.run()
 
-    r = lister.scheduler.search_tasks(task_type="load-git")
+def test_lister_gitlab(swh_lister, requests_mock_datadir):
+    swh_lister.run()
+
+    r = swh_lister.scheduler.search_tasks(task_type="load-git")
     assert len(r) == 10
 
     for row in r:

swh/lister/gnu/tests/test_lister.py

@@ -1,20 +1,26 @@
-# Copyright (C) 2019 The Software Heritage developers
+# Copyright (C) 2019-2020 The Software Heritage developers
 # See the AUTHORS file at the top-level directory of this distribution
 # License: GNU General Public License version 3, or any later version
 # See top-level LICENSE file for more information
 
 import logging
 
+import pytest
+
 
 logger = logging.getLogger(__name__)
 
 
-def test_gnu_lister(swh_listers, requests_mock_datadir):
-    lister = swh_listers["gnu"]
+@pytest.fixture
+def lister_under_test():
+    return "gnu"
 
-    lister.run()
 
-    r = lister.scheduler.search_tasks(task_type="load-archive-files")
+@pytest.fixture
+def test_gnu_lister(swh_lister, requests_mock_datadir):
+    swh_lister.run()
+
+    r = swh_lister.scheduler.search_tasks(task_type="load-archive-files")
     assert len(r) == 383
 
     for row in r:

swh/lister/npm/tests/conftest.py

@@ -7,11 +7,14 @@ import pytest
 
 
 @pytest.fixture
-def lister_npm(swh_listers):
-    lister = swh_listers["npm"]
+def lister_under_test():
+    return "npm"
 
-    # Add the load-deb-package in the scheduler backend
-    lister.scheduler.create_task_type(
+
+@pytest.fixture
+def lister_npm(swh_lister):
+    # Add the load-npm in the scheduler backend
+    swh_lister.scheduler.create_task_type(
         {
             "type": "load-npm",
             "description": "Load npm package",
@@ -20,4 +23,4 @@ def lister_npm(swh_listers):
         }
     )
 
-    return lister
+    return swh_lister

swh/lister/packagist/tests/conftest.py

@@ -7,11 +7,14 @@ import pytest
 
 
 @pytest.fixture
-def lister_packagist(swh_listers):
-    lister = swh_listers["packagist"]
+def lister_under_test():
+    return "packagist"
 
+
+@pytest.fixture
+def lister_packagist(swh_lister):
     # Amend the scheduler with an unknown yet load-packagist task type
-    lister.scheduler.create_task_type(
+    swh_lister.scheduler.create_task_type(
         {
             "type": "load-packagist",
             "description": "Load packagist origin",
@@ -20,4 +23,4 @@ def lister_packagist(swh_listers):
         }
     )
 
-    return lister
+    return swh_lister

swh/lister/phabricator/tests/conftest.py

@@ -7,13 +7,16 @@ import pytest
 
 
 @pytest.fixture
-def lister_phabricator(swh_listers):
-    lister = swh_listers["phabricator"]
+def lister_under_test():
+    return "phabricator"
 
+
+@pytest.fixture
+def lister_phabricator(swh_lister):
     # Amend the credentials
-    lister.config = {
+    swh_lister.config = {
         "cache_responses": False,
-        "credentials": {"phabricator": {lister.instance: [{"password": "foo"}]}},
+        "credentials": {"phabricator": {swh_lister.instance: [{"password": "foo"}]}},
     }
 
-    return lister
+    return swh_lister

swh/lister/pypi/tests/conftest.py

@@ -7,11 +7,14 @@ import pytest
 
 
 @pytest.fixture
-def lister_pypi(swh_listers):
-    lister = swh_listers["pypi"]
+def lister_under_test():
+    return "pypi"
 
-    # Add the load-deb-package in the scheduler backend
-    lister.scheduler.create_task_type(
+
+@pytest.fixture
+def lister_pypi(swh_lister):
+    # Add the load-pypi in the scheduler backend
+    swh_lister.scheduler.create_task_type(
         {
             "type": "load-pypi",
             "description": "Load PyPI package",
@@ -20,4 +23,4 @@ def lister_pypi(swh_listers):
         }
     )
 
-    return lister
+    return swh_lister

swh/lister/pytest_plugin.py

@@ -29,33 +29,15 @@ def lister_db_url(postgresql_proc, postgresql):
 
 
 @pytest.fixture
-def listers_to_instantiate():
-    """Fixture to define what listers to instantiate. Because some need dedicated setup.
-
-    """
-    return set(SUPPORTED_LISTERS) - {"launchpad"}
+def lister_under_test():
+    """Fixture to determine which lister to test"""
+    return "core"
 
 
 @pytest.fixture
-def swh_listers(
-    mock_get_scheduler, lister_db_url, swh_scheduler, listers_to_instantiate
-):
-    listers = {}
-
-    # Prepare schema for all listers
-    for lister_name in listers_to_instantiate:
-        lister = get_lister(lister_name, db_url=lister_db_url)
-        listers[lister_name] = lister
+def swh_lister(mock_get_scheduler, lister_db_url, swh_scheduler, lister_under_test):
+    assert lister_under_test in SUPPORTED_LISTERS
+    lister = get_lister(lister_under_test, db_url=lister_db_url)
     initialize(create_engine(lister_db_url), drop_tables=True)
 
-    # Add the load-archive-files expected by some listers (gnu, cran, ...)
-    swh_scheduler.create_task_type(
-        {
-            "type": "load-archive-files",
-            "description": "Load archive files.",
-            "backend_name": "swh.loader.package.tasks.LoadArchive",
-            "default_interval": "1 day",
-        }
-    )
-
-    return listers
+    return lister

swh/lister/tests/test_cli.py

@@ -7,7 +7,7 @@ import pytest
 
 
 from swh.lister.core.lister_base import ListerBase
-from swh.lister.cli import get_lister
+from swh.lister.cli import get_lister, SUPPORTED_LISTERS
 
 from .test_utils import init_db
 
@@ -20,11 +20,14 @@ def test_get_lister_wrong_input():
     assert "Invalid lister" in str(e.value)
 
 
-def test_get_lister(mock_get_scheduler, listers_to_instantiate):
+def test_get_lister(mock_get_scheduler):
     """Instantiating a supported lister should be ok
 
     """
     db_url = init_db().url()
+    # exclude listers because they need special instantiation treatment unrelated to
+    # this test (launchpad: network mock, gnu: scheduler load task)
+    listers_to_instantiate = set(SUPPORTED_LISTERS) - {"launchpad", "gnu"}
     for lister_name in listers_to_instantiate:
         lst = get_lister(lister_name, db_url)
         assert isinstance(lst, ListerBase)