diff --git a/swh/lister/cran/lister.py b/swh/lister/cran/lister.py index 3a46acf..fdb83e0 100644 --- a/swh/lister/cran/lister.py +++ b/swh/lister/cran/lister.py @@ -60,6 +60,7 @@ class CRANLister(SimpleLister): policy = kwargs.get('policy', 'oneshot') package = kwargs.get('name') version = kwargs.get('version') + assert origin_type == 'tar' return create_task_dict( 'load-%s' % origin_type, policy, package, origin_url, version, diff --git a/swh/lister/cran/tests/conftest.py b/swh/lister/cran/tests/conftest.py index 507fef9..34741f2 100644 --- a/swh/lister/cran/tests/conftest.py +++ b/swh/lister/cran/tests/conftest.py @@ -1 +1,23 @@ +# Copyright (C) 2019 The Software Heritage developers +# See the AUTHORS file at the top-level directory of this distribution +# License: GNU General Public License version 3, or any later version +# See top-level LICENSE file for more information + +import pytest + from swh.lister.core.tests.conftest import * # noqa + + +@pytest.fixture +def lister_cran(swh_listers): + lister = swh_listers['cran'] + + # Add the load-deb-package in the scheduler backend + lister.scheduler.create_task_type({ + 'type': 'load-tar', + 'description': 'Load archive files', + 'backend_name': 'swh.loader.package.tasks.LoadArchive', + 'default_interval': '1 day', + }) + + return lister diff --git a/swh/lister/cran/tests/test_lister.py b/swh/lister/cran/tests/test_lister.py index 688a6de..338579f 100644 --- a/swh/lister/cran/tests/test_lister.py +++ b/swh/lister/cran/tests/test_lister.py @@ -28,18 +28,16 @@ def test_cran_compute_package_url_failure(): @patch('swh.lister.cran.lister.read_cran_data') -def test_cran_lister_cran(mock_cran, datadir, swh_listers): - lister = swh_listers['cran'] - +def test_cran_lister_cran(mock_cran, datadir, lister_cran): with open(path.join(datadir, 'list-r-packages.json')) as f: data = json.loads(f.read()) mock_cran.return_value = data assert len(data) == 6 - lister.run() + lister_cran.run() - r = lister.scheduler.search_tasks(task_type='load-tar') + r = lister_cran.scheduler.search_tasks(task_type='load-tar') assert len(r) == 6 for row in r: diff --git a/swh/lister/gnu/lister.py b/swh/lister/gnu/lister.py index 0e99a58..a3e1714 100644 --- a/swh/lister/gnu/lister.py +++ b/swh/lister/gnu/lister.py @@ -49,6 +49,7 @@ class GNULister(SimpleLister): """ artifacts = self.gnu_tree.artifacts[origin_url] + assert origin_type == 'tar' return utils.create_task_dict( 'load-%s' % origin_type, kwargs.get('policy', 'oneshot'), diff --git a/swh/lister/gnu/tests/conftest.py b/swh/lister/gnu/tests/conftest.py index 507fef9..a766127 100644 --- a/swh/lister/gnu/tests/conftest.py +++ b/swh/lister/gnu/tests/conftest.py @@ -1 +1,23 @@ +# Copyright (C) 2019 The Software Heritage developers +# See the AUTHORS file at the top-level directory of this distribution +# License: GNU General Public License version 3, or any later version +# See top-level LICENSE file for more information + +import pytest + from swh.lister.core.tests.conftest import * # noqa + + +@pytest.fixture +def lister_gnu(swh_listers): + lister = swh_listers['gnu'] + + # Add the load-deb-package in the scheduler backend + lister.scheduler.create_task_type({ + 'type': 'load-tar', + 'description': 'Load archive files', + 'backend_name': 'swh.loader.package.tasks.LoadArchive', + 'default_interval': '1 day', + }) + + return lister diff --git a/swh/lister/gnu/tests/test_lister.py b/swh/lister/gnu/tests/test_lister.py index 327c73d..9176ae2 100644 --- a/swh/lister/gnu/tests/test_lister.py +++ b/swh/lister/gnu/tests/test_lister.py @@ -9,12 +9,10 @@ import logging logger = logging.getLogger(__name__) -def test_gnu_lister(swh_listers, requests_mock_datadir): - lister = swh_listers['gnu'] +def test_gnu_lister(lister_gnu, requests_mock_datadir): + lister_gnu.run() - lister.run() - - r = lister.scheduler.search_tasks(task_type='load-tar') + r = lister_gnu.scheduler.search_tasks(task_type='load-tar') assert len(r) == 383 for row in r: diff --git a/swh/lister/npm/tests/conftest.py b/swh/lister/npm/tests/conftest.py index 507fef9..a7f2433 100644 --- a/swh/lister/npm/tests/conftest.py +++ b/swh/lister/npm/tests/conftest.py @@ -1 +1,23 @@ +# Copyright (C) 2019 The Software Heritage developers +# See the AUTHORS file at the top-level directory of this distribution +# License: GNU General Public License version 3, or any later version +# See top-level LICENSE file for more information + +import pytest + from swh.lister.core.tests.conftest import * # noqa + + +@pytest.fixture +def lister_npm(swh_listers): + lister = swh_listers['npm'] + + # Add the load-deb-package in the scheduler backend + lister.scheduler.create_task_type({ + 'type': 'load-npm', + 'description': 'Load npm package', + 'backend_name': 'swh.loader.package.tasks.LoadNpm', + 'default_interval': '1 day', + }) + + return lister diff --git a/swh/lister/npm/tests/test_lister.py b/swh/lister/npm/tests/test_lister.py index 281631a..33e0ea7 100644 --- a/swh/lister/npm/tests/test_lister.py +++ b/swh/lister/npm/tests/test_lister.py @@ -77,19 +77,17 @@ def check_tasks(tasks: List[Any]): assert row['priority'] is None -def test_lister_npm_basic_listing(swh_listers, requests_mock_datadir): - lister = swh_listers['npm'] +def test_lister_npm_basic_listing(lister_npm, requests_mock_datadir): + lister_npm.run() - lister.run() - - tasks = lister.scheduler.search_tasks(task_type='load-npm') + tasks = lister_npm.scheduler.search_tasks(task_type='load-npm') assert len(tasks) == 100 check_tasks(tasks) -def test_lister_npm_listing_pagination(swh_listers, requests_mock_datadir): - lister = swh_listers['npm'] +def test_lister_npm_listing_pagination(lister_npm, requests_mock_datadir): + lister = lister_npm # Patch per page pagination lister.per_page = 10 + 1 lister.PATH_TEMPLATE = lister.PATH_TEMPLATE.replace( diff --git a/swh/lister/pypi/tests/conftest.py b/swh/lister/pypi/tests/conftest.py index 507fef9..50a4239 100644 --- a/swh/lister/pypi/tests/conftest.py +++ b/swh/lister/pypi/tests/conftest.py @@ -1 +1,23 @@ +# Copyright (C) 2019 The Software Heritage developers +# See the AUTHORS file at the top-level directory of this distribution +# License: GNU General Public License version 3, or any later version +# See top-level LICENSE file for more information + +import pytest + from swh.lister.core.tests.conftest import * # noqa + + +@pytest.fixture +def lister_pypi(swh_listers): + lister = swh_listers['pypi'] + + # Add the load-deb-package in the scheduler backend + lister.scheduler.create_task_type({ + 'type': 'load-pypi', + 'description': 'Load PyPI package', + 'backend_name': 'swh.loader.package.tasks.LoadPyPI', + 'default_interval': '1 day', + }) + + return lister diff --git a/swh/lister/pypi/tests/test_lister.py b/swh/lister/pypi/tests/test_lister.py index bcc3043..a67dd22 100644 --- a/swh/lister/pypi/tests/test_lister.py +++ b/swh/lister/pypi/tests/test_lister.py @@ -4,12 +4,10 @@ # See top-level LICENSE file for more information -def test_pypi_lister(swh_listers, requests_mock_datadir): - lister = swh_listers['pypi'] +def test_pypi_lister(lister_pypi, requests_mock_datadir): + lister_pypi.run() - lister.run() - - r = lister.scheduler.search_tasks(task_type='load-pypi') + r = lister_pypi.scheduler.search_tasks(task_type='load-pypi') assert len(r) == 4 for row in r: