diff --git a/swh/lister/core/tests/test_lister.py b/swh/lister/core/tests/test_lister.py index f1ea8d7..b7ae9e5 100644 --- a/swh/lister/core/tests/test_lister.py +++ b/swh/lister/core/tests/test_lister.py @@ -9,9 +9,9 @@ from unittest.mock import Mock, patch import requests_mock from sqlalchemy import create_engine -from testing.postgresql import Postgresql from swh.lister.core.abstractattribute import AbstractAttribute +from swh.lister.tests.test_utils import init_db def noop(*args, **kwargs): @@ -166,9 +166,7 @@ class HttpListerTester(HttpListerTesterBase, abc.ABC): @requests_mock.Mocker() def test_fetch_multiple_pages_yesdb(self, http_mocker): http_mocker.get(self.test_re, text=self.mock_response) - initdb_args = Postgresql.DEFAULT_SETTINGS['initdb_args'] - initdb_args = ' '.join([initdb_args, '-E UTF-8']) - db = Postgresql(initdb_args=initdb_args) + db = init_db() fl = self.get_fl(override_config={ 'lister': { diff --git a/swh/lister/tests/test_cli.py b/swh/lister/tests/test_cli.py new file mode 100644 index 0000000..d0b9c50 --- /dev/null +++ b/swh/lister/tests/test_cli.py @@ -0,0 +1,95 @@ +# Copyright (C) 2019 The Software Heritage developers +# See the AUTHORS file at the top-level directory of this distribution +# License: GNU General Public License version 3, or any later version +# See top-level LICENSE file for more information + +import pytest + +from swh.lister.core.lister_base import ListerBase +from swh.lister.cli import new_lister, SUPPORTED_LISTERS, DEFAULT_BASEURLS + +from .test_utils import init_db + + +def test_new_lister_wrong_input(): + """Unsupported lister should raise""" + with pytest.raises(ValueError) as e: + new_lister('unknown', 'db-url') + + assert "Invalid lister" in str(e.value) + + +def test_new_lister(): + """Instantiating a supported lister should be ok + + """ + db_url = init_db().url() + supported_listers_with_init = {'npm', 'debian'} + supported_listers = set(SUPPORTED_LISTERS) - supported_listers_with_init + for lister_name in supported_listers: + lst, drop_fn, init_fn, insert_data_fn = new_lister(lister_name, db_url) + + assert isinstance(lst, ListerBase) + assert drop_fn is None + assert init_fn is not None + assert insert_data_fn is None + + for lister_name in supported_listers_with_init: + lst, drop_fn, init_fn, insert_data_fn = new_lister(lister_name, db_url) + + assert isinstance(lst, ListerBase) + assert drop_fn is None + assert init_fn is not None + assert insert_data_fn is not None + + for lister_name in supported_listers_with_init: + lst, drop_fn, init_fn, insert_data_fn = new_lister(lister_name, db_url, + drop_tables=True) + + assert isinstance(lst, ListerBase) + assert drop_fn is not None + assert init_fn is not None + assert insert_data_fn is not None + + +def test_new_lister_override(): + """Overriding the lister configuration should populate its config + + """ + db_url = init_db().url() + + listers = { + 'gitlab': ('api_baseurl', 'https://gitlab.uni/api/v4/'), + 'phabricator': ('forge_url', 'https://somewhere.org'), + 'cgit': ('url_prefix', 'https://some-cgit.eu/'), + } + + # check the override ends up defined in the lister + for lister_name, (url_key, url_value) in listers.items(): + lst, drop_fn, init_fn, insert_data_fn = new_lister( + lister_name, db_url, **{ + 'api_baseurl': url_value, + 'priority': 'high', + 'policy': 'oneshot', + }) + + assert getattr(lst, url_key) == url_value + assert lst.config['priority'] == 'high' + assert lst.config['policy'] == 'oneshot' + + # check the default urls are used and not the override (since it's not + # passed) + for lister_name, (url_key, url_value) in listers.items(): + lst, drop_fn, init_fn, insert_data_fn = new_lister(lister_name, db_url) + + # no override so this does not end up in lister's configuration + assert url_key not in lst.config + + # then the default base url is used + default_url = DEFAULT_BASEURLS[lister_name] + if isinstance(default_url, tuple): # cgit implementation detail... + default_url = default_url[1] + + assert getattr(lst, url_key) == default_url + assert 'priority' not in lst.config + assert 'oneshot' not in lst.config diff --git a/swh/lister/tests/test_utils.py b/swh/lister/tests/test_utils.py index 5d9f476..1fe7e7a 100644 --- a/swh/lister/tests/test_utils.py +++ b/swh/lister/tests/test_utils.py @@ -1,9 +1,11 @@ -# Copyright (C) 2018 the Software Heritage developers +# Copyright (C) 2018-2019 the Software Heritage developers # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import unittest +from testing.postgresql import Postgresql + from swh.lister import utils @@ -22,3 +24,15 @@ class UtilsTest(unittest.TestCase): with self.assertRaises(TypeError): list(utils.split_range(100, None)) + + +def init_db(): + """Factorize the db_url instantiation + + Returns: + db object to ease db manipulation + + """ + initdb_args = Postgresql.DEFAULT_SETTINGS['initdb_args'] + initdb_args = ' '.join([initdb_args, '-E UTF-8']) + return Postgresql(initdb_args=initdb_args)