diff --git a/conftest.py b/conftest.py index ea09c62..b4775dc 100644 --- a/conftest.py +++ b/conftest.py @@ -1,4 +1,4 @@ -# Copyright (C) 2020-2022 The Software Heritage developers +# Copyright (C) 2020-2024 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information @@ -13,8 +13,6 @@ os.environ["LC_ALL"] = "C.UTF-8" @pytest.fixture(autouse=True) -def tenacity_wait(mocker): - # Stops tenacity from blocking lister tests for 50x errors - from swh.lister.pattern import Lister - - mocker.patch.object(Lister.http_request.retry, "sleep") +def mock_sleep(mocker): + # Stops tenacity from blocking lister tests when retrying + return mocker.patch("time.sleep") diff --git a/requirements.txt b/requirements.txt index 6c03204..e8bf14c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,6 +10,6 @@ python_debian repomd requests setuptools -tenacity >= 6.2 +tenacity >= 8.4.2 testing.postgresql toml diff --git a/swh/lister/bitbucket/tests/test_lister.py b/swh/lister/bitbucket/tests/test_lister.py index 7a78ae2..7882d31 100644 --- a/swh/lister/bitbucket/tests/test_lister.py +++ b/swh/lister/bitbucket/tests/test_lister.py @@ -1,4 +1,4 @@ -# Copyright (C) 2017-2023 The Software Heritage developers +# Copyright (C) 2017-2024 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information @@ -109,8 +109,6 @@ def test_bitbucket_lister_rate_limit_hit( lister = BitbucketLister(scheduler=swh_scheduler, page_size=10) - mocker.patch.object(lister.http_request.retry, "sleep") - stats = lister.run() assert stats.pages == 2 @@ -200,8 +198,6 @@ def test_bitbucket_lister_buggy_page( lister = BitbucketLister(scheduler=swh_scheduler, page_size=10) - mocker.patch.object(lister.http_request.retry, "sleep") - stats = lister.run() assert stats.pages == 2 diff --git a/swh/lister/cgit/tests/test_lister.py b/swh/lister/cgit/tests/test_lister.py index 44953bb..1be417c 100644 --- a/swh/lister/cgit/tests/test_lister.py +++ b/swh/lister/cgit/tests/test_lister.py @@ -1,4 +1,4 @@ -# Copyright (C) 2019-2023 The Software Heritage developers +# Copyright (C) 2019-2024 The Software Heritage developers # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information @@ -276,8 +276,6 @@ def test_lister_cgit_get_pages_with_pages_and_retry( lister_cgit = CGitLister(swh_scheduler, url=url) - mocker.patch.object(lister_cgit.http_request.retry, "sleep") - repos: List[List[str]] = list(lister_cgit.get_pages()) flattened_repos = sum(repos, []) # we should have 16 repos (listed on 3 pages) diff --git a/swh/lister/gitiles/tests/test_lister.py b/swh/lister/gitiles/tests/test_lister.py index 2b12117..2fb20f5 100644 --- a/swh/lister/gitiles/tests/test_lister.py +++ b/swh/lister/gitiles/tests/test_lister.py @@ -1,4 +1,4 @@ -# Copyright (C) 2023 The Software Heritage developers +# Copyright (C) 2023-2024 The Software Heritage developers # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information @@ -104,7 +104,5 @@ def test_lister_gitiles_get_pages_with_pages_and_retry( lister_gitiles = GitilesLister(swh_scheduler, url=url) - mocker.patch.object(lister_gitiles.http_request.retry, "sleep") - pages: List[str] = list(lister_gitiles.get_pages()) assert len(pages) == 7 diff --git a/swh/lister/gitlab/tests/test_lister.py b/swh/lister/gitlab/tests/test_lister.py index f8dd504..4cd4b06 100644 --- a/swh/lister/gitlab/tests/test_lister.py +++ b/swh/lister/gitlab/tests/test_lister.py @@ -1,4 +1,4 @@ -# Copyright (C) 2017-2022 The Software Heritage developers +# Copyright (C) 2017-2024 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information @@ -212,7 +212,9 @@ def test_lister_gitlab_incremental(swh_scheduler, requests_mock, datadir): assert listed_origin.last_update is not None -def test_lister_gitlab_rate_limit(swh_scheduler, requests_mock, datadir, mocker): +def test_lister_gitlab_rate_limit( + swh_scheduler, requests_mock, datadir, mocker, mock_sleep +): """Gitlab lister supports rate-limit""" instance = "gite.lirmm.fr" url = api_url(instance) @@ -240,9 +242,6 @@ def test_lister_gitlab_rate_limit(swh_scheduler, requests_mock, datadir, mocker) additional_matcher=_match_request, ) - # To avoid this test being too slow, we mock sleep within the retry behavior - mock_sleep = mocker.patch.object(lister.get_page_result.retry, "sleep") - listed_result = lister.run() expected_nb_origins = len(response1) + len(response2) @@ -253,7 +252,7 @@ def test_lister_gitlab_rate_limit(swh_scheduler, requests_mock, datadir, mocker) @pytest.mark.parametrize("status_code", [502, 503, 520]) def test_lister_gitlab_http_errors( - swh_scheduler, requests_mock, datadir, mocker, status_code + swh_scheduler, requests_mock, datadir, mocker, status_code, mock_sleep ): """Gitlab lister should retry requests when encountering HTTP 50x errors""" instance = "gite.lirmm.fr" @@ -281,9 +280,6 @@ def test_lister_gitlab_http_errors( additional_matcher=_match_request, ) - # To avoid this test being too slow, we mock sleep within the retry behavior - mock_sleep = mocker.patch.object(lister.get_page_result.retry, "sleep") - listed_result = lister.run() expected_nb_origins = len(response1) + len(response2) diff --git a/swh/lister/gitweb/tests/test_lister.py b/swh/lister/gitweb/tests/test_lister.py index 30f3dc9..6353421 100644 --- a/swh/lister/gitweb/tests/test_lister.py +++ b/swh/lister/gitweb/tests/test_lister.py @@ -1,4 +1,4 @@ -# Copyright (C) 2023 The Software Heritage developers +# Copyright (C) 2023-2024 The Software Heritage developers # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information @@ -105,8 +105,6 @@ def test_lister_gitweb_get_pages_with_pages_and_retry( lister_gitweb = GitwebLister(swh_scheduler, url=url) - mocker.patch.object(lister_gitweb.http_request.retry, "sleep") - pages: List[List[str]] = list(lister_gitweb.get_pages()) flattened_repos = sum(pages, []) assert len(pages) == 1 diff --git a/swh/lister/golang/tests/test_lister.py b/swh/lister/golang/tests/test_lister.py index 94cea57..a6d24bf 100644 --- a/swh/lister/golang/tests/test_lister.py +++ b/swh/lister/golang/tests/test_lister.py @@ -1,4 +1,4 @@ -# Copyright (C) 2022-2023 The Software Heritage developers +# Copyright (C) 2022-2024 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information @@ -97,10 +97,7 @@ def _generate_responses(datadir, requests_mock): requests_mock.get(GolangLister.GOLANG_MODULES_INDEX_URL, responses) -def test_golang_lister(swh_scheduler, mocker, requests_mock, datadir): - # Exponential retries take a long time, so stub time.sleep - mocked_sleep = mocker.patch.object(GolangLister.http_request.retry, "sleep") - +def test_golang_lister(swh_scheduler, mocker, requests_mock, datadir, mock_sleep): # first listing, should return one origin per package lister = GolangLister(scheduler=swh_scheduler) @@ -126,7 +123,7 @@ def test_golang_lister(swh_scheduler, mocker, requests_mock, datadir): # Test `time.sleep` is called with exponential retries assert_sleep_calls( - mocker, mocked_sleep, [1, WAIT_EXP_BASE, 1, WAIT_EXP_BASE, 1, WAIT_EXP_BASE] + mocker, mock_sleep, [1, WAIT_EXP_BASE, 1, WAIT_EXP_BASE, 1, WAIT_EXP_BASE] ) # doing it all again (without incremental) should give us the same result diff --git a/swh/lister/launchpad/tests/test_lister.py b/swh/lister/launchpad/tests/test_lister.py index d6081bf..50e0b43 100644 --- a/swh/lister/launchpad/tests/test_lister.py +++ b/swh/lister/launchpad/tests/test_lister.py @@ -1,4 +1,4 @@ -# Copyright (C) 2020-2022 The Software Heritage developers +# Copyright (C) 2020-2024 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information @@ -234,8 +234,6 @@ def test_launchpad_lister_raise_during_listing( swh_scheduler, mocker, launchpad_response1, launchpad_bzr_response ): lister = LaunchpadLister(scheduler=swh_scheduler) - # Exponential retries take a long time, so stub time.sleep - mocker.patch.object(lister._page_request.retry, "sleep") mock_getRepositories, mock_getBranches = _mock_launchpad( mocker, diff --git a/swh/lister/maven/tests/test_lister.py b/swh/lister/maven/tests/test_lister.py index 81a57ee..5f879de 100644 --- a/swh/lister/maven/tests/test_lister.py +++ b/swh/lister/maven/tests/test_lister.py @@ -1,4 +1,4 @@ -# Copyright (C) 2021-2022 The Software Heritage developers +# Copyright (C) 2021-2024 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information @@ -80,11 +80,6 @@ def network_requests_mock(requests_mock, requests_mock_datadir, maven_index_full requests_mock.get(INDEX_URL, content=maven_index_full) -@pytest.fixture(autouse=True) -def retry_sleep_mock(mocker): - mocker.patch.object(MavenLister.http_request.retry, "sleep") - - def test_maven_full_listing(swh_scheduler): """Covers full listing of multiple pages, checking page results and listed origins, statelessness.""" diff --git a/swh/lister/npm/tests/test_lister.py b/swh/lister/npm/tests/test_lister.py index 7c4fa93..ea27778 100644 --- a/swh/lister/npm/tests/test_lister.py +++ b/swh/lister/npm/tests/test_lister.py @@ -1,4 +1,4 @@ -# Copyright (C) 2018-2022 The Software Heritage developers +# Copyright (C) 2018-2024 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information @@ -35,11 +35,6 @@ def npm_incremental_listing_page2(datadir): return json.loads(Path(datadir, "npm_incremental_page2.json").read_text()) -@pytest.fixture(autouse=True) -def retry_sleep_mock(mocker): - mocker.patch.object(NpmLister.http_request.retry, "sleep") - - def _check_listed_npm_packages(lister, packages, scheduler_origins): for package in packages: package_name = package["doc"]["name"] diff --git a/swh/lister/phabricator/tests/test_lister.py b/swh/lister/phabricator/tests/test_lister.py index acea2b7..3b10bc0 100644 --- a/swh/lister/phabricator/tests/test_lister.py +++ b/swh/lister/phabricator/tests/test_lister.py @@ -1,4 +1,4 @@ -# Copyright (C) 2019-2022 The Software Heritage developers +# Copyright (C) 2019-2024 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information @@ -27,11 +27,6 @@ def phabricator_repositories_page2(datadir): ) -@pytest.fixture(autouse=True) -def retry_sleep_mock(mocker): - mocker.patch.object(PhabricatorLister.http_request.retry, "sleep") - - def test_get_repo_url(phabricator_repositories_page1): repos = phabricator_repositories_page1["result"]["data"] for repo in repos: diff --git a/swh/lister/sourceforge/tests/test_lister.py b/swh/lister/sourceforge/tests/test_lister.py index 4261ce5..a7b767e 100644 --- a/swh/lister/sourceforge/tests/test_lister.py +++ b/swh/lister/sourceforge/tests/test_lister.py @@ -1,4 +1,4 @@ -# Copyright (C) 2021-2023 The Software Heritage developers +# Copyright (C) 2021-2024 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information @@ -367,12 +367,11 @@ def test_sourceforge_lister_incremental(swh_scheduler, requests_mock, datadir, m _check_listed_origins(lister, swh_scheduler) -def test_sourceforge_lister_retry(swh_scheduler, requests_mock, mocker, datadir): +def test_sourceforge_lister_retry( + swh_scheduler, requests_mock, mocker, datadir, mock_sleep +): lister = SourceForgeLister(scheduler=swh_scheduler) - # Exponential retries take a long time, so stub time.sleep - mocked_sleep = mocker.patch.object(lister.http_request.retry, "sleep") - requests_mock.get( MAIN_SITEMAP_URL, [ @@ -431,18 +430,15 @@ def test_sourceforge_lister_retry(swh_scheduler, requests_mock, mocker, datadir) _check_listed_origins(lister, swh_scheduler) # Test `time.sleep` is called with exponential retries - assert_sleep_calls(mocker, mocked_sleep, [1, WAIT_EXP_BASE, 1, 1]) + assert_sleep_calls(mocker, mock_sleep, [1, WAIT_EXP_BASE, 1, 1]) @pytest.mark.parametrize("status_code", [500, 503, 504, 403, 404]) def test_sourceforge_lister_http_error( - swh_scheduler, requests_mock, status_code, mocker + swh_scheduler, requests_mock, status_code, mocker, mock_sleep ): lister = SourceForgeLister(scheduler=swh_scheduler) - # Exponential retries take a long time, so stub time.sleep - mocked_sleep = mocker.patch.object(lister.http_request.retry, "sleep") - requests_mock.get(MAIN_SITEMAP_URL, status_code=status_code) with pytest.raises(HTTPError): @@ -452,7 +448,7 @@ def test_sourceforge_lister_http_error( if status_code >= 500: exp_retries = [1.0, 10.0, 100.0, 1000.0] - assert_sleep_calls(mocker, mocked_sleep, exp_retries) + assert_sleep_calls(mocker, mock_sleep, exp_retries) @pytest.mark.parametrize("status_code", [500, 503, 504, 403, 404]) @@ -460,8 +456,6 @@ def test_sourceforge_lister_project_error( datadir, swh_scheduler, requests_mock, status_code, mocker ): lister = SourceForgeLister(scheduler=swh_scheduler) - # Exponential retries take a long time, so stub time.sleep - mocker.patch.object(lister.http_request.retry, "sleep") requests_mock.get( MAIN_SITEMAP_URL, diff --git a/swh/lister/stagit/tests/test_lister.py b/swh/lister/stagit/tests/test_lister.py index 15fbdcb..dc810a0 100644 --- a/swh/lister/stagit/tests/test_lister.py +++ b/swh/lister/stagit/tests/test_lister.py @@ -1,4 +1,4 @@ -# Copyright (C) 2023 The Software Heritage developers +# Copyright (C) 2023-2024 The Software Heritage developers # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information @@ -97,8 +97,6 @@ def test_lister_stagit_get_pages_with_pages_and_retry( lister_stagit = StagitLister(swh_scheduler, url=url) - mocker.patch.object(lister_stagit.http_request.retry, "sleep") - pages: List[List[str]] = list(lister_stagit.get_pages()) flattened_repos = sum(pages, []) assert len(pages) == 1 diff --git a/swh/lister/tuleap/tests/test_lister.py b/swh/lister/tuleap/tests/test_lister.py index 722e708..0194941 100644 --- a/swh/lister/tuleap/tests/test_lister.py +++ b/swh/lister/tuleap/tests/test_lister.py @@ -1,4 +1,4 @@ -# Copyright (C) 2021-2022 The Software Heritage developers +# Copyright (C) 2021-2024 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information @@ -92,11 +92,6 @@ def tuleap_repo_3(datadir) -> Tuple[str, Dict[str, str], List[RepoPage], List[st return text, headers, page_results, origin_urls -@pytest.fixture(autouse=True) -def retry_sleep_mock(mocker): - mocker.patch.object(TuleapLister.http_request.retry, "sleep") - - def check_listed_origins(lister_urls: List[str], scheduler_origins: List[ListedOrigin]): """Asserts that the two collections have the same origin URLs.