From a7607abcf928e46b21406c4e6ab7203000c9616e Mon Sep 17 00:00:00 2001 From: Antoine Lambert Date: Thu, 27 Jun 2024 11:40:58 +0200 Subject: [PATCH] tests: Fix mocking of sleep calls with tenacity 8.4.2 Latest tenacity release adds some internal changes that broke the mocking of sleep calls in tests. Fix it by directly mocking time.sleep (was not working previously). --- conftest.py | 10 ++++------ requirements.txt | 2 +- swh/lister/bitbucket/tests/test_lister.py | 6 +----- swh/lister/cgit/tests/test_lister.py | 4 +--- swh/lister/gitiles/tests/test_lister.py | 4 +--- swh/lister/gitlab/tests/test_lister.py | 14 +++++--------- swh/lister/gitweb/tests/test_lister.py | 4 +--- swh/lister/golang/tests/test_lister.py | 9 +++------ swh/lister/launchpad/tests/test_lister.py | 4 +--- swh/lister/maven/tests/test_lister.py | 7 +------ swh/lister/npm/tests/test_lister.py | 7 +------ swh/lister/phabricator/tests/test_lister.py | 7 +------ swh/lister/sourceforge/tests/test_lister.py | 20 +++++++------------- swh/lister/stagit/tests/test_lister.py | 4 +--- swh/lister/tuleap/tests/test_lister.py | 7 +------ 15 files changed, 30 insertions(+), 79 deletions(-) diff --git a/conftest.py b/conftest.py index ea09c62..b4775dc 100644 --- a/conftest.py +++ b/conftest.py @@ -1,4 +1,4 @@ -# Copyright (C) 2020-2022 The Software Heritage developers +# Copyright (C) 2020-2024 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information @@ -13,8 +13,6 @@ os.environ["LC_ALL"] = "C.UTF-8" @pytest.fixture(autouse=True) -def tenacity_wait(mocker): - # Stops tenacity from blocking lister tests for 50x errors - from swh.lister.pattern import Lister - - mocker.patch.object(Lister.http_request.retry, "sleep") +def mock_sleep(mocker): + # Stops tenacity from blocking lister tests when retrying + return mocker.patch("time.sleep") diff --git a/requirements.txt b/requirements.txt index 6c03204..e8bf14c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,6 +10,6 @@ python_debian repomd requests setuptools -tenacity >= 6.2 +tenacity >= 8.4.2 testing.postgresql toml diff --git a/swh/lister/bitbucket/tests/test_lister.py b/swh/lister/bitbucket/tests/test_lister.py index 7a78ae2..7882d31 100644 --- a/swh/lister/bitbucket/tests/test_lister.py +++ b/swh/lister/bitbucket/tests/test_lister.py @@ -1,4 +1,4 @@ -# Copyright (C) 2017-2023 The Software Heritage developers +# Copyright (C) 2017-2024 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information @@ -109,8 +109,6 @@ def test_bitbucket_lister_rate_limit_hit( lister = BitbucketLister(scheduler=swh_scheduler, page_size=10) - mocker.patch.object(lister.http_request.retry, "sleep") - stats = lister.run() assert stats.pages == 2 @@ -200,8 +198,6 @@ def test_bitbucket_lister_buggy_page( lister = BitbucketLister(scheduler=swh_scheduler, page_size=10) - mocker.patch.object(lister.http_request.retry, "sleep") - stats = lister.run() assert stats.pages == 2 diff --git a/swh/lister/cgit/tests/test_lister.py b/swh/lister/cgit/tests/test_lister.py index 44953bb..1be417c 100644 --- a/swh/lister/cgit/tests/test_lister.py +++ b/swh/lister/cgit/tests/test_lister.py @@ -1,4 +1,4 @@ -# Copyright (C) 2019-2023 The Software Heritage developers +# Copyright (C) 2019-2024 The Software Heritage developers # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information @@ -276,8 +276,6 @@ def test_lister_cgit_get_pages_with_pages_and_retry( lister_cgit = CGitLister(swh_scheduler, url=url) - mocker.patch.object(lister_cgit.http_request.retry, "sleep") - repos: List[List[str]] = list(lister_cgit.get_pages()) flattened_repos = sum(repos, []) # we should have 16 repos (listed on 3 pages) diff --git a/swh/lister/gitiles/tests/test_lister.py b/swh/lister/gitiles/tests/test_lister.py index 2b12117..2fb20f5 100644 --- a/swh/lister/gitiles/tests/test_lister.py +++ b/swh/lister/gitiles/tests/test_lister.py @@ -1,4 +1,4 @@ -# Copyright (C) 2023 The Software Heritage developers +# Copyright (C) 2023-2024 The Software Heritage developers # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information @@ -104,7 +104,5 @@ def test_lister_gitiles_get_pages_with_pages_and_retry( lister_gitiles = GitilesLister(swh_scheduler, url=url) - mocker.patch.object(lister_gitiles.http_request.retry, "sleep") - pages: List[str] = list(lister_gitiles.get_pages()) assert len(pages) == 7 diff --git a/swh/lister/gitlab/tests/test_lister.py b/swh/lister/gitlab/tests/test_lister.py index f8dd504..4cd4b06 100644 --- a/swh/lister/gitlab/tests/test_lister.py +++ b/swh/lister/gitlab/tests/test_lister.py @@ -1,4 +1,4 @@ -# Copyright (C) 2017-2022 The Software Heritage developers +# Copyright (C) 2017-2024 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information @@ -212,7 +212,9 @@ def test_lister_gitlab_incremental(swh_scheduler, requests_mock, datadir): assert listed_origin.last_update is not None -def test_lister_gitlab_rate_limit(swh_scheduler, requests_mock, datadir, mocker): +def test_lister_gitlab_rate_limit( + swh_scheduler, requests_mock, datadir, mocker, mock_sleep +): """Gitlab lister supports rate-limit""" instance = "gite.lirmm.fr" url = api_url(instance) @@ -240,9 +242,6 @@ def test_lister_gitlab_rate_limit(swh_scheduler, requests_mock, datadir, mocker) additional_matcher=_match_request, ) - # To avoid this test being too slow, we mock sleep within the retry behavior - mock_sleep = mocker.patch.object(lister.get_page_result.retry, "sleep") - listed_result = lister.run() expected_nb_origins = len(response1) + len(response2) @@ -253,7 +252,7 @@ def test_lister_gitlab_rate_limit(swh_scheduler, requests_mock, datadir, mocker) @pytest.mark.parametrize("status_code", [502, 503, 520]) def test_lister_gitlab_http_errors( - swh_scheduler, requests_mock, datadir, mocker, status_code + swh_scheduler, requests_mock, datadir, mocker, status_code, mock_sleep ): """Gitlab lister should retry requests when encountering HTTP 50x errors""" instance = "gite.lirmm.fr" @@ -281,9 +280,6 @@ def test_lister_gitlab_http_errors( additional_matcher=_match_request, ) - # To avoid this test being too slow, we mock sleep within the retry behavior - mock_sleep = mocker.patch.object(lister.get_page_result.retry, "sleep") - listed_result = lister.run() expected_nb_origins = len(response1) + len(response2) diff --git a/swh/lister/gitweb/tests/test_lister.py b/swh/lister/gitweb/tests/test_lister.py index 30f3dc9..6353421 100644 --- a/swh/lister/gitweb/tests/test_lister.py +++ b/swh/lister/gitweb/tests/test_lister.py @@ -1,4 +1,4 @@ -# Copyright (C) 2023 The Software Heritage developers +# Copyright (C) 2023-2024 The Software Heritage developers # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information @@ -105,8 +105,6 @@ def test_lister_gitweb_get_pages_with_pages_and_retry( lister_gitweb = GitwebLister(swh_scheduler, url=url) - mocker.patch.object(lister_gitweb.http_request.retry, "sleep") - pages: List[List[str]] = list(lister_gitweb.get_pages()) flattened_repos = sum(pages, []) assert len(pages) == 1 diff --git a/swh/lister/golang/tests/test_lister.py b/swh/lister/golang/tests/test_lister.py index 94cea57..a6d24bf 100644 --- a/swh/lister/golang/tests/test_lister.py +++ b/swh/lister/golang/tests/test_lister.py @@ -1,4 +1,4 @@ -# Copyright (C) 2022-2023 The Software Heritage developers +# Copyright (C) 2022-2024 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information @@ -97,10 +97,7 @@ def _generate_responses(datadir, requests_mock): requests_mock.get(GolangLister.GOLANG_MODULES_INDEX_URL, responses) -def test_golang_lister(swh_scheduler, mocker, requests_mock, datadir): - # Exponential retries take a long time, so stub time.sleep - mocked_sleep = mocker.patch.object(GolangLister.http_request.retry, "sleep") - +def test_golang_lister(swh_scheduler, mocker, requests_mock, datadir, mock_sleep): # first listing, should return one origin per package lister = GolangLister(scheduler=swh_scheduler) @@ -126,7 +123,7 @@ def test_golang_lister(swh_scheduler, mocker, requests_mock, datadir): # Test `time.sleep` is called with exponential retries assert_sleep_calls( - mocker, mocked_sleep, [1, WAIT_EXP_BASE, 1, WAIT_EXP_BASE, 1, WAIT_EXP_BASE] + mocker, mock_sleep, [1, WAIT_EXP_BASE, 1, WAIT_EXP_BASE, 1, WAIT_EXP_BASE] ) # doing it all again (without incremental) should give us the same result diff --git a/swh/lister/launchpad/tests/test_lister.py b/swh/lister/launchpad/tests/test_lister.py index d6081bf..50e0b43 100644 --- a/swh/lister/launchpad/tests/test_lister.py +++ b/swh/lister/launchpad/tests/test_lister.py @@ -1,4 +1,4 @@ -# Copyright (C) 2020-2022 The Software Heritage developers +# Copyright (C) 2020-2024 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information @@ -234,8 +234,6 @@ def test_launchpad_lister_raise_during_listing( swh_scheduler, mocker, launchpad_response1, launchpad_bzr_response ): lister = LaunchpadLister(scheduler=swh_scheduler) - # Exponential retries take a long time, so stub time.sleep - mocker.patch.object(lister._page_request.retry, "sleep") mock_getRepositories, mock_getBranches = _mock_launchpad( mocker, diff --git a/swh/lister/maven/tests/test_lister.py b/swh/lister/maven/tests/test_lister.py index 81a57ee..5f879de 100644 --- a/swh/lister/maven/tests/test_lister.py +++ b/swh/lister/maven/tests/test_lister.py @@ -1,4 +1,4 @@ -# Copyright (C) 2021-2022 The Software Heritage developers +# Copyright (C) 2021-2024 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information @@ -80,11 +80,6 @@ def network_requests_mock(requests_mock, requests_mock_datadir, maven_index_full requests_mock.get(INDEX_URL, content=maven_index_full) -@pytest.fixture(autouse=True) -def retry_sleep_mock(mocker): - mocker.patch.object(MavenLister.http_request.retry, "sleep") - - def test_maven_full_listing(swh_scheduler): """Covers full listing of multiple pages, checking page results and listed origins, statelessness.""" diff --git a/swh/lister/npm/tests/test_lister.py b/swh/lister/npm/tests/test_lister.py index 7c4fa93..ea27778 100644 --- a/swh/lister/npm/tests/test_lister.py +++ b/swh/lister/npm/tests/test_lister.py @@ -1,4 +1,4 @@ -# Copyright (C) 2018-2022 The Software Heritage developers +# Copyright (C) 2018-2024 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information @@ -35,11 +35,6 @@ def npm_incremental_listing_page2(datadir): return json.loads(Path(datadir, "npm_incremental_page2.json").read_text()) -@pytest.fixture(autouse=True) -def retry_sleep_mock(mocker): - mocker.patch.object(NpmLister.http_request.retry, "sleep") - - def _check_listed_npm_packages(lister, packages, scheduler_origins): for package in packages: package_name = package["doc"]["name"] diff --git a/swh/lister/phabricator/tests/test_lister.py b/swh/lister/phabricator/tests/test_lister.py index acea2b7..3b10bc0 100644 --- a/swh/lister/phabricator/tests/test_lister.py +++ b/swh/lister/phabricator/tests/test_lister.py @@ -1,4 +1,4 @@ -# Copyright (C) 2019-2022 The Software Heritage developers +# Copyright (C) 2019-2024 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information @@ -27,11 +27,6 @@ def phabricator_repositories_page2(datadir): ) -@pytest.fixture(autouse=True) -def retry_sleep_mock(mocker): - mocker.patch.object(PhabricatorLister.http_request.retry, "sleep") - - def test_get_repo_url(phabricator_repositories_page1): repos = phabricator_repositories_page1["result"]["data"] for repo in repos: diff --git a/swh/lister/sourceforge/tests/test_lister.py b/swh/lister/sourceforge/tests/test_lister.py index 4261ce5..a7b767e 100644 --- a/swh/lister/sourceforge/tests/test_lister.py +++ b/swh/lister/sourceforge/tests/test_lister.py @@ -1,4 +1,4 @@ -# Copyright (C) 2021-2023 The Software Heritage developers +# Copyright (C) 2021-2024 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information @@ -367,12 +367,11 @@ def test_sourceforge_lister_incremental(swh_scheduler, requests_mock, datadir, m _check_listed_origins(lister, swh_scheduler) -def test_sourceforge_lister_retry(swh_scheduler, requests_mock, mocker, datadir): +def test_sourceforge_lister_retry( + swh_scheduler, requests_mock, mocker, datadir, mock_sleep +): lister = SourceForgeLister(scheduler=swh_scheduler) - # Exponential retries take a long time, so stub time.sleep - mocked_sleep = mocker.patch.object(lister.http_request.retry, "sleep") - requests_mock.get( MAIN_SITEMAP_URL, [ @@ -431,18 +430,15 @@ def test_sourceforge_lister_retry(swh_scheduler, requests_mock, mocker, datadir) _check_listed_origins(lister, swh_scheduler) # Test `time.sleep` is called with exponential retries - assert_sleep_calls(mocker, mocked_sleep, [1, WAIT_EXP_BASE, 1, 1]) + assert_sleep_calls(mocker, mock_sleep, [1, WAIT_EXP_BASE, 1, 1]) @pytest.mark.parametrize("status_code", [500, 503, 504, 403, 404]) def test_sourceforge_lister_http_error( - swh_scheduler, requests_mock, status_code, mocker + swh_scheduler, requests_mock, status_code, mocker, mock_sleep ): lister = SourceForgeLister(scheduler=swh_scheduler) - # Exponential retries take a long time, so stub time.sleep - mocked_sleep = mocker.patch.object(lister.http_request.retry, "sleep") - requests_mock.get(MAIN_SITEMAP_URL, status_code=status_code) with pytest.raises(HTTPError): @@ -452,7 +448,7 @@ def test_sourceforge_lister_http_error( if status_code >= 500: exp_retries = [1.0, 10.0, 100.0, 1000.0] - assert_sleep_calls(mocker, mocked_sleep, exp_retries) + assert_sleep_calls(mocker, mock_sleep, exp_retries) @pytest.mark.parametrize("status_code", [500, 503, 504, 403, 404]) @@ -460,8 +456,6 @@ def test_sourceforge_lister_project_error( datadir, swh_scheduler, requests_mock, status_code, mocker ): lister = SourceForgeLister(scheduler=swh_scheduler) - # Exponential retries take a long time, so stub time.sleep - mocker.patch.object(lister.http_request.retry, "sleep") requests_mock.get( MAIN_SITEMAP_URL, diff --git a/swh/lister/stagit/tests/test_lister.py b/swh/lister/stagit/tests/test_lister.py index 15fbdcb..dc810a0 100644 --- a/swh/lister/stagit/tests/test_lister.py +++ b/swh/lister/stagit/tests/test_lister.py @@ -1,4 +1,4 @@ -# Copyright (C) 2023 The Software Heritage developers +# Copyright (C) 2023-2024 The Software Heritage developers # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information @@ -97,8 +97,6 @@ def test_lister_stagit_get_pages_with_pages_and_retry( lister_stagit = StagitLister(swh_scheduler, url=url) - mocker.patch.object(lister_stagit.http_request.retry, "sleep") - pages: List[List[str]] = list(lister_stagit.get_pages()) flattened_repos = sum(pages, []) assert len(pages) == 1 diff --git a/swh/lister/tuleap/tests/test_lister.py b/swh/lister/tuleap/tests/test_lister.py index 722e708..0194941 100644 --- a/swh/lister/tuleap/tests/test_lister.py +++ b/swh/lister/tuleap/tests/test_lister.py @@ -1,4 +1,4 @@ -# Copyright (C) 2021-2022 The Software Heritage developers +# Copyright (C) 2021-2024 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information @@ -92,11 +92,6 @@ def tuleap_repo_3(datadir) -> Tuple[str, Dict[str, str], List[RepoPage], List[st return text, headers, page_results, origin_urls -@pytest.fixture(autouse=True) -def retry_sleep_mock(mocker): - mocker.patch.object(TuleapLister.http_request.retry, "sleep") - - def check_listed_origins(lister_urls: List[str], scheduler_origins: List[ListedOrigin]): """Asserts that the two collections have the same origin URLs.