reonokiy/swh-lister
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

tests: Fix mocking of sleep calls with tenacity 8.4.2

Browse source 
Latest tenacity release adds some internal changes that broke the
mocking of sleep calls in tests.

Fix it by directly mocking time.sleep (was not working previously).
This commit is contained in:
Antoine Lambert 2024-06-27 11:40:58 +02:00
parent 323e277482
commit a7607abcf9
15 changed files with 30 additions and 79 deletions
Download patch file Download diff file Expand all files Collapse all files

6
swh/lister/bitbucket/tests/test_lister.py
View file

@ -1,4 +1,4 @@
# Copyright (C) 2017-2023 The Software Heritage developers
# Copyright (C) 2017-2024 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
@ -109,8 +109,6 @@ def test_bitbucket_lister_rate_limit_hit(
lister = BitbucketLister(scheduler=swh_scheduler, page_size=10)
mocker.patch.object(lister.http_request.retry, "sleep")
stats = lister.run()
assert stats.pages == 2
@ -200,8 +198,6 @@ def test_bitbucket_lister_buggy_page(
lister = BitbucketLister(scheduler=swh_scheduler, page_size=10)
mocker.patch.object(lister.http_request.retry, "sleep")
stats = lister.run()
assert stats.pages == 2

4
swh/lister/cgit/tests/test_lister.py
View file

@ -1,4 +1,4 @@
# Copyright (C) 2019-2023 The Software Heritage developers
# Copyright (C) 2019-2024 The Software Heritage developers
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
@ -276,8 +276,6 @@ def test_lister_cgit_get_pages_with_pages_and_retry(
lister_cgit = CGitLister(swh_scheduler, url=url)
mocker.patch.object(lister_cgit.http_request.retry, "sleep")
repos: List[List[str]] = list(lister_cgit.get_pages())
flattened_repos = sum(repos, [])
# we should have 16 repos (listed on 3 pages)

4
swh/lister/gitiles/tests/test_lister.py
View file

@ -1,4 +1,4 @@
# Copyright (C) 2023 The Software Heritage developers
# Copyright (C) 2023-2024 The Software Heritage developers
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
@ -104,7 +104,5 @@ def test_lister_gitiles_get_pages_with_pages_and_retry(
lister_gitiles = GitilesLister(swh_scheduler, url=url)
mocker.patch.object(lister_gitiles.http_request.retry, "sleep")
pages: List[str] = list(lister_gitiles.get_pages())
assert len(pages) == 7

14
swh/lister/gitlab/tests/test_lister.py
View file

@ -1,4 +1,4 @@
# Copyright (C) 2017-2022 The Software Heritage developers
# Copyright (C) 2017-2024 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
@ -212,7 +212,9 @@ def test_lister_gitlab_incremental(swh_scheduler, requests_mock, datadir):
assert listed_origin.last_update is not None
def test_lister_gitlab_rate_limit(swh_scheduler, requests_mock, datadir, mocker):
def test_lister_gitlab_rate_limit(
swh_scheduler, requests_mock, datadir, mocker, mock_sleep
):
"""Gitlab lister supports rate-limit"""
instance = "gite.lirmm.fr"
url = api_url(instance)
@ -240,9 +242,6 @@ def test_lister_gitlab_rate_limit(swh_scheduler, requests_mock, datadir, mocker)
additional_matcher=_match_request,
)
# To avoid this test being too slow, we mock sleep within the retry behavior
mock_sleep = mocker.patch.object(lister.get_page_result.retry, "sleep")
listed_result = lister.run()
expected_nb_origins = len(response1) + len(response2)
@ -253,7 +252,7 @@ def test_lister_gitlab_rate_limit(swh_scheduler, requests_mock, datadir, mocker)
@pytest.mark.parametrize("status_code", [502, 503, 520])
def test_lister_gitlab_http_errors(
swh_scheduler, requests_mock, datadir, mocker, status_code
swh_scheduler, requests_mock, datadir, mocker, status_code, mock_sleep
):
"""Gitlab lister should retry requests when encountering HTTP 50x errors"""
instance = "gite.lirmm.fr"
@ -281,9 +280,6 @@ def test_lister_gitlab_http_errors(
additional_matcher=_match_request,
)
# To avoid this test being too slow, we mock sleep within the retry behavior
mock_sleep = mocker.patch.object(lister.get_page_result.retry, "sleep")
listed_result = lister.run()
expected_nb_origins = len(response1) + len(response2)

4
swh/lister/gitweb/tests/test_lister.py
View file

@ -1,4 +1,4 @@
# Copyright (C) 2023 The Software Heritage developers
# Copyright (C) 2023-2024 The Software Heritage developers
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
@ -105,8 +105,6 @@ def test_lister_gitweb_get_pages_with_pages_and_retry(
lister_gitweb = GitwebLister(swh_scheduler, url=url)
mocker.patch.object(lister_gitweb.http_request.retry, "sleep")
pages: List[List[str]] = list(lister_gitweb.get_pages())
flattened_repos = sum(pages, [])
assert len(pages) == 1

9
swh/lister/golang/tests/test_lister.py
View file

@ -1,4 +1,4 @@
# Copyright (C) 2022-2023 The Software Heritage developers
# Copyright (C) 2022-2024 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
@ -97,10 +97,7 @@ def _generate_responses(datadir, requests_mock):
requests_mock.get(GolangLister.GOLANG_MODULES_INDEX_URL, responses)
def test_golang_lister(swh_scheduler, mocker, requests_mock, datadir):
# Exponential retries take a long time, so stub time.sleep
mocked_sleep = mocker.patch.object(GolangLister.http_request.retry, "sleep")
def test_golang_lister(swh_scheduler, mocker, requests_mock, datadir, mock_sleep):
# first listing, should return one origin per package
lister = GolangLister(scheduler=swh_scheduler)
@ -126,7 +123,7 @@ def test_golang_lister(swh_scheduler, mocker, requests_mock, datadir):
# Test `time.sleep` is called with exponential retries
assert_sleep_calls(
mocker, mocked_sleep, [1, WAIT_EXP_BASE, 1, WAIT_EXP_BASE, 1, WAIT_EXP_BASE]
mocker, mock_sleep, [1, WAIT_EXP_BASE, 1, WAIT_EXP_BASE, 1, WAIT_EXP_BASE]
)
# doing it all again (without incremental) should give us the same result

4
swh/lister/launchpad/tests/test_lister.py
View file

@ -1,4 +1,4 @@
# Copyright (C) 2020-2022 The Software Heritage developers
# Copyright (C) 2020-2024 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
@ -234,8 +234,6 @@ def test_launchpad_lister_raise_during_listing(
swh_scheduler, mocker, launchpad_response1, launchpad_bzr_response
):
lister = LaunchpadLister(scheduler=swh_scheduler)
# Exponential retries take a long time, so stub time.sleep
mocker.patch.object(lister._page_request.retry, "sleep")
mock_getRepositories, mock_getBranches = _mock_launchpad(
mocker,

7
swh/lister/maven/tests/test_lister.py
View file

@ -1,4 +1,4 @@
# Copyright (C) 2021-2022 The Software Heritage developers
# Copyright (C) 2021-2024 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
@ -80,11 +80,6 @@ def network_requests_mock(requests_mock, requests_mock_datadir, maven_index_full
requests_mock.get(INDEX_URL, content=maven_index_full)
@pytest.fixture(autouse=True)
def retry_sleep_mock(mocker):
mocker.patch.object(MavenLister.http_request.retry, "sleep")
def test_maven_full_listing(swh_scheduler):
"""Covers full listing of multiple pages, checking page results and listed
origins, statelessness."""

7
swh/lister/npm/tests/test_lister.py
View file

@ -1,4 +1,4 @@
# Copyright (C) 2018-2022 The Software Heritage developers
# Copyright (C) 2018-2024 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
@ -35,11 +35,6 @@ def npm_incremental_listing_page2(datadir):
return json.loads(Path(datadir, "npm_incremental_page2.json").read_text())
@pytest.fixture(autouse=True)
def retry_sleep_mock(mocker):
mocker.patch.object(NpmLister.http_request.retry, "sleep")
def _check_listed_npm_packages(lister, packages, scheduler_origins):
for package in packages:
package_name = package["doc"]["name"]

7
swh/lister/phabricator/tests/test_lister.py
View file

@ -1,4 +1,4 @@
# Copyright (C) 2019-2022 The Software Heritage developers
# Copyright (C) 2019-2024 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
@ -27,11 +27,6 @@ def phabricator_repositories_page2(datadir):
)
@pytest.fixture(autouse=True)
def retry_sleep_mock(mocker):
mocker.patch.object(PhabricatorLister.http_request.retry, "sleep")
def test_get_repo_url(phabricator_repositories_page1):
repos = phabricator_repositories_page1["result"]["data"]
for repo in repos:

20
swh/lister/sourceforge/tests/test_lister.py
View file

@ -1,4 +1,4 @@
# Copyright (C) 2021-2023 The Software Heritage developers
# Copyright (C) 2021-2024 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
@ -367,12 +367,11 @@ def test_sourceforge_lister_incremental(swh_scheduler, requests_mock, datadir, m
_check_listed_origins(lister, swh_scheduler)
def test_sourceforge_lister_retry(swh_scheduler, requests_mock, mocker, datadir):
def test_sourceforge_lister_retry(
swh_scheduler, requests_mock, mocker, datadir, mock_sleep
):
lister = SourceForgeLister(scheduler=swh_scheduler)
# Exponential retries take a long time, so stub time.sleep
mocked_sleep = mocker.patch.object(lister.http_request.retry, "sleep")
requests_mock.get(
MAIN_SITEMAP_URL,
[
@ -431,18 +430,15 @@ def test_sourceforge_lister_retry(swh_scheduler, requests_mock, mocker, datadir)
_check_listed_origins(lister, swh_scheduler)
# Test `time.sleep` is called with exponential retries
assert_sleep_calls(mocker, mocked_sleep, [1, WAIT_EXP_BASE, 1, 1])
assert_sleep_calls(mocker, mock_sleep, [1, WAIT_EXP_BASE, 1, 1])
@pytest.mark.parametrize("status_code", [500, 503, 504, 403, 404])
def test_sourceforge_lister_http_error(
swh_scheduler, requests_mock, status_code, mocker
swh_scheduler, requests_mock, status_code, mocker, mock_sleep
):
lister = SourceForgeLister(scheduler=swh_scheduler)
# Exponential retries take a long time, so stub time.sleep
mocked_sleep = mocker.patch.object(lister.http_request.retry, "sleep")
requests_mock.get(MAIN_SITEMAP_URL, status_code=status_code)
with pytest.raises(HTTPError):
@ -452,7 +448,7 @@ def test_sourceforge_lister_http_error(
if status_code >= 500:
exp_retries = [1.0, 10.0, 100.0, 1000.0]
assert_sleep_calls(mocker, mocked_sleep, exp_retries)
assert_sleep_calls(mocker, mock_sleep, exp_retries)
@pytest.mark.parametrize("status_code", [500, 503, 504, 403, 404])
@ -460,8 +456,6 @@ def test_sourceforge_lister_project_error(
datadir, swh_scheduler, requests_mock, status_code, mocker
):
lister = SourceForgeLister(scheduler=swh_scheduler)
# Exponential retries take a long time, so stub time.sleep
mocker.patch.object(lister.http_request.retry, "sleep")
requests_mock.get(
MAIN_SITEMAP_URL,

4
swh/lister/stagit/tests/test_lister.py
View file

@ -1,4 +1,4 @@
# Copyright (C) 2023 The Software Heritage developers
# Copyright (C) 2023-2024 The Software Heritage developers
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
@ -97,8 +97,6 @@ def test_lister_stagit_get_pages_with_pages_and_retry(
lister_stagit = StagitLister(swh_scheduler, url=url)
mocker.patch.object(lister_stagit.http_request.retry, "sleep")
pages: List[List[str]] = list(lister_stagit.get_pages())
flattened_repos = sum(pages, [])
assert len(pages) == 1

7
swh/lister/tuleap/tests/test_lister.py
View file

@ -1,4 +1,4 @@
# Copyright (C) 2021-2022 The Software Heritage developers
# Copyright (C) 2021-2024 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
@ -92,11 +92,6 @@ def tuleap_repo_3(datadir) -> Tuple[str, Dict[str, str], List[RepoPage], List[st
return text, headers, page_results, origin_urls
@pytest.fixture(autouse=True)
def retry_sleep_mock(mocker):
mocker.patch.object(TuleapLister.http_request.retry, "sleep")
def check_listed_origins(lister_urls: List[str], scheduler_origins: List[ListedOrigin]):
"""Asserts that the two collections have the same origin URLs.