Use generic HTTP retry policy by default and rename dedicated decorator

Instead of retrying HTTP requests only for 429 status code by default,
prefer to use the generic retry policy enabling to also retry for status
codes >= 500 but also on ConnectionError exceptions.

Rename throttling_retry decorator to http_retry to reflect this change.
This commit is contained in:
Antoine Lambert 2022-09-21 16:56:34 +02:00
parent 9b3e565cf7
commit 9c55acd286
23 changed files with 71 additions and 62 deletions

View file

@ -16,7 +16,7 @@ import requests
from tenacity.before_sleep import before_sleep_log
from swh.core.github.utils import GitHubSession
from swh.lister.utils import throttling_retry
from swh.lister.utils import http_retry
from swh.scheduler.interface import SchedulerInterface
from swh.scheduler.model import ListedOrigin
@ -112,7 +112,7 @@ class MavenLister(Lister[MavenListerState, RepoPage]):
def state_to_dict(self, state: MavenListerState) -> Dict[str, Any]:
return asdict(state)
@throttling_retry(before_sleep=before_sleep_log(logger, logging.WARNING))
@http_retry(before_sleep=before_sleep_log(logger, logging.WARNING))
def page_request(self, url: str, params: Dict[str, Any]) -> requests.Response:
logger.info("Fetching URL %s with params %s", url, params)

View file

@ -125,6 +125,11 @@ def network_requests_mock(
requests_mock.get(URL_POM_3, content=maven_pom_3)
@pytest.fixture(autouse=True)
def retry_sleep_mock(mocker):
mocker.patch.object(MavenLister.page_request.retry, "sleep")
def test_maven_full_listing(swh_scheduler):
"""Covers full listing of multiple pages, checking page results and listed
origins, statelessness."""