diff --git a/swh/lister/github/lister.py b/swh/lister/github/lister.py index 1560a45..a3740a4 100644 --- a/swh/lister/github/lister.py +++ b/swh/lister/github/lister.py @@ -13,6 +13,7 @@ from urllib.parse import parse_qs, urlparse import iso8601 import requests +from tenacity import retry, retry_any, retry_if_exception_type, wait_exponential from swh.scheduler.interface import SchedulerInterface from swh.scheduler.model import ListedOrigin @@ -41,6 +42,14 @@ class RateLimited(Exception): self.response = response +@retry( + wait=wait_exponential(multiplier=1, min=4, max=10), + retry=retry_any( + # ChunkedEncodingErrors happen when the TLS connection gets reset, e.g. + # when running the lister on a connection with high latency + retry_if_exception_type(requests.exceptions.ChunkedEncodingError), + ), +) def github_request( url: str, token: Optional[str] = None, session: Optional[requests.Session] = None ) -> requests.Response: