From cfd4169bd8c68d2a6fbd3287d81c694d07b4858e Mon Sep 17 00:00:00 2001
From: Nicolas Dandrimont <nicolas@dandrimont.eu>
Date: Thu, 25 Feb 2021 21:09:53 +0100
Subject: [PATCH] Retry GitHub requests on ChunkEncodingErrors

These happen, sometimes, when the connection to the GitHub server
resets, e.g. because of congestion on a slow link.
---
 swh/lister/github/lister.py | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/swh/lister/github/lister.py b/swh/lister/github/lister.py
index 1560a45..a3740a4 100644
--- a/swh/lister/github/lister.py
+++ b/swh/lister/github/lister.py
@@ -13,6 +13,7 @@ from urllib.parse import parse_qs, urlparse
 
 import iso8601
 import requests
+from tenacity import retry, retry_any, retry_if_exception_type, wait_exponential
 
 from swh.scheduler.interface import SchedulerInterface
 from swh.scheduler.model import ListedOrigin
@@ -41,6 +42,14 @@ class RateLimited(Exception):
         self.response = response
 
 
+@retry(
+    wait=wait_exponential(multiplier=1, min=4, max=10),
+    retry=retry_any(
+        # ChunkedEncodingErrors happen when the TLS connection gets reset, e.g.
+        # when running the lister on a connection with high latency
+        retry_if_exception_type(requests.exceptions.ChunkedEncodingError),
+    ),
+)
 def github_request(
     url: str, token: Optional[str] = None, session: Optional[requests.Session] = None
 ) -> requests.Response: