From 9a78c80b5cdc670e7e5dc8041e40fc4f73abf347 Mon Sep 17 00:00:00 2001 From: Stefano Zacchiroli Date: Sun, 26 Apr 2015 16:29:36 +0200 Subject: [PATCH] lister.py: handle temporary connection error increase MAX_SLEEP a bit, as it is now used for both connection-level and throttle-level retries --- ghlister/lister.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/ghlister/lister.py b/ghlister/lister.py index c4b09e4..0486998 100644 --- a/ghlister/lister.py +++ b/ghlister/lister.py @@ -18,8 +18,9 @@ from ghlister.models import Repository GH_API_URL = 'https://api.github.com' -MAX_RETRIES = 5 +MAX_RETRIES = 7 MAX_SLEEP = 3600 # 1 hour +CONN_SLEEP = 10 REPO_API_URL_RE = re.compile(r'^.*/repositories\?since=(\d+)') @@ -50,7 +51,15 @@ def gh_api_request(path, username=None, password=None, headers={}): retries_left = MAX_RETRIES while retries_left > 0: logging.debug('sending API request: %s' % path) - r = requests.get(GH_API_URL + path, **params) + try: + r = requests.get(GH_API_URL + path, **params) + except requests.exceptions.ConnectionError: + # network-level connection error, try again + logging.warn('connection error upon %s: sleep for %d seconds' % + (path, CONN_SLEEP)) + time.sleep(CONN_SLEEP) + retries_left -= 1 + continue if r.ok: # all went well, do not retry break