reonokiy/swh-lister
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

github.lister: Use Retry-After header when rate limit reached

Browse source 
Following the github's documentation [1]

[1] https://developer.github.com/v3/guides/best-practices-for-integrators/#dealing-with-abuse-rate-limits

Related to T2170
This commit is contained in:
Antoine R. Dumont (@ardumont) 2020-01-16 18:43:58 +01:00
parent 4761773631
commit 4b383abc56
No known key found for this signature in database
GPG key ID: 52E2E9840D10C3B8
2 changed files with 4 additions and 9 deletions
Download patch file Download diff file Expand all files Collapse all files

6
swh/lister/github/lister.py
View file

@ -1,10 +1,9 @@
# Copyright (C) 2017-2019 The Software Heritage developers
# Copyright (C) 2017-2020 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
import re
import time
from typing import Any
@ -39,8 +38,7 @@ class GitHubLister(IndexingHttpLister):
return False, 0
reqs_remaining = int(x_rate_limit_remaining)
if response.status_code == 403 and reqs_remaining == 0:
reset_at = int(response.headers['X-RateLimit-Reset'])
delay = min(reset_at - time.time(), 3600)
delay = int(response.headers['Retry-After'])
return True, delay
return False, 0

7
swh/lister/github/tests/test_lister.py
View file

@ -1,4 +1,4 @@
# Copyright (C) 2017-2019 The Software Heritage developers
# Copyright (C) 2017-2020 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
@ -8,8 +8,6 @@ import unittest
import requests_mock
from datetime import datetime, timedelta
from swh.lister.core.tests.test_lister import HttpListerTester
from swh.lister.github.lister import GitHubLister
@ -45,8 +43,7 @@ class GitHubListerTester(HttpListerTester, unittest.TestCase):
self.rate_limit += 1
context.status_code = 403
context.headers['X-RateLimit-Remaining'] = '0'
one_second = int((datetime.now() + timedelta(seconds=1.5)).timestamp())
context.headers['X-RateLimit-Reset'] = str(one_second)
context.headers['Retry-After'] = '1' # 1 second
return '{"error":"dummy"}'
@requests_mock.Mocker()