reonokiy/swh-lister
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

swh.lister.gitlab: Make rate limit check optional

Browse source 
Samples:
- https://0xacab.org/api/v4/projects/
- https://framagit.org/api/v4/projects/
- https://salsa.debian.org/api/v4/projects/
- https://gitlab.com/api/v4/projects/
- https://gitlab.freedesktop.org/api/v4/projects/
- https://gitlab.gnome.org/api/v4/projects/
- https://gitlab.inria.fr/api/v4/projects/

Related T989
This commit is contained in:
Antoine R. Dumont (@ardumont) 2018-07-04 14:51:10 +02:00
parent 79cd00737f
commit 3ca566776f
No known key found for this signature in database
GPG key ID: 52E2E9840D10C3B8
1 changed files with 8 additions and 6 deletions
Download patch file Download diff file Expand all files Collapse all files

14
swh/lister/gitlab/lister.py
View file

@ -87,14 +87,16 @@ class GitLabLister(SWHPagingHttpLister):
}
def transport_quota_check(self, response):
"""Deal with rate limit
"""Deal with rate limit if any.
"""
reqs_remaining = int(response.headers['RateLimit-Remaining'])
if response.status_code == 403 and reqs_remaining == 0:
reset_at = int(response.headers['RateLimit-Reset'])
delay = min(reset_at - time.time(), 3600)
return True, delay
# not all gitlab instance have rate limit
if 'RateLimit-Remaining' in response.headers:
reqs_remaining = int(response.headers['RateLimit-Remaining'])
if response.status_code == 403 and reqs_remaining == 0:
reset_at = int(response.headers['RateLimit-Reset'])
delay = min(reset_at - time.time(), 3600)
return True, delay
return False, 0
def get_next_target_from_response(self, response):