reonokiy/swh-lister
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

lister_base: cache all responses

Browse source
This commit is contained in:
Nicolas Dandrimont 2017-09-04 17:10:05 +02:00
parent 3c7cfe7d78
commit 7c6ac9df0d
1 changed files with 4 additions and 4 deletions
Download patch file Download diff file Expand all files Collapse all files

8
swh/lister/core/lister_base.py
View file

@ -260,6 +260,7 @@ class SWHListerBase(abc.ABC, config.SWHConfig):
server response
"""
retries_left = self.MAX_RETRIES
do_cache = self.config['cache_responses']
while retries_left > 0:
try:
r = self.transport_request(identifier)
@ -271,6 +272,9 @@ class SWHListerBase(abc.ABC, config.SWHConfig):
retries_left -= 1
continue
if do_cache:
self.save_response(r)
# detect throttling
must_retry, delay = self.transport_quota_check(r)
if must_retry:
@ -285,10 +289,6 @@ class SWHListerBase(abc.ABC, config.SWHConfig):
if not retries_left:
logging.warn('giving up on %s: max retries exceeded' % identifier)
do_cache = self.config['cache_responses']
if do_cache:
self.save_response(r)
return r
def db_query_equal(self, key, value):