From 7c6ac9df0de6c964958c9c54d0de692162321dca Mon Sep 17 00:00:00 2001 From: Nicolas Dandrimont Date: Mon, 4 Sep 2017 17:10:05 +0200 Subject: [PATCH] lister_base: cache all responses --- swh/lister/core/lister_base.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/swh/lister/core/lister_base.py b/swh/lister/core/lister_base.py index 515b4b5..9339898 100644 --- a/swh/lister/core/lister_base.py +++ b/swh/lister/core/lister_base.py @@ -260,6 +260,7 @@ class SWHListerBase(abc.ABC, config.SWHConfig): server response """ retries_left = self.MAX_RETRIES + do_cache = self.config['cache_responses'] while retries_left > 0: try: r = self.transport_request(identifier) @@ -271,6 +272,9 @@ class SWHListerBase(abc.ABC, config.SWHConfig): retries_left -= 1 continue + if do_cache: + self.save_response(r) + # detect throttling must_retry, delay = self.transport_quota_check(r) if must_retry: @@ -285,10 +289,6 @@ class SWHListerBase(abc.ABC, config.SWHConfig): if not retries_left: logging.warn('giving up on %s: max retries exceeded' % identifier) - do_cache = self.config['cache_responses'] - if do_cache: - self.save_response(r) - return r def db_query_equal(self, key, value):