From ccd0525c9b5f4e3dd00adfdc58b8161597c26ed1 Mon Sep 17 00:00:00 2001 From: "Antoine R. Dumont (@ardumont)" Date: Wed, 11 Jul 2018 15:50:36 +0200 Subject: [PATCH] swh.lister: Do not hardcode the index notion into parameter names --- swh/lister/core/indexing_lister.py | 14 ++++++------ swh/lister/core/paging_lister.py | 32 ++++++++++++++-------------- swh/lister/core/tasks.py | 4 ++-- swh/lister/core/tests/test_lister.py | 8 +++---- 4 files changed, 29 insertions(+), 29 deletions(-) diff --git a/swh/lister/core/indexing_lister.py b/swh/lister/core/indexing_lister.py index d086382..f76c4fc 100644 --- a/swh/lister/core/indexing_lister.py +++ b/swh/lister/core/indexing_lister.py @@ -152,23 +152,23 @@ class SWHIndexingLister(SWHListerBase): for repo in deleted_repos: repo.task_id = None - def run(self, min_index=None, max_index=None): + def run(self, min_bound=None, max_bound=None): """Main entry function. Sequentially fetches repository data from the service according to the basic outline in the class docstring, continually fetching sublists until either there is no next index reference given or the given next index is greater - than the desired max_index. + than the desired max_bound. Args: - min_index (indexable type): optional index to start from - max_index (indexable type): optional index to stop at + min_bound (indexable type): optional index to start from + max_bound (indexable type): optional index to stop at Returns: nothing """ - index = min_index or '' + index = min_bound or '' loop_count = 0 - self.min_index = min_index - self.max_index = max_index + self.min_index = min_bound + self.max_index = max_bound while self.is_within_bounds(index, self.min_index, self.max_index): logging.info('listing repos starting at %s' % index) diff --git a/swh/lister/core/paging_lister.py b/swh/lister/core/paging_lister.py index 9779975..d65d4a4 100644 --- a/swh/lister/core/paging_lister.py +++ b/swh/lister/core/paging_lister.py @@ -79,41 +79,41 @@ class PageByPageLister(SWHListerBase): # You probably don't need to override anything below this line. - def run(self, min_index=None, max_index=None): + def run(self, min_bound=None, max_bound=None): """Main entry function. Sequentially fetches repository data from the service according to the basic outline in the class docstring. Continually fetching sublists until either there - is no next index reference given or the given next index is - greater than the desired max_index. + is no next page reference given or the given next page is + greater than the desired max_page. Args: - min_index (indexable type): optional index to start from - max_index (indexable type): optional index to stop at + min_bound: optional page to start from + max_bound: optional page to stop at Returns: nothing """ - index = min_index or '' + page = min_bound or '' loop_count = 0 - self.min_index = min_index - self.max_index = max_index + self.min_page = min_bound + self.max_page = max_bound - while self.is_within_bounds(index, self.min_index, self.max_index): - logging.info('listing repos starting at %s' % index) + while self.is_within_bounds(page, self.min_page, self.max_page): + logging.info('listing repos starting at %s' % page) - response, injected_repos = self.ingest_data(index) - next_index = self.get_next_target_from_response(response) + response, injected_repos = self.ingest_data(page) + next_page = self.get_next_target_from_response(response) # termination condition - if (next_index is None) or (next_index == index): - logging.info('stopping after index %s, no next link found' % - index) + if (next_page is None) or (next_page == page): + logging.info('stopping after page %s, no next link found' % + page) break else: - index = next_index + page = next_page loop_count += 1 if loop_count == 20: diff --git a/swh/lister/core/tasks.py b/swh/lister/core/tasks.py index 4373eaa..a8305e7 100644 --- a/swh/lister/core/tasks.py +++ b/swh/lister/core/tasks.py @@ -59,7 +59,7 @@ class RangeListerTask(ListerTaskBase): """ def run_task(self, start, end, *args, **kwargs): lister = self.new_lister(*args, **kwargs) - return lister.run(min_index=start, max_index=end) + return lister.run(min_bound=start, max_bound=end) # Indexing Lister tasks derivatives (cf. {github/bitbucket}/tasks) @@ -71,7 +71,7 @@ class IndexingDiscoveryListerTask(ListerTaskBase): """ def run_task(self, *args, **kwargs): lister = self.new_lister(*args, **kwargs) - return lister.run(min_index=lister.db_last_index(), max_index=None) + return lister.run(min_bound=lister.db_last_index(), max_bound=None) class IndexingRefreshListerTask(ListerTaskBase): diff --git a/swh/lister/core/tests/test_lister.py b/swh/lister/core/tests/test_lister.py index 9d8f5e9..e4b8785 100644 --- a/swh/lister/core/tests/test_lister.py +++ b/swh/lister/core/tests/test_lister.py @@ -189,7 +189,7 @@ class HttpListerTesterBase(abc.ABC): self.disable_storage_and_scheduler(fl) self.disable_db(fl) - fl.run(min_index=1, max_index=1) # stores no results + fl.run(min_bound=1, max_bound=1) # stores no results @istest def test_fetch_one_nodb(self, http_mocker): @@ -199,7 +199,7 @@ class HttpListerTesterBase(abc.ABC): self.disable_storage_and_scheduler(fl) self.disable_db(fl) - fl.run(min_index=self.first_index, max_index=self.first_index) + fl.run(min_bound=self.first_index, max_bound=self.first_index) @istest def test_fetch_multiple_pages_nodb(self, http_mocker): @@ -209,7 +209,7 @@ class HttpListerTesterBase(abc.ABC): self.disable_storage_and_scheduler(fl) self.disable_db(fl) - fl.run(min_index=self.first_index) + fl.run(min_bound=self.first_index) def init_db(self, db, model): engine = create_engine(db.url()) @@ -231,7 +231,7 @@ class HttpListerTesterBase(abc.ABC): # did not succeed yet if not hasattr(fl, 'db_last_index'): # gitlab lister cannot pass here return - fl.run(min_index=self.first_index) + fl.run(min_bound=self.first_index) self.assertEqual(fl.db_last_index(), self.last_index) partitions = fl.db_partition_indices(5)