reonokiy/swh-lister
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

swh.lister: Do not hardcode the index notion into parameter names

Browse source
This commit is contained in:
Antoine R. Dumont (@ardumont) 2018-07-11 15:50:36 +02:00
parent b6c5865ab1
commit ccd0525c9b
No known key found for this signature in database
GPG key ID: 52E2E9840D10C3B8
4 changed files with 29 additions and 29 deletions
Download patch file Download diff file Expand all files Collapse all files

14
swh/lister/core/indexing_lister.py
View file

@ -152,23 +152,23 @@ class SWHIndexingLister(SWHListerBase):
for repo in deleted_repos:
repo.task_id = None
def run(self, min_index=None, max_index=None):
def run(self, min_bound=None, max_bound=None):
"""Main entry function. Sequentially fetches repository data
from the service according to the basic outline in the class
docstring, continually fetching sublists until either there
is no next index reference given or the given next index is greater
than the desired max_index.
than the desired max_bound.
Args:
min_index (indexable type): optional index to start from
max_index (indexable type): optional index to stop at
min_bound (indexable type): optional index to start from
max_bound (indexable type): optional index to stop at
Returns:
nothing
"""
index = min_index or ''
index = min_bound or ''
loop_count = 0
self.min_index = min_index
self.max_index = max_index
self.min_index = min_bound
self.max_index = max_bound
while self.is_within_bounds(index, self.min_index, self.max_index):
logging.info('listing repos starting at %s' % index)

32
swh/lister/core/paging_lister.py
View file

@ -79,41 +79,41 @@ class PageByPageLister(SWHListerBase):
# You probably don't need to override anything below this line.
def run(self, min_index=None, max_index=None):
def run(self, min_bound=None, max_bound=None):
"""Main entry function. Sequentially fetches repository data from the
service according to the basic outline in the class
docstring. Continually fetching sublists until either there
is no next index reference given or the given next index is
greater than the desired max_index.
is no next page reference given or the given next page is
greater than the desired max_page.
Args:
min_index (indexable type): optional index to start from
max_index (indexable type): optional index to stop at
min_bound: optional page to start from
max_bound: optional page to stop at
Returns:
nothing
"""
index = min_index or ''
page = min_bound or ''
loop_count = 0
self.min_index = min_index
self.max_index = max_index
self.min_page = min_bound
self.max_page = max_bound
while self.is_within_bounds(index, self.min_index, self.max_index):
logging.info('listing repos starting at %s' % index)
while self.is_within_bounds(page, self.min_page, self.max_page):
logging.info('listing repos starting at %s' % page)
response, injected_repos = self.ingest_data(index)
next_index = self.get_next_target_from_response(response)
response, injected_repos = self.ingest_data(page)
next_page = self.get_next_target_from_response(response)
# termination condition
if (next_index is None) or (next_index == index):
logging.info('stopping after index %s, no next link found' %
index)
if (next_page is None) or (next_page == page):
logging.info('stopping after page %s, no next link found' %
page)
break
else:
index = next_index
page = next_page
loop_count += 1
if loop_count == 20:

4
swh/lister/core/tasks.py
View file

@ -59,7 +59,7 @@ class RangeListerTask(ListerTaskBase):
"""
def run_task(self, start, end, *args, **kwargs):
lister = self.new_lister(*args, **kwargs)
return lister.run(min_index=start, max_index=end)
return lister.run(min_bound=start, max_bound=end)
# Indexing Lister tasks derivatives (cf. {github/bitbucket}/tasks)
@ -71,7 +71,7 @@ class IndexingDiscoveryListerTask(ListerTaskBase):
"""
def run_task(self, *args, **kwargs):
lister = self.new_lister(*args, **kwargs)
return lister.run(min_index=lister.db_last_index(), max_index=None)
return lister.run(min_bound=lister.db_last_index(), max_bound=None)
class IndexingRefreshListerTask(ListerTaskBase):

8
swh/lister/core/tests/test_lister.py
View file

@ -189,7 +189,7 @@ class HttpListerTesterBase(abc.ABC):
self.disable_storage_and_scheduler(fl)
self.disable_db(fl)
fl.run(min_index=1, max_index=1) # stores no results
fl.run(min_bound=1, max_bound=1) # stores no results
@istest
def test_fetch_one_nodb(self, http_mocker):
@ -199,7 +199,7 @@ class HttpListerTesterBase(abc.ABC):
self.disable_storage_and_scheduler(fl)
self.disable_db(fl)
fl.run(min_index=self.first_index, max_index=self.first_index)
fl.run(min_bound=self.first_index, max_bound=self.first_index)
@istest
def test_fetch_multiple_pages_nodb(self, http_mocker):
@ -209,7 +209,7 @@ class HttpListerTesterBase(abc.ABC):
self.disable_storage_and_scheduler(fl)
self.disable_db(fl)
fl.run(min_index=self.first_index)
fl.run(min_bound=self.first_index)
def init_db(self, db, model):
engine = create_engine(db.url())
@ -231,7 +231,7 @@ class HttpListerTesterBase(abc.ABC):
# did not succeed yet
if not hasattr(fl, 'db_last_index'): # gitlab lister cannot pass here
return
fl.run(min_index=self.first_index)
fl.run(min_bound=self.first_index)
self.assertEqual(fl.db_last_index(), self.last_index)
partitions = fl.db_partition_indices(5)