reonokiy/swh-lister
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Small refactoring in indexing_lister

Browse source 
- use a generator instead of a while loop
- declare a local logger and use it instead of the root logger.
This commit is contained in:
David Douard 2018-12-21 16:36:25 +01:00
parent 0583b0e685
commit dd104759ae
1 changed files with 23 additions and 26 deletions
Download patch file Download diff file Expand all files Collapse all files

49
swh/lister/core/indexing_lister.py
View file

@ -4,12 +4,15 @@
import abc
import logging
from itertools import count
from sqlalchemy import func
from .lister_transports import SWHListerHttpTransport
from .lister_base import SWHListerBase
logger = logging.getLogger(__name__)
class SWHIndexingLister(SWHListerBase):
"""Lister* intermediate class for any service that follows the pattern:
@ -165,39 +168,33 @@ class SWHIndexingLister(SWHListerBase):
Returns:
nothing
"""
index = min_bound or ''
loop_count = 0
self.min_index = min_bound
self.max_index = max_bound
while self.is_within_bounds(index, self.min_index, self.max_index):
logging.info('listing repos starting at %s' % index)
def ingest_indexes():
index = min_bound or ''
for i in count(1):
response, injected_repos = self.ingest_data(index)
if not response and not injected_repos:
logger.info('No response from api server, stopping')
return
response, injected_repos = self.ingest_data(index)
if not response and not injected_repos:
logging.info('No response from api server, stopping')
break
next_index = self.get_next_target_from_response(response)
# Determine if any repos were deleted, and disable their tasks.
keep_these = list(injected_repos.keys())
self.disable_deleted_repo_tasks(index, next_index, keep_these)
next_index = self.get_next_target_from_response(response)
# Determine if any repos were deleted, and disable their tasks.
keep_these = [k for k in injected_repos.keys()]
self.disable_deleted_repo_tasks(index, next_index, keep_these)
# termination condition
if (next_index is None) or (next_index == index):
logging.info('stopping after index %s, no next link found' %
index)
break
else:
# termination condition
if next_index is None or next_index == index:
logger.info('stopping after index %s, no next link found' %
index)
return
index = next_index
yield i
loop_count += 1
if loop_count == 20:
logging.info('flushing updates')
loop_count = 0
for i in ingest_indexes():
if (i % 20) == 0:
logger.info('flushing updates')
self.db_session.commit()
self.db_session = self.mk_session()