reonokiy/swh-lister
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

swh.lister.gitlab: Add Incremental lister behavior

Browse source 
Related T989
This commit is contained in:
Antoine R. Dumont (@ardumont) 2018-07-06 09:26:15 +02:00
parent ccd0525c9b
commit 847a8d341a
No known key found for this signature in database
GPG key ID: 52E2E9840D10C3B8
3 changed files with 43 additions and 4 deletions
Download patch file Download diff file Expand all files Collapse all files

4
swh/lister/gitlab/lister.py
View file

@ -12,7 +12,7 @@ from .models import GitLabModel
class GitLabLister(PageByPageHttpLister):
# Template path expecting an integer that represents the page id
PATH_TEMPLATE = '/projects?page=%d&order_by=id&sort=asc&simple=true'
PATH_TEMPLATE = '/projects?page=%d&order_by=id'
API_URL_INDEX_RE = re.compile(r'^.*/projects.*page=(\d+).*')
MODEL = GitLabModel
LISTER_NAME = 'gitlab'
@ -103,7 +103,7 @@ class GitLabLister(PageByPageHttpLister):
return None
def get_pages_information(self):
"""Determine some pages information.
"""Determine pages information.
"""
response = self.transport_head(identifier=1)

24
swh/lister/gitlab/tasks.py
View file

@ -17,13 +17,16 @@ class GitLabListerTask(ListerTaskBase):
class RangeGitLabLister(GitLabListerTask, RangeListerTask):
"""GitLab lister working on specified range (start, end) arguments.
"""Range GitLab lister (list available origins on specified range)
"""
task_queue = 'swh_lister_gitlab_refresh'
class FullGitLabRelister(GitLabListerTask):
"""Full GitLab lister (list all available origins from the api).
"""
task_queue = 'swh_lister_gitlab_refresh'
def run_task(self, *args, **kwargs):
@ -41,3 +44,22 @@ class FullGitLabRelister(GitLabListerTask):
range_task = RangeGitLabLister()
group(range_task.s(minv, maxv, *args, **kwargs)
for minv, maxv in ranges)()
class IncrementalGitLabLister(ListerTaskBase):
"""Incremental GitLab lister (list only new available origins).
"""
task_queue = 'swh_lister_gitlab_discover'
def new_lister(self, api_baseurl='https://gitlab.com/api/v4',
instance='gitlab.com',):
# will invert the order of the lister's result
return GitLabLister(instance=instance, api_baseurl=api_baseurl,
sort='desc')
def run_task(self, *args, **kwargs):
lister = self.new_lister(*args, **kwargs)
# will check for existing data and exit when found
return lister.run(min_bound=None, max_bound=None,
check_existence=True)