plugins: add support for scheduler's task-type declaration
Add a new register-task-types cli that will create missing task-type entries in the scheduler according to: - only create missing task-types (do not update them), but check that the backend_name field is consistent, - each SWHTask-based task declared in a module listed in the 'task_modules' plugin registry field will be checked and added if needed; tasks which name start wit an underscore will not be added, - added task-type will have: - the 'type' field is derived from the task's function name (with underscores replaced with dashes), - the description field is the first line of that function's docstring, - default values as provided by the swh.lister.cli.DEFAULT_TASK_TYPE (with a simple pattern matching to have decent default values for full/incremental tasks), - these default values can be overloaded via the 'task_type' plugin registry entry. For this, we had to rename all tasks names (eg. `cran_lister` -> `list_cran`). Comes with some tests.
This commit is contained in:
parent
e3c0ea9d90
commit
8d9deeb8f8
14 changed files with 233 additions and 41 deletions
|
@ -22,7 +22,8 @@ def new_lister(api_baseurl='https://gitlab.com/api/v4',
|
|||
|
||||
|
||||
@app.task(name=__name__ + '.IncrementalGitLabLister')
|
||||
def incremental_gitlab_lister(**lister_args):
|
||||
def list_gitlab_incremental(**lister_args):
|
||||
"""Incremental update of a GitLab instance"""
|
||||
lister_args['sort'] = 'desc'
|
||||
lister = new_lister(**lister_args)
|
||||
total_pages = lister.get_pages_information()[1]
|
||||
|
@ -31,23 +32,19 @@ def incremental_gitlab_lister(**lister_args):
|
|||
|
||||
|
||||
@app.task(name=__name__ + '.RangeGitLabLister')
|
||||
def range_gitlab_lister(start, end, **lister_args):
|
||||
def _range_gitlab_lister(start, end, **lister_args):
|
||||
lister = new_lister(**lister_args)
|
||||
lister.run(min_bound=start, max_bound=end)
|
||||
|
||||
|
||||
@app.task(name=__name__ + '.FullGitLabRelister', bind=True)
|
||||
def full_gitlab_relister(self, **lister_args):
|
||||
"""Full lister
|
||||
|
||||
This should be renamed as such.
|
||||
|
||||
"""
|
||||
def list_gitlab_full(self, **lister_args):
|
||||
"""Full update of a GitLab instance"""
|
||||
lister = new_lister(**lister_args)
|
||||
_, total_pages, _ = lister.get_pages_information()
|
||||
ranges = list(utils.split_range(total_pages, NBPAGES))
|
||||
random.shuffle(ranges)
|
||||
promise = group(range_gitlab_lister.s(minv, maxv, **lister_args)
|
||||
promise = group(_range_gitlab_lister.s(minv, maxv, **lister_args)
|
||||
for minv, maxv in ranges)()
|
||||
self.log.debug('%s OK (spawned %s subtasks)' % (self.name, len(ranges)))
|
||||
try:
|
||||
|
@ -58,5 +55,5 @@ def full_gitlab_relister(self, **lister_args):
|
|||
|
||||
|
||||
@app.task(name=__name__ + '.ping')
|
||||
def ping():
|
||||
def _ping():
|
||||
return 'OK'
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue