listers: Allow to override policy and priority for scheduled tasks

Prior to this commit, the policy and priority were hard-coded.
The default values are now the old hard-coded values.

This will allow to develop a cli to trigger forges listing with oneshot policy
and some priority tasks. Thus ingesting those faster and without manual
interventation as we currently do.

swh/lister/core/lister_base.py

@@ -391,8 +391,10 @@ class ListerBase(abc.ABC, config.SWHConfig):
             the same information in a different form
         """
         _type = 'load-%s' % origin_type
-        _policy = 'recurring'
-        return utils.create_task_dict(_type, _policy, origin_url)
+        _policy = kwargs.get('policy', 'recurring')
+        priority = kwargs.get('priority')
+        kw = {'priority': priority} if priority else {}
+        return utils.create_task_dict(_type, _policy, origin_url, **kw)
 
     def string_pattern_check(self, a, b, c=None):
         """When comparing indexable types in is_within_bounds, complex strings
@@ -460,6 +462,12 @@ class ListerBase(abc.ABC, config.SWHConfig):
         for m in models_list:
             ir = injected_repos[m['uid']]
             if not ir.task_id:
+                # Patching the model instance to add the policy/priority task
+                # scheduling
+                if 'policy' in self.config:
+                    m['policy'] = self.config['policy']
+                if 'priority' in self.config:
+                    m['priority'] = self.config['priority']
                 task_dict = self.task_dict(**m)
                 tasks[_task_key(task_dict)] = (ir, m, task_dict)
 

swh/lister/cran/lister.py

@@ -27,7 +27,8 @@ class CRANLister(SimpleLister):
         needed for the ingestion task creation.
         """
         return create_task_dict(
-            'load-%s' % origin_type, 'recurring',
+            'load-%s' % origin_type,
+            kwargs.get('policy', 'recurring'),
             kwargs.get('name'), origin_url, kwargs.get('version'),
             project_metadata=self.descriptions[kwargs.get('name')])
 

swh/lister/gnu/lister.py

@@ -32,7 +32,8 @@ class GNULister(SimpleLister):
         needed for the ingestion task creation.
         """
         return utils.create_task_dict(
-            'load-%s' % origin_type, 'recurring', kwargs.get('name'),
+            'load-%s' % origin_type, kwargs.get('policy', 'recurring'),
+            kwargs.get('name'),
             origin_url, tarballs=self.tarballs[kwargs.get('name')])
 
     def get_file(self):

swh/lister/packagist/lister.py

@@ -51,7 +51,8 @@ class PackagistLister(ListerOnePageApiTransport, SimpleLister):
         needed for the ingestion task creation.
 
         """
-        return utils.create_task_dict('load-%s' % origin_type, 'recurring',
+        return utils.create_task_dict('load-%s' % origin_type,
+                                      kwargs.get('policy', 'recurring'),
                                       kwargs.get('name'), origin_url)
 
     def list_packages(self, response):

swh/lister/pypi/lister.py

@@ -30,7 +30,7 @@ class PyPILister(ListerOnePageApiTransport, SimpleLister):
 
         """
         _type = 'load-%s' % origin_type
-        _policy = 'recurring'
+        _policy = kwargs.get('policy', 'recurring')
         project_name = kwargs.get('name')
         project_metadata_url = kwargs.get('html_url')
         return utils.create_task_dict(