base: Create a new base class for the github loader

debian/control

@@ -11,6 +11,7 @@ Build-Depends: debhelper (>= 9),
                python3-redis,
                python3-setuptools,
                python3-swh.core,
+               python3-swh.scheduler (>= 0.0.10~),
                python3-swh.storage,
                python3-vcversioner
 Standards-Version: 3.9.6
@@ -18,5 +19,7 @@ Homepage: https://forge.softwareheritage.org/diffusion/DLSGH/
 
 Package: python3-swh.lister.github
 Architecture: all
-Depends: ${misc:Depends}, ${python3:Depends}
+Depends: python3-swh.scheduler (>= 0.0.10~),
+         ${misc:Depends},
+         ${python3:Depends}
 Description: Software Heritage GitHub lister

swh/lister/github/base.py

@@ -0,0 +1,63 @@
+# Copyright (C) 2016 The Software Heritage developers
+# License: GNU General Public License version 3, or any later version
+# See top-level LICENSE file for more information
+
+from swh.core import config
+from swh.storage import get_storage
+from swh.scheduler.backend import SchedulerBackend
+
+
+class SWHLister(config.SWHConfig):
+    CONFIG_BASE_FILENAME = None
+
+    DEFAULT_CONFIG = {
+        'storage_class': ('str', 'remote_storage'),
+        'storage_args': ('list[str]', ['http://localhost:5000/']),
+
+        'scheduling_db': ('str', 'dbname=swh-scheduler'),
+    }
+
+    ADDITIONAL_CONFIG = {}
+
+    def __init__(self):
+        self.config = self.parse_config_file(
+            additional_configs=[self.ADDITIONAL_CONFIG])
+
+        self.storage = get_storage(self.config['storage_class'],
+                                   self.config['storage_args'])
+
+        self.scheduler = SchedulerBackend(
+            scheduling_db=self.config['scheduling_db'],
+        )
+
+    def create_origins(self, origins):
+        """Create the origins listed, and return their ids.
+
+        Args:
+            origins: a list of origins
+        Returns:
+            a list of origin ids
+        """
+
+        raise NotImplementedError
+
+    def create_tasks(self, tasks):
+        """Create the tasks specified, and return their ids.
+
+        Args:
+            tasks (list of dict): a list of task specifications:
+                type (str): the task type
+                arguments (dict): the arguments for the task runner
+                    args (list of str): arguments
+                    kwargs (dict str -> str): keyword arguments
+                next_run (datetime.datetime): when to schedule the next run
+        Returns:
+            a list of task ids
+        """
+        returned_tasks = self.scheduler.create_tasks(tasks)
+        id_index = self.scheduler.task_keys.index('id')
+        return [returned_task[id_index] for returned_task in returned_tasks]
+
+    def disable_tasks(self, task_ids):
+        """Disable the tasks identified by the given ids"""
+        self.scheduler.disable_tasks(task_ids)