debian.lister: Add integration test which checks scheduled tasks

Related T2032

swh/lister/debian/__init__.py

@@ -1,14 +1,32 @@
-# Copyright (C) 2019 the Software Heritage developers
+# Copyright (C) 2019 The Software Heritage developers
+# See the AUTHORS file at the top-level directory of this distribution
 # License: GNU General Public License version 3, or any later version
 # See top-level LICENSE file for more information
 
+from typing import Any, Mapping, Optional
 
-def debian_init(db_engine, override_conf=None):
+
+def debian_init(db_engine, lister=None,
+                override_conf: Optional[Mapping[str, Any]] = None,
+                distributions: Optional[str] = ['stretch', 'buster'],
+                area_names: Optional[str] = ['main', 'contrib', 'non-free']):
+    """Initialize the debian data model.
+
+    Args:
+        db_engine: SQLAlchemy manipulation database object
+        lister: Debian lister instance. None by default.
+        override_conf: Override conf to pass to instantiate a lister.
+            None by default
+        distributions: Default distribution to build
+
+
+    """
     from swh.storage.schemata.distribution import (
         Distribution, Area)
-    from .lister import DebianLister
 
-    lister = DebianLister(override_config=override_conf)
+    if lister is None:
+        from .lister import DebianLister
+        lister = DebianLister(override_config=override_conf)
 
     if not lister.db_session\
                  .query(Distribution)\
@@ -22,8 +40,8 @@ def debian_init(db_engine, override_conf=None):
         lister.db_session.add(d)
 
         areas = []
-        for distribution_name in ['stretch', 'buster']:
-            for area_name in ['main', 'contrib', 'non-free']:
+        for distribution_name in distributions:
+            for area_name in area_names:
                 areas.append(Area(
                     name='%s/%s' % (distribution_name, area_name),
                     distribution=d,
@@ -32,7 +50,7 @@ def debian_init(db_engine, override_conf=None):
         lister.db_session.commit()
 
 
-def register():
+def register() -> Mapping[str, Any]:
     from .lister import DebianLister
     return {'models': [DebianLister.MODEL],
             'lister': DebianLister,

swh/lister/debian/lister.py

@@ -1,4 +1,5 @@
-# Copyright (C) 2017 the Software Heritage developers
+# Copyright (C) 2017-2019 The Software Heritage developers
+# See the AUTHORS file at the top-level directory of this distribution
 # License: GNU General Public License version 3, or any later version
 # See top-level LICENSE file for more information
 
@@ -28,6 +29,9 @@ decompressors = {
 }
 
 
+logger = logging.getLogger(__name__)
+
+
 class DebianLister(ListerHttpTransport, ListerBase):
     MODEL = Package
     PATH_TEMPLATE = None
@@ -209,8 +213,8 @@ class DebianLister(ListerHttpTransport, ListerBase):
 
         date = date or datetime.datetime.now(tz=datetime.timezone.utc)
 
-        logging.debug('Creating snapshot for distribution %s on date %s' %
-                      (distribution, date))
+        logger.debug('Creating snapshot for distribution %s on date %s' %
+                     (distribution, date))
 
         snapshot = DistributionSnapshot(date=date, distribution=distribution)
 
@@ -222,7 +226,7 @@ class DebianLister(ListerHttpTransport, ListerBase):
 
             self.area = area
 
-            logging.debug('Processing area %s' % area)
+            logger.debug('Processing area %s' % area)
 
             _, new_area_packages = self.ingest_data(None)
             area_snapshot = AreaSnapshot(snapshot=snapshot, area=area)

swh/lister/debian/tests/conftest.py

@@ -1 +1,30 @@
+# Copyright (C) 2019 The Software Heritage developers
+# See the AUTHORS file at the top-level directory of this distribution
+# License: GNU General Public License version 3, or any later version
+# See top-level LICENSE file for more information
+
+import pytest
+
 from swh.lister.core.tests.conftest import *  # noqa
+
+from swh.lister.debian import debian_init
+
+
+@pytest.fixture
+def lister_debian(swh_listers):
+    lister = swh_listers['debian']
+
+    # Initialize the debian data model
+    debian_init(lister.db_engine, lister=lister,
+                distributions=['stretch'],
+                area_names=['main', 'contrib'])
+
+    # Add the load-deb-package in the scheduler backend
+    lister.scheduler.create_task_type({
+        'type': 'load-deb-package',
+        'description': 'Load a Debian package',
+        'backend_name': 'swh.loader.debian.tasks.LoaderDebianPackage',
+        'default_interval': '1 day',
+    })
+
+    return lister

swh/lister/debian/tests/test_lister.py

@@ -0,0 +1,35 @@
+# Copyright (C) 2019 The Software Heritage developers
+# See the AUTHORS file at the top-level directory of this distribution
+# License: GNU General Public License version 3, or any later version
+# See top-level LICENSE file for more information
+
+import logging
+
+
+logger = logging.getLogger(__name__)
+
+
+def test_lister_debian(lister_debian, datadir, requests_mock_datadir):
+    """Simple debian listing should create scheduled tasks
+
+    """
+    # Run the lister
+    lister_debian.run(distribution="Debian")
+
+    r = lister_debian.scheduler.search_tasks(task_type='load-deb-package')
+    assert len(r) == 151
+
+    for row in r:
+        assert row['type'] == 'load-deb-package'
+        # arguments check
+        args = row['arguments']['args']
+        assert len(args) == 0
+
+        # kwargs
+        kwargs = row['arguments']['kwargs']
+        assert set(kwargs.keys()) == {'origin', 'date', 'packages'}
+
+        logger.debug('kwargs: %s', kwargs)
+
+        assert row['policy'] == 'oneshot'
+        assert row['priority'] is None