lister.debian: Make debian init step idempotent and up-to-date

swh/lister/debian/__init__.py

@@ -3,46 +3,62 @@
 # License: GNU General Public License version 3, or any later version
 # See top-level LICENSE file for more information
 
+import logging
+
 from typing import Any, List, Mapping
 
 
+logger = logging.getLogger(__name__)
+
+
 def debian_init(db_engine,
                 override_conf: Mapping[str, Any] = {},
-                distributions: List[str] = ['stretch', 'buster'],
-                area_names: List[str] = ['main', 'contrib', 'non-free']):
+                distribution_name: str = 'Debian',
+                suites: List[str] = ['stretch', 'buster', 'bullseye'],
+                components: List[str] = ['main', 'contrib', 'non-free']):
     """Initialize the debian data model.
 
     Args:
         db_engine: SQLAlchemy manipulation database object
         override_conf: Override conf to pass to instantiate a lister
-        distributions: Default distribution to build
-
+        distribution_name: Distribution to initialize
+        suites: Default suites to register with the lister
+        components: Default components to register per suite
 
     """
-    distribution_name = 'Debian'
     from swh.lister.debian.models import Distribution, Area
     from sqlalchemy.orm import sessionmaker
     db_session = sessionmaker(bind=db_engine)()
-
-    existing_distrib = db_session \
-        .query(Distribution) \
+    distrib = db_session.query(Distribution) \
         .filter(Distribution.name == distribution_name) \
         .one_or_none()
-    if not existing_distrib:
-        distrib = Distribution(name=distribution_name,
-                               type='deb',
-                               mirror_uri='http://deb.debian.org/debian/')
+
+    if distrib is None:
+        distrib = Distribution(
+            name=distribution_name, type='deb',
+            mirror_uri='http://deb.debian.org/debian/'
+        )
         db_session.add(distrib)
 
-        for distribution_name in distributions:
-            for area_name in area_names:
-                area = Area(
-                    name='%s/%s' % (distribution_name, area_name),
-                    distribution=distrib,
-                )
-                db_session.add(area)
+    # Check the existing
+    existing_area = db_session.query(Area) \
+        .filter(Area.distribution == distrib) \
+        .all()
+    existing_area = set([a.name for a in existing_area])
 
-        db_session.commit()
+    logger.debug('Area already known: %s', ', '.join(existing_area))
+
+    # Create only the new ones
+    for suite in suites:
+        for component in components:
+            area_name = f'{suite}/{component}'
+            if area_name in existing_area:
+                logger.debug("Area '%s' already set, skipping", area_name)
+                continue
+            area = Area(name=area_name, distribution=distrib)
+            db_session.add(area)
+
+    db_session.commit()
     db_session.close()
 
 

swh/lister/debian/tests/conftest.py

@@ -19,9 +19,9 @@ def lister_debian(swh_listers):
     lister = swh_listers['debian']
 
     # Initialize the debian data model
-    debian_init(lister.db_engine,
-                distributions=['stretch'],
-                area_names=['main', 'contrib'])
+    debian_init(
+        lister.db_engine, suites=['stretch'], components=['main', 'contrib']
+    )
 
     # Add the load-deb-package in the scheduler backend
     lister.scheduler.create_task_type({

swh/lister/debian/tests/test_init.py

@@ -0,0 +1,77 @@
+# Copyright (C) 2019  The Software Heritage developers
+# See the AUTHORS file at the top-level directory of this distribution
+# License: GNU General Public License version 3, or any later version
+# See top-level LICENSE file for more information
+
+import pytest
+
+from swh.lister.debian import debian_init
+
+from swh.lister.debian.models import Distribution, Area
+
+
+@pytest.fixture
+def engine(session):
+    session.autoflush = False
+    return session.bind
+
+
+def test_debian_init_step(engine, session):
+    distribution_name = 'KaliLinux'
+
+    distrib = session.query(Distribution) \
+        .filter(Distribution.name == distribution_name) \
+        .one_or_none()
+    assert distrib is None
+
+    all_area = session.query(Area).all()
+    assert all_area == []
+
+    suites = ['wheezy', 'jessie']
+    components = ['main', 'contrib']
+
+    debian_init(engine, distribution_name=distribution_name,
+                suites=suites, components=components)
+    distrib = session.query(Distribution) \
+        .filter(Distribution.name == distribution_name) \
+        .one_or_none()
+
+    assert distrib is not None
+    assert distrib.name == distribution_name
+    assert distrib.type == 'deb'
+    assert distrib.mirror_uri == 'http://deb.debian.org/debian/'
+
+    all_area = session.query(Area).all()
+    assert len(all_area) == 2 * 2, "2 suites * 2 components per suite"
+
+    expected_area_names = []
+    for suite in suites:
+        for component in components:
+            expected_area_names.append(f'{suite}/{component}')
+
+    for area in all_area:
+        area.id = None
+        assert area.distribution == distrib
+        assert area.name in expected_area_names
+
+    # check idempotency (on exact same call)
+
+    debian_init(engine, distribution_name=distribution_name,
+                suites=suites, components=components)
+
+    distribs = session.query(Distribution) \
+        .filter(Distribution.name == distribution_name) \
+        .all()
+
+    assert len(distribs) == 1
+    distrib = distribs[0]
+
+    all_area = session.query(Area).all()
+    assert len(all_area) == 2 * 2, "2 suites * 2 components per suite"
+
+    # Add a new suite
+    debian_init(engine, distribution_name=distribution_name,
+                suites=['lenny'], components=components)
+
+    all_area = [a.name for a in session.query(Area).all()]
+    assert len(all_area) == (2 + 1) * 2, "3 suites * 2 components per suite"