gitlab.lister: Add integration test which checks scheduled tasks

Related T2032

swh/lister/gitlab/lister.py

@@ -1,4 +1,5 @@
-# Copyright (C) 2018-2019 the Software Heritage developers
+# Copyright (C) 2018-2019 The Software Heritage developers
+# See the AUTHORS file at the top-level directory of this distribution
 # License: GNU General Public License version 3, or any later version
 # See top-level LICENSE file for more information
 

swh/lister/gitlab/tests/conftest.py

@@ -1 +1,6 @@
+# Copyright (C) 2019 The Software Heritage developers
+# See the AUTHORS file at the top-level directory of this distribution
+# License: GNU General Public License version 3, or any later version
+# See top-level LICENSE file for more information
+
 from swh.lister.core.tests.conftest import *  # noqa

swh/lister/gitlab/tests/data/gitlab.com/api_response.json

@@ -0,0 +1 @@
+api_v4__projects,page=0,order_by=id,sort=asc,per_page=20

swh/lister/gitlab/tests/test_lister.py

@@ -1,7 +1,9 @@
-# Copyright (C) 2017-2019 the Software Heritage developers
+# Copyright (C) 2017-2019 The Software Heritage developers
+# See the AUTHORS file at the top-level directory of this distribution
 # License: GNU General Public License version 3, or any later version
 # See top-level LICENSE file for more information
 
+import logging
 import re
 import unittest
 from datetime import datetime, timedelta
@@ -10,12 +12,15 @@ from swh.lister.core.tests.test_lister import HttpListerTesterBase
 from swh.lister.gitlab.lister import GitLabLister
 
 
+logger = logging.getLogger(__name__)
+
+
 class GitLabListerTester(HttpListerTesterBase, unittest.TestCase):
     Lister = GitLabLister
     test_re = re.compile(r'^.*/projects.*page=(\d+).*')
     lister_subdir = 'gitlab'
-    good_api_response_file = 'api_response.json'
-    bad_api_response_file = 'api_empty_response.json'
+    good_api_response_file = 'data/gitlab.com/api_response.json'
+    bad_api_response_file = 'data/gitlab.com/api_empty_response.json'
     first_index = 1
     entries_per_page = 10
     convert_type = int
@@ -36,3 +41,28 @@ class GitLabListerTester(HttpListerTesterBase, unittest.TestCase):
         one_second = int((datetime.now() + timedelta(seconds=1.5)).timestamp())
         context.headers['RateLimit-Reset'] = str(one_second)
         return '{"error":"dummy"}'
+
+
+def test_lister_gitlab(swh_listers, requests_mock_datadir):
+    lister = swh_listers['gitlab']
+
+    lister.run()
+
+    r = lister.scheduler.search_tasks(task_type='load-git')
+    assert len(r) == 10
+
+    for row in r:
+        assert row['type'] == 'load-git'
+        # arguments check
+        args = row['arguments']['args']
+        assert len(args) == 1
+
+        url = args[0]
+        assert url.startswith('https://gitlab.com')
+
+        # kwargs
+        kwargs = row['arguments']['kwargs']
+        assert kwargs == {}
+
+        assert row['policy'] == 'recurring'
+        assert row['priority'] is None