bitbucket.lister: Add integration test which checks scheduled tasks

Related T2032

swh/lister/bitbucket/lister.py

@@ -1,4 +1,5 @@
-# Copyright (C) 2017-2019 the Software Heritage developers
+# Copyright (C) 2017-2019 The Software Heritage developers
+# See the AUTHORS file at the top-level directory of this distribution
 # License: GNU General Public License version 3, or any later version
 # See top-level LICENSE file for more information
 

swh/lister/bitbucket/tests/data/api.bitbucket.org/response.json

@@ -0,0 +1 @@
+2.0_repositories,after=1970-01-01T00%3A00%3A00%2B00%3A00,pagelen=100

swh/lister/bitbucket/tests/test_lister.py

@@ -1,4 +1,5 @@
-# Copyright (C) 2017-2019 the Software Heritage developers
+# Copyright (C) 2017-2019 The Software Heritage developers
+# See the AUTHORS file at the top-level directory of this distribution
 # License: GNU General Public License version 3, or any later version
 # See top-level LICENSE file for more information
 
@@ -28,8 +29,8 @@ class BitBucketListerTester(HttpListerTester, unittest.TestCase):
     Lister = BitBucketLister
     test_re = re.compile(r'/repositories\?after=([^?&]+)')
     lister_subdir = 'bitbucket'
-    good_api_response_file = 'api_response.json'
-    bad_api_response_file = 'api_empty_response.json'
+    good_api_response_file = 'data/api.bitbucket.org/response.json'
+    bad_api_response_file = 'data/api.bitbucket.org/empty_response.json'
     first_index = convert_type('2008-07-12T07:44:01.476818+00:00')
     last_index = convert_type('2008-07-19T06:16:43.044743+00:00')
     entries_per_page = 10
@@ -71,3 +72,31 @@ class BitBucketListerTester(HttpListerTester, unittest.TestCase):
         self.assertFalse(fl.is_within_bounds(
             iso8601.parse_date('2008-07-11'),
             self.first_index, self.last_index))
+
+
+def test_lister_bitbucket(swh_listers, requests_mock_datadir):
+    """Simple bitbucket listing should create scheduled tasks
+
+    """
+    lister = swh_listers['bitbucket']
+
+    lister.run()
+
+    r = lister.scheduler.search_tasks(task_type='load-hg')
+    assert len(r) == 10
+
+    for row in r:
+        assert row['type'] == 'load-hg'
+        # arguments check
+        args = row['arguments']['args']
+        assert len(args) == 1
+
+        url = args[0]
+        assert url.startswith('https://bitbucket.org')
+
+        # kwargs
+        kwargs = row['arguments']['kwargs']
+        assert kwargs == {}
+
+        assert row['policy'] == 'recurring'
+        assert row['priority'] is None

swh/lister/core/lister_base.py

@@ -382,6 +382,7 @@ class ListerBase(abc.ABC, config.SWHConfig):
         Returns:
             the same information in a different form
         """
+        logger.debug('origin-url: %s, type: %s', origin_url, origin_type)
         _type = 'load-%s' % origin_type
         _policy = kwargs.get('policy', 'recurring')
         priority = kwargs.get('priority')