From f92ac83646125b09d6c5876ebefca3d750011c31 Mon Sep 17 00:00:00 2001 From: "Antoine R. Dumont (@ardumont)" Date: Sat, 12 Oct 2019 03:39:47 +0200 Subject: [PATCH] bitbucket.lister: Add integration test which checks scheduled tasks Related T2032 --- swh/lister/bitbucket/lister.py | 3 +- ...-01-01T00%3A00%3A00%2B00%3A00,pagelen=100} | 0 .../api.bitbucket.org/empty_response.json} | 0 .../data/api.bitbucket.org/response.json | 1 + .../{test_bb_lister.py => test_lister.py} | 35 +++++++++++++++++-- swh/lister/core/lister_base.py | 1 + 6 files changed, 36 insertions(+), 4 deletions(-) rename swh/lister/bitbucket/tests/{api_response.json => data/api.bitbucket.org/2.0_repositories,after=1970-01-01T00%3A00%3A00%2B00%3A00,pagelen=100} (100%) rename swh/lister/bitbucket/tests/{api_empty_response.json => data/api.bitbucket.org/empty_response.json} (100%) create mode 120000 swh/lister/bitbucket/tests/data/api.bitbucket.org/response.json rename swh/lister/bitbucket/tests/{test_bb_lister.py => test_lister.py} (70%) diff --git a/swh/lister/bitbucket/lister.py b/swh/lister/bitbucket/lister.py index 5da11b1..559117c 100644 --- a/swh/lister/bitbucket/lister.py +++ b/swh/lister/bitbucket/lister.py @@ -1,4 +1,5 @@ -# Copyright (C) 2017-2019 the Software Heritage developers +# Copyright (C) 2017-2019 The Software Heritage developers +# See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information diff --git a/swh/lister/bitbucket/tests/api_response.json b/swh/lister/bitbucket/tests/data/api.bitbucket.org/2.0_repositories,after=1970-01-01T00%3A00%3A00%2B00%3A00,pagelen=100 similarity index 100% rename from swh/lister/bitbucket/tests/api_response.json rename to swh/lister/bitbucket/tests/data/api.bitbucket.org/2.0_repositories,after=1970-01-01T00%3A00%3A00%2B00%3A00,pagelen=100 diff --git a/swh/lister/bitbucket/tests/api_empty_response.json b/swh/lister/bitbucket/tests/data/api.bitbucket.org/empty_response.json similarity index 100% rename from swh/lister/bitbucket/tests/api_empty_response.json rename to swh/lister/bitbucket/tests/data/api.bitbucket.org/empty_response.json diff --git a/swh/lister/bitbucket/tests/data/api.bitbucket.org/response.json b/swh/lister/bitbucket/tests/data/api.bitbucket.org/response.json new file mode 120000 index 0000000..f420e5f --- /dev/null +++ b/swh/lister/bitbucket/tests/data/api.bitbucket.org/response.json @@ -0,0 +1 @@ +2.0_repositories,after=1970-01-01T00%3A00%3A00%2B00%3A00,pagelen=100 \ No newline at end of file diff --git a/swh/lister/bitbucket/tests/test_bb_lister.py b/swh/lister/bitbucket/tests/test_lister.py similarity index 70% rename from swh/lister/bitbucket/tests/test_bb_lister.py rename to swh/lister/bitbucket/tests/test_lister.py index 72ee097..a3cbbe6 100644 --- a/swh/lister/bitbucket/tests/test_bb_lister.py +++ b/swh/lister/bitbucket/tests/test_lister.py @@ -1,4 +1,5 @@ -# Copyright (C) 2017-2019 the Software Heritage developers +# Copyright (C) 2017-2019 The Software Heritage developers +# See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information @@ -28,8 +29,8 @@ class BitBucketListerTester(HttpListerTester, unittest.TestCase): Lister = BitBucketLister test_re = re.compile(r'/repositories\?after=([^?&]+)') lister_subdir = 'bitbucket' - good_api_response_file = 'api_response.json' - bad_api_response_file = 'api_empty_response.json' + good_api_response_file = 'data/api.bitbucket.org/response.json' + bad_api_response_file = 'data/api.bitbucket.org/empty_response.json' first_index = convert_type('2008-07-12T07:44:01.476818+00:00') last_index = convert_type('2008-07-19T06:16:43.044743+00:00') entries_per_page = 10 @@ -71,3 +72,31 @@ class BitBucketListerTester(HttpListerTester, unittest.TestCase): self.assertFalse(fl.is_within_bounds( iso8601.parse_date('2008-07-11'), self.first_index, self.last_index)) + + +def test_lister_bitbucket(swh_listers, requests_mock_datadir): + """Simple bitbucket listing should create scheduled tasks + + """ + lister = swh_listers['bitbucket'] + + lister.run() + + r = lister.scheduler.search_tasks(task_type='load-hg') + assert len(r) == 10 + + for row in r: + assert row['type'] == 'load-hg' + # arguments check + args = row['arguments']['args'] + assert len(args) == 1 + + url = args[0] + assert url.startswith('https://bitbucket.org') + + # kwargs + kwargs = row['arguments']['kwargs'] + assert kwargs == {} + + assert row['policy'] == 'recurring' + assert row['priority'] is None diff --git a/swh/lister/core/lister_base.py b/swh/lister/core/lister_base.py index 501b774..eb4c039 100644 --- a/swh/lister/core/lister_base.py +++ b/swh/lister/core/lister_base.py @@ -382,6 +382,7 @@ class ListerBase(abc.ABC, config.SWHConfig): Returns: the same information in a different form """ + logger.debug('origin-url: %s, type: %s', origin_url, origin_type) _type = 'load-%s' % origin_type _policy = kwargs.get('policy', 'recurring') priority = kwargs.get('priority')