From 0b8b1419e14a57df22862857aa225c736c1d2baa Mon Sep 17 00:00:00 2001 From: "Antoine R. Dumont (@ardumont)" Date: Sat, 12 Oct 2019 03:28:39 +0200 Subject: [PATCH] github.lister: Add integration test which checks scheduled tasks Related T2032 --- swh/lister/github/lister.py | 8 +++- .../api.github.com/empty_response.json} | 0 .../data/api.github.com/first_response.json | 1 + .../api.github.com/next_response.json} | 0 .../api.github.com/repositories,since=0} | 0 .../{test_gh_lister.py => test_lister.py} | 38 +++++++++++++++++-- 6 files changed, 41 insertions(+), 6 deletions(-) rename swh/lister/github/tests/{api_empty_response.json => data/api.github.com/empty_response.json} (100%) create mode 120000 swh/lister/github/tests/data/api.github.com/first_response.json rename swh/lister/github/tests/{api_next_response.json => data/api.github.com/next_response.json} (100%) rename swh/lister/github/tests/{api_first_response.json => data/api.github.com/repositories,since=0} (100%) rename swh/lister/github/tests/{test_gh_lister.py => test_lister.py} (61%) diff --git a/swh/lister/github/lister.py b/swh/lister/github/lister.py index 859dead..9d22172 100644 --- a/swh/lister/github/lister.py +++ b/swh/lister/github/lister.py @@ -1,4 +1,5 @@ -# Copyright (C) 2017-2019 the Software Heritage developers +# Copyright (C) 2017-2019 The Software Heritage developers +# See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information @@ -31,7 +32,10 @@ class GitHubLister(IndexingHttpLister): } def transport_quota_check(self, response): - reqs_remaining = int(response.headers['X-RateLimit-Remaining']) + x_rate_limit_remaining = response.headers.get('X-RateLimit-Remaining') + if not x_rate_limit_remaining: + return False, 0 + reqs_remaining = int(x_rate_limit_remaining) if response.status_code == 403 and reqs_remaining == 0: reset_at = int(response.headers['X-RateLimit-Reset']) delay = min(reset_at - time.time(), 3600) diff --git a/swh/lister/github/tests/api_empty_response.json b/swh/lister/github/tests/data/api.github.com/empty_response.json similarity index 100% rename from swh/lister/github/tests/api_empty_response.json rename to swh/lister/github/tests/data/api.github.com/empty_response.json diff --git a/swh/lister/github/tests/data/api.github.com/first_response.json b/swh/lister/github/tests/data/api.github.com/first_response.json new file mode 120000 index 0000000..6070594 --- /dev/null +++ b/swh/lister/github/tests/data/api.github.com/first_response.json @@ -0,0 +1 @@ +repositories,since=0 \ No newline at end of file diff --git a/swh/lister/github/tests/api_next_response.json b/swh/lister/github/tests/data/api.github.com/next_response.json similarity index 100% rename from swh/lister/github/tests/api_next_response.json rename to swh/lister/github/tests/data/api.github.com/next_response.json diff --git a/swh/lister/github/tests/api_first_response.json b/swh/lister/github/tests/data/api.github.com/repositories,since=0 similarity index 100% rename from swh/lister/github/tests/api_first_response.json rename to swh/lister/github/tests/data/api.github.com/repositories,since=0 diff --git a/swh/lister/github/tests/test_gh_lister.py b/swh/lister/github/tests/test_lister.py similarity index 61% rename from swh/lister/github/tests/test_gh_lister.py rename to swh/lister/github/tests/test_lister.py index c6aca16..cc61bc0 100644 --- a/swh/lister/github/tests/test_gh_lister.py +++ b/swh/lister/github/tests/test_lister.py @@ -1,4 +1,5 @@ -# Copyright (C) 2017-2019 the Software Heritage developers +# Copyright (C) 2017-2019 The Software Heritage developers +# See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information @@ -17,8 +18,8 @@ class GitHubListerTester(HttpListerTester, unittest.TestCase): Lister = GitHubLister test_re = re.compile(r'/repositories\?since=([^?&]+)') lister_subdir = 'github' - good_api_response_file = 'api_first_response.json' - bad_api_response_file = 'api_empty_response.json' + good_api_response_file = 'data/api.github.com/first_response.json' + bad_api_response_file = 'data/api.github.com/empty_response.json' first_index = 0 last_index = 369 entries_per_page = 100 @@ -50,4 +51,33 @@ class GitHubListerTester(HttpListerTester, unittest.TestCase): @requests_mock.Mocker() def test_scheduled_tasks(self, http_mocker): - self.scheduled_tasks_test('api_next_response.json', 876, http_mocker) + self.scheduled_tasks_test( + 'data/api.github.com/next_response.json', 876, http_mocker) + + +def test_lister_github(swh_listers, requests_mock_datadir): + """Simple github listing should create scheduled tasks + + """ + lister = swh_listers['github'] + + lister.run() + + r = lister.scheduler.search_tasks(task_type='load-git') + assert len(r) == 100 + + for row in r: + assert row['type'] == 'load-git' + # arguments check + args = row['arguments']['args'] + assert len(args) == 1 + + url = args[0] + assert url.startswith('https://github.com') + + # kwargs + kwargs = row['arguments']['kwargs'] + assert kwargs == {} + + assert row['policy'] == 'recurring' + assert row['priority'] is None