From 5ab9d67d675aa072a1e3f68d6c72ad6c1afa2c55 Mon Sep 17 00:00:00 2001 From: "Antoine R. Dumont (@ardumont)" Date: Fri, 6 Dec 2019 17:35:24 +0100 Subject: [PATCH] core: Align listers' task output (hg/git tasks) with expected format Related to T2134 Related to D2409 Related to D2410 --- swh/lister/bitbucket/lister.py | 1 + ...fter=1970-01-01T00:00:00+00:00,pagelen=100 | 2 +- swh/lister/bitbucket/tests/test_lister.py | 34 +++++++++++++------ swh/lister/cgit/tests/test_lister.py | 10 +++--- swh/lister/core/lister_base.py | 9 ++--- swh/lister/github/tests/test_lister.py | 8 ++--- swh/lister/gitlab/tests/test_lister.py | 8 ++--- swh/lister/packagist/lister.py | 4 +-- swh/lister/phabricator/tests/test_lister.py | 8 ++--- 9 files changed, 46 insertions(+), 38 deletions(-) diff --git a/swh/lister/bitbucket/lister.py b/swh/lister/bitbucket/lister.py index 0c78af0..a480532 100644 --- a/swh/lister/bitbucket/lister.py +++ b/swh/lister/bitbucket/lister.py @@ -10,6 +10,7 @@ from datetime import datetime, timezone from typing import Any from urllib import parse + from swh.lister.bitbucket.models import BitBucketModel from swh.lister.core.indexing_lister import IndexingHttpLister diff --git a/swh/lister/bitbucket/tests/data/https_api.bitbucket.org/2.0_repositories,after=1970-01-01T00:00:00+00:00,pagelen=100 b/swh/lister/bitbucket/tests/data/https_api.bitbucket.org/2.0_repositories,after=1970-01-01T00:00:00+00:00,pagelen=100 index 080a2fe..9e64695 100644 --- a/swh/lister/bitbucket/tests/data/https_api.bitbucket.org/2.0_repositories,after=1970-01-01T00:00:00+00:00,pagelen=100 +++ b/swh/lister/bitbucket/tests/data/https_api.bitbucket.org/2.0_repositories,after=1970-01-01T00:00:00+00:00,pagelen=100 @@ -82,7 +82,7 @@ "description": "Basic files and directory structure for a C++ project. Intended as a starting point for a new project. Includes a basic cross platform core library." }, { - "scm": "hg", + "scm": "git", "website": "", "has_wiki": true, "name": "mercurialeclipse", diff --git a/swh/lister/bitbucket/tests/test_lister.py b/swh/lister/bitbucket/tests/test_lister.py index 9e2378a..eda17d6 100644 --- a/swh/lister/bitbucket/tests/test_lister.py +++ b/swh/lister/bitbucket/tests/test_lister.py @@ -74,7 +74,7 @@ class BitBucketListerTester(HttpListerTester, unittest.TestCase): def test_lister_bitbucket(swh_listers, requests_mock_datadir): - """Simple bitbucket listing should create scheduled tasks + """Simple bitbucket listing should create scheduled tasks (git, hg) """ lister = swh_listers['bitbucket'] @@ -82,20 +82,32 @@ def test_lister_bitbucket(swh_listers, requests_mock_datadir): lister.run() r = lister.scheduler.search_tasks(task_type='load-hg') - assert len(r) == 10 + assert len(r) == 9 for row in r: - assert row['type'] == 'load-hg' - # arguments check args = row['arguments']['args'] - assert len(args) == 1 - - url = args[0] - assert url.startswith('https://bitbucket.org') - - # kwargs kwargs = row['arguments']['kwargs'] - assert kwargs == {} + + assert len(args) == 0 + assert len(kwargs) == 1 + url = kwargs['url'] + + assert url.startswith('https://bitbucket.org') + + assert row['policy'] == 'recurring' + assert row['priority'] is None + + r = lister.scheduler.search_tasks(task_type='load-git') + assert len(r) == 1 + + for row in r: + args = row['arguments']['args'] + kwargs = row['arguments']['kwargs'] + assert len(args) == 0 + assert len(kwargs) == 1 + url = kwargs['url'] + + assert url.startswith('https://bitbucket.org') assert row['policy'] == 'recurring' assert row['priority'] is None diff --git a/swh/lister/cgit/tests/test_lister.py b/swh/lister/cgit/tests/test_lister.py index a140cdd..ca8ddd5 100644 --- a/swh/lister/cgit/tests/test_lister.py +++ b/swh/lister/cgit/tests/test_lister.py @@ -57,14 +57,14 @@ def test_lister_run(requests_mock_datadir, swh_listers): assert row['type'] == 'load-git' # arguments check args = row['arguments']['args'] - assert len(args) == 1 - - url = args[0] - assert url.startswith('https://git.tizen') + assert len(args) == 0 # kwargs kwargs = row['arguments']['kwargs'] - assert kwargs == {} + assert len(kwargs) == 1 + url = kwargs['url'] + assert url.startswith('https://git.tizen') + assert row['policy'] == 'recurring' assert row['priority'] is None diff --git a/swh/lister/core/lister_base.py b/swh/lister/core/lister_base.py index a9f1e02..e4253af 100644 --- a/swh/lister/core/lister_base.py +++ b/swh/lister/core/lister_base.py @@ -1,4 +1,4 @@ -# Copyright (C) 2015-2018 the Software Heritage developers +# Copyright (C) 2015-2019 the Software Heritage developers # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information @@ -13,7 +13,7 @@ import time from sqlalchemy import create_engine, func from sqlalchemy.orm import sessionmaker -from typing import Any, Type, Union +from typing import Any, Dict, Type, Union from swh.core import config from swh.scheduler import get_scheduler, utils @@ -376,7 +376,8 @@ class ListerBase(abc.ABC, config.SWHConfig): return sql_repo - def task_dict(self, origin_type, origin_url, **kwargs): + def task_dict(self, origin_type: str, + origin_url: str, **kwargs) -> Dict[str, Any]: """Return special dict format for the tasks list Args: @@ -390,7 +391,7 @@ class ListerBase(abc.ABC, config.SWHConfig): _policy = kwargs.get('policy', 'recurring') priority = kwargs.get('priority') kw = {'priority': priority} if priority else {} - return utils.create_task_dict(_type, _policy, origin_url, **kw) + return utils.create_task_dict(_type, _policy, url=origin_url, **kw) def string_pattern_check(self, a, b, c=None): """When comparing indexable types in is_within_bounds, complex strings diff --git a/swh/lister/github/tests/test_lister.py b/swh/lister/github/tests/test_lister.py index db11c35..f2c085f 100644 --- a/swh/lister/github/tests/test_lister.py +++ b/swh/lister/github/tests/test_lister.py @@ -70,14 +70,12 @@ def test_lister_github(swh_listers, requests_mock_datadir): assert row['type'] == 'load-git' # arguments check args = row['arguments']['args'] - assert len(args) == 1 - - url = args[0] - assert url.startswith('https://github.com') + assert len(args) == 0 # kwargs kwargs = row['arguments']['kwargs'] - assert kwargs == {} + url = kwargs['url'] + assert url.startswith('https://github.com') assert row['policy'] == 'recurring' assert row['priority'] is None diff --git a/swh/lister/gitlab/tests/test_lister.py b/swh/lister/gitlab/tests/test_lister.py index 2201bea..0d02423 100644 --- a/swh/lister/gitlab/tests/test_lister.py +++ b/swh/lister/gitlab/tests/test_lister.py @@ -55,14 +55,12 @@ def test_lister_gitlab(swh_listers, requests_mock_datadir): assert row['type'] == 'load-git' # arguments check args = row['arguments']['args'] - assert len(args) == 1 - - url = args[0] - assert url.startswith('https://gitlab.com') + assert len(args) == 0 # kwargs kwargs = row['arguments']['kwargs'] - assert kwargs == {} + url = kwargs['url'] + assert url.startswith('https://gitlab.com') assert row['policy'] == 'recurring' assert row['priority'] is None diff --git a/swh/lister/packagist/lister.py b/swh/lister/packagist/lister.py index fa2d581..5461ed6 100644 --- a/swh/lister/packagist/lister.py +++ b/swh/lister/packagist/lister.py @@ -7,7 +7,7 @@ import json import logging import random -from typing import Any, List, Mapping +from typing import Any, Dict, List, Mapping from swh.scheduler import utils from swh.lister.core.simple_lister import SimpleLister @@ -60,7 +60,7 @@ class PackagistLister(ListerOnePageApiTransport, SimpleLister): SimpleLister.__init__(self, override_config=override_config) def task_dict(self, origin_type: str, origin_url: str, - **kwargs: Mapping[str, str]) -> Mapping[str, str]: + **kwargs: Mapping[str, str]) -> Dict[str, Any]: """Return task format dict This is overridden from the lister_base as more information is diff --git a/swh/lister/phabricator/tests/test_lister.py b/swh/lister/phabricator/tests/test_lister.py index a433226..dcf76c0 100644 --- a/swh/lister/phabricator/tests/test_lister.py +++ b/swh/lister/phabricator/tests/test_lister.py @@ -129,14 +129,12 @@ def test_phabricator_lister(lister_phabricator, requests_mock_datadir): assert row['type'] == 'load-git' # arguments check args = row['arguments']['args'] - assert len(args) == 1 - - url = args[0] - assert lister.instance in url + assert len(args) == 0 # kwargs kwargs = row['arguments']['kwargs'] - assert kwargs == {} + url = kwargs['url'] + assert lister.instance in url assert row['policy'] == 'recurring' assert row['priority'] is None