core: Align listers' task output (hg/git tasks) with expected format

Related to T2134
Related to D2409
Related to D2410

swh/lister/bitbucket/lister.py

@@ -10,6 +10,7 @@ from datetime import datetime, timezone
 from typing import Any
 from urllib import parse
 
+
 from swh.lister.bitbucket.models import BitBucketModel
 from swh.lister.core.indexing_lister import IndexingHttpLister
 

swh/lister/bitbucket/tests/data/https_api.bitbucket.org/2.0_repositories,after=1970-01-01T00:00:00+00:00,pagelen=100

@@ -82,7 +82,7 @@
             "description": "Basic files and directory structure for a C++ project. Intended as a starting point for a new project. Includes a basic cross platform core library."
         },
         {
-            "scm": "hg",
+            "scm": "git",
             "website": "",
             "has_wiki": true,
             "name": "mercurialeclipse",

swh/lister/bitbucket/tests/test_lister.py

@@ -74,7 +74,7 @@ class BitBucketListerTester(HttpListerTester, unittest.TestCase):
 
 
 def test_lister_bitbucket(swh_listers, requests_mock_datadir):
-    """Simple bitbucket listing should create scheduled tasks
+    """Simple bitbucket listing should create scheduled tasks (git, hg)
 
     """
     lister = swh_listers['bitbucket']
@@ -82,20 +82,32 @@ def test_lister_bitbucket(swh_listers, requests_mock_datadir):
     lister.run()
 
     r = lister.scheduler.search_tasks(task_type='load-hg')
-    assert len(r) == 10
+    assert len(r) == 9
 
     for row in r:
-        assert row['type'] == 'load-hg'
-        # arguments check
         args = row['arguments']['args']
-        assert len(args) == 1
-
-        url = args[0]
-        assert url.startswith('https://bitbucket.org')
-
-        # kwargs
         kwargs = row['arguments']['kwargs']
-        assert kwargs == {}
+
+        assert len(args) == 0
+        assert len(kwargs) == 1
+        url = kwargs['url']
+
+        assert url.startswith('https://bitbucket.org')
+
+        assert row['policy'] == 'recurring'
+        assert row['priority'] is None
+
+    r = lister.scheduler.search_tasks(task_type='load-git')
+    assert len(r) == 1
+
+    for row in r:
+        args = row['arguments']['args']
+        kwargs = row['arguments']['kwargs']
+        assert len(args) == 0
+        assert len(kwargs) == 1
+        url = kwargs['url']
+
+        assert url.startswith('https://bitbucket.org')
 
         assert row['policy'] == 'recurring'
         assert row['priority'] is None

swh/lister/cgit/tests/test_lister.py

@@ -57,14 +57,14 @@ def test_lister_run(requests_mock_datadir, swh_listers):
         assert row['type'] == 'load-git'
         # arguments check
         args = row['arguments']['args']
-        assert len(args) == 1
-
-        url = args[0]
-        assert url.startswith('https://git.tizen')
+        assert len(args) == 0
 
         # kwargs
         kwargs = row['arguments']['kwargs']
-        assert kwargs == {}
+        assert len(kwargs) == 1
+        url = kwargs['url']
+        assert url.startswith('https://git.tizen')
+
         assert row['policy'] == 'recurring'
         assert row['priority'] is None
 

swh/lister/core/lister_base.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2015-2018 the Software Heritage developers
+# Copyright (C) 2015-2019 the Software Heritage developers
 # License: GNU General Public License version 3, or any later version
 # See top-level LICENSE file for more information
 
@@ -13,7 +13,7 @@ import time
 
 from sqlalchemy import create_engine, func
 from sqlalchemy.orm import sessionmaker
-from typing import Any, Type, Union
+from typing import Any, Dict, Type, Union
 
 from swh.core import config
 from swh.scheduler import get_scheduler, utils
@@ -376,7 +376,8 @@ class ListerBase(abc.ABC, config.SWHConfig):
 
         return sql_repo
 
-    def task_dict(self, origin_type, origin_url, **kwargs):
+    def task_dict(self, origin_type: str,
+                  origin_url: str, **kwargs) -> Dict[str, Any]:
         """Return special dict format for the tasks list
 
         Args:
@@ -390,7 +391,7 @@ class ListerBase(abc.ABC, config.SWHConfig):
         _policy = kwargs.get('policy', 'recurring')
         priority = kwargs.get('priority')
         kw = {'priority': priority} if priority else {}
-        return utils.create_task_dict(_type, _policy, origin_url, **kw)
+        return utils.create_task_dict(_type, _policy, url=origin_url, **kw)
 
     def string_pattern_check(self, a, b, c=None):
         """When comparing indexable types in is_within_bounds, complex strings

swh/lister/github/tests/test_lister.py

@@ -70,14 +70,12 @@ def test_lister_github(swh_listers, requests_mock_datadir):
         assert row['type'] == 'load-git'
         # arguments check
         args = row['arguments']['args']
-        assert len(args) == 1
-
-        url = args[0]
-        assert url.startswith('https://github.com')
+        assert len(args) == 0
 
         # kwargs
         kwargs = row['arguments']['kwargs']
-        assert kwargs == {}
+        url = kwargs['url']
+        assert url.startswith('https://github.com')
 
         assert row['policy'] == 'recurring'
         assert row['priority'] is None

swh/lister/gitlab/tests/test_lister.py

@@ -55,14 +55,12 @@ def test_lister_gitlab(swh_listers, requests_mock_datadir):
         assert row['type'] == 'load-git'
         # arguments check
         args = row['arguments']['args']
-        assert len(args) == 1
-
-        url = args[0]
-        assert url.startswith('https://gitlab.com')
+        assert len(args) == 0
 
         # kwargs
         kwargs = row['arguments']['kwargs']
-        assert kwargs == {}
+        url = kwargs['url']
+        assert url.startswith('https://gitlab.com')
 
         assert row['policy'] == 'recurring'
         assert row['priority'] is None

swh/lister/packagist/lister.py

@@ -7,7 +7,7 @@ import json
 import logging
 import random
 
-from typing import Any, List, Mapping
+from typing import Any, Dict, List, Mapping
 
 from swh.scheduler import utils
 from swh.lister.core.simple_lister import SimpleLister
@@ -60,7 +60,7 @@ class PackagistLister(ListerOnePageApiTransport, SimpleLister):
         SimpleLister.__init__(self, override_config=override_config)
 
     def task_dict(self, origin_type: str, origin_url: str,
-                  **kwargs: Mapping[str, str]) -> Mapping[str, str]:
+                  **kwargs: Mapping[str, str]) -> Dict[str, Any]:
         """Return task format dict
 
         This is overridden from the lister_base as more information is

swh/lister/phabricator/tests/test_lister.py

@@ -129,14 +129,12 @@ def test_phabricator_lister(lister_phabricator, requests_mock_datadir):
         assert row['type'] == 'load-git'
         # arguments check
         args = row['arguments']['args']
-        assert len(args) == 1
-
-        url = args[0]
-        assert lister.instance in url
+        assert len(args) == 0
 
         # kwargs
         kwargs = row['arguments']['kwargs']
-        assert kwargs == {}
+        url = kwargs['url']
+        assert lister.instance in url
 
         assert row['policy'] == 'recurring'
         assert row['priority'] is None