phabricator: Add test for new lister implementation

Also remove no longer used JSON files.

swh/lister/phabricator/tests/data/api_empty_response.json

@@ -1,8 +0,0 @@
-{
-  "result": {
-    "data": [],
-    "cursor": {
-      "after": null
-    }
-  }
-}

swh/lister/phabricator/tests/data/api_response_undefined_protocol.json

@@ -1,60 +0,0 @@
-{
-  "id": 8,
-  "type": "REPO",
-  "phid": "PHID-REPO-ge2icigfu5ijk2whqfbl",
-  "fields": {
-    "name": "Blender Libraries",
-    "vcs": "svn",
-    "callsign": "BL",
-    "shortName": null,
-    "status": "active",
-    "isImporting": false,
-    "almanacServicePHID": null,
-    "spacePHID": null,
-    "dateCreated": 1385564674,
-    "dateModified": 1468574079,
-    "policy": {
-      "view": "public",
-      "edit": "admin",
-      "diffusion.push": "PHID-PROJ-hclk7tvd6pmpjmqastjl"
-    }
-  },
-  "attachments": {
-    "uris": {
-      "uris": [
-        {
-          "id": "70",
-          "type": "RURI",
-          "phid": "PHID-RURI-h7zdbkud6why4xrb2s2e",
-          "fields": {
-            "repositoryPHID": "PHID-REPO-ge2icigfu5ijk2whqfbl",
-            "uri": {
-              "raw": "https://svn.blender.org/svnroot/bf-blender/",
-              "display": "https://svn.blender.org/svnroot/bf-blender/",
-              "effective": "https://svn.blender.org/svnroot/bf-blender/",
-              "normalized": "svn.blender.org/svnroot/bf-blender"
-            },
-            "io": {
-              "raw": "observe",
-              "default": "none",
-              "effective": "observe"
-            },
-            "display": {
-              "raw": "always",
-              "default": "never",
-              "effective": "always"
-            },
-            "credentialPHID": null,
-            "disabled": false,
-            "builtin": {
-              "protocol": null,
-              "identifier": null
-            },
-            "dateCreated": "1467894515",
-            "dateModified": "1468574079"
-          }
-        }
-      ]
-    }
-  }
-}

swh/lister/phabricator/tests/data/https_forge.softwareheritage.org/api_diffusion.repository.search,order=oldest,attachments[uris]=1,after=,api.token=foo

@@ -1 +0,0 @@
-../api_first_response.json

swh/lister/phabricator/tests/test_lister.py

@@ -1,32 +1,56 @@
-# Copyright (C) 2019  The Software Heritage developers
+# Copyright (C) 2019-2021  The Software Heritage developers
 # See the AUTHORS file at the top-level directory of this distribution
 # License: GNU General Public License version 3, or any later version
 # See top-level LICENSE file for more information
 
-import importlib.resources
 import json
+from pathlib import Path
 
+import pytest
+
+from swh.lister import USER_AGENT
 from swh.lister.phabricator.lister import PhabricatorLister, get_repo_url
 
 
-def test_get_repo_url():
-    with importlib.resources.open_text(
-        "swh.lister.phabricator.tests.data", "api_first_response.json"
-    ) as f:
-        api_response = json.load(f)
-    repos = api_response["result"]["data"]
+@pytest.fixture
+def phabricator_repositories_page1(datadir):
+    return json.loads(
+        Path(datadir, "phabricator_api_repositories_page1.json").read_text()
+    )
+
+
+@pytest.fixture
+def phabricator_repositories_page2(datadir):
+    return json.loads(
+        Path(datadir, "phabricator_api_repositories_page2.json").read_text()
+    )
+
+
+def test_get_repo_url(phabricator_repositories_page1):
+    repos = phabricator_repositories_page1["result"]["data"]
     for repo in repos:
         expected_name = "https://forge.softwareheritage.org/source/%s.git" % (
             repo["fields"]["shortName"]
         )
         assert get_repo_url(repo["attachments"]["uris"]["uris"]) == expected_name
 
-    with importlib.resources.open_text(
-        "swh.lister.phabricator.tests.data", "api_response_undefined_protocol.json",
-    ) as f:
-        repo = json.load(f)
+
+def test_get_repo_url_undefined_protocol():
+    undefined_protocol_uris = [
+        {
+            "fields": {
+                "uri": {
+                    "raw": "https://svn.blender.org/svnroot/bf-blender/",
+                    "display": "https://svn.blender.org/svnroot/bf-blender/",
+                    "effective": "https://svn.blender.org/svnroot/bf-blender/",
+                    "normalized": "svn.blender.org/svnroot/bf-blender",
+                },
+                "builtin": {"protocol": None, "identifier": None},
+            },
+        }
+    ]
     expected_name = "https://svn.blender.org/svnroot/bf-blender/"
-    assert get_repo_url(repo["attachments"]["uris"]["uris"]) == expected_name
+    assert get_repo_url(undefined_protocol_uris) == expected_name
 
 
 def test_lister_url_param(swh_scheduler):
@@ -47,3 +71,43 @@ def test_lister_url_param(swh_scheduler):
         expected_url = f"{FORGE_BASE_URL}{API_REPOSITORY_PATH}"
 
         assert lister.url == expected_url
+
+
+def test_lister(
+    swh_scheduler,
+    requests_mock,
+    phabricator_repositories_page1,
+    phabricator_repositories_page2,
+):
+    FORGE_BASE_URL = "https://forge.softwareheritage.org"
+    API_TOKEN = "foo"
+
+    lister = PhabricatorLister(
+        scheduler=swh_scheduler, url=FORGE_BASE_URL, instance="swh", api_token=API_TOKEN
+    )
+
+    def match_request(request):
+        return (
+            request.headers.get("User-Agent") == USER_AGENT
+            and f"api.token={API_TOKEN}" in request.body
+        )
+
+    requests_mock.post(
+        f"{FORGE_BASE_URL}{lister.API_REPOSITORY_PATH}",
+        [
+            {"json": phabricator_repositories_page1},
+            {"json": phabricator_repositories_page2},
+        ],
+        additional_matcher=match_request,
+    )
+
+    stats = lister.run()
+
+    expected_nb_origins = len(phabricator_repositories_page1["result"]["data"]) * 2
+
+    assert stats.pages == 2
+    assert stats.origins == expected_nb_origins
+
+    scheduler_origins = swh_scheduler.get_listed_origins(lister.lister_obj.id).origins
+
+    assert len(scheduler_origins) == expected_nb_origins