reonokiy/swh-lister
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

packagist: Canonicalize github origins

Browse source 
In particular, there seems to be a negligeable number of origins
using SSH instead of HTTPS, which the git loader cannot deal with.
This commit is contained in:
Valentin Lorentz 2022-10-13 13:05:34 +02:00
parent 82b936a277
commit a681f2f405
8 changed files with 640 additions and 5 deletions
Download patch file Download diff file Expand all files Collapse all files

2
requirements-swh.txt
View file

@ -1,2 +1,2 @@
swh.core[db,github] >= 2.15
swh.core[db,github] >= 2.16.1
swh.scheduler >= 0.8

12
swh/lister/packagist/lister.py
View file

@ -11,6 +11,7 @@ from typing import Any, Dict, Iterator, List, Optional
import iso8601
import requests
from swh.core.github.utils import GitHubSession
from swh.scheduler.interface import SchedulerInterface
from swh.scheduler.model import ListedOrigin
@ -63,6 +64,10 @@ class PackagistLister(Lister[PackagistListerState, PackagistPageType]):
self.session.headers.update({"Accept": "application/json"})
self.listing_date = datetime.now().astimezone(tz=timezone.utc)
self.github_session = GitHubSession(
credentials=self.credentials,
user_agent=str(self.session.headers["User-Agent"]),
)
def state_from_dict(self, d: Dict[str, Any]) -> PackagistListerState:
last_listing_date = d.get("last_listing_date")
@ -144,6 +149,13 @@ class PackagistLister(Lister[PackagistListerState, PackagistPageType]):
if visit_type is None or origin_url is None or origin_url in origin_urls:
continue
if visit_type == "git":
# Non-github urls will be returned as is, github ones will be canonical
# ones
origin_url = (
self.github_session.get_canonical_url(origin_url) or origin_url
)
# bitbucket closed its mercurial hosting service, those origins can not be
# loaded into the archive anymore
if visit_type == "hg" and origin_url.startswith("https://bitbucket.org/"):

1
swh/lister/packagist/tests/data/https_api.github.com/repos_gitlky_wx_article Normal file
View file

@ -0,0 +1 @@
{"html_url": "https://github.com/gitlky/wx_article"}

130
swh/lister/packagist/tests/data/https_api.github.com/repos_spryker-eco_computop-api Normal file
View file

@ -0,0 +1,130 @@
{
"id": 133818271,
"node_id": "MDEwOlJlcG9zaXRvcnkxMzM4MTgyNzE=",
"name": "computop-api",
"full_name": "spryker-eco/computop-api",
"private": false,
"owner": {
"login": "spryker-eco",
"id": 25103059,
"node_id": "MDEyOk9yZ2FuaXphdGlvbjI1MTAzMDU5",
"avatar_url": "https://avatars.githubusercontent.com/u/25103059?v=4",
"gravatar_id": "",
"url": "https://api.github.com/users/spryker-eco",
"html_url": "https://github.com/spryker-eco",
"followers_url": "https://api.github.com/users/spryker-eco/followers",
"following_url": "https://api.github.com/users/spryker-eco/following{/other_user}",
"gists_url": "https://api.github.com/users/spryker-eco/gists{/gist_id}",
"starred_url": "https://api.github.com/users/spryker-eco/starred{/owner}{/repo}",
"subscriptions_url": "https://api.github.com/users/spryker-eco/subscriptions",
"organizations_url": "https://api.github.com/users/spryker-eco/orgs",
"repos_url": "https://api.github.com/users/spryker-eco/repos",
"events_url": "https://api.github.com/users/spryker-eco/events{/privacy}",
"received_events_url": "https://api.github.com/users/spryker-eco/received_events",
"type": "Organization",
"site_admin": false
},
"html_url": "https://github.com/spryker-eco/computop-api",
"description": "Spryker Eco ComputopApi module",
"fork": false,
"url": "https://api.github.com/repos/spryker-eco/computop-api",
"forks_url": "https://api.github.com/repos/spryker-eco/computop-api/forks",
"keys_url": "https://api.github.com/repos/spryker-eco/computop-api/keys{/key_id}",
"collaborators_url": "https://api.github.com/repos/spryker-eco/computop-api/collaborators{/collaborator}",
"teams_url": "https://api.github.com/repos/spryker-eco/computop-api/teams",
"hooks_url": "https://api.github.com/repos/spryker-eco/computop-api/hooks",
"issue_events_url": "https://api.github.com/repos/spryker-eco/computop-api/issues/events{/number}",
"events_url": "https://api.github.com/repos/spryker-eco/computop-api/events",
"assignees_url": "https://api.github.com/repos/spryker-eco/computop-api/assignees{/user}",
"branches_url": "https://api.github.com/repos/spryker-eco/computop-api/branches{/branch}",
"tags_url": "https://api.github.com/repos/spryker-eco/computop-api/tags",
"blobs_url": "https://api.github.com/repos/spryker-eco/computop-api/git/blobs{/sha}",
"git_tags_url": "https://api.github.com/repos/spryker-eco/computop-api/git/tags{/sha}",
"git_refs_url": "https://api.github.com/repos/spryker-eco/computop-api/git/refs{/sha}",
"trees_url": "https://api.github.com/repos/spryker-eco/computop-api/git/trees{/sha}",
"statuses_url": "https://api.github.com/repos/spryker-eco/computop-api/statuses/{sha}",
"languages_url": "https://api.github.com/repos/spryker-eco/computop-api/languages",
"stargazers_url": "https://api.github.com/repos/spryker-eco/computop-api/stargazers",
"contributors_url": "https://api.github.com/repos/spryker-eco/computop-api/contributors",
"subscribers_url": "https://api.github.com/repos/spryker-eco/computop-api/subscribers",
"subscription_url": "https://api.github.com/repos/spryker-eco/computop-api/subscription",
"commits_url": "https://api.github.com/repos/spryker-eco/computop-api/commits{/sha}",
"git_commits_url": "https://api.github.com/repos/spryker-eco/computop-api/git/commits{/sha}",
"comments_url": "https://api.github.com/repos/spryker-eco/computop-api/comments{/number}",
"issue_comment_url": "https://api.github.com/repos/spryker-eco/computop-api/issues/comments{/number}",
"contents_url": "https://api.github.com/repos/spryker-eco/computop-api/contents/{+path}",
"compare_url": "https://api.github.com/repos/spryker-eco/computop-api/compare/{base}...{head}",
"merges_url": "https://api.github.com/repos/spryker-eco/computop-api/merges",
"archive_url": "https://api.github.com/repos/spryker-eco/computop-api/{archive_format}{/ref}",
"downloads_url": "https://api.github.com/repos/spryker-eco/computop-api/downloads",
"issues_url": "https://api.github.com/repos/spryker-eco/computop-api/issues{/number}",
"pulls_url": "https://api.github.com/repos/spryker-eco/computop-api/pulls{/number}",
"milestones_url": "https://api.github.com/repos/spryker-eco/computop-api/milestones{/number}",
"notifications_url": "https://api.github.com/repos/spryker-eco/computop-api/notifications{?since,all,participating}",
"labels_url": "https://api.github.com/repos/spryker-eco/computop-api/labels{/name}",
"releases_url": "https://api.github.com/repos/spryker-eco/computop-api/releases{/id}",
"deployments_url": "https://api.github.com/repos/spryker-eco/computop-api/deployments",
"created_at": "2018-05-17T13:34:07Z",
"updated_at": "2021-12-28T13:55:55Z",
"pushed_at": "2022-03-18T14:05:09Z",
"git_url": "git://github.com/spryker-eco/computop-api.git",
"ssh_url": "git@github.com:spryker-eco/computop-api.git",
"clone_url": "https://github.com/spryker-eco/computop-api.git",
"svn_url": "https://github.com/spryker-eco/computop-api",
"homepage": "https://spryker.com",
"size": 198,
"stargazers_count": 0,
"watchers_count": 0,
"language": "PHP",
"has_issues": true,
"has_projects": false,
"has_downloads": true,
"has_wiki": true,
"has_pages": false,
"forks_count": 0,
"mirror_url": null,
"archived": false,
"disabled": false,
"open_issues_count": 0,
"license": {
"key": "mit",
"name": "MIT License",
"spdx_id": "MIT",
"url": "https://api.github.com/licenses/mit",
"node_id": "MDc6TGljZW5zZTEz"
},
"allow_forking": true,
"is_template": false,
"web_commit_signoff_required": false,
"topics": [
],
"visibility": "public",
"forks": 0,
"open_issues": 0,
"watchers": 0,
"default_branch": "master",
"temp_clone_token": null,
"organization": {
"login": "spryker-eco",
"id": 25103059,
"node_id": "MDEyOk9yZ2FuaXphdGlvbjI1MTAzMDU5",
"avatar_url": "https://avatars.githubusercontent.com/u/25103059?v=4",
"gravatar_id": "",
"url": "https://api.github.com/users/spryker-eco",
"html_url": "https://github.com/spryker-eco",
"followers_url": "https://api.github.com/users/spryker-eco/followers",
"following_url": "https://api.github.com/users/spryker-eco/following{/other_user}",
"gists_url": "https://api.github.com/users/spryker-eco/gists{/gist_id}",
"starred_url": "https://api.github.com/users/spryker-eco/starred{/owner}{/repo}",
"subscriptions_url": "https://api.github.com/users/spryker-eco/subscriptions",
"organizations_url": "https://api.github.com/users/spryker-eco/orgs",
"repos_url": "https://api.github.com/users/spryker-eco/repos",
"events_url": "https://api.github.com/users/spryker-eco/events{/privacy}",
"received_events_url": "https://api.github.com/users/spryker-eco/received_events",
"type": "Organization",
"site_admin": false
},
"network_count": 0,
"subscribers_count": 33
}

104
swh/lister/packagist/tests/data/https_api.github.com/repos_ycms_module-main Normal file
View file

@ -0,0 +1,104 @@
{
"id": 38592537,
"node_id": "MDEwOlJlcG9zaXRvcnkzODU5MjUzNw==",
"name": "module-main",
"full_name": "GameCHN/module-main",
"private": false,
"owner": {
"login": "GameCHN",
"id": 13175811,
"node_id": "MDQ6VXNlcjEzMTc1ODEx",
"avatar_url": "https://avatars.githubusercontent.com/u/13175811?v=4",
"gravatar_id": "",
"url": "https://api.github.com/users/GameCHN",
"html_url": "https://github.com/GameCHN",
"followers_url": "https://api.github.com/users/GameCHN/followers",
"following_url": "https://api.github.com/users/GameCHN/following{/other_user}",
"gists_url": "https://api.github.com/users/GameCHN/gists{/gist_id}",
"starred_url": "https://api.github.com/users/GameCHN/starred{/owner}{/repo}",
"subscriptions_url": "https://api.github.com/users/GameCHN/subscriptions",
"organizations_url": "https://api.github.com/users/GameCHN/orgs",
"repos_url": "https://api.github.com/users/GameCHN/repos",
"events_url": "https://api.github.com/users/GameCHN/events{/privacy}",
"received_events_url": "https://api.github.com/users/GameCHN/received_events",
"type": "User",
"site_admin": false
},
"html_url": "https://github.com/GameCHN/module-main",
"description": null,
"fork": false,
"url": "https://api.github.com/repos/GameCHN/module-main",
"forks_url": "https://api.github.com/repos/GameCHN/module-main/forks",
"keys_url": "https://api.github.com/repos/GameCHN/module-main/keys{/key_id}",
"collaborators_url": "https://api.github.com/repos/GameCHN/module-main/collaborators{/collaborator}",
"teams_url": "https://api.github.com/repos/GameCHN/module-main/teams",
"hooks_url": "https://api.github.com/repos/GameCHN/module-main/hooks",
"issue_events_url": "https://api.github.com/repos/GameCHN/module-main/issues/events{/number}",
"events_url": "https://api.github.com/repos/GameCHN/module-main/events",
"assignees_url": "https://api.github.com/repos/GameCHN/module-main/assignees{/user}",
"branches_url": "https://api.github.com/repos/GameCHN/module-main/branches{/branch}",
"tags_url": "https://api.github.com/repos/GameCHN/module-main/tags",
"blobs_url": "https://api.github.com/repos/GameCHN/module-main/git/blobs{/sha}",
"git_tags_url": "https://api.github.com/repos/GameCHN/module-main/git/tags{/sha}",
"git_refs_url": "https://api.github.com/repos/GameCHN/module-main/git/refs{/sha}",
"trees_url": "https://api.github.com/repos/GameCHN/module-main/git/trees{/sha}",
"statuses_url": "https://api.github.com/repos/GameCHN/module-main/statuses/{sha}",
"languages_url": "https://api.github.com/repos/GameCHN/module-main/languages",
"stargazers_url": "https://api.github.com/repos/GameCHN/module-main/stargazers",
"contributors_url": "https://api.github.com/repos/GameCHN/module-main/contributors",
"subscribers_url": "https://api.github.com/repos/GameCHN/module-main/subscribers",
"subscription_url": "https://api.github.com/repos/GameCHN/module-main/subscription",
"commits_url": "https://api.github.com/repos/GameCHN/module-main/commits{/sha}",
"git_commits_url": "https://api.github.com/repos/GameCHN/module-main/git/commits{/sha}",
"comments_url": "https://api.github.com/repos/GameCHN/module-main/comments{/number}",
"issue_comment_url": "https://api.github.com/repos/GameCHN/module-main/issues/comments{/number}",
"contents_url": "https://api.github.com/repos/GameCHN/module-main/contents/{+path}",
"compare_url": "https://api.github.com/repos/GameCHN/module-main/compare/{base}...{head}",
"merges_url": "https://api.github.com/repos/GameCHN/module-main/merges",
"archive_url": "https://api.github.com/repos/GameCHN/module-main/{archive_format}{/ref}",
"downloads_url": "https://api.github.com/repos/GameCHN/module-main/downloads",
"issues_url": "https://api.github.com/repos/GameCHN/module-main/issues{/number}",
"pulls_url": "https://api.github.com/repos/GameCHN/module-main/pulls{/number}",
"milestones_url": "https://api.github.com/repos/GameCHN/module-main/milestones{/number}",
"notifications_url": "https://api.github.com/repos/GameCHN/module-main/notifications{?since,all,participating}",
"labels_url": "https://api.github.com/repos/GameCHN/module-main/labels{/name}",
"releases_url": "https://api.github.com/repos/GameCHN/module-main/releases{/id}",
"deployments_url": "https://api.github.com/repos/GameCHN/module-main/deployments",
"created_at": "2015-07-06T02:08:07Z",
"updated_at": "2015-07-06T02:09:45Z",
"pushed_at": "2015-08-23T04:42:42Z",
"git_url": "git://github.com/GameCHN/module-main.git",
"ssh_url": "git@github.com:GameCHN/module-main.git",
"clone_url": "https://github.com/GameCHN/module-main.git",
"svn_url": "https://github.com/GameCHN/module-main",
"homepage": null,
"size": 172,
"stargazers_count": 0,
"watchers_count": 0,
"language": "PHP",
"has_issues": true,
"has_projects": true,
"has_downloads": true,
"has_wiki": true,
"has_pages": false,
"forks_count": 0,
"mirror_url": null,
"archived": false,
"disabled": false,
"open_issues_count": 0,
"license": null,
"allow_forking": true,
"is_template": false,
"web_commit_signoff_required": false,
"topics": [
],
"visibility": "public",
"forks": 0,
"open_issues": 0,
"watchers": 0,
"default_branch": "master",
"temp_clone_token": null,
"network_count": 0,
"subscribers_count": 2
}

309
swh/lister/packagist/tests/data/idevlab_essential.json Normal file
View file

@ -0,0 +1,309 @@
{
"packages": {
"idevlab/essential": {
"1.0.1": {
"name": "idevlab/essential",
"description": "All the methods and tools useful for the development of the various idevlab projects.",
"keywords": [],
"homepage": "",
"version": "1.0.1",
"version_normalized": "1.0.1.0",
"license": [],
"authors": [
{
"name": "Florian Sinama",
"email": "f.sinama@gmail.com"
}
],
"source": {
"url": "git@gitlab.com:idevlab/Essential.git",
"type": "git",
"reference": "6ff62de2e789aae308f3ff6fb11ea5955c806e19"
},
"dist": {
"url": "https://gitlab.com/api/v4/projects/idevlab%2FEssential/repository/archive.zip?sha=6ff62de2e789aae308f3ff6fb11ea5955c806e19",
"type": "zip",
"shasum": "",
"reference": "6ff62de2e789aae308f3ff6fb11ea5955c806e19"
},
"type": "library",
"time": "2022-08-17T21:57:20+00:00",
"autoload": {
"psr-4": {
"Idevlab\\Essential\\": "src/"
}
},
"require": {
"php": "^8.1"
},
"uid": 6622550
},
"1.1.0": {
"name": "idevlab/essential",
"description": "All the methods and tools useful for the development of the various idevlab projects.",
"keywords": [],
"homepage": "",
"version": "1.1.0",
"version_normalized": "1.1.0.0",
"license": [],
"authors": [
{
"name": "Florian Sinama",
"email": "f.sinama@gmail.com"
}
],
"source": {
"url": "git@gitlab.com:idevlab/Essential.git",
"type": "git",
"reference": "e69bdd42f03d7c453be072eef0c62f7cfeae2af8"
},
"dist": {
"url": "https://gitlab.com/api/v4/projects/idevlab%2FEssential/repository/archive.zip?sha=e69bdd42f03d7c453be072eef0c62f7cfeae2af8",
"type": "zip",
"shasum": "",
"reference": "e69bdd42f03d7c453be072eef0c62f7cfeae2af8"
},
"type": "library",
"time": "2022-08-17T22:11:20+00:00",
"autoload": {
"psr-4": {
"Idevlab\\Essential\\": "src/"
}
},
"require": {
"php": "^8.1",
"ext-mbstring": "*"
},
"uid": 6622551
},
"1.1.1": {
"name": "idevlab/essential",
"description": "All the methods and tools useful for the development of the various idevlab projects.",
"keywords": [],
"homepage": "",
"version": "1.1.1",
"version_normalized": "1.1.1.0",
"license": [],
"authors": [
{
"name": "Florian Sinama",
"email": "f.sinama@gmail.com"
}
],
"source": {
"url": "git@gitlab.com:idevlab/Essential.git",
"type": "git",
"reference": "c6f5113ffad27402a9cafe593efd518517bc321c"
},
"dist": {
"url": "https://gitlab.com/api/v4/projects/idevlab%2FEssential/repository/archive.zip?sha=c6f5113ffad27402a9cafe593efd518517bc321c",
"type": "zip",
"shasum": "",
"reference": "c6f5113ffad27402a9cafe593efd518517bc321c"
},
"type": "library",
"time": "2022-08-17T22:31:14+00:00",
"autoload": {
"psr-4": {
"Idevlab\\Essential\\": "src/"
}
},
"require": {
"php": "^8.1",
"ext-mbstring": "*"
},
"uid": 6622552
},
"1.2.0": {
"name": "idevlab/essential",
"description": "All the methods and tools useful for the development of the various idevlab projects.",
"keywords": [],
"homepage": "",
"version": "1.2.0",
"version_normalized": "1.2.0.0",
"license": [],
"authors": [
{
"name": "Florian Sinama",
"email": "f.sinama@gmail.com"
}
],
"source": {
"url": "git@gitlab.com:idevlab/Essential.git",
"type": "git",
"reference": "263fd95fcadfbfd4af6108749be0699d69d4df90"
},
"dist": {
"url": "https://gitlab.com/api/v4/projects/idevlab%2FEssential/repository/archive.zip?sha=263fd95fcadfbfd4af6108749be0699d69d4df90",
"type": "zip",
"shasum": "",
"reference": "263fd95fcadfbfd4af6108749be0699d69d4df90"
},
"type": "library",
"time": "2022-10-12T10:34:29+00:00",
"autoload": {
"psr-4": {
"Idevlab\\Essential\\": "src/"
}
},
"require": {
"php": "^8.1",
"ext-mbstring": "*"
},
"uid": 6624846
},
"dev-develop": {
"name": "idevlab/essential",
"description": "All the methods and tools useful for the development of the various idevlab projects.",
"keywords": [],
"homepage": "",
"version": "dev-develop",
"version_normalized": "dev-develop",
"license": [],
"authors": [
{
"name": "Florian Sinama",
"email": "f.sinama@gmail.com"
}
],
"source": {
"url": "git@gitlab.com:idevlab/Essential.git",
"type": "git",
"reference": "8125bcc747e1bf5086a3195f74a682b7be0aea6a"
},
"dist": {
"url": "https://gitlab.com/api/v4/projects/idevlab%2FEssential/repository/archive.zip?sha=8125bcc747e1bf5086a3195f74a682b7be0aea6a",
"type": "zip",
"shasum": "",
"reference": "8125bcc747e1bf5086a3195f74a682b7be0aea6a"
},
"type": "library",
"time": "2022-10-12T10:34:29+00:00",
"autoload": {
"psr-4": {
"Idevlab\\Essential\\": "src/"
}
},
"require": {
"php": "^8.1",
"ext-mbstring": "*"
},
"uid": 6622554
},
"dev-main": {
"name": "idevlab/essential",
"description": "All the methods and tools useful for the development of the various idevlab projects.",
"keywords": [],
"homepage": "",
"version": "dev-main",
"version_normalized": "dev-main",
"license": [],
"authors": [
{
"name": "Florian Sinama",
"email": "f.sinama@gmail.com"
}
],
"source": {
"url": "git@gitlab.com:idevlab/Essential.git",
"type": "git",
"reference": "263fd95fcadfbfd4af6108749be0699d69d4df90"
},
"dist": {
"url": "https://gitlab.com/api/v4/projects/idevlab%2FEssential/repository/archive.zip?sha=263fd95fcadfbfd4af6108749be0699d69d4df90",
"type": "zip",
"shasum": "",
"reference": "263fd95fcadfbfd4af6108749be0699d69d4df90"
},
"type": "library",
"time": "2022-10-12T10:34:29+00:00",
"autoload": {
"psr-4": {
"Idevlab\\Essential\\": "src/"
}
},
"default-branch": true,
"require": {
"php": "^8.1",
"ext-mbstring": "*"
},
"uid": 6622553
},
"dev-release/1.2.0": {
"name": "idevlab/essential",
"description": "All the methods and tools useful for the development of the various idevlab projects.",
"keywords": [],
"homepage": "",
"version": "dev-release/1.2.0",
"version_normalized": "dev-release/1.2.0",
"license": [],
"authors": [
{
"name": "Florian Sinama",
"email": "f.sinama@gmail.com"
}
],
"source": {
"url": "git@gitlab.com:idevlab/Essential.git",
"type": "git",
"reference": "1688c6cb5f3b0e9f21a5bbcb9f9951138ab51a66"
},
"dist": {
"url": "https://gitlab.com/api/v4/projects/idevlab%2FEssential/repository/archive.zip?sha=1688c6cb5f3b0e9f21a5bbcb9f9951138ab51a66",
"type": "zip",
"shasum": "",
"reference": "1688c6cb5f3b0e9f21a5bbcb9f9951138ab51a66"
},
"type": "library",
"time": "2022-10-12T10:34:14+00:00",
"autoload": {
"psr-4": {
"Idevlab\\Essential\\": "src/"
}
},
"require": {
"php": "^8.1",
"ext-mbstring": "*"
},
"uid": 6624845
},
"v1.0.0": {
"name": "idevlab/essential",
"description": "All the methods and tools useful for the development of the various idevlab projects.",
"keywords": [],
"homepage": "",
"version": "v1.0.0",
"version_normalized": "1.0.0.0",
"license": [],
"authors": [
{
"name": "Florian Sinama",
"email": "f.sinama@gmail.com"
}
],
"source": {
"url": "git@gitlab.com:idevlab/Essential.git",
"type": "git",
"reference": "b86767e1baf9ed9c218abcba963361876d55138a"
},
"dist": {
"url": "https://gitlab.com/api/v4/projects/idevlab%2FEssential/repository/archive.zip?sha=b86767e1baf9ed9c218abcba963361876d55138a",
"type": "zip",
"shasum": "",
"reference": "b86767e1baf9ed9c218abcba963361876d55138a"
},
"type": "library",
"time": "2022-08-17T21:34:05+00:00",
"autoload": {
"psr-4": {
"Idevlab\\Essential\\": "src/"
}
},
"require": {
"php": "^8.1"
},
"uid": 6622549
}
}
}
}

41
swh/lister/packagist/tests/data/ycms_module-main.json Normal file
View file

@ -0,0 +1,41 @@
{
"packages": {
"ycms/module-main": {
"dev-master": {
"name": "ycms/module-main",
"description": "",
"keywords": [],
"homepage": "",
"version": "dev-master",
"version_normalized": "9999999-dev",
"license": [],
"authors": [
{
"name": "YCMS Labs",
"email": "ycms.net@gmail.com"
}
],
"source": {
"type": "git",
"url": "git@github.com:ycms/module-main.git",
"reference": "1173796881fbd7202009a68a2a59a5150bf2dbc6"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/ycms/module-main/zipball/1173796881fbd7202009a68a2a59a5150bf2dbc6",
"reference": "1173796881fbd7202009a68a2a59a5150bf2dbc6",
"shasum": ""
},
"type": "wordpress-plugin",
"time": "2015-08-23T04:42:33+00:00",
"autoload": {
"psr-4": {
"YC\\Main\\": ""
}
},
"default-branch": true,
"uid": 4064797
}
}
}
}

46
swh/lister/packagist/tests/test_lister.py
View file

@ -14,6 +14,7 @@ _packages_list = {
"ljjackson/linnworks",
"lky/wx_article",
"spryker-eco/computop-api",
"idevlab/essential",
]
}
@ -32,7 +33,7 @@ def _request_with_if_modified_since(request):
return request.headers.get("If-Modified-Since") is not None
def test_packagist_lister(swh_scheduler, requests_mock, datadir):
def test_packagist_lister(swh_scheduler, requests_mock, datadir, requests_mock_datadir):
# first listing, should return one origin per package
lister = PackagistLister(scheduler=swh_scheduler)
requests_mock.get(lister.PACKAGIST_PACKAGES_LIST_URL, json=_packages_list)
@ -53,20 +54,25 @@ def test_packagist_lister(swh_scheduler, requests_mock, datadir):
expected_origins = {
(
"https://github.com/gitlky/wx_article.git",
"https://github.com/gitlky/wx_article", # standard case
"git",
datetime.datetime.fromisoformat("2018-08-30T07:37:09+00:00"),
),
(
"https://github.com/ljjackson/linnworks.git",
"https://github.com/ljjackson/linnworks.git", # API goes 404
"git",
datetime.datetime.fromisoformat("2018-11-01T21:45:50+00:00"),
),
(
"https://github.com/spryker-eco/computop-api.git",
"https://github.com/spryker-eco/computop-api", # SSH URL in manifest
"git",
datetime.datetime.fromisoformat("2020-06-22T15:50:29+00:00"),
),
(
"git@gitlab.com:idevlab/Essential.git", # not GitHub
"git",
datetime.datetime.fromisoformat("2022-10-12T10:34:29+00:00"),
),
}
assert expected_origins == {
@ -150,6 +156,38 @@ def test_packagist_lister_package_with_bitbucket_hg_origin(
assert stats.origins == 0
def test_packagist_lister_package_normalize_github_origin(
swh_scheduler, requests_mock, datadir, requests_mock_datadir
):
package_name = "ycms/module-main"
lister = PackagistLister(scheduler=swh_scheduler)
requests_mock.get(
lister.PACKAGIST_PACKAGES_LIST_URL, json={"packageNames": [package_name]}
)
requests_mock.get(
f"{lister.PACKAGIST_REPO_BASE_URL}/{package_name}.json",
additional_matcher=_request_without_if_modified_since,
json=_package_metadata(datadir, package_name),
)
stats = lister.run()
assert stats.pages == 1
assert stats.origins == 1
expected_origins = {
(
"https://github.com/GameCHN/module-main",
"git",
datetime.datetime.fromisoformat("2015-08-23T04:42:33+00:00"),
),
}
assert expected_origins == {
(o.url, o.visit_type, o.last_update)
for o in swh_scheduler.get_listed_origins(lister.lister_obj.id).results
}
def test_lister_from_configfile(swh_scheduler_config, mocker):
load_from_envvar = mocker.patch("swh.lister.pattern.load_from_envvar")
load_from_envvar.return_value = {