nixguix: Add support for pseudo url with missing schema
Related to T3294 Related to T3781
This commit is contained in:
parent
0f8f293f96
commit
5daead68ad
3 changed files with 23 additions and 7 deletions
|
@ -129,8 +129,7 @@ def is_tarball(urls: List[str], request: Optional[Any] = None) -> Tuple[bool, st
|
|||
url = urls[index]
|
||||
|
||||
try:
|
||||
is_tar = _is_tarball(url)
|
||||
return is_tar, urls[0]
|
||||
return _is_tarball(url), urls[0]
|
||||
except IndexError:
|
||||
if request is None:
|
||||
raise ArtifactNatureUndetected(
|
||||
|
@ -285,15 +284,25 @@ class NixGuixLister(StatelessLister[PageResult]):
|
|||
)
|
||||
elif artifact_type == "url":
|
||||
# It's either a tarball or a file
|
||||
urls = artifact.get("urls")
|
||||
if not urls:
|
||||
origin_urls = artifact.get("urls")
|
||||
if not origin_urls:
|
||||
# Nothing to fetch
|
||||
logger.warning("Skipping url <%s>: empty artifact", artifact)
|
||||
continue
|
||||
|
||||
assert urls is not None
|
||||
assert origin_urls is not None
|
||||
|
||||
# Deal with urls with empty scheme (basic fallback to http)
|
||||
urls = []
|
||||
for url in origin_urls:
|
||||
urlparsed = urlparse(url)
|
||||
if urlparsed.scheme == "":
|
||||
logger.warning("Missing scheme for <%s>, fallback to http", url)
|
||||
fixed_url = f"http://{url}"
|
||||
else:
|
||||
fixed_url = url
|
||||
urls.append(fixed_url)
|
||||
|
||||
# FIXME: T3294: Fix missing scheme in urls
|
||||
origin, *fallback_urls = urls
|
||||
|
||||
integrity = artifact.get("integrity")
|
||||
|
|
|
@ -29,6 +29,13 @@
|
|||
],
|
||||
"integrity": "sha256-bss09x9yOnuW+Q5BHHjf8nNcCNxCKMdl9/2/jKSFcrQ="
|
||||
},
|
||||
{
|
||||
"type": "url",
|
||||
"urls": [
|
||||
"www.roudoudou.com/export/cpc/rasm/rasm_v0117_src.zip"
|
||||
],
|
||||
"integrity": "sha256-wAEswtkl3ulAw3zq4perrGS6Wlww5XXnQYsEAoYT9fI="
|
||||
},
|
||||
{
|
||||
"type": "url",
|
||||
"urls": [
|
||||
|
|
|
@ -146,7 +146,7 @@ def test_is_tarball_complex_with_content_type_result(
|
|||
assert origin == url
|
||||
|
||||
|
||||
def test_lister_nixguix(datadir, swh_scheduler, requests_mock):
|
||||
def test_lister_nixguix_ok(datadir, swh_scheduler, requests_mock):
|
||||
"""NixGuixLister should list all origins per visit type"""
|
||||
url = "https://nix-community.github.io/nixpkgs-swh/sources-unstable.json"
|
||||
origin_upstream = "https://github.com/NixOS/nixpkgs"
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue