nixguix: Deal with connection error with server
When that arises, we skip the origins. Related to T3781
This commit is contained in:
parent
d92474bbda
commit
0f8f293f96
3 changed files with 19 additions and 6 deletions
|
@ -25,7 +25,7 @@ from typing import Any, Dict, Iterator, List, Optional, Tuple, Union
|
|||
from urllib.parse import urlparse
|
||||
|
||||
import requests
|
||||
from requests.exceptions import InvalidSchema, SSLError
|
||||
from requests.exceptions import ConnectionError, InvalidSchema, SSLError
|
||||
|
||||
from swh.core.github.utils import GitHubSession
|
||||
from swh.core.tarball import MIMETYPE_TO_ARCHIVE_FORMAT
|
||||
|
@ -143,7 +143,7 @@ def is_tarball(urls: List[str], request: Optional[Any] = None) -> Tuple[bool, st
|
|||
|
||||
try:
|
||||
response = request.head(url)
|
||||
except (InvalidSchema, SSLError):
|
||||
except (InvalidSchema, SSLError, ConnectionError):
|
||||
raise ArtifactNatureUndetected(
|
||||
f"Cannot determine artifact type from url <{url}>"
|
||||
)
|
||||
|
|
|
@ -20,6 +20,13 @@
|
|||
],
|
||||
"integrity": "sha256-bss09x9yOnuW+Q5BHHjf8nNcCNxCKMdl9/2/jKSFcrQ="
|
||||
},
|
||||
{
|
||||
"type": "url",
|
||||
"urls": [
|
||||
"https://git-tails.immerda.ch/onioncircuits"
|
||||
],
|
||||
"integrity": "sha256-lV3xiWUZmSnt4LW0ni/sUyC/bbtaxkTzvFLFtJKLuI4="
|
||||
},
|
||||
{
|
||||
"type": "url",
|
||||
"urls": [ "unknown://example.org/wrong-scheme-so-skipped.txt" ],
|
||||
|
|
|
@ -11,6 +11,7 @@ from typing import Dict, List
|
|||
|
||||
import pytest
|
||||
import requests
|
||||
from requests.exceptions import ConnectionError, InvalidSchema, SSLError
|
||||
|
||||
from swh.lister import TARBALL_EXTENSIONS
|
||||
from swh.lister.nixguix.lister import (
|
||||
|
@ -229,16 +230,21 @@ def test_lister_nixguix_mostly_noop(datadir, swh_scheduler, requests_mock):
|
|||
"https://crates.io/api/v1/0.1.5/no-extension-and-head-404-so-skipped",
|
||||
status_code=404,
|
||||
)
|
||||
# Will raise for that origin, this will get ignored as we cannot determine anything
|
||||
# Invalid schema for that origin (and no extension), so skip origin
|
||||
# from its name
|
||||
requests_mock.head(
|
||||
"ftp://ftp.ourproject.org/file-with-no-extension",
|
||||
exc=requests.exceptions.InvalidSchema,
|
||||
exc=InvalidSchema,
|
||||
)
|
||||
# Cannot communicate with an expired cert so skip
|
||||
# Cannot communicate with an expired cert, so skip origin
|
||||
requests_mock.head(
|
||||
"https://code.9front.org/hg/plan9front",
|
||||
exc=requests.exceptions.SSLError,
|
||||
exc=SSLError,
|
||||
)
|
||||
# Cannot connect to the site, so skip origin
|
||||
requests_mock.head(
|
||||
"https://git-tails.immerda.ch/onioncircuits",
|
||||
exc=ConnectionError,
|
||||
)
|
||||
|
||||
listed_result = lister.run()
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue