nixguix: Deal with connection error with server

When that arises, we skip the origins.

Related to T3781
This commit is contained in:
Antoine R. Dumont (@ardumont) 2022-10-04 14:57:01 +02:00
parent d92474bbda
commit 0f8f293f96
No known key found for this signature in database
GPG key ID: 52E2E9840D10C3B8
3 changed files with 19 additions and 6 deletions

View file

@ -25,7 +25,7 @@ from typing import Any, Dict, Iterator, List, Optional, Tuple, Union
from urllib.parse import urlparse
import requests
from requests.exceptions import InvalidSchema, SSLError
from requests.exceptions import ConnectionError, InvalidSchema, SSLError
from swh.core.github.utils import GitHubSession
from swh.core.tarball import MIMETYPE_TO_ARCHIVE_FORMAT
@ -143,7 +143,7 @@ def is_tarball(urls: List[str], request: Optional[Any] = None) -> Tuple[bool, st
try:
response = request.head(url)
except (InvalidSchema, SSLError):
except (InvalidSchema, SSLError, ConnectionError):
raise ArtifactNatureUndetected(
f"Cannot determine artifact type from url <{url}>"
)

View file

@ -20,6 +20,13 @@
],
"integrity": "sha256-bss09x9yOnuW+Q5BHHjf8nNcCNxCKMdl9/2/jKSFcrQ="
},
{
"type": "url",
"urls": [
"https://git-tails.immerda.ch/onioncircuits"
],
"integrity": "sha256-lV3xiWUZmSnt4LW0ni/sUyC/bbtaxkTzvFLFtJKLuI4="
},
{
"type": "url",
"urls": [ "unknown://example.org/wrong-scheme-so-skipped.txt" ],

View file

@ -11,6 +11,7 @@ from typing import Dict, List
import pytest
import requests
from requests.exceptions import ConnectionError, InvalidSchema, SSLError
from swh.lister import TARBALL_EXTENSIONS
from swh.lister.nixguix.lister import (
@ -229,16 +230,21 @@ def test_lister_nixguix_mostly_noop(datadir, swh_scheduler, requests_mock):
"https://crates.io/api/v1/0.1.5/no-extension-and-head-404-so-skipped",
status_code=404,
)
# Will raise for that origin, this will get ignored as we cannot determine anything
# Invalid schema for that origin (and no extension), so skip origin
# from its name
requests_mock.head(
"ftp://ftp.ourproject.org/file-with-no-extension",
exc=requests.exceptions.InvalidSchema,
exc=InvalidSchema,
)
# Cannot communicate with an expired cert so skip
# Cannot communicate with an expired cert, so skip origin
requests_mock.head(
"https://code.9front.org/hg/plan9front",
exc=requests.exceptions.SSLError,
exc=SSLError,
)
# Cannot connect to the site, so skip origin
requests_mock.head(
"https://git-tails.immerda.ch/onioncircuits",
exc=ConnectionError,
)
listed_result = lister.run()