reonokiy/swh-lister
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Send package artifact checksums to loaders when info is available

Browse source 
In listers collecting artifacts for each package to load, add artifacts
checksums, when that info is available, in parameters sent to loaders
in order to check downloaded artifact integrity.
This commit is contained in:
Antoine Lambert 2022-09-29 17:23:33 +02:00
parent 6f40d2c1a5
commit fa1205c4df
9 changed files with 487 additions and 143 deletions
Download patch file Download diff file Expand all files Collapse all files

8
swh/lister/arch/lister.py
View file

@ -453,6 +453,14 @@ class ArchLister(StatelessLister[ArchListerPage]):
"length": version["length"],
}
)
if version["version"] == origin["version"]:
artifacts[-1]["checksums"] = {
"md5": origin["data"]["md5sum"],
"sha256": origin["data"]["sha256sum"],
}
else:
artifacts[-1]["checksums"] = {"length": version["length"]}
arch_metadata.append(
{
"version": version["version"],

515
swh/lister/arch/tests/test_lister.py
View file

File diff suppressed because it is too large Load diff

1
swh/lister/cpan/tests/test_lister.py
View file

@ -2,6 +2,7 @@
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
from swh.lister.cpan.lister import CpanLister
expected_origins = [

2
swh/lister/cran/list_all_packages.R
View file

@ -4,6 +4,6 @@
# all the packages of R and their description, then convert the API
# response to JSON string and print it
db <- tools::CRAN_package_db()[, c("Package", "Version", "Packaged")]
db <- tools::CRAN_package_db()[, c("Package", "Version", "Packaged", "MD5sum")]
dbjson <- jsonlite::toJSON(db)
print(dbjson)

1
swh/lister/cran/lister.py
View file

@ -68,6 +68,7 @@ class CRANLister(StatelessLister[PageType]):
"url": artifact_url,
"version": package_info["Version"],
"package": package_info["Package"],
"checksums": {"md5": package_info["MD5sum"]},
}
]
},

48
swh/lister/cran/tests/data/list-r-packages.json
View file

@ -1,40 +1,32 @@
[
{
"Package": "SeleMix",
"Version": "1.0.2",
"Packaged": "2020-11-28 22:16:43 UTC; Teresa"
"Package": "cNORM",
"Version": "3.0.2",
"Packaged": "2022-06-12 08:46:39 UTC; gbpa005",
"MD5sum": "d878686afc17b990e500dc88afb3a990"
},
{
"Package": "plink",
"Version": "1.5-1",
"Packaged": "2017-04-26 11:36:15 UTC; Jonathan"
"Package": "CNprep",
"Version": "2.2",
"Packaged": "2022-05-23 23:58:37 UTC; Astrid",
"MD5sum": "4b6ddc37df607c79b7fb50a96a57197f"
},
{
"Package": "jsonlite",
"Version": "1.7.2",
"Packaged": "2020-12-09 13:54:18 UTC; jeroen"
"Package": "CNPS",
"Version": "1.0.0",
"Packaged": "2021-05-21 16:55:04 UTC; Surface",
"MD5sum": "deac071a9387e3a296481d041e6d09ee"
},
{
"Package": "Records",
"Version": "1.0",
"Packaged": "2012-10-29 08:57:37 UTC; ripley"
"Package": "cns",
"Version": "0.1.0",
"Packaged": "2021-07-16 19:30:51 UTC; nfultz",
"MD5sum": "3ad5a474260dbacb889be461b826a73b"
},
{
"Package": "scRNAtools",
"Version": "1.0",
"Packaged": "2018-07-04 00:49:45 UTC; dell"
},
{
"Package": "Deriv",
"Version": "4.1.2",
"Packaged": "2020-12-10 11:12:28 UTC; sokol"
},
{
"Package": "BayesValidate",
"Version": "0.0",
"Packaged": "Thu Mar 30 10:48:35 2006; hornik"
"Package": "cnum",
"Version": "0.1.3",
"Packaged": "2021-01-11 13:24:52 UTC; Elgar",
"MD5sum": "3cb5ab3fdaf4277d1ebfbe147e8990e1"
}
]

1
swh/lister/cran/tests/test_lister.py
View file

@ -109,6 +109,7 @@ def test_cran_lister_cran(datadir, swh_scheduler, mocker):
"url": artifact_url,
"version": package_info["Version"],
"package": package_info["Package"],
"checksums": {"md5": package_info["MD5sum"]},
}
]
}

13
swh/lister/puppet/lister.py
View file

@ -80,6 +80,18 @@ class PuppetLister(StatelessLister[PuppetListerPage]):
for release in entry["releases"]:
# Build an artifact entry following original-artifacts-json specification
# https://docs.softwareheritage.org/devel/swh-storage/extrinsic-metadata-specification.html#original-artifacts-json # noqa: B950
checksums = {}
if release["version"] == entry["current_release"]["version"]:
# checksums are only available for current release
for checksum in ("md5", "sha256"):
checksums[checksum] = entry["current_release"][
f"file_{checksum}"
]
else:
# use file length as basic content check instead
checksums["length"] = release["file_size"]
artifacts[release["version"]] = {
"filename": release["file_uri"].split("/")[-1],
"url": urljoin(self.BASE_URL, release["file_uri"]),
@ -87,6 +99,7 @@ class PuppetLister(StatelessLister[PuppetListerPage]):
"last_update": datetime.strptime(
release["created_at"], dt_parse_pattern
).isoformat(),
"checksums": checksums,
}
yield ListedOrigin(

41
swh/lister/puppet/tests/test_lister.py
View file

@ -2,60 +2,87 @@
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
from swh.lister.puppet.lister import PuppetLister
# flake8: noqa: B950
expected_origins = {
"https://forge.puppet.com/modules/electrical/file_concat": {
"artifacts": {
"1.0.0": {
"url": "https://forgeapi.puppet.com/v3/files/electrical-file_concat-1.0.0.tar.gz", # noqa: B950
"url": "https://forgeapi.puppet.com/v3/files/electrical-file_concat-1.0.0.tar.gz",
"version": "1.0.0",
"filename": "electrical-file_concat-1.0.0.tar.gz",
"last_update": "2015-04-09T12:03:13-07:00",
"checksums": {
"length": 13289,
},
},
"1.0.1": {
"url": "https://forgeapi.puppet.com/v3/files/electrical-file_concat-1.0.1.tar.gz", # noqa: B950
"url": "https://forgeapi.puppet.com/v3/files/electrical-file_concat-1.0.1.tar.gz",
"version": "1.0.1",
"filename": "electrical-file_concat-1.0.1.tar.gz",
"last_update": "2015-04-17T01:03:46-07:00",
"checksums": {
"md5": "74901a89544134478c2dfde5efbb7f14",
"sha256": "15e973613ea038d8a4f60bafe2d678f88f53f3624c02df3157c0043f4a400de6",
},
},
}
},
"https://forge.puppet.com/modules/puppetlabs/puppetdb": {
"artifacts": {
"1.0.0": {
"url": "https://forgeapi.puppet.com/v3/files/puppetlabs-puppetdb-1.0.0.tar.gz", # noqa: B950
"url": "https://forgeapi.puppet.com/v3/files/puppetlabs-puppetdb-1.0.0.tar.gz",
"version": "1.0.0",
"filename": "puppetlabs-puppetdb-1.0.0.tar.gz",
"last_update": "2012-09-19T16:51:22-07:00",
"checksums": {
"length": 16336,
},
},
"7.9.0": {
"url": "https://forgeapi.puppet.com/v3/files/puppetlabs-puppetdb-7.9.0.tar.gz", # noqa: B950
"url": "https://forgeapi.puppet.com/v3/files/puppetlabs-puppetdb-7.9.0.tar.gz",
"version": "7.9.0",
"filename": "puppetlabs-puppetdb-7.9.0.tar.gz",
"last_update": "2021-06-24T07:48:54-07:00",
"checksums": {
"length": 42773,
},
},
"7.10.0": {
"url": "https://forgeapi.puppet.com/v3/files/puppetlabs-puppetdb-7.10.0.tar.gz", # noqa: B950
"url": "https://forgeapi.puppet.com/v3/files/puppetlabs-puppetdb-7.10.0.tar.gz",
"version": "7.10.0",
"filename": "puppetlabs-puppetdb-7.10.0.tar.gz",
"last_update": "2021-12-16T14:57:46-08:00",
"checksums": {
"md5": "e91a2074ca8d94a8b3ff7f6c8bbf12bc",
"sha256": "49b1a542fbd2a1378c16cb04809e0f88bf4f3e45979532294fb1f03f56c97fbb",
},
},
}
},
"https://forge.puppet.com/modules/saz/memcached": {
"artifacts": {
"1.0.0": {
"url": "https://forgeapi.puppet.com/v3/files/saz-memcached-1.0.0.tar.gz", # noqa: B950
"url": "https://forgeapi.puppet.com/v3/files/saz-memcached-1.0.0.tar.gz",
"version": "1.0.0",
"filename": "saz-memcached-1.0.0.tar.gz",
"last_update": "2011-11-20T13:40:30-08:00",
"checksums": {
"length": 2472,
},
},
"8.1.0": {
"url": "https://forgeapi.puppet.com/v3/files/saz-memcached-8.1.0.tar.gz", # noqa: B950
"url": "https://forgeapi.puppet.com/v3/files/saz-memcached-8.1.0.tar.gz",
"version": "8.1.0",
"filename": "saz-memcached-8.1.0.tar.gz",
"last_update": "2022-07-11T03:34:55-07:00",
"checksums": {
"md5": "aadf80fba5848909429eb002ee1927ea",
"sha256": "883d6186e91c2c3fed13ae2009c3aa596657f6707b76f1f7efc6203c6e4ae986",
},
},
}
},