reonokiy/swh-lister
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Puppet: Switch artifacts from dict to list

Browse source
This commit is contained in:
Franck Bret 2022-10-25 10:09:10 +02:00
parent ca4ab7f277
commit 8355fee25f
3 changed files with 92 additions and 69 deletions
Download patch file Download diff file Expand all files Collapse all files

27
swh/lister/puppet/__init__.py
View file

@ -38,7 +38,7 @@ this pattern::
"https://forge.puppet.com/modules/{owner}/{pkgname}"
For each origin `last_update`is set via the module "updated_at" value.
For each origin `last_update` is set via the module "updated_at" value.
As the api also returns all existing versions for a package, we build an `artifacts`
dict in `extra_loader_arguments` with the archive tarball corresponding to each
existing versions.
@ -47,20 +47,27 @@ Example for ``file_concat`` module located at
https://forge.puppet.com/modules/electrical/file_concat::
{
"artifacts": {
"1.0.0": {
"url": "https://forgeapi.puppet.com/v3/files/electrical-file_concat-1.0.0.tar.gz", # noqa: B950
"version": "1.0.0",
"filename": "electrical-file_concat-1.0.0.tar.gz",
"last_update": "2015-04-09T12:03:13-07:00",
},
"1.0.1": {
"artifacts": [
{
"url": "https://forgeapi.puppet.com/v3/files/electrical-file_concat-1.0.1.tar.gz", # noqa: B950
"version": "1.0.1",
"filename": "electrical-file_concat-1.0.1.tar.gz",
"last_update": "2015-04-17T01:03:46-07:00",
"checksums": {
"md5": "74901a89544134478c2dfde5efbb7f14",
"sha256": "15e973613ea038d8a4f60bafe2d678f88f53f3624c02df3157c0043f4a400de6", # noqa: B950
},
},
}
{
"url": "https://forgeapi.puppet.com/v3/files/electrical-file_concat-1.0.0.tar.gz", # noqa: B950
"version": "1.0.0",
"filename": "electrical-file_concat-1.0.0.tar.gz",
"last_update": "2015-04-09T12:03:13-07:00",
"checksums": {
"length": 13289,
},
},
],
}
Running tests

22
swh/lister/puppet/lister.py
View file

@ -76,7 +76,7 @@ class PuppetLister(StatelessLister[PuppetListerPage]):
pkgname = entry["name"]
owner = entry["owner"]["slug"]
url = f"https://forge.puppet.com/modules/{owner}/{pkgname}"
artifacts = {}
artifacts = []
for release in entry["releases"]:
# Build an artifact entry following original-artifacts-json specification
# https://docs.softwareheritage.org/devel/swh-storage/extrinsic-metadata-specification.html#original-artifacts-json # noqa: B950
@ -92,15 +92,17 @@ class PuppetLister(StatelessLister[PuppetListerPage]):
# use file length as basic content check instead
checksums["length"] = release["file_size"]
artifacts[release["version"]] = {
"filename": release["file_uri"].split("/")[-1],
"url": urljoin(self.BASE_URL, release["file_uri"]),
"version": release["version"],
"last_update": datetime.strptime(
release["created_at"], dt_parse_pattern
).isoformat(),
"checksums": checksums,
}
artifacts.append(
{
"filename": release["file_uri"].split("/")[-1],
"url": urljoin(self.BASE_URL, release["file_uri"]),
"version": release["version"],
"last_update": datetime.strptime(
release["created_at"], dt_parse_pattern
).isoformat(),
"checksums": checksums,
}
)
yield ListedOrigin(
lister_id=self.lister_obj.id,

112
swh/lister/puppet/tests/test_lister.py
View file

@ -7,19 +7,11 @@ from swh.lister.puppet.lister import PuppetLister
# flake8: noqa: B950
expected_origins = {
"https://forge.puppet.com/modules/electrical/file_concat": {
"artifacts": {
"1.0.0": {
"url": "https://forgeapi.puppet.com/v3/files/electrical-file_concat-1.0.0.tar.gz",
"version": "1.0.0",
"filename": "electrical-file_concat-1.0.0.tar.gz",
"last_update": "2015-04-09T12:03:13-07:00",
"checksums": {
"length": 13289,
},
},
"1.0.1": {
expected_origins = [
{
"url": "https://forge.puppet.com/modules/electrical/file_concat",
"artifacts": [
{
"url": "https://forgeapi.puppet.com/v3/files/electrical-file_concat-1.0.1.tar.gz",
"version": "1.0.1",
"filename": "electrical-file_concat-1.0.1.tar.gz",
@ -29,29 +21,21 @@ expected_origins = {
"sha256": "15e973613ea038d8a4f60bafe2d678f88f53f3624c02df3157c0043f4a400de6",
},
},
}
},
"https://forge.puppet.com/modules/puppetlabs/puppetdb": {
"artifacts": {
"1.0.0": {
"url": "https://forgeapi.puppet.com/v3/files/puppetlabs-puppetdb-1.0.0.tar.gz",
{
"url": "https://forgeapi.puppet.com/v3/files/electrical-file_concat-1.0.0.tar.gz",
"version": "1.0.0",
"filename": "puppetlabs-puppetdb-1.0.0.tar.gz",
"last_update": "2012-09-19T16:51:22-07:00",
"filename": "electrical-file_concat-1.0.0.tar.gz",
"last_update": "2015-04-09T12:03:13-07:00",
"checksums": {
"length": 16336,
"length": 13289,
},
},
"7.9.0": {
"url": "https://forgeapi.puppet.com/v3/files/puppetlabs-puppetdb-7.9.0.tar.gz",
"version": "7.9.0",
"filename": "puppetlabs-puppetdb-7.9.0.tar.gz",
"last_update": "2021-06-24T07:48:54-07:00",
"checksums": {
"length": 42773,
},
},
"7.10.0": {
],
},
{
"url": "https://forge.puppet.com/modules/puppetlabs/puppetdb",
"artifacts": [
{
"url": "https://forgeapi.puppet.com/v3/files/puppetlabs-puppetdb-7.10.0.tar.gz",
"version": "7.10.0",
"filename": "puppetlabs-puppetdb-7.10.0.tar.gz",
@ -61,20 +45,30 @@ expected_origins = {
"sha256": "49b1a542fbd2a1378c16cb04809e0f88bf4f3e45979532294fb1f03f56c97fbb",
},
},
}
},
"https://forge.puppet.com/modules/saz/memcached": {
"artifacts": {
"1.0.0": {
"url": "https://forgeapi.puppet.com/v3/files/saz-memcached-1.0.0.tar.gz",
"version": "1.0.0",
"filename": "saz-memcached-1.0.0.tar.gz",
"last_update": "2011-11-20T13:40:30-08:00",
{
"url": "https://forgeapi.puppet.com/v3/files/puppetlabs-puppetdb-7.9.0.tar.gz",
"version": "7.9.0",
"filename": "puppetlabs-puppetdb-7.9.0.tar.gz",
"last_update": "2021-06-24T07:48:54-07:00",
"checksums": {
"length": 2472,
"length": 42773,
},
},
"8.1.0": {
{
"url": "https://forgeapi.puppet.com/v3/files/puppetlabs-puppetdb-1.0.0.tar.gz",
"version": "1.0.0",
"filename": "puppetlabs-puppetdb-1.0.0.tar.gz",
"last_update": "2012-09-19T16:51:22-07:00",
"checksums": {
"length": 16336,
},
},
],
},
{
"url": "https://forge.puppet.com/modules/saz/memcached",
"artifacts": [
{
"url": "https://forgeapi.puppet.com/v3/files/saz-memcached-8.1.0.tar.gz",
"version": "8.1.0",
"filename": "saz-memcached-8.1.0.tar.gz",
@ -84,9 +78,18 @@ expected_origins = {
"sha256": "883d6186e91c2c3fed13ae2009c3aa596657f6707b76f1f7efc6203c6e4ae986",
},
},
}
{
"url": "https://forgeapi.puppet.com/v3/files/saz-memcached-1.0.0.tar.gz",
"version": "1.0.0",
"filename": "saz-memcached-1.0.0.tar.gz",
"last_update": "2011-11-20T13:40:30-08:00",
"checksums": {
"length": 2472,
},
},
],
},
}
]
def test_puppet_lister(datadir, requests_mock_datadir, swh_scheduler):
@ -100,7 +103,18 @@ def test_puppet_lister(datadir, requests_mock_datadir, swh_scheduler):
assert len(scheduler_origins) == len(expected_origins)
for origin in scheduler_origins:
assert origin.visit_type == "puppet"
assert origin.url in expected_origins
assert origin.extra_loader_arguments == expected_origins[origin.url]
assert [
(
scheduled.visit_type,
scheduled.url,
scheduled.extra_loader_arguments["artifacts"],
)
for scheduled in sorted(scheduler_origins, key=lambda scheduled: scheduled.url)
] == [
(
"puppet",
expected["url"],
expected["artifacts"],
)
for expected in sorted(expected_origins, key=lambda expected: expected["url"])
]