reonokiy/swh-lister
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Puppet: Lister for Puppet modules

Browse source 
The puppet lister retrieves origins from from https://forge.puppet.com/modules

Related T4519
This commit is contained in:
Franck Bret 2022-09-26 11:25:23 +02:00
parent a4aec3894e
commit cd596eb2b4
9 changed files with 910 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

1
setup.py
View file

@ -77,6 +77,7 @@ setup(
lister.packagist=swh.lister.packagist:register
lister.phabricator=swh.lister.phabricator:register
lister.pubdev=swh.lister.pubdev:register
lister.puppet=swh.lister.puppet:register
lister.pypi=swh.lister.pypi:register
lister.sourceforge=swh.lister.sourceforge:register
lister.tuleap=swh.lister.tuleap:register

101
swh/lister/puppet/__init__.py Normal file
View file

@ -0,0 +1,101 @@
# Copyright (C) 2022 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
"""
Puppet lister
=============
The Puppet lister list origins from `Puppet Forge`_.
Puppet Forge is a package manager for Puppet modules.
As of September 2022 `Puppet Forge`_ list 6917 package names.
Origins retrieving strategy
---------------------------
To get a list of all package names we call an `http api endpoint`_ which have a
`getModules`_ endpoint.
It returns a paginated list of results and a `next` url.
The api follow `OpenApi 3.0 specifications`.
Page listing
------------
Each page returns a list of ``results`` which are raw data from api response.
The results size is 100 as 100 is the maximum limit size allowed by the api.
Origins from page
-----------------
The lister yields one hundred origin url per page.
Origin url is the html page corresponding to a package name on the forge, following
this pattern::
"https://forge.puppet.com/modules/{owner}/{pkgname}"
For each origin `last_update`is set via the module "updated_at" value.
As the api also returns all existing versions for a package, we build an `artifacts`
dict in `extra_loader_arguments` with the archive tarball corresponding to each
existing versions.
Example for ``file_concat`` module located at
https://forge.puppet.com/modules/electrical/file_concat::
{
"artifacts": {
"1.0.0": {
"url": "https://forgeapi.puppet.com/v3/files/electrical-file_concat-1.0.0.tar.gz", # noqa: B950
"version": "1.0.0",
"filename": "electrical-file_concat-1.0.0.tar.gz",
"last_update": "2015-04-09T12:03:13-07:00",
},
"1.0.1": {
"url": "https://forgeapi.puppet.com/v3/files/electrical-file_concat-1.0.1.tar.gz", # noqa: B950
"version": "1.0.1",
"filename": "electrical-file_concat-1.0.1.tar.gz",
"last_update": "2015-04-17T01:03:46-07:00",
},
}
}
Running tests
-------------
Activate the virtualenv and run from within swh-lister directory::
pytest -s -vv --log-cli-level=DEBUG swh/lister/puppet/tests
Testing with Docker
-------------------
Change directory to swh/docker then launch the docker environment::
docker compose up -d
Then schedule a Puppet listing task::
docker compose exec swh-scheduler swh scheduler task add -p oneshot list-puppet
You can follow lister execution by displaying logs of swh-lister service::
docker compose logs -f swh-lister
.. _Puppet Forge: https://forge.puppet.com/
.. _http api endpoint: https://forgeapi.puppet.com/
.. _getModules: https://forgeapi.puppet.com/#tag/Module-Operations/operation/getModules
"""
def register():
from .lister import PuppetLister
return {
"lister": PuppetLister,
"task_modules": ["%s.tasks" % __name__],
}

98
swh/lister/puppet/lister.py Normal file
View file

@ -0,0 +1,98 @@
# Copyright (C) 2022 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
from datetime import datetime
import logging
from typing import Any, Dict, Iterator, List, Optional
from urllib.parse import urljoin
from swh.scheduler.interface import SchedulerInterface
from swh.scheduler.model import ListedOrigin
from ..pattern import CredentialsType, StatelessLister
logger = logging.getLogger(__name__)
# Aliasing the page results returned by `get_pages` method from the lister.
PuppetListerPage = List[Dict[str, Any]]
class PuppetLister(StatelessLister[PuppetListerPage]):
"""The Puppet lister list origins from 'Puppet Forge'"""
LISTER_NAME = "puppet"
VISIT_TYPE = "puppet"
INSTANCE = "puppet"
BASE_URL = "https://forgeapi.puppet.com/"
def __init__(
self,
scheduler: SchedulerInterface,
credentials: Optional[CredentialsType] = None,
):
super().__init__(
scheduler=scheduler,
credentials=credentials,
instance=self.INSTANCE,
url=self.BASE_URL,
)
def get_pages(self) -> Iterator[PuppetListerPage]:
"""Yield an iterator which returns 'page'
It request the http api endpoint to get a paginated results of modules,
and retrieve a `next` url. It ends when `next` json value is `null`.
Open Api specification for getModules endpoint:
https://forgeapi.puppet.com/#tag/Module-Operations/operation/getModules
"""
# limit = 100 is the max value for pagination
limit: int = 100
response = self.http_request(
f"{self.BASE_URL}v3/modules", params={"limit": limit}
)
data: Dict[str, Any] = response.json()
yield data["results"]
while data["pagination"]["next"]:
response = self.http_request(
urljoin(self.BASE_URL, data["pagination"]["next"])
)
data = response.json()
yield data["results"]
def get_origins_from_page(self, page: PuppetListerPage) -> Iterator[ListedOrigin]:
"""Iterate on all pages and yield ListedOrigin instances."""
assert self.lister_obj.id is not None
dt_parse_pattern = "%Y-%m-%d %H:%M:%S %z"
for entry in page:
last_update = datetime.strptime(entry["updated_at"], dt_parse_pattern)
pkgname = entry["name"]
owner = entry["owner"]["slug"]
url = f"https://forge.puppet.com/modules/{owner}/{pkgname}"
artifacts = {}
for release in entry["releases"]:
# Build an artifact entry following original-artifacts-json specification
# https://docs.softwareheritage.org/devel/swh-storage/extrinsic-metadata-specification.html#original-artifacts-json # noqa: B950
artifacts[release["version"]] = {
"filename": release["file_uri"].split("/")[-1],
"url": urljoin(self.BASE_URL, release["file_uri"]),
"version": release["version"],
"last_update": datetime.strptime(
release["created_at"], dt_parse_pattern
).isoformat(),
}
yield ListedOrigin(
lister_id=self.lister_obj.id,
visit_type=self.VISIT_TYPE,
url=url,
last_update=last_update,
extra_loader_arguments={"artifacts": artifacts},
)

19
swh/lister/puppet/tasks.py Normal file
View file

@ -0,0 +1,19 @@
# Copyright (C) 2022 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
from celery import shared_task
from swh.lister.puppet.lister import PuppetLister
@shared_task(name=__name__ + ".PuppetListerTask")
def list_puppet(**lister_args):
"""Lister task for Puppet"""
return PuppetLister.from_configfile(**lister_args).run().dict()
@shared_task(name=__name__ + ".ping")
def _ping():
return "OK"

0
swh/lister/puppet/tests/__init__.py Normal file
View file

391
swh/lister/puppet/tests/data/https_forgeapi.puppet.com/v3_modules,limit=100 Normal file
View file

File diff suppressed because one or more lines are too long

190
swh/lister/puppet/tests/data/https_forgeapi.puppet.com/v3_modules,limit=100,offset=100 Normal file
View file

@ -0,0 +1,190 @@
{
"pagination": {
"limit": 100,
"offset": 100,
"first": "/v3/modules?limit=100&offset=0",
"previous": "/v3/modules?limit=100&offset=0",
"current": "/v3/modules?limit=100&offset=100",
"next": null,
"total": 7301
},
"results": [
{
"uri": "/v3/modules/electrical-file_concat",
"slug": "electrical-file_concat",
"name": "file_concat",
"downloads": 2293802,
"created_at": "2015-04-09 12:03:13 -0700",
"updated_at": "2016-12-28 20:00:02 -0800",
"deprecated_at": null,
"deprecated_for": null,
"superseded_by": null,
"supported": false,
"endorsement": null,
"module_group": "base",
"owner": {
"uri": "/v3/users/electrical",
"slug": "electrical",
"username": "electrical",
"gravatar_id": "46dbd1ee4484b8e993466bd2209858cf"
},
"premium": false,
"current_release": {
"uri": "/v3/releases/electrical-file_concat-1.0.1",
"slug": "electrical-file_concat-1.0.1",
"module": {
"uri": "/v3/modules/electrical-file_concat",
"slug": "electrical-file_concat",
"name": "file_concat",
"deprecated_at": null,
"owner": {
"uri": "/v3/users/electrical",
"slug": "electrical",
"username": "electrical",
"gravatar_id": "46dbd1ee4484b8e993466bd2209858cf"
}
},
"version": "1.0.1",
"metadata": {
"name": "electrical-file_concat",
"version": "1.0.1",
"author": "electrical",
"summary": "Library for concatenating different files into 1",
"license": "Apache License, Version 2.0",
"source": "https://github.com/electrical/puppet-lib-file_concat",
"project_page": "https://github.com/electrical/puppet-lib-file_concat",
"issues_url": "https://github.com/electrical/puppet-lib-file_concat/issues",
"operatingsystem_support": [
{
"operatingsystem": "RedHat",
"operatingsystemrelease": [
"5",
"6",
"7"
]
},
{
"operatingsystem": "CentOS",
"operatingsystemrelease": [
"5",
"6",
"7"
]
},
{
"operatingsystem": "OracleLinux",
"operatingsystemrelease": [
"5",
"6",
"7"
]
},
{
"operatingsystem": "Scientific",
"operatingsystemrelease": [
"5",
"6",
"7"
]
},
{
"operatingsystem": "SLES",
"operatingsystemrelease": [
"10 SP4",
"11 SP1",
"12"
]
},
{
"operatingsystem": "Debian",
"operatingsystemrelease": [
"6",
"7"
]
},
{
"operatingsystem": "Ubuntu",
"operatingsystemrelease": [
"10.04",
"12.04",
"14.04"
]
},
{
"operatingsystem": "Solaris",
"operatingsystemrelease": [
"10",
"11"
]
}
],
"requirements": [
{
"name": "pe",
"version_requirement": "3.x"
},
{
"name": "puppet",
"version_requirement": "3.x"
}
],
"description": "Library for concatenating different files into 1",
"dependencies": [
]
},
"tags": [
],
"supported": false,
"pdk": false,
"validation_score": 62,
"file_uri": "/v3/files/electrical-file_concat-1.0.1.tar.gz",
"file_size": 13335,
"file_md5": "74901a89544134478c2dfde5efbb7f14",
"file_sha256": "15e973613ea038d8a4f60bafe2d678f88f53f3624c02df3157c0043f4a400de6",
"downloads": 2291838,
"readme": "# puppet-lib-file_concat\n\n#### Table of Contents\n\n1. [Overview](#overview)\n2. [Usage - Configuration options and additional functionality](#usage)\n3. [Limitations - OS compatibility, etc.](#limitations)\n4. [Development - Guide for contributing to the module](#development)\n\n## Overview\n\nLibrary for concatenating multiple files into 1.\n\n## Usage\n\n### Creating a file fragment\n\nCreates a file fragment to be collected by file_concat based on the tag.\n\nExample with exported resource:\n\n @@file_fragment { \"uniqe_name_${::fqdn}\":\n tag => 'unique_tag', # Mandatory.\n order => 10, # Optional. Defaults to 10.\n content => 'some content' # OR\n content => template('template.erb') # OR\n source => 'puppet:///path/to/file'\n }\n\n### Concatenating file fragments into one file\n\nGets all the file fragments and puts these into the target file.\nThis will mostly be used with exported resources.\n\nexample:\n \n File_fragment <<| tag == 'unique_tag' |>>\n\n file_concat { '/tmp/file':\n tag => 'unique_tag', # Mandatory\n path => '/tmp/file', # Optional. If given it overrides the resource name.\n owner => 'root', # Optional. Defaults to undef.\n group => 'root', # Optional. Defaults to undef.\n mode => '0644' # Optional. Defaults to undef.\n order => 'numeric' # Optional. Set to 'numeric' or 'alpha'. Defaults to numeric.\n replace => true # Optional. Boolean Value. Defaults to true.\n backup => false # Optional. true, false, 'puppet', or a string. Defaults to 'puppet' for Filebucketing.\n }\n\n## Limitations\n\n## Development\n\n",
"changelog": "##1.0.1 ( Apr 17, 2015 )\n\n###Summary\nBugfix release\n\n####Features\n\n####Bugfixes\n* Fix windows support by not defaulting owner,group and mode values\n\n####Changes\n\n####Testing changes\n\n####Known bugs\n\n\n##1.0.0 ( Apr 09, 2015 )\n\n###Summary\nMajor release.\nThe module has been moved from the ispavailability account on Forge to elecrical.\n\n####Features\n* Major refactoring to enhance functionality\n* Re-use existing file resource to avoid code duplication\n* Make the module more compatiable with puppetlabs-concat\n* Support array of sources\n\n####Bugfixes\n\n####Changes\n\n####Testing changes\n* Add centos 7 acceptance testing\n* Add tests for user/group/mode options\n\n####Known bugs\n\n##0.3.0 ( Mar 26, 2015 )\n\n###Summary\nThis release adds windows support to the library.\n\n####Features\n* Added windows support to the library.\n\n####Bugfixes\n\n####Changes\n\n####Testing changes\n\n####Known bugs\n\n##0.2.1 ( Mar 25, 2015 )\n\n###Summary\nBugfix release\n\n####Features\n\n####Bugfixes\n* Fix a bug caused by some refactoring\n\n####Changes\n\n####Testing changes\n\n####Known bugs\n* Windows is not supported yet\n\n##0.2.0 ( Mar 25, 2015 )\n\n###Summary\nWith this release Ive done several code cleanups and added some basic tests.\nAlso support for puppet-server has been fixed\n\n####Features\n\n####Bugfixes\n* Remove unnecessary require which fixed support for puppet-server\n\n####Changes\n* Added some basic files\n* Implemented rubocop for style checking\n\n####Testing changes\n* Implemented basic acceptance tests\n\n####Known bugs\n* Windows is not supported yet\n\n##0.1.0 ( Jan 21, 2014 )\n Rewrite of the fragment ordering part.\n Fragments are now first ordered based on the order number and then on the resource name.\n Convert `order` parameter to string to support integer values when using Hiera/YAML ( PR#3 by Michael G. Noll )\n\n##0.0.2 ( Mar 03, 2013 )\n Adding source variable option to file_fragment type\n\n##0.0.1 ( Jan 13, 2013 )\n Initial release of the module\n",
"license": "Copyright (c) 2013-2015 Richard Pijnenbug <richard@ispavailability.com>\nCopyright (c) 2012 Simon Effenberg <savar@schuldeigen.de>\nCopyright (c) 2012 Uwe Stuehler <uwe@bsdx.de>\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n",
"reference": null,
"malware_scan": null,
"tasks": [
],
"plans": [
],
"created_at": "2015-04-17 01:03:46 -0700",
"updated_at": "2019-07-03 15:45:50 -0700",
"deleted_at": null,
"deleted_for": null
},
"releases": [
{
"uri": "/v3/releases/electrical-file_concat-1.0.1",
"slug": "electrical-file_concat-1.0.1",
"version": "1.0.1",
"supported": false,
"created_at": "2015-04-17 01:03:46 -0700",
"deleted_at": null,
"file_uri": "/v3/files/electrical-file_concat-1.0.1.tar.gz",
"file_size": 13335
},
{
"uri": "/v3/releases/electrical-file_concat-1.0.0",
"slug": "electrical-file_concat-1.0.0",
"version": "1.0.0",
"supported": false,
"created_at": "2015-04-09 12:03:13 -0700",
"deleted_at": null,
"file_uri": "/v3/files/electrical-file_concat-1.0.0.tar.gz",
"file_size": 13289
}
],
"feedback_score": null,
"homepage_url": "https://github.com/electrical/puppet-lib-file_concat",
"issues_url": "https://github.com/electrical/puppet-lib-file_concat/issues"
}
]
}

79
swh/lister/puppet/tests/test_lister.py Normal file
View file

@ -0,0 +1,79 @@
# Copyright (C) 2022 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
from swh.lister.puppet.lister import PuppetLister
expected_origins = {
"https://forge.puppet.com/modules/electrical/file_concat": {
"artifacts": {
"1.0.0": {
"url": "https://forgeapi.puppet.com/v3/files/electrical-file_concat-1.0.0.tar.gz", # noqa: B950
"version": "1.0.0",
"filename": "electrical-file_concat-1.0.0.tar.gz",
"last_update": "2015-04-09T12:03:13-07:00",
},
"1.0.1": {
"url": "https://forgeapi.puppet.com/v3/files/electrical-file_concat-1.0.1.tar.gz", # noqa: B950
"version": "1.0.1",
"filename": "electrical-file_concat-1.0.1.tar.gz",
"last_update": "2015-04-17T01:03:46-07:00",
},
}
},
"https://forge.puppet.com/modules/puppetlabs/puppetdb": {
"artifacts": {
"1.0.0": {
"url": "https://forgeapi.puppet.com/v3/files/puppetlabs-puppetdb-1.0.0.tar.gz", # noqa: B950
"version": "1.0.0",
"filename": "puppetlabs-puppetdb-1.0.0.tar.gz",
"last_update": "2012-09-19T16:51:22-07:00",
},
"7.9.0": {
"url": "https://forgeapi.puppet.com/v3/files/puppetlabs-puppetdb-7.9.0.tar.gz", # noqa: B950
"version": "7.9.0",
"filename": "puppetlabs-puppetdb-7.9.0.tar.gz",
"last_update": "2021-06-24T07:48:54-07:00",
},
"7.10.0": {
"url": "https://forgeapi.puppet.com/v3/files/puppetlabs-puppetdb-7.10.0.tar.gz", # noqa: B950
"version": "7.10.0",
"filename": "puppetlabs-puppetdb-7.10.0.tar.gz",
"last_update": "2021-12-16T14:57:46-08:00",
},
}
},
"https://forge.puppet.com/modules/saz/memcached": {
"artifacts": {
"1.0.0": {
"url": "https://forgeapi.puppet.com/v3/files/saz-memcached-1.0.0.tar.gz", # noqa: B950
"version": "1.0.0",
"filename": "saz-memcached-1.0.0.tar.gz",
"last_update": "2011-11-20T13:40:30-08:00",
},
"8.1.0": {
"url": "https://forgeapi.puppet.com/v3/files/saz-memcached-8.1.0.tar.gz", # noqa: B950
"version": "8.1.0",
"filename": "saz-memcached-8.1.0.tar.gz",
"last_update": "2022-07-11T03:34:55-07:00",
},
}
},
}
def test_puppet_lister(datadir, requests_mock_datadir, swh_scheduler):
lister = PuppetLister(scheduler=swh_scheduler)
res = lister.run()
assert res.pages == 2
assert res.origins == 1 + 1 + 1
scheduler_origins = swh_scheduler.get_listed_origins(lister.lister_obj.id).results
assert len(scheduler_origins) == len(expected_origins)
for origin in scheduler_origins:
assert origin.visit_type == "puppet"
assert origin.url in expected_origins
assert origin.extra_loader_arguments == expected_origins[origin.url]

31
swh/lister/puppet/tests/test_tasks.py Normal file
View file

@ -0,0 +1,31 @@
# Copyright (C) 2022 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
from swh.lister.pattern import ListerStats
def test_puppet_ping(swh_scheduler_celery_app, swh_scheduler_celery_worker):
res = swh_scheduler_celery_app.send_task("swh.lister.puppet.tasks.ping")
assert res
res.wait()
assert res.successful()
assert res.result == "OK"
def test_puppet_lister(swh_scheduler_celery_app, swh_scheduler_celery_worker, mocker):
# setup the mocked PuppetLister
lister = mocker.patch("swh.lister.puppet.tasks.PuppetLister")
lister.from_configfile.return_value = lister
stats = ListerStats(pages=42, origins=42)
lister.run.return_value = stats
res = swh_scheduler_celery_app.send_task("swh.lister.puppet.tasks.PuppetListerTask")
assert res
res.wait()
assert res.successful()
assert res.result == stats.dict()
lister.from_configfile.assert_called_once_with()
lister.run.assert_called_once_with()