reonokiy/swh-lister
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

rubygems: Use gems database dump to improve listing output

Browse source 
Instead of using an undocumented rubygems HTTP endpoint that only
gives us the names of the gems, prefer to exploit the daily PostgreSQL
dump of the rubygems.org database.

It enables to list all gems but also all versions of a gem and its
release artifacts. For each relase artifact, the following info are
extracted: version, download URL, sha256 checksum, release date
plus a couple of extra metadata.

The lister will now set list of artifacts and list of metadata as extra
loader arguments when sending a listed origin to the scheduler database.
A last_update date is also computed which should ensure loading tasks
for rubygems will be scheduled only when new releases are available since
last loadings.

To be noted, the lister will spawn a temporary postgres instance so this
require the initdb executable from postgres server installation to be
available in the execution environment.

Related to T1777
This commit is contained in:
Antoine Lambert 2022-10-06 17:51:27 +02:00
parent c22f41a6d7
commit 108816f232
8 changed files with 378 additions and 50 deletions
Download patch file Download diff file Expand all files Collapse all files

6
mypy.ini
View file

@ -42,3 +42,9 @@ ignore_missing_imports = True
[mypy-dulwich.*]
ignore_missing_imports = True
[mypy-testing.postgresql.*]
ignore_missing_imports = True
[mypy-psycopg2.*]
ignore_missing_imports = True

2
requirements.txt
View file

@ -7,3 +7,5 @@ launchpadlib
tenacity >= 6.2
lxml
dulwich
testing.postgresql
psycopg2

201
swh/lister/rubygems/lister.py
View file

@ -3,8 +3,20 @@
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
import base64
from datetime import timezone
import gzip
import logging
from typing import Iterator, List, Optional, Text
import os
import shutil
import subprocess
import tarfile
import tempfile
from typing import Any, Dict, Iterator, Optional, Tuple
from bs4 import BeautifulSoup
import psycopg2
from testing.postgresql import Postgresql
from swh.scheduler.interface import SchedulerInterface
from swh.scheduler.model import ListedOrigin
@ -13,18 +25,39 @@ from ..pattern import CredentialsType, StatelessLister
logger = logging.getLogger(__name__)
# Aliasing the page results returned by `get_pages` method from the lister.
RubyGemsListerPage = Text
RubyGemsListerPage = Dict[str, Any]
class RubyGemsLister(StatelessLister[RubyGemsListerPage]):
"""Lister for RubyGems.org, the Ruby community’s gem hosting service."""
"""Lister for RubyGems.org, the Ruby community's gem hosting service.
Instead of querying rubygems.org Web API, it uses gems data from the
daily PostreSQL database dump of rubygems. It enables to gather all
interesting info about a gem and its release artifacts (version number,
download URL, checksums, release date) in an efficient way and without
flooding rubygems Web API with numerous HTTP requests (as there is more
than 187000 gems available on 07/10/2022).
"""
LISTER_NAME = "rubygems"
VISIT_TYPE = "rubygems"
INSTANCE = "rubygems"
INDEX_URL = "https://rubygems.org/versions"
RUBY_GEMS_POSTGRES_DUMP_BASE_URL = (
"https://s3-us-west-2.amazonaws.com/rubygems-dumps"
)
RUBY_GEMS_POSTGRES_DUMP_LIST_URL = (
f"{RUBY_GEMS_POSTGRES_DUMP_BASE_URL}?prefix=production/public_postgresql"
)
RUBY_GEM_DOWNLOAD_URL_PATTERN = "https://rubygems.org/downloads/{gem}-{version}.gem"
RUBY_GEM_ORIGIN_URL_PATTERN = "https://rubygems.org/gems/{gem}"
RUBY_GEM_EXTRINSIC_METADATA_URL_PATTERN = (
"https://rubygems.org/api/v2/rubygems/{gem}/versions/{version}.json"
)
DB_NAME = "rubygems"
DUMP_SQL_PATH = "public_postgresql/databases/PostgreSQL.sql.gz"
def __init__(
self,
@ -35,41 +68,147 @@ class RubyGemsLister(StatelessLister[RubyGemsListerPage]):
scheduler=scheduler,
credentials=credentials,
instance=self.INSTANCE,
url=self.INDEX_URL,
url=self.RUBY_GEMS_POSTGRES_DUMP_BASE_URL,
)
def get_latest_dump_file(self) -> str:
response = self.http_request(self.RUBY_GEMS_POSTGRES_DUMP_LIST_URL)
xml = BeautifulSoup(response.content, "xml")
contents = xml.find_all("Contents")
return contents[-1].find("Key").text
def create_rubygems_db(
self, postgresql: Postgresql
) -> Tuple[str, psycopg2._psycopg.connection]:
logger.debug("Creating rubygems database")
db_dsn = postgresql.dsn()
db_url = postgresql.url().replace(db_dsn["database"], self.DB_NAME)
db = psycopg2.connect(**db_dsn)
db.set_isolation_level(psycopg2.extensions.ISOLATION_LEVEL_AUTOCOMMIT)
with db.cursor() as cursor:
cursor.execute(f"CREATE DATABASE {self.DB_NAME}")
db_dsn["database"] = self.DB_NAME
db = psycopg2.connect(**db_dsn)
db.set_isolation_level(psycopg2.extensions.ISOLATION_LEVEL_AUTOCOMMIT)
with db.cursor() as cursor:
cursor.execute("CREATE EXTENSION IF NOT EXISTS hstore")
return db_url, db
def populate_rubygems_db(self, db_url: str):
dump_file = self.get_latest_dump_file()
dump_id = dump_file.split("/")[2]
response = self.http_request(f"{self.url}/{dump_file}", stream=True)
with tempfile.TemporaryDirectory() as temp_dir:
logger.debug(
"Downloading latest rubygems database dump: %s (%s bytes)",
dump_id,
response.headers["content-length"],
)
dump_file = os.path.join(temp_dir, "rubygems_dump.tar")
with open(dump_file, "wb") as dump:
for chunk in response.iter_content(chunk_size=1024):
dump.write(chunk)
with tarfile.open(dump_file) as dump_tar:
dump_tar.extractall(temp_dir)
logger.debug("Populating rubygems database with dump %s", dump_id)
psql = subprocess.Popen(
["psql", "-q", db_url],
stdin=subprocess.PIPE,
)
# passing value of gzip.open as stdin of subprocess.run makes the process
# read raw data instead of decompressed data so we have to use a pipe
with gzip.open(os.path.join(temp_dir, self.DUMP_SQL_PATH), "rb") as sql:
shutil.copyfileobj(sql, psql.stdin) # type: ignore
# denote end of read file
psql.stdin.close() # type: ignore
psql.wait()
def get_pages(self) -> Iterator[RubyGemsListerPage]:
"""Yield an iterator which returns 'page'
# spawn a temporary postgres instance (require initdb executable in environment)
with Postgresql() as postgresql:
db_url, db = self.create_rubygems_db(postgresql)
self.populate_rubygems_db(db_url)
It uses the index file located at `https://rubygems.org/versions`
to get a list of package names. Each page returns an origin url based on
the following pattern::
https://rubygems.org/gems/{pkgname}
"""
package_names: List[str] = []
response = self.http_request(url=self.url)
data = response.content.decode()
# remove the first 3 lines (file headers + first package named '-')
for line in data.splitlines()[3:]:
package_names.append(line.split(" ")[0])
# Remove duplicates
package_names_set: List[str] = list(set(package_names))
for pkgname in package_names_set:
yield f"https://rubygems.org/gems/{pkgname}"
with db.cursor() as cursor:
cursor.execute("SELECT id, name from rubygems")
for gem_id, gem_name in cursor.fetchall():
logger.debug("Processing gem named %s", gem_name[1])
with db.cursor() as cursor_v:
cursor_v.execute(
"SELECT authors, built_at, number, sha256, size from versions "
"where rubygem_id = %s",
(gem_id,),
)
versions = [
{
"number": number,
"url": self.RUBY_GEM_DOWNLOAD_URL_PATTERN.format(
gem=gem_name, version=number
),
"date": built_at.replace(tzinfo=timezone.utc),
"authors": authors,
"sha256": (
base64.decodebytes(sha256.encode()).hex()
if sha256
else None
),
"size": size,
}
for authors, built_at, number, sha256, size in cursor_v.fetchall()
]
if versions:
yield {
"name": gem_name,
"versions": versions,
}
def get_origins_from_page(self, page: RubyGemsListerPage) -> Iterator[ListedOrigin]:
"""Iterate on all pages and yield ListedOrigin instances."""
assert self.lister_obj.id is not None
artifacts = []
rubygem_metadata = []
for version in page["versions"]:
artifacts.append(
{
"version": version["number"],
"filename": version["url"].split("/")[-1],
"url": version["url"],
"checksums": (
{"sha256": version["sha256"]} if version["sha256"] else {}
),
"length": version["size"],
}
)
rubygem_metadata.append(
{
"version": version["number"],
"date": version["date"].isoformat(),
"authors": version["authors"],
"extrinsic_metadata_url": (
self.RUBY_GEM_EXTRINSIC_METADATA_URL_PATTERN.format(
gem=page["name"], version=version["number"]
)
),
}
)
yield ListedOrigin(
lister_id=self.lister_obj.id,
visit_type=self.VISIT_TYPE,
url=page,
last_update=None,
url=self.RUBY_GEM_ORIGIN_URL_PATTERN.format(gem=page["name"]),
last_update=max(version["date"] for version in page["versions"]),
extra_loader_arguments={
"artifacts": artifacts,
"rubygem_metadata": rubygem_metadata,
},
)

6
swh/lister/rubygems/tests/data/https_rubygems.org/versions
View file

@ -1,6 +0,0 @@
created_at: 2022-09-01T00:00:05Z
---
- 1 05d0116933ba44b0b5d0ee19bfd35ccc
mercurial-ruby 0.3.0,0.4.0,0.5.0,0.6.0,0.6.1,0.7.0,0.7.1,0.7.2,0.7.3,0.7.4,0.7.5,0.7.6,0.7.7,0.7.8,0.7.9,0.7.10,0.7.11,0.7.12 3ea9d3b3f1010f06d292dcfcc799f260
mercurial-wrapper 0.8.4,0.8.5 b6541e48f15eafc0b50fa694cdbffc22
mercurius 0.0.1,0.0.2,0.0.3,0.0.5,0.0.6,0.0.7,0.0.8,0.0.9,0.1.0,0.1.1,0.1.2,0.1.3,0.1.4,0.1.5,0.1.6,0.1.7,0.1.8,0.1.9,0.2.0,0.2.1 9a388c7c57d2ed4a879ab42520d91ffd

22
swh/lister/rubygems/tests/data/rubygems_dumps.xml Normal file
View file

@ -0,0 +1,22 @@
<?xml version="1.0" encoding="UTF-8"?>
<ListBucketResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
<Name>rubygems-dumps</Name>
<Prefix>production/public_postgresql</Prefix>
<Marker></Marker>
<MaxKeys>1000</MaxKeys>
<IsTruncated>false</IsTruncated>
<Contents>
<Key>production/public_postgresql/2022.10.05.06.10.11/public_postgresql.tar</Key>
<LastModified>2022-10-05T06:11:15.000Z</LastModified>
<ETag>&quot;d1c447a2a490225c2d59061e60ed86e9-75&quot;</ETag>
<Size>391653888</Size>
<StorageClass>STANDARD</StorageClass>
</Contents>
<Contents>
<Key>production/public_postgresql/2022.10.06.06.10.05/public_postgresql.tar</Key>
<LastModified>2022-10-06T06:11:11.000Z</LastModified>
<ETag>&quot;2ccd9340e4f802ec982e4cd00db2d168-75&quot;</ETag>
<Size>390047744</Size>
<StorageClass>STANDARD</StorageClass>
</Contents>
</ListBucketResult>

BIN
swh/lister/rubygems/tests/data/rubygems_pgsql_dump.tar Normal file
View file

Binary file not shown.

38
swh/lister/rubygems/tests/data/small_rubygems_dump.sh Normal file
View file

@ -0,0 +1,38 @@
#!/bin/bash
# this script requires a PostgreSQL server running on host,
# it enables to generate the rubygems_pgsql_dump.tar file used in tests data
# which contains a very small subset of gems for testing purpose
cd /tmp
# download rubygems load-pg-dump utility script
curl -O https://raw.githubusercontent.com/rubygems/rubygems.org/1c8cf7e079e56f709e7fc8f4b2398637e41815f2/script/load-pg-dump
# download latest rubygems pgsql dump and load rubygems db in local pgsql server
./load-pg-dump -c rubygems_dump.tar
# remove all rows in the rubygems db not related to gem haar_joke or l33tify
# those gems have few releases so that is why they have been picked
# also drop tables not needed by the rubygems lister
cleanup_script=$(cat <<- EOF
with t as (
select id from rubygems where name = 'haar_joke'
),
t2 as (
select id from rubygems where name = 'l33tify'
) delete from versions where rubygem_id != (select id from t) and rubygem_id != (select id from t2);
delete from rubygems where name != 'haar_joke' and name != 'l33tify';
drop table dependencies;
drop table gem_downloads;
drop table linksets;
EOF
)
echo $cleanup_script | psql rubygems
# create the rubygems_pgsql_dump.tar file
mkdir -p public_postgresql/databases
pg_dump rubygems | gzip -c > public_postgresql/databases/PostgreSQL.sql.gz
tar -cvf rubygems_pgsql_dump.tar public_postgresql

153
swh/lister/rubygems/tests/test_lister.py
View file

@ -2,26 +2,153 @@
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
# flake8: noqa: B950
from pathlib import Path
import iso8601
import pytest
from swh.lister.rubygems.lister import RubyGemsLister
from swh.scheduler.model import ListedOrigin
expected_origins = [
"https://rubygems.org/gems/mercurial-ruby",
"https://rubygems.org/gems/mercurial-wrapper",
"https://rubygems.org/gems/mercurius",
]
DUMP_FILEPATH = "production/public_postgresql/2022.10.06.06.10.05/public_postgresql.tar"
def test_rubygems_lister(datadir, requests_mock_datadir, swh_scheduler):
@pytest.fixture
def expected_listed_origins():
return [
{
"url": "https://rubygems.org/gems/haar_joke",
"visit_type": "rubygems",
"last_update": iso8601.parse_date("2016-11-05T00:00:00+00:00"),
"extra_loader_arguments": {
"artifacts": [
{
"url": "https://rubygems.org/downloads/haar_joke-0.0.2.gem",
"length": 8704,
"version": "0.0.2",
"filename": "haar_joke-0.0.2.gem",
"checksums": {
"sha256": "85a8cf5f41890e9605265eeebfe9e99aa0350a01a3c799f9f55a0615a31a2f5f"
},
},
{
"url": "https://rubygems.org/downloads/haar_joke-0.0.1.gem",
"length": 8704,
"version": "0.0.1",
"filename": "haar_joke-0.0.1.gem",
"checksums": {
"sha256": "a2ee7052fb8ffcfc4ec0fdb77fae9a36e473f859af196a36870a0f386b5ab55e"
},
},
],
"rubygem_metadata": [
{
"date": "2016-11-05T00:00:00+00:00",
"authors": "Gemma Gotch",
"version": "0.0.2",
"extrinsic_metadata_url": "https://rubygems.org/api/v2/rubygems/haar_joke/versions/0.0.2.json",
},
{
"date": "2016-07-23T00:00:00+00:00",
"authors": "Gemma Gotch",
"version": "0.0.1",
"extrinsic_metadata_url": "https://rubygems.org/api/v2/rubygems/haar_joke/versions/0.0.1.json",
},
],
},
},
{
"url": "https://rubygems.org/gems/l33tify",
"visit_type": "rubygems",
"last_update": iso8601.parse_date("2014-11-14T00:00:00+00:00"),
"extra_loader_arguments": {
"artifacts": [
{
"url": "https://rubygems.org/downloads/l33tify-0.0.2.gem",
"length": 6144,
"version": "0.0.2",
"filename": "l33tify-0.0.2.gem",
"checksums": {
"sha256": "0087a21fb6161bba8892df40de3b5e27404f941658084413b8fde49db2bc7c9f"
},
},
{
"url": "https://rubygems.org/downloads/l33tify-0.0.3.gem",
"length": 6144,
"version": "0.0.3",
"filename": "l33tify-0.0.3.gem",
"checksums": {
"sha256": "4502097ddf2657d561ce0f527ef1f49f1658c8a0968ab8cc853273138f8382a2"
},
},
{
"url": "https://rubygems.org/downloads/l33tify-0.0.1.gem",
"length": 6144,
"version": "0.0.1",
"filename": "l33tify-0.0.1.gem",
"checksums": {
"sha256": "5abfb737ce5cf561726f2f7cc1ba0f0e4f865f8b7283192e05eb3f246d3dbbca"
},
},
],
"rubygem_metadata": [
{
"date": "2014-11-14T00:00:00+00:00",
"authors": "E Alexander Liedtke",
"version": "0.0.2",
"extrinsic_metadata_url": "https://rubygems.org/api/v2/rubygems/l33tify/versions/0.0.2.json",
},
{
"date": "2014-11-14T00:00:00+00:00",
"authors": "E Alexander Liedtke",
"version": "0.0.3",
"extrinsic_metadata_url": "https://rubygems.org/api/v2/rubygems/l33tify/versions/0.0.3.json",
},
{
"date": "2014-11-14T00:00:00+00:00",
"authors": "E Alexander Liedtke",
"version": "0.0.1",
"extrinsic_metadata_url": "https://rubygems.org/api/v2/rubygems/l33tify/versions/0.0.1.json",
},
],
},
},
]
@pytest.fixture(autouse=True)
def network_requests_mock(datadir, requests_mock):
requests_mock.get(
RubyGemsLister.RUBY_GEMS_POSTGRES_DUMP_LIST_URL,
content=Path(datadir, "rubygems_dumps.xml").read_bytes(),
)
content = Path(datadir, "rubygems_pgsql_dump.tar").read_bytes()
requests_mock.get(
f"{RubyGemsLister.RUBY_GEMS_POSTGRES_DUMP_BASE_URL}/{DUMP_FILEPATH}",
content=content,
headers={"content-length": str(len(content))},
)
@pytest.mark.db
def test_rubygems_lister(swh_scheduler, expected_listed_origins):
lister = RubyGemsLister(scheduler=swh_scheduler)
res = lister.run()
assert res.pages == 3
assert res.origins == 1 + 1 + 1
assert res.pages == 2
assert res.origins == 2
scheduler_origins = swh_scheduler.get_listed_origins(lister.lister_obj.id).results
assert len(scheduler_origins) == len(expected_origins)
for origin in scheduler_origins:
assert origin.visit_type == "rubygems"
assert origin.url in expected_origins
assert [
{
"url": origin.url,
"visit_type": origin.visit_type,
"last_update": origin.last_update,
"extra_loader_arguments": origin.extra_loader_arguments,
}
for origin in scheduler_origins
] == expected_listed_origins