diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index ce247f1..5e13001 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -32,6 +32,7 @@ repos: stages: [commit] - id: codespell name: Check commit message spelling + args: [-L crate] stages: [commit-msg] - repo: local diff --git a/requirements.txt b/requirements.txt index e8bf14c..dead79e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,6 +3,7 @@ dateparser dulwich iso8601 launchpadlib +looseversion lxml psycopg2 pyreadr diff --git a/swh/lister/crates/lister.py b/swh/lister/crates/lister.py index a17ce7d..e31756c 100644 --- a/swh/lister/crates/lister.py +++ b/swh/lister/crates/lister.py @@ -15,7 +15,7 @@ from typing import Any, Dict, Iterator, List, Optional from urllib.parse import urlparse import iso8601 -from packaging.version import parse as parse_version +from looseversion import LooseVersion2 from swh.scheduler.interface import SchedulerInterface from swh.scheduler.model import ListedOrigin @@ -201,7 +201,7 @@ class CratesLister(Lister[CratesListerState, CratesListerPage]): for name, item in dataset.items(): page = [] # sort crate versions - versions: list = sorted(item["versions"].keys(), key=parse_version) + versions = sorted(item["versions"].keys(), key=LooseVersion2) for version in versions: v = item["versions"][version] diff --git a/swh/lister/crates/tests/data/fake_crates_repository_init.sh b/swh/lister/crates/tests/data/fake_crates_repository_init.sh index b58d195..8078fad 100755 --- a/swh/lister/crates/tests/data/fake_crates_repository_init.sh +++ b/swh/lister/crates/tests/data/fake_crates_repository_init.sh @@ -17,6 +17,7 @@ echo -e '''created_at,description,documentation,downloads,homepage,id,max_upload ''' > data/crates.csv echo -e '''checksum,crate_id,crate_size,created_at,downloads,features,id,license,links,num,published_by,updated_at,yanked +d879626d5babe4ca6c4ec953d712e28d939672b325a4f9352f28ca3c82568a15,1339,,2014-12-18 06:56:46.88489,845,{},1321,MIT/Apache-2.0,,0.1.3-experimental,,2017-11-30 05:24:37.146115,f 398952a2f6cd1d22bc1774fd663808e32cf36add0280dee5cdd84a8fff2db944,2233,,2015-05-27 23:19:16.848643,1961,{},10855,MIT/Apache-2.0,,0.1.0,,2017-11-30 03:37:17.449539,f 343bd0171ee23346506db6f4c64525de6d72f0e8cc533f83aea97f3e7488cbf9,545,,2014-12-18 06:56:46.88489,845,{},1321,MIT/Apache-2.0,,0.1.2,,2017-11-30 02:29:20.01125,f 6e229ed392842fa93c1d76018d197b7e1b74250532bafb37b0e1d121a92d4cf7,1339,,2015-02-03 11:15:19.001762,8211,{},4371,MIT/Apache-2.0,,0.1.2,,2017-11-30 03:14:27.545115,f diff --git a/swh/lister/crates/tests/data/https_static.crates.io/db-dump.tar.gz b/swh/lister/crates/tests/data/https_static.crates.io/db-dump.tar.gz index bd74c75..12b3082 100644 Binary files a/swh/lister/crates/tests/data/https_static.crates.io/db-dump.tar.gz and b/swh/lister/crates/tests/data/https_static.crates.io/db-dump.tar.gz differ diff --git a/swh/lister/crates/tests/data/https_static.crates.io/db-dump.tar.gz_visit1 b/swh/lister/crates/tests/data/https_static.crates.io/db-dump.tar.gz_visit1 index 0b7dd38..7164da3 100644 Binary files a/swh/lister/crates/tests/data/https_static.crates.io/db-dump.tar.gz_visit1 and b/swh/lister/crates/tests/data/https_static.crates.io/db-dump.tar.gz_visit1 differ diff --git a/swh/lister/crates/tests/test_lister.py b/swh/lister/crates/tests/test_lister.py index 387ca26..ebc9220 100644 --- a/swh/lister/crates/tests/test_lister.py +++ b/swh/lister/crates/tests/test_lister.py @@ -26,6 +26,14 @@ expected_origins = [ "filename": "rand-0.1.2.crate", "url": "https://static.crates.io/crates/rand/rand-0.1.2.crate", }, + { + "version": "0.1.3-experimental", + "checksums": { + "sha256": "d879626d5babe4ca6c4ec953d712e28d939672b325a4f9352f28ca3c82568a15", # noqa: B950 + }, + "filename": "rand-0.1.3-experimental.crate", + "url": "https://static.crates.io/crates/rand/rand-0.1.3-experimental.crate", + }, ], "crates_metadata": [ { @@ -38,6 +46,11 @@ expected_origins = [ "last_update": "2017-11-30 03:14:27.545115", "yanked": False, }, + { + "version": "0.1.3-experimental", + "last_update": "2017-11-30 05:24:37.146115", + "yanked": False, + }, ], }, {