reonokiy/swh-lister
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

add opam lister

Browse source
This commit is contained in:
zapashcanon 2021-05-17 15:31:00 +02:00
parent bf7d44db3c
commit fe01d08cd9
No known key found for this signature in database
GPG key ID: 8981C3C62D1D28F1
22 changed files with 680 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

1
CONTRIBUTORS
View file

@ -5,3 +5,4 @@ Yann Gautier
Sushant Sushant
Hezekiah Maina
Boris Baldassari
Léo Andrès

1
setup.py
View file

@ -65,6 +65,7 @@ setup(
lister.gnu=swh.lister.gnu:register
lister.launchpad=swh.lister.launchpad:register
lister.npm=swh.lister.npm:register
lister.opam=swh.lister.opam:register
lister.packagist=swh.lister.packagist:register
lister.phabricator=swh.lister.phabricator:register
lister.pypi=swh.lister.pypi:register

12
swh/lister/opam/__init__.py Normal file
View file

@ -0,0 +1,12 @@
# Copyright (C) 2021 the Software Heritage developers
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
def register():
from .lister import OpamLister
return {
"lister": OpamLister,
"task_modules": ["%s.tasks" % __name__],
}

95
swh/lister/opam/lister.py Normal file
View file

@ -0,0 +1,95 @@
# Copyright (C) 2021 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
import io
import logging
import os
from subprocess import PIPE, Popen, call
import tempfile
from typing import Iterator
from swh.lister.pattern import StatelessLister
from swh.scheduler.interface import SchedulerInterface
from swh.scheduler.model import ListedOrigin
from ..pattern import CredentialsType
logger = logging.getLogger(__name__)
PageType = str
class OpamLister(StatelessLister[PageType]):
"""
List all repositories hosted on an opam repository.
On initialisation, we create an opam root, with no ocaml compiler (no switch)
as we won't need it and it's costly. In this opam root, we add a single opam
repository (url) and give it a name (instance). Then, to get pages, we just ask
opam to list all the packages for our opam repository in our opam root.
Args:
url: base URL of an opam repository
(for instance https://opam.ocaml.org)
instance: string identifier for the listed repository
"""
# Part of the lister API, that identifies this lister
LISTER_NAME = "opam"
def __init__(
self,
scheduler: SchedulerInterface,
url: str,
instance: str,
credentials: CredentialsType = None,
):
super().__init__(
scheduler=scheduler, credentials=credentials, url=url, instance=instance,
)
self.env = os.environ.copy()
self.env["OPAMROOT"] = tempfile.mkdtemp(prefix="swh_opam_lister")
call(
["opam", "init", "--reinit", "--bare", "--no-setup", instance, url],
env=self.env,
)
def get_pages(self) -> Iterator[PageType]:
proc = Popen(
[
"opam",
"list",
"--all",
"--no-switch",
"--repos",
self.instance,
"--normalise",
"--short",
],
env=self.env,
stdout=PIPE,
)
if proc.stdout is not None:
for line in io.TextIOWrapper(proc.stdout):
yield line.rstrip("\n")
def get_origins_from_page(self, page: PageType) -> Iterator[ListedOrigin]:
"""Convert a page of OpamLister repositories into a list of ListedOrigins"""
assert self.lister_obj.id is not None
# a page is just a package name
url = f"opam+{self.url}/packages/{page}/"
yield ListedOrigin(
lister_id=self.lister_obj.id,
visit_type="opam",
url=url,
last_update=None,
extra_loader_arguments={
"opam_root": self.env["OPAMROOT"],
"opam_instance": self.instance,
"opam_url": self.url,
"opam_package": page,
},
)

18
swh/lister/opam/tasks.py Normal file
View file

@ -0,0 +1,18 @@
# Copyright (C) 2021 the Software Heritage developers
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
from celery import shared_task
from swh.lister.opam.lister import OpamLister
@shared_task(name=__name__ + ".OpamListerTask")
def list_opam(**lister_args):
"""Lister task for the Opam registry"""
return OpamLister.from_configfile(**lister_args).run().dict()
@shared_task(name=__name__ + ".ping")
def _ping():
return "OK"

0
swh/lister/opam/tests/__init__.py Normal file
View file

38
swh/lister/opam/tests/data/fake_opam_repo/packages/agrid/agrid.0.1/opam vendored Normal file
View file

@ -0,0 +1,38 @@
opam-version: "2.0"
synopsis: "Adjustable grid (two dimensional array) library"
description:
"Adjustable grids are two dimensional arrays whose width/height can be changed by adding or removing row/column at either end (one at a time)."
maintainer: ["OCamlPro <contact@ocamlpro.com>"]
authors: ["OCamlPro <contact@ocamlpro.com>"]
license: "ISC"
homepage: "https://github.com/ocamlpro/agrid"
bug-reports: "https://github.com/ocamlpro/agrid/issues"
depends: [
"ocaml" {>= "4.05"}
"dune" {>= "2.7"}
"flex-array" {>= "1.2"}
"bisect_ppx" {with-test & >= "2.6" & dev}
"odoc" {with-doc}
]
build: [
["dune" "subst"] {dev}
[
"dune"
"build"
"-p"
name
"-j"
jobs
"@install"
"@runtest" {with-test}
"@doc" {with-doc}
]
]
dev-repo: "git+https://github.com/ocamlpro/agrid.git"
url {
src: "https://github.com/OCamlPro/agrid/archive/0.1.tar.gz"
checksum: [
"sha256=ea82546711a6abdd4edf8bc3052041498cae9c2e5a9e147e29820da4eac4beb4"
"sha512=f53b2c095e3607e53f92d4e7e13848e9e34bd866837335e7d9341dbb468ac46ffbcd2002d1bf1105e2f6060f57871aef7ce8e65594855447fafb72ad32b076b7"
]
}

57
swh/lister/opam/tests/data/fake_opam_repo/packages/calculon/calculon.0.1/opam vendored Normal file
View file

@ -0,0 +1,57 @@
opam-version: "2.0"
maintainer: "c-cube"
authors: ["Armael" "Enjolras" "c-cube"]
homepage: "https://github.com/c-cube/calculon"
bug-reports: "https://github.com/c-cube/calculon/issues"
tags: ["irc" "bot" "factoids"]
dev-repo: "git+http://github.com/c-cube/calculon.git"
build: [
[
"./configure"
"--bindir"
"%{bin}%"
"--%{uri+lambdasoup+cohttp+atdgen:enable}%-web"
"--%{re+sequence:enable}%-extras"
]
[make "build"]
[make "test"] {with-test}
[make "doc"] {with-doc}
]
install: [make "install"]
remove: ["ocamlfind" "remove" "calculon"]
depends: [
"ocaml" {>= "4.02.0"}
"ocamlfind" {build}
"base-bytes"
"base-unix"
"result"
"lwt"
"irc-client" {>= "0.4.0" & < "0.6.0"}
"tls"
"yojson"
"containers" {>= "1.0" & < "2.0"}
"ISO8601"
"re" {>= "1.5.0" & < "1.7.2"}
"stringext"
]
depopts: [
"uri"
"cohttp"
"atdgen"
"lambdasoup"
"sequence"
]
conflicts: [
"cohttp" {>= "1.0.0"}
]
synopsis:
"Library for writing IRC bots in OCaml, a collection of plugins, and a dramatic robotic actor."
description: """
The core library is called `calculon` and revolves around
the concept of commands that react to user messages. See the README for a small
tutorial on how to write your own plugins."""
flags: light-uninstall
url {
src: "https://github.com/c-cube/calculon/archive/0.1.tar.gz"
checksum: "md5=24fb37bb915717a0497962fda9fecc5c"
}

46
swh/lister/opam/tests/data/fake_opam_repo/packages/calculon/calculon.0.2/opam vendored Normal file
View file

@ -0,0 +1,46 @@
opam-version: "2.0"
maintainer: "c-cube"
authors: ["Armael" "Enjolras" "c-cube"]
homepage: "https://github.com/c-cube/calculon"
bug-reports: "https://github.com/c-cube/calculon/issues"
tags: ["irc" "bot" "factoids"]
dev-repo: "git+https://github.com/c-cube/calculon.git"
build: [
["jbuilder" "build" "-p" name]
["jbuilder" "runtest" "-p" name] {with-test}
["jbuilder" "build" "@doc" "-p" name] {with-doc}
]
depends: [
"ocaml" {>= "4.02.0"}
"jbuilder"
"base-bytes"
"base-unix"
"result"
"lwt"
"irc-client" {>= "0.4.0" & < "0.6.0"}
"tls"
"yojson"
"containers" {>= "1.0" & < "2.1"}
"ISO8601"
"stringext"
"re"
]
depopts: [
"uri"
"cohttp"
"cohttp-lwt"
"cohttp-lwt-unix"
"atdgen"
"lambdasoup"
"sequence"
]
synopsis:
"Library for writing IRC bots in OCaml, a collection of plugins, and a dramatic robotic actor."
description: """
The core library is called `calculon` and revolves around
the concept of commands that react to user messages. See the README for a small
tutorial on how to write your own plugins."""
url {
src: "https://github.com/c-cube/calculon/archive/0.2.tar.gz"
checksum: "md5=df6692d28d27ae6f87c773739b758fb8"
}

40
swh/lister/opam/tests/data/fake_opam_repo/packages/calculon/calculon.0.3/opam vendored Normal file
View file

@ -0,0 +1,40 @@
opam-version: "2.0"
maintainer: "c-cube"
authors: ["Armael" "Enjolras" "c-cube"]
homepage: "https://github.com/c-cube/calculon"
bug-reports: "https://github.com/c-cube/calculon/issues"
tags: ["irc" "bot" "factoids"]
dev-repo: "git+https://github.com/c-cube/calculon.git"
build: [
["jbuilder" "build" "-p" name]
["jbuilder" "runtest" "-p" name] {with-test}
["jbuilder" "build" "@doc" "-p" name] {with-doc}
]
depends: [
"ocaml" {>= "4.02.0"}
"jbuilder"
"base-bytes"
"base-unix"
"result"
"lwt"
"irc-client" {>= "0.6.0"}
"irc-client-lwt"
"irc-client-tls"
"tls"
"yojson"
"containers" {>= "1.0" & < "3.0"}
"ISO8601"
"stringext"
"re" {>= "1.7.2"}
"odoc" {with-doc}
]
synopsis:
"Library for writing IRC bots in OCaml, a collection of plugins, and a dramatic robotic actor."
description: """
The core library is called `calculon` and revolves around
the concept of commands that react to user messages. See the README for a small
tutorial on how to write your own plugins."""
url {
src: "https://github.com/c-cube/calculon/archive/0.3.tar.gz"
checksum: "md5=635a554938d42319beeb6d70a85800ff"
}

40
swh/lister/opam/tests/data/fake_opam_repo/packages/calculon/calculon.0.4/opam vendored Normal file
View file

@ -0,0 +1,40 @@
opam-version: "2.0"
synopsis: "Library for writing IRC bots in OCaml and a collection of plugins"
maintainer: "c-cube"
authors: ["Armael" "Enjolras" "c-cube"]
tags: ["irc" "bot" "factoids"]
homepage: "https://github.com/c-cube/calculon"
bug-reports: "https://github.com/c-cube/calculon/issues"
depends: [
"dune"
"base-bytes"
"base-unix"
"result"
"lwt"
"irc-client" {>= "0.6.0"}
"irc-client-lwt"
"irc-client-tls"
"tls"
"yojson"
"containers" {>= "1.0" & < "3.0"}
"ISO8601"
"stringext"
"x509" {< "0.10.0"}
"re" {>= "1.7.2"}
"odoc" {with-doc}
"ocaml" {>= "4.03.0"}
"mdx" {with-test}
]
build: [
["dune" "build" "-p" name "-j" jobs]
["dune" "build" "@doc" "-p" name] {with-doc}
["dune" "runtest" "-p" name "-j" jobs] {with-test}
]
dev-repo: "git+https://github.com/c-cube/calculon.git"
url {
src: "https://github.com/c-cube/calculon/archive/v0.4.tar.gz"
checksum: [
"md5=e53363833b7e3620a2ce7ffb9a27715d"
"sha512=c997fd52fd277e8a2d50f266f12182afa96f72bb2f161a6ac1b3372fc9ebbcc12c7593f90cbb5873456b8b96d3488e5c52d071723bbaa05097710d643241d70b"
]
}

38
swh/lister/opam/tests/data/fake_opam_repo/packages/calculon/calculon.0.5/opam vendored Normal file
View file

@ -0,0 +1,38 @@
opam-version: "2.0"
synopsis: "Library for writing IRC bots in OCaml and a collection of plugins"
authors: ["Armael" "Enjolras" "c-cube"]
maintainer: "c-cube"
build: [
["dune" "build" "-p" name "-j" jobs]
[ "dune" "build" "@doc" "-p" name] {with-doc}
[ "dune" "runtest" "-p" name "-j" jobs] {with-test}
]
depends: [
"dune" { >= "1.1" }
"base-bytes"
"base-unix"
"result"
"lwt"
"irc-client" { >= "0.6.0" }
"irc-client-lwt"
"irc-client-lwt-ssl"
"yojson"
"containers" { >= "1.0" & < "3.0" }
"ISO8601"
"stringext"
"re" { >= "1.7.2" }
"iter"
"odoc" {with-doc}
"ocaml" { >= "4.03.0" }
]
tags: [ "irc" "bot" "factoids" ]
homepage: "https://github.com/c-cube/calculon"
bug-reports: "https://github.com/c-cube/calculon/issues"
dev-repo: "git+https://github.com/c-cube/calculon.git"
url {
src: "https://github.com/c-cube/calculon/archive/v0.5.tar.gz"
checksum: [
"md5=831b8d45ac76bfa1118e7e954bfd4474"
"sha512=b7e856d88a2c34f2f7bb2c5c8f416ef99e29ccd46a9016e5f7fefc838df6fcb5daffd45170b606562a2ba15e910421884071e6e19fa90b23f412f45d85cc7d5a"
]
}

39
swh/lister/opam/tests/data/fake_opam_repo/packages/calculon/calculon.0.6/opam vendored Normal file
View file

@ -0,0 +1,39 @@
opam-version: "2.0"
synopsis: "Library for writing IRC bots in OCaml and a collection of plugins"
authors: ["Armael" "Enjolras" "c-cube"]
maintainer: "c-cube"
build: [
["dune" "build" "-p" name "-j" jobs]
["dune" "build" "@doc" "-p" name] {with-doc}
["dune" "runtest" "-p" name "-j" jobs] {with-test}
]
depends: [
"dune" { >= "1.1" }
"base-unix"
"lwt"
"irc-client" { >= "0.6.0" }
"irc-client-lwt"
"irc-client-lwt-ssl"
"logs"
"yojson" { >= "1.6" }
"containers" { >= "1.0" & < "3.0" }
"ISO8601"
"stringext"
"re" { >= "1.7.2" }
"odoc" {with-doc}
"ocaml" { >= "4.03.0" }
]
depopts: [
"iter"
]
tags: [ "irc" "bot" "factoids" ]
homepage: "https://github.com/c-cube/calculon"
bug-reports: "https://github.com/c-cube/calculon/issues"
dev-repo: "git+https://github.com/c-cube/calculon.git"
url {
src: "https://github.com/c-cube/calculon/archive/v0.6.tar.gz"
checksum: [
"md5=16ad257c16f82a2acfab5e10c7c8ef8a"
"sha512=c460994c0ffabf0d756cdbb0cdd77b99d3b4844f597c894bb2c8ed22d1038b44f1be4d76721a956a76cc953d915fb76324f0fdb385e6a2e531da7fd4cc832836"
]
}

38
swh/lister/opam/tests/data/fake_opam_repo/packages/directories/directories.0.1/opam vendored Normal file
View file

@ -0,0 +1,38 @@
opam-version: "2.0"
synopsis:
"An OCaml library that provides configuration, cache and data paths (and more!) following the suitable conventions on Linux, macOS and Windows"
description:
"directories is an OCaml library that provides configuration, cache and data paths (and more!) following the suitable conventions on Linux, macOS and Windows. It is inspired by similar libraries for other languages such as directories-jvm. The following conventions are used: XDG Base Directory Specification and xdg-user-dirs on Linux, Known Folders on Windows, Standard Directories on macOS."
maintainer: ["OCamlPro <contact@ocamlpro.com>"]
authors: ["OCamlPro <contact@ocamlpro.com>"]
license: "ISC"
homepage: "https://github.com/ocamlpro/directories"
bug-reports: "https://github.com/ocamlpro/directories/issues"
depends: [
"dune" {>= "2.1"}
"ocaml" {>= "4.07.0"}
"ctypes-foreign" {>= "0.4.0" | "os" != "win32" | "os" != "mingw"}
"ctypes" {>= "0.17.1" | "os" != "win32" | "os" != "mingw"}
]
build: [
["dune" "subst"] {dev}
[
"dune"
"build"
"-p"
name
"-j"
jobs
"@install"
"@runtest" {with-test}
"@doc" {with-doc}
]
]
dev-repo: "git+https://github.com/ocamlpro/directories.git"
url {
src: "https://github.com/OCamlPro/directories/archive/0.1.tar.gz"
checksum: [
"sha256=89aa3586af3c38aea17302cfb0b6243afdf0fd90b2eb1f86fb69990917304445"
"sha512=3302f8c8c2c3ecc217f199e97d8d32d1ef14af9dd3763cbc1030522832ce375192422f7b6b6acd8cd8399d96ccd52d5e5cfffac3cb392f1cf8b9f67374c1cf7c"
]
}

38
swh/lister/opam/tests/data/fake_opam_repo/packages/directories/directories.0.2/opam vendored Normal file
View file

@ -0,0 +1,38 @@
opam-version: "2.0"
synopsis:
"An OCaml library that provides configuration, cache and data paths (and more!) following the suitable conventions on Linux, macOS and Windows"
description:
"directories is an OCaml library that provides configuration, cache and data paths (and more!) following the suitable conventions on Linux, macOS and Windows. It is inspired by similar libraries for other languages such as directories-jvm. The following conventions are used: XDG Base Directory Specification and xdg-user-dirs on Linux, Known Folders on Windows, Standard Directories on macOS."
maintainer: ["OCamlPro <contact@ocamlpro.com>"]
authors: ["OCamlPro <contact@ocamlpro.com>"]
license: "ISC"
homepage: "https://github.com/ocamlpro/directories"
bug-reports: "https://github.com/ocamlpro/directories/issues"
depends: [
"dune" {>= "2.1"}
"ocaml" {>= "4.07.0"}
"ctypes-foreign" {>= "0.4.0" & (os = "win32" | os = "cygwin")}
"ctypes" {>= "0.17.1" & (os = "win32" | os = "cygwin")}
]
build: [
["dune" "subst"] {dev}
[
"dune"
"build"
"-p"
name
"-j"
jobs
"@install"
"@runtest" {with-test}
"@doc" {with-doc}
]
]
dev-repo: "git+https://github.com/ocamlpro/directories.git"
url {
src: "https://github.com/OCamlPro/directories/archive/0.2.tar.gz"
checksum: [
"sha256=af81e7bd7dd7125eb5168226def47e6e2287f3210ba33c015b58123a1877c40d"
"sha512=30c0db12eb453f3549bf52c202d6e6a8d12e8f4f07b92d8fc68309beb4dcf66b610eab49ac4bd3e1ddd487cd1539790c92f9511c5c06d33c2bf966148c08aa82"
]
}

43
swh/lister/opam/tests/data/fake_opam_repo/packages/directories/directories.0.3/opam vendored Normal file
View file

@ -0,0 +1,43 @@
opam-version: "2.0"
synopsis:
"An OCaml library that provides configuration, cache and data paths (and more!) following the suitable conventions on Linux, macOS and Windows"
description: """
directories is an OCaml library that provides configuration, cache and data paths (and more!) following the suitable conventions on Linux, macOS and Windows.
It is inspired by similar libraries for other languages such as directories-jvm.
The following conventions are used:
- XDG Base Directory Specification and xdg-user-dirs on Linux
- Known Folders on Windows
- Standard Directories on macOS.
"""
maintainer: ["OCamlPro <contact@ocamlpro.com>"]
authors: ["OCamlPro <contact@ocamlpro.com>"]
license: "ISC"
homepage: "https://github.com/ocamlpro/directories"
bug-reports: "https://github.com/ocamlpro/directories/issues"
depends: [
"dune" {>= "2.1"}
"ocaml" {>= "4.07.0"}
"ctypes" {>= "0.17.1" & (os = "win32" | os = "cygwin")}
]
build: [
["dune" "subst"] {dev}
[
"dune"
"build"
"-p"
name
"-j"
jobs
"@install"
"@runtest" {with-test}
"@doc" {with-doc}
]
]
dev-repo: "git+https://github.com/ocamlpro/directories.git"
url {
src: "https://github.com/OCamlPro/directories/archive/0.3.tar.gz"
checksum: [
"sha256=9b37d1d43e3b06f3b68ebe57651ea2cf5a9db6d5bdb1a0afe65786bcdcc8bf11"
"sha512=9d1634a0c44dd74dc3005154ebbc22c39bb0c3e1477f3e67c9fc49cc32059ac0f496816c06785f204f60ee67136c998f526ba0c9bf4232d040ff30bd89411fb3"
]
}

37
swh/lister/opam/tests/data/fake_opam_repo/packages/ocb/ocb.0.1/opam vendored Normal file
View file

@ -0,0 +1,37 @@
opam-version: "2.0"
synopsis: "SVG badge generator"
description:
"An OCaml library for SVG badge generation. There's also a command-line tool provided."
maintainer: ["OCamlPro <contact@ocamlpro.com>"]
authors: ["OCamlPro <contact@ocamlpro.com>"]
license: "ISC"
homepage: "https://ocamlpro.github.io/ocb/"
doc: "https://ocamlpro.github.io/ocb/api/"
bug-reports: "https://github.com/OCamlPro/ocb/issues"
depends: [
"ocaml" {>= "4.05"}
"dune" {>= "2.0"}
"odoc" {with-doc}
]
build: [
["dune" "subst"] {dev}
[
"dune"
"build"
"-p"
name
"-j"
jobs
"@install"
"@runtest" {with-test}
"@doc" {with-doc}
]
]
dev-repo: "git+https://github.com/OCamlPro/ocb.git"
url {
src: "https://github.com/OCamlPro/ocb/archive/0.1.tar.gz"
checksum: [
"sha256=aa27684fbda1b8036ae7e3c87de33a98a9cd2662bcc91c8447e00e41476b6a46"
"sha512=1260344f184dd8c8074b0439dbcc8a5d59550a654c249cd61913d4c150c664f37b76195ddca38f7f6646d08bddb320ceb8d420508450b4f09a233cd5c22e6b9b"
]
}

1
swh/lister/opam/tests/data/fake_opam_repo/repo Normal file
View file

@ -0,0 +1 @@
opam-version: "2.0"

1
swh/lister/opam/tests/data/fake_opam_repo/version Normal file
View file

@ -0,0 +1 @@
0.9.0

65
swh/lister/opam/tests/test_lister.py Normal file
View file

@ -0,0 +1,65 @@
# Copyright (C) 2021 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
import io
from unittest.mock import MagicMock
from swh.lister.opam.lister import OpamLister
def test_urls(swh_scheduler, mocker):
instance_url = "https://opam.ocaml.org"
lister = OpamLister(swh_scheduler, url=instance_url, instance="opam")
mocked_popen = MagicMock()
mocked_popen.stdout = io.BytesIO(b"bar\nbaz\nfoo\n")
# replaces the real Popen with a fake one
mocker.patch("swh.lister.opam.lister.Popen", return_value=mocked_popen)
# call the lister and get all listed origins urls
stats = lister.run()
assert stats.pages == 3
assert stats.origins == 3
scheduler_origins = swh_scheduler.get_listed_origins(lister.lister_obj.id).results
expected_urls = [
f"opam+{instance_url}/packages/bar/",
f"opam+{instance_url}/packages/baz/",
f"opam+{instance_url}/packages/foo/",
]
result_urls = [origin.url for origin in scheduler_origins]
assert expected_urls == result_urls
def test_opam_binary(datadir, swh_scheduler):
instance_url = f"file://{datadir}/fake_opam_repo"
lister = OpamLister(swh_scheduler, url=instance_url, instance="fake")
stats = lister.run()
assert stats.pages == 4
assert stats.origins == 4
scheduler_origins = swh_scheduler.get_listed_origins(lister.lister_obj.id).results
expected_urls = [
f"opam+{instance_url}/packages/agrid/",
f"opam+{instance_url}/packages/calculon/",
f"opam+{instance_url}/packages/directories/",
f"opam+{instance_url}/packages/ocb/",
]
result_urls = [origin.url for origin in scheduler_origins]
assert expected_urls == result_urls

31
swh/lister/opam/tests/test_tasks.py Normal file
View file

@ -0,0 +1,31 @@
# Copyright (C) 2021 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
from swh.lister.pattern import ListerStats
def test_opam_ping(swh_scheduler_celery_app, swh_scheduler_celery_worker):
res = swh_scheduler_celery_app.send_task("swh.lister.opam.tasks.ping")
assert res
res.wait()
assert res.successful()
assert res.result == "OK"
def test_opam_lister(swh_scheduler_celery_app, swh_scheduler_celery_worker, mocker):
# setup the mocked OpamLister
lister = mocker.patch("swh.lister.opam.tasks.OpamLister")
lister.from_configfile.return_value = lister
stats = ListerStats(pages=42, origins=42)
lister.run.return_value = stats
res = swh_scheduler_celery_app.send_task("swh.lister.opam.tasks.OpamListerTask")
assert res
res.wait()
assert res.successful()
assert res.result == stats.dict()
lister.from_configfile.assert_called_once_with()
lister.run.assert_called_once_with()

1
swh/lister/tests/test_cli.py
View file

@ -17,6 +17,7 @@ lister_args = {
"gitea": {"url": "https://try.gitea.io/api/v1/",},
"tuleap": {"url": "https://tuleap.net",},
"gitlab": {"url": "https://gitlab.ow2.org/api/v4", "instance": "ow2",},
"opam": {"url": "https://opam.ocaml.org", "instance": "opam"},
}