reonokiy/swh-lister
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Enable black

Browse source 
- blackify all the python files,
- enable black in pre-commit,
- add a black tox environment.
This commit is contained in:
David Douard 2020-04-08 16:31:22 +02:00
parent 1ae75166c7
commit 93a4d8b784
97 changed files with 1734 additions and 1642 deletions
Download patch file Download diff file Expand all files Collapse all files

9
swh/lister/cran/__init__.py
View file

@ -7,7 +7,8 @@ def register():
from .models import CRANModel
from .lister import CRANLister
return {'models': [CRANModel],
'lister': CRANLister,
'task_modules': ['%s.tasks' % __name__],
}
return {
"models": [CRANModel],
"lister": CRANLister,
"task_modules": ["%s.tasks" % __name__],
}

69
swh/lister/cran/lister.py
View file

@ -19,16 +19,23 @@ from swh.scheduler.utils import create_task_dict
logger = logging.getLogger(__name__)
CRAN_MIRROR = 'https://cran.r-project.org'
CRAN_MIRROR = "https://cran.r-project.org"
class CRANLister(SimpleLister):
MODEL = CRANModel
LISTER_NAME = 'cran'
instance = 'cran'
LISTER_NAME = "cran"
instance = "cran"
def task_dict(self, origin_type, origin_url, version=None, html_url=None,
policy=None, **kwargs):
def task_dict(
self,
origin_type,
origin_url,
version=None,
html_url=None,
policy=None,
**kwargs,
):
"""Return task format dict. This creates tasks with args and kwargs
set, for example::
@ -43,15 +50,15 @@ class CRANLister(SimpleLister):
"""
if not policy:
policy = 'oneshot'
policy = "oneshot"
artifact_url = html_url
assert origin_type == 'tar'
assert origin_type == "tar"
return create_task_dict(
'load-cran', policy,
url=origin_url, artifacts=[{
'url': artifact_url,
'version': version
}], retries_left=3
"load-cran",
policy,
url=origin_url,
artifacts=[{"url": artifact_url, "version": version}],
retries_left=3,
)
def safely_issue_request(self, identifier):
@ -91,23 +98,22 @@ class CRANLister(SimpleLister):
"""
return read_cran_data()
def get_model_from_repo(
self, repo: Mapping[str, str]) -> Mapping[str, str]:
def get_model_from_repo(self, repo: Mapping[str, str]) -> Mapping[str, str]:
"""Transform from repository representation to model
"""
logger.debug('repo: %s', repo)
logger.debug("repo: %s", repo)
origin_url, artifact_url = compute_origin_urls(repo)
package = repo['Package']
version = repo['Version']
package = repo["Package"]
version = repo["Version"]
return {
'uid': f'{package}-{version}',
'name': package,
'full_name': repo['Title'],
'version': version,
'html_url': artifact_url,
'origin_url': origin_url,
'origin_type': 'tar',
"uid": f"{package}-{version}",
"name": package,
"full_name": repo["Title"],
"version": version,
"html_url": artifact_url,
"origin_url": origin_url,
"origin_type": "tar",
}
@ -115,11 +121,10 @@ def read_cran_data() -> List[Mapping[str, str]]:
"""Execute r script to read cran listing.
"""
filepath = pkg_resources.resource_filename('swh.lister.cran',
'list_all_packages.R')
logger.debug('script list-all-packages.R path: %s', filepath)
filepath = pkg_resources.resource_filename("swh.lister.cran", "list_all_packages.R")
logger.debug("script list-all-packages.R path: %s", filepath)
response = subprocess.run(filepath, stdout=subprocess.PIPE, shell=False)
return json.loads(response.stdout.decode('utf-8'))
return json.loads(response.stdout.decode("utf-8"))
def compute_origin_urls(repo: Mapping[str, str]) -> Tuple[str, str]:
@ -132,8 +137,8 @@ def compute_origin_urls(repo: Mapping[str, str]) -> Tuple[str, str]:
the tuple project url, artifact url
"""
package = repo['Package']
version = repo['Version']
origin_url = f'{CRAN_MIRROR}/package={package}'
artifact_url = f'{CRAN_MIRROR}/src/contrib/{package}_{version}.tar.gz'
package = repo["Package"]
version = repo["Version"]
origin_url = f"{CRAN_MIRROR}/package={package}"
artifact_url = f"{CRAN_MIRROR}/src/contrib/{package}_{version}.tar.gz"
return origin_url, artifact_url

3
swh/lister/cran/models.py
View file

@ -11,7 +11,8 @@ class CRANModel(ModelBase):
"""a CRAN repository representation
"""
__tablename__ = 'cran_repo'
__tablename__ = "cran_repo"
uid = Column(String, primary_key=True)
version = Column(String)

8
swh/lister/cran/tasks.py
View file

@ -7,12 +7,12 @@ from celery import shared_task
from swh.lister.cran.lister import CRANLister
@shared_task(name=__name__ + '.CRANListerTask')
@shared_task(name=__name__ + ".CRANListerTask")
def list_cran(**lister_args):
'''Lister task for the CRAN registry'''
"""Lister task for the CRAN registry"""
return CRANLister(**lister_args).run()
@shared_task(name=__name__ + '.ping')
@shared_task(name=__name__ + ".ping")
def _ping():
return 'OK'
return "OK"

16
swh/lister/cran/tests/conftest.py
View file

@ -10,14 +10,16 @@ from swh.lister.core.tests.conftest import * # noqa
@pytest.fixture
def lister_cran(swh_listers):
lister = swh_listers['cran']
lister = swh_listers["cran"]
# Add the load-deb-package in the scheduler backend
lister.scheduler.create_task_type({
'type': 'load-cran',
'description': 'Load a CRAN package',
'backend_name': 'swh.loader.package.cran.tasks.LoaderCRAN',
'default_interval': '1 day',
})
lister.scheduler.create_task_type(
{
"type": "load-cran",
"description": "Load a CRAN package",
"backend_name": "swh.loader.package.cran.tasks.LoaderCRAN",
"default_interval": "1 day",
}
)
return lister

49
swh/lister/cran/tests/test_lister.py
View file

@ -13,28 +13,25 @@ from swh.lister.cran.lister import compute_origin_urls, CRAN_MIRROR
def test_cran_compute_origin_urls():
pack = 'something'
vers = '0.0.1'
origin_url, artifact_url = compute_origin_urls({
'Package': pack,
'Version': vers,
})
pack = "something"
vers = "0.0.1"
origin_url, artifact_url = compute_origin_urls({"Package": pack, "Version": vers,})
assert origin_url == f'{CRAN_MIRROR}/package={pack}'
assert artifact_url == f'{CRAN_MIRROR}/src/contrib/{pack}_{vers}.tar.gz'
assert origin_url == f"{CRAN_MIRROR}/package={pack}"
assert artifact_url == f"{CRAN_MIRROR}/src/contrib/{pack}_{vers}.tar.gz"
def test_cran_compute_origin_urls_failure():
for incomplete_repo in [{'Version': '0.0.1'}, {'Package': 'package'}, {}]:
for incomplete_repo in [{"Version": "0.0.1"}, {"Package": "package"}, {}]:
with pytest.raises(KeyError):
compute_origin_urls(incomplete_repo)
@patch('swh.lister.cran.lister.read_cran_data')
@patch("swh.lister.cran.lister.read_cran_data")
def test_cran_lister_cran(mock_cran, datadir, lister_cran):
lister = lister_cran
with open(path.join(datadir, 'list-r-packages.json')) as f:
with open(path.join(datadir, "list-r-packages.json")) as f:
data = json.loads(f.read())
mock_cran.return_value = data
@ -42,31 +39,33 @@ def test_cran_lister_cran(mock_cran, datadir, lister_cran):
lister.run()
r = lister.scheduler.search_tasks(task_type='load-cran')
r = lister.scheduler.search_tasks(task_type="load-cran")
assert len(r) == 6
for row in r:
assert row['type'] == 'load-cran'
assert row["type"] == "load-cran"
# arguments check
args = row['arguments']['args']
args = row["arguments"]["args"]
assert len(args) == 0
# kwargs
kwargs = row['arguments']['kwargs']
kwargs = row["arguments"]["kwargs"]
assert len(kwargs) == 2
assert set(kwargs.keys()) == {'url', 'artifacts'}
assert set(kwargs.keys()) == {"url", "artifacts"}
artifacts = kwargs['artifacts']
artifacts = kwargs["artifacts"]
assert len(artifacts) == 1
assert set(artifacts[0].keys()) == {'url', 'version'}
assert set(artifacts[0].keys()) == {"url", "version"}
assert row['policy'] == 'oneshot'
assert row['retries_left'] == 3
assert row["policy"] == "oneshot"
assert row["retries_left"] == 3
origin_url = kwargs['url']
record = lister.db_session \
.query(lister.MODEL) \
.filter(origin_url == origin_url).first()
origin_url = kwargs["url"]
record = (
lister.db_session.query(lister.MODEL)
.filter(origin_url == origin_url)
.first()
)
assert record
assert record.uid == f'{record.name}-{record.version}'
assert record.uid == f"{record.name}-{record.version}"

10
swh/lister/cran/tests/test_tasks.py
View file

@ -2,22 +2,20 @@ from unittest.mock import patch
def test_ping(swh_app, celery_session_worker):
res = swh_app.send_task(
'swh.lister.cran.tasks.ping')
res = swh_app.send_task("swh.lister.cran.tasks.ping")
assert res
res.wait()
assert res.successful()
assert res.result == 'OK'
assert res.result == "OK"
@patch('swh.lister.cran.tasks.CRANLister')
@patch("swh.lister.cran.tasks.CRANLister")
def test_lister(lister, swh_app, celery_session_worker):
# setup the mocked CRANLister
lister.return_value = lister
lister.run.return_value = None
res = swh_app.send_task(
'swh.lister.cran.tasks.CRANListerTask')
res = swh_app.send_task("swh.lister.cran.tasks.CRANListerTask")
assert res
res.wait()
assert res.successful()