reonokiy/swh-lister
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Enable black

Browse source 
- blackify all the python files,
- enable black in pre-commit,
- add a black tox environment.
This commit is contained in:
David Douard 2020-04-08 16:31:22 +02:00
parent 1ae75166c7
commit 93a4d8b784
97 changed files with 1734 additions and 1642 deletions
Download patch file Download diff file Expand all files Collapse all files

9
swh/lister/packagist/__init__.py
View file

@ -7,7 +7,8 @@ def register():
from .models import PackagistModel
from .lister import PackagistLister
return {'models': [PackagistModel],
'lister': PackagistLister,
'task_modules': ['%s.tasks' % __name__],
}
return {
"models": [PackagistModel],
"lister": PackagistLister,
"task_modules": ["%s.tasks" % __name__],
}

42
swh/lister/packagist/lister.py
View file

@ -23,7 +23,7 @@ def compute_package_url(repo_name: str) -> str:
"""Compute packgist package url from repo name.
"""
return 'https://repo.packagist.org/p/%s.json' % repo_name
return "https://repo.packagist.org/p/%s.json" % repo_name
class PackagistLister(ListerOnePageApiTransport, SimpleLister):
@ -52,17 +52,19 @@ class PackagistLister(ListerOnePageApiTransport, SimpleLister):
'https://repo.packagist.org/p/hypejunction/hypegamemechanics.json'
"""
MODEL = PackagistModel
LISTER_NAME = 'packagist'
PAGE = 'https://packagist.org/packages/list.json'
instance = 'packagist'
LISTER_NAME = "packagist"
PAGE = "https://packagist.org/packages/list.json"
instance = "packagist"
def __init__(self, override_config=None):
ListerOnePageApiTransport .__init__(self)
ListerOnePageApiTransport.__init__(self)
SimpleLister.__init__(self, override_config=override_config)
def task_dict(self, origin_type: str, origin_url: str,
**kwargs: Mapping[str, str]) -> Dict[str, Any]:
def task_dict(
self, origin_type: str, origin_url: str, **kwargs: Mapping[str, str]
) -> Dict[str, Any]:
"""Return task format dict
This is overridden from the lister_base as more information is
@ -70,18 +72,20 @@ class PackagistLister(ListerOnePageApiTransport, SimpleLister):
"""
return utils.create_task_dict(
'load-%s' % origin_type,
kwargs.get('policy', 'recurring'),
kwargs.get('name'), origin_url,
retries_left=3)
"load-%s" % origin_type,
kwargs.get("policy", "recurring"),
kwargs.get("name"),
origin_url,
retries_left=3,
)
def list_packages(self, response: Any) -> List[str]:
"""List the actual packagist origins from the response.
"""
response = json.loads(response.text)
packages = [name for name in response['packageNames']]
logger.debug('Number of packages: %s', len(packages))
packages = [name for name in response["packageNames"]]
logger.debug("Number of packages: %s", len(packages))
random.shuffle(packages)
return packages
@ -91,10 +95,10 @@ class PackagistLister(ListerOnePageApiTransport, SimpleLister):
"""
url = compute_package_url(repo_name)
return {
'uid': repo_name,
'name': repo_name,
'full_name': repo_name,
'html_url': url,
'origin_url': url,
'origin_type': 'packagist',
"uid": repo_name,
"name": repo_name,
"full_name": repo_name,
"html_url": url,
"origin_url": url,
"origin_type": "packagist",
}

3
swh/lister/packagist/models.py
View file

@ -11,6 +11,7 @@ class PackagistModel(ModelBase):
"""a Packagist repository representation
"""
__tablename__ = 'packagist_repo'
__tablename__ = "packagist_repo"
uid = Column(String, primary_key=True)

8
swh/lister/packagist/tasks.py
View file

@ -7,12 +7,12 @@ from celery import shared_task
from .lister import PackagistLister
@shared_task(name=__name__ + '.PackagistListerTask')
@shared_task(name=__name__ + ".PackagistListerTask")
def list_packagist(**lister_args):
'List the packagist (php) registry'
"List the packagist (php) registry"
PackagistLister(**lister_args).run()
@shared_task(name=__name__ + '.ping')
@shared_task(name=__name__ + ".ping")
def _ping():
return 'OK'
return "OK"

16
swh/lister/packagist/tests/conftest.py
View file

@ -10,14 +10,16 @@ from swh.lister.core.tests.conftest import * # noqa
@pytest.fixture
def lister_packagist(swh_listers):
lister = swh_listers['packagist']
lister = swh_listers["packagist"]
# Amend the scheduler with an unknown yet load-packagist task type
lister.scheduler.create_task_type({
'type': 'load-packagist',
'description': 'Load packagist origin',
'backend_name': 'swh.loader.package.tasks.LoaderPackagist',
'default_interval': '1 day',
})
lister.scheduler.create_task_type(
{
"type": "load-packagist",
"description": "Load packagist origin",
"backend_name": "swh.loader.package.tasks.LoaderPackagist",
"default_interval": "1 day",
}
)
return lister

61
swh/lister/packagist/tests/test_lister.py
View file

@ -12,27 +12,29 @@ from swh.lister.packagist.lister import PackagistLister, compute_package_url
from swh.lister.core.tests.test_lister import HttpSimpleListerTester
expected_packages = ['0.0.0/composer-include-files', '0.0.0/laravel-env-shim',
'0.0.1/try-make-package', '0099ff/dialogflowphp',
'00f100/array_dot']
expected_packages = [
"0.0.0/composer-include-files",
"0.0.0/laravel-env-shim",
"0.0.1/try-make-package",
"0099ff/dialogflowphp",
"00f100/array_dot",
]
expected_model = {
'uid': '0099ff/dialogflowphp',
'name': '0099ff/dialogflowphp',
'full_name': '0099ff/dialogflowphp',
'html_url':
'https://repo.packagist.org/p/0099ff/dialogflowphp.json',
'origin_url':
'https://repo.packagist.org/p/0099ff/dialogflowphp.json',
'origin_type': 'packagist',
}
"uid": "0099ff/dialogflowphp",
"name": "0099ff/dialogflowphp",
"full_name": "0099ff/dialogflowphp",
"html_url": "https://repo.packagist.org/p/0099ff/dialogflowphp.json",
"origin_url": "https://repo.packagist.org/p/0099ff/dialogflowphp.json",
"origin_type": "packagist",
}
class PackagistListerTester(HttpSimpleListerTester, unittest.TestCase):
Lister = PackagistLister
PAGE = 'https://packagist.org/packages/list.json'
lister_subdir = 'packagist'
good_api_response_file = 'data/https_packagist.org/packages_list.json'
PAGE = "https://packagist.org/packages/list.json"
lister_subdir = "packagist"
good_api_response_file = "data/https_packagist.org/packages_list.json"
entries = 5
@requests_mock.Mocker()
@ -52,40 +54,41 @@ class PackagistListerTester(HttpSimpleListerTester, unittest.TestCase):
"""
fl = self.get_fl()
model = fl.transport_response_simplified(['0099ff/dialogflowphp'])
model = fl.transport_response_simplified(["0099ff/dialogflowphp"])
assert len(model) == 1
for key, values in model[0].items():
assert values == expected_model[key]
@patch('swh.lister.packagist.lister.utils.create_task_dict')
@patch("swh.lister.packagist.lister.utils.create_task_dict")
def test_task_dict(self, mock_create_tasks):
"""Test the task creation of lister
"""
fl = self.get_fl()
fl.task_dict(origin_type='packagist', origin_url='https://abc',
name='test_pack')
fl.task_dict(
origin_type="packagist", origin_url="https://abc", name="test_pack"
)
mock_create_tasks.assert_called_once_with(
'load-packagist', 'recurring', 'test_pack', 'https://abc',
retries_left=3)
"load-packagist", "recurring", "test_pack", "https://abc", retries_left=3
)
def test_compute_package_url():
expected_url = 'https://repo.packagist.org/p/hello.json'
actual_url = compute_package_url('hello')
expected_url = "https://repo.packagist.org/p/hello.json"
actual_url = compute_package_url("hello")
assert actual_url == expected_url
def test_packagist_lister(lister_packagist, requests_mock_datadir):
lister_packagist.run()
r = lister_packagist.scheduler.search_tasks(task_type='load-packagist')
r = lister_packagist.scheduler.search_tasks(task_type="load-packagist")
assert len(r) == 5
for row in r:
assert row['type'] == 'load-packagist'
assert row["type"] == "load-packagist"
# arguments check
args = row['arguments']['args']
args = row["arguments"]["args"]
assert len(args) == 2
package = args[0]
@ -95,8 +98,8 @@ def test_packagist_lister(lister_packagist, requests_mock_datadir):
assert url == expected_url
# kwargs
kwargs = row['arguments']['kwargs']
kwargs = row["arguments"]["kwargs"]
assert kwargs == {}
assert row['policy'] == 'recurring'
assert row['priority'] is None
assert row["policy"] == "recurring"
assert row["priority"] is None

10
swh/lister/packagist/tests/test_tasks.py
View file

@ -6,22 +6,20 @@ from unittest.mock import patch
def test_ping(swh_app, celery_session_worker):
res = swh_app.send_task(
'swh.lister.packagist.tasks.ping')
res = swh_app.send_task("swh.lister.packagist.tasks.ping")
assert res
res.wait()
assert res.successful()
assert res.result == 'OK'
assert res.result == "OK"
@patch('swh.lister.packagist.tasks.PackagistLister')
@patch("swh.lister.packagist.tasks.PackagistLister")
def test_lister(lister, swh_app, celery_session_worker):
# setup the mocked PackagistLister
lister.return_value = lister
lister.run.return_value = None
res = swh_app.send_task(
'swh.lister.packagist.tasks.PackagistListerTask')
res = swh_app.send_task("swh.lister.packagist.tasks.PackagistListerTask")
assert res
res.wait()
assert res.successful()