reonokiy/swh-lister
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

sourceforge/tasks: Allow incremental listing

Browse source 
Related to T3310
This commit is contained in:
Antoine R. Dumont (@ardumont) 2021-05-07 17:02:29 +02:00
parent 7282647bb2
commit 2ff549e125
No known key found for this signature in database
GPG key ID: 52E2E9840D10C3B8
2 changed files with 29 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

6
swh/lister/sourceforge/tasks.py
View file

@ -15,6 +15,12 @@ def list_sourceforge_full() -> Dict[str, int]:
return SourceForgeLister.from_configfile().run().dict()
@shared_task(name=__name__ + ".IncrementalSourceForgeLister")
def list_sourceforge_incremental() -> Dict[str, int]:
"""Full update of a SourceForge instance"""
return SourceForgeLister.from_configfile(incremental=True).run().dict()
@shared_task(name=__name__ + ".ping")
def _ping():
return "OK"

24
swh/lister/sourceforge/tests/test_tasks.py
View file

@ -5,6 +5,8 @@
from swh.lister.pattern import ListerStats
lister_module = "swh.lister.sourceforge.tasks.SourceForgeLister"
def test_sourceforge_ping(swh_scheduler_celery_app, swh_scheduler_celery_worker):
res = swh_scheduler_celery_app.send_task("swh.lister.sourceforge.tasks.ping")
@ -18,7 +20,7 @@ def test_sourceforge_full_lister_task(
swh_scheduler_celery_app, swh_scheduler_celery_worker, mocker
):
stats = ListerStats(pages=10, origins=900)
mock_lister = mocker.patch("swh.lister.sourceforge.tasks.SourceForgeLister")
mock_lister = mocker.patch(lister_module)
mock_lister.from_configfile.return_value = mock_lister
mock_lister.run.return_value = stats
@ -32,3 +34,23 @@ def test_sourceforge_full_lister_task(
mock_lister.from_configfile.assert_called_once()
mock_lister.run.assert_called_once()
assert res.result == stats.dict()
def test_incremental_listing(
swh_scheduler_celery_app, swh_scheduler_celery_worker, mocker
):
stats = ListerStats(pages=1, origins=90)
mock_lister = mocker.patch(lister_module)
mock_lister.from_configfile.return_value = mock_lister
mock_lister.run.return_value = stats
res = swh_scheduler_celery_app.send_task(
"swh.lister.sourceforge.tasks.IncrementalSourceForgeLister"
)
assert res
res.wait()
assert res.successful()
mock_lister.from_configfile.assert_called_once()
mock_lister.run.assert_called_once()
assert res.result == stats.dict()