reonokiy/swh-lister
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

pattern: Store termination date to scheduler database at end of listing

Browse source 
It enables to track last lister execution date and will be used to schedule
first visits with high priority for listed origins.

Related to swh/devel/swh-scheduler#4687.
This commit is contained in:
Antoine Lambert 2024-10-09 18:32:23 +02:00
parent 927aebbd0b
commit 7609ebf7e1
5 changed files with 35 additions and 13 deletions
Download patch file Download diff file Expand all files Collapse all files

17
swh/lister/tests/test_pattern.py
View file

@ -1,4 +1,4 @@
# Copyright (C) 2020-2021 The Software Heritage developers
# Copyright (C) 2020-2024 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
@ -151,21 +151,34 @@ def test_run(swh_scheduler):
update_date = lister.lister_obj.updated
assert lister.lister_obj.last_listing_finished_at is None
run_result = lister.run()
assert run_result.pages == 2
assert run_result.origins == 20
stored_lister = swh_scheduler.get_or_create_lister(
name="test-pattern-lister", instance_name="example.com"
name=lister.lister_obj.name, instance_name=lister.lister_obj.instance_name
)
# Check that the finalize operation happened
assert stored_lister.updated > update_date
assert stored_lister.current_state["updated"] == "yes"
assert stored_lister.last_listing_finished_at is not None
last_listing_finished_at = stored_lister.last_listing_finished_at
check_listed_origins(swh_scheduler, lister, stored_lister)
lister.run()
stored_lister = swh_scheduler.get_or_create_lister(
name=lister.lister_obj.name, instance_name=lister.lister_obj.instance_name
)
assert stored_lister.last_listing_finished_at > last_listing_finished_at
class InstantiableStatelessLister(pattern.StatelessLister[PageType]):
LISTER_NAME = "test-stateless-lister"