pattern: Ensure accurate origin counts returned by run method

Previously, the run method was returning the total count of ListedOrigin
objects sent to scheduler database.

However, some listers can send multiple ListedOrigin objects for a given
origin URL during the listing process, for instance when an origin is
contained in multiple pages (e.g. gogs listing) or when the listing
is gathering multiple versions of an origin spread across multiple
pages (e.g. maven listing).

This changes ensures an accurate count of listed origins by maintaining
a set of origin URLs associated to the sent ListedOrigin objects.

swh/lister/pattern.py

@@ -7,7 +7,7 @@ from __future__ import annotations
 
 from dataclasses import dataclass
 import logging
-from typing import Any, Dict, Generic, Iterable, Iterator, List, Optional, TypeVar
+from typing import Any, Dict, Generic, Iterable, Iterator, List, Optional, Set, TypeVar
 from urllib.parse import urlparse
 
 import requests
@@ -128,6 +128,8 @@ class Lister(Generic[StateType, PageType]):
             {"User-Agent": USER_AGENT_TEMPLATE % self.LISTER_NAME}
         )
 
+        self.recorded_origins: Set[str] = set()
+
     @http_retry(before_sleep=before_sleep_log(logger, logging.WARNING))
     def http_request(self, url: str, method="GET", **kwargs) -> requests.Response:
 
@@ -154,12 +156,15 @@ class Lister(Generic[StateType, PageType]):
 
         """
         full_stats = ListerStats()
+        self.recorded_origins = set()
 
         try:
             for page in self.get_pages():
                 full_stats.pages += 1
                 origins = self.get_origins_from_page(page)
-                full_stats.origins += self.send_origins(origins)
+                sent_origins = self.send_origins(origins)
+                self.recorded_origins.update(sent_origins)
+                full_stats.origins = len(self.recorded_origins)
                 self.commit_page(page)
         finally:
             self.finalize()
@@ -255,18 +260,18 @@ class Lister(Generic[StateType, PageType]):
         """
         pass
 
-    def send_origins(self, origins: Iterable[model.ListedOrigin]) -> int:
+    def send_origins(self, origins: Iterable[model.ListedOrigin]) -> List[str]:
         """Record a list of :class:`model.ListedOrigin` in the scheduler.
 
         Returns:
-          the number of listed origins recorded in the scheduler
+          the list of origin URLs recorded in scheduler database
         """
-        count = 0
+        recorded_origins = []
         for batch_origins in grouper(origins, n=1000):
             ret = self.scheduler.record_listed_origins(batch_origins)
-            count += len(ret)
+            recorded_origins += [origin.url for origin in ret]
 
-        return count
+        return recorded_origins
 
     @classmethod
     def from_config(cls, scheduler: Dict[str, Any], **config: Any):