lister: Make sure lister that requires github tokens can use it
Deploying the nixguix lister, I realized that even though the credentials configuration is properly set for all listers, the listers actually requiring github origin canonicalization do not have access to the github credentials. It's lost during the constructor to only focus on the lister's credentials. Which currently translates to listers being rate-limited. This commit fixes it by pushing the self.github_session instantiation in the constructor when the lister explicitely requires the github session. Hence lifting the rate limit for maven, packagist, nixguix, and github listers. Related to infra/sysadm-environment#4655
This commit is contained in:
parent
81688ca17e
commit
92d494261f
6 changed files with 20 additions and 28 deletions
|
@ -14,6 +14,7 @@ import requests
|
|||
from tenacity.before_sleep import before_sleep_log
|
||||
|
||||
from swh.core.config import load_from_envvar
|
||||
from swh.core.github.utils import GitHubSession
|
||||
from swh.core.utils import grouper
|
||||
from swh.scheduler import get_scheduler, model
|
||||
from swh.scheduler.interface import SchedulerInterface
|
||||
|
@ -93,6 +94,7 @@ class Lister(Generic[StateType, PageType]):
|
|||
"""
|
||||
|
||||
LISTER_NAME: str = ""
|
||||
github_session: Optional[GitHubSession] = None
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
|
@ -100,6 +102,7 @@ class Lister(Generic[StateType, PageType]):
|
|||
url: str,
|
||||
instance: Optional[str] = None,
|
||||
credentials: CredentialsType = None,
|
||||
with_github_session: bool = False,
|
||||
):
|
||||
if not self.LISTER_NAME:
|
||||
raise ValueError("Must set the LISTER_NAME attribute on Lister classes")
|
||||
|
@ -127,6 +130,14 @@ class Lister(Generic[StateType, PageType]):
|
|||
self.session.headers.update(
|
||||
{"User-Agent": USER_AGENT_TEMPLATE % self.LISTER_NAME}
|
||||
)
|
||||
self.github_session: Optional[GitHubSession] = (
|
||||
GitHubSession(
|
||||
credentials=credentials.get("github", {}).get("github", []),
|
||||
user_agent=str(self.session.headers["User-Agent"]),
|
||||
)
|
||||
if with_github_session
|
||||
else None
|
||||
)
|
||||
|
||||
self.recorded_origins: Set[str] = set()
|
||||
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue