diff --git a/swh/lister/bitbucket/lister.py b/swh/lister/bitbucket/lister.py index 5378983..e067148 100644 --- a/swh/lister/bitbucket/lister.py +++ b/swh/lister/bitbucket/lister.py @@ -7,7 +7,7 @@ import logging import iso8601 from datetime import datetime, timezone -from typing import Any, Dict, List, Optional, Union +from typing import Any, Dict, List, Optional from urllib import parse from requests import Response @@ -46,7 +46,7 @@ class BitBucketLister(IndexingHttpLister): } def get_next_target_from_response(self, response: Response - ) -> Union[None, datetime]: + ) -> Optional[datetime]: """This will read the 'next' link from the api response if any and return it as a datetime. @@ -69,7 +69,7 @@ class BitBucketLister(IndexingHttpLister): repos = response.json()['values'] return [self.get_model_from_repo(repo) for repo in repos] - def request_uri(self, identifier: datetime) -> str: + def request_uri(self, identifier: datetime) -> str: # type: ignore identifier_str = parse.quote(identifier.isoformat()) return super().request_uri(identifier_str or '1970-01-01') diff --git a/swh/lister/cgit/lister.py b/swh/lister/cgit/lister.py index 1f5545c..bf37ea3 100644 --- a/swh/lister/cgit/lister.py +++ b/swh/lister/cgit/lister.py @@ -8,9 +8,9 @@ from urllib.parse import urlparse, urljoin from bs4 import BeautifulSoup from requests import Session -# from requests.structures import CaseInsensitiveDict + from requests.adapters import HTTPAdapter -from typing import Any, Dict, Generator, Union +from typing import Any, Dict, Generator, Optional from .models import CGitModel from swh.core.utils import grouper @@ -60,9 +60,9 @@ class CGitLister(ListerBase): """Lister class for CGit repositories. Args: - url : main URL of the CGit instance, i.e. url of the index + url (str): main URL of the CGit instance, i.e. url of the index of published git repositories on this instance. - instance : Name of cgit instance. Defaults to url's hostname + instance (str): Name of cgit instance. Defaults to url's hostname if unset. """ @@ -96,7 +96,7 @@ class CGitLister(ListerBase): return {'status': status} - def get_repos(self) -> Generator: + def get_repos(self) -> Generator[str, None, None]: """Generate git 'project' URLs found on the current CGit server """ @@ -118,7 +118,7 @@ class CGitLister(ListerBase): # no pager, or no next page next_page = None - def build_model(self, repo_url: str) -> Union[None, Dict[str, Any]]: + def build_model(self, repo_url: str) -> Optional[Dict[str, Any]]: """Given the URL of a git repo project page on a CGit server, return the repo description (dict) suitable for insertion in the db. """ diff --git a/swh/lister/core/indexing_lister.py b/swh/lister/core/indexing_lister.py index 8611e7f..d13933d 100644 --- a/swh/lister/core/indexing_lister.py +++ b/swh/lister/core/indexing_lister.py @@ -13,7 +13,8 @@ from .lister_transports import ListerHttpTransport from .lister_base import ListerBase from requests import Response -from typing import Any, Dict, List, Tuple, Optional +from typing import Any, Dict, List, Tuple, Optional, Union +from datetime import datetime logger = logging.getLogger(__name__) @@ -58,7 +59,9 @@ class IndexingLister(ListerBase): """ @abc.abstractmethod - def get_next_target_from_response(self, response: Response): + def get_next_target_from_response( + self, response: Response + ) -> Union[Optional[datetime], Optional[str], Optional[int]]: """Find the next server endpoint identifier given the entire response. Implementation of this method depends on the server API spec @@ -184,8 +187,7 @@ class IndexingLister(ListerBase): return t[0] return None - def disable_deleted_repo_tasks( - self, start, end, keep_these): + def disable_deleted_repo_tasks(self, start, end, keep_these): """Disable tasks for repos that no longer exist between start and end. Args: diff --git a/swh/lister/core/lister_transports.py b/swh/lister/core/lister_transports.py index 8c7ceb3..f4d6920 100644 --- a/swh/lister/core/lister_transports.py +++ b/swh/lister/core/lister_transports.py @@ -12,7 +12,7 @@ import logging import requests import xmltodict -from typing import Optional, Union, Dict, Any +from typing import Optional, Union, Dict, Any, List from requests import Response from swh.lister import USER_AGENT_TEMPLATE, __version__ @@ -49,7 +49,7 @@ class ListerHttpTransport(abc.ABC): 'User-Agent': USER_AGENT_TEMPLATE % self.lister_version } - def request_instance_credentials(self): + def request_instance_credentials(self) -> List[Dict[str, Any]]: """Returns dictionary of any credentials configuration needed by the forge instance to list. @@ -82,23 +82,23 @@ class ListerHttpTransport(abc.ABC): list of credential dicts for the current lister. """ - all_creds = self.config.get('credentials') + all_creds = self.config.get('credentials') # type: ignore if not all_creds: return [] - lister_creds = all_creds.get(self.LISTER_NAME, {}) - creds = lister_creds.get(self.instance, []) + lister_creds = all_creds.get(self.LISTER_NAME, {}) # type: ignore + creds = lister_creds.get(self.instance, []) # type: ignore return creds - def request_uri(self, identifier): + def request_uri(self, identifier: str) -> str: """Get the full request URI given the transport_request identifier. MAY BE OVERRIDDEN if something more complex than the PATH_TEMPLATE is required. """ - path = self.PATH_TEMPLATE % identifier + path = self.PATH_TEMPLATE % identifier # type: ignore return self.url + path - def request_params(self, identifier: int) -> Dict[str, Any]: + def request_params(self, identifier: str) -> Dict[str, Any]: """Get the full parameters passed to requests given the transport_request identifier. @@ -155,7 +155,7 @@ class ListerHttpTransport(abc.ABC): self.lister_version = __version__ def _transport_action( - self, identifier: int, method: str = 'get') -> Response: + self, identifier: str, method: str = 'get') -> Response: """Permit to ask information to the api prior to actually executing query. @@ -179,13 +179,13 @@ class ListerHttpTransport(abc.ABC): raise FetchError(response) return response - def transport_head(self, identifier: int) -> Response: + def transport_head(self, identifier: str) -> Response: """Retrieve head information on api. """ return self._transport_action(identifier, method='head') - def transport_request(self, identifier: int) -> Response: + def transport_request(self, identifier: str) -> Response: """Implements ListerBase.transport_request for HTTP using Requests. Retrieve get information on api. diff --git a/swh/lister/gitlab/lister.py b/swh/lister/gitlab/lister.py index 3d8d3d7..d3e45bf 100644 --- a/swh/lister/gitlab/lister.py +++ b/swh/lister/gitlab/lister.py @@ -76,7 +76,7 @@ class GitLabLister(PageByPageHttpLister): """Determine pages information. """ - response = self.transport_head(identifier=1) + response = self.transport_head(identifier=1) # type: ignore if not response.ok: raise ValueError( 'Problem during information fetch: %s' % response.status_code) diff --git a/swh/lister/phabricator/lister.py b/swh/lister/phabricator/lister.py index a32a6c5..bfb4a95 100644 --- a/swh/lister/phabricator/lister.py +++ b/swh/lister/phabricator/lister.py @@ -33,7 +33,7 @@ class PhabricatorLister(IndexingHttpLister): instance = urllib.parse.urlparse(self.url).hostname self.instance = instance - def request_params(self, identifier: int) -> Dict[str, Any]: + def request_params(self, identifier: str) -> Dict[str, Any]: """Override the default params behavior to retrieve the api token Credentials are stored as: