review corrections

swh/lister/bitbucket/lister.py

@@ -7,7 +7,7 @@ import logging
 import iso8601
 
 from datetime import datetime, timezone
-from typing import Any, Dict, List, Optional, Union
+from typing import Any, Dict, List, Optional
 from urllib import parse
 from requests import Response
 
@@ -46,7 +46,7 @@ class BitBucketLister(IndexingHttpLister):
         }
 
     def get_next_target_from_response(self, response: Response
-                                      ) -> Union[None, datetime]:
+                                      ) -> Optional[datetime]:
         """This will read the 'next' link from the api response if any
            and return it as a datetime.
 
@@ -69,7 +69,7 @@ class BitBucketLister(IndexingHttpLister):
         repos = response.json()['values']
         return [self.get_model_from_repo(repo) for repo in repos]
 
-    def request_uri(self, identifier: datetime) -> str:
+    def request_uri(self, identifier: datetime) -> str:  # type: ignore
         identifier_str = parse.quote(identifier.isoformat())
         return super().request_uri(identifier_str or '1970-01-01')
 

swh/lister/cgit/lister.py

@@ -8,9 +8,9 @@ from urllib.parse import urlparse, urljoin
 
 from bs4 import BeautifulSoup
 from requests import Session
-# from requests.structures import CaseInsensitiveDict
+
 from requests.adapters import HTTPAdapter
-from typing import Any, Dict, Generator, Union
+from typing import Any, Dict, Generator, Optional
 from .models import CGitModel
 
 from swh.core.utils import grouper
@@ -60,9 +60,9 @@ class CGitLister(ListerBase):
         """Lister class for CGit repositories.
 
         Args:
-            url : main URL of the CGit instance, i.e. url of the index
+            url (str): main URL of the CGit instance, i.e. url of the index
                 of published git repositories on this instance.
-            instance : Name of cgit instance. Defaults to url's hostname
+            instance (str): Name of cgit instance. Defaults to url's hostname
                 if unset.
 
         """
@@ -96,7 +96,7 @@ class CGitLister(ListerBase):
 
         return {'status': status}
 
-    def get_repos(self) -> Generator:
+    def get_repos(self) -> Generator[str, None, None]:
         """Generate git 'project' URLs found on the current CGit server
 
         """
@@ -118,7 +118,7 @@ class CGitLister(ListerBase):
                 # no pager, or no next page
                 next_page = None
 
-    def build_model(self, repo_url: str) -> Union[None, Dict[str, Any]]:
+    def build_model(self, repo_url: str) -> Optional[Dict[str, Any]]:
         """Given the URL of a git repo project page on a CGit server,
         return the repo description (dict) suitable for insertion in the db.
         """

swh/lister/core/indexing_lister.py

@@ -13,7 +13,8 @@ from .lister_transports import ListerHttpTransport
 from .lister_base import ListerBase
 
 from requests import Response
-from typing import Any, Dict, List, Tuple, Optional
+from typing import Any, Dict, List, Tuple, Optional, Union
+from datetime import datetime
 
 logger = logging.getLogger(__name__)
 
@@ -58,7 +59,9 @@ class IndexingLister(ListerBase):
     """
 
     @abc.abstractmethod
-    def get_next_target_from_response(self, response: Response):
+    def get_next_target_from_response(
+            self, response: Response
+    ) -> Union[Optional[datetime], Optional[str], Optional[int]]:
         """Find the next server endpoint identifier given the entire response.
 
         Implementation of this method depends on the server API spec
@@ -184,8 +187,7 @@ class IndexingLister(ListerBase):
             return t[0]
         return None
 
-    def disable_deleted_repo_tasks(
-            self, start, end, keep_these):
+    def disable_deleted_repo_tasks(self, start, end, keep_these):
         """Disable tasks for repos that no longer exist between start and end.
 
         Args:

swh/lister/core/lister_transports.py

@@ -12,7 +12,7 @@ import logging
 import requests
 import xmltodict
 
-from typing import Optional, Union, Dict, Any
+from typing import Optional, Union, Dict, Any, List
 from requests import Response
 
 from swh.lister import USER_AGENT_TEMPLATE, __version__
@@ -49,7 +49,7 @@ class ListerHttpTransport(abc.ABC):
             'User-Agent': USER_AGENT_TEMPLATE % self.lister_version
         }
 
-    def request_instance_credentials(self):
+    def request_instance_credentials(self) -> List[Dict[str, Any]]:
         """Returns dictionary of any credentials configuration needed by the
         forge instance to list.
 
@@ -82,23 +82,23 @@ class ListerHttpTransport(abc.ABC):
             list of credential dicts for the current lister.
 
         """
-        all_creds = self.config.get('credentials')
+        all_creds = self.config.get('credentials')  # type: ignore
         if not all_creds:
             return []
-        lister_creds = all_creds.get(self.LISTER_NAME, {})
-        creds = lister_creds.get(self.instance, [])
+        lister_creds = all_creds.get(self.LISTER_NAME, {})  # type: ignore
+        creds = lister_creds.get(self.instance, [])  # type: ignore
         return creds
 
-    def request_uri(self, identifier):
+    def request_uri(self, identifier: str) -> str:
         """Get the full request URI given the transport_request identifier.
 
         MAY BE OVERRIDDEN if something more complex than the PATH_TEMPLATE is
         required.
         """
-        path = self.PATH_TEMPLATE % identifier
+        path = self.PATH_TEMPLATE % identifier  # type: ignore
         return self.url + path
 
-    def request_params(self, identifier: int) -> Dict[str, Any]:
+    def request_params(self, identifier: str) -> Dict[str, Any]:
         """Get the full parameters passed to requests given the
         transport_request identifier.
 
@@ -155,7 +155,7 @@ class ListerHttpTransport(abc.ABC):
         self.lister_version = __version__
 
     def _transport_action(
-            self, identifier: int, method: str = 'get') -> Response:
+            self, identifier: str, method: str = 'get') -> Response:
         """Permit to ask information to the api prior to actually executing
            query.
 
@@ -179,13 +179,13 @@ class ListerHttpTransport(abc.ABC):
                 raise FetchError(response)
             return response
 
-    def transport_head(self, identifier: int) -> Response:
+    def transport_head(self, identifier: str) -> Response:
         """Retrieve head information on api.
 
         """
         return self._transport_action(identifier, method='head')
 
-    def transport_request(self, identifier: int) -> Response:
+    def transport_request(self, identifier: str) -> Response:
         """Implements ListerBase.transport_request for HTTP using Requests.
 
         Retrieve get information on api.

swh/lister/gitlab/lister.py

@@ -76,7 +76,7 @@ class GitLabLister(PageByPageHttpLister):
         """Determine pages information.
 
         """
-        response = self.transport_head(identifier=1)
+        response = self.transport_head(identifier=1)  # type: ignore
         if not response.ok:
             raise ValueError(
                 'Problem during information fetch: %s' % response.status_code)

swh/lister/phabricator/lister.py

@@ -33,7 +33,7 @@ class PhabricatorLister(IndexingHttpLister):
             instance = urllib.parse.urlparse(self.url).hostname
         self.instance = instance
 
-    def request_params(self, identifier: int) -> Dict[str, Any]:
+    def request_params(self, identifier: str) -> Dict[str, Any]:
         """Override the default params behavior to retrieve the api token
 
         Credentials are stored as: