typing: minimal changes to make a no-op mypy run pass

.gitignore

@@ -11,3 +11,4 @@ dist/
 version.txt
 swh/lister/_version.py
 .tox/
+.mypy_cache/

MANIFEST.in

@@ -7,3 +7,4 @@ include version.txt
 include swh/lister/cran/list_all_packages.R
 recursive-include swh/lister/*/tests/ *.json *.html *.txt *.* *
 recursive-include swh/lister/*/tests/data/ *.* *
+recursive-include swh py.typed

mypy.ini

@@ -0,0 +1,39 @@
+[mypy]
+namespace_packages = True
+warn_unused_ignores = True
+
+# support for sqlalchemy magic: see https://github.com/dropbox/sqlalchemy-stubs
+plugins = sqlmypy
+
+
+# 3rd party libraries without stubs (yet)
+
+[mypy-bs4.*]
+ignore_missing_imports = True
+
+[mypy-celery.*]
+ignore_missing_imports = True
+
+[mypy-debian.*]
+ignore_missing_imports = True
+
+[mypy-iso8601.*]
+ignore_missing_imports = True
+
+[mypy-pkg_resources.*]
+ignore_missing_imports = True
+
+[mypy-pytest.*]
+ignore_missing_imports = True
+
+[mypy-requests_mock.*]
+ignore_missing_imports = True
+
+[mypy-testing.postgresql.*]
+ignore_missing_imports = True
+
+[mypy-urllib3.util.*]
+ignore_missing_imports = True
+
+[mypy-xmltodict.*]
+ignore_missing_imports = True

requirements-test.txt

@@ -2,3 +2,4 @@ pytest
 pytest-postgresql
 requests_mock
 testing.postgresql
+sqlalchemy-stubs

swh/__init__.py

@@ -1 +1,4 @@
-__path__ = __import__('pkgutil').extend_path(__path__, __name__)
+from pkgutil import extend_path
+from typing import Iterable
+
+__path__ = extend_path(__path__, __name__)  # type: Iterable[str]

swh/lister/bitbucket/lister.py

@@ -7,6 +7,7 @@ import logging
 import iso8601
 
 from datetime import datetime, timezone
+from typing import Any
 from urllib import parse
 
 from swh.lister.bitbucket.models import BitBucketModel
@@ -22,7 +23,7 @@ class BitBucketLister(IndexingHttpLister):
     LISTER_NAME = 'bitbucket'
     DEFAULT_URL = 'https://api.bitbucket.org/2.0'
     instance = 'bitbucket'
-    default_min_bound = datetime.fromtimestamp(0, timezone.utc)
+    default_min_bound = datetime.fromtimestamp(0, timezone.utc)  # type: Any
 
     def __init__(self, url=None, override_config=None, per_page=100):
         super().__init__(url=url, override_config=override_config)

swh/lister/bitbucket/tests/test_lister.py

@@ -7,7 +7,6 @@ import re
 import unittest
 
 from datetime import timedelta
-
 from urllib.parse import unquote
 
 import iso8601
@@ -17,7 +16,7 @@ from swh.lister.bitbucket.lister import BitBucketLister
 from swh.lister.core.tests.test_lister import HttpListerTester
 
 
-def convert_type(req_index):
+def _convert_type(req_index):
     """Convert the req_index to its right type according to the model's
        "indexable" column.
 
@@ -31,17 +30,17 @@ class BitBucketListerTester(HttpListerTester, unittest.TestCase):
     lister_subdir = 'bitbucket'
     good_api_response_file = 'data/https_api.bitbucket.org/response.json'
     bad_api_response_file = 'data/https_api.bitbucket.org/empty_response.json'
-    first_index = convert_type('2008-07-12T07:44:01.476818+00:00')
-    last_index = convert_type('2008-07-19T06:16:43.044743+00:00')
+    first_index = _convert_type('2008-07-12T07:44:01.476818+00:00')
+    last_index = _convert_type('2008-07-19T06:16:43.044743+00:00')
     entries_per_page = 10
-    convert_type = staticmethod(convert_type)
+    convert_type = _convert_type
 
     def request_index(self, request):
         """(Override) This is needed to emulate the listing bootstrap
         when no min_bound is provided to run
         """
         m = self.test_re.search(request.path_url)
-        idx = convert_type(m.group(1))
+        idx = _convert_type(m.group(1))
         if idx == self.Lister.default_min_bound:
             idx = self.first_index
         return idx

swh/lister/core/abstractattribute.py

@@ -16,7 +16,8 @@ class AbstractAttribute:
 
         import abc
         class ClassContainingAnAbstractAttribute(abc.ABC):
-            foo = AbstractAttribute('descriptive docstring for foo')
+            foo: Union[AbstractAttribute, Any] = \
+                AbstractAttribute('docstring for foo')
 
     """
     __isabstractmethod__ = True

swh/lister/core/lister_base.py

@@ -13,6 +13,7 @@ import time
 
 from sqlalchemy import create_engine, func
 from sqlalchemy.orm import sessionmaker
+from typing import Any, Type, Union
 
 from swh.core import config
 from swh.scheduler import get_scheduler, utils
@@ -64,10 +65,12 @@ class ListerBase(abc.ABC, config.SWHConfig):
             def is_within_bounds
     """
 
-    MODEL = AbstractAttribute('Subclass type (not instance)'
-                              ' of swh.lister.core.models.ModelBase'
-                              ' customized for a specific service.')
-    LISTER_NAME = AbstractAttribute("Lister's name")
+    MODEL = AbstractAttribute(
+        'Subclass type (not instance) of swh.lister.core.models.ModelBase '
+        'customized for a specific service.'
+    )  # type: Union[AbstractAttribute, Type[Any]]
+    LISTER_NAME = AbstractAttribute(
+        "Lister's name")  # type: Union[AbstractAttribute, str]
 
     def transport_request(self, identifier):
         """Given a target endpoint identifier to query, try once to request it.

swh/lister/core/lister_transports.py

@@ -12,6 +12,8 @@ import logging
 import requests
 import xmltodict
 
+from typing import Optional, Union
+
 try:
     from swh.lister._version import __version__
 except ImportError:
@@ -29,14 +31,14 @@ class ListerHttpTransport(abc.ABC):
 
     To be used in conjunction with ListerBase or a subclass of it.
     """
-    DEFAULT_URL = None
-    PATH_TEMPLATE = AbstractAttribute('string containing a python string'
-                                      ' format pattern that produces the API'
-                                      ' endpoint path for listing stored'
-                                      ' repositories when given an index.'
-                                      ' eg. "/repositories?after=%s".'
-                                      'To be implemented in the API-specific'
-                                      ' class inheriting this.')
+    DEFAULT_URL = None  # type: Optional[str]
+    PATH_TEMPLATE = \
+        AbstractAttribute(
+            'string containing a python string format pattern that produces'
+            ' the API endpoint path for listing stored repositories when given'
+            ' an index, e.g., "/repositories?after=%s". To be implemented in'
+            ' the API-specific class inheriting this.'
+        )  # type: Union[AbstractAttribute, Optional[str]]
 
     EXPECTED_STATUS_CODES = (200, 429, 403, 404)
 
@@ -217,8 +219,9 @@ class ListerOnePageApiTransport(ListerHttpTransport):
        To be used in conjunction with ListerBase or a subclass of it.
 
     """
-    PAGE = AbstractAttribute("The server api's unique page to retrieve and "
-                             "parse for information")
+    PAGE = AbstractAttribute(
+        "URL of the API's unique page to retrieve and parse "
+        "for information")  # type: Union[AbstractAttribute, str]
     PATH_TEMPLATE = None  # we do not use it
 
     def __init__(self, url=None):

swh/lister/core/models.py

@@ -8,6 +8,7 @@ import logging
 
 from sqlalchemy import Column, DateTime, Integer, String
 from sqlalchemy.ext.declarative import DeclarativeMeta
+from typing import Type, Union
 
 from .abstractattribute import AbstractAttribute
 
@@ -24,9 +25,12 @@ class ABCSQLMeta(abc.ABCMeta, DeclarativeMeta):
 class ModelBase(SQLBase, metaclass=ABCSQLMeta):
     """a common repository"""
     __abstract__ = True
-    __tablename__ = AbstractAttribute
+    __tablename__ = \
+        AbstractAttribute  # type: Union[Type[AbstractAttribute], str]
 
-    uid = AbstractAttribute('Column(<uid_type>, primary_key=True)')
+    uid = AbstractAttribute(
+        'Column(<uid_type>, primary_key=True)'
+    )  # type: Union[AbstractAttribute, Column]
 
     name = Column(String, index=True)
     full_name = Column(String, index=True)
@@ -45,11 +49,14 @@ class ModelBase(SQLBase, metaclass=ABCSQLMeta):
 
 class IndexingModelBase(ModelBase, metaclass=ABCSQLMeta):
     __abstract__ = True
-    __tablename__ = AbstractAttribute
+    __tablename__ = \
+        AbstractAttribute  # type: Union[Type[AbstractAttribute], str]
 
     # The value used for sorting, segmenting, or api query paging,
     # because uids aren't always sequential.
-    indexable = AbstractAttribute('Column(<indexable_type>, index=True)')
+    indexable = AbstractAttribute(
+        'Column(<indexable_type>, index=True)'
+    )  # type: Union[AbstractAttribute, Column]
 
 
 def initialize(db_engine, drop_tables=False, **kwargs):

swh/lister/core/tests/test_abstractattribute.py

@@ -5,13 +5,15 @@
 import abc
 import unittest
 
+from typing import Any
+
 from swh.lister.core.abstractattribute import AbstractAttribute
 
 
 class BaseClass(abc.ABC):
-    v1 = AbstractAttribute
-    v2 = AbstractAttribute()
-    v3 = AbstractAttribute('changed docstring')
+    v1 = AbstractAttribute  # type: Any
+    v2 = AbstractAttribute()  # type: Any
+    v3 = AbstractAttribute('changed docstring')  # type: Any
     v4 = 'qux'
 
 

swh/lister/core/tests/test_lister.py

@@ -10,6 +10,7 @@ from unittest.mock import Mock, patch
 
 import requests_mock
 from sqlalchemy import create_engine
+from typing import Any, Callable, Optional, Pattern, Type, Union
 
 from swh.lister.core.abstractattribute import AbstractAttribute
 from swh.lister.tests.test_utils import init_db
@@ -28,9 +29,12 @@ class HttpListerTesterBase(abc.ABC):
     to customize for a specific listing service.
 
     """
-    Lister = AbstractAttribute('The lister class to test')
-    lister_subdir = AbstractAttribute('bitbucket, github, etc.')
-    good_api_response_file = AbstractAttribute('Example good response body')
+    Lister = AbstractAttribute(
+        'Lister class to test')  # type: Union[AbstractAttribute, Type[Any]]
+    lister_subdir = AbstractAttribute(
+        'bitbucket, github, etc.')  # type: Union[AbstractAttribute, str]
+    good_api_response_file = AbstractAttribute(
+        'Example good response body')  # type: Union[AbstractAttribute, str]
     LISTER_NAME = 'fake-lister'
 
     # May need to override this if the headers are used for something
@@ -157,13 +161,21 @@ class HttpListerTester(HttpListerTesterBase, abc.ABC):
     to customize for a specific listing service.
 
     """
-    last_index = AbstractAttribute('Last index in good_api_response')
-    first_index = AbstractAttribute('First index in good_api_response')
-    bad_api_response_file = AbstractAttribute('Example bad response body')
-    entries_per_page = AbstractAttribute('Number of results in good response')
-    test_re = AbstractAttribute('Compiled regex matching the server url. Must'
-                                ' capture the index value.')
-    convert_type = str
+    last_index = AbstractAttribute(
+        'Last index '
+        'in good_api_response')  # type: Union[AbstractAttribute, int]
+    first_index = AbstractAttribute(
+        'First index in '
+        ' good_api_response')  # type: Union[AbstractAttribute, Optional[int]]
+    bad_api_response_file = AbstractAttribute(
+        'Example bad response body')  # type: Union[AbstractAttribute, str]
+    entries_per_page = AbstractAttribute(
+        'Number of results in '
+        'good response')  # type: Union[AbstractAttribute, int]
+    test_re = AbstractAttribute(
+        'Compiled regex matching the server url. Must capture the '
+        'index value.')  # type: Union[AbstractAttribute, Pattern]
+    convert_type = str  # type: Callable[..., Any]
     """static method used to convert the "request_index" to its right type (for
        indexing listers for example, this is in accordance with the model's
        "indexable" column).
@@ -343,9 +355,12 @@ class HttpSimpleListerTester(HttpListerTesterBase, abc.ABC):
     to customize for a specific listing service.
 
     """
-    entries = AbstractAttribute('Number of results in good response')
-    PAGE = AbstractAttribute("The server api's unique page to retrieve and "
-                             "parse for information")
+    entries = AbstractAttribute(
+        'Number of results '
+        'in good response')  # type: Union[AbstractAttribute, int]
+    PAGE = AbstractAttribute(
+        "URL of the server api's unique page to retrieve and "
+        "parse for information")  # type: Union[AbstractAttribute, str]
 
     def get_fl(self, override_config=None):
         """Retrieve an instance of fake lister (fl).

swh/lister/cran/lister.py

@@ -42,7 +42,7 @@ def compute_package_url(repo: Mapping[str, str]) -> str:
 
     """
     return 'https://cran.r-project.org/src/contrib' \
-        '/%(Package)s_%(Version)s.tar.gz' % repo
+        '/%(Package)s_%(Version)s.tar.gz'.format(repo)
 
 
 class CRANLister(SimpleLister):

swh/lister/github/lister.py

@@ -6,6 +6,8 @@
 import re
 import time
 
+from typing import Any
+
 from swh.lister.core.indexing_lister import IndexingHttpLister
 from swh.lister.github.models import GitHubModel
 
@@ -17,7 +19,7 @@ class GitHubLister(IndexingHttpLister):
     API_URL_INDEX_RE = re.compile(r'^.*/repositories\?since=(\d+)')
     LISTER_NAME = 'github'
     instance = 'github'  # There is only 1 instance of such lister
-    default_min_bound = 0
+    default_min_bound = 0  # type: Any
 
     def get_model_from_repo(self, repo):
         return {

swh/lister/phabricator/lister.py

@@ -20,7 +20,8 @@ logger = logging.getLogger(__name__)
 
 class PhabricatorLister(IndexingHttpLister):
     PATH_TEMPLATE = '?order=oldest&attachments[uris]=1&after=%s'
-    DEFAULT_URL = 'https://forge.softwareheritage.org/api/diffusion.repository.search'  # noqa
+    DEFAULT_URL = \
+        'https://forge.softwareheritage.org/api/diffusion.repository.search'
     MODEL = PhabricatorModel
     LISTER_NAME = 'phabricator'
 

swh/lister/py.typed

@@ -0,0 +1 @@
+# Marker file for PEP 561.

tox.ini

@@ -1,5 +1,5 @@
 [tox]
-envlist=flake8,py3
+envlist=flake8,mypy,py3
 
 [testenv:py3]
 deps =
@@ -25,3 +25,11 @@ deps =
   flake8
 commands =
   {envpython} -m flake8
+
+[testenv:mypy]
+skip_install = true
+deps =
+  .[testing]
+  mypy
+commands =
+  mypy swh