From 22f71812945edf41e76142cbcdcd34ba91d97131 Mon Sep 17 00:00:00 2001 From: Antoine Lambert Date: Thu, 17 Sep 2020 17:47:55 +0200 Subject: [PATCH] python: Reorder imports with isort Related to T2610 --- setup.py | 6 +++--- swh/lister/__init__.py | 2 +- swh/lister/bitbucket/__init__.py | 2 +- swh/lister/bitbucket/lister.py | 7 +++---- swh/lister/bitbucket/models.py | 2 +- swh/lister/bitbucket/tasks.py | 1 + swh/lister/bitbucket/tests/test_lister.py | 3 +-- swh/lister/bitbucket/tests/test_tasks.py | 4 ++-- swh/lister/cgit/__init__.py | 2 +- swh/lister/cgit/lister.py | 9 ++++----- swh/lister/cli.py | 9 +++++---- swh/lister/core/indexing_lister.py | 11 +++++------ swh/lister/core/lister_base.py | 5 ++--- swh/lister/core/lister_transports.py | 10 ++++------ swh/lister/core/models.py | 3 +-- swh/lister/core/page_by_page_lister.py | 2 +- swh/lister/core/simple_lister.py | 3 +-- swh/lister/core/tests/test_abstractattribute.py | 3 +-- swh/lister/core/tests/test_lister.py | 2 +- swh/lister/cran/__init__.py | 2 +- swh/lister/cran/lister.py | 6 ++---- swh/lister/cran/tests/test_lister.py | 6 +++--- swh/lister/debian/__init__.py | 5 ++--- swh/lister/debian/lister.py | 11 +++++------ swh/lister/debian/models.py | 2 +- swh/lister/debian/tests/conftest.py | 1 - swh/lister/debian/tests/test_init.py | 3 +-- swh/lister/debian/tests/test_lister.py | 1 - swh/lister/debian/tests/test_models.py | 2 +- swh/lister/debian/utils.py | 2 +- swh/lister/gitea/__init__.py | 2 +- swh/lister/gitea/lister.py | 11 ++++++----- swh/lister/gitea/tasks.py | 1 - swh/lister/gitea/tests/test_tasks.py | 2 +- swh/lister/github/__init__.py | 2 +- swh/lister/github/lister.py | 5 ++--- swh/lister/github/models.py | 2 +- swh/lister/github/tests/test_tasks.py | 4 ++-- swh/lister/gitlab/__init__.py | 2 +- swh/lister/gitlab/lister.py | 6 +++--- swh/lister/gitlab/tasks.py | 1 - swh/lister/gitlab/tests/test_lister.py | 4 +--- swh/lister/gitlab/tests/test_tasks.py | 2 +- swh/lister/gnu/__init__.py | 2 +- swh/lister/gnu/lister.py | 9 ++++----- swh/lister/gnu/tests/test_lister.py | 1 - swh/lister/gnu/tests/test_tree.py | 8 ++++---- swh/lister/gnu/tree.py | 9 ++++----- swh/lister/launchpad/__init__.py | 2 +- swh/lister/launchpad/lister.py | 13 +++++++------ swh/lister/launchpad/models.py | 2 +- swh/lister/launchpad/tests/conftest.py | 4 +++- swh/lister/npm/__init__.py | 2 +- swh/lister/npm/lister.py | 7 ++++--- swh/lister/npm/models.py | 4 ++-- swh/lister/npm/tasks.py | 4 ++-- swh/lister/npm/tests/test_lister.py | 10 ++++------ swh/lister/packagist/__init__.py | 2 +- swh/lister/packagist/lister.py | 6 ++---- swh/lister/packagist/tests/test_lister.py | 7 +++---- swh/lister/phabricator/__init__.py | 2 +- swh/lister/phabricator/lister.py | 9 +++------ swh/lister/phabricator/models.py | 2 +- swh/lister/phabricator/tests/test_lister.py | 6 ++---- swh/lister/pypi/__init__.py | 2 +- swh/lister/pypi/lister.py | 12 ++++++------ swh/lister/pytest_plugin.py | 4 +--- swh/lister/tests/test_cli.py | 3 +-- swh/lister/tests/test_utils.py | 1 - 69 files changed, 136 insertions(+), 168 deletions(-) diff --git a/setup.py b/setup.py index 02d9066..bac190c 100755 --- a/setup.py +++ b/setup.py @@ -4,10 +4,10 @@ # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information -from setuptools import setup, find_packages - -from os import path from io import open +from os import path + +from setuptools import find_packages, setup here = path.abspath(path.dirname(__file__)) diff --git a/swh/lister/__init__.py b/swh/lister/__init__.py index b37e946..7589a96 100644 --- a/swh/lister/__init__.py +++ b/swh/lister/__init__.py @@ -4,8 +4,8 @@ # See top-level LICENSE file for more information import logging -import pkg_resources +import pkg_resources logger = logging.getLogger(__name__) diff --git a/swh/lister/bitbucket/__init__.py b/swh/lister/bitbucket/__init__.py index 917c7bd..3b61fc9 100644 --- a/swh/lister/bitbucket/__init__.py +++ b/swh/lister/bitbucket/__init__.py @@ -4,8 +4,8 @@ def register(): - from .models import BitBucketModel from .lister import BitBucketLister + from .models import BitBucketModel return { "models": [BitBucketModel], diff --git a/swh/lister/bitbucket/lister.py b/swh/lister/bitbucket/lister.py index d64281c..3ab5916 100644 --- a/swh/lister/bitbucket/lister.py +++ b/swh/lister/bitbucket/lister.py @@ -3,18 +3,17 @@ # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information -import logging -import iso8601 - from datetime import datetime, timezone +import logging from typing import Any, Dict, List, Optional from urllib import parse + +import iso8601 from requests import Response from swh.lister.bitbucket.models import BitBucketModel from swh.lister.core.indexing_lister import IndexingHttpLister - logger = logging.getLogger(__name__) diff --git a/swh/lister/bitbucket/models.py b/swh/lister/bitbucket/models.py index dca32f7..454b76d 100644 --- a/swh/lister/bitbucket/models.py +++ b/swh/lister/bitbucket/models.py @@ -2,7 +2,7 @@ # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information -from sqlalchemy import Column, String, DateTime +from sqlalchemy import Column, DateTime, String from swh.lister.core.models import IndexingModelBase diff --git a/swh/lister/bitbucket/tasks.py b/swh/lister/bitbucket/tasks.py index 68cae21..cd3f15f 100644 --- a/swh/lister/bitbucket/tasks.py +++ b/swh/lister/bitbucket/tasks.py @@ -3,6 +3,7 @@ # See top-level LICENSE file for more information import random + from celery import group, shared_task from .lister import BitBucketLister diff --git a/swh/lister/bitbucket/tests/test_lister.py b/swh/lister/bitbucket/tests/test_lister.py index 3757dc8..355103d 100644 --- a/swh/lister/bitbucket/tests/test_lister.py +++ b/swh/lister/bitbucket/tests/test_lister.py @@ -3,10 +3,9 @@ # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information +from datetime import timedelta import re import unittest - -from datetime import timedelta from urllib.parse import unquote import iso8601 diff --git a/swh/lister/bitbucket/tests/test_tasks.py b/swh/lister/bitbucket/tests/test_tasks.py index 3f02b9a..1297e2f 100644 --- a/swh/lister/bitbucket/tests/test_tasks.py +++ b/swh/lister/bitbucket/tests/test_tasks.py @@ -4,10 +4,10 @@ # See top-level LICENSE file for more information from time import sleep -from celery.result import GroupResult - from unittest.mock import patch +from celery.result import GroupResult + def test_ping(swh_scheduler_celery_app, swh_scheduler_celery_worker): res = swh_scheduler_celery_app.send_task("swh.lister.bitbucket.tasks.ping") diff --git a/swh/lister/cgit/__init__.py b/swh/lister/cgit/__init__.py index f5f9cf6..50cc8b5 100644 --- a/swh/lister/cgit/__init__.py +++ b/swh/lister/cgit/__init__.py @@ -4,8 +4,8 @@ def register(): - from .models import CGitModel from .lister import CGitLister + from .models import CGitModel return { "models": [CGitModel], diff --git a/swh/lister/cgit/lister.py b/swh/lister/cgit/lister.py index 5d88edc..6b8484d 100644 --- a/swh/lister/cgit/lister.py +++ b/swh/lister/cgit/lister.py @@ -2,21 +2,20 @@ # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information -import re import logging -from urllib.parse import urlparse, urljoin +import re +from typing import Any, Dict, Generator, Optional +from urllib.parse import urljoin, urlparse from bs4 import BeautifulSoup from requests import Session - from requests.adapters import HTTPAdapter -from typing import Any, Dict, Generator, Optional -from .models import CGitModel from swh.core.utils import grouper from swh.lister import USER_AGENT from swh.lister.core.lister_base import ListerBase +from .models import CGitModel logger = logging.getLogger(__name__) diff --git a/swh/lister/cli.py b/swh/lister/cli.py index f8cc623..523b273 100644 --- a/swh/lister/cli.py +++ b/swh/lister/cli.py @@ -3,17 +3,17 @@ # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information +from copy import deepcopy +import logging + # WARNING: do not import unnecessary things here to keep cli startup time under # control import os -import logging -from copy import deepcopy import click from swh.core.cli import CONTEXT_SETTINGS -from swh.lister import get_lister, SUPPORTED_LISTERS, LISTERS - +from swh.lister import LISTERS, SUPPORTED_LISTERS, get_lister logger = logging.getLogger(__name__) @@ -82,6 +82,7 @@ def db_init(ctx, drop_tables): """ from sqlalchemy import create_engine + from swh.lister.core.models import initialize cfg = ctx.obj["config"] diff --git a/swh/lister/core/indexing_lister.py b/swh/lister/core/indexing_lister.py index f7c6aa4..c1df1f2 100644 --- a/swh/lister/core/indexing_lister.py +++ b/swh/lister/core/indexing_lister.py @@ -3,18 +3,17 @@ # See top-level LICENSE file for more information import abc -import logging +from datetime import datetime from itertools import count +import logging +from typing import Any, Dict, List, Optional, Tuple, Union import dateutil +from requests import Response from sqlalchemy import func -from .lister_transports import ListerHttpTransport from .lister_base import ListerBase - -from requests import Response -from typing import Any, Dict, List, Tuple, Optional, Union -from datetime import datetime +from .lister_transports import ListerHttpTransport logger = logging.getLogger(__name__) diff --git a/swh/lister/core/lister_base.py b/swh/lister/core/lister_base.py index ff6e086..76ba592 100644 --- a/swh/lister/core/lister_base.py +++ b/swh/lister/core/lister_base.py @@ -10,10 +10,11 @@ import logging import os import re import time +from typing import Any, Dict, List, Optional, Type, Union +from requests import Response from sqlalchemy import create_engine, func from sqlalchemy.orm import sessionmaker -from typing import Any, Dict, List, Type, Union, Optional from swh.core import config from swh.core.utils import grouper @@ -21,8 +22,6 @@ from swh.scheduler import get_scheduler, utils from .abstractattribute import AbstractAttribute -from requests import Response - logger = logging.getLogger(__name__) diff --git a/swh/lister/core/lister_transports.py b/swh/lister/core/lister_transports.py index a1027ac..f5f9e80 100644 --- a/swh/lister/core/lister_transports.py +++ b/swh/lister/core/lister_transports.py @@ -3,24 +3,22 @@ # See top-level LICENSE file for more information import abc -import random from datetime import datetime from email.utils import parsedate -from pprint import pformat import logging +from pprint import pformat +import random +from typing import Any, Dict, List, Optional, Union import requests -import xmltodict - -from typing import Optional, Union, Dict, Any, List from requests import Response +import xmltodict from swh.lister import USER_AGENT_TEMPLATE, __version__ from .abstractattribute import AbstractAttribute from .lister_base import FetchError - logger = logging.getLogger(__name__) diff --git a/swh/lister/core/models.py b/swh/lister/core/models.py index 7e87d78..8d3d381 100644 --- a/swh/lister/core/models.py +++ b/swh/lister/core/models.py @@ -5,14 +5,13 @@ import abc from datetime import datetime import logging +from typing import Type, Union from sqlalchemy import Column, DateTime, Integer, String from sqlalchemy.ext.declarative import DeclarativeMeta, declarative_base -from typing import Type, Union from .abstractattribute import AbstractAttribute - SQLBase = declarative_base() diff --git a/swh/lister/core/page_by_page_lister.py b/swh/lister/core/page_by_page_lister.py index 1f38a2a..2db8a53 100644 --- a/swh/lister/core/page_by_page_lister.py +++ b/swh/lister/core/page_by_page_lister.py @@ -5,8 +5,8 @@ import abc import logging -from .lister_transports import ListerHttpTransport from .lister_base import ListerBase +from .lister_transports import ListerHttpTransport class PageByPageLister(ListerBase): diff --git a/swh/lister/core/simple_lister.py b/swh/lister/core/simple_lister.py index 9612986..d690b2d 100644 --- a/swh/lister/core/simple_lister.py +++ b/swh/lister/core/simple_lister.py @@ -4,12 +4,11 @@ # See top-level LICENSE file for more information import logging - from typing import Any, List from swh.core import utils -from .lister_base import ListerBase +from .lister_base import ListerBase logger = logging.getLogger(__name__) diff --git a/swh/lister/core/tests/test_abstractattribute.py b/swh/lister/core/tests/test_abstractattribute.py index 113ee0a..d804f21 100644 --- a/swh/lister/core/tests/test_abstractattribute.py +++ b/swh/lister/core/tests/test_abstractattribute.py @@ -3,9 +3,8 @@ # See top-level LICENSE file for more information import abc -import unittest - from typing import Any +import unittest from swh.lister.core.abstractattribute import AbstractAttribute diff --git a/swh/lister/core/tests/test_lister.py b/swh/lister/core/tests/test_lister.py index a835b1e..cb6cf2b 100644 --- a/swh/lister/core/tests/test_lister.py +++ b/swh/lister/core/tests/test_lister.py @@ -5,12 +5,12 @@ import abc import datetime import time +from typing import Any, Callable, Optional, Pattern, Type, Union from unittest import TestCase from unittest.mock import Mock, patch import requests_mock from sqlalchemy import create_engine -from typing import Any, Callable, Optional, Pattern, Type, Union import swh.lister from swh.lister.core.abstractattribute import AbstractAttribute diff --git a/swh/lister/cran/__init__.py b/swh/lister/cran/__init__.py index 3fa6586..69605f0 100644 --- a/swh/lister/cran/__init__.py +++ b/swh/lister/cran/__init__.py @@ -4,8 +4,8 @@ def register(): - from .models import CRANModel from .lister import CRANLister + from .models import CRANModel return { "models": [CRANModel], diff --git a/swh/lister/cran/lister.py b/swh/lister/cran/lister.py index 6f9e738..e22bba1 100644 --- a/swh/lister/cran/lister.py +++ b/swh/lister/cran/lister.py @@ -5,17 +5,15 @@ import json import logging -import pkg_resources import subprocess - from typing import List, Mapping, Tuple -from swh.lister.cran.models import CRANModel +import pkg_resources from swh.lister.core.simple_lister import SimpleLister +from swh.lister.cran.models import CRANModel from swh.scheduler.utils import create_task_dict - logger = logging.getLogger(__name__) diff --git a/swh/lister/cran/tests/test_lister.py b/swh/lister/cran/tests/test_lister.py index d7ce6a4..9512970 100644 --- a/swh/lister/cran/tests/test_lister.py +++ b/swh/lister/cran/tests/test_lister.py @@ -4,12 +4,12 @@ # See top-level LICENSE file for more information import json -import pytest - from os import path from unittest.mock import patch -from swh.lister.cran.lister import compute_origin_urls, CRAN_MIRROR +import pytest + +from swh.lister.cran.lister import CRAN_MIRROR, compute_origin_urls def test_cran_compute_origin_urls(): diff --git a/swh/lister/debian/__init__.py b/swh/lister/debian/__init__.py index 9e201b0..d121a86 100644 --- a/swh/lister/debian/__init__.py +++ b/swh/lister/debian/__init__.py @@ -4,10 +4,8 @@ # See top-level LICENSE file for more information import logging - from typing import Any, List, Mapping - logger = logging.getLogger(__name__) @@ -28,9 +26,10 @@ def debian_init( components: Default components to register per suite """ - from swh.lister.debian.models import Distribution, Area from sqlalchemy.orm import sessionmaker + from swh.lister.debian.models import Area, Distribution + db_session = sessionmaker(bind=db_engine)() distrib = ( db_session.query(Distribution) diff --git a/swh/lister/debian/lister.py b/swh/lister/debian/lister.py index 7355fc7..758c656 100644 --- a/swh/lister/debian/lister.py +++ b/swh/lister/debian/lister.py @@ -7,15 +7,17 @@ import bz2 from collections import defaultdict import datetime import gzip -import lzma import logging +import lzma +from typing import Any, Dict, Mapping, Optional from debian.deb822 import Sources +from requests import Response from sqlalchemy.orm import joinedload, load_only from sqlalchemy.schema import CreateTable, DropTable -from typing import Mapping, Optional, Dict, Any -from requests import Response +from swh.lister.core.lister_base import FetchError, ListerBase +from swh.lister.core.lister_transports import ListerHttpTransport from swh.lister.debian.models import ( AreaSnapshot, Distribution, @@ -24,9 +26,6 @@ from swh.lister.debian.models import ( TempPackage, ) -from swh.lister.core.lister_base import ListerBase, FetchError -from swh.lister.core.lister_transports import ListerHttpTransport - decompressors = { "gz": lambda f: gzip.GzipFile(fileobj=f), "bz2": bz2.BZ2File, diff --git a/swh/lister/debian/models.py b/swh/lister/debian/models.py index 9335b82..e1ffe39 100644 --- a/swh/lister/debian/models.py +++ b/swh/lister/debian/models.py @@ -6,6 +6,7 @@ import binascii from collections import defaultdict import datetime +from typing import Any, Mapping from sqlalchemy import ( Boolean, @@ -19,7 +20,6 @@ from sqlalchemy import ( Table, UniqueConstraint, ) -from typing import Any, Mapping try: from sqlalchemy import JSON diff --git a/swh/lister/debian/tests/conftest.py b/swh/lister/debian/tests/conftest.py index d624f8d..8adc690 100644 --- a/swh/lister/debian/tests/conftest.py +++ b/swh/lister/debian/tests/conftest.py @@ -4,7 +4,6 @@ # See top-level LICENSE file for more information import pytest - from pytest_postgresql.janitor import DatabaseJanitor from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker diff --git a/swh/lister/debian/tests/test_init.py b/swh/lister/debian/tests/test_init.py index 860c4ec..3048613 100644 --- a/swh/lister/debian/tests/test_init.py +++ b/swh/lister/debian/tests/test_init.py @@ -6,8 +6,7 @@ import pytest from swh.lister.debian import debian_init - -from swh.lister.debian.models import Distribution, Area +from swh.lister.debian.models import Area, Distribution @pytest.fixture diff --git a/swh/lister/debian/tests/test_lister.py b/swh/lister/debian/tests/test_lister.py index 8694d8d..4fb4169 100644 --- a/swh/lister/debian/tests/test_lister.py +++ b/swh/lister/debian/tests/test_lister.py @@ -5,7 +5,6 @@ import logging - logger = logging.getLogger(__name__) diff --git a/swh/lister/debian/tests/test_models.py b/swh/lister/debian/tests/test_models.py index 43d1555..25cf995 100644 --- a/swh/lister/debian/tests/test_models.py +++ b/swh/lister/debian/tests/test_models.py @@ -5,7 +5,7 @@ import pytest -from swh.lister.debian.models import Distribution, Area +from swh.lister.debian.models import Area, Distribution def test_area_index_uris_deb(session): diff --git a/swh/lister/debian/utils.py b/swh/lister/debian/utils.py index f6c4ca8..777b401 100644 --- a/swh/lister/debian/utils.py +++ b/swh/lister/debian/utils.py @@ -6,8 +6,8 @@ import logging import click -from swh.lister.debian.models import Distribution, Area, SQLBase from swh.lister.debian.lister import DebianLister +from swh.lister.debian.models import Area, Distribution, SQLBase @click.group() diff --git a/swh/lister/gitea/__init__.py b/swh/lister/gitea/__init__.py index 87d5c77..c2ddbb2 100644 --- a/swh/lister/gitea/__init__.py +++ b/swh/lister/gitea/__init__.py @@ -4,8 +4,8 @@ def register(): - from .models import GiteaModel from .lister import GiteaLister + from .models import GiteaModel return { "models": [GiteaModel], diff --git a/swh/lister/gitea/lister.py b/swh/lister/gitea/lister.py index 9c8a511..a386d5f 100644 --- a/swh/lister/gitea/lister.py +++ b/swh/lister/gitea/lister.py @@ -3,14 +3,15 @@ # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information +import re +from typing import Any, Dict, List, MutableMapping, Optional, Tuple + +from requests import Response +from urllib3.util import parse_url + from ..core.page_by_page_lister import PageByPageHttpLister from .models import GiteaModel -from typing import Any, Dict, List, Tuple, MutableMapping, Optional -from requests import Response -import re -from urllib3.util import parse_url - class GiteaLister(PageByPageHttpLister): # Template path expecting an integer that represents the page id diff --git a/swh/lister/gitea/tasks.py b/swh/lister/gitea/tasks.py index 098f822..33961ef 100644 --- a/swh/lister/gitea/tasks.py +++ b/swh/lister/gitea/tasks.py @@ -9,7 +9,6 @@ from celery import group, shared_task from .. import utils from .lister import GiteaLister - NBPAGES = 10 diff --git a/swh/lister/gitea/tests/test_tasks.py b/swh/lister/gitea/tests/test_tasks.py index c780510..92ff475 100644 --- a/swh/lister/gitea/tests/test_tasks.py +++ b/swh/lister/gitea/tests/test_tasks.py @@ -4,9 +4,9 @@ # See top-level LICENSE file for more information from time import sleep +from unittest.mock import call, patch from celery.result import GroupResult -from unittest.mock import patch, call from swh.lister.gitea.tasks import NBPAGES from swh.lister.utils import split_range diff --git a/swh/lister/github/__init__.py b/swh/lister/github/__init__.py index 1704bc8..1123a6c 100644 --- a/swh/lister/github/__init__.py +++ b/swh/lister/github/__init__.py @@ -4,8 +4,8 @@ def register(): - from .models import GitHubModel from .lister import GitHubLister + from .models import GitHubModel return { "models": [GitHubModel], diff --git a/swh/lister/github/lister.py b/swh/lister/github/lister.py index 0f3b71d..0094aaf 100644 --- a/swh/lister/github/lister.py +++ b/swh/lister/github/lister.py @@ -4,14 +4,13 @@ # See top-level LICENSE file for more information import re +from typing import Any, Dict, List, Optional, Tuple -from typing import Any, Dict, List, Tuple, Optional +from requests import Response from swh.lister.core.indexing_lister import IndexingHttpLister from swh.lister.github.models import GitHubModel -from requests import Response - class GitHubLister(IndexingHttpLister): PATH_TEMPLATE = "/repositories?since=%d" diff --git a/swh/lister/github/models.py b/swh/lister/github/models.py index 58de011..3c55e5d 100644 --- a/swh/lister/github/models.py +++ b/swh/lister/github/models.py @@ -2,7 +2,7 @@ # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information -from sqlalchemy import Column, Boolean, Integer +from sqlalchemy import Boolean, Column, Integer from swh.lister.core.models import IndexingModelBase diff --git a/swh/lister/github/tests/test_tasks.py b/swh/lister/github/tests/test_tasks.py index 409ed6e..aef8fd7 100644 --- a/swh/lister/github/tests/test_tasks.py +++ b/swh/lister/github/tests/test_tasks.py @@ -1,8 +1,8 @@ from time import sleep -from celery.result import GroupResult - from unittest.mock import patch +from celery.result import GroupResult + def test_ping(swh_scheduler_celery_app, swh_scheduler_celery_worker): res = swh_scheduler_celery_app.send_task("swh.lister.github.tasks.ping") diff --git a/swh/lister/gitlab/__init__.py b/swh/lister/gitlab/__init__.py index 5ddf416..eec7bc8 100644 --- a/swh/lister/gitlab/__init__.py +++ b/swh/lister/gitlab/__init__.py @@ -4,8 +4,8 @@ def register(): - from .models import GitLabModel from .lister import GitLabLister + from .models import GitLabModel return { "models": [GitLabModel], diff --git a/swh/lister/gitlab/lister.py b/swh/lister/gitlab/lister.py index ca70bf4..b3a3880 100644 --- a/swh/lister/gitlab/lister.py +++ b/swh/lister/gitlab/lister.py @@ -4,14 +4,14 @@ # See top-level LICENSE file for more information import time +from typing import Any, Dict, List, MutableMapping, Optional, Tuple, Union + +from requests import Response from urllib3.util import parse_url from ..core.page_by_page_lister import PageByPageHttpLister from .models import GitLabModel -from typing import Any, Dict, List, Tuple, Union, MutableMapping, Optional -from requests import Response - class GitLabLister(PageByPageHttpLister): # Template path expecting an integer that represents the page id diff --git a/swh/lister/gitlab/tasks.py b/swh/lister/gitlab/tasks.py index 85866bd..356ed20 100644 --- a/swh/lister/gitlab/tasks.py +++ b/swh/lister/gitlab/tasks.py @@ -9,7 +9,6 @@ from celery import group, shared_task from .. import utils from .lister import GitLabLister - NBPAGES = 10 diff --git a/swh/lister/gitlab/tests/test_lister.py b/swh/lister/gitlab/tests/test_lister.py index c22e9bf..151def6 100644 --- a/swh/lister/gitlab/tests/test_lister.py +++ b/swh/lister/gitlab/tests/test_lister.py @@ -3,18 +3,16 @@ # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information +from datetime import datetime, timedelta import logging import re import unittest -from datetime import datetime, timedelta - import pytest from swh.lister.core.tests.test_lister import HttpListerTesterBase from swh.lister.gitlab.lister import GitLabLister - logger = logging.getLogger(__name__) diff --git a/swh/lister/gitlab/tests/test_tasks.py b/swh/lister/gitlab/tests/test_tasks.py index 466da99..3e3b4e5 100644 --- a/swh/lister/gitlab/tests/test_tasks.py +++ b/swh/lister/gitlab/tests/test_tasks.py @@ -4,9 +4,9 @@ # See top-level LICENSE file for more information from time import sleep +from unittest.mock import call, patch from celery.result import GroupResult -from unittest.mock import patch, call from swh.lister.gitea.tasks import NBPAGES from swh.lister.utils import split_range diff --git a/swh/lister/gnu/__init__.py b/swh/lister/gnu/__init__.py index 8ff20bb..eec08d4 100644 --- a/swh/lister/gnu/__init__.py +++ b/swh/lister/gnu/__init__.py @@ -4,8 +4,8 @@ def register(): - from .models import GNUModel from .lister import GNULister + from .models import GNUModel return { "models": [GNUModel], diff --git a/swh/lister/gnu/lister.py b/swh/lister/gnu/lister.py index 3390078..e6582fb 100644 --- a/swh/lister/gnu/lister.py +++ b/swh/lister/gnu/lister.py @@ -3,15 +3,14 @@ # See top-level LICENSE file for more information import logging +from typing import Any, Dict, List + +from requests import Response -from swh.scheduler import utils from swh.lister.core.simple_lister import SimpleLister - from swh.lister.gnu.models import GNUModel from swh.lister.gnu.tree import GNUTree - -from typing import Any, Dict, List -from requests import Response +from swh.scheduler import utils logger = logging.getLogger(__name__) diff --git a/swh/lister/gnu/tests/test_lister.py b/swh/lister/gnu/tests/test_lister.py index 68c0bc5..8dedfa9 100644 --- a/swh/lister/gnu/tests/test_lister.py +++ b/swh/lister/gnu/tests/test_lister.py @@ -7,7 +7,6 @@ import logging import pytest - logger = logging.getLogger(__name__) diff --git a/swh/lister/gnu/tests/test_tree.py b/swh/lister/gnu/tests/test_tree.py index f09fe9e..ba11e9a 100644 --- a/swh/lister/gnu/tests/test_tree.py +++ b/swh/lister/gnu/tests/test_tree.py @@ -4,17 +4,17 @@ # See top-level LICENSE file for more information import json +from os import path import pytest -from os import path from swh.lister.gnu.tree import ( GNUTree, - find_artifacts, check_filename_is_archive, - load_raw_data, - get_version, + find_artifacts, format_date, + get_version, + load_raw_data, ) diff --git a/swh/lister/gnu/tree.py b/swh/lister/gnu/tree.py index 0e47293..f666cb5 100644 --- a/swh/lister/gnu/tree.py +++ b/swh/lister/gnu/tree.py @@ -3,19 +3,18 @@ # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information +from datetime import datetime import gzip import json import logging -import requests -import re - -from datetime import datetime from os import path from pathlib import Path -from pytz import utc +import re from typing import Any, List, Mapping, Sequence, Tuple from urllib.parse import urlparse +from pytz import utc +import requests logger = logging.getLogger(__name__) diff --git a/swh/lister/launchpad/__init__.py b/swh/lister/launchpad/__init__.py index 5dd5484..ac9c66b 100644 --- a/swh/lister/launchpad/__init__.py +++ b/swh/lister/launchpad/__init__.py @@ -4,8 +4,8 @@ def register(): - from .models import LaunchpadModel from .lister import LaunchpadLister + from .models import LaunchpadModel return { "models": [LaunchpadModel], diff --git a/swh/lister/launchpad/lister.py b/swh/lister/launchpad/lister.py index 2beaf15..de11e93 100644 --- a/swh/lister/launchpad/lister.py +++ b/swh/lister/launchpad/lister.py @@ -3,17 +3,18 @@ # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information -from typing import Optional, Dict, List, Union, Tuple, Any - -from swh.lister.core.lister_base import ListerBase -from .models import LaunchpadModel - +from datetime import datetime, timedelta from itertools import count +from typing import Any, Dict, List, Optional, Tuple, Union + from launchpadlib.launchpad import Launchpad # type: ignore from lazr.restfulclient.resource import Collection, Entry # type: ignore -from datetime import datetime, timedelta from sqlalchemy import func +from swh.lister.core.lister_base import ListerBase + +from .models import LaunchpadModel + class LaunchpadLister(ListerBase): MODEL = LaunchpadModel diff --git a/swh/lister/launchpad/models.py b/swh/lister/launchpad/models.py index 9fae311..daa0a4b 100644 --- a/swh/lister/launchpad/models.py +++ b/swh/lister/launchpad/models.py @@ -2,7 +2,7 @@ # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information -from sqlalchemy import Column, String, Date +from sqlalchemy import Column, Date, String from swh.lister.core.models import ModelBase diff --git a/swh/lister/launchpad/tests/conftest.py b/swh/lister/launchpad/tests/conftest.py index 655d858..bd69f08 100644 --- a/swh/lister/launchpad/tests/conftest.py +++ b/swh/lister/launchpad/tests/conftest.py @@ -7,10 +7,12 @@ from datetime import datetime import json import os from unittest.mock import patch + import pytest +from sqlalchemy.engine import create_engine + from swh.lister import get_lister from swh.lister.core.models import initialize -from sqlalchemy.engine import create_engine @pytest.fixture diff --git a/swh/lister/npm/__init__.py b/swh/lister/npm/__init__.py index 0d10210..e2ec1bf 100644 --- a/swh/lister/npm/__init__.py +++ b/swh/lister/npm/__init__.py @@ -4,8 +4,8 @@ def register(): - from .models import NpmVisitModel, NpmModel from .lister import NpmLister + from .models import NpmModel, NpmVisitModel return { "models": [NpmVisitModel, NpmModel], diff --git a/swh/lister/npm/lister.py b/swh/lister/npm/lister.py index 15c2556..485842b 100644 --- a/swh/lister/npm/lister.py +++ b/swh/lister/npm/lister.py @@ -2,13 +2,14 @@ # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information +from typing import Any, Dict, List, Optional + +from requests import Response + from swh.lister.core.indexing_lister import IndexingHttpLister from swh.lister.npm.models import NpmModel from swh.scheduler.utils import create_task_dict -from typing import Any, Dict, Optional, List -from requests import Response - class NpmListerBase(IndexingHttpLister): """List packages available in the npm registry in a paginated way diff --git a/swh/lister/npm/models.py b/swh/lister/npm/models.py index 08f7d6e..e8d88ab 100644 --- a/swh/lister/npm/models.py +++ b/swh/lister/npm/models.py @@ -2,9 +2,9 @@ # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information -from sqlalchemy import Column, String, DateTime, Integer, BigInteger, Sequence +from sqlalchemy import BigInteger, Column, DateTime, Integer, Sequence, String -from swh.lister.core.models import IndexingModelBase, SQLBase, ABCSQLMeta +from swh.lister.core.models import ABCSQLMeta, IndexingModelBase, SQLBase class NpmVisitModel(SQLBase, metaclass=ABCSQLMeta): diff --git a/swh/lister/npm/tasks.py b/swh/lister/npm/tasks.py index b0d06d4..5e977b4 100644 --- a/swh/lister/npm/tasks.py +++ b/swh/lister/npm/tasks.py @@ -2,12 +2,12 @@ # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information -from datetime import datetime from contextlib import contextmanager +from datetime import datetime from celery import shared_task -from swh.lister.npm.lister import NpmLister, NpmIncrementalLister +from swh.lister.npm.lister import NpmIncrementalLister, NpmLister from swh.lister.npm.models import NpmVisitModel diff --git a/swh/lister/npm/tests/test_lister.py b/swh/lister/npm/tests/test_lister.py index 5a28a6d..6566ec8 100644 --- a/swh/lister/npm/tests/test_lister.py +++ b/swh/lister/npm/tests/test_lister.py @@ -4,16 +4,14 @@ # See top-level LICENSE file for more information import logging - import re -import requests_mock +from typing import Any, List import unittest +import requests_mock + from swh.lister.core.tests.test_lister import HttpListerTesterBase -from swh.lister.npm.lister import NpmLister, NpmIncrementalLister - -from typing import Any, List - +from swh.lister.npm.lister import NpmIncrementalLister, NpmLister logger = logging.getLogger(__name__) diff --git a/swh/lister/packagist/__init__.py b/swh/lister/packagist/__init__.py index a97ede1..262008f 100644 --- a/swh/lister/packagist/__init__.py +++ b/swh/lister/packagist/__init__.py @@ -4,8 +4,8 @@ def register(): - from .models import PackagistModel from .lister import PackagistLister + from .models import PackagistModel return { "models": [PackagistModel], diff --git a/swh/lister/packagist/lister.py b/swh/lister/packagist/lister.py index e7b9709..e49a99c 100644 --- a/swh/lister/packagist/lister.py +++ b/swh/lister/packagist/lister.py @@ -6,16 +6,14 @@ import json import logging import random - from typing import Any, Dict, List, Mapping -from swh.scheduler import utils -from swh.lister.core.simple_lister import SimpleLister from swh.lister.core.lister_transports import ListerOnePageApiTransport +from swh.lister.core.simple_lister import SimpleLister +from swh.scheduler import utils from .models import PackagistModel - logger = logging.getLogger(__name__) diff --git a/swh/lister/packagist/tests/test_lister.py b/swh/lister/packagist/tests/test_lister.py index 3bfff49..808910f 100644 --- a/swh/lister/packagist/tests/test_lister.py +++ b/swh/lister/packagist/tests/test_lister.py @@ -4,13 +4,12 @@ # See top-level LICENSE file for more information import unittest -import requests_mock - from unittest.mock import patch -from swh.lister.packagist.lister import PackagistLister, compute_package_url -from swh.lister.core.tests.test_lister import HttpSimpleListerTester +import requests_mock +from swh.lister.core.tests.test_lister import HttpSimpleListerTester +from swh.lister.packagist.lister import PackagistLister, compute_package_url expected_packages = [ "0.0.0/composer-include-files", diff --git a/swh/lister/phabricator/__init__.py b/swh/lister/phabricator/__init__.py index 3f5ff29..967a23d 100644 --- a/swh/lister/phabricator/__init__.py +++ b/swh/lister/phabricator/__init__.py @@ -4,8 +4,8 @@ def register(): - from .models import PhabricatorModel from .lister import PhabricatorLister + from .models import PhabricatorModel return { "models": [PhabricatorModel], diff --git a/swh/lister/phabricator/lister.py b/swh/lister/phabricator/lister.py index 89487ae..9db17b4 100644 --- a/swh/lister/phabricator/lister.py +++ b/swh/lister/phabricator/lister.py @@ -2,21 +2,18 @@ # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information +from collections import defaultdict import logging import random - +from typing import Any, Dict, List, Optional import urllib.parse -from collections import defaultdict - +from requests import Response from sqlalchemy import func from swh.lister.core.indexing_lister import IndexingHttpLister from swh.lister.phabricator.models import PhabricatorModel -from typing import Any, Dict, List, Optional -from requests import Response - logger = logging.getLogger(__name__) diff --git a/swh/lister/phabricator/models.py b/swh/lister/phabricator/models.py index 676be83..5a41909 100644 --- a/swh/lister/phabricator/models.py +++ b/swh/lister/phabricator/models.py @@ -2,7 +2,7 @@ # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information -from sqlalchemy import Column, String, Integer +from sqlalchemy import Column, Integer, String from swh.lister.core.models import IndexingModelBase diff --git a/swh/lister/phabricator/tests/test_lister.py b/swh/lister/phabricator/tests/test_lister.py index 6b95af0..e7d2bfa 100644 --- a/swh/lister/phabricator/tests/test_lister.py +++ b/swh/lister/phabricator/tests/test_lister.py @@ -3,17 +3,15 @@ # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information -import re import json import logging +import re import unittest import requests_mock from swh.lister.core.tests.test_lister import HttpListerTester -from swh.lister.phabricator.lister import PhabricatorLister -from swh.lister.phabricator.lister import get_repo_url - +from swh.lister.phabricator.lister import PhabricatorLister, get_repo_url logger = logging.getLogger(__name__) diff --git a/swh/lister/pypi/__init__.py b/swh/lister/pypi/__init__.py index 6266e58..3ea7a3f 100644 --- a/swh/lister/pypi/__init__.py +++ b/swh/lister/pypi/__init__.py @@ -4,8 +4,8 @@ def register(): - from .models import PyPIModel from .lister import PyPILister + from .models import PyPIModel return { "models": [PyPIModel], diff --git a/swh/lister/pypi/lister.py b/swh/lister/pypi/lister.py index e7223e7..c372732 100644 --- a/swh/lister/pypi/lister.py +++ b/swh/lister/pypi/lister.py @@ -4,16 +4,16 @@ # See top-level LICENSE file for more information import random +from typing import Any, Dict + +from requests import Response import xmltodict -from .models import PyPIModel - -from swh.scheduler import utils -from swh.lister.core.simple_lister import SimpleLister from swh.lister.core.lister_transports import ListerOnePageApiTransport +from swh.lister.core.simple_lister import SimpleLister +from swh.scheduler import utils -from typing import Any, Dict -from requests import Response +from .models import PyPIModel class PyPILister(ListerOnePageApiTransport, SimpleLister): diff --git a/swh/lister/pytest_plugin.py b/swh/lister/pytest_plugin.py index e153d15..8e235f1 100644 --- a/swh/lister/pytest_plugin.py +++ b/swh/lister/pytest_plugin.py @@ -6,13 +6,11 @@ import logging import pytest - from sqlalchemy import create_engine -from swh.lister import get_lister, SUPPORTED_LISTERS +from swh.lister import SUPPORTED_LISTERS, get_lister from swh.lister.core.models import initialize - logger = logging.getLogger(__name__) diff --git a/swh/lister/tests/test_cli.py b/swh/lister/tests/test_cli.py index 302c3aa..e71a2fa 100644 --- a/swh/lister/tests/test_cli.py +++ b/swh/lister/tests/test_cli.py @@ -5,9 +5,8 @@ import pytest - +from swh.lister.cli import SUPPORTED_LISTERS, get_lister from swh.lister.core.lister_base import ListerBase -from swh.lister.cli import get_lister, SUPPORTED_LISTERS from .test_utils import init_db diff --git a/swh/lister/tests/test_utils.py b/swh/lister/tests/test_utils.py index defde1a..0d965e3 100644 --- a/swh/lister/tests/test_utils.py +++ b/swh/lister/tests/test_utils.py @@ -3,7 +3,6 @@ # See top-level LICENSE file for more information import pytest - from testing.postgresql import Postgresql from swh.lister import utils