reonokiy/swh-lister
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Enable black

Browse source 
- blackify all the python files,
- enable black in pre-commit,
- add a black tox environment.
This commit is contained in:
David Douard 2020-04-08 16:31:22 +02:00
parent 1ae75166c7
commit 93a4d8b784
97 changed files with 1734 additions and 1642 deletions
Download patch file Download diff file Expand all files Collapse all files

9
swh/lister/bitbucket/__init__.py
View file

@ -7,7 +7,8 @@ def register():
from .models import BitBucketModel
from .lister import BitBucketLister
return {'models': [BitBucketModel],
'lister': BitBucketLister,
'task_modules': ['%s.tasks' % __name__],
}
return {
"models": [BitBucketModel],
"lister": BitBucketLister,
"task_modules": ["%s.tasks" % __name__],
}

51
swh/lister/bitbucket/lister.py
View file

@ -19,34 +19,33 @@ logger = logging.getLogger(__name__)
class BitBucketLister(IndexingHttpLister):
PATH_TEMPLATE = '/repositories?after=%s'
PATH_TEMPLATE = "/repositories?after=%s"
MODEL = BitBucketModel
LISTER_NAME = 'bitbucket'
DEFAULT_URL = 'https://api.bitbucket.org/2.0'
instance = 'bitbucket'
LISTER_NAME = "bitbucket"
DEFAULT_URL = "https://api.bitbucket.org/2.0"
instance = "bitbucket"
default_min_bound = datetime.fromtimestamp(0, timezone.utc) # type: Any
def __init__(self, url: str = None,
override_config=None, per_page: int = 100) -> None:
def __init__(
self, url: str = None, override_config=None, per_page: int = 100
) -> None:
super().__init__(url=url, override_config=override_config)
per_page = self.config.get('per_page', per_page)
per_page = self.config.get("per_page", per_page)
self.PATH_TEMPLATE = '%s&pagelen=%s' % (
self.PATH_TEMPLATE, per_page)
self.PATH_TEMPLATE = "%s&pagelen=%s" % (self.PATH_TEMPLATE, per_page)
def get_model_from_repo(self, repo: Dict) -> Dict[str, Any]:
return {
'uid': repo['uuid'],
'indexable': iso8601.parse_date(repo['created_on']),
'name': repo['name'],
'full_name': repo['full_name'],
'html_url': repo['links']['html']['href'],
'origin_url': repo['links']['clone'][0]['href'],
'origin_type': repo['scm'],
"uid": repo["uuid"],
"indexable": iso8601.parse_date(repo["created_on"]),
"name": repo["name"],
"full_name": repo["full_name"],
"html_url": repo["links"]["html"]["href"],
"origin_url": repo["links"]["clone"][0]["href"],
"origin_type": repo["scm"],
}
def get_next_target_from_response(self, response: Response
) -> Optional[datetime]:
def get_next_target_from_response(self, response: Response) -> Optional[datetime]:
"""This will read the 'next' link from the api response if any
and return it as a datetime.
@ -58,23 +57,23 @@ class BitBucketLister(IndexingHttpLister):
"""
body = response.json()
next_ = body.get('next')
next_ = body.get("next")
if next_ is not None:
next_ = parse.urlparse(next_)
return iso8601.parse_date(parse.parse_qs(next_.query)['after'][0])
return iso8601.parse_date(parse.parse_qs(next_.query)["after"][0])
return None
def transport_response_simplified(self, response: Response
) -> List[Dict[str, Any]]:
repos = response.json()['values']
def transport_response_simplified(self, response: Response) -> List[Dict[str, Any]]:
repos = response.json()["values"]
return [self.get_model_from_repo(repo) for repo in repos]
def request_uri(self, identifier: datetime) -> str: # type: ignore
identifier_str = parse.quote(identifier.isoformat())
return super().request_uri(identifier_str or '1970-01-01')
return super().request_uri(identifier_str or "1970-01-01")
def is_within_bounds(self, inner: int, lower: Optional[int] = None,
upper: Optional[int] = None) -> bool:
def is_within_bounds(
self, inner: int, lower: Optional[int] = None, upper: Optional[int] = None
) -> bool:
# values are expected to be datetimes
if lower is None and upper is None:
ret = True

3
swh/lister/bitbucket/models.py
View file

@ -9,7 +9,8 @@ from swh.lister.core.models import IndexingModelBase
class BitBucketModel(IndexingModelBase):
"""a BitBucket repository"""
__tablename__ = 'bitbucket_repo'
__tablename__ = "bitbucket_repo"
uid = Column(String, primary_key=True)
indexable = Column(DateTime(timezone=True), index=True)

23
swh/lister/bitbucket/tasks.py
View file

@ -10,20 +10,20 @@ from .lister import BitBucketLister
GROUP_SPLIT = 10000
@shared_task(name=__name__ + '.IncrementalBitBucketLister')
@shared_task(name=__name__ + ".IncrementalBitBucketLister")
def list_bitbucket_incremental(**lister_args):
'''Incremental update of the BitBucket forge'''
"""Incremental update of the BitBucket forge"""
lister = BitBucketLister(**lister_args)
return lister.run(min_bound=lister.db_last_index(), max_bound=None)
@shared_task(name=__name__ + '.RangeBitBucketLister')
@shared_task(name=__name__ + ".RangeBitBucketLister")
def _range_bitbucket_lister(start, end, **lister_args):
lister = BitBucketLister(**lister_args)
return lister.run(min_bound=start, max_bound=end)
@shared_task(name=__name__ + '.FullBitBucketRelister', bind=True)
@shared_task(name=__name__ + ".FullBitBucketRelister", bind=True)
def list_bitbucket_full(self, split=None, **lister_args):
"""Full update of the BitBucket forge
@ -33,21 +33,22 @@ def list_bitbucket_full(self, split=None, **lister_args):
lister = BitBucketLister(**lister_args)
ranges = lister.db_partition_indices(split or GROUP_SPLIT)
if not ranges:
self.log.info('Nothing to list')
self.log.info("Nothing to list")
return
random.shuffle(ranges)
promise = group(_range_bitbucket_lister.s(minv, maxv, **lister_args)
for minv, maxv in ranges)()
self.log.debug('%s OK (spawned %s subtasks)', (self.name, len(ranges)))
promise = group(
_range_bitbucket_lister.s(minv, maxv, **lister_args) for minv, maxv in ranges
)()
self.log.debug("%s OK (spawned %s subtasks)", (self.name, len(ranges)))
try:
promise.save() # so that we can restore the GroupResult in tests
except (NotImplementedError, AttributeError):
self.log.info('Unable to call save_group with current result backend.')
self.log.info("Unable to call save_group with current result backend.")
# FIXME: what to do in terms of return here?
return promise.id
@shared_task(name=__name__ + '.ping')
@shared_task(name=__name__ + ".ping")
def _ping():
return 'OK'
return "OK"

71
swh/lister/bitbucket/tests/test_lister.py
View file

@ -26,12 +26,12 @@ def _convert_type(req_index):
class BitBucketListerTester(HttpListerTester, unittest.TestCase):
Lister = BitBucketLister
test_re = re.compile(r'/repositories\?after=([^?&]+)')
lister_subdir = 'bitbucket'
good_api_response_file = 'data/https_api.bitbucket.org/response.json'
bad_api_response_file = 'data/https_api.bitbucket.org/empty_response.json'
first_index = _convert_type('2008-07-12T07:44:01.476818+00:00')
last_index = _convert_type('2008-07-19T06:16:43.044743+00:00')
test_re = re.compile(r"/repositories\?after=([^?&]+)")
lister_subdir = "bitbucket"
good_api_response_file = "data/https_api.bitbucket.org/response.json"
bad_api_response_file = "data/https_api.bitbucket.org/empty_response.json"
first_index = _convert_type("2008-07-12T07:44:01.476818+00:00")
last_index = _convert_type("2008-07-19T06:16:43.044743+00:00")
entries_per_page = 10
convert_type = _convert_type
@ -57,57 +57,64 @@ class BitBucketListerTester(HttpListerTester, unittest.TestCase):
self.disable_db(fl)
# stores no results
fl.run(min_bound=self.first_index - timedelta(days=3),
max_bound=self.first_index)
fl.run(
min_bound=self.first_index - timedelta(days=3), max_bound=self.first_index
)
def test_is_within_bounds(self):
fl = self.get_fl()
self.assertTrue(fl.is_within_bounds(
iso8601.parse_date('2008-07-15'),
self.first_index, self.last_index))
self.assertFalse(fl.is_within_bounds(
iso8601.parse_date('2008-07-20'),
self.first_index, self.last_index))
self.assertFalse(fl.is_within_bounds(
iso8601.parse_date('2008-07-11'),
self.first_index, self.last_index))
self.assertTrue(
fl.is_within_bounds(
iso8601.parse_date("2008-07-15"), self.first_index, self.last_index
)
)
self.assertFalse(
fl.is_within_bounds(
iso8601.parse_date("2008-07-20"), self.first_index, self.last_index
)
)
self.assertFalse(
fl.is_within_bounds(
iso8601.parse_date("2008-07-11"), self.first_index, self.last_index
)
)
def test_lister_bitbucket(swh_listers, requests_mock_datadir):
"""Simple bitbucket listing should create scheduled tasks (git, hg)
"""
lister = swh_listers['bitbucket']
lister = swh_listers["bitbucket"]
lister.run()
r = lister.scheduler.search_tasks(task_type='load-hg')
r = lister.scheduler.search_tasks(task_type="load-hg")
assert len(r) == 9
for row in r:
args = row['arguments']['args']
kwargs = row['arguments']['kwargs']
args = row["arguments"]["args"]
kwargs = row["arguments"]["kwargs"]
assert len(args) == 0
assert len(kwargs) == 1
url = kwargs['url']
url = kwargs["url"]
assert url.startswith('https://bitbucket.org')
assert url.startswith("https://bitbucket.org")
assert row['policy'] == 'recurring'
assert row['priority'] is None
assert row["policy"] == "recurring"
assert row["priority"] is None
r = lister.scheduler.search_tasks(task_type='load-git')
r = lister.scheduler.search_tasks(task_type="load-git")
assert len(r) == 1
for row in r:
args = row['arguments']['args']
kwargs = row['arguments']['kwargs']
args = row["arguments"]["args"]
kwargs = row["arguments"]["kwargs"]
assert len(args) == 0
assert len(kwargs) == 1
url = kwargs['url']
url = kwargs["url"]
assert url.startswith('https://bitbucket.org')
assert url.startswith("https://bitbucket.org")
assert row['policy'] == 'recurring'
assert row['priority'] is None
assert row["policy"] == "recurring"
assert row["priority"] is None

29
swh/lister/bitbucket/tests/test_tasks.py
View file

@ -5,23 +5,21 @@ from unittest.mock import patch
def test_ping(swh_app, celery_session_worker):
res = swh_app.send_task(
'swh.lister.bitbucket.tasks.ping')
res = swh_app.send_task("swh.lister.bitbucket.tasks.ping")
assert res
res.wait()
assert res.successful()
assert res.result == 'OK'
assert res.result == "OK"
@patch('swh.lister.bitbucket.tasks.BitBucketLister')
@patch("swh.lister.bitbucket.tasks.BitBucketLister")
def test_incremental(lister, swh_app, celery_session_worker):
# setup the mocked BitbucketLister
lister.return_value = lister
lister.db_last_index.return_value = 42
lister.run.return_value = None
res = swh_app.send_task(
'swh.lister.bitbucket.tasks.IncrementalBitBucketLister')
res = swh_app.send_task("swh.lister.bitbucket.tasks.IncrementalBitBucketLister")
assert res
res.wait()
assert res.successful()
@ -31,15 +29,15 @@ def test_incremental(lister, swh_app, celery_session_worker):
lister.run.assert_called_once_with(min_bound=42, max_bound=None)
@patch('swh.lister.bitbucket.tasks.BitBucketLister')
@patch("swh.lister.bitbucket.tasks.BitBucketLister")
def test_range(lister, swh_app, celery_session_worker):
# setup the mocked BitbucketLister
lister.return_value = lister
lister.run.return_value = None
res = swh_app.send_task(
'swh.lister.bitbucket.tasks.RangeBitBucketLister',
kwargs=dict(start=12, end=42))
"swh.lister.bitbucket.tasks.RangeBitBucketLister", kwargs=dict(start=12, end=42)
)
assert res
res.wait()
assert res.successful()
@ -49,16 +47,14 @@ def test_range(lister, swh_app, celery_session_worker):
lister.run.assert_called_once_with(min_bound=12, max_bound=42)
@patch('swh.lister.bitbucket.tasks.BitBucketLister')
@patch("swh.lister.bitbucket.tasks.BitBucketLister")
def test_relister(lister, swh_app, celery_session_worker):
# setup the mocked BitbucketLister
lister.return_value = lister
lister.run.return_value = None
lister.db_partition_indices.return_value = [
(i, i+9) for i in range(0, 50, 10)]
lister.db_partition_indices.return_value = [(i, i + 9) for i in range(0, 50, 10)]
res = swh_app.send_task(
'swh.lister.bitbucket.tasks.FullBitBucketRelister')
res = swh_app.send_task("swh.lister.bitbucket.tasks.FullBitBucketRelister")
assert res
res.wait()
@ -85,5 +81,6 @@ def test_relister(lister, swh_app, celery_session_worker):
# lister.run should have been called once per partition interval
for i in range(5):
assert (dict(min_bound=10*i, max_bound=10*i + 9),) \
in lister.run.call_args_list
assert (
dict(min_bound=10 * i, max_bound=10 * i + 9),
) in lister.run.call_args_list