reonokiy/swh-lister
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Refactor lister code

Browse source 
Streamline production of new listers by aggressively moving core
functionality into progressively inherited (A->B->C) base classes
with the transport layer abstracted.
This should make common individual forge listers straightforward to
produce with minimal customization. Github and Bitbucket listers
can be used as examples of the indexing type.
This commit is contained in:
Avi Kelman (fiendish) 2017-03-06 11:34:37 +01:00
parent a6e43f2777
commit 68d77fd43f
24 changed files with 8835 additions and 477 deletions
Download patch file Download diff file Expand all files Collapse all files

36
swh/lister/bitbucket/lister.py Normal file
View file

@ -0,0 +1,36 @@
# Copyright (C) 2017 the Software Heritage developers
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
from urllib import parse
from swh.lister.bitbucket.models import BitBucketModel
from swh.lister.core.indexing_lister import SWHIndexingHttpLister
class BitBucketLister(SWHIndexingHttpLister):
PATH_TEMPLATE = '/repositories?after=%s'
MODEL = BitBucketModel
def get_model_from_repo(self, repo):
return {
'uid': repo['uuid'],
'indexable': repo['created_on'],
'name': repo['name'],
'full_name': repo['full_name'],
'html_url': repo['links']['html']['href'],
'origin_url': repo['links']['clone'][0]['href'],
'origin_type': repo['scm'],
'description': repo['description']
}
def get_next_target_from_response(self, response):
body = response.json()
if 'next' in body:
return parse.unquote(body['next'].split('after=')[1])
else:
return None
def transport_response_simplified(self, response):
repos = response.json()['values']
return [self.get_model_from_repo(repo) for repo in repos]

15
swh/lister/bitbucket/models.py Normal file
View file

@ -0,0 +1,15 @@
# Copyright (C) 2017 the Software Heritage developers
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
from sqlalchemy import Column, String
from swh.lister.core.models import ModelBase
class BitBucketModel(ModelBase):
"""a BitBucket repository"""
__tablename__ = 'bitbucket_repos'
uid = Column(String, primary_key=True)
indexable = Column(String, index=True)

28
swh/lister/bitbucket/tasks.py Normal file
View file

@ -0,0 +1,28 @@
# Copyright (C) 2017 the Software Heritage developers
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
from swh.lister.core.tasks import (IndexingDiscoveryListerTask,
IndexingRangeListerTask,
IndexingRefreshListerTask, ListerTaskBase)
from .lister import BitBucketLister
class BitBucketListerTask(ListerTaskBase):
def new_lister(self):
return BitBucketLister(lister_name='bitbucket.com',
api_baseurl='https://api.bitbucket.org/2.0')
class IncrementalBitBucketLister(BitBucketListerTask,
IndexingDiscoveryListerTask):
task_queue = 'swh_lister_bitbucket_discover'
class RangeBitBucketLister(BitBucketListerTask, IndexingRangeListerTask):
task_queue = 'swh_lister_bitbucket_refresh'
class FullBitBucketRelister(BitBucketListerTask, IndexingRefreshListerTask):
task_queue = 'swh_lister_bitbucket_refresh'

4
swh/lister/bitbucket/tests/api_empty_response.json Normal file
View file

@ -0,0 +1,4 @@
{
"pagelen": 10,
"values": []
}

806
swh/lister/bitbucket/tests/api_response.json Normal file
View file

@ -0,0 +1,806 @@
{
"pagelen": 10,
"values": [
{
"scm": "hg",
"website": "",
"has_wiki": true,
"name": "app-template",
"links": {
"watchers": {
"href": "https://api.bitbucket.org/2.0/repositories/bebac/app-template/watchers"
},
"branches": {
"href": "https://api.bitbucket.org/2.0/repositories/bebac/app-template/refs/branches"
},
"tags": {
"href": "https://api.bitbucket.org/2.0/repositories/bebac/app-template/refs/tags"
},
"commits": {
"href": "https://api.bitbucket.org/2.0/repositories/bebac/app-template/commits"
},
"clone": [
{
"href": "https://bitbucket.org/bebac/app-template",
"name": "https"
},
{
"href": "ssh://hg@bitbucket.org/bebac/app-template",
"name": "ssh"
}
],
"self": {
"href": "https://api.bitbucket.org/2.0/repositories/bebac/app-template"
},
"html": {
"href": "https://bitbucket.org/bebac/app-template"
},
"avatar": {
"href": "https://bitbucket.org/bebac/app-template/avatar/32/"
},
"hooks": {
"href": "https://api.bitbucket.org/2.0/repositories/bebac/app-template/hooks"
},
"forks": {
"href": "https://api.bitbucket.org/2.0/repositories/bebac/app-template/forks"
},
"downloads": {
"href": "https://api.bitbucket.org/2.0/repositories/bebac/app-template/downloads"
},
"pullrequests": {
"href": "https://api.bitbucket.org/2.0/repositories/bebac/app-template/pullrequests"
}
},
"fork_policy": "allow_forks",
"uuid": "{0cf80a6e-e91f-4a4c-a61b-8c8ff51ca3ec}",
"language": "c++",
"created_on": "2008-07-12T07:44:01.476818+00:00",
"full_name": "bebac/app-template",
"has_issues": true,
"owner": {
"username": "bebac",
"display_name": "Benny Bach",
"type": "user",
"uuid": "{d1a83a2a-be1b-4034-8c1d-386a6690cddb}",
"links": {
"self": {
"href": "https://api.bitbucket.org/2.0/users/bebac"
},
"html": {
"href": "https://bitbucket.org/bebac/"
},
"avatar": {
"href": "https://bitbucket.org/account/bebac/avatar/32/"
}
}
},
"updated_on": "2011-10-05T15:36:19.409008+00:00",
"size": 71548,
"type": "repository",
"slug": "app-template",
"is_private": false,
"description": "Basic files and directory structure for a C++ project. Intended as a starting point for a new project. Includes a basic cross platform core library."
},
{
"scm": "hg",
"website": "",
"has_wiki": true,
"name": "mercurialeclipse",
"links": {
"watchers": {
"href": "https://api.bitbucket.org/2.0/repositories/bastiand/mercurialeclipse/watchers"
},
"branches": {
"href": "https://api.bitbucket.org/2.0/repositories/bastiand/mercurialeclipse/refs/branches"
},
"tags": {
"href": "https://api.bitbucket.org/2.0/repositories/bastiand/mercurialeclipse/refs/tags"
},
"commits": {
"href": "https://api.bitbucket.org/2.0/repositories/bastiand/mercurialeclipse/commits"
},
"clone": [
{
"href": "https://bitbucket.org/bastiand/mercurialeclipse",
"name": "https"
},
{
"href": "ssh://hg@bitbucket.org/bastiand/mercurialeclipse",
"name": "ssh"
}
],
"self": {
"href": "https://api.bitbucket.org/2.0/repositories/bastiand/mercurialeclipse"
},
"html": {
"href": "https://bitbucket.org/bastiand/mercurialeclipse"
},
"avatar": {
"href": "https://bitbucket.org/bastiand/mercurialeclipse/avatar/32/"
},
"hooks": {
"href": "https://api.bitbucket.org/2.0/repositories/bastiand/mercurialeclipse/hooks"
},
"forks": {
"href": "https://api.bitbucket.org/2.0/repositories/bastiand/mercurialeclipse/forks"
},
"downloads": {
"href": "https://api.bitbucket.org/2.0/repositories/bastiand/mercurialeclipse/downloads"
},
"pullrequests": {
"href": "https://api.bitbucket.org/2.0/repositories/bastiand/mercurialeclipse/pullrequests"
}
},
"fork_policy": "allow_forks",
"uuid": "{f7a08670-bdd1-4465-aa97-7a5ce8d1a25b}",
"language": "",
"created_on": "2008-07-12T09:37:06.254721+00:00",
"full_name": "bastiand/mercurialeclipse",
"has_issues": false,
"owner": {
"username": "bastiand",
"display_name": "Bastian Doetsch",
"type": "user",
"uuid": "{3742cd48-adad-4205-ab0d-04fc992a1728}",
"links": {
"self": {
"href": "https://api.bitbucket.org/2.0/users/bastiand"
},
"html": {
"href": "https://bitbucket.org/bastiand/"
},
"avatar": {
"href": "https://bitbucket.org/account/bastiand/avatar/32/"
}
}
},
"updated_on": "2011-09-17T02:36:59.062596+00:00",
"size": 6445145,
"type": "repository",
"slug": "mercurialeclipse",
"is_private": false,
"description": "my own repo for MercurialEclipse."
},
{
"scm": "hg",
"website": "",
"has_wiki": true,
"name": "sandboxpublic",
"links": {
"watchers": {
"href": "https://api.bitbucket.org/2.0/repositories/aleax/sandboxpublic/watchers"
},
"branches": {
"href": "https://api.bitbucket.org/2.0/repositories/aleax/sandboxpublic/refs/branches"
},
"tags": {
"href": "https://api.bitbucket.org/2.0/repositories/aleax/sandboxpublic/refs/tags"
},
"commits": {
"href": "https://api.bitbucket.org/2.0/repositories/aleax/sandboxpublic/commits"
},
"clone": [
{
"href": "https://bitbucket.org/aleax/sandboxpublic",
"name": "https"
},
{
"href": "ssh://hg@bitbucket.org/aleax/sandboxpublic",
"name": "ssh"
}
],
"self": {
"href": "https://api.bitbucket.org/2.0/repositories/aleax/sandboxpublic"
},
"html": {
"href": "https://bitbucket.org/aleax/sandboxpublic"
},
"avatar": {
"href": "https://bitbucket.org/aleax/sandboxpublic/avatar/32/"
},
"hooks": {
"href": "https://api.bitbucket.org/2.0/repositories/aleax/sandboxpublic/hooks"
},
"forks": {
"href": "https://api.bitbucket.org/2.0/repositories/aleax/sandboxpublic/forks"
},
"downloads": {
"href": "https://api.bitbucket.org/2.0/repositories/aleax/sandboxpublic/downloads"
},
"pullrequests": {
"href": "https://api.bitbucket.org/2.0/repositories/aleax/sandboxpublic/pullrequests"
}
},
"fork_policy": "allow_forks",
"uuid": "{452c716c-a1ce-42bc-a95b-d38da49cbb37}",
"language": "",
"created_on": "2008-07-14T01:59:23.568048+00:00",
"full_name": "aleax/sandboxpublic",
"has_issues": true,
"owner": {
"username": "aleax",
"display_name": "Alex Martelli",
"type": "user",
"uuid": "{1155d94d-fb48-43fe-a431-ec07c900b636}",
"links": {
"self": {
"href": "https://api.bitbucket.org/2.0/users/aleax"
},
"html": {
"href": "https://bitbucket.org/aleax/"
},
"avatar": {
"href": "https://bitbucket.org/account/aleax/avatar/32/"
}
}
},
"updated_on": "2012-06-22T21:55:28.753727+00:00",
"size": 3120,
"type": "repository",
"slug": "sandboxpublic",
"is_private": false,
"description": "to help debug ACLs for private vs public bitbucket repos"
},
{
"scm": "hg",
"website": "",
"has_wiki": true,
"name": "otrsfix-ng",
"links": {
"watchers": {
"href": "https://api.bitbucket.org/2.0/repositories/adiakin/otrsfix-ng/watchers"
},
"branches": {
"href": "https://api.bitbucket.org/2.0/repositories/adiakin/otrsfix-ng/refs/branches"
},
"tags": {
"href": "https://api.bitbucket.org/2.0/repositories/adiakin/otrsfix-ng/refs/tags"
},
"commits": {
"href": "https://api.bitbucket.org/2.0/repositories/adiakin/otrsfix-ng/commits"
},
"clone": [
{
"href": "https://bitbucket.org/adiakin/otrsfix-ng",
"name": "https"
},
{
"href": "ssh://hg@bitbucket.org/adiakin/otrsfix-ng",
"name": "ssh"
}
],
"self": {
"href": "https://api.bitbucket.org/2.0/repositories/adiakin/otrsfix-ng"
},
"html": {
"href": "https://bitbucket.org/adiakin/otrsfix-ng"
},
"avatar": {
"href": "https://bitbucket.org/adiakin/otrsfix-ng/avatar/32/"
},
"hooks": {
"href": "https://api.bitbucket.org/2.0/repositories/adiakin/otrsfix-ng/hooks"
},
"forks": {
"href": "https://api.bitbucket.org/2.0/repositories/adiakin/otrsfix-ng/forks"
},
"downloads": {
"href": "https://api.bitbucket.org/2.0/repositories/adiakin/otrsfix-ng/downloads"
},
"pullrequests": {
"href": "https://api.bitbucket.org/2.0/repositories/adiakin/otrsfix-ng/pullrequests"
}
},
"fork_policy": "allow_forks",
"uuid": "{05b1b9dc-a7b6-46d6-ae1b-e66a17aa4f55}",
"language": "",
"created_on": "2008-07-15T06:14:39.306314+00:00",
"full_name": "adiakin/otrsfix-ng",
"has_issues": true,
"owner": {
"username": "adiakin",
"display_name": "adiakin",
"type": "user",
"uuid": "{414012b5-1ac9-4096-9f46-8893cfa3cda5}",
"links": {
"self": {
"href": "https://api.bitbucket.org/2.0/users/adiakin"
},
"html": {
"href": "https://bitbucket.org/adiakin/"
},
"avatar": {
"href": "https://bitbucket.org/account/adiakin/avatar/32/"
}
}
},
"updated_on": "2016-06-02T18:56:34.868302+00:00",
"size": 211631,
"type": "repository",
"slug": "otrsfix-ng",
"is_private": false,
"description": "OTRS greasemonkey extension"
},
{
"scm": "hg",
"website": "",
"has_wiki": true,
"name": "pida-pypaned",
"links": {
"watchers": {
"href": "https://api.bitbucket.org/2.0/repositories/aafshar/pida-pypaned/watchers"
},
"branches": {
"href": "https://api.bitbucket.org/2.0/repositories/aafshar/pida-pypaned/refs/branches"
},
"tags": {
"href": "https://api.bitbucket.org/2.0/repositories/aafshar/pida-pypaned/refs/tags"
},
"commits": {
"href": "https://api.bitbucket.org/2.0/repositories/aafshar/pida-pypaned/commits"
},
"clone": [
{
"href": "https://bitbucket.org/aafshar/pida-pypaned",
"name": "https"
},
{
"href": "ssh://hg@bitbucket.org/aafshar/pida-pypaned",
"name": "ssh"
}
],
"self": {
"href": "https://api.bitbucket.org/2.0/repositories/aafshar/pida-pypaned"
},
"html": {
"href": "https://bitbucket.org/aafshar/pida-pypaned"
},
"avatar": {
"href": "https://bitbucket.org/aafshar/pida-pypaned/avatar/32/"
},
"hooks": {
"href": "https://api.bitbucket.org/2.0/repositories/aafshar/pida-pypaned/hooks"
},
"forks": {
"href": "https://api.bitbucket.org/2.0/repositories/aafshar/pida-pypaned/forks"
},
"downloads": {
"href": "https://api.bitbucket.org/2.0/repositories/aafshar/pida-pypaned/downloads"
},
"pullrequests": {
"href": "https://api.bitbucket.org/2.0/repositories/aafshar/pida-pypaned/pullrequests"
}
},
"fork_policy": "allow_forks",
"uuid": "{94cb830a-1784-4e51-9791-8f5cc93990a9}",
"language": "",
"created_on": "2008-07-16T22:47:38.682491+00:00",
"full_name": "aafshar/pida-pypaned",
"has_issues": true,
"owner": {
"username": "aafshar",
"display_name": "Ali Afshar",
"type": "user",
"uuid": "{bcb87110-6a92-41fc-b95c-680feeea5512}",
"links": {
"self": {
"href": "https://api.bitbucket.org/2.0/users/aafshar"
},
"html": {
"href": "https://bitbucket.org/aafshar/"
},
"avatar": {
"href": "https://bitbucket.org/account/aafshar/avatar/32/"
}
}
},
"updated_on": "2012-06-22T21:55:42.451431+00:00",
"size": 4680652,
"type": "repository",
"slug": "pida-pypaned",
"is_private": false,
"description": ""
},
{
"scm": "hg",
"website": "",
"has_wiki": true,
"name": "TLOMM-testing",
"links": {
"watchers": {
"href": "https://api.bitbucket.org/2.0/repositories/tgrimley/tlomm-testing/watchers"
},
"branches": {
"href": "https://api.bitbucket.org/2.0/repositories/tgrimley/tlomm-testing/refs/branches"
},
"tags": {
"href": "https://api.bitbucket.org/2.0/repositories/tgrimley/tlomm-testing/refs/tags"
},
"commits": {
"href": "https://api.bitbucket.org/2.0/repositories/tgrimley/tlomm-testing/commits"
},
"clone": [
{
"href": "https://bitbucket.org/tgrimley/tlomm-testing",
"name": "https"
},
{
"href": "ssh://hg@bitbucket.org/tgrimley/tlomm-testing",
"name": "ssh"
}
],
"self": {
"href": "https://api.bitbucket.org/2.0/repositories/tgrimley/tlomm-testing"
},
"html": {
"href": "https://bitbucket.org/tgrimley/tlomm-testing"
},
"avatar": {
"href": "https://bitbucket.org/tgrimley/tlomm-testing/avatar/32/"
},
"hooks": {
"href": "https://api.bitbucket.org/2.0/repositories/tgrimley/tlomm-testing/hooks"
},
"forks": {
"href": "https://api.bitbucket.org/2.0/repositories/tgrimley/tlomm-testing/forks"
},
"downloads": {
"href": "https://api.bitbucket.org/2.0/repositories/tgrimley/tlomm-testing/downloads"
},
"pullrequests": {
"href": "https://api.bitbucket.org/2.0/repositories/tgrimley/tlomm-testing/pullrequests"
}
},
"fork_policy": "allow_forks",
"uuid": "{95283ca1-f77e-40d6-b3ed-5bfae6ed2d15}",
"language": "",
"created_on": "2008-07-18T21:05:17.750587+00:00",
"full_name": "tgrimley/tlomm-testing",
"has_issues": true,
"owner": {
"username": "tgrimley",
"display_name": "Thomas Grimley",
"type": "user",
"uuid": "{c958a08f-4669-4c77-81ec-4e2faa8ebf35}",
"links": {
"self": {
"href": "https://api.bitbucket.org/2.0/users/tgrimley"
},
"html": {
"href": "https://bitbucket.org/tgrimley/"
},
"avatar": {
"href": "https://bitbucket.org/account/tgrimley/avatar/32/"
}
}
},
"updated_on": "2012-06-22T21:55:46.627825+00:00",
"size": 3128,
"type": "repository",
"slug": "tlomm-testing",
"is_private": false,
"description": "File related to testing functionality of TLOMM->TLOTTS transition"
},
{
"scm": "hg",
"website": "",
"has_wiki": true,
"name": "test",
"links": {
"watchers": {
"href": "https://api.bitbucket.org/2.0/repositories/tingle/test/watchers"
},
"branches": {
"href": "https://api.bitbucket.org/2.0/repositories/tingle/test/refs/branches"
},
"tags": {
"href": "https://api.bitbucket.org/2.0/repositories/tingle/test/refs/tags"
},
"commits": {
"href": "https://api.bitbucket.org/2.0/repositories/tingle/test/commits"
},
"clone": [
{
"href": "https://bitbucket.org/tingle/test",
"name": "https"
},
{
"href": "ssh://hg@bitbucket.org/tingle/test",
"name": "ssh"
}
],
"self": {
"href": "https://api.bitbucket.org/2.0/repositories/tingle/test"
},
"html": {
"href": "https://bitbucket.org/tingle/test"
},
"avatar": {
"href": "https://bitbucket.org/tingle/test/avatar/32/"
},
"hooks": {
"href": "https://api.bitbucket.org/2.0/repositories/tingle/test/hooks"
},
"forks": {
"href": "https://api.bitbucket.org/2.0/repositories/tingle/test/forks"
},
"downloads": {
"href": "https://api.bitbucket.org/2.0/repositories/tingle/test/downloads"
},
"pullrequests": {
"href": "https://api.bitbucket.org/2.0/repositories/tingle/test/pullrequests"
}
},
"fork_policy": "allow_forks",
"uuid": "{457953ec-fe87-41b9-b659-94756fb40ece}",
"language": "",
"created_on": "2008-07-18T22:24:31.984981+00:00",
"full_name": "tingle/test",
"has_issues": true,
"owner": {
"username": "tingle",
"display_name": "tingle",
"type": "user",
"uuid": "{dddce42b-bd19-417b-90ff-72cdbfb6eb7d}",
"links": {
"self": {
"href": "https://api.bitbucket.org/2.0/users/tingle"
},
"html": {
"href": "https://bitbucket.org/tingle/"
},
"avatar": {
"href": "https://bitbucket.org/account/tingle/avatar/32/"
}
}
},
"updated_on": "2012-06-22T21:55:49.860564+00:00",
"size": 3090,
"type": "repository",
"slug": "test",
"is_private": false,
"description": ""
},
{
"scm": "hg",
"website": "http://shaze.myopenid.com/",
"has_wiki": true,
"name": "Repository",
"links": {
"watchers": {
"href": "https://api.bitbucket.org/2.0/repositories/Shaze/repository/watchers"
},
"branches": {
"href": "https://api.bitbucket.org/2.0/repositories/Shaze/repository/refs/branches"
},
"tags": {
"href": "https://api.bitbucket.org/2.0/repositories/Shaze/repository/refs/tags"
},
"commits": {
"href": "https://api.bitbucket.org/2.0/repositories/Shaze/repository/commits"
},
"clone": [
{
"href": "https://bitbucket.org/Shaze/repository",
"name": "https"
},
{
"href": "ssh://hg@bitbucket.org/Shaze/repository",
"name": "ssh"
}
],
"self": {
"href": "https://api.bitbucket.org/2.0/repositories/Shaze/repository"
},
"html": {
"href": "https://bitbucket.org/Shaze/repository"
},
"avatar": {
"href": "https://bitbucket.org/Shaze/repository/avatar/32/"
},
"hooks": {
"href": "https://api.bitbucket.org/2.0/repositories/Shaze/repository/hooks"
},
"forks": {
"href": "https://api.bitbucket.org/2.0/repositories/Shaze/repository/forks"
},
"downloads": {
"href": "https://api.bitbucket.org/2.0/repositories/Shaze/repository/downloads"
},
"pullrequests": {
"href": "https://api.bitbucket.org/2.0/repositories/Shaze/repository/pullrequests"
}
},
"fork_policy": "allow_forks",
"uuid": "{3c0b8076-caef-465a-8d08-a184459f659b}",
"language": "",
"created_on": "2008-07-18T22:39:51.380134+00:00",
"full_name": "Shaze/repository",
"has_issues": true,
"owner": {
"username": "Shaze",
"display_name": "Shaze",
"type": "user",
"uuid": "{f57817e9-bfe4-4c65-84dd-662152430323}",
"links": {
"self": {
"href": "https://api.bitbucket.org/2.0/users/Shaze"
},
"html": {
"href": "https://bitbucket.org/Shaze/"
},
"avatar": {
"href": "https://bitbucket.org/account/Shaze/avatar/32/"
}
}
},
"updated_on": "2012-06-22T21:55:51.570502+00:00",
"size": 3052,
"type": "repository",
"slug": "repository",
"is_private": false,
"description": "Mine, all mine!"
},
{
"scm": "hg",
"website": "http://bitbucket.org/copiesofcopies/identifox/",
"has_wiki": true,
"name": "identifox",
"links": {
"watchers": {
"href": "https://api.bitbucket.org/2.0/repositories/uncryptic/identifox/watchers"
},
"branches": {
"href": "https://api.bitbucket.org/2.0/repositories/uncryptic/identifox/refs/branches"
},
"tags": {
"href": "https://api.bitbucket.org/2.0/repositories/uncryptic/identifox/refs/tags"
},
"commits": {
"href": "https://api.bitbucket.org/2.0/repositories/uncryptic/identifox/commits"
},
"clone": [
{
"href": "https://bitbucket.org/uncryptic/identifox",
"name": "https"
},
{
"href": "ssh://hg@bitbucket.org/uncryptic/identifox",
"name": "ssh"
}
],
"self": {
"href": "https://api.bitbucket.org/2.0/repositories/uncryptic/identifox"
},
"html": {
"href": "https://bitbucket.org/uncryptic/identifox"
},
"avatar": {
"href": "https://bitbucket.org/uncryptic/identifox/avatar/32/"
},
"hooks": {
"href": "https://api.bitbucket.org/2.0/repositories/uncryptic/identifox/hooks"
},
"forks": {
"href": "https://api.bitbucket.org/2.0/repositories/uncryptic/identifox/forks"
},
"downloads": {
"href": "https://api.bitbucket.org/2.0/repositories/uncryptic/identifox/downloads"
},
"pullrequests": {
"href": "https://api.bitbucket.org/2.0/repositories/uncryptic/identifox/pullrequests"
}
},
"fork_policy": "allow_forks",
"uuid": "{78a1a080-a77e-4d0d-823a-b107484477a8}",
"language": "",
"created_on": "2008-07-19T00:33:14.065946+00:00",
"full_name": "uncryptic/identifox",
"has_issues": true,
"owner": {
"username": "uncryptic",
"display_name": "Uncryptic Communications",
"type": "user",
"uuid": "{db87bb9a-9980-4840-bd4a-61f7748a56b4}",
"links": {
"self": {
"href": "https://api.bitbucket.org/2.0/users/uncryptic"
},
"html": {
"href": "https://bitbucket.org/uncryptic/"
},
"avatar": {
"href": "https://bitbucket.org/account/uncryptic/avatar/32/"
}
}
},
"updated_on": "2008-07-19T00:33:14+00:00",
"size": 1918,
"type": "repository",
"slug": "identifox",
"is_private": false,
"description": "TwitterFox, modified to work with Identi.ca, including cosmetic and subtle code changes. For the most part, the code is nearly identical to the TwitterFox base: http://www.naan.net/trac/wiki/TwitterFox"
},
{
"scm": "hg",
"website": "http://rforce.rubyforge.org",
"has_wiki": false,
"name": "rforce",
"links": {
"watchers": {
"href": "https://api.bitbucket.org/2.0/repositories/undees/rforce/watchers"
},
"branches": {
"href": "https://api.bitbucket.org/2.0/repositories/undees/rforce/refs/branches"
},
"tags": {
"href": "https://api.bitbucket.org/2.0/repositories/undees/rforce/refs/tags"
},
"commits": {
"href": "https://api.bitbucket.org/2.0/repositories/undees/rforce/commits"
},
"clone": [
{
"href": "https://bitbucket.org/undees/rforce",
"name": "https"
},
{
"href": "ssh://hg@bitbucket.org/undees/rforce",
"name": "ssh"
}
],
"self": {
"href": "https://api.bitbucket.org/2.0/repositories/undees/rforce"
},
"html": {
"href": "https://bitbucket.org/undees/rforce"
},
"avatar": {
"href": "https://bitbucket.org/undees/rforce/avatar/32/"
},
"hooks": {
"href": "https://api.bitbucket.org/2.0/repositories/undees/rforce/hooks"
},
"forks": {
"href": "https://api.bitbucket.org/2.0/repositories/undees/rforce/forks"
},
"downloads": {
"href": "https://api.bitbucket.org/2.0/repositories/undees/rforce/downloads"
},
"pullrequests": {
"href": "https://api.bitbucket.org/2.0/repositories/undees/rforce/pullrequests"
}
},
"fork_policy": "allow_forks",
"uuid": "{ec2ffee7-bfcd-4e95-83c8-22ac31e69fa3}",
"language": "",
"created_on": "2008-07-19T06:16:43.044743+00:00",
"full_name": "undees/rforce",
"has_issues": false,
"owner": {
"username": "undees",
"display_name": "Ian Dees",
"type": "user",
"uuid": "{6ff66a34-6412-4f28-bf57-707a2a5c6d7b}",
"links": {
"self": {
"href": "https://api.bitbucket.org/2.0/users/undees"
},
"html": {
"href": "https://bitbucket.org/undees/"
},
"avatar": {
"href": "https://bitbucket.org/account/undees/avatar/32/"
}
}
},
"updated_on": "2015-02-09T00:48:15.408680+00:00",
"size": 338402,
"type": "repository",
"slug": "rforce",
"is_private": false,
"description": "A simple, usable binding to the SalesForce API."
}
],
"next": "https://api.bitbucket.org/2.0/repositories?after=2008-07-19T19%3A53%3A07.031845%2B00%3A00"
}

20
swh/lister/bitbucket/tests/test_bb_lister.py Normal file
View file

@ -0,0 +1,20 @@
# Copyright (C) 2017 the Software Heritage developers
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
import re
import unittest
from swh.lister.bitbucket.lister import BitBucketLister
from swh.lister.core.tests.test_lister import IndexingHttpListerTesterBase
class BitBucketListerTester(IndexingHttpListerTesterBase, unittest.TestCase):
Lister = BitBucketLister
test_re = re.compile(r'/repositories\?after=([^?&]+)')
lister_subdir = 'bitbucket'
good_api_response_file = 'api_response.json'
bad_api_response_file = 'api_empty_response.json'
first_index = '2008-07-12T07:44:01.476818+00:00'
last_index = '2008-07-19T06:16:43.044743+00:00'
entries_per_page = 10