bitbucket: Allow to specify the number of repos per api request

This is independent but still, it somehow fixes the issue occurring on T1826.

Related T1826

swh/lister/bitbucket/lister.py

@@ -15,12 +15,22 @@ from swh.lister.core.indexing_lister import IndexingHttpLister
 logger = logging.getLogger(__name__)
 
 
+DEFAULT_BITBUCKET_PAGE = 10
+
+
 class BitBucketLister(IndexingHttpLister):
     PATH_TEMPLATE = '/repositories?after=%s'
     MODEL = BitBucketModel
     LISTER_NAME = 'bitbucket'
     instance = 'bitbucket'
 
+    def __init__(self, api_baseurl, override_config=None, per_page=100):
+        super().__init__(
+            api_baseurl=api_baseurl, override_config=override_config)
+        if per_page != DEFAULT_BITBUCKET_PAGE:
+            self.PATH_TEMPLATE = '%s&pagelen=%s' % (
+                self.PATH_TEMPLATE, per_page)
+
     def get_model_from_repo(self, repo):
         return {
             'uid': repo['uuid'],
@@ -76,11 +86,10 @@ class BitBucketLister(IndexingHttpLister):
             else:
                 ret = lower <= inner <= upper
         except Exception as e:
-            logger.error(str(e) + ': %s, %s, %s' %
-                         (('inner=%s%s' % (type(inner), inner)),
-                          ('lower=%s%s' % (type(lower), lower)),
-                          ('upper=%s%s' % (type(upper), upper)))
-                         )
+            logger.error(str(e) + ': %s, %s, %s',
+                         ('inner=%s%s' % (type(inner), inner)),
+                         ('lower=%s%s' % (type(lower), lower)),
+                         ('upper=%s%s' % (type(upper), upper)))
             raise
 
         return ret

swh/lister/bitbucket/tasks.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2017-2018 the Software Heritage developers
+# Copyright (C) 2017-2019 the Software Heritage developers
 # License: GNU General Public License version 3, or any later version
 # See top-level LICENSE file for more information
 
@@ -12,8 +12,8 @@ from .lister import BitBucketLister
 GROUP_SPLIT = 10000
 
 
-def new_lister(api_baseurl='https://api.bitbucket.org/2.0'):
-    return BitBucketLister(api_baseurl=api_baseurl)
+def new_lister(api_baseurl='https://api.bitbucket.org/2.0', per_page=100):
+    return BitBucketLister(api_baseurl=api_baseurl, per_page=per_page)
 
 
 @app.task(name=__name__ + '.IncrementalBitBucketLister')

swh/lister/bitbucket/tests/test_tasks.py

@@ -26,7 +26,8 @@ def test_incremental(lister, swh_app, celery_session_worker):
     res.wait()
     assert res.successful()
 
-    lister.assert_called_once_with(api_baseurl='https://api.bitbucket.org/2.0')
+    lister.assert_called_once_with(
+        api_baseurl='https://api.bitbucket.org/2.0', per_page=100)
     lister.db_last_index.assert_called_once_with()
     lister.run.assert_called_once_with(min_bound=42, max_bound=None)
 
@@ -44,7 +45,8 @@ def test_range(lister, swh_app, celery_session_worker):
     res.wait()
     assert res.successful()
 
-    lister.assert_called_once_with(api_baseurl='https://api.bitbucket.org/2.0')
+    lister.assert_called_once_with(
+        api_baseurl='https://api.bitbucket.org/2.0', per_page=100)
     lister.db_last_index.assert_not_called()
     lister.run.assert_called_once_with(min_bound=12, max_bound=42)
 
@@ -74,7 +76,8 @@ def test_relister(lister, swh_app, celery_session_worker):
             break
         sleep(1)
 
-    lister.assert_called_with(api_baseurl='https://api.bitbucket.org/2.0')
+    lister.assert_called_with(
+        api_baseurl='https://api.bitbucket.org/2.0', per_page=100)
 
     # one by the FullBitbucketRelister task
     # + 5 for the RangeBitbucketLister subtasks