swh.lister.gitlab: Make the lister instance parametric

This impacts:
- the credentials reading which needs to be indexed per instance
- the models since a new instance column needs to referenced and
  indexed

Related T989

swh/lister/gitlab/lister.py

@@ -2,6 +2,7 @@
 # License: GNU General Public License version 3, or any later version
 # See top-level LICENSE file for more information
 
+import random
 import re
 import time
 
@@ -21,6 +22,55 @@ class GitLabLister(SWHIndexingHttpLister):
     # (method disable_deleted_repo_tasks)
     MODEL = GitLabModel
 
+    @property
+    def ADDITIONAL_CONFIG(self):
+        """Override additional config as the 'credentials' structure change
+           between the ancestor classes and the subclass.
+
+           cf. request_params method below
+
+        """
+        return {
+            'lister_db_url':
+                ('str', 'postgresql:///lister-%s' % self.lister_name),
+            'credentials':  # credentials is a dict
+                ('dict', {}),
+            'cache_responses':
+                ('bool', False),
+            'cache_dir':
+                ('str', '~/.cache/swh/lister/%s' % self.lister_name),
+        }
+
+    def request_params(self, identifier):
+        """Get the full parameters passed to requests given the
+        transport_request identifier.
+
+        For the gitlab lister, the 'credentials' entries is configured
+        per instance. For example:
+
+        - credentials:
+          - gitlab.com:
+            - username: user0
+              password: <pass>
+            - username: user1
+              password: <pass>
+            - ...
+          - other-gitlab-instance:
+            ...
+
+        """
+        params = {
+            'headers': self.request_headers() or {}
+        }
+        # Retrieve the credentials per instance
+        creds = self.config['credentials']
+        if creds:
+            creds_lister = creds[self.lister_name]
+            auth = random.choice(creds_lister) if creds else None
+            if auth:
+                params['auth'] = (auth['username'], auth['password'])
+        return params
+
     def filter_before_inject(self, models_list):
         """We cannot filter so returns the models_list as is.
 
@@ -29,6 +79,7 @@ class GitLabLister(SWHIndexingHttpLister):
 
     def get_model_from_repo(self, repo):
         return {
+            'instance': self.lister_name,
             'uid': repo['id'],
             'indexable': repo['id'],
             'name': repo['name'],

swh/lister/gitlab/models.py

@@ -2,19 +2,31 @@
 # License: GNU General Public License version 3, or any later version
 # See top-level LICENSE file for more information
 
-from sqlalchemy import Column, Boolean, Integer
+from sqlalchemy import Column, Boolean, Integer, String
 
 from ..core.models import ModelBase
 
 
 class GitLabModel(ModelBase):
-    """a Gitlab repository"""
-    __tablename__ = 'main_gitlab_repos'
+    """a Gitlab repository from a gitlab instance
 
-    uid = Column(Integer, primary_key=True)
+    """
+    __tablename__ = 'gitlab_repo'
+
+    id = Column(Integer, primary_key=True)
+    uid = Column(Integer, index=True)
+    instance = Column(String, index=True)
     indexable = Column(Integer, index=True)
     fork = Column(Boolean)
 
-    def __init__(self, *args, **kwargs):
+    def __init__(self, uid=None, indexable=None, name=None,
+                 full_name=None, html_url=None, origin_url=None,
+                 origin_type=None, description=None, task_id=None,
+                 origin_id=None, instance=None, **kwargs):
+        super().__init__(uid=uid, indexable=indexable, name=name,
+                         full_name=full_name, html_url=html_url,
+                         origin_url=origin_url, origin_type=origin_type,
+                         description=description, task_id=task_id,
+                         origin_id=origin_id)
         self.fork = kwargs.pop('fork', False)
-        super().__init__(*args, **kwargs)
+        self.instance = instance