From 124ebf34df48e65e68e6c1a43cb5eeb744215c3d Mon Sep 17 00:00:00 2001
From: Nicolas Dandrimont <nicolas@dandrimont.eu>
Date: Thu, 17 Mar 2016 17:50:27 +0100
Subject: [PATCH] processors: Schedule jobs for users

---
 swh/lister/github/processors.py | 30 ++++++++++++++++++++++++++++--
 1 file changed, 28 insertions(+), 2 deletions(-)

diff --git a/swh/lister/github/processors.py b/swh/lister/github/processors.py
index 4ab6abc..84e3a7d 100644
--- a/swh/lister/github/processors.py
+++ b/swh/lister/github/processors.py
@@ -25,7 +25,18 @@ def repositories(item, queue, session, credentials, storage):
             'url': repos['links']['next']['url'],
         })
 
-    storage_utils.update_repo_entities(storage, repos['data'])
+    users = {}
+    for repo in repos['data']:
+        users[repo['owner']['id']] = repo['owner']
+
+    for id, user in users.items():
+        jid = 'user-%d' % id
+        if not queue.client.jobs[jid]:
+            req_queue.push(queue, {
+                'type': 'user',
+                'user_login': user['login'],
+                'user_id': id,
+            }, jid=jid)
 
     for repo in repos['data']:
         if not repo['fork']:
@@ -35,6 +46,8 @@ def repositories(item, queue, session, credentials, storage):
                 'repo_id': repo['id'],
             })
 
+    storage_utils.update_repo_entities(storage, repos['data'])
+
 
 def repository(item, queue, session, credentials, storage):
     print('Processing repository %s (%s)' % (item['repo_name'],
@@ -76,7 +89,18 @@ def forks(item, queue, session, credentials, storage):
     forks = github_api.forks(item['repo_id'], item['forks_page'], session,
                              credentials)
 
-    storage_utils.update_repo_entities(storage, forks['data'])
+    users = {}
+    for repo in forks['data']:
+        users[repo['owner']['id']] = repo['owner']
+
+    for id, user in users.items():
+        jid = 'user-%d' % id
+        if not queue.client.jobs[jid]:
+            req_queue.push(queue, {
+                'type': 'user',
+                'user_login': user['login'],
+                'user_id': id,
+            }, jid=jid)
 
     if item['check_next'] and 'next' in forks['links']:
         req_queue.push(queue, {
@@ -87,6 +111,8 @@ def forks(item, queue, session, credentials, storage):
             'check_next': True,
         })
 
+    storage_utils.update_repo_entities(storage, forks['data'])
+
 
 def user(item, queue, session, credentials, storage):
     print('Processing user %s (%s)' % (item['user_login'], item['user_id']))