reonokiy/swh-lister
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

processors: Schedule jobs for users

Browse source
This commit is contained in:
Nicolas Dandrimont 2016-03-17 17:50:27 +01:00
parent 6fbabbe586
commit 124ebf34df
1 changed files with 28 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

30
swh/lister/github/processors.py
View file

@ -25,7 +25,18 @@ def repositories(item, queue, session, credentials, storage):
'url': repos['links']['next']['url'],
})
storage_utils.update_repo_entities(storage, repos['data'])
users = {}
for repo in repos['data']:
users[repo['owner']['id']] = repo['owner']
for id, user in users.items():
jid = 'user-%d' % id
if not queue.client.jobs[jid]:
req_queue.push(queue, {
'type': 'user',
'user_login': user['login'],
'user_id': id,
}, jid=jid)
for repo in repos['data']:
if not repo['fork']:
@ -35,6 +46,8 @@ def repositories(item, queue, session, credentials, storage):
'repo_id': repo['id'],
})
storage_utils.update_repo_entities(storage, repos['data'])
def repository(item, queue, session, credentials, storage):
print('Processing repository %s (%s)' % (item['repo_name'],
@ -76,7 +89,18 @@ def forks(item, queue, session, credentials, storage):
forks = github_api.forks(item['repo_id'], item['forks_page'], session,
credentials)
storage_utils.update_repo_entities(storage, forks['data'])
users = {}
for repo in forks['data']:
users[repo['owner']['id']] = repo['owner']
for id, user in users.items():
jid = 'user-%d' % id
if not queue.client.jobs[jid]:
req_queue.push(queue, {
'type': 'user',
'user_login': user['login'],
'user_id': id,
}, jid=jid)
if item['check_next'] and 'next' in forks['links']:
req_queue.push(queue, {
@ -87,6 +111,8 @@ def forks(item, queue, session, credentials, storage):
'check_next': True,
})
storage_utils.update_repo_entities(storage, forks['data'])
def user(item, queue, session, credentials, storage):
print('Processing user %s (%s)' % (item['user_login'], item['user_id']))