diff --git a/bin/ghlister b/bin/ghlister index 078d4f6..8f7f3eb 100755 --- a/bin/ghlister +++ b/bin/ghlister @@ -8,10 +8,13 @@ import argparse import configparser import logging import os +import sys + from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker from ghlister import lister, models +from ghlister.db_utils import session_scope DEFAULT_CONF = { @@ -73,6 +76,9 @@ def parse_args(): help='interval of repository IDs to list, ' + 'in N-M format; either N or M can be omitted.') + list_cli = subcli.add_parser('catchup', + help='catchup with new repos since last time') + args = cli.parse_args() return args @@ -108,3 +114,16 @@ if __name__ == '__main__': mk_session, min_id=args.interval[0], max_id=args.interval[1]) + elif args.action == 'catchup': + with session_scope(mk_session) as db_session: + last_known_id = lister.last_repo_id(db_session) + if last_known_id is not None: + logging.info('catching up from last known repo id: %d' % + last_known_id) + lister.fetch(conf, + mk_session, + min_id=last_known_id + 1, + max_id=None) + else: + logging.error('Cannot catchup: no last known id found. Abort.') + sys.exit(2) diff --git a/ghlister/lister.py b/ghlister/lister.py index 0486998..9d19189 100644 --- a/ghlister/lister.py +++ b/ghlister/lister.py @@ -12,6 +12,7 @@ import requests import time from pprint import pformat +from sqlalchemy import func from ghlister.db_utils import session_scope from ghlister.models import Repository @@ -89,6 +90,14 @@ def lookup_repo(db_session, repo_id): .first() +def last_repo_id(db_session): + t = db_session.query(func.max(Repository.id)) \ + .first() + if t is not None: + return t[0] + # else: return None + + INJECT_KEYS = ['id', 'name', 'full_name', 'html_url', 'description', 'fork']