ghlister: add "catchup" action to continue from last time

bin/ghlister

@@ -8,10 +8,13 @@ import argparse
 import configparser
 import logging
 import os
+import sys
+
 from sqlalchemy import create_engine
 from sqlalchemy.orm import sessionmaker
 
 from ghlister import lister, models
+from ghlister.db_utils import session_scope
 
 
 DEFAULT_CONF = {
@@ -73,6 +76,9 @@ def parse_args():
                           help='interval of repository IDs to list, '
                           + 'in N-M format; either N or M can be omitted.')
 
+    list_cli = subcli.add_parser('catchup',
+                                 help='catchup with new repos since last time')
+
     args = cli.parse_args()
 
     return args
@@ -108,3 +114,16 @@ if __name__ == '__main__':
                      mk_session,
                      min_id=args.interval[0],
                      max_id=args.interval[1])
+    elif args.action == 'catchup':
+        with session_scope(mk_session) as db_session:
+            last_known_id = lister.last_repo_id(db_session)
+            if last_known_id is not None:
+                logging.info('catching up from last known repo id: %d' %
+                             last_known_id)
+                lister.fetch(conf,
+                             mk_session,
+                             min_id=last_known_id + 1,
+                             max_id=None)
+            else:
+                logging.error('Cannot catchup: no last known id found. Abort.')
+                sys.exit(2)

ghlister/lister.py

@@ -12,6 +12,7 @@ import requests
 import time
 
 from pprint import pformat
+from sqlalchemy import func
 
 from ghlister.db_utils import session_scope
 from ghlister.models import Repository
@@ -89,6 +90,14 @@ def lookup_repo(db_session, repo_id):
                      .first()
 
 
+def last_repo_id(db_session):
+    t = db_session.query(func.max(Repository.id)) \
+                  .first()
+    if t is not None:
+        return t[0]
+    # else: return None
+
+
 INJECT_KEYS = ['id', 'name', 'full_name', 'html_url', 'description', 'fork']