reonokiy/swh-lister
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

swh.lister.github: Refactor to use swh.storage instead of sqlalchemy

Browse source
This commit is contained in:
Nicolas Dandrimont 2016-03-09 19:03:35 +01:00
parent f13764ba36
commit 533f6fa1a3
20 changed files with 600 additions and 694 deletions
Download patch file Download diff file Expand all files Collapse all files

41
bin/batch
View file

@ -1,41 +0,0 @@
#!/bin/bash
# Copyright (C) 2015 Stefano Zacchiroli <zack@upsilon.cc>
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
export https_proxy="127.0.0.1:8118" # use Tor
export PYTHONPATH=`pwd`
DBNAME=github
DBCONN="-p 5433"
psql="psql $DBCONN --no-psqlrc --pset t --pset format=unaligned ${DBNAME}"
BATCH_NO="$1"
shift
if [ -z "$BATCH_NO" ] ; then
echo "Usage: batch MILLION_NO [ MIN_ID | continue ]"
exit 2
fi
MIN_ID="$1"
shift
min_id=$[ ($BATCH_NO - 1) * 1000000 + 1 ]
max_id=$[ $BATCH_NO * 1000000 ]
# allow min_id override on the command line
if [ "$MIN_ID" = "continue" ] ; then
last_id=$(echo "select max(id) from repos where ${min_id} <= id and id <= ${max_id}" | $psql)
if [ "$last_id" -eq "$last_id" ] 2> /dev/null ; then # is an integer?
echo "Continuing from last known id ${last_id}"
min_id=$last_id
fi
elif [ -n "$MIN_ID" ] ; then
min_id=$[ $MIN_ID > $min_id ? $MIN_ID : $min_id ]
fi
cmd="bin/ghlister list ${min_id}-${max_id}"
echo Running $cmd ...
$cmd