diff --git a/bin/batch b/bin/batch index e19c11f..9796387 100755 --- a/bin/batch +++ b/bin/batch @@ -4,26 +4,35 @@ # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information -PROXY="127.0.0.1:8118" # use Tor +export https_proxy="127.0.0.1:8118" # use Tor +export PYTHONPATH=`pwd` + +DBNAME=github +DBCONN="-p 5433" + +psql="psql $DBCONN --no-psqlrc --pset t --pset format=unaligned ${DBNAME}" BATCH_NO="$1" shift if [ -z "$BATCH_NO" ] ; then - echo "Usage: batch MILLION_NO [MIN_ID]" + echo "Usage: batch MILLION_NO [ MIN_ID | continue ]" exit 2 fi MIN_ID="$1" shift -export https_proxy=$PROXY -export PYTHONPATH=`pwd` - min_id=$[ ($BATCH_NO - 1) * 1000000 + 1 ] max_id=$[ $BATCH_NO * 1000000 ] # allow min_id override on the command line -if [ -n "$MIN_ID" ] ; then +if [ "$MIN_ID" = "continue" ] ; then + last_id=$(echo "select max(id) from repos where ${min_id} <= id and id <= ${max_id}" | $psql) + if [ "$last_id" -eq "$last_id" ] 2> /dev/null ; then # is an integer? + echo "Continuing from last known id ${last_id}" + min_id=$last_id + fi +elif [ -n "$MIN_ID" ] ; then min_id=$[ $MIN_ID > $min_id ? $MIN_ID : $min_id ] fi