reonokiy/swh-lister
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

bin/batch: add support for continuing from last id

Browse source
This commit is contained in:
Stefano Zacchiroli 2015-04-28 10:31:30 +02:00
parent 99f2c06c95
commit a65ab0cf2a
1 changed files with 15 additions and 6 deletions
Download patch file Download diff file Expand all files Collapse all files

21
bin/batch
View file

@ -4,26 +4,35 @@
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
PROXY="127.0.0.1:8118" # use Tor
export https_proxy="127.0.0.1:8118" # use Tor
export PYTHONPATH=`pwd`
DBNAME=github
DBCONN="-p 5433"
psql="psql $DBCONN --no-psqlrc --pset t --pset format=unaligned ${DBNAME}"
BATCH_NO="$1"
shift
if [ -z "$BATCH_NO" ] ; then
echo "Usage: batch MILLION_NO [MIN_ID]"
echo "Usage: batch MILLION_NO [ MIN_ID | continue ]"
exit 2
fi
MIN_ID="$1"
shift
export https_proxy=$PROXY
export PYTHONPATH=`pwd`
min_id=$[ ($BATCH_NO - 1) * 1000000 + 1 ]
max_id=$[ $BATCH_NO * 1000000 ]
# allow min_id override on the command line
if [ -n "$MIN_ID" ] ; then
if [ "$MIN_ID" = "continue" ] ; then
last_id=$(echo "select max(id) from repos where ${min_id} <= id and id <= ${max_id}" | $psql)
if [ "$last_id" -eq "$last_id" ] 2> /dev/null ; then # is an integer?
echo "Continuing from last known id ${last_id}"
min_id=$last_id
fi
elif [ -n "$MIN_ID" ] ; then
min_id=$[ $MIN_ID > $min_id ? $MIN_ID : $min_id ]
fi