updated testnet launcher helper scripts to manage restarting nodes as needed

scripts/tn_bounce.sh

 #!/bin/bash
-pkill -9 eosd
-sleep 3
-now=`date +'%Y_%m_%d_%H_%M_%S'`
-DD=`ls -d tn_data_??`
-log=$DD/stderr.$now.txt
-nohup programs/eosd/eosd --resync --data-dir $DD > $DD/stdout.txt  2> $log &
-echo $! > $DD/eosd.pid
-rm $DD/stderr.txt
-ln -s stderr.$now.txt $DD/stderr.txt
+#
+# tn_bounce is used to restart a node that is acting badly or is down.
+# usage: tn_bounce.sh [arglist]
+# arglist will be passed to the node's command line. First with no modifiers
+# then with --replay and then a third time with --resync
+#
+# the data directory and log file are set by this script. Do not pass them on
+# the command line.
+#
+# in most cases, simply running ./tn_bounce.sh is sufficient.
+#
+
+cd $EOSIO_HOME
+
+prog=eosd
+if [ ! \( -f programs/$prog/$prog \) ]; then
+    prog=eosiod
+fi
+
+if [ -z "$EOSIO_TN_NODE" ]; then
+    DD=`ls -d tn_data_??`
+    ddcount=`echo $DD | wc -w`
+    if [ $ddcount -ne 1 ]; then
+        echo $HOSTNAME has $ddcount data directories, bounce not possible. Set environment variable
+        echo EOSIO_TN_NODE to the 2-digit node id number to specify which node to bounce. For example:
+        echo EOSIO_TN_NODE=06 $0 \<options\>
+        cd -
+        exit 1
+    fi
+else
+    DD=tn_data_$EOSIO_TN_NODE
+    if [ ! \( -d $DD \) ]; then
+        echo no directory named $PWD/$DD
+        cd -
+        exit 1
+    fi
+fi
+
+echo DD = $DD
+
+export EOSIO_TN_RESTART_DATA_DIR=$DD
+bash $EOSIO_HOME/scripts/tn_down.sh
+bash $EOSIO_HOME/scripts/tn_up.sh $*
+unset EOSIO_TN_RESTART_DATA_DIR
+
+cd -

scripts/tn_down.sh

+#!/bin/bash
+#
+# tn_bounce is used to restart a node that is acting badly or is down.
+# usage: tn_bounce.sh [arglist]
+# arglist will be passed to the node's command line. First with no modifiers
+# then with --replay and then a third time with --resync
+#
+# the data directory and log file are set by this script. Do not pass them on
+# the command line.
+#
+# in most cases, simply running ./tn_bounce.sh is sufficient.
+#
+
+
+prog=""
+RD=""
+for p in eosd eosiod; do
+    prog=$p
+    RD=$EOSIO_HOME/bin
+    if [ -f $RD/$prog ]; then
+        break;
+    else
+        RD=$EOSIO_HOME/programs/$prog
+        if [ -f $RD/$prog ]; then
+            break;
+        fi
+    fi
+    prog=""
+    RD=""
+done
+
+if [ \( -z "$prog" \) -o \( -z "RD" \) ]; then
+    echo unable to locate binary for eosd or eosiod
+    exit 1
+fi
+
+if [ -z "$EOSIO_TN_RESTART_DATA_DIR" ]; then
+    echo data directory not set
+    exit 1
+fi
+
+DD=$EOSIO_TN_RESTART_DATA_DIR;
+runtest=""
+if [ $DD = "all" ]; then
+    runtest=$prog
+else
+    runtest=`cat $DD/$prog.pid`
+fi
+echo runtest =  $runtest
+running=`ps -e | grep $runtest | grep -cv grep `
+
+if [ $running -ne 0 ]; then
+    echo killing $prog
+
+    if [ $runtest = $prog ]; then
+        kill -15 $runtest
+    else
+        pkill -15 $runtest
+    fi
+
+    for (( a = 10; $a; a = $(($a - 1)) )); do
+        echo waiting for safe termination, pass $((11 - $a))
+        sleep 2
+        running=`ps -e | grep $runtest | grep -cv grep`
+        echo running = $running
+        if [ $running -eq 0 ]; then
+            break;
+        fi
+    done
+fi
+
+if [ $running -ne 0 ]; then
+    echo killing $prog with SIGTERM failed, trying with SIGKILL
+    if [ $runtest = $prog ]; then
+        kill -9 $runtest
+    else
+        pkill -9 $runtest
+    fi
+fi

scripts/tn_roll.sh

+#!/bin/bash
+#
+# tn_bounce is used to restart a node that is acting badly or is down.
+# usage: tn_bounce.sh [arglist]
+# arglist will be passed to the node's command line. First with no modifiers
+# then with --replay and then a third time with --resync
+#
+# the data directory and log file are set by this script. Do not pass them on
+# the command line.
+#
+# in most cases, simply running ./tn_bounce.sh is sufficient.
+#
+
+cd $EOSIO_HOME
+
+prog=eosd
+if [ ! \( -f programs/$prog/$prog \) ]; then
+    prog=eosiod
+fi
+
+if [ -z "$EOSIO_TN_NODE" ]; then
+    DD=`ls -d tn_data_??`
+    ddcount=`echo $DD | wc -w`
+    if [ $ddcount -ne 1 ]; then
+        DD="all"
+    fi
+else
+    DD=tn_data_$EOSIO_TN_NODE
+    if [ ! \( -d $DD \) ]; then
+        echo no directory named $PWD/$DD
+        cd -
+        exit 1
+    fi
+fi
+
+prog=""
+RD=""
+for p in eosd eosiod; do
+    prog=$p
+    RD=$EOSIO_HOME/bin
+    if [ -f $RD/$prog ]; then
+        break;
+    else
+        RD=$EOSIO_HOME/programs/$prog
+        if [ -f $RD/$prog ]; then
+            break;
+        fi
+    fi
+    prog=""
+    RD=""
+done
+
+if [ \( -z "$prog" \) -o \( -z "$RD" \) ]; then
+    echo unable to locate binary for eosd or eosiod
+    exit 1
+fi
+
+SDIR=staging/eos
+if [ -e $RD/$prog ]; then
+    s1=`md5sum $RD/$prog`
+    s2=`md5sum $SDIR/$prog`
+    if [ "$s1" == "$s2" ]; then
+        echo $HOSTNAME no update $SDIR/$p
+        exit 1;
+    fi
+fi
+
+
+echo DD = $DD
+
+export EOSIO_TN_RESTART_DATA_DIR=$DD
+bash $EOSIO_HOME/scripts/tn_down.sh
+
+cp $SDIR/$RD/$prog $RD/$prog
+
+bash $EOSIO_HOME/scripts/tn_up.sh $*
+unset EOSIO_TN_RESTART_DATA_DIR
+
+cd -

scripts/tn_up.sh

+#!/bin/bash
+#
+# tn_up is a helper script used to start a node that was previously stopped
+# usage: tn_bounce.sh [ [arglist]
+# arglist will be passed to the node's command line. First with no modifiers
+# then with --replay and then a third time with --resync
+#
+# the data directory and log file are set by this script. Do not pass them on
+# the command line.
+#
+# in most cases, simply running ./tn_bounce.sh is sufficient.
+#
+
+connected="0"
+
+relaunch() {
+    prog=$1; shift
+    DD=$1; shift
+    RD=$1; shift
+
+    now=`date +'%Y_%m_%d_%H_%M_%S'`
+    log=$DD/stderr.$now.txt
+
+    nohup $RD/$prog $* --data-dir $DD > $DD/stdout.txt  2> $log &
+    pid=$!
+    echo $pid > $DD/$prog.pid
+    for (( a = 10; $a; a = $(($a - 1)) )); do
+        echo checking viability pass $((11 - $a))
+        sleep 2
+        running=`ps -hp $pid`
+        if [ -z "$running" ]; then
+            break;
+        fi
+        connected=`grep -c "net_plugin.cpp:.*connection" $log`
+        if [ "$connected" -ne 0 ]; then
+            break;
+        fi
+    done
+}
+
+prog=""
+RD=""
+for p in eosd eosiod; do
+    prog=$p
+    RD=$EOSIO_HOME/bin
+    if [ -f $RD/$prog ]; then
+        break;
+    else
+        RD=$EOSIO_HOME/programs/$prog
+        if [ -f $RD/$prog ]; then
+            break;
+        fi
+    fi
+    prog=""
+    RD=""
+done
+
+if [ \( -z "$prog" \) -o \( -z "$RD" \) ]; then
+    echo unable to locate binary for eosd or eosiod
+    exit 1
+fi
+
+
+if [ -z "$EOSIO_TN_RESTART_DATA_DIR" ]; then
+    echo data directory not set
+    exit 1
+fi
+
+DD=$EOSIO_TN_RESTART_DATA_DIR;
+
+echo starting with no modifiers
+relaunch $prog $DD $RD $*
+if [ \( -z "$running" \) -o \( "$connected" -eq 0 \) ]; then
+    echo base relaunch failed, trying with replay
+    relaunch $prog $DD $RD $* --replay
+    if [ \( -z "$running" \) -o \( "$connected" -eq 0 \) ]; then
+        echo relaunch replay failed, trying with resync
+        relaunch $prog $DD $RD $* --resync
+    fi
+fi
+
+rm $DD/stderr.txt
+ln -s stderr.$now.txt $DD/stderr.txt