add barrier (#2309)

tests/benchmark/pwgan/run_benchmark.sh

@@ -43,6 +43,7 @@ function _train(){
         log_parse_file="mylog/workerlog.0" ;;
     *) echo "choose run_mode(sp or mp)"; exit 1;
     esac
+    bash tests/test_tipc/barrier.sh
 # 以下不用修改
     timeout 15m ${train_cmd} > ${log_file} 2>&1
     if [ $? -ne 0 ];then

tests/test_tipc/barrier.sh

+set -ex
+NNODES=${PADDLE_TRAINERS_NUM:-"1"}
+PYTHON=${PYTHON:-"python"}
+TIMEOUT=${1:-"10m"}
+
+if [[ "$NNODES" -gt 1 ]]; then
+  while ! timeout "$TIMEOUT" "$PYTHON" -m paddle.distributed.launch run_check; do
+    echo "Retry barrier ......"
+  done
+fi