submit_local.sh.in 4.4 KB
Newer Older
Z
zhangjinchao01 已提交
1 2 3 4 5 6
#!/bin/bash

function version(){
        echo "PaddlePaddle @PADDLE_VERSION@, compiled with"
        echo "    with_avx: @WITH_AVX@"
        echo "    with_gpu: @WITH_GPU@"
7
        echo "    with_mkl: @WITH_MKL@"
T
tensor-tang 已提交
8
        echo "    with_mkldnn: @WITH_MKLDNN@"
Z
zhangjinchao01 已提交
9 10 11 12 13 14
        echo "    with_double: @WITH_DOUBLE@"
        echo "    with_python: @WITH_PYTHON@"
        echo "    with_rdma: @WITH_RDMA@"
        echo "    with_timer: @WITH_TIMER@"
}

Y
Yu Yang 已提交
15
function ver2num() {
Y
Yu Yang 已提交
16
  set -e
Y
Yu Yang 已提交
17 18 19 20 21 22 23 24 25 26 27 28
  # convert version to number.
  if [ -z "$1" ]; then # empty argument
    printf "%03d%03d%03d%03d%03d" 0
  else
    local VERN=$(echo $1 | sed 's#v##g' | sed 's#\.# #g' \
        | sed 's#a# 0 #g' | sed 's#b# 1 #g' | sed 's#rc# 2 #g')
    if [ `echo $VERN | wc -w` -eq 3 ] ; then
      printf "%03d%03d%03d%03d%03d" $VERN 999 999
    else
      printf "%03d%03d%03d%03d%03d" $VERN
    fi
  fi
Y
Yu Yang 已提交
29
  set +e
Y
Yu Yang 已提交
30 31
}

32 33
function cpu_config() {
  # auto set KMP_AFFINITY and OMP_DYNAMIC from Hyper Threading Status
34 35
  # only when MKL enabled
  if [ "@WITH_MKL@" == "OFF" ]; then
36 37
    return 0
  fi
T
tensor-tang 已提交
38 39 40 41 42
  platform="`uname -s`"
  ht=0
  if [ $platform == "Linux" ]; then
    ht=`lscpu |grep "per core"|awk -F':' '{print $2}'|xargs`
  elif [ $platform == "Darwin" ]; then
43
    if [ `sysctl -n hw.physicalcpu` -eq `sysctl -n hw.logicalcpu` ]; then
T
tensor-tang 已提交
44 45 46 47 48 49
      # HT is OFF
      ht=1
    fi
  else
    return 0
  fi
50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69
  if [ $ht -eq 1 ]; then # HT is OFF
    if [ -z "$KMP_AFFINITY" ]; then
      export KMP_AFFINITY="granularity=fine,compact,0,0"
    fi
    if [ -z "$OMP_DYNAMIC" ]; then
      export OMP_DYNAMIC="FALSE"
    fi
  else # HT is ON
    if [ -z "$KMP_AFFINITY" ]; then
      export KMP_AFFINITY="granularity=fine,compact,1,0"
    fi
    if [ -z "$OMP_DYNAMIC" ]; then
      export OMP_DYNAMIC="True"
    fi
  fi
}

function threads_config() {
  # auto set OMP_NUM_THREADS and MKL_NUM_THREADS
  # according to trainer_count and total processors
70
  # only when MKL enabled
T
tensor-tang 已提交
71
  # auto set OPENBLAS_NUM_THREADS when do not use MKL
T
tensor-tang 已提交
72 73 74 75 76 77 78 79 80
  platform="`uname -s`"
  processors=0
  if [ $platform == "Linux" ]; then
    processors=`grep "processor" /proc/cpuinfo|sort -u|wc -l`
  elif [ $platform == "Darwin" ]; then
    processors=`sysctl -n hw.logicalcpu`
  else
    return 0
  fi
81 82 83 84
  trainers=`grep -Eo 'trainer_count.[0-9]+' <<< "$@" |grep -Eo '[0-9]+'|xargs`
  if [ -z $trainers ]; then
    trainers=1
  fi
85 86 87 88
  threads=$((processors / trainers))
  if [ $threads -eq 0 ]; then
    threads=1
  fi
T
tensor-tang 已提交
89 90 91 92 93 94 95 96 97 98 99
  if [ "@WITH_MKL@" == "ON" ]; then
    if [ -z "$OMP_NUM_THREADS" ]; then
      export OMP_NUM_THREADS=$threads
    fi
    if [ -z "$MKL_NUM_THREADS" ]; then
      export MKL_NUM_THREADS=$threads
    fi
  else
    if [ -z "$OPENBLAS_NUM_THREADS" ]; then
      export OPENBLAS_NUM_THREADS=$threads
    fi
L
Luo Tao 已提交
100 101 102
    if [ $threads -gt 1 ] && [ -z "$OPENBLAS_MAIN_FREE" ]; then
      export OPENBLAS_MAIN_FREE=1
    fi
103
  fi
T
tensor-tang 已提交
104
  
105 106
}

Y
Yu Yang 已提交
107 108 109 110
PADDLE_CONF_HOME="$HOME/.config/paddle"
mkdir -p ${PADDLE_CONF_HOME}

if [ -z "${PADDLE_NO_STAT+x}" ]; then
Y
Yu Yang 已提交
111
    SERVER_VER=`curl -m 5 -X POST --data content="{ \"version\": \"@PADDLE_VERSION@\" }"\
Y
Yu Yang 已提交
112 113 114 115
        -b ${PADDLE_CONF_HOME}/paddle.cookie \
        -c ${PADDLE_CONF_HOME}/paddle.cookie \
        http://api.paddlepaddle.org/version 2>/dev/null`
    if [ $? -eq 0 ] && [ "$(ver2num @PADDLE_VERSION@)" -lt  $(ver2num $SERVER_VER) ]; then
116
      echo "Paddle release a new version ${SERVER_VER}, you can get the install package in http://www.paddlepaddle.org"
Y
Yu Yang 已提交
117 118 119
    fi
fi

120
PADDLE_BIN_PATH="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
Z
zhangjinchao01 已提交
121 122 123 124 125 126 127 128 129 130 131 132 133

if [ ! -z "${DEBUGGER}" ]; then
    echo "Using debug command ${DEBUGGER}"
fi

CUDNN_LIB_PATH="@CUDNN_LIB_PATH@"

if [ ! -z "${CUDNN_LIB_PATH}" ]; then
    export LD_LIBRARY_PATH=${CUDNN_LIB_PATH}:${LD_LIBRARY_PATH}
fi

export PYTHONPATH=${PWD}:${PYTHONPATH}

Y
Yu Yang 已提交
134 135 136 137 138 139 140 141

# Check python lib installed or not.
pip --help > /dev/null
if [ $? -ne 0 ]; then
    echo "pip should be installed to run paddle."
    exit 1
fi

142 143 144 145 146 147 148
if [ "@WITH_GPU@" == "ON" ]; then
    PADDLE_NAME="paddlepaddle-gpu"
else 
    PADDLE_NAME="paddlepaddle"
fi

INSTALLED_VERSION=`pip freeze 2>/dev/null | grep "^${PADDLE_NAME}==" | sed 's/.*==//g'`
Y
Yu Yang 已提交
149

150
if [ -z "${INSTALLED_VERSION}" ]; then
Y
Yu Yang 已提交
151 152 153 154 155 156 157 158 159 160 161
   INSTALLED_VERSION="0.0.0"  # not installed
fi
cat <<EOF | python -
from distutils.version import LooseVersion
import sys
if LooseVersion("${INSTALLED_VERSION}") < LooseVersion("@PADDLE_VERSION@"):
  sys.exit(1)
else:
  sys.exit(0)
EOF

T
tensor-tang 已提交
162
cpu_config
T
tensor-tang 已提交
163
# echo $KMP_AFFINITY $OMP_DYNAMIC
Y
Yu Yang 已提交
164

Z
zhangjinchao01 已提交
165 166 167 168 169
case "$1" in
    "version")
        version
        ;;
    *)
T
Tao Luo 已提交
170
        version
Z
zhangjinchao01 已提交
171 172
        ;;
 esac