未验证 提交 d00e1ed5 编写于 作者: T Tao Luo 提交者: GitHub

Merge pull request #7104 from tensor-tang/openblas

auto set openblas env
...@@ -8,15 +8,19 @@ function clock_to_seconds() { ...@@ -8,15 +8,19 @@ function clock_to_seconds() {
} }
function infer() { function infer() {
unset OMP_NUM_THREADS MKL_NUM_THREADS OMP_DYNAMIC KMP_AFFINITY
topology=$1 topology=$1
layer_num=$2 layer_num=$2
bs=$3 bs=$3
thread=`nproc` trainers=`nproc`
if [ $thread -gt $bs ]; then if [ $trainers -gt $bs ]; then
thread=$bs trainers=$bs
fi fi
log="logs/infer-${topology}-${layer_num}-${thread}openblas-${bs}.log" log="logs/infer-${topology}-${layer_num}-${trainers}openblas-${bs}.log"
threads=$((`nproc` / trainers))
if [ $threads -eq 0 ]; then
threads=1
fi
export OPENBLAS_NUM_THREADS=$threads
models_in="models/${topology}-${layer_num}/pass-00000/" models_in="models/${topology}-${layer_num}/pass-00000/"
if [ ! -d $models_in ]; then if [ ! -d $models_in ]; then
...@@ -28,7 +32,7 @@ function infer() { ...@@ -28,7 +32,7 @@ function infer() {
--config="${topology}.py" \ --config="${topology}.py" \
--use_mkldnn=False \ --use_mkldnn=False \
--use_gpu=False \ --use_gpu=False \
--trainer_count=$thread \ --trainer_count=$trainers \
--log_period=$log_period \ --log_period=$log_period \
--config_args="batch_size=${bs},layer_num=${layer_num},is_infer=True,num_samples=256" \ --config_args="batch_size=${bs},layer_num=${layer_num},is_infer=True,num_samples=256" \
--init_model_path=$models_in \ --init_model_path=$models_in \
......
set -e set -e
function train() { function train() {
unset OMP_NUM_THREADS MKL_NUM_THREADS OMP_DYNAMIC KMP_AFFINITY export OPENBLAS_NUM_THREADS=1
topology=$1 topology=$1
layer_num=$2 layer_num=$2
bs=$3 bs=$3
......
...@@ -71,9 +71,7 @@ function threads_config() { ...@@ -71,9 +71,7 @@ function threads_config() {
# auto set OMP_NUM_THREADS and MKL_NUM_THREADS # auto set OMP_NUM_THREADS and MKL_NUM_THREADS
# according to trainer_count and total processors # according to trainer_count and total processors
# only when MKL enabled # only when MKL enabled
if [ "@WITH_MKL@" == "OFF" ]; then # auto set OPENBLAS_NUM_THREADS when do not use MKL
return 0
fi
processors=`grep "processor" /proc/cpuinfo|sort -u|wc -l` processors=`grep "processor" /proc/cpuinfo|sort -u|wc -l`
trainers=`grep -Eo 'trainer_count.[0-9]+' <<< "$@" |grep -Eo '[0-9]+'|xargs` trainers=`grep -Eo 'trainer_count.[0-9]+' <<< "$@" |grep -Eo '[0-9]+'|xargs`
if [ -z $trainers ]; then if [ -z $trainers ]; then
...@@ -83,12 +81,19 @@ function threads_config() { ...@@ -83,12 +81,19 @@ function threads_config() {
if [ $threads -eq 0 ]; then if [ $threads -eq 0 ]; then
threads=1 threads=1
fi fi
if [ -z "$OMP_NUM_THREADS" ]; then if [ "@WITH_MKL@" == "ON" ]; then
export OMP_NUM_THREADS=$threads if [ -z "$OMP_NUM_THREADS" ]; then
fi export OMP_NUM_THREADS=$threads
if [ -z "$MKL_NUM_THREADS" ]; then fi
export MKL_NUM_THREADS=$threads if [ -z "$MKL_NUM_THREADS" ]; then
export MKL_NUM_THREADS=$threads
fi
else
if [ -z "$OPENBLAS_NUM_THREADS" ]; then
export OPENBLAS_NUM_THREADS=$threads
fi
fi fi
} }
PADDLE_CONF_HOME="$HOME/.config/paddle" PADDLE_CONF_HOME="$HOME/.config/paddle"
...@@ -150,7 +155,7 @@ fi ...@@ -150,7 +155,7 @@ fi
case "$1" in case "$1" in
"train") "train")
threads_config $@ threads_config $@
# echo $OMP_NUM_THREADS $MKL_NUM_THREADS # echo $OMP_NUM_THREADS $MKL_NUM_THREADS $OPENBLAS_NUM_THREADS
${DEBUGGER} $PADDLE_BIN_PATH/paddle_trainer ${@:2} ${DEBUGGER} $PADDLE_BIN_PATH/paddle_trainer ${@:2}
;; ;;
"merge_model") "merge_model")
......
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册