Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
溜达小灰鼠
Paddle
提交
6cf7f1e4
P
Paddle
项目概览
溜达小灰鼠
/
Paddle
与 Fork 源项目一致
Fork自
PaddlePaddle / Paddle
通知
1
Star
1
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
1
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
P
Paddle
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
1
Issue
1
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
未验证
提交
6cf7f1e4
编写于
11月 16, 2017
作者:
T
Tao Luo
提交者:
GitHub
11月 16, 2017
浏览文件
操作
浏览文件
下载
差异文件
Merge pull request #5671 from tensor-tang/autocpu
auto set cpu env when mkldnn or mklml enabled
上级
a76b6144
6337007e
变更
3
隐藏空白更改
内联
并排
Showing
3 changed file
with
78 addition
and
5 deletion
+78
-5
benchmark/paddle/image/run_mkldnn.sh
benchmark/paddle/image/run_mkldnn.sh
+1
-5
paddle/scripts/submit_local.sh.in
paddle/scripts/submit_local.sh.in
+52
-0
python/paddle/v2/__init__.py
python/paddle/v2/__init__.py
+25
-0
未找到文件。
benchmark/paddle/image/run_mkldnn.sh
浏览文件 @
6cf7f1e4
set
-e
function
train
()
{
unset
OMP_NUM_THREADS MKL_NUM_THREADS
export
OMP_DYNAMIC
=
"FALSE"
export
KMP_AFFINITY
=
"granularity=fine,compact,0,0"
unset
OMP_NUM_THREADS MKL_NUM_THREADS OMP_DYNAMIC KMP_AFFINITY
topology
=
$1
layer_num
=
$2
bs
=
$3
...
...
@@ -14,8 +12,6 @@ function train() {
elif
[
$4
==
"False"
]
;
then
thread
=
`
nproc
`
# each trainer_count use only 1 core to avoid conflict
export
OMP_NUM_THREADS
=
1
export
MKL_NUM_THREADS
=
1
log
=
"logs/
${
topology
}
-
${
layer_num
}
-
${
thread
}
mklml-
${
bs
}
.log"
else
echo
"Wrong input
$3
, use True or False."
...
...
paddle/scripts/submit_local.sh.in
浏览文件 @
6cf7f1e4
...
...
@@ -43,6 +43,54 @@ function ver2num() {
set
+e
}
function
cpu_config
()
{
# auto set KMP_AFFINITY and OMP_DYNAMIC from Hyper Threading Status
# only when MKLDNN or MKLML enabled
if
[
"@WITH_MKLDNN@"
==
"OFF"
]
&&
[
"@WITH_MKLML@"
==
"OFF"
]
;
then
return
0
fi
ht
=
`
lscpu |grep
"per core"
|awk
-F
':'
'{print $2}'
|xargs
`
if
[
$ht
-eq
1
]
;
then
# HT is OFF
if
[
-z
"
$KMP_AFFINITY
"
]
;
then
export
KMP_AFFINITY
=
"granularity=fine,compact,0,0"
fi
if
[
-z
"
$OMP_DYNAMIC
"
]
;
then
export
OMP_DYNAMIC
=
"FALSE"
fi
else
# HT is ON
if
[
-z
"
$KMP_AFFINITY
"
]
;
then
export
KMP_AFFINITY
=
"granularity=fine,compact,1,0"
fi
if
[
-z
"
$OMP_DYNAMIC
"
]
;
then
export
OMP_DYNAMIC
=
"True"
fi
fi
}
function
threads_config
()
{
# auto set OMP_NUM_THREADS and MKL_NUM_THREADS
# according to trainer_count and total processors
# only when MKLDNN or MKLML enabled
if
[
"@WITH_MKLDNN@"
==
"OFF"
]
&&
[
"@WITH_MKLML@"
==
"OFF"
]
;
then
return
0
fi
processors
=
`
grep
"processor"
/proc/cpuinfo|sort
-u
|wc
-l
`
trainers
=
`
grep
-Eo
'trainer_count.[0-9]+'
<<<
"
$@
"
|grep
-Eo
'[0-9]+'
|xargs
`
if
[
-z
$trainers
]
;
then
trainers
=
1
fi
threads
=
$((
processors
/
trainers
))
if
[
$threads
-eq
0
]
;
then
threads
=
1
fi
if
[
-z
"
$OMP_NUM_THREADS
"
]
;
then
export
OMP_NUM_THREADS
=
$threads
fi
if
[
-z
"
$MKL_NUM_THREADS
"
]
;
then
export
MKL_NUM_THREADS
=
$threads
fi
}
PADDLE_CONF_HOME
=
"
$HOME
/.config/paddle"
mkdir
-p
${
PADDLE_CONF_HOME
}
...
...
@@ -92,9 +140,13 @@ else:
sys.exit(0)
EOF
cpu_config
# echo $KMP_AFFINITY $OMP_DYNAMIC
case
"
$1
"
in
"train"
)
threads_config
$@
# echo $OMP_NUM_THREADS $MKL_NUM_THREADS
${
DEBUGGER
}
$PADDLE_BIN_PATH
/paddle_trainer
${
@
:2
}
;;
"merge_model"
)
...
...
python/paddle/v2/__init__.py
浏览文件 @
6cf7f1e4
...
...
@@ -76,6 +76,31 @@ def init(**kwargs):
for
key
in
args_dict
.
keys
():
args
.
append
(
'--%s=%s'
%
(
key
,
str
(
args_dict
[
key
])))
# auto set cpu environment
def
set_env
(
key
,
value
):
'''If the key has not been set in the environment, set it with value.'''
assert
isinstance
(
key
,
str
)
assert
isinstance
(
value
,
str
)
envset
=
os
.
environ
.
get
(
key
)
if
envset
is
None
:
os
.
environ
[
key
]
=
value
ht
=
os
.
popen
(
"lscpu |grep
\"
per core
\"
|awk -F':' '{print $2}'|xargs"
)
ht
=
int
(
ht
.
read
())
if
ht
==
1
:
# ht is off
set_env
(
"OMP_DYNAMIC"
,
"false"
)
set_env
(
"KMP_AFFINITY"
,
"granularity=fine,compact,0,0"
)
else
:
set_env
(
"OMP_DYNAMIC"
,
"true"
)
set_env
(
"KMP_AFFINITY"
,
"granularity=fine,compact,1,0"
)
processors
=
os
.
popen
(
"grep
\"
processor
\"
/proc/cpuinfo|sort -u|wc -l"
)
processors
=
int
(
processors
.
read
())
trainers
=
kwargs
.
get
(
'trainer_count'
,
1
)
threads
=
processors
/
trainers
threads
=
'1'
if
threads
<
1
else
str
(
threads
)
set_env
(
"OMP_NUM_THREADS"
,
threads
)
set_env
(
"MKL_NUM_THREADS"
,
threads
)
if
'use_gpu'
in
kwargs
:
cp
.
g_command_config_args
[
'use_gpu'
]
=
kwargs
[
'use_gpu'
]
if
'use_mkldnn'
in
kwargs
:
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录