Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
wmsofts
Paddle
提交
3508bd28
P
Paddle
项目概览
wmsofts
/
Paddle
与 Fork 源项目一致
Fork自
PaddlePaddle / Paddle
通知
1
Star
0
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
P
Paddle
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
未验证
提交
3508bd28
编写于
7月 08, 2021
作者:
D
dyning
提交者:
GitHub
7月 08, 2021
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
Add the op def for elementwise_mul and enhance layer_norm_fuse_pass (#33560)
上级
11f5a400
变更
3
隐藏空白更改
内联
并排
Showing
3 changed file
with
158 addition
and
18 deletion
+158
-18
paddle/fluid/framework/ir/layer_norm_fuse_pass.cc
paddle/fluid/framework/ir/layer_norm_fuse_pass.cc
+126
-0
paddle/fluid/framework/ir/layer_norm_fuse_pass.h
paddle/fluid/framework/ir/layer_norm_fuse_pass.h
+1
-0
paddle/fluid/framework/ir/layer_norm_fuse_pass_tester.cc
paddle/fluid/framework/ir/layer_norm_fuse_pass_tester.cc
+31
-18
未找到文件。
paddle/fluid/framework/ir/layer_norm_fuse_pass.cc
浏览文件 @
3508bd28
...
@@ -99,6 +99,122 @@ void addIntermediateOut(Node* op_node, const std::string& out_name,
...
@@ -99,6 +99,122 @@ void addIntermediateOut(Node* op_node, const std::string& out_name,
}
// namespace
}
// namespace
LayerNormFusePass
::
LayerNormFusePass
()
{
AddOpCompat
(
OpCompat
(
"layer_norm"
))
.
AddInput
(
"X"
)
.
IsTensor
()
.
End
()
.
AddInput
(
"Scale"
)
.
IsTensor
()
.
End
()
.
AddInput
(
"Bias"
)
.
IsTensor
()
.
End
()
.
AddOutput
(
"Y"
)
.
IsTensor
()
.
End
()
.
AddOutput
(
"Mean"
)
.
IsTensor
()
.
IsOptional
()
.
End
()
.
AddOutput
(
"Variance"
)
.
IsTensor
()
.
IsOptional
()
.
End
()
.
AddAttr
(
"epsilon"
)
.
IsNumGE
(
0.0
f
)
.
IsNumLE
(
0.001
f
)
.
End
()
.
AddAttr
(
"begin_norm_axis"
)
.
IsNumGT
(
0
)
.
End
();
AddOpCompat
(
OpCompat
(
"reduce_mean"
))
.
AddInput
(
"X"
)
.
IsTensor
()
.
End
()
.
AddOutput
(
"Out"
)
.
IsTensor
()
.
End
()
.
AddAttr
(
"dim"
)
.
IsType
<
std
::
vector
<
int
>>
()
.
End
()
.
AddAttr
(
"keep_dim"
)
.
IsBoolEQ
(
true
)
.
End
();
AddOpCompat
(
OpCompat
(
"sqrt"
))
.
AddInput
(
"X"
)
.
IsTensor
()
.
End
()
.
AddOutput
(
"Out"
)
.
IsTensor
()
.
End
();
AddOpCompat
(
OpCompat
(
"elementwise_sub"
))
.
AddInput
(
"X"
)
.
IsTensor
()
.
End
()
.
AddInput
(
"Y"
)
.
IsTensor
()
.
End
()
.
AddOutput
(
"Out"
)
.
IsTensor
()
.
End
()
.
AddAttr
(
"axis"
)
.
IsNumEQ
(
1
)
.
End
();
AddOpCompat
(
OpCompat
(
"elementwise_pow"
))
.
AddInput
(
"X"
)
.
IsTensor
()
.
End
()
.
AddInput
(
"Y"
)
.
IsTensor
()
.
End
()
.
AddOutput
(
"Out"
)
.
IsTensor
()
.
End
()
.
AddAttr
(
"axis"
)
.
IsNumEQ
(
1
)
.
End
();
AddOpCompat
(
OpCompat
(
"elementwise_add"
))
.
AddInput
(
"X"
)
.
IsTensor
()
.
End
()
.
AddInput
(
"Y"
)
.
IsTensor
()
.
End
()
.
AddOutput
(
"Out"
)
.
IsTensor
()
.
End
()
.
AddAttr
(
"axis"
)
.
IsNumEQ
(
1
)
.
End
();
AddOpCompat
(
OpCompat
(
"elementwise_div"
))
.
AddInput
(
"X"
)
.
IsTensor
()
.
End
()
.
AddInput
(
"Y"
)
.
IsTensor
()
.
End
()
.
AddOutput
(
"Out"
)
.
IsTensor
()
.
End
()
.
AddAttr
(
"axis"
)
.
IsNumEQ
(
1
)
.
End
();
AddOpCompat
(
OpCompat
(
"elementwise_mul"
))
.
AddInput
(
"X"
)
.
IsTensor
()
.
End
()
.
AddInput
(
"Y"
)
.
IsTensor
()
.
End
()
.
AddOutput
(
"Out"
)
.
IsTensor
()
.
End
()
.
AddAttr
(
"axis"
)
.
IsNumEQ
(
1
)
.
End
();
}
void
LayerNormFusePass
::
ApplyImpl
(
Graph
*
graph
)
const
{
void
LayerNormFusePass
::
ApplyImpl
(
Graph
*
graph
)
const
{
PADDLE_ENFORCE_NOT_NULL
(
graph
,
PADDLE_ENFORCE_NOT_NULL
(
graph
,
platform
::
errors
::
InvalidArgument
(
platform
::
errors
::
InvalidArgument
(
...
@@ -117,6 +233,10 @@ void LayerNormFusePass::ApplyImpl(Graph* graph) const {
...
@@ -117,6 +233,10 @@ void LayerNormFusePass::ApplyImpl(Graph* graph) const {
int
found_layer_norm_count
=
0
;
int
found_layer_norm_count
=
0
;
auto
handler
=
[
&
](
const
GraphPatternDetector
::
subgraph_t
&
subgraph
,
auto
handler
=
[
&
](
const
GraphPatternDetector
::
subgraph_t
&
subgraph
,
Graph
*
g
)
{
Graph
*
g
)
{
if
(
!
IsCompat
(
subgraph
,
g
))
{
LOG
(
WARNING
)
<<
"Pass in op compat failed."
;
return
;
}
VLOG
(
4
)
<<
"Fuse LayerNorm from subgraph."
;
VLOG
(
4
)
<<
"Fuse LayerNorm from subgraph."
;
GET_IR_NODE_FROM_SUBGRAPH
(
x
,
x
,
layer_norm_pattern
);
GET_IR_NODE_FROM_SUBGRAPH
(
x
,
x
,
layer_norm_pattern
);
GET_IR_NODE_FROM_SUBGRAPH
(
x_mean
,
x_mean
,
layer_norm_pattern
);
GET_IR_NODE_FROM_SUBGRAPH
(
x_mean
,
x_mean
,
layer_norm_pattern
);
...
@@ -205,6 +325,12 @@ void LayerNormFusePass::ApplyImpl(Graph* graph) const {
...
@@ -205,6 +325,12 @@ void LayerNormFusePass::ApplyImpl(Graph* graph) const {
ln_op_desc
.
SetAttr
(
"begin_norm_axis"
,
static_cast
<
int
>
(
x_shape
.
size
()
-
1
));
ln_op_desc
.
SetAttr
(
"begin_norm_axis"
,
static_cast
<
int
>
(
x_shape
.
size
()
-
1
));
ln_op_desc
.
SetAttr
(
"epsilon"
,
*
(
eps_tensor
->
data
<
float
>
()));
ln_op_desc
.
SetAttr
(
"epsilon"
,
*
(
eps_tensor
->
data
<
float
>
()));
ln_op_desc
.
SetAttr
(
"is_test"
,
true
);
ln_op_desc
.
SetAttr
(
"is_test"
,
true
);
if
(
!
IsCompat
(
ln_op_desc
))
{
LOG
(
WARNING
)
<<
"layer norm pass in out layer_norm op compat failed."
;
return
;
}
Node
*
ln_op
=
g
->
CreateOpNode
(
&
ln_op_desc
);
Node
*
ln_op
=
g
->
CreateOpNode
(
&
ln_op_desc
);
addIntermediateOut
(
ln_op
,
"Mean"
,
scope_name_
,
g
);
addIntermediateOut
(
ln_op
,
"Mean"
,
scope_name_
,
g
);
...
...
paddle/fluid/framework/ir/layer_norm_fuse_pass.h
浏览文件 @
3508bd28
...
@@ -70,6 +70,7 @@ namespace ir {
...
@@ -70,6 +70,7 @@ namespace ir {
*/
*/
class
LayerNormFusePass
:
public
FusePassBase
{
class
LayerNormFusePass
:
public
FusePassBase
{
public:
public:
LayerNormFusePass
();
virtual
~
LayerNormFusePass
()
{}
virtual
~
LayerNormFusePass
()
{}
protected:
protected:
...
...
paddle/fluid/framework/ir/layer_norm_fuse_pass_tester.cc
浏览文件 @
3508bd28
...
@@ -66,12 +66,16 @@ class LayerNormFuseTest {
...
@@ -66,12 +66,16 @@ class LayerNormFuseTest {
x_mean
->
SetAttr
(
"keep_dim"
,
true
);
x_mean
->
SetAttr
(
"keep_dim"
,
true
);
x_mean
->
SetAttr
(
"reduce_all"
,
false
);
x_mean
->
SetAttr
(
"reduce_all"
,
false
);
test
::
CreateOp
(
&
m_prog
,
"elementwise_sub"
,
auto
*
x_sub
=
test
::
CreateOp
(
&
m_prog
,
"elementwise_sub"
,
{{
"X"
,
"x"
},
{
"Y"
,
"x_mean_out"
}},
{{
"X"
,
"x"
},
{
"Y"
,
"x_mean_out"
}},
{{
"Out"
,
"x_sub_mean_out"
}},
false
);
{{
"Out"
,
"x_sub_mean_out"
}},
false
);
test
::
CreateOp
(
&
m_prog
,
"elementwise_pow"
,
x_sub
->
SetAttr
(
"axis"
,
1
);
{{
"X"
,
"x_sub_mean_out"
},
{
"Y"
,
"sqr_pow"
}},
{{
"Out"
,
"x_sub_mean_sqr_out"
}},
false
);
auto
*
x_pow
=
test
::
CreateOp
(
&
m_prog
,
"elementwise_pow"
,
{{
"X"
,
"x_sub_mean_out"
},
{
"Y"
,
"sqr_pow"
}},
{{
"Out"
,
"x_sub_mean_sqr_out"
}},
false
);
x_pow
->
SetAttr
(
"axis"
,
1
);
auto
*
std_dev
=
auto
*
std_dev
=
test
::
CreateOp
(
&
m_prog
,
"reduce_mean"
,
{{
"X"
,
"x_sub_mean_sqr_out"
}},
test
::
CreateOp
(
&
m_prog
,
"reduce_mean"
,
{{
"X"
,
"x_sub_mean_sqr_out"
}},
{{
"Out"
,
"std_dev_out"
}},
false
);
{{
"Out"
,
"std_dev_out"
}},
false
);
...
@@ -79,20 +83,29 @@ class LayerNormFuseTest {
...
@@ -79,20 +83,29 @@ class LayerNormFuseTest {
std_dev
->
SetAttr
(
"keep_dim"
,
true
);
std_dev
->
SetAttr
(
"keep_dim"
,
true
);
std_dev
->
SetAttr
(
"reduce_all"
,
false
);
std_dev
->
SetAttr
(
"reduce_all"
,
false
);
test
::
CreateOp
(
&
m_prog
,
"elementwise_add"
,
auto
*
x_add
=
test
::
CreateOp
(
&
m_prog
,
"elementwise_add"
,
{{
"X"
,
"std_dev_out"
},
{
"Y"
,
"eps"
}},
{{
"X"
,
"std_dev_out"
},
{
"Y"
,
"eps"
}},
{{
"Out"
,
"std_dev_eps_out"
}},
false
);
{{
"Out"
,
"std_dev_eps_out"
}},
false
);
x_add
->
SetAttr
(
"axis"
,
1
);
test
::
CreateOp
(
&
m_prog
,
"sqrt"
,
{{
"X"
,
"std_dev_eps_out"
}},
test
::
CreateOp
(
&
m_prog
,
"sqrt"
,
{{
"X"
,
"std_dev_eps_out"
}},
{{
"Out"
,
"std_dev_eps_sqrt_out"
}},
false
);
{{
"Out"
,
"std_dev_eps_sqrt_out"
}},
false
);
test
::
CreateOp
(
&
m_prog
,
"elementwise_div"
,
{{
"X"
,
"x_sub_mean_out"
},
{
"Y"
,
"std_dev_eps_sqrt_out"
}},
auto
*
x_div
=
{{
"Out"
,
"division_out"
}},
false
);
test
::
CreateOp
(
&
m_prog
,
"elementwise_div"
,
test
::
CreateOp
(
&
m_prog
,
"elementwise_mul"
,
{{
"X"
,
"x_sub_mean_out"
},
{
"Y"
,
"std_dev_eps_sqrt_out"
}},
{{
"X"
,
"division_out"
},
{
"Y"
,
"gamma"
}},
{{
"Out"
,
"division_out"
}},
false
);
{{
"Out"
,
"scale_out"
}},
false
);
x_div
->
SetAttr
(
"axis"
,
1
);
test
::
CreateOp
(
&
m_prog
,
"elementwise_add"
,
{{
"X"
,
"scale_out"
},
{
"Y"
,
"beta"
}},
{{
"Out"
,
"shift_out"
}},
auto
*
x_mul
=
test
::
CreateOp
(
&
m_prog
,
"elementwise_mul"
,
false
);
{{
"X"
,
"division_out"
},
{
"Y"
,
"gamma"
}},
{{
"Out"
,
"scale_out"
}},
false
);
x_mul
->
SetAttr
(
"axis"
,
1
);
auto
*
x_add_v1
=
test
::
CreateOp
(
&
m_prog
,
"elementwise_add"
,
{{
"X"
,
"scale_out"
},
{
"Y"
,
"beta"
}},
{{
"Out"
,
"shift_out"
}},
false
);
x_add_v1
->
SetAttr
(
"axis"
,
1
);
}
}
template
<
typename
Func
>
template
<
typename
Func
>
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录