Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
PaddlePaddle
Paddle
提交
4a2b0ae4
P
Paddle
项目概览
PaddlePaddle
/
Paddle
大约 1 年 前同步成功
通知
2299
Star
20931
Fork
5422
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
1423
列表
看板
标记
里程碑
合并请求
543
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
P
Paddle
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
1,423
Issue
1,423
列表
看板
标记
里程碑
合并请求
543
合并请求
543
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
提交
4a2b0ae4
编写于
11月 21, 2017
作者:
A
Abhinav Arora
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
Implementing the MSRA initializer for rectifier units
上级
55ecd6d2
变更
2
隐藏空白更改
内联
并排
Showing
2 changed file
with
187 addition
and
0 deletion
+187
-0
python/paddle/v2/fluid/initializer.py
python/paddle/v2/fluid/initializer.py
+83
-0
python/paddle/v2/fluid/tests/test_initializer.py
python/paddle/v2/fluid/tests/test_initializer.py
+104
-0
未找到文件。
python/paddle/v2/fluid/initializer.py
浏览文件 @
4a2b0ae4
...
@@ -285,3 +285,86 @@ class XavierInitializer(Initializer):
...
@@ -285,3 +285,86 @@ class XavierInitializer(Initializer):
})
})
var
.
op
=
op
var
.
op
=
op
return
op
return
op
class
MSRAInitializer
(
Initializer
):
"""Implements the MSRA initializer a.k.a. Kaiming Initializer
This class implements the weight initialization from the paper
Delving Deep into Rectifiers: Surpassing Human-Level Performance on
ImageNet Classification[1] by Kaiming He, Xiangyu Zhang, Shaoqing Ren
and Jian Sun. This is a robust initialization method that particularly
considers the rectifier nonlinearities. In case of Uniform distribution,
the range is [-x, x], where x = sqrt(6 / fan_in). In case of Normal
distribution, the mean is 0 and the standard deviation
is sqrt(2/ fan_in).
References:
[1] Delving Deep into Rectifiers: Surpassing Human-Level Performance
on ImageNet Classification
(https://arxiv.org/abs/1502.01852)
"""
def
__init__
(
self
,
uniform
=
True
,
fan_in
=
None
,
seed
=
0
):
"""Constructor for MSRAInitializer
Args:
uniform: whether to use uniform or normal distribution
fan_in: fan_in for MSRAInitializer. If None, it is
inferred from the variable.
seed: random seed
Note: It is recommended to set fan_in to None for most cases.
"""
assert
uniform
is
not
None
assert
seed
is
not
None
super
(
MSRAInitializer
,
self
).
__init__
()
self
.
_uniform
=
uniform
self
.
_fan_in
=
fan_in
self
.
_seed
=
seed
def
__call__
(
self
,
var
,
block
):
"""Add MSRA initialization ops for a variable
Args:
var: Variable that needs to be initialized
block: The block in which initialization ops
should be added
Returns:
the initialization op
"""
assert
isinstance
(
var
,
framework
.
Variable
)
assert
isinstance
(
block
,
framework
.
Block
)
f_in
,
f_out
=
self
.
_compute_fans
(
var
)
# If fan_in is passed, use it
fan_in
=
f_in
if
self
.
_fan_in
is
None
else
self
.
_fan_in
if
self
.
_uniform
:
limit
=
np
.
sqrt
(
6.0
/
float
(
fan_in
))
op
=
block
.
prepend_op
(
type
=
"uniform_random"
,
outputs
=
{
"Out"
:
var
},
attrs
=
{
"shape"
:
var
.
shape
,
"data_type"
:
int
(
var
.
data_type
),
"min"
:
-
limit
,
"max"
:
limit
,
"seed"
:
self
.
_seed
})
else
:
std
=
np
.
sqrt
(
2.0
/
float
(
fan_in
))
op
=
block
.
prepend_op
(
type
=
"gaussian_random"
,
outputs
=
{
"Out"
:
var
},
attrs
=
{
"shape"
:
var
.
shape
,
"data_type"
:
int
(
var
.
data_type
),
"mean"
:
0.0
,
"std"
:
std
,
"seed"
:
self
.
_seed
})
var
.
op
=
op
return
op
python/paddle/v2/fluid/tests/test_initializer.py
浏览文件 @
4a2b0ae4
...
@@ -223,5 +223,109 @@ class TestXavierInitializer(unittest.TestCase):
...
@@ -223,5 +223,109 @@ class TestXavierInitializer(unittest.TestCase):
self
.
assertEqual
(
init_op
.
attr
(
'seed'
),
134
)
self
.
assertEqual
(
init_op
.
attr
(
'seed'
),
134
)
class
TestMSRAInitializer
(
unittest
.
TestCase
):
def
test_uniform_msra_initializer
(
self
):
"""Test MSRA initializer with uniform distribution on
for matrix multiply.
"""
program
=
framework
.
Program
()
block
=
program
.
global_block
()
param
=
block
.
create_parameter
(
dtype
=
"float32"
,
shape
=
[
5
,
10
],
lod_level
=
0
,
name
=
"param"
,
initializer
=
initializer
.
MSRAInitializer
())
self
.
assertEqual
(
len
(
block
.
ops
),
1
)
init_op
=
block
.
ops
[
0
]
self
.
assertEqual
(
init_op
.
type
,
'uniform_random'
)
limit
=
np
.
sqrt
(
6.0
/
param
.
shape
[
0
])
self
.
assertAlmostEqual
(
init_op
.
attr
(
'min'
),
-
limit
,
delta
=
DELTA
)
self
.
assertAlmostEqual
(
init_op
.
attr
(
'max'
),
limit
,
delta
=
DELTA
)
self
.
assertEqual
(
init_op
.
attr
(
'seed'
),
0
)
def
test_uniform_msra_initializer_conv
(
self
):
"""Test MSRA initializer with uniform distribution on
for convolutions.
"""
program
=
framework
.
Program
()
block
=
program
.
global_block
()
param
=
block
.
create_parameter
(
dtype
=
"float32"
,
shape
=
[
5
,
10
,
15
,
20
],
lod_level
=
0
,
name
=
"param"
,
initializer
=
initializer
.
MSRAInitializer
())
self
.
assertEqual
(
len
(
block
.
ops
),
1
)
init_op
=
block
.
ops
[
0
]
self
.
assertEqual
(
init_op
.
type
,
'uniform_random'
)
receptive_field_size
=
float
(
15
*
20
)
limit
=
np
.
sqrt
(
6.0
/
(
param
.
shape
[
1
]
*
receptive_field_size
))
self
.
assertAlmostEqual
(
init_op
.
attr
(
'min'
),
-
limit
,
delta
=
DELTA
)
self
.
assertAlmostEqual
(
init_op
.
attr
(
'max'
),
limit
,
delta
=
DELTA
)
self
.
assertEqual
(
init_op
.
attr
(
'seed'
),
0
)
def
test_normal_msra_initializer
(
self
):
"""Test MSRA initializer with normal distribution on
for matrix multiply.
"""
program
=
framework
.
Program
()
block
=
program
.
global_block
()
param
=
block
.
create_parameter
(
dtype
=
"float32"
,
shape
=
[
5
,
10
],
lod_level
=
0
,
name
=
"param"
,
initializer
=
initializer
.
MSRAInitializer
(
uniform
=
False
))
self
.
assertEqual
(
len
(
block
.
ops
),
1
)
init_op
=
block
.
ops
[
0
]
self
.
assertEqual
(
init_op
.
type
,
'gaussian_random'
)
std
=
np
.
sqrt
(
2.0
/
param
.
shape
[
0
])
self
.
assertAlmostEqual
(
init_op
.
attr
(
'mean'
),
0.0
,
delta
=
DELTA
)
self
.
assertAlmostEqual
(
init_op
.
attr
(
'std'
),
std
,
delta
=
DELTA
)
self
.
assertEqual
(
init_op
.
attr
(
'seed'
),
0
)
def
test_normal_msra_initializer_conv
(
self
):
"""Test MSRA initializer with normal distribution on
for convolutions.
"""
program
=
framework
.
Program
()
block
=
program
.
global_block
()
param
=
block
.
create_parameter
(
dtype
=
"float32"
,
shape
=
[
5
,
10
,
15
,
20
],
lod_level
=
0
,
name
=
"param"
,
initializer
=
initializer
.
MSRAInitializer
(
uniform
=
False
))
self
.
assertEqual
(
len
(
block
.
ops
),
1
)
init_op
=
block
.
ops
[
0
]
self
.
assertEqual
(
init_op
.
type
,
'gaussian_random'
)
receptive_field_size
=
float
(
15
*
20
)
std
=
np
.
sqrt
(
2.0
/
(
param
.
shape
[
1
]
*
receptive_field_size
))
self
.
assertAlmostEqual
(
init_op
.
attr
(
'mean'
),
0.0
,
delta
=
DELTA
)
self
.
assertAlmostEqual
(
init_op
.
attr
(
'std'
),
std
,
delta
=
DELTA
)
self
.
assertEqual
(
init_op
.
attr
(
'seed'
),
0
)
def
test_msra_initializer_supplied_arguments
(
self
):
"""Test the MSRA initializer with supplied arguments
"""
program
=
framework
.
Program
()
block
=
program
.
global_block
()
block
.
create_parameter
(
dtype
=
"float32"
,
shape
=
[
5
,
10
],
lod_level
=
0
,
name
=
"param"
,
initializer
=
initializer
.
MSRAInitializer
(
fan_in
=
12
,
seed
=
134
))
self
.
assertEqual
(
len
(
block
.
ops
),
1
)
init_op
=
block
.
ops
[
0
]
self
.
assertEqual
(
init_op
.
type
,
'uniform_random'
)
limit
=
np
.
sqrt
(
6.0
/
12
)
self
.
assertAlmostEqual
(
init_op
.
attr
(
'min'
),
-
limit
,
delta
=
DELTA
)
self
.
assertAlmostEqual
(
init_op
.
attr
(
'max'
),
limit
,
delta
=
DELTA
)
self
.
assertEqual
(
init_op
.
attr
(
'seed'
),
134
)
if
__name__
==
'__main__'
:
if
__name__
==
'__main__'
:
unittest
.
main
()
unittest
.
main
()
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录