Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
afeixing77
Paddle
提交
19639e31
P
Paddle
项目概览
afeixing77
/
Paddle
与 Fork 源项目一致
Fork自
PaddlePaddle / Paddle
通知
1
Star
0
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
P
Paddle
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
未验证
提交
19639e31
编写于
7月 24, 2018
作者:
F
fengjiayi
提交者:
GitHub
7月 24, 2018
浏览文件
操作
浏览文件
下载
差异文件
Merge pull request #12254 from JiayiFeng/fix_lr_decay
Fix learning rate scheduler performance issue
上级
7c046ae7
4cba5500
变更
2
隐藏空白更改
内联
并排
Showing
2 changed file
with
59 addition
and
65 deletion
+59
-65
python/paddle/fluid/layers/learning_rate_scheduler.py
python/paddle/fluid/layers/learning_rate_scheduler.py
+52
-59
python/paddle/fluid/tests/unittests/test_learning_rate_scheduler.py
...dle/fluid/tests/unittests/test_learning_rate_scheduler.py
+7
-6
未找到文件。
python/paddle/fluid/layers/learning_rate_scheduler.py
浏览文件 @
19639e31
...
@@ -62,10 +62,10 @@ def noam_decay(d_model, warmup_steps):
...
@@ -62,10 +62,10 @@ def noam_decay(d_model, warmup_steps):
The decayed learning rate.
The decayed learning rate.
"""
"""
global_step
=
_decay_step_counter
(
1
)
global_step
=
_decay_step_counter
(
1
)
with
init_on_cpu
():
a
=
global_step
**-
0.5
a
=
global_step
**-
0.5
b
=
(
warmup_steps
**-
1.5
)
*
global_step
b
=
(
warmup_steps
**-
1.5
)
*
global_step
lr_value
=
(
d_model
**-
0.5
)
*
ops
.
elementwise_min
(
a
,
b
)
lr_value
=
(
d_model
**-
0.5
)
*
ops
.
elementwise_min
(
a
,
b
)
return
lr_value
return
lr_value
...
@@ -108,12 +108,10 @@ def exponential_decay(learning_rate, decay_steps, decay_rate, staircase=False):
...
@@ -108,12 +108,10 @@ def exponential_decay(learning_rate, decay_steps, decay_rate, staircase=False):
"""
"""
global_step
=
_decay_step_counter
()
global_step
=
_decay_step_counter
()
with
init_on_cpu
():
div_res
=
global_step
/
decay_steps
# update learning_rate
if
staircase
:
div_res
=
global_step
/
decay_steps
div_res
=
ops
.
floor
(
div_res
)
if
staircase
:
decayed_lr
=
learning_rate
*
(
decay_rate
**
div_res
)
div_res
=
ops
.
floor
(
div_res
)
decayed_lr
=
learning_rate
*
(
decay_rate
**
div_res
)
return
decayed_lr
return
decayed_lr
...
@@ -138,11 +136,10 @@ def natural_exp_decay(learning_rate, decay_steps, decay_rate, staircase=False):
...
@@ -138,11 +136,10 @@ def natural_exp_decay(learning_rate, decay_steps, decay_rate, staircase=False):
"""
"""
global_step
=
_decay_step_counter
()
global_step
=
_decay_step_counter
()
with
init_on_cpu
():
div_res
=
global_step
/
decay_steps
div_res
=
global_step
/
decay_steps
if
staircase
:
if
staircase
:
div_res
=
ops
.
floor
(
div_res
)
div_res
=
ops
.
floor
(
div_res
)
decayed_lr
=
learning_rate
*
ops
.
exp
(
-
1
*
decay_rate
*
div_res
)
decayed_lr
=
learning_rate
*
ops
.
exp
(
-
1
*
decay_rate
*
div_res
)
return
decayed_lr
return
decayed_lr
...
@@ -184,12 +181,11 @@ def inverse_time_decay(learning_rate, decay_steps, decay_rate, staircase=False):
...
@@ -184,12 +181,11 @@ def inverse_time_decay(learning_rate, decay_steps, decay_rate, staircase=False):
"""
"""
global_step
=
_decay_step_counter
()
global_step
=
_decay_step_counter
()
with
init_on_cpu
():
div_res
=
global_step
/
decay_steps
div_res
=
global_step
/
decay_steps
if
staircase
:
if
staircase
:
div_res
=
ops
.
floor
(
div_res
)
div_res
=
ops
.
floor
(
div_res
)
decayed_lr
=
learning_rate
/
(
1
+
decay_rate
*
div_res
)
decayed_lr
=
learning_rate
/
(
1
+
decay_rate
*
div_res
)
return
decayed_lr
return
decayed_lr
...
@@ -224,25 +220,22 @@ def polynomial_decay(learning_rate,
...
@@ -224,25 +220,22 @@ def polynomial_decay(learning_rate,
"""
"""
global_step
=
_decay_step_counter
()
global_step
=
_decay_step_counter
()
with
init_on_cpu
():
if
cycle
:
if
cycle
:
div_res
=
ops
.
ceil
(
global_step
/
decay_steps
)
div_res
=
ops
.
ceil
(
global_step
/
decay_steps
)
zero_var
=
tensor
.
fill_constant
(
shape
=
[
1
],
dtype
=
'float32'
,
value
=
0.0
)
zero_var
=
tensor
.
fill_constant
(
one_var
=
tensor
.
fill_constant
(
shape
=
[
1
],
dtype
=
'float32'
,
value
=
1.0
)
shape
=
[
1
],
dtype
=
'float32'
,
value
=
0.0
)
one_var
=
tensor
.
fill_constant
(
shape
=
[
1
],
dtype
=
'float32'
,
value
=
1.0
)
with
control_flow
.
Switch
()
as
switch
:
with
switch
.
case
(
global_step
==
zero_var
):
tensor
.
assign
(
input
=
one_var
,
output
=
div_res
)
decay_steps
=
decay_steps
*
div_res
else
:
decay_steps_var
=
tensor
.
fill_constant
(
shape
=
[
1
],
dtype
=
'float32'
,
value
=
float
(
decay_steps
))
global_step
=
ops
.
elementwise_min
(
x
=
global_step
,
y
=
decay_steps_var
)
decayed_lr
=
(
learning_rate
-
end_learning_rate
)
*
\
with
control_flow
.
Switch
()
as
switch
:
((
1
-
global_step
/
decay_steps
)
**
power
)
+
end_learning_rate
with
switch
.
case
(
global_step
==
zero_var
):
tensor
.
assign
(
input
=
one_var
,
output
=
div_res
)
decay_steps
=
decay_steps
*
div_res
else
:
decay_steps_var
=
tensor
.
fill_constant
(
shape
=
[
1
],
dtype
=
'float32'
,
value
=
float
(
decay_steps
))
global_step
=
ops
.
elementwise_min
(
x
=
global_step
,
y
=
decay_steps_var
)
decayed_lr
=
(
learning_rate
-
end_learning_rate
)
*
\
((
1
-
global_step
/
decay_steps
)
**
power
)
+
end_learning_rate
return
decayed_lr
return
decayed_lr
...
@@ -277,28 +270,28 @@ def piecewise_decay(boundaries, values):
...
@@ -277,28 +270,28 @@ def piecewise_decay(boundaries, values):
global_step
=
_decay_step_counter
()
global_step
=
_decay_step_counter
()
with
init_on_cpu
():
lr
=
tensor
.
create_global_var
(
lr
=
tensor
.
create_global_var
(
shape
=
[
1
],
shape
=
[
1
],
value
=
0.0
,
value
=
0.0
,
dtype
=
'float32'
,
dtype
=
'float32'
,
persistable
=
True
,
persistable
=
True
,
name
=
"learning_rate"
)
name
=
"learning_rate"
)
with
control_flow
.
Switch
()
as
switch
:
with
control_flow
.
Switch
()
as
switch
:
for
i
in
range
(
len
(
boundaries
)):
for
i
in
range
(
len
(
boundaries
)):
boundary_val
=
tensor
.
fill_constant
(
boundary_val
=
tensor
.
fill_constant
(
shape
=
[
1
],
dtype
=
'float32'
,
value
=
float
(
boundaries
[
i
]))
value_var
=
tensor
.
fill_constant
(
shape
=
[
1
],
dtype
=
'float32'
,
value
=
float
(
values
[
i
]))
with
switch
.
case
(
global_step
<
boundary_val
):
tensor
.
assign
(
value_var
,
lr
)
last_value_var
=
tensor
.
fill_constant
(
shape
=
[
1
],
shape
=
[
1
],
dtype
=
'float32'
,
dtype
=
'float32'
,
value
=
float
(
values
[
len
(
values
)
-
1
]))
value
=
float
(
boundaries
[
i
]),
with
switch
.
default
():
force_cpu
=
True
)
tensor
.
assign
(
last_value_var
,
lr
)
value_var
=
tensor
.
fill_constant
(
shape
=
[
1
],
dtype
=
'float32'
,
value
=
float
(
values
[
i
]))
with
switch
.
case
(
global_step
<
boundary_val
):
tensor
.
assign
(
value_var
,
lr
)
last_value_var
=
tensor
.
fill_constant
(
shape
=
[
1
],
dtype
=
'float32'
,
value
=
float
(
values
[
len
(
values
)
-
1
]))
with
switch
.
default
():
tensor
.
assign
(
last_value_var
,
lr
)
return
lr
return
lr
...
@@ -333,9 +326,9 @@ def append_LARS(params_grads, learning_rate, weight_decay):
...
@@ -333,9 +326,9 @@ def append_LARS(params_grads, learning_rate, weight_decay):
grad_norm
=
ops
.
sqrt
(
nn
.
reduce_sum
(
input
=
ops
.
square
(
grad
)))
grad_norm
=
ops
.
sqrt
(
nn
.
reduce_sum
(
input
=
ops
.
square
(
grad
)))
if
type
(
param_lr
)
==
float
and
param_lr
==
1.0
:
if
type
(
param_lr
)
==
float
and
param_lr
==
1.0
:
decayed_lr
=
learning_rate
*
param_norm
\
decayed_lr
=
learning_rate
*
param_norm
\
/
_balanced_weight
(
param_norm
,
grad_norm
)
/
_balanced_weight
(
param_norm
,
grad_norm
)
else
:
else
:
decayed_lr
=
learning_rate
*
param_lr
*
param_norm
\
decayed_lr
=
learning_rate
*
param_lr
*
param_norm
\
/
_balanced_weight
(
param_norm
,
grad_norm
)
/
_balanced_weight
(
param_norm
,
grad_norm
)
# set back param local learning rate
# set back param local learning rate
param
.
optimize_attr
[
'learning_rate'
]
=
decayed_lr
param
.
optimize_attr
[
'learning_rate'
]
=
decayed_lr
python/paddle/fluid/tests/unittests/test_learning_rate_scheduler.py
浏览文件 @
19639e31
...
@@ -91,20 +91,21 @@ class TestLearningRateDecay(unittest.TestCase):
...
@@ -91,20 +91,21 @@ class TestLearningRateDecay(unittest.TestCase):
def
check_decay_with_place
(
self
,
place
,
python_decay_fn
,
fluid_decay_fn
,
def
check_decay_with_place
(
self
,
place
,
python_decay_fn
,
fluid_decay_fn
,
kwargs
):
kwargs
):
main_prog
=
fluid
.
Program
()
startup_prog
=
fluid
.
Program
()
decayed_lr
=
fluid_decay_fn
(
**
kwargs
)
with
fluid
.
program_guard
(
main_prog
,
startup_prog
):
decayed_lr
=
fluid_decay_fn
(
**
kwargs
)
place
=
fluid
.
CPUPlace
()
place
=
fluid
.
CPUPlace
()
exe
=
fluid
.
Executor
(
place
)
exe
=
fluid
.
Executor
(
place
)
exe
.
run
(
fluid
.
default_startup_program
()
)
exe
.
run
(
startup_prog
)
fluid
.
memory_optimize
(
fluid
.
default_main_program
()
)
fluid
.
memory_optimize
(
main_prog
)
for
step
in
range
(
10
):
for
step
in
range
(
10
):
lr_val
,
=
exe
.
run
(
fluid
.
default_main_program
(),
lr_val
,
=
exe
.
run
(
main_prog
,
feed
=
{},
fetch_list
=
[
decayed_lr
])
feed
=
{},
fetch_list
=
[
decayed_lr
])
python_decayed_lr
=
python_decay_fn
(
python_decayed_lr
=
python_decay_fn
(
global_step
=
float
(
step
),
**
kwargs
)
global_step
=
float
(
step
),
**
kwargs
)
self
.
assertAlmostEqual
(
self
.
assertAlmostEqual
(
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录