From b0149dc35724ecd6e3d5242e5a69a1c4a02afabe Mon Sep 17 00:00:00 2001 From: zhangxuefei Date: Thu, 18 Apr 2019 18:48:48 +0800 Subject: [PATCH] Modify the linear_warmup_decay to linear_decay --- demo/sequence-labeling/sequence_label.py | 2 +- demo/text-classification/README.md | 4 ++-- demo/text-classification/text_classifier.py | 2 +- docs/API/Strategy.md | 4 ++-- paddlehub/finetune/optimization.py | 6 +++--- paddlehub/finetune/strategy.py | 4 ++-- 6 files changed, 11 insertions(+), 11 deletions(-) diff --git a/demo/sequence-labeling/sequence_label.py b/demo/sequence-labeling/sequence_label.py index 1b59b054..ebbf281c 100644 --- a/demo/sequence-labeling/sequence_label.py +++ b/demo/sequence-labeling/sequence_label.py @@ -69,7 +69,7 @@ if __name__ == '__main__': strategy = hub.AdamWeightDecayStrategy( weight_decay=args.weight_decay, learning_rate=args.learning_rate, - lr_scheduler="linear_warmup_decay", + lr_scheduler="linear_decay", ) # Setup runing config for PaddleHub Finetune API diff --git a/demo/text-classification/README.md b/demo/text-classification/README.md index 84d6d427..63e11147 100644 --- a/demo/text-classification/README.md +++ b/demo/text-classification/README.md @@ -111,7 +111,7 @@ strategy = hub.AdamWeightDecayStrategy( learning_rate=5e-5, weight_decay=0.01, warmup_proportion=0.0, - lr_scheduler="linear_warmup_decay", + lr_scheduler="linear_decay", ) config = hub.RunConfig(use_cuda=True, num_epoch=3, batch_size=32, strategy=strategy) @@ -124,7 +124,7 @@ hub.finetune_and_eval(task=cls_task, data_reader=reader, feed_list=feed_list, co `learning_rate`: Finetune过程中的最大学习率; `weight_decay`: 模型的正则项参数，默认0.01，如果模型有过拟合倾向，可适当调高这一参数; `warmup_proportion`: 如果warmup_proportion>0, 例如0.1, 则学习率会在前10%的steps中线性增长至最高值learning_rate; -`lr_scheduler`: 有两种策略可选(1) `linear_warmup_decay`策略学习率会在最高点后以线性方式衰减; `noam_decay`策略学习率会在最高点以多项式形式衰减； +`lr_scheduler`: 有两种策略可选(1) `linear_decay`策略学习率会在最高点后以线性方式衰减; `noam_decay`策略学习率会在最高点以多项式形式衰减； #### 运行配置 `RunConfig` 主要控制Finetune的训练，包含以下可控制的参数: diff --git a/demo/text-classification/text_classifier.py b/demo/text-classification/text_classifier.py index ba52f09d..0e379d66 100644 --- a/demo/text-classification/text_classifier.py +++ b/demo/text-classification/text_classifier.py @@ -78,7 +78,7 @@ if __name__ == '__main__': strategy = hub.AdamWeightDecayStrategy( weight_decay=args.weight_decay, learning_rate=args.learning_rate, - lr_scheduler="linear_warmup_decay", + lr_scheduler="linear_decay", ) # Setup runing config for PaddleHub Finetune API diff --git a/docs/API/Strategy.md b/docs/API/Strategy.md index 2408dff6..b91b6d60 100644 --- a/docs/API/Strategy.md +++ b/docs/API/Strategy.md @@ -3,13 +3,13 @@ ---- 在PaddleHub中，Strategy代表了在对[Task](https://github.com/PaddlePaddle/PaddleHub/tree/develop/docs/API/Task.md)进行Finetune时，应该使用怎样的策略。这里的策略，包含了对预训练参数使用怎样的学习率，使用哪种类型的优化器，使用什么类型的正则化等 -## `class paddlehub.finetune.strategy.AdamWeightDecayStrategy(learning_rate=1e-4, lr_scheduler="linear_warmup_decay", warmup_proportion=0.0, weight_decay=0.01, optimizer_name=None)` +## `class paddlehub.finetune.strategy.AdamWeightDecayStrategy(learning_rate=1e-4, lr_scheduler="linear_decay", warmup_proportion=0.0, weight_decay=0.01, optimizer_name=None)` 基于Adam优化器的学习率衰减策略 > ### 参数 > * learning_rate: 全局学习率。默认为1e-4 > -> * lr_scheduler: 学习率调度方法。默认为"linear_warmup_decay" +> * lr_scheduler: 学习率调度方法。默认为"linear_decay" > > * warmup_proportion: warmup所占比重 > diff --git a/paddlehub/finetune/optimization.py b/paddlehub/finetune/optimization.py index e19e4210..d438660c 100644 --- a/paddlehub/finetune/optimization.py +++ b/paddlehub/finetune/optimization.py @@ -27,18 +27,18 @@ def adam_weight_decay_optimization(loss, learning_rate, main_program, weight_decay, - scheduler='linear_warmup_decay'): + scheduler='linear_decay'): if warmup_steps > 0: if scheduler == 'noam_decay': scheduled_lr = fluid.layers.learning_rate_scheduler\ .noam_decay(1/(warmup_steps *(learning_rate ** 2)), warmup_steps) - elif scheduler == 'linear_warmup_decay': + elif scheduler == 'linear_decay': scheduled_lr = linear_warmup_decay(learning_rate, warmup_steps, num_train_steps) else: raise ValueError("Unkown learning rate scheduler, should be " - "'noam_decay' or 'linear_warmup_decay'") + "'noam_decay' or 'linear_decay'") optimizer = fluid.optimizer.Adam(learning_rate=scheduled_lr) else: optimizer = fluid.optimizer.Adam(learning_rate=learning_rate) diff --git a/paddlehub/finetune/strategy.py b/paddlehub/finetune/strategy.py index a6dc7afc..453f8f5b 100644 --- a/paddlehub/finetune/strategy.py +++ b/paddlehub/finetune/strategy.py @@ -64,14 +64,14 @@ class DefaultStrategy(object): class AdamWeightDecayStrategy(DefaultStrategy): def __init__(self, learning_rate=1e-4, - lr_scheduler="linear_warmup_decay", + lr_scheduler="linear_decay", warmup_proportion=0.0, weight_decay=0.01, optimizer_name=None): super().__init__( learning_rate=learning_rate, optimizer_name=optimizer_name) # check strategy correctness - if lr_scheduler not in ["linear_warmup_decay", "noam_decay"]: + if lr_scheduler not in ["linear_decay", "noam_decay"]: raise ValueError("lr_scheduler {} is not setup " "correctly".format(lr_scheduler)) self._lr_scheduler = lr_scheduler -- GitLab