diff --git a/demo/sequence-labeling/sequence_label.py b/demo/sequence-labeling/sequence_label.py index 1b59b05454ea4fad03573200b768de0ac65ea0a1..ebbf281cd5f65341eafd22ffed3eb1d0d3693aae 100644 --- a/demo/sequence-labeling/sequence_label.py +++ b/demo/sequence-labeling/sequence_label.py @@ -69,7 +69,7 @@ if __name__ == '__main__': strategy = hub.AdamWeightDecayStrategy( weight_decay=args.weight_decay, learning_rate=args.learning_rate, - lr_scheduler="linear_warmup_decay", + lr_scheduler="linear_decay", ) # Setup runing config for PaddleHub Finetune API diff --git a/demo/text-classification/README.md b/demo/text-classification/README.md index 84d6d42787e153745f0e9bbb25b7cb88cff47011..63e111472a1f4fbae88bba48f8ae3d11d8e97356 100644 --- a/demo/text-classification/README.md +++ b/demo/text-classification/README.md @@ -111,7 +111,7 @@ strategy = hub.AdamWeightDecayStrategy( learning_rate=5e-5, weight_decay=0.01, warmup_proportion=0.0, - lr_scheduler="linear_warmup_decay", + lr_scheduler="linear_decay", ) config = hub.RunConfig(use_cuda=True, num_epoch=3, batch_size=32, strategy=strategy) @@ -124,7 +124,7 @@ hub.finetune_and_eval(task=cls_task, data_reader=reader, feed_list=feed_list, co `learning_rate`: Finetune过程中的最大学习率; `weight_decay`: 模型的正则项参数，默认0.01，如果模型有过拟合倾向，可适当调高这一参数; `warmup_proportion`: 如果warmup_proportion>0, 例如0.1, 则学习率会在前10%的steps中线性增长至最高值learning_rate; -`lr_scheduler`: 有两种策略可选(1) `linear_warmup_decay`策略学习率会在最高点后以线性方式衰减; `noam_decay`策略学习率会在最高点以多项式形式衰减； +`lr_scheduler`: 有两种策略可选(1) `linear_decay`策略学习率会在最高点后以线性方式衰减; `noam_decay`策略学习率会在最高点以多项式形式衰减； #### 运行配置 `RunConfig` 主要控制Finetune的训练，包含以下可控制的参数: diff --git a/demo/text-classification/text_classifier.py b/demo/text-classification/text_classifier.py index ba52f09df6fd6ac23f548b9e66b609c58dc54891..0e379d663cec8ade11c2fd3fded8fd1eccd2317d 100644 --- a/demo/text-classification/text_classifier.py +++ b/demo/text-classification/text_classifier.py @@ -78,7 +78,7 @@ if __name__ == '__main__': strategy = hub.AdamWeightDecayStrategy( weight_decay=args.weight_decay, learning_rate=args.learning_rate, - lr_scheduler="linear_warmup_decay", + lr_scheduler="linear_decay", ) # Setup runing config for PaddleHub Finetune API diff --git a/docs/API/Strategy.md b/docs/API/Strategy.md index 2408dff684ef0112dd119eed01de15dd71ebd5ab..b91b6d6044cf9653d1995cd0462d6d58ebc1c991 100644 --- a/docs/API/Strategy.md +++ b/docs/API/Strategy.md @@ -3,13 +3,13 @@ ---- 在PaddleHub中，Strategy代表了在对[Task](https://github.com/PaddlePaddle/PaddleHub/tree/develop/docs/API/Task.md)进行Finetune时，应该使用怎样的策略。这里的策略，包含了对预训练参数使用怎样的学习率，使用哪种类型的优化器，使用什么类型的正则化等 -## `class paddlehub.finetune.strategy.AdamWeightDecayStrategy(learning_rate=1e-4, lr_scheduler="linear_warmup_decay", warmup_proportion=0.0, weight_decay=0.01, optimizer_name=None)` +## `class paddlehub.finetune.strategy.AdamWeightDecayStrategy(learning_rate=1e-4, lr_scheduler="linear_decay", warmup_proportion=0.0, weight_decay=0.01, optimizer_name=None)` 基于Adam优化器的学习率衰减策略 > ### 参数 > * learning_rate: 全局学习率。默认为1e-4 > -> * lr_scheduler: 学习率调度方法。默认为"linear_warmup_decay" +> * lr_scheduler: 学习率调度方法。默认为"linear_decay" > > * warmup_proportion: warmup所占比重 > diff --git a/paddlehub/finetune/optimization.py b/paddlehub/finetune/optimization.py index e19e421022dff2a28a799d1a1edec765911900e3..d438660cd79db8d8d78f972390904891607e075c 100644 --- a/paddlehub/finetune/optimization.py +++ b/paddlehub/finetune/optimization.py @@ -27,18 +27,18 @@ def adam_weight_decay_optimization(loss, learning_rate, main_program, weight_decay, - scheduler='linear_warmup_decay'): + scheduler='linear_decay'): if warmup_steps > 0: if scheduler == 'noam_decay': scheduled_lr = fluid.layers.learning_rate_scheduler\ .noam_decay(1/(warmup_steps *(learning_rate ** 2)), warmup_steps) - elif scheduler == 'linear_warmup_decay': + elif scheduler == 'linear_decay': scheduled_lr = linear_warmup_decay(learning_rate, warmup_steps, num_train_steps) else: raise ValueError("Unkown learning rate scheduler, should be " - "'noam_decay' or 'linear_warmup_decay'") + "'noam_decay' or 'linear_decay'") optimizer = fluid.optimizer.Adam(learning_rate=scheduled_lr) else: optimizer = fluid.optimizer.Adam(learning_rate=learning_rate) diff --git a/paddlehub/finetune/strategy.py b/paddlehub/finetune/strategy.py index 4015d16fda3150d973b8b1ead12fe572814fa6b9..c01ba5fe5515bcd476de4e001a40c67ee5d99b98 100644 --- a/paddlehub/finetune/strategy.py +++ b/paddlehub/finetune/strategy.py @@ -88,14 +88,14 @@ class DefaultStrategy(object): class AdamWeightDecayStrategy(DefaultStrategy): def __init__(self, learning_rate=1e-4, - lr_scheduler="linear_warmup_decay", + lr_scheduler="linear_decay", warmup_proportion=0.0, weight_decay=0.01, optimizer_name=None): super().__init__( learning_rate=learning_rate, optimizer_name=optimizer_name) # check strategy correctness - if lr_scheduler not in ["linear_warmup_decay", "noam_decay"]: + if lr_scheduler not in ["linear_decay", "noam_decay"]: raise ValueError("lr_scheduler {} is not setup " "correctly".format(lr_scheduler)) self._lr_scheduler = lr_scheduler