From f97029220ecd5721a6241ff1eaf7f88f609e1e69 Mon Sep 17 00:00:00 2001 From: littletomatodonkey Date: Tue, 24 Mar 2020 10:34:49 +0800 Subject: [PATCH] add cosine decay strategy (#387) Add cosine decay lr strategy. --- ppdet/optimizer.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/ppdet/optimizer.py b/ppdet/optimizer.py index e695aad04..e7a6eb8ab 100644 --- a/ppdet/optimizer.py +++ b/ppdet/optimizer.py @@ -58,6 +58,26 @@ class PiecewiseDecay(object): return fluid.layers.piecewise_decay(self.milestones, values) +@serializable +class CosineDecay(object): + """ + Cosine learning rate decay + + Args: + max_iters (float): max iterations for the training process. + if you commbine cosine decay with warmup, it is recommended that + the max_iter is much larger than the warmup iter + """ + + def __init__(self, max_iters=180000): + self.max_iters = max_iters + + def __call__(self, base_lr=None, learning_rate=None): + assert base_lr is not None, "either base LR or values should be provided" + lr = fluid.layers.cosine_decay(base_lr, 1, self.max_iters) + return lr + + @serializable class LinearWarmup(object): """ -- GitLab