From bba88f6791aa8a4f2e1d3683541bb48f00eb4e6e Mon Sep 17 00:00:00 2001
From: minghaoBD <79566150+minghaoBD@users.noreply.github.com>
Date: Wed, 17 Nov 2021 17:52:16 +0800
Subject: [PATCH] fix warmup+decay LR (#4611)

---
 ppdet/optimizer.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/ppdet/optimizer.py b/ppdet/optimizer.py
index fcdcbd8d6..7fdcddd69 100644
--- a/ppdet/optimizer.py
+++ b/ppdet/optimizer.py
@@ -56,11 +56,13 @@ class CosineDecay(object):
         max_iters = self.max_epochs * int(step_per_epoch)
 
         if boundary is not None and value is not None and self.use_warmup:
+            warmup_iters = len(boundary)
             for i in range(int(boundary[-1]), max_iters):
                 boundary.append(i)
 
-                decayed_lr = base_lr * 0.5 * (
-                    math.cos(i * math.pi / max_iters) + 1)
+                decayed_lr = base_lr * 0.5 * (math.cos(
+                    (i - warmup_iters) * math.pi /
+                    (max_iters - warmup_iters)) + 1)
                 value.append(decayed_lr)
             return optimizer.lr.PiecewiseDecay(boundary, value)
 
-- 
GitLab