diff --git a/ppdet/optimizer.py b/ppdet/optimizer.py
index fcdcbd8d64475949e211fa09f3c4baffc16abf9a..7fdcddd69311e2ba94b9ae7beff88ac53d96b93e 100644
--- a/ppdet/optimizer.py
+++ b/ppdet/optimizer.py
@@ -56,11 +56,13 @@ class CosineDecay(object):
         max_iters = self.max_epochs * int(step_per_epoch)
 
         if boundary is not None and value is not None and self.use_warmup:
+            warmup_iters = len(boundary)
             for i in range(int(boundary[-1]), max_iters):
                 boundary.append(i)
 
-                decayed_lr = base_lr * 0.5 * (
-                    math.cos(i * math.pi / max_iters) + 1)
+                decayed_lr = base_lr * 0.5 * (math.cos(
+                    (i - warmup_iters) * math.pi /
+                    (max_iters - warmup_iters)) + 1)
                 value.append(decayed_lr)
             return optimizer.lr.PiecewiseDecay(boundary, value)