diff --git a/python/paddle/fluid/optimizer.py b/python/paddle/fluid/optimizer.py
index 65fef50ead6f3da7abde844747e4e81f45c96207..214f47afa1e4e29d53b11ccc035b4d840f963591 100644
--- a/python/paddle/fluid/optimizer.py
+++ b/python/paddle/fluid/optimizer.py
@@ -1158,7 +1158,8 @@ class ModelAverage(Optimizer):
                 self.params_grads.append((param, grad))
 
         for param, grad in self.params_grads:
-            self._append_average_accumulate_op(param)
+            with param.block.program.optimized_guard(param):
+                self._append_average_accumulate_op(param)
 
         self.apply_program = Program()
         block = self.apply_program.global_block()