diff --git a/paddle/fluid/framework/details/build_strategy.cc b/paddle/fluid/framework/details/build_strategy.cc
index cb660cb8c2c3906c023f68e66ca5538aa6f6897e..50426526023fd7604d01f33ec96e7cb0a875e911 100644
--- a/paddle/fluid/framework/details/build_strategy.cc
+++ b/paddle/fluid/framework/details/build_strategy.cc
@@ -153,6 +153,7 @@ std::unique_ptr<ir::Graph> BuildStrategy::Apply(
       pass->Erase("local_scopes");
       pass->SetNotOwned<const std::vector<Scope *>>("local_scopes",
                                                     &local_scopes);
+      pass->Erase("num_parallel_devices");
       pass->Set<size_t>("num_parallel_devices",
                         new size_t(num_parallel_devices));
 
diff --git a/python/paddle/fluid/tests/unittests/test_parallel_executor_crf.py b/python/paddle/fluid/tests/unittests/test_parallel_executor_crf.py
index 3e4490aa58e819e49bcc3c452bc1acd69137c76c..41286ba08cadbcd45b3f63b00ed076ccac18f8df 100644
--- a/python/paddle/fluid/tests/unittests/test_parallel_executor_crf.py
+++ b/python/paddle/fluid/tests/unittests/test_parallel_executor_crf.py
@@ -211,7 +211,9 @@ class TestCRFModel(unittest.TestCase):
                 use_cuda=True)
 
         self.check_network_convergence(
-            is_sparse=False, build_strategy=build_strategy, use_cuda=False)
+            is_sparse=False,
+            build_strategy=self._new_build_strategy(),
+            use_cuda=False)
 
     def test_update_sparse_parameter_reduce(self):
         if core.is_compiled_with_cuda():