diff --git a/paddle/fluid/framework/parallel_executor.cc b/paddle/fluid/framework/parallel_executor.cc
index 25b31f8636136e4b787e7e3892498fc52246cb1f..ea5ce3f2e9c8cc70e02788e8a84d29d266ded8c9 100644
--- a/paddle/fluid/framework/parallel_executor.cc
+++ b/paddle/fluid/framework/parallel_executor.cc
@@ -502,6 +502,7 @@ void ParallelExecutor::BCastParamsToGPUs(
     }
   }
 
+  // Debug code, bias should be 1.0f.
   for (auto &pair : member_->local_scopes_) {
     member_->GetNCCLCtx(pair.first).ctx_->Wait();
 
diff --git a/python/paddle/fluid/tests/unittests/test_parallel_executor.py b/python/paddle/fluid/tests/unittests/test_parallel_executor.py
index 85a9f7697fa03840fed2ec9c1cacfc2c246e0b45..2a614700b0a5b60d052b6085df6399f2edb9ba30 100644
--- a/python/paddle/fluid/tests/unittests/test_parallel_executor.py
+++ b/python/paddle/fluid/tests/unittests/test_parallel_executor.py
@@ -45,7 +45,12 @@ class ParallelExecutor(unittest.TestCase):
                 lod_levels=[0, 0],
                 dtypes=['float32', 'int64'])
             img, label = fluid.layers.read_file(reader)
-            hidden = fluid.layers.fc(img, size=200, act='tanh')
+            hidden = fluid.layers.fc(
+                img,
+                size=200,
+                act='tanh',
+                bias_attr=fluid.ParamAttr(
+                    initializer=fluid.initializer.Constant(value=1.0)))
             prediction = fluid.layers.fc(hidden, size=10, act='softmax')
             loss = fluid.layers.cross_entropy(input=prediction, label=label)
             loss = fluid.layers.mean(loss)