diff --git a/paddle/gserver/tests/LayerGradUtil.cpp b/paddle/gserver/tests/LayerGradUtil.cpp
index 66aafba844dd2a943410230335b207d300ccd24d..15b8cedeb83167417a6f6b529f29f1ff0bf37edd 100644
--- a/paddle/gserver/tests/LayerGradUtil.cpp
+++ b/paddle/gserver/tests/LayerGradUtil.cpp
@@ -498,12 +498,9 @@ void initTestLayer(TestConfig testConf,
         paraConfig.add_dims((*layerMap)[input.input_layer_name()]->getSize());
         paraConfig.add_dims(testConf.layerConfig.size());
       }
-      if (testConf.hasParamInitialValue) {
-        paraConfig.set_initial_mean(testConf.paramInitialMean);
-        paraConfig.set_initial_std(testConf.paramInitialStd);
-      } else {
-        paraConfig.set_initial_std(1);
-      }
+      CHECK_GE(testConf.paramInitialStd, 0);
+      paraConfig.set_initial_mean(testConf.paramInitialMean);
+      paraConfig.set_initial_std(testConf.paramInitialStd);
       initParameter(paraName, paraSize, inputDef.isStatic, false, paraConfig);
     }
   }
diff --git a/paddle/gserver/tests/LayerGradUtil.h b/paddle/gserver/tests/LayerGradUtil.h
index 9dbd2027578796972d56ee2ff485e91bf527fa1e..d299b4dd09418589514d99a72f83e1103ace7de1 100644
--- a/paddle/gserver/tests/LayerGradUtil.h
+++ b/paddle/gserver/tests/LayerGradUtil.h
@@ -127,7 +127,6 @@ struct TestConfig {
   size_t biasSize;
   real paramInitialMean;
   real paramInitialStd;
-  bool hasParamInitialValue;
   bool testAccumulate;
   bool testState;
   bool staticBias;
@@ -136,7 +135,6 @@ struct TestConfig {
       : biasSize(0),
         paramInitialMean(0.0),
         paramInitialStd(1.0),
-        hasParamInitialValue(false),
         testAccumulate(true),
         testState(false),
         staticBias(false),
diff --git a/paddle/gserver/tests/test_LayerGrad.cpp b/paddle/gserver/tests/test_LayerGrad.cpp
index 6441e08b48d2a339766540e74f905d63121fcfe4..bf0136a10f144abbd25fd6c032d7b17defd57b4b 100644
--- a/paddle/gserver/tests/test_LayerGrad.cpp
+++ b/paddle/gserver/tests/test_LayerGrad.cpp
@@ -1661,7 +1661,6 @@ TEST(Layer, PadLayer) {
 
 TEST(Layer, CrossChannelNormLayer) {
   TestConfig config;
-  config.hasParamInitialValue = true;
   config.paramInitialMean = 1.;
   config.paramInitialStd = 0.;
   config.layerConfig.set_type("norm");