diff --git a/paddle/gserver/layers/CostLayer.cpp b/paddle/gserver/layers/CostLayer.cpp
index 69d5830dd2a1afb93948eacec1cb4309cf8c6109..6bfdea3c6e3f7cb80b620564f8229d954d773f04 100644
--- a/paddle/gserver/layers/CostLayer.cpp
+++ b/paddle/gserver/layers/CostLayer.cpp
@@ -217,10 +217,10 @@ void SmoothL1CostLayer::forwardImp(Matrix& output,
     targetCpu->copyFrom(target);
     outputCpu->copyFrom(output);
     labelCpu->copyFrom(*label.value);
-    targetCpu->smoothL1(*outputCpu, *labelCpu);
+    targetCpu->smoothL1(*outputCpu, *labelCpu, 1.0);
     target.copyFrom(*targetCpu);
   } else {
-    target.smoothL1(output, *label.value);
+    target.smoothL1(output, *label.value, 1.0);
   }
 }
 
@@ -238,10 +238,10 @@ void SmoothL1CostLayer::backwardImp(Matrix& output,
     outputGCpu->copyFrom(outputG);
     outputCpu->copyFrom(output);
     labelCpu->copyFrom(*label.value);
-    outputGCpu->smoothL1Bp(*outputCpu, *labelCpu);
+    outputGCpu->smoothL1Bp(*outputCpu, *labelCpu, 1.0);
     outputG.copyFrom(*outputGCpu);
   } else {
-    outputG.smoothL1Bp(output, *label.value);
+    outputG.smoothL1Bp(output, *label.value, 1.0);
   }
 }
 
diff --git a/paddle/math/Matrix.cpp b/paddle/math/Matrix.cpp
index 6ac61be0bf1b7a4e308705617faf5af2886a4082..c910146164ebfb0737583c72c48ce6dbc5b49939 100644
--- a/paddle/math/Matrix.cpp
+++ b/paddle/math/Matrix.cpp
@@ -3606,7 +3606,7 @@ void CpuMatrix::sumOfSquaresBp(Matrix& output, Matrix& label) {
   }
 }
 
-void CpuMatrix::smoothL1(Matrix& output, Matrix& label) {
+void CpuMatrix::smoothL1(Matrix& output, Matrix& label, real destScale) {
   CHECK(output.useGpu_ == false && label.useGpu_ == false)
       << "Matrix type are not equal";
 
@@ -3624,6 +3624,7 @@ void CpuMatrix::smoothL1(Matrix& output, Matrix& label) {
   for (size_t i = 0; i < numSamples; ++i, out += dim, lbl += dim) {
     for (size_t j = 0; j < dim; ++j) {
       real absVal = std::fabs(out[j] - lbl[j]);
+      cost[i] *= destScale;
       if (absVal < 1.0)
         cost[i] += 0.5 * absVal * absVal;
       else
@@ -3632,7 +3633,7 @@ void CpuMatrix::smoothL1(Matrix& output, Matrix& label) {
   }
 }
 
-void CpuMatrix::smoothL1Bp(Matrix& output, Matrix& label) {
+void CpuMatrix::smoothL1Bp(Matrix& output, Matrix& label, real destScale) {
   CHECK(output.useGpu_ == false && label.useGpu_ == false)
       << "Matrix type are not equal";
 
@@ -3650,6 +3651,7 @@ void CpuMatrix::smoothL1Bp(Matrix& output, Matrix& label) {
   for (size_t i = 0; i < numSamples; ++i, out += dim, grad += dim, lbl += dim) {
     for (size_t j = 0; j < dim; ++j) {
       real val = out[j] - lbl[j];
+      grad[j] *= destScale;
       if (std::fabs(val) < 1) {
         grad[j] += val;
       } else {
diff --git a/paddle/math/Matrix.h b/paddle/math/Matrix.h
index 3252adb19e4c2e48f86c3c811bfc7d75fd06a8f7..748be850b4c902d1b48c1dafbb0d5ea2bf197e6e 100644
--- a/paddle/math/Matrix.h
+++ b/paddle/math/Matrix.h
@@ -789,11 +789,11 @@ public:
     LOG(FATAL) << "Not implemented";
   }
 
-  virtual void smoothL1(Matrix& output, Matrix& label) {
+  virtual void smoothL1(Matrix& output, Matrix& label, real destScale) {
     LOG(FATAL) << "Not implemented";
   }
 
-  virtual void smoothL1Bp(Matrix& outputV, Matrix& label) {
+  virtual void smoothL1Bp(Matrix& outputV, Matrix& label, real destScale) {
     LOG(FATAL) << "Not implemented";
   }
 
@@ -1736,8 +1736,8 @@ public:
   /// gradient of sumOfSquares.
   void sumOfSquaresBp(Matrix& outputV, Matrix& label);
 
-  void smoothL1(Matrix& output, Matrix& label);
-  void smoothL1Bp(Matrix& output, Matrix& label);
+  void smoothL1(Matrix& output, Matrix& label, real destScale);
+  void smoothL1Bp(Matrix& output, Matrix& label, real destScale);
 
   void tanh(Matrix& output);
   void tanhDerivative(Matrix& output);