diff --git a/paddle/fluid/operators/math/cpu_vec.h b/paddle/fluid/operators/math/cpu_vec.h
index e74e84055a9f297081e5853d7a70abaeb7594c3f..a2e2b5a7fed03a480ce81fba03806560824f8ef3 100644
--- a/paddle/fluid/operators/math/cpu_vec.h
+++ b/paddle/fluid/operators/math/cpu_vec.h
@@ -82,7 +82,7 @@ inline void vec_tanh(const int n, const T* x, T* y) {
   for (int i = 0; i < n; ++i) {
     y[i] = static_cast<T>(2) * x[i];
   }
-  vec_exp<T>(n, y, y);
+  vec_sigmoid<T>(n, y, y);
   for (int i = 0; i < n; ++i) {
     y[i] = static_cast<T>(2) * y[i] - static_cast<T>(1);
   }
diff --git a/paddle/fluid/operators/math/cpu_vec_test.cc b/paddle/fluid/operators/math/cpu_vec_test.cc
index ab4858984d233977c462170b909e2a028ce5a1fb..0888e44fa655f1e8f9dc774bb71efcf3e234c142 100644
--- a/paddle/fluid/operators/math/cpu_vec_test.cc
+++ b/paddle/fluid/operators/math/cpu_vec_test.cc
@@ -68,7 +68,7 @@ void RandomVec(const int n, T* a) {
   std::mt19937 rng(seed++);
   std::uniform_real_distribution<double> uniform_dist(0, 1);
   const T lower = static_cast<T>(-20.f);
-  const T upper = static_cast<T>(-20.f);
+  const T upper = static_cast<T>(20.f);
   for (int i = 0; i < n; ++i) {
     a[i] = static_cast<T>(uniform_dist(rng) * (upper - lower) + lower);
   }