diff --git a/paddle/operators/cross_entropy_op.cu b/paddle/operators/cross_entropy_op.cu
index 2392c3d5ed98a0c406b9d9d6807a1578828f9e0f..5f5d2692670b80de9af2a0e8c679f42e35a27426 100644
--- a/paddle/operators/cross_entropy_op.cu
+++ b/paddle/operators/cross_entropy_op.cu
@@ -20,6 +20,21 @@ namespace operators {
 
 using Tensor = framework::Tensor;
 
+template <typename T>
+struct clipping_log {
+  __host__ __device__ T operator()(const T x) {
+    PADDLE_ASSERT(std::is_floating_point<T>::value);
+    const T kApproInf = 1e20;
+    if (x == INFINITY) {
+      return kApproInf;
+    }
+    if (x == -INFINITY) {
+      return -kApproInf;
+    }
+    return x;
+  }
+};
+
 template <typename T>
 __global__ void CrossEntropyKernel(T* Y, const T* X, const int* label,
                                    const int N, const int D) {
@@ -28,10 +43,11 @@ __global__ void CrossEntropyKernel(T* Y, const T* X, const int* label,
   for (int i = blockIdx.x * blockDim.x + threadIdx.x; i < N;
        i += blockDim.x * gridDim.x) {
     PADDLE_ASSERT(label[i] >= 0 && label[i] < D);
-    Y[i] = -log(X[i * D + label[i]]);
+    Y[i] = -clipping_log<T>()(X[i * D + label[i]]);
   }
 }
 
+// TODO(qingqing): make zero setting an common function.
 template <typename T>
 __global__ void zero(T* X, const int N) {
   for (int i = blockIdx.x * blockDim.x + threadIdx.x; i < N;
@@ -98,7 +114,6 @@ class OnehotCrossEntropyGradientOpCUDAKernel : public framework::OpKernel {
     int D = X->dims()[1];
     int block = 512;
     int grid = (N * D + block - 1) / block;
-    // TODO(qingqing): make zero an common function.
     zero<T><<<grid, block>>>(dXdata, N * D);
 
     grid = (N + block - 1) / block;
diff --git a/paddle/operators/cross_entropy_op.h b/paddle/operators/cross_entropy_op.h
index 261cbe2d423ab5fcb7bbd75ceb1c397035a1b2bc..e95f5e11678e2bc5506c7bc52b9ad3dcbb6bba79 100644
--- a/paddle/operators/cross_entropy_op.h
+++ b/paddle/operators/cross_entropy_op.h
@@ -21,7 +21,7 @@ namespace operators {
 using Tensor = framework::Tensor;
 
 template <typename T>
-T tolerable_value(T x) {
+T tolerable_value(const T x) {
   static_assert(std::is_floating_point<T>::value,
                 "tolerable_value works only on float, "
                 "double and double double.");
@@ -85,6 +85,7 @@ class OnehotCrossEntropyGradientOpKernel : public framework::OpKernel {
     const int batch_size = X->dims()[0];
     const int class_num = X->dims()[1];
 
+    // TODO(qingqing): make zero setting an common function.
     memset(dXdata, 0, sizeof(T) * batch_size * class_num);
     for (int i = 0; i < batch_size; ++i) {
       int index = i * class_num + label_data[i];
diff --git a/python/paddle/v2/framework/tests/op_test_util.py b/python/paddle/v2/framework/tests/op_test_util.py
index dd65e0f2dc23d3f657ff16c55fb297dae210b2d7..ae23108dfa4461f3cb11f077277246716f51d6d7 100644
--- a/python/paddle/v2/framework/tests/op_test_util.py
+++ b/python/paddle/v2/framework/tests/op_test_util.py
@@ -64,7 +64,8 @@ class OpTestMeta(type):
                     actual = numpy.array(scope.find_var(out_name).get_tensor())
                     expect = self.outputs[out_name]
                     self.assertTrue(
-                        numpy.allclose(actual, expect),
+                        numpy.allclose(
+                            actual, expect, atol=1e-04),
                         "output name: " + out_name + "has diff")
 
         obj.test_all = test_all
diff --git a/python/paddle/v2/framework/tests/test_cross_entropy_op.py b/python/paddle/v2/framework/tests/test_cross_entropy_op.py
index 5557e0d35820dffec9597596cfbff6ed53b7d550..d4277f2a42ce2e66e37405ccd3b2ee444d403d1a 100644
--- a/python/paddle/v2/framework/tests/test_cross_entropy_op.py
+++ b/python/paddle/v2/framework/tests/test_cross_entropy_op.py
@@ -8,9 +8,8 @@ class TestCrossEntropy(unittest.TestCase):
     __metaclass__ = OpTestMeta
 
     def setUp(self):
-        # TODO this unit test is not passed
         self.type = "onehot_cross_entropy"
-        batch_size = 100
+        batch_size = 30
         class_num = 10
         X = numpy.random.random((batch_size, class_num)).astype("float32")
         label = 5 * numpy.ones(batch_size).astype("int32")
@@ -24,7 +23,7 @@ class TestCrossEntropy(unittest.TestCase):
 class CrossEntropyGradOpTest(GradientChecker):
     def test_check_grad(self):
         op = create_op("onehot_cross_entropy")
-        batch_size = 100
+        batch_size = 30
         class_num = 10
         inputs = {
             "X": numpy.random.uniform(