diff --git a/paddle/fluid/operators/clip_by_norm_op.h b/paddle/fluid/operators/clip_by_norm_op.h
index a8d1e8e4661bff87da4c982d638fe56ef8e56ec7..fb21d9fec90cab167ebdd3c271c2506e8ea6faa9 100644
--- a/paddle/fluid/operators/clip_by_norm_op.h
+++ b/paddle/fluid/operators/clip_by_norm_op.h
@@ -81,7 +81,12 @@ class ClipByNormKernel : public framework::OpKernel<T> {
         *context.template device_context<DeviceContext>().eigen_device();
 
     auto temp = (x_norm <= max_norm).template cast<T>();
-    auto scaling = temp + (static_cast<T>(1) - temp) * max_norm / x_norm;
+    auto epsilon =
+        ((x_norm <= static_cast<T>(1e-30)).all().template cast<T>()) *
+        static_cast<T>(1e-6);
+
+    auto scaling =
+        temp + (static_cast<T>(1) - temp) * max_norm / (x_norm + epsilon);
     Eigen::array<int, 1> one_dim{{1}};
     Eigen::DSizes<int, 1> m_dsize(input->numel());
     if (context.GetPlace() == platform::CPUPlace()) {