diff --git a/paddle/phi/kernels/xpu/pool_grad_kernel.cc b/paddle/phi/kernels/xpu/pool_grad_kernel.cc
index 4eeef936216effd2a71a60ee514a4c39d3a05ebd..b03be5dd9449cb42892cfd27ff73864cb8afc81f 100644
--- a/paddle/phi/kernels/xpu/pool_grad_kernel.cc
+++ b/paddle/phi/kernels/xpu/pool_grad_kernel.cc
@@ -112,30 +112,6 @@ void Pool2dGradKernel(const Context& ctx,
           true);
 
     } else if (pooling_type == "avg") {
-      // When output dim is 1 * 1 (1 * 1 * 1 in pool_3d), use scale
-      // and broadcast kernels to get same output, but better performance.
-      if (out_h == 1 && out_w == 1 && std::is_same<T, float>::value) {
-        xpu::ctx_guard RAII_GUARD(ctx.x_context());
-        float scale = 1.0 / (in_h * in_w);
-        float* scaled_dy = RAII_GUARD.alloc_l3_or_gm<float>(n * c);
-        r = xpu::scale(ctx.x_context(),
-                       dout.data<float>(),
-                       scaled_dy,
-                       n * c,
-                       true,
-                       scale,
-                       0.0);
-        PADDLE_ENFORCE_XDNN_SUCCESS(r, "scale");
-
-        r = xpu::broadcast(ctx.x_context(),
-                           scaled_dy,
-                           dx->data<float>(),
-                           {n, c, 1, 1},
-                           {n, c, in_h, in_w});
-        PADDLE_ENFORCE_XDNN_SUCCESS(r, "broadcast");
-
-        return;
-      }
       r = xpu::adaptive_avg_pool2d_grad<XPUType>(
           ctx.x_context(),
           reinterpret_cast<const XPUType*>(dout.data<T>()),