replace spatial with per_activation mode for bn op to improve perf (#33887)

paddle/fluid/operators/batch_norm_op.cu

@@ -225,11 +225,17 @@ class BatchNormKernel<platform::CUDADeviceContext, T>
 #elif CUDNN_VERSION_MIN(7, 0, 1)
     if (FLAGS_cudnn_batchnorm_spatial_persistent) {
       mode_ = CUDNN_BATCHNORM_SPATIAL_PERSISTENT;
+    } else if (H == 1 && W == 1) {
+      mode_ = CUDNN_BATCHNORM_PER_ACTIVATION;
     } else {
       mode_ = CUDNN_BATCHNORM_SPATIAL;
     }
 #else
-    mode_ = CUDNN_BATCHNORM_SPATIAL;
+    if (H == 1 && W == 1) {
+      mode_ = CUDNN_BATCHNORM_PER_ACTIVATION;
+    } else {
+      mode_ = CUDNN_BATCHNORM_SPATIAL;
+    }
 #endif  // CUDNN_VERSION_MIN(7, 0, 1)
 
     VLOG(3) << "Setting descriptors.";
@@ -989,11 +995,17 @@ class BatchNormGradKernel<platform::CUDADeviceContext, T>
 #elif CUDNN_VERSION_MIN(7, 0, 1)
       if (FLAGS_cudnn_batchnorm_spatial_persistent) {
         mode_ = CUDNN_BATCHNORM_SPATIAL_PERSISTENT;
+      } else if (H == 1 && W == 1) {
+        mode_ = CUDNN_BATCHNORM_PER_ACTIVATION;
       } else {
         mode_ = CUDNN_BATCHNORM_SPATIAL;
       }
 #else
-      mode_ = CUDNN_BATCHNORM_SPATIAL;
+      if (H == 1 && W == 1) {
+        mode_ = CUDNN_BATCHNORM_PER_ACTIVATION;
+      } else {
+        mode_ = CUDNN_BATCHNORM_SPATIAL;
+      }
 #endif  // CUDNN_VERSION_MIN(7, 0, 1)
 
 #ifdef PADDLE_WITH_HIP