diff --git a/cmake/operators.cmake b/cmake/operators.cmake
index c9d0f80da2962ee1578af0f6f06f07438898f399..5636342ef72f21ab03eb1ddca27559b39c23d719 100644
--- a/cmake/operators.cmake
+++ b/cmake/operators.cmake
@@ -84,8 +84,9 @@ function(op_library TARGET)
     endif()
     if (WIN32)
     # remove windows unsupported op, because windows has no nccl, no warpctc such ops.
-    foreach(windows_unsupport_op "nccl_op" "gen_nccl_id_op" "warpctc_op" "hierarchical_sigmoid_op"
-     "crf_decoding_op" "select_op" "lstmp_op" "gru_op" "fusion_gru_op" "lstm_op" "fusion_lstm_op" "cumsum_op"
+    foreach(windows_unsupport_op "nccl_op" "gen_nccl_id_op" "warpctc_op"
+#     "hierarchical_sigmoid_op" "cumsum_op"
+#     "crf_decoding_op" "select_op" "lstmp_op" "gru_op" "fusion_gru_op" "lstm_op" "fusion_lstm_op"
       "fusion_seqconv_eltadd_relu_op" "channel_send_op" "channel_create_op" "channel_close_op" "channel_recv_op")
         if ("${TARGET}" STREQUAL "${windows_unsupport_op}")
           return()
diff --git a/paddle/fluid/operators/math/detail/activation_functions.h b/paddle/fluid/operators/math/detail/activation_functions.h
index 42fb45a8a5ea9d83b9d98001d234cc4d228b1e2e..2b3d38d95a18fad9b76e616cdf2cb6c3eb07da3a 100644
--- a/paddle/fluid/operators/math/detail/activation_functions.h
+++ b/paddle/fluid/operators/math/detail/activation_functions.h
@@ -16,12 +16,6 @@ limitations under the License. */
 #include <math.h>
 #include <string>
 
-#ifdef _WIN32
-#undef __AVX__
-#undef __AVX__2
-#endif
-
-
 #include "paddle/fluid/platform/enforce.h"
 #include "paddle/fluid/platform/hostdevice.h"
 
diff --git a/paddle/fluid/platform/cpu_info.h b/paddle/fluid/platform/cpu_info.h
index 6810a1651a14cdb2080af846b21cad242b70bf35..1b4840d9a117b3aa3cf3c056cd841503ad8a78b3 100644
--- a/paddle/fluid/platform/cpu_info.h
+++ b/paddle/fluid/platform/cpu_info.h
@@ -14,6 +14,11 @@ limitations under the License. */
 
 #pragma once
 
+#ifdef _WIN32
+#undef __AVX__
+#undef __AVX2__
+#endif
+
 #include <stddef.h>
 
 namespace paddle {
diff --git a/paddle/fluid/platform/enforce.h b/paddle/fluid/platform/enforce.h
index a251bfcd9914422cb6300adbbcdef3dfa79f441c..c03bbd59ac922faf9264f0211bf4a358d75c29ad 100644
--- a/paddle/fluid/platform/enforce.h
+++ b/paddle/fluid/platform/enforce.h
@@ -14,6 +14,11 @@ limitations under the License. */
 
 #pragma once
 
+#ifdef _WIN32
+#undef __AVX__
+#undef __AVX2__
+#endif
+
 #ifdef __GNUC__
 #include <cxxabi.h>  // for __cxa_demangle
 #endif               // __GNUC__