diff --git a/modules/imgproc/src/color.cpp b/modules/imgproc/src/color.cpp
index 5d4a943dafc5e065c0077f2b44da8be9b33a8297..33ab41f4f8e531aa68ae31a67df86f2a050edd6f 100644
--- a/modules/imgproc/src/color.cpp
+++ b/modules/imgproc/src/color.cpp
@@ -1674,7 +1674,9 @@ struct RGB2Gray<float>
     bool haveSIMD;
 };
 
-#else
+#endif // CV_SSE2
+
+#if !CV_NEON && !CV_SSE4_1
 
 template<> struct RGB2Gray<ushort>
 {
@@ -1698,7 +1700,7 @@ template<> struct RGB2Gray<ushort>
     int coeffs[3];
 };
 
-#endif
+#endif // !CV_NEON && !CV_SSE4_1
 
 ///////////////////////////////////// RGB <-> YCrCb //////////////////////////////////////