diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig
index ca91e50bdb109dbfd41f3ddae23036a08d14c6fd..0de793cf21443bda1653a1bebd61ad0da4c60e36 100644
--- a/arch/x86/Kconfig
+++ b/arch/x86/Kconfig
@@ -11,7 +11,6 @@ config 64BIT
 
 config X86_32
 	def_bool !64BIT
-	select HAVE_FUNCTION_RET_TRACER
 
 config X86_64
 	def_bool 64BIT
@@ -30,6 +29,7 @@ config X86
 	select HAVE_FTRACE_MCOUNT_RECORD
 	select HAVE_DYNAMIC_FTRACE
 	select HAVE_FUNCTION_TRACER
+	select HAVE_FUNCTION_RET_TRACER if X86_32
 	select HAVE_FUNCTION_TRACE_MCOUNT_TEST
 	select HAVE_KVM if ((X86_32 && !X86_VOYAGER && !X86_VISWS && !X86_NUMAQ) || X86_64)
 	select HAVE_ARCH_KGDB if !X86_VOYAGER