diff --git a/compiler/rustc_const_eval/src/interpret/operand.rs b/compiler/rustc_const_eval/src/interpret/operand.rs
index 47d128307d3b9d4e662a0f7534cdc34191fcd955..75d987b655366af0a1244703cdb62c56484d38cb 100644
--- a/compiler/rustc_const_eval/src/interpret/operand.rs
+++ b/compiler/rustc_const_eval/src/interpret/operand.rs
@@ -71,7 +71,7 @@ pub fn new_dyn_trait(
     }
 
     #[inline]
-    #[track_caller]
+    #[cfg_attr(debug_assertions, track_caller)] // only in debug builds due to perf (see #98980)
     pub fn to_scalar_or_uninit(self) -> ScalarMaybeUninit<Tag> {
         match self {
             Immediate::Scalar(val) => val,
@@ -80,13 +80,13 @@ pub fn to_scalar_or_uninit(self) -> ScalarMaybeUninit<Tag> {
     }
 
     #[inline]
-    #[track_caller]
+    #[cfg_attr(debug_assertions, track_caller)] // only in debug builds due to perf (see #98980)
     pub fn to_scalar(self) -> InterpResult<'tcx, Scalar<Tag>> {
         self.to_scalar_or_uninit().check_init()
     }
 
     #[inline]
-    #[track_caller]
+    #[cfg_attr(debug_assertions, track_caller)] // only in debug builds due to perf (see #98980)
     pub fn to_scalar_or_uninit_pair(self) -> (ScalarMaybeUninit<Tag>, ScalarMaybeUninit<Tag>) {
         match self {
             Immediate::ScalarPair(val1, val2) => (val1, val2),
@@ -95,7 +95,7 @@ pub fn to_scalar_or_uninit_pair(self) -> (ScalarMaybeUninit<Tag>, ScalarMaybeUni
     }
 
     #[inline]
-    #[track_caller]
+    #[cfg_attr(debug_assertions, track_caller)] // only in debug builds due to perf (see #98980)
     pub fn to_scalar_pair(self) -> InterpResult<'tcx, (Scalar<Tag>, Scalar<Tag>)> {
         let (val1, val2) = self.to_scalar_or_uninit_pair();
         Ok((val1.check_init()?, val2.check_init()?))