diff --git a/drivers/gpu/drm/i915/intel_display.c b/drivers/gpu/drm/i915/intel_display.c
index 853697fc4d4b7f96d474e30da9594d53f49a4e8e..3218455a7ade03d893e66bba9228e8ca42f56b02 100644
--- a/drivers/gpu/drm/i915/intel_display.c
+++ b/drivers/gpu/drm/i915/intel_display.c
@@ -10907,7 +10907,6 @@ intel_modeset_compute_config(struct drm_crtc *crtc,
 	}
 	intel_dump_pipe_config(to_intel_crtc(crtc), pipe_config,
 			       "[modeset]");
-	to_intel_crtc(crtc)->new_config = pipe_config;
 
 out:
 	return pipe_config;
@@ -10933,6 +10932,9 @@ static int __intel_set_mode(struct drm_crtc *crtc,
 
 	*saved_mode = crtc->mode;
 
+	if (modeset_pipes)
+		to_intel_crtc(crtc)->new_config = pipe_config;
+
 	/*
 	 * See if the config requires any additional preparation, e.g.
 	 * to adjust global state with pipes off.  We need to do this
@@ -11466,12 +11468,12 @@ static int intel_crtc_set_config(struct drm_mode_set *set)
 		ret = PTR_ERR(pipe_config);
 		goto fail;
 	} else if (pipe_config) {
-		if (to_intel_crtc(set->crtc)->new_config->has_audio !=
+		if (pipe_config->has_audio !=
 		    to_intel_crtc(set->crtc)->config.has_audio)
 			config->mode_changed = true;
 
 		/* Force mode sets for any infoframe stuff */
-		if (to_intel_crtc(set->crtc)->new_config->has_infoframe ||
+		if (pipe_config->has_infoframe ||
 		    to_intel_crtc(set->crtc)->config.has_infoframe)
 			config->mode_changed = true;
 	}