drm/amd/display: dsc validate fail not pass to atomic check

[Why] when 4k@144hz dp connect to dp1.4 dsc mst hub, requested
bandwidth exceeds caps of dsc hub. but dsc bw valid functions,
increase_dsc_bpp, try_disable_dsc, pre_validate_dsc,
compute_mst_dsc_configs_for_state, do not return false to
atomic check. this cause user mode initiate mode set to kernel,
then cause kernel assert, system hang.

[How] dsc bandwidth valid functions return pass or fail to atomic
check.
Reviewed-by: NWayne Lin <Wayne.Lin@amd.com>
Reviewed-by: NRodrigo Siqueira <Rodrigo.Siqueira@amd.com>
Acked-by: NHamza Mahfooz <hamza.mahfooz@amd.com>
Signed-off-by: Nhersen wu <hersenxs.wu@amd.com>
Tested-by: NDaniel Wheeler <daniel.wheeler@amd.com>
Signed-off-by: NAlex Deucher <alexander.deucher@amd.com>

drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c

@@ -11255,7 +11255,10 @@ static int amdgpu_dm_atomic_check(struct drm_device *dev,
 				}
 			}
 		}
-		pre_validate_dsc(state, &dm_state, vars);
+		if (!pre_validate_dsc(state, &dm_state, vars)) {
+			ret = -EINVAL;
+			goto fail;
+		}
 	}
 #endif
 	for_each_oldnew_crtc_in_state(state, crtc, old_crtc_state, new_crtc_state, i) {
@@ -11501,6 +11504,7 @@ static int amdgpu_dm_atomic_check(struct drm_device *dev,
 #if defined(CONFIG_DRM_AMD_DC_DCN)
 		if (!compute_mst_dsc_configs_for_state(state, dm_state->context, vars)) {
 			DRM_DEBUG_DRIVER("compute_mst_dsc_configs_for_state() failed\n");
+			ret = -EINVAL;
 			goto fail;
 		}
 

drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_mst_types.c

@@ -670,7 +670,7 @@ static int bpp_x16_from_pbn(struct dsc_mst_fairness_params param, int pbn)
 	return dsc_config.bits_per_pixel;
 }
 
-static void increase_dsc_bpp(struct drm_atomic_state *state,
+static bool increase_dsc_bpp(struct drm_atomic_state *state,
 			     struct dc_link *dc_link,
 			     struct dsc_mst_fairness_params *params,
 			     struct dsc_mst_fairness_vars *vars,
@@ -730,7 +730,7 @@ static void increase_dsc_bpp(struct drm_atomic_state *state,
 							  params[next_index].port,
 							  vars[next_index].pbn,
 							  pbn_per_timeslot) < 0)
-				return;
+				return false;
 			if (!drm_dp_mst_atomic_check(state)) {
 				vars[next_index].bpp_x16 = bpp_x16_from_pbn(params[next_index], vars[next_index].pbn);
 			} else {
@@ -740,7 +740,7 @@ static void increase_dsc_bpp(struct drm_atomic_state *state,
 								  params[next_index].port,
 								  vars[next_index].pbn,
 								  pbn_per_timeslot) < 0)
-					return;
+					return false;
 			}
 		} else {
 			vars[next_index].pbn += initial_slack[next_index];
@@ -749,7 +749,7 @@ static void increase_dsc_bpp(struct drm_atomic_state *state,
 							  params[next_index].port,
 							  vars[next_index].pbn,
 							  pbn_per_timeslot) < 0)
-				return;
+				return false;
 			if (!drm_dp_mst_atomic_check(state)) {
 				vars[next_index].bpp_x16 = params[next_index].bw_range.max_target_bpp_x16;
 			} else {
@@ -759,16 +759,17 @@ static void increase_dsc_bpp(struct drm_atomic_state *state,
 								  params[next_index].port,
 								  vars[next_index].pbn,
 								  pbn_per_timeslot) < 0)
-					return;
+					return false;
 			}
 		}
 
 		bpp_increased[next_index] = true;
 		remaining_to_increase--;
 	}
+	return true;
 }
 
-static void try_disable_dsc(struct drm_atomic_state *state,
+static bool try_disable_dsc(struct drm_atomic_state *state,
 			    struct dc_link *dc_link,
 			    struct dsc_mst_fairness_params *params,
 			    struct dsc_mst_fairness_vars *vars,
@@ -816,7 +817,7 @@ static void try_disable_dsc(struct drm_atomic_state *state,
 						  params[next_index].port,
 						  vars[next_index].pbn,
 						  dm_mst_get_pbn_divider(dc_link)) < 0)
-			return;
+			return false;
 
 		if (!drm_dp_mst_atomic_check(state)) {
 			vars[next_index].dsc_enabled = false;
@@ -828,12 +829,13 @@ static void try_disable_dsc(struct drm_atomic_state *state,
 							  params[next_index].port,
 							  vars[next_index].pbn,
 							  dm_mst_get_pbn_divider(dc_link)) < 0)
-				return;
+				return false;
 		}
 
 		tried[next_index] = true;
 		remaining_to_try--;
 	}
+	return true;
 }
 
 static bool compute_mst_dsc_configs_for_link(struct drm_atomic_state *state,
@@ -949,9 +951,11 @@ static bool compute_mst_dsc_configs_for_link(struct drm_atomic_state *state,
 		return false;
 
 	/* Optimize degree of compression */
-	increase_dsc_bpp(state, dc_link, params, vars, count, k);
+	if (!increase_dsc_bpp(state, dc_link, params, vars, count, k))
+		return false;
 
-	try_disable_dsc(state, dc_link, params, vars, count, k);
+	if (!try_disable_dsc(state, dc_link, params, vars, count, k))
+		return false;
 
 	set_dsc_configs_from_fairness_vars(params, vars, count, k);
 
@@ -1223,21 +1227,22 @@ static bool is_dsc_precompute_needed(struct drm_atomic_state *state)
 	return ret;
 }
 
-void pre_validate_dsc(struct drm_atomic_state *state,
+bool pre_validate_dsc(struct drm_atomic_state *state,
 		      struct dm_atomic_state **dm_state_ptr,
 		      struct dsc_mst_fairness_vars *vars)
 {
 	int i;
 	struct dm_atomic_state *dm_state;
 	struct dc_state *local_dc_state = NULL;
+	int ret = 0;
 
 	if (!is_dsc_precompute_needed(state)) {
 		DRM_INFO_ONCE("DSC precompute is not needed.\n");
-		return;
+		return true;
 	}
 	if (dm_atomic_get_state(state, dm_state_ptr)) {
 		DRM_INFO_ONCE("dm_atomic_get_state() failed\n");
-		return;
+		return false;
 	}
 	dm_state = *dm_state_ptr;
 
@@ -1249,7 +1254,7 @@ void pre_validate_dsc(struct drm_atomic_state *state,
 
 	local_dc_state = kmemdup(dm_state->context, sizeof(struct dc_state), GFP_KERNEL);
 	if (!local_dc_state)
-		return;
+		return false;
 
 	for (i = 0; i < local_dc_state->stream_count; i++) {
 		struct dc_stream_state *stream = dm_state->context->streams[i];
@@ -1275,11 +1280,19 @@ void pre_validate_dsc(struct drm_atomic_state *state,
 								&state->crtcs[ind].new_state->mode,
 								dm_new_conn_state,
 								dm_old_crtc_state->stream);
+			if (local_dc_state->streams[i] == NULL) {
+				ret = -EINVAL;
+				break;
+			}
 		}
 	}
 
+	if (ret != 0)
+		goto clean_exit;
+
 	if (!pre_compute_mst_dsc_configs_for_state(state, local_dc_state, vars)) {
 		DRM_INFO_ONCE("pre_compute_mst_dsc_configs_for_state() failed\n");
+		ret = -EINVAL;
 		goto clean_exit;
 	}
 
@@ -1309,5 +1322,7 @@ void pre_validate_dsc(struct drm_atomic_state *state,
 	}
 
 	kfree(local_dc_state);
+
+	return (ret == 0);
 }
 #endif

drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_mst_types.h

@@ -59,7 +59,7 @@ bool compute_mst_dsc_configs_for_state(struct drm_atomic_state *state,
 
 bool needs_dsc_aux_workaround(struct dc_link *link);
 
-void pre_validate_dsc(struct drm_atomic_state *state,
+bool pre_validate_dsc(struct drm_atomic_state *state,
 		      struct dm_atomic_state **dm_state_ptr,
 		      struct dsc_mst_fairness_vars *vars);