diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c
index 68ccaedad0f7531031ae1a8c8d90589856db3709..19180aaa8bf273b3f948b84aaec2a57d967fa96a 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c
@@ -358,8 +358,6 @@ static int psp_load_fw(struct amdgpu_device *adev)
 	if (ret)
 		goto failed_mem;
 
-	amdgpu_bo_free_kernel(&psp->fence_buf_bo,
-			      &psp->fence_buf_mc_addr, &psp->fence_buf);
 	kfree(cmd);
 
 	return 0;
@@ -423,6 +421,10 @@ static int psp_hw_fini(void *handle)
 		amdgpu_bo_free_kernel(&psp->fw_pri_bo,
 				      &psp->fw_pri_mc_addr, &psp->fw_pri_buf);
 
+	if (psp->fence_buf_bo)
+		amdgpu_bo_free_kernel(&psp->fence_buf_bo,
+				      &psp->fence_buf_mc_addr, &psp->fence_buf);
+
 	return 0;
 }