Merge pull request #2109 from notr1ch/encoder-error-feedback

obs-ffmpeg: Encoder error feedback

libobs/obs-encoder.c

@@ -276,6 +276,8 @@ static void obs_encoder_actually_destroy(obs_encoder_t *encoder)
 		obs_context_data_free(&encoder->context);
 		if (encoder->owns_info_id)
 			bfree((void *)encoder->info.id);
+		if (encoder->last_error_message)
+			bfree(encoder->last_error_message);
 		bfree(encoder);
 	}
 }
@@ -1484,3 +1486,25 @@ bool obs_encoder_paused(const obs_encoder_t *encoder)
 		       ? os_atomic_load_bool(&encoder->paused)
 		       : false;
 }
+
+const char *obs_encoder_get_last_error(obs_encoder_t *encoder)
+{
+	if (!obs_encoder_valid(encoder, "obs_encoder_get_last_error"))
+		return NULL;
+
+	return encoder->last_error_message;
+}
+
+void obs_encoder_set_last_error(obs_encoder_t *encoder, const char *message)
+{
+	if (!obs_encoder_valid(encoder, "obs_encoder_set_last_error"))
+		return;
+
+	if (encoder->last_error_message)
+		bfree(encoder->last_error_message);
+
+	if (message)
+		encoder->last_error_message = bstrdup(message);
+	else
+		encoder->last_error_message = NULL;
+}

libobs/obs-internal.h

@@ -1076,6 +1076,7 @@ struct obs_encoder {
 	struct pause_data pause;
 
 	const char *profile_encoder_encode_name;
+	char *last_error_message;
 };
 
 extern struct obs_encoder_info *find_encoder(const char *id);

libobs/obs-output.c

@@ -1979,6 +1979,9 @@ static inline bool initialize_audio_encoders(obs_output_t *output,
 {
 	for (size_t i = 0; i < num_mixes; i++) {
 		if (!obs_encoder_initialize(output->audio_encoders[i])) {
+			obs_output_set_last_error(
+				output, obs_encoder_get_last_error(
+						output->audio_encoders[i]));
 			return false;
 		}
 	}
@@ -2038,8 +2041,12 @@ bool obs_output_initialize_encoders(obs_output_t *output, uint32_t flags)
 
 	if (!encoded)
 		return false;
-	if (has_video && !obs_encoder_initialize(output->video_encoder))
+	if (has_video && !obs_encoder_initialize(output->video_encoder)) {
+		obs_output_set_last_error(
+			output,
+			obs_encoder_get_last_error(output->video_encoder));
 		return false;
+	}
 	if (has_audio && !initialize_audio_encoders(output, num_mixes))
 		return false;
 

libobs/obs.h

@@ -2102,6 +2102,10 @@ EXPORT void *obs_encoder_create_rerouted(obs_encoder_t *encoder,
 /** Returns whether encoder is paused */
 EXPORT bool obs_encoder_paused(const obs_encoder_t *output);
 
+EXPORT const char *obs_encoder_get_last_error(obs_encoder_t *encoder);
+EXPORT void obs_encoder_set_last_error(obs_encoder_t *encoder,
+				       const char *message);
+
 /* ------------------------------------------------------------------------- */
 /* Stream Services */
 

plugins/obs-ffmpeg/jim-nvenc-helpers.c

 #include "jim-nvenc.h"
 #include <util/platform.h>
 #include <util/threading.h>
+#include <util/dstr.h>
 
 static void *nvenc_lib = NULL;
 static pthread_mutex_t init_mutex = PTHREAD_MUTEX_INITIALIZER;
@@ -9,18 +10,46 @@ NV_CREATE_INSTANCE_FUNC nv_create_instance = NULL;
 
 #define error(format, ...) blog(LOG_ERROR, "[jim-nvenc] " format, ##__VA_ARGS__)
 
-static inline bool nv_failed(NVENCSTATUS err, const char *func,
-			     const char *call)
+bool nv_failed(obs_encoder_t *encoder, NVENCSTATUS err, const char *func,
+	       const char *call)
 {
-	if (err == NV_ENC_SUCCESS)
+	struct dstr error_message = {0};
+
+	switch (err) {
+	case NV_ENC_SUCCESS:
 		return false;
 
+	case NV_ENC_ERR_OUT_OF_MEMORY:
+		obs_encoder_set_last_error(
+			encoder,
+			"NVENC Error: Too many concurrent sessions. "
+			"Try closing other recording software which might "
+			"be using NVENC such as Windows 10 Game DVR.");
+		break;
+
+	case NV_ENC_ERR_UNSUPPORTED_DEVICE:
+		obs_encoder_set_last_error(
+			encoder,
+			"NVENC Error: Unsupported device. Check your "
+			"video card supports NVENC and that the drivers are "
+			"up to date.");
+		break;
+
+	default:
+		dstr_printf(&error_message,
+			    "NVENC Error: %s: %s failed: %d (%s)", func, call,
+			    (int)err, nv_error_name(err));
+		obs_encoder_set_last_error(encoder, error_message.array);
+		dstr_free(&error_message);
+		break;
+	}
+
 	error("%s: %s failed: %d (%s)", func, call, (int)err,
 	      nv_error_name(err));
 	return true;
 }
 
-#define NV_FAILED(x) nv_failed(x, __FUNCTION__, #x)
+#define NV_FAILED(e, x) nv_failed(e, x, __FUNCTION__, #x)
 
 bool load_nvenc_lib(void)
 {
@@ -83,7 +112,7 @@ const char *nv_error_name(NVENCSTATUS err)
 	return "Unknown Error";
 }
 
-static inline bool init_nvenc_internal(void)
+static inline bool init_nvenc_internal(obs_encoder_t *encoder)
 {
 	static bool initialized = false;
 	static bool success = false;
@@ -95,17 +124,26 @@ static inline bool init_nvenc_internal(void)
 	NV_MAX_VER_FUNC nv_max_ver = (NV_MAX_VER_FUNC)load_nv_func(
 		"NvEncodeAPIGetMaxSupportedVersion");
 	if (!nv_max_ver) {
+		obs_encoder_set_last_error(
+			encoder,
+			"Missing NvEncodeAPIGetMaxSupportedVersion, check "
+			"your video card drivers are up to date.");
 		return false;
 	}
 
 	uint32_t ver = 0;
-	if (NV_FAILED(nv_max_ver(&ver))) {
+	if (NV_FAILED(encoder, nv_max_ver(&ver))) {
 		return false;
 	}
 
 	uint32_t cur_ver = (NVENCAPI_MAJOR_VERSION << 4) |
 			   NVENCAPI_MINOR_VERSION;
 	if (cur_ver > ver) {
+		obs_encoder_set_last_error(
+			encoder,
+			"Your current video card driver does not support "
+			"this NVENC version, please update your drivers.");
+
 		error("Current driver version does not support this NVENC "
 		      "version, please upgrade your driver");
 		return false;
@@ -114,10 +152,13 @@ static inline bool init_nvenc_internal(void)
 	nv_create_instance = (NV_CREATE_INSTANCE_FUNC)load_nv_func(
 		"NvEncodeAPICreateInstance");
 	if (!nv_create_instance) {
+		obs_encoder_set_last_error(
+			encoder, "Missing NvEncodeAPICreateInstance, check "
+				 "your video card drivers are up to date.");
 		return false;
 	}
 
-	if (NV_FAILED(nv_create_instance(&nv))) {
+	if (NV_FAILED(encoder, nv_create_instance(&nv))) {
 		return false;
 	}
 
@@ -125,12 +166,12 @@ static inline bool init_nvenc_internal(void)
 	return true;
 }
 
-bool init_nvenc(void)
+bool init_nvenc(obs_encoder_t *encoder)
 {
 	bool success;
 
 	pthread_mutex_lock(&init_mutex);
-	success = init_nvenc_internal();
+	success = init_nvenc_internal(encoder);
 	pthread_mutex_unlock(&init_mutex);
 
 	return success;

plugins/obs-ffmpeg/jim-nvenc.c

@@ -81,18 +81,7 @@ struct nv_bitstream {
 	HANDLE event;
 };
 
-static inline bool nv_failed(struct nvenc_data *enc, NVENCSTATUS err,
-			     const char *func, const char *call)
-{
-	if (err == NV_ENC_SUCCESS)
-		return false;
-
-	error("%s: %s failed: %d (%s)", func, call, (int)err,
-	      nv_error_name(err));
-	return true;
-}
-
-#define NV_FAILED(x) nv_failed(enc, x, __FUNCTION__, #x)
+#define NV_FAILED(x) nv_failed(enc->encoder, x, __FUNCTION__, #x)
 
 static bool nv_bitstream_init(struct nvenc_data *enc, struct nv_bitstream *bs)
 {
@@ -400,7 +389,8 @@ static bool init_encoder(struct nvenc_data *enc, obs_data_t *settings)
 	err = nv.nvEncGetEncodePresetConfig(enc->session,
 					    NV_ENC_CODEC_H264_GUID, nv_preset,
 					    &preset_config);
-	if (nv_failed(enc, err, __FUNCTION__, "nvEncGetEncodePresetConfig")) {
+	if (nv_failed(enc->encoder, err, __FUNCTION__,
+		      "nvEncGetEncodePresetConfig")) {
 		return false;
 	}
 
@@ -587,7 +577,7 @@ static void *nvenc_create(obs_data_t *settings, obs_encoder_t *encoder)
 	if (!obs_nv12_tex_active()) {
 		goto fail;
 	}
-	if (!init_nvenc()) {
+	if (!init_nvenc(encoder)) {
 		goto fail;
 	}
 	if (NV_FAILED(nv_create_instance(&init))) {
@@ -766,7 +756,8 @@ static bool get_encoded_packet(struct nvenc_data *enc, bool finalize)
 		if (nvtex->mapped_res) {
 			NVENCSTATUS err;
 			err = nv.nvEncUnmapInputResource(s, nvtex->mapped_res);
-			if (nv_failed(enc, err, __FUNCTION__, "unmap")) {
+			if (nv_failed(enc->encoder, err, __FUNCTION__,
+				      "unmap")) {
 				return false;
 			}
 			nvtex->mapped_res = NULL;
@@ -859,7 +850,8 @@ static bool nvenc_encode_tex(void *data, uint32_t handle, int64_t pts,
 
 	err = nv.nvEncEncodePicture(enc->session, &params);
 	if (err != NV_ENC_SUCCESS && err != NV_ENC_ERR_NEED_MORE_INPUT) {
-		nv_failed(enc, err, __FUNCTION__, "nvEncEncodePicture");
+		nv_failed(enc->encoder, err, __FUNCTION__,
+			  "nvEncEncodePicture");
 		return false;
 	}
 

plugins/obs-ffmpeg/jim-nvenc.h

@@ -12,4 +12,6 @@ typedef NVENCSTATUS(NVENCAPI *NV_CREATE_INSTANCE_FUNC)(
 extern const char *nv_error_name(NVENCSTATUS err);
 extern NV_ENCODE_API_FUNCTION_LIST nv;
 extern NV_CREATE_INSTANCE_FUNC nv_create_instance;
-extern bool init_nvenc(void);
+extern bool init_nvenc(obs_encoder_t *encoder);
+bool nv_failed(obs_encoder_t *encoder, NVENCSTATUS err, const char *func,
+	       const char *call);

plugins/obs-ffmpeg/obs-ffmpeg-audio-encoders.c

@@ -18,6 +18,7 @@
 #include <util/base.h>
 #include <util/circlebuf.h>
 #include <util/darray.h>
+#include <util/dstr.h>
 #include <obs-module.h>
 
 #include <libavutil/opt.h>
@@ -143,6 +144,11 @@ static bool initialize_codec(struct enc_encoder *enc)
 
 	ret = avcodec_open2(enc->context, enc->codec, NULL);
 	if (ret < 0) {
+		struct dstr error_message = {0};
+		dstr_printf(&error_message, "Failed to open AAC codec: %s",
+			    av_err2str(ret));
+		obs_encoder_set_last_error(enc->encoder, error_message.array);
+		dstr_free(&error_message);
 		warn("Failed to open AAC codec: %s", av_err2str(ret));
 		return false;
 	}

plugins/obs-ffmpeg/obs-ffmpeg-nvenc.c

@@ -90,6 +90,27 @@ static bool nvenc_init_codec(struct nvenc_encoder *enc)
 
 	ret = avcodec_open2(enc->context, enc->nvenc, NULL);
 	if (ret < 0) {
+		struct dstr error_message = {0};
+
+		// special case for common NVENC error
+		if (ret == AVERROR_EXTERNAL) {
+			dstr_printf(&error_message,
+				    "Failed to open NVENC codec: %s\r\n\r\n"
+				    "Check your video drivers are up to "
+				    "date. Disable other software that may "
+				    "be using NVENC such as NVIDIA "
+				    "ShadowPlay or Windows 10 Game "
+				    "DVR.",
+				    av_err2str(ret));
+		} else {
+			dstr_printf(
+				&error_message,
+				"Failed to open NVENC codec: %s\r\n\r\n"
+				"Please check your video drivers are up to date.",
+				av_err2str(ret));
+		}
+		obs_encoder_set_last_error(enc->encoder, error_message.array);
+		dstr_free(&error_message);
 		warn("Failed to open NVENC codec: %s", av_err2str(ret));
 		return false;
 	}
@@ -296,6 +317,8 @@ static void *nvenc_create(obs_data_t *settings, obs_encoder_t *encoder)
 	blog(LOG_INFO, "---------------------------------");
 
 	if (!enc->nvenc) {
+		obs_encoder_set_last_error(encoder,
+					   "Couldn't find NVENC encoder");
 		warn("Couldn't find encoder");
 		goto fail;
 	}