obs-ffmpeg: Use new encoder error handling functions

libobs/obs.h

@@ -2104,7 +2104,7 @@ EXPORT bool obs_encoder_paused(const obs_encoder_t *output);
 
 EXPORT const char *obs_encoder_get_last_error(obs_encoder_t *encoder);
 EXPORT void obs_encoder_set_last_error(obs_encoder_t *encoder,
-				      const char *message);
+				       const char *message);
 
 /* ------------------------------------------------------------------------- */
 /* Stream Services */

plugins/obs-ffmpeg/jim-nvenc-helpers.c

 #include "jim-nvenc.h"
 #include <util/platform.h>
 #include <util/threading.h>
+#include <util/dstr.h>
 
 static void *nvenc_lib = NULL;
 static pthread_mutex_t init_mutex = PTHREAD_MUTEX_INITIALIZER;
@@ -9,18 +10,46 @@ NV_CREATE_INSTANCE_FUNC nv_create_instance = NULL;
 
 #define error(format, ...) blog(LOG_ERROR, "[jim-nvenc] " format, ##__VA_ARGS__)
 
-static inline bool nv_failed(NVENCSTATUS err, const char *func,
-			     const char *call)
+bool nv_failed(obs_encoder_t *encoder, NVENCSTATUS err, const char *func,
+	       const char *call)
 {
-	if (err == NV_ENC_SUCCESS)
+	struct dstr error_message = {0};
+
+	switch (err) {
+	case NV_ENC_SUCCESS:
 		return false;
 
+	case NV_ENC_ERR_OUT_OF_MEMORY:
+		obs_encoder_set_last_error(
+			encoder,
+			"NVENC Error: Too many concurrent sessions. "
+			"Try closing other recording software which might "
+			"be using NVENC such as Windows 10 Game DVR.");
+		break;
+
+	case NV_ENC_ERR_UNSUPPORTED_DEVICE:
+		obs_encoder_set_last_error(
+			encoder,
+			"NVENC Error: Unsupported device. Check your "
+			"video card supports NVENC and that the drivers are "
+			"up to date.");
+		break;
+
+	default:
+		dstr_printf(&error_message,
+			    "NVENC Error: %s: %s failed: %d (%s)", func, call,
+			    (int)err, nv_error_name(err));
+		obs_encoder_set_last_error(encoder, error_message.array);
+		dstr_free(&error_message);
+		break;
+	}
+
 	error("%s: %s failed: %d (%s)", func, call, (int)err,
 	      nv_error_name(err));
 	return true;
 }
 
-#define NV_FAILED(x) nv_failed(x, __FUNCTION__, #x)
+#define NV_FAILED(e, x) nv_failed(e, x, __FUNCTION__, #x)
 
 bool load_nvenc_lib(void)
 {
@@ -83,7 +112,7 @@ const char *nv_error_name(NVENCSTATUS err)
 	return "Unknown Error";
 }
 
-static inline bool init_nvenc_internal(void)
+static inline bool init_nvenc_internal(obs_encoder_t *encoder)
 {
 	static bool initialized = false;
 	static bool success = false;
@@ -95,17 +124,26 @@ static inline bool init_nvenc_internal(void)
 	NV_MAX_VER_FUNC nv_max_ver = (NV_MAX_VER_FUNC)load_nv_func(
 		"NvEncodeAPIGetMaxSupportedVersion");
 	if (!nv_max_ver) {
+		obs_encoder_set_last_error(
+			encoder,
+			"Missing NvEncodeAPIGetMaxSupportedVersion, check "
+			"your video card drivers are up to date.");
 		return false;
 	}
 
 	uint32_t ver = 0;
-	if (NV_FAILED(nv_max_ver(&ver))) {
+	if (NV_FAILED(encoder, nv_max_ver(&ver))) {
 		return false;
 	}
 
 	uint32_t cur_ver = (NVENCAPI_MAJOR_VERSION << 4) |
 			   NVENCAPI_MINOR_VERSION;
 	if (cur_ver > ver) {
+		obs_encoder_set_last_error(
+			encoder,
+			"Your current video card driver does not support "
+			"this NVENC version, please update your drivers.");
+
 		error("Current driver version does not support this NVENC "
 		      "version, please upgrade your driver");
 		return false;
@@ -114,10 +152,13 @@ static inline bool init_nvenc_internal(void)
 	nv_create_instance = (NV_CREATE_INSTANCE_FUNC)load_nv_func(
 		"NvEncodeAPICreateInstance");
 	if (!nv_create_instance) {
+		obs_encoder_set_last_error(
+			encoder, "Missing NvEncodeAPICreateInstance, check "
+				 "your video card drivers are up to date.");
 		return false;
 	}
 
-	if (NV_FAILED(nv_create_instance(&nv))) {
+	if (NV_FAILED(encoder, nv_create_instance(&nv))) {
 		return false;
 	}
 
@@ -125,12 +166,12 @@ static inline bool init_nvenc_internal(void)
 	return true;
 }
 
-bool init_nvenc(void)
+bool init_nvenc(obs_encoder_t *encoder)
 {
 	bool success;
 
 	pthread_mutex_lock(&init_mutex);
-	success = init_nvenc_internal();
+	success = init_nvenc_internal(encoder);
 	pthread_mutex_unlock(&init_mutex);
 
 	return success;

plugins/obs-ffmpeg/jim-nvenc.c

@@ -81,18 +81,7 @@ struct nv_bitstream {
 	HANDLE event;
 };
 
-static inline bool nv_failed(struct nvenc_data *enc, NVENCSTATUS err,
-			     const char *func, const char *call)
-{
-	if (err == NV_ENC_SUCCESS)
-		return false;
-
-	error("%s: %s failed: %d (%s)", func, call, (int)err,
-	      nv_error_name(err));
-	return true;
-}
-
-#define NV_FAILED(x) nv_failed(enc, x, __FUNCTION__, #x)
+#define NV_FAILED(x) nv_failed(enc->encoder, x, __FUNCTION__, #x)
 
 static bool nv_bitstream_init(struct nvenc_data *enc, struct nv_bitstream *bs)
 {
@@ -400,7 +389,8 @@ static bool init_encoder(struct nvenc_data *enc, obs_data_t *settings)
 	err = nv.nvEncGetEncodePresetConfig(enc->session,
 					    NV_ENC_CODEC_H264_GUID, nv_preset,
 					    &preset_config);
-	if (nv_failed(enc, err, __FUNCTION__, "nvEncGetEncodePresetConfig")) {
+	if (nv_failed(enc->encoder, err, __FUNCTION__,
+		      "nvEncGetEncodePresetConfig")) {
 		return false;
 	}
 
@@ -587,7 +577,7 @@ static void *nvenc_create(obs_data_t *settings, obs_encoder_t *encoder)
 	if (!obs_nv12_tex_active()) {
 		goto fail;
 	}
-	if (!init_nvenc()) {
+	if (!init_nvenc(encoder)) {
 		goto fail;
 	}
 	if (NV_FAILED(nv_create_instance(&init))) {
@@ -766,7 +756,8 @@ static bool get_encoded_packet(struct nvenc_data *enc, bool finalize)
 		if (nvtex->mapped_res) {
 			NVENCSTATUS err;
 			err = nv.nvEncUnmapInputResource(s, nvtex->mapped_res);
-			if (nv_failed(enc, err, __FUNCTION__, "unmap")) {
+			if (nv_failed(enc->encoder, err, __FUNCTION__,
+				      "unmap")) {
 				return false;
 			}
 			nvtex->mapped_res = NULL;
@@ -859,7 +850,8 @@ static bool nvenc_encode_tex(void *data, uint32_t handle, int64_t pts,
 
 	err = nv.nvEncEncodePicture(enc->session, &params);
 	if (err != NV_ENC_SUCCESS && err != NV_ENC_ERR_NEED_MORE_INPUT) {
-		nv_failed(enc, err, __FUNCTION__, "nvEncEncodePicture");
+		nv_failed(enc->encoder, err, __FUNCTION__,
+			  "nvEncEncodePicture");
 		return false;
 	}
 

plugins/obs-ffmpeg/jim-nvenc.h

@@ -12,4 +12,6 @@ typedef NVENCSTATUS(NVENCAPI *NV_CREATE_INSTANCE_FUNC)(
 extern const char *nv_error_name(NVENCSTATUS err);
 extern NV_ENCODE_API_FUNCTION_LIST nv;
 extern NV_CREATE_INSTANCE_FUNC nv_create_instance;
-extern bool init_nvenc(void);
+extern bool init_nvenc(obs_encoder_t *encoder);
+bool nv_failed(obs_encoder_t *encoder, NVENCSTATUS err, const char *func,
+	       const char *call);

plugins/obs-ffmpeg/obs-ffmpeg-audio-encoders.c

@@ -18,6 +18,7 @@
 #include <util/base.h>
 #include <util/circlebuf.h>
 #include <util/darray.h>
+#include <util/dstr.h>
 #include <obs-module.h>
 
 #include <libavutil/opt.h>
@@ -143,6 +144,11 @@ static bool initialize_codec(struct enc_encoder *enc)
 
 	ret = avcodec_open2(enc->context, enc->codec, NULL);
 	if (ret < 0) {
+		struct dstr error_message = {0};
+		dstr_printf(&error_message, "Failed to open AAC codec: %s",
+			    av_err2str(ret));
+		obs_encoder_set_last_error(enc->encoder, error_message.array);
+		dstr_free(&error_message);
 		warn("Failed to open AAC codec: %s", av_err2str(ret));
 		return false;
 	}

plugins/obs-ffmpeg/obs-ffmpeg-nvenc.c

@@ -90,6 +90,14 @@ static bool nvenc_init_codec(struct nvenc_encoder *enc)
 
 	ret = avcodec_open2(enc->context, enc->nvenc, NULL);
 	if (ret < 0) {
+		struct dstr error_message = {0};
+		dstr_printf(
+			&error_message,
+			"Failed to open NVENC codec: %s"
+			"\r\n\r\nPlease check your video drivers are up to date.",
+			av_err2str(ret));
+		obs_encoder_set_last_error(enc->encoder, error_message.array);
+		dstr_free(&error_message);
 		warn("Failed to open NVENC codec: %s", av_err2str(ret));
 		return false;
 	}
@@ -296,6 +304,8 @@ static void *nvenc_create(obs_data_t *settings, obs_encoder_t *encoder)
 	blog(LOG_INFO, "---------------------------------");
 
 	if (!enc->nvenc) {
+		obs_encoder_set_last_error(encoder,
+					   "Couldn't find NVENC encoder");
 		warn("Couldn't find encoder");
 		goto fail;
 	}