obs-ffmpeg: Add texture-based NVENC encoder implementation

Adds a texture-based NVENC implementation which passes OBS NV12 output
textures directly to NVENC without downloading them off of the GPU,
increasing NVENC performance by a significant margin.

If NV12 textures are unavailable or the new encoder fails to initialize
for whatever reason, it will fall back to the FFmpeg NVENC
implementation safely.

plugins/obs-ffmpeg/CMakeLists.txt

@@ -29,6 +29,14 @@ if(UNIX AND NOT APPLE)
 		${LIBVA_LBRARIES})
 endif()
 
+if(WIN32)
+	list(APPEND obs-ffmpeg_SOURCES
+		jim-nvenc.c
+		jim-nvenc-helpers.c)
+	list(APPEND obs-ffmpeg_HEADERS
+		jim-nvenc.h)
+endif()
+
 add_library(obs-ffmpeg MODULE
 	${obs-ffmpeg_HEADERS}
 	${obs-ffmpeg_SOURCES})

plugins/obs-ffmpeg/jim-nvenc-helpers.c

+#include "jim-nvenc.h"
+#include <util/platform.h>
+#include <util/threading.h>
+
+static void *nvenc_lib = NULL;
+static pthread_mutex_t init_mutex = PTHREAD_MUTEX_INITIALIZER;
+NV_ENCODE_API_FUNCTION_LIST nv = {NV_ENCODE_API_FUNCTION_LIST_VER};
+NV_CREATE_INSTANCE_FUNC nv_create_instance = NULL;
+
+bool load_nvenc_lib(void)
+{
+	if (sizeof(void*) == 8) {
+		nvenc_lib = os_dlopen("nvEncodeAPI64.dll");
+	} else {
+		nvenc_lib = os_dlopen("nvEncodeAPI.dll");
+	}
+
+	return !!nvenc_lib;
+}
+
+static void *load_nv_func(const char *func)
+{
+	void *func_ptr = os_dlsym(nvenc_lib, func);
+	if (!func_ptr) {
+		error("Could not load function: %s", func);
+	}
+	return func_ptr;
+}
+
+typedef NVENCSTATUS (NVENCAPI *NV_MAX_VER_FUNC)(uint32_t*);
+
+const char *nv_error_name(NVENCSTATUS err)
+{
+#define RETURN_CASE(x) \
+	case x: return #x
+
+	switch (err) {
+	RETURN_CASE(NV_ENC_SUCCESS);
+	RETURN_CASE(NV_ENC_ERR_NO_ENCODE_DEVICE);
+	RETURN_CASE(NV_ENC_ERR_UNSUPPORTED_DEVICE);
+	RETURN_CASE(NV_ENC_ERR_INVALID_ENCODERDEVICE);
+	RETURN_CASE(NV_ENC_ERR_INVALID_DEVICE);
+	RETURN_CASE(NV_ENC_ERR_DEVICE_NOT_EXIST);
+	RETURN_CASE(NV_ENC_ERR_INVALID_PTR);
+	RETURN_CASE(NV_ENC_ERR_INVALID_EVENT);
+	RETURN_CASE(NV_ENC_ERR_INVALID_PARAM);
+	RETURN_CASE(NV_ENC_ERR_INVALID_CALL);
+	RETURN_CASE(NV_ENC_ERR_OUT_OF_MEMORY);
+	RETURN_CASE(NV_ENC_ERR_ENCODER_NOT_INITIALIZED);
+	RETURN_CASE(NV_ENC_ERR_UNSUPPORTED_PARAM);
+	RETURN_CASE(NV_ENC_ERR_LOCK_BUSY);
+	RETURN_CASE(NV_ENC_ERR_NOT_ENOUGH_BUFFER);
+	RETURN_CASE(NV_ENC_ERR_INVALID_VERSION);
+	RETURN_CASE(NV_ENC_ERR_MAP_FAILED);
+	RETURN_CASE(NV_ENC_ERR_NEED_MORE_INPUT);
+	RETURN_CASE(NV_ENC_ERR_ENCODER_BUSY);
+	RETURN_CASE(NV_ENC_ERR_EVENT_NOT_REGISTERD);
+	RETURN_CASE(NV_ENC_ERR_GENERIC);
+	RETURN_CASE(NV_ENC_ERR_INCOMPATIBLE_CLIENT_KEY);
+	RETURN_CASE(NV_ENC_ERR_UNIMPLEMENTED);
+	RETURN_CASE(NV_ENC_ERR_RESOURCE_REGISTER_FAILED);
+	RETURN_CASE(NV_ENC_ERR_RESOURCE_NOT_REGISTERED);
+	RETURN_CASE(NV_ENC_ERR_RESOURCE_NOT_MAPPED);
+	}
+#undef RETURN_CASE
+
+	return "Unknown Error";
+}
+
+static inline bool init_nvenc_internal(void)
+{
+	static bool initialized = false;
+	static bool success = false;
+
+	if (initialized)
+		return success;
+	initialized = true;
+
+	NV_MAX_VER_FUNC nv_max_ver = (NV_MAX_VER_FUNC)
+		load_nv_func("NvEncodeAPIGetMaxSupportedVersion");
+	if (!nv_max_ver) {
+		return false;
+	}
+
+	uint32_t ver = 0;
+	if (NV_FAILED(nv_max_ver(&ver))) {
+		return false;
+	}
+
+	uint32_t cur_ver =
+		(NVENCAPI_MAJOR_VERSION << 4) | NVENCAPI_MINOR_VERSION;
+	if (cur_ver > ver) {
+		error("Current driver version does not support this NVENC "
+				"version, please upgrade your driver");
+		return false;
+	}
+
+	nv_create_instance = (NV_CREATE_INSTANCE_FUNC)
+		load_nv_func("NvEncodeAPICreateInstance");
+	if (!nv_create_instance) {
+		return false;
+	}
+
+	if (NV_FAILED(nv_create_instance(&nv))) {
+		return false;
+	}
+
+	success = true;
+	return true;
+}
+
+bool init_nvenc(void)
+{
+	bool success;
+
+	pthread_mutex_lock(&init_mutex);
+	success = init_nvenc_internal();
+	pthread_mutex_unlock(&init_mutex);
+
+	return success;
+}
+
+extern struct obs_encoder_info nvenc_info;
+
+void jim_nvenc_load(void)
+{
+	pthread_mutex_init(&init_mutex, NULL);
+	obs_register_encoder(&nvenc_info);
+}
+
+void jim_nvenc_unload(void)
+{
+	pthread_mutex_destroy(&init_mutex);
+}

plugins/obs-ffmpeg/jim-nvenc.h

+#pragma once
+
+#define WIN32_LEAN_AND_MEAN
+#include <windows.h>
+
+#include <obs-module.h>
+#include "nvEncodeAPI.h"
+
+#define do_log(level, format, ...) \
+	blog(level, "[jim-nvenc] " format, ##__VA_ARGS__)
+
+#define error(format, ...) do_log(LOG_ERROR,   format, ##__VA_ARGS__)
+#define warn(format, ...)  do_log(LOG_WARNING, format, ##__VA_ARGS__)
+#define info(format, ...)  do_log(LOG_INFO,    format, ##__VA_ARGS__)
+#define debug(format, ...) do_log(LOG_DEBUG,   format, ##__VA_ARGS__)
+
+typedef NVENCSTATUS (NVENCAPI *NV_CREATE_INSTANCE_FUNC)(NV_ENCODE_API_FUNCTION_LIST*);
+
+extern const char *nv_error_name(NVENCSTATUS err);
+extern NV_ENCODE_API_FUNCTION_LIST nv;
+extern NV_CREATE_INSTANCE_FUNC nv_create_instance;
+extern bool init_nvenc(void);
+
+static inline bool nv_failed(NVENCSTATUS err, const char *func,
+		const char *call)
+{
+	if (err == NV_ENC_SUCCESS)
+		return false;
+
+	error("%s: %s failed: %d (%s)", func, call, (int)err,
+			nv_error_name(err));
+	return true;
+}
+
+#define NV_FAILED(x) nv_failed(x, __FUNCTION__, #x)

plugins/obs-ffmpeg/obs-ffmpeg.c

@@ -223,6 +223,10 @@ finish:
 }
 #endif
 
+#ifdef _WIN32
+extern bool load_nvenc_lib(void);
+#endif
+
 static bool nvenc_supported(void)
 {
 	av_register_all();
@@ -241,11 +245,9 @@ static bool nvenc_supported(void)
 	if (!nvenc_device_available()) {
 		goto cleanup;
 	}
-
-	if (sizeof(void*) == 8) {
-		lib = os_dlopen("nvEncodeAPI64.dll");
-	} else {
-		lib = os_dlopen("nvEncodeAPI.dll");
+	if (load_nvenc_lib()) {
+		success = true;
+		goto finish;
 	}
 #else
 	lib = os_dlopen("libnvidia-encode.so.1");
@@ -258,6 +260,7 @@ static bool nvenc_supported(void)
 cleanup:
 	if (lib)
 		os_dlclose(lib);
+finish:
 	profile_end(nvenc_check_name);
 	return success;
 }
@@ -272,6 +275,11 @@ static bool vaapi_supported(void)
 }
 #endif
 
+#ifdef _WIN32
+extern void jim_nvenc_load(void);
+extern void jim_nvenc_unload(void);
+#endif
+
 bool obs_module_load(void)
 {
 	da_init(active_log_contexts);
@@ -288,6 +296,11 @@ bool obs_module_load(void)
 #ifndef __APPLE__
 	if (nvenc_supported()) {
 		blog(LOG_INFO, "NVENC supported");
+#ifdef _WIN32
+		if (get_win_ver_int() > 0x0601) {
+			jim_nvenc_load();
+		}
+#endif
 		obs_register_encoder(&nvenc_encoder_info);
 	}
 #if !defined(_WIN32) && defined(LIBAVUTIL_VAAPI_AVAILABLE)
@@ -317,4 +330,8 @@ void obs_module_unload(void)
 
 	da_free(active_log_contexts);
 	da_free(cached_log_contexts);
+
+#ifdef _WIN32
+	jim_nvenc_unload();
+#endif
 }