obs-ffmpeg: Improve NVENC detection

Certain NVIDIA GPUs don't support NVENC, but ship with the NVENC
library, causing OBS to mistakenly think that NVENC is available when it
actually isn't.

Closes jp9000/obs-studio#1087

plugins/obs-ffmpeg/obs-ffmpeg.c

@@ -116,6 +116,8 @@ cleanup:
 	destroy_log_context(log_context);
 }
 
+static const char *nvenc_check_name = "nvenc_check";
+
 static bool nvenc_supported(void)
 {
 	AVCodec *nvenc = avcodec_find_encoder_by_name("nvenc_h264");
@@ -133,8 +135,35 @@ static bool nvenc_supported(void)
 #else
 	lib = os_dlopen("libnvidia-encode.so.1");
 #endif
+	if (!lib)
+		return false;
+
 	os_dlclose(lib);
-	return !!lib;
+
+	bool success = false;
+	profile_start(nvenc_check_name);
+
+	AVCodecContext *context = avcodec_alloc_context3(nvenc);
+	if (!context)
+		goto cleanup;
+
+	context->bit_rate = 5000;
+	context->width = 640;
+	context->height = 480;
+	context->time_base = (AVRational) { 1, 25 };
+	context->pix_fmt = AV_PIX_FMT_YUV420P;
+
+	if (avcodec_open2(context, nvenc, NULL) < 0)
+		goto cleanup;
+
+	success = true;
+
+cleanup:
+	if (context)
+		avcodec_free_context(&context);
+
+	profile_end(nvenc_check_name);
+	return success;
 }
 
 bool obs_module_load(void)