obs-ffmpeg: Change NVENC to use rate control

Instead of using an option that turns CBR on/off, adds rate control
methods: VBR, CBR, CQP, Lossless.

This moves lossless from being a preset to being a rate control method.

plugins/obs-ffmpeg/data/locale/en-US.ini

@@ -2,8 +2,9 @@ FFmpegOutput="FFmpeg Output"
 FFmpegAAC="FFmpeg Default AAC Encoder"
 Bitrate="Bitrate"
 Preset="Preset"
-UseCBR="Use CBR"
+RateControl="Rate Control"
 KeyframeIntervalSec="Keyframe Interval (seconds, 0=auto)"
+Lossless="Lossless"
 
 NVENC.Use2Pass="Use Two-Pass Encoding"
 NVENC.Preset.default="Default"
@@ -13,8 +14,6 @@ NVENC.Preset.bd="Bluray"
 NVENC.Preset.ll="Low-Latency"
 NVENC.Preset.llhq="Low-Latency High Quality"
 NVENC.Preset.llhp="Low-Latency High Performance"
-NVENC.Preset.lossless="Lossless"
-NVENC.Preset.losslesshp="Lossless High Performance"
 NVENC.Level="Level"
 
 FFmpegSource="Media Source"

plugins/obs-ffmpeg/obs-ffmpeg-nvenc.c

@@ -117,44 +117,78 @@ static bool nvenc_init_codec(struct nvenc_encoder *enc)
 	return true;
 }
 
+enum RC_MODE {
+	RC_MODE_CBR,
+	RC_MODE_VBR,
+	RC_MODE_CQP,
+	RC_MODE_LOSSLESS
+};
+
 static bool nvenc_update(void *data, obs_data_t *settings)
 {
 	struct nvenc_encoder *enc = data;
 
+	const char *rc = obs_data_get_string(settings, "rate_control");
 	int bitrate = (int)obs_data_get_int(settings, "bitrate");
+	int cqp = (int)obs_data_get_int(settings, "cqp");
 	int keyint_sec = (int)obs_data_get_int(settings, "keyint_sec");
 	const char *preset = obs_data_get_string(settings, "preset");
 	const char *profile = obs_data_get_string(settings, "profile");
 	const char *level = obs_data_get_string(settings, "level");
-	bool cbr = obs_data_get_bool(settings, "cbr");
 	bool twopass = obs_data_get_bool(settings, "2pass");
 	int gpu = (int)obs_data_get_int(settings, "gpu");
+	bool cbr_override = obs_data_get_bool(settings, "cbr");
 
 	video_t *video = obs_encoder_video(enc->encoder);
 	const struct video_output_info *voi = video_output_get_info(video);
 	struct video_scale_info info;
 
+	/* XXX: "cbr" setting has been deprecated */
+	if (cbr_override) {
+		warn("\"cbr\" setting has been deprecated for all encoders!  "
+		     "Please set \"rate_control\" to \"CBR\" instead.  "
+		     "Forcing CBR mode.  "
+		     "(Note to all: this is why you shouldn't use strings for "
+		     "common settings)");
+		rc = "CBR";
+	}
+
 	info.format = voi->format;
 	info.colorspace = voi->colorspace;
 	info.range = voi->range;
 
 	nvenc_video_info(enc, &info);
+	av_opt_set_int(enc->context->priv_data, "cbr", false, 0);
 
-	av_opt_set(enc->context->priv_data, "preset", preset, 0);
-	av_opt_set(enc->context->priv_data, "level", level, 0);
-	av_opt_set_int(enc->context->priv_data, "2pass", twopass, 0);
-	av_opt_set_int(enc->context->priv_data, "gpu", gpu, 0);
+	if (astrcmpi(rc, "cqp") == 0) {
+		bitrate = 0;
+		enc->context->global_quality = cqp;
 
-	if (astrcmp_n(preset, "lossless", 8) == 0) {
-		enc->context->bit_rate = 0;
+	} else if (astrcmpi(rc, "lossless") == 0) {
 		bitrate = 0;
-	} else {
-		enc->context->bit_rate = bitrate * 1000;
+		cqp = 0;
 
-		av_opt_set(enc->context->priv_data, "profile", profile, 0);
-		av_opt_set_int(enc->context->priv_data, "cbr", cbr, 0);
+		bool hp = (astrcmpi(preset, "hp") == 0 ||
+		           astrcmpi(preset, "llhp") == 0);
+
+		av_opt_set(enc->context->priv_data, "profile",
+				hp ? "losslesshp" : "lossless", 0);
+
+	} else if (astrcmpi(rc, "vbr") != 0) { /* CBR by default */
+		av_opt_set_int(enc->context->priv_data, "cbr", true, 0);
+		enc->context->rc_max_rate = bitrate * 1000;
+		enc->context->rc_min_rate = bitrate * 1000;
+		cqp = 0;
 	}
 
+
+	av_opt_set(enc->context->priv_data, "preset", preset, 0);
+	av_opt_set(enc->context->priv_data, "level", level, 0);
+	av_opt_set_int(enc->context->priv_data, "2pass", twopass, 0);
+	av_opt_set_int(enc->context->priv_data, "gpu", gpu, 0);
+
+	enc->context->bit_rate = bitrate * 1000;
+	enc->context->rc_buffer_size = bitrate * 1000;
 	enc->context->width = obs_encoder_get_width(enc->encoder);
 	enc->context->height = obs_encoder_get_height(enc->encoder);
 	enc->context->time_base = (AVRational){voi->fps_den, voi->fps_num};
@@ -173,20 +207,20 @@ static bool nvenc_update(void *data, obs_data_t *settings)
 	enc->height = enc->context->height;
 
 	info("settings:\n"
-	     "\tbitrate:     %d\n"
-	     "\tkeyint:      %d\n"
-	     "\tpreset:      %s\n"
-	     "\tprofile:     %s\n"
-	     "\tlevel:       %s\n"
-	     "\twidth:       %d\n"
-	     "\theight:      %d\n"
-	     "\tcbr:         %s\n"
-	     "\t2-pass:      %s\n"
-	     "\tGPU:         %d\n",
-	     bitrate, enc->context->gop_size,
+	     "\trate_control: %s\n"
+	     "\tbitrate:      %d\n"
+	     "\tcqp:          %d\n"
+	     "\tkeyint:       %d\n"
+	     "\tpreset:       %s\n"
+	     "\tprofile:      %s\n"
+	     "\tlevel:        %s\n"
+	     "\twidth:        %d\n"
+	     "\theight:       %d\n"
+	     "\t2-pass:       %s\n"
+	     "\tGPU:          %d\n",
+	     rc, bitrate, cqp, enc->context->gop_size,
 	     preset, profile, level,
 	     enc->context->width, enc->context->height,
-	     cbr ? "true" : "false",
 	     twopass ? "true" : "false",
 	     gpu);
 
@@ -321,21 +355,36 @@ static void nvenc_defaults(obs_data_t *settings)
 {
 	obs_data_set_default_int(settings, "bitrate", 2500);
 	obs_data_set_default_int(settings, "keyint_sec", 0);
+	obs_data_set_default_int(settings, "cqp", 23);
+	obs_data_set_default_string(settings, "rate_control", "CBR");
 	obs_data_set_default_string(settings, "preset", "default");
 	obs_data_set_default_string(settings, "profile", "main");
 	obs_data_set_default_string(settings, "level", "auto");
-	obs_data_set_default_bool(settings, "cbr", false);
 	obs_data_set_default_bool(settings, "2pass", true);
 	obs_data_set_default_int(settings, "gpu", 0);
 }
 
-static bool preset_modified(obs_properties_t *props, obs_property_t *p,
+static bool rate_control_modified(obs_properties_t *ppts, obs_property_t *p,
 		obs_data_t *settings)
 {
-	const char *preset = obs_data_get_string(settings, "preset");
-	bool lossless = preset && astrcmp_n(preset, "lossless", 8) == 0;
-	p = obs_properties_get(props, "bitrate");
-	obs_property_set_visible(p, !lossless);
+	const char *rc = obs_data_get_string(settings, "rate_control");
+	bool cqp = astrcmpi(rc, "CQP") == 0;
+	bool lossless = astrcmpi(rc, "lossless") == 0;
+	size_t count;
+
+	p = obs_properties_get(ppts, "bitrate");
+	obs_property_set_visible(p, !cqp && !lossless);
+	p = obs_properties_get(ppts, "cqp");
+	obs_property_set_visible(p, cqp);
+
+	p = obs_properties_get(ppts, "preset");
+	count = obs_property_list_item_count(p);
+
+	for (size_t i = 0; i < count; i++) {
+		bool compatible = (i == 0 || i == 2);
+		obs_property_list_item_disable(p, i, lossless && !compatible);
+	}
+
 	return true;
 }
 
@@ -346,9 +395,22 @@ static obs_properties_t *nvenc_properties(void *unused)
 	obs_properties_t *props = obs_properties_create();
 	obs_property_t *p;
 
+	p = obs_properties_add_list(props, "rate_control",
+			obs_module_text("RateControl"),
+			OBS_COMBO_TYPE_LIST, OBS_COMBO_FORMAT_STRING);
+	obs_property_list_add_string(p, "CBR", "CBR");
+	obs_property_list_add_string(p, "VBR", "VBR");
+	obs_property_list_add_string(p, "CQP", "CQP");
+	obs_property_list_add_string(p, obs_module_text("Lossless"),
+			"lossless");
+
+	obs_property_set_modified_callback(p, rate_control_modified);
+
 	obs_properties_add_int(props, "bitrate",
 			obs_module_text("Bitrate"), 50, 300000, 50);
 
+	obs_properties_add_int(props, "cqp", "CQP", 0, 50, 1);
+
 	obs_properties_add_int(props, "keyint_sec",
 			obs_module_text("KeyframeIntervalSec"), 0, 10, 1);
 
@@ -365,12 +427,8 @@ static obs_properties_t *nvenc_properties(void *unused)
 	add_preset("ll");
 	add_preset("llhq");
 	add_preset("llhp");
-	add_preset("lossless");
-	add_preset("losslesshp");
 #undef add_preset
 
-	obs_property_set_modified_callback(p, preset_modified);
-
 	p = obs_properties_add_list(props, "profile", obs_module_text("Profile"),
 			OBS_COMBO_TYPE_LIST, OBS_COMBO_FORMAT_STRING);
 
@@ -411,7 +469,6 @@ static obs_properties_t *nvenc_properties(void *unused)
 
 	obs_properties_add_bool(props, "2pass",
 			obs_module_text("NVENC.Use2Pass"));
-	obs_properties_add_bool(props, "cbr", obs_module_text("UseCBR"));
 	obs_properties_add_int(props, "gpu", obs_module_text("GPU"), 0, 8, 1);
 
 	return props;