obs-ffmpeg: use avcodec_find_best_pix_fmt_of_list

The built-in format selector failed in certain cases like UtVideo now
using a differently packeg RGB format. FFmpeg has a format selection
functionality built-in that does pick the correct format however
(avcodec_find_best_pix_fmt_of_list), so we can simply use that instead.

plugins/obs-ffmpeg/CMakeLists.txt

@@ -21,8 +21,7 @@ set(obs-ffmpeg_config_HEADERS
 
 set(obs-ffmpeg_HEADERS
 	obs-ffmpeg-formats.h
-	obs-ffmpeg-compat.h
-	closest-pixel-format.h)
+	obs-ffmpeg-compat.h)
 
 set(obs-ffmpeg_SOURCES
 	obs-ffmpeg.c

plugins/obs-ffmpeg/closest-pixel-format.h

-#pragma once
-
-static const enum AVPixelFormat i420_formats[] = {
-	AV_PIX_FMT_YUV420P, AV_PIX_FMT_NV12,    AV_PIX_FMT_NV21,
-	AV_PIX_FMT_YUYV422, AV_PIX_FMT_UYVY422, AV_PIX_FMT_YUV422P,
-	AV_PIX_FMT_YUV444P, AV_PIX_FMT_NONE};
-
-static const enum AVPixelFormat nv12_formats[] = {
-	AV_PIX_FMT_NV12,    AV_PIX_FMT_NV21,    AV_PIX_FMT_YUV420P,
-	AV_PIX_FMT_YUYV422, AV_PIX_FMT_UYVY422, AV_PIX_FMT_YUV444P,
-	AV_PIX_FMT_NONE};
-
-static const enum AVPixelFormat i444_formats[] = {
-	AV_PIX_FMT_YUV444P, AV_PIX_FMT_RGBA,    AV_PIX_FMT_BGRA,
-	AV_PIX_FMT_YUYV422, AV_PIX_FMT_UYVY422, AV_PIX_FMT_NV12,
-	AV_PIX_FMT_NV21,    AV_PIX_FMT_NONE};
-
-static const enum AVPixelFormat yuy2_formats[] = {
-	AV_PIX_FMT_YUYV422, AV_PIX_FMT_UYVY422, AV_PIX_FMT_NV12,
-	AV_PIX_FMT_NV21,    AV_PIX_FMT_YUV420P, AV_PIX_FMT_YUV444P,
-	AV_PIX_FMT_NONE};
-
-static const enum AVPixelFormat uyvy_formats[] = {
-	AV_PIX_FMT_UYVY422, AV_PIX_FMT_YUYV422, AV_PIX_FMT_NV12,
-	AV_PIX_FMT_NV21,    AV_PIX_FMT_YUV420P, AV_PIX_FMT_YUV444P,
-	AV_PIX_FMT_NONE};
-
-static const enum AVPixelFormat rgba_formats[] = {
-	AV_PIX_FMT_RGBA,    AV_PIX_FMT_BGRA,    AV_PIX_FMT_YUV444P,
-	AV_PIX_FMT_YUYV422, AV_PIX_FMT_UYVY422, AV_PIX_FMT_NV12,
-	AV_PIX_FMT_NV21,    AV_PIX_FMT_NONE};
-
-static const enum AVPixelFormat bgra_formats[] = {
-	AV_PIX_FMT_BGRA,    AV_PIX_FMT_RGBA,    AV_PIX_FMT_YUV444P,
-	AV_PIX_FMT_YUYV422, AV_PIX_FMT_UYVY422, AV_PIX_FMT_NV12,
-	AV_PIX_FMT_NV21,    AV_PIX_FMT_NONE};
-
-static enum AVPixelFormat get_best_format(const enum AVPixelFormat *best,
-					  const enum AVPixelFormat *formats)
-{
-	while (*best != AV_PIX_FMT_NONE) {
-		enum AVPixelFormat best_format = *best;
-		const enum AVPixelFormat *cur_formats = formats;
-
-		while (*cur_formats != AV_PIX_FMT_NONE) {
-			enum AVPixelFormat avail_format = *cur_formats;
-
-			if (best_format == avail_format)
-				return best_format;
-
-			cur_formats++;
-		}
-
-		best++;
-	}
-
-	return AV_PIX_FMT_NONE;
-}
-
-static inline enum AVPixelFormat
-get_closest_format(enum AVPixelFormat format, const enum AVPixelFormat *formats)
-{
-	enum AVPixelFormat best_format = AV_PIX_FMT_NONE;
-
-	if (!formats || formats[0] == AV_PIX_FMT_NONE)
-		return format;
-
-	switch ((int)format) {
-
-	case AV_PIX_FMT_YUV444P:
-		best_format = get_best_format(i444_formats, formats);
-		break;
-	case AV_PIX_FMT_YUV420P:
-		best_format = get_best_format(i420_formats, formats);
-		break;
-	case AV_PIX_FMT_NV12:
-		best_format = get_best_format(nv12_formats, formats);
-		break;
-	case AV_PIX_FMT_YUYV422:
-		best_format = get_best_format(yuy2_formats, formats);
-		break;
-	case AV_PIX_FMT_UYVY422:
-		best_format = get_best_format(uyvy_formats, formats);
-		break;
-	case AV_PIX_FMT_RGBA:
-		best_format = get_best_format(rgba_formats, formats);
-		break;
-	case AV_PIX_FMT_BGRA:
-		best_format = get_best_format(bgra_formats, formats);
-		break;
-	}
-
-	return (best_format == AV_PIX_FMT_NONE) ? formats[0] : best_format;
-}

plugins/obs-ffmpeg/obs-ffmpeg-output.c

@@ -28,7 +28,6 @@
 #include <libswscale/swscale.h>
 
 #include "obs-ffmpeg-formats.h"
-#include "closest-pixel-format.h"
 #include "obs-ffmpeg-compat.h"
 
 struct ffmpeg_cfg {
@@ -279,8 +278,8 @@ static bool create_video_stream(struct ffmpeg_data *data)
 			data->config.video_encoder))
 		return false;
 
-	closest_format =
-		get_closest_format(data->config.format, data->vcodec->pix_fmts);
+	closest_format = avcodec_find_best_pix_fmt_of_list(
+		data->vcodec->pix_fmts, data->config.format, 0, NULL);
 
 	context = data->video->codec;
 	context->bit_rate = data->config.video_bitrate * 1000;