From d4cd8e7f6a49aeb6728af9acc1d9ac3eda431094 Mon Sep 17 00:00:00 2001 From: Mark Thompson Date: Thu, 2 Jun 2016 19:06:08 +0100 Subject: [PATCH] vaapi_encode_h26[45]: Reject bitrate targets higher than 2^31 --- libavcodec/vaapi_encode_h264.c | 6 ++++++ libavcodec/vaapi_encode_h265.c | 6 ++++++ 2 files changed, 12 insertions(+) diff --git a/libavcodec/vaapi_encode_h264.c b/libavcodec/vaapi_encode_h264.c index dc7774b9f1..2e2052bb71 100644 --- a/libavcodec/vaapi_encode_h264.c +++ b/libavcodec/vaapi_encode_h264.c @@ -731,6 +731,12 @@ static av_cold int vaapi_encode_h264_init_constant_bitrate(AVCodecContext *avctx int hrd_buffer_size; int hrd_initial_buffer_fullness; + if (avctx->bit_rate > INT32_MAX) { + av_log(avctx, AV_LOG_ERROR, "Target bitrate of 2^31 bps or " + "higher is not supported.\n"); + return AVERROR(EINVAL); + } + if (avctx->rc_buffer_size) hrd_buffer_size = avctx->rc_buffer_size; else diff --git a/libavcodec/vaapi_encode_h265.c b/libavcodec/vaapi_encode_h265.c index 17cd900b30..e4d21f0f2f 100644 --- a/libavcodec/vaapi_encode_h265.c +++ b/libavcodec/vaapi_encode_h265.c @@ -1158,6 +1158,12 @@ static av_cold int vaapi_encode_h265_init_constant_bitrate(AVCodecContext *avctx int hrd_buffer_size; int hrd_initial_buffer_fullness; + if (avctx->bit_rate > INT32_MAX) { + av_log(avctx, AV_LOG_ERROR, "Target bitrate of 2^31 bps or " + "higher is not supported.\n"); + return AVERROR(EINVAL); + } + if (avctx->rc_buffer_size) hrd_buffer_size = avctx->rc_buffer_size; else -- GitLab