diff --git a/libavcodec/cbs_av1_syntax_template.c b/libavcodec/cbs_av1_syntax_template.c
index b04cd51d5586432a4e04b04b4a329c3abe0c10c4..806b302de6afa6219bb93f8eb7dd0e9171a3f3c5 100644
--- a/libavcodec/cbs_av1_syntax_template.c
+++ b/libavcodec/cbs_av1_syntax_template.c
@@ -419,16 +419,17 @@ static int FUNC(frame_size_with_refs)(CodedBitstreamContext *ctx, RWContext *rw,
     for (i = 0; i < AV1_REFS_PER_FRAME; i++) {
         flags(found_ref[i], 1, i);
         if (current->found_ref[i]) {
-            AV1ReferenceFrameState *ref =
-                &priv->ref[current->ref_frame_idx[i]];
+            AV1ReferenceFrameState *ref;
 
-            if (!ref->valid) {
+            if (current->ref_frame_idx[i] < 0 ||
+                !priv->ref[current->ref_frame_idx[i]].valid) {
                 av_log(ctx->log_ctx, AV_LOG_ERROR,
                        "Missing reference frame needed for frame size "
                        "(ref = %d, ref_frame_idx = %d).\n",
                        i, current->ref_frame_idx[i]);
                 return AVERROR_INVALIDDATA;
             }
+            ref = &priv->ref[current->ref_frame_idx[i]];
 
             priv->upscaled_width = ref->upscaled_width;
             priv->frame_width    = ref->frame_width;