h2645_parse: support badly muxed mp4 streams

Some streams contain an additional AnnexB NAL inside the mp4/nalff NALU.
This commonly occurs in interlaced streams where both fields are packed
into the same MP4 NAL with an AnnexB startcode in between.

Port handling of this format from the previous h264 nal handling.

Fixes trac #5529

libavcodec/h2645_parse.c

@@ -250,6 +250,7 @@ int ff_h2645_packet_split(H2645Packet *pkt, const uint8_t *buf, int length,
                           enum AVCodecID codec_id)
 {
     int consumed, ret = 0;
+    const uint8_t *next_avc = is_nalff ? buf : buf + length;
 
     pkt->nb_nals = 0;
     while (length >= 4) {
@@ -257,7 +258,7 @@ int ff_h2645_packet_split(H2645Packet *pkt, const uint8_t *buf, int length,
         int extract_length = 0;
         int skip_trailing_zeros = 1;
 
-        if (is_nalff) {
+        if (buf >= next_avc) {
             int i;
             for (i = 0; i < nal_length_size; i++)
                 extract_length = (extract_length << 8) | buf[i];
@@ -268,6 +269,7 @@ int ff_h2645_packet_split(H2645Packet *pkt, const uint8_t *buf, int length,
                 av_log(logctx, AV_LOG_ERROR, "Invalid NAL unit size.\n");
                 return AVERROR_INVALIDDATA;
             }
+            next_avc = buf + extract_length;
         } else {
             /* search start code */
             while (buf[0] != 0 || buf[1] != 0 || buf[2] != 1) {
@@ -282,12 +284,21 @@ int ff_h2645_packet_split(H2645Packet *pkt, const uint8_t *buf, int length,
                         av_log(logctx, AV_LOG_ERROR, "No start code is found.\n");
                         return AVERROR_INVALIDDATA;
                     }
-                }
+                } else if (buf >= (next_avc - 3))
+                    break;
             }
 
             buf           += 3;
             length        -= 3;
             extract_length = length;
+
+            if (buf >= next_avc) {
+                /* skip to the start of the next NAL */
+                int offset = next_avc - buf;
+                buf    += offset;
+                length -= offset;
+                continue;
+            }
         }
 
         if (pkt->nals_allocated < pkt->nb_nals + 1) {
@@ -315,6 +326,11 @@ int ff_h2645_packet_split(H2645Packet *pkt, const uint8_t *buf, int length,
         if (consumed < 0)
             return consumed;
 
+        if (is_nalff && (extract_length != consumed) && extract_length)
+            av_log(logctx, AV_LOG_DEBUG,
+                   "NALFF: Consumed only %d bytes instead of %d\n",
+                   consumed, extract_length);
+
         pkt->nb_nals++;
 
         /* see commit 3566042a0 */