ac3: detect dba errors and prevent writing past end of array

Originally committed as revision 16034 to svn://svn.ffmpeg.org/ffmpeg/trunk

libavcodec/ac3.c

@@ -80,7 +80,7 @@ void ff_ac3_bit_alloc_calc_psd(int8_t *exp, int start, int end, int16_t *psd,
     } while (end > band_start_tab[k]);
 }
 
-void ff_ac3_bit_alloc_calc_mask(AC3BitAllocParameters *s, int16_t *band_psd,
+int ff_ac3_bit_alloc_calc_mask(AC3BitAllocParameters *s, int16_t *band_psd,
                                 int start, int end, int fast_gain, int is_lfe,
                                 int dba_mode, int dba_nsegs, uint8_t *dba_offsets,
                                 uint8_t *dba_lengths, uint8_t *dba_values,
@@ -156,9 +156,13 @@ void ff_ac3_bit_alloc_calc_mask(AC3BitAllocParameters *s, int16_t *band_psd,
 
     if (dba_mode == DBA_REUSE || dba_mode == DBA_NEW) {
         int band, seg, delta;
+        if (dba_nsegs >= 8)
+            return -1;
         band = 0;
-        for (seg = 0; seg < FFMIN(8, dba_nsegs); seg++) {
-            band = FFMIN(49, band + dba_offsets[seg]);
+        for (seg = 0; seg < dba_nsegs; seg++) {
+            band += dba_offsets[seg];
+            if (band >= 50 || dba_lengths[seg] > 50-band)
+                return -1;
             if (dba_values[seg] >= 4) {
                 delta = (dba_values[seg] - 3) << 7;
             } else {
@@ -170,6 +174,7 @@ void ff_ac3_bit_alloc_calc_mask(AC3BitAllocParameters *s, int16_t *band_psd,
             }
         }
     }
+    return 0;
 }
 
 void ff_ac3_bit_alloc_calc_bap(int16_t *mask, int16_t *psd, int start, int end,

libavcodec/ac3.h

@@ -149,8 +149,9 @@ void ff_ac3_bit_alloc_calc_psd(int8_t *exp, int start, int end, int16_t *psd,
  * @param[in]  dba_lengths  length of each segment
  * @param[in]  dba_values   delta bit allocation for each segment
  * @param[out] mask         calculated masking curve
+ * @return returns 0 for success, non-zero for error
  */
-void ff_ac3_bit_alloc_calc_mask(AC3BitAllocParameters *s, int16_t *band_psd,
+int ff_ac3_bit_alloc_calc_mask(AC3BitAllocParameters *s, int16_t *band_psd,
                                 int start, int end, int fast_gain, int is_lfe,
                                 int dba_mode, int dba_nsegs, uint8_t *dba_offsets,
                                 uint8_t *dba_lengths, uint8_t *dba_values,

libavcodec/ac3dec.c

@@ -1133,12 +1133,15 @@ static int decode_audio_block(AC3DecodeContext *s, int blk)
         if(bit_alloc_stages[ch] > 1) {
             /* Compute excitation function, Compute masking curve, and
                Apply delta bit allocation */
-            ff_ac3_bit_alloc_calc_mask(&s->bit_alloc_params, s->band_psd[ch],
+            if (ff_ac3_bit_alloc_calc_mask(&s->bit_alloc_params, s->band_psd[ch],
                                        s->start_freq[ch], s->end_freq[ch],
                                        s->fast_gain[ch], (ch == s->lfe_ch),
                                        s->dba_mode[ch], s->dba_nsegs[ch],
                                        s->dba_offsets[ch], s->dba_lengths[ch],
-                                       s->dba_values[ch], s->mask[ch]);
+                                       s->dba_values[ch], s->mask[ch])) {
+                av_log(s->avctx, AV_LOG_ERROR, "error in bit allocation\n");
+                return -1;
+            }
         }
         if(bit_alloc_stages[ch] > 0) {
             /* Compute bit allocation */