testing all dquant for qp_rd fixes the qp runaway to 31 bug

Originally committed as revision 7126 to svn://svn.ffmpeg.org/ffmpeg/trunk

libavcodec/mpegvideo.c

@@ -5226,9 +5226,10 @@ static int encode_thread(AVCodecContext *c, void *arg){
                 if(s->flags & CODEC_FLAG_QP_RD){
                     if(best_s.mv_type==MV_TYPE_16X16 && !(best_s.mv_dir&MV_DIRECT)){
                         const int last_qp= backup_s.qscale;
-                        int dquant, dir, qp, dc[6];
+                        int dquant, qpi, qp, dc[6];
                         DCTELEM ac[6][16];
                         const int mvdir= (best_s.mv_dir&MV_DIR_BACKWARD) ? 1 : 0;
+                        static const int dquant_tab[4]={-1,1,-2,2};
 
                         assert(backup_s.dquant == 0);
 
@@ -5241,12 +5242,11 @@ static int encode_thread(AVCodecContext *c, void *arg){
                         s->mv[1][0][0] = best_s.mv[1][0][0];
                         s->mv[1][0][1] = best_s.mv[1][0][1];
 
-                        dir= s->pict_type == B_TYPE ? 2 : 1;
-                        if(last_qp + dir > s->avctx->qmax) dir= -dir;
-                        for(dquant= dir; dquant<=2 && dquant>=-2; dquant += dir){
+                        qpi = s->pict_type == B_TYPE ? 2 : 0;
+                        for(dquant= dquant_tab[qpi]; qpi<4; qpi++){
                             qp= last_qp + dquant;
                             if(qp < s->avctx->qmin || qp > s->avctx->qmax)
-                                break;
+                                continue;
                             backup_s.dquant= dquant;
                             if(s->mb_intra && s->dc_val[0]){
                                 for(i=0; i<6; i++){
@@ -5264,11 +5264,6 @@ static int encode_thread(AVCodecContext *c, void *arg){
                                         memcpy(s->ac_val[0][s->block_index[i]], ac[i], sizeof(DCTELEM)*16);
                                     }
                                 }
-                                if(dir > 0 && dquant==dir){
-                                    dquant= 0;
-                                    dir= -dir;
-                                }else
-                                    break;
                             }
                         }
                         qp= best_s.qscale;