Commit 4a15e557 authored by Scott LaVarnway's avatar Scott LaVarnway

Possible bug in vp8cx_encode_intra_macro_block

vp8_pick_intra4x4mby_modes uses the passed in distortion
for an early breakout.  The best distortion was never saved
and the distortion for TM_PRED was always used.

Change-Id: Idbaf73027408a4bba26601713725191a5d7b325e
parent 60fde4d3
......@@ -1211,13 +1211,15 @@ int vp8cx_encode_intra_macro_block(VP8_COMP *cpi, MACROBLOCK *x, TOKENEXTRA **t)
int rate2, distortion2;
int rate2, best_distortion;
MB_PREDICTION_MODE mode, best_mode = DC_PRED;
int this_rd;
Error16x16 = INT_MAX;
for (mode = DC_PRED; mode <= TM_PRED; mode ++)
int distortion2;
x->e_mbd.mode_info_context->mbmi.mode = mode;
distortion2 = VARIANCE_INVOKE(&cpi->rtcd.variance, get16x16prederror)(x->src.y_buffer, x->src.y_stride, x->e_mbd.predictor, 16, 0x7fffffff);
......@@ -1228,15 +1230,16 @@ int vp8cx_encode_intra_macro_block(VP8_COMP *cpi, MACROBLOCK *x, TOKENEXTRA **t)
Error16x16 = this_rd;
best_mode = mode;
best_distortion = distortion2;
vp8_pick_intra4x4mby_modes(IF_RTCD(&cpi->rtcd), x, &rate2, &distortion2);
vp8_pick_intra4x4mby_modes(IF_RTCD(&cpi->rtcd), x, &rate2, &best_distortion);
if (distortion2 == INT_MAX)
if (best_distortion == INT_MAX)
Error4x4 = INT_MAX;
Error4x4 = RD_ESTIMATE(x->rdmult, x->rddiv, rate2, distortion2);
Error4x4 = RD_ESTIMATE(x->rdmult, x->rddiv, rate2, best_distortion);
if (Error4x4 < Error16x16)
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment