Commit cb969895 authored by John Koleszar's avatar John Koleszar
Browse files

fix uninitialized read in encode breakout

Change I3430820b performed an uninitialized read when
encode_breakout == 0, since the sum and sse wouldn't be set:

   if(x->encode_breakout)
       VARIANCE_INVOKE(..., get16x16var)(..., &sum, &sse);
   if (cpi->active_map_enabled && x->active_ptr[0] == 0) {
       ...
   } else if (sse < x->encode_breakout)

Change-Id: I915eb76d1227b4b6d1137a0dedf2c143860098a2
parent c63fc881
......@@ -1642,8 +1642,6 @@ int vp8_rd_pick_inter_mode(VP8_COMP *cpi, MACROBLOCK *x, int recon_yoffset, int
//int intermodecost[MAX_MODES];
MB_PREDICTION_MODE uv_intra_mode;
int sse;
int sum;
int uvintra_eob = 0;
int tteob = 0;
int force_no_skip = 0;
......@@ -2238,36 +2236,44 @@ int vp8_rd_pick_inter_mode(VP8_COMP *cpi, MACROBLOCK *x, int recon_yoffset, int
vp8_set_mbmode_and_mvs(x, this_mode, &mode_mv[this_mode]);
vp8_build_inter_predictors_mby(&x->e_mbd);
if(x->encode_breakout)
VARIANCE_INVOKE(&cpi->rtcd.variance, get16x16var)(x->src.y_buffer, x->src.y_stride, x->e_mbd.predictor, 16, (unsigned int *)(&sse), &sum);
if (cpi->active_map_enabled && x->active_ptr[0] == 0) {
x->skip = 1;
}
else if (sse < x->encode_breakout)
else if (x->encode_breakout)
{
// Check u and v to make sure skip is ok
int sse2 = 0;
int sum, sse;
// add dc check
if (abs(sum) < (cpi->common.Y2dequant[0][0] << 2))
VARIANCE_INVOKE(&cpi->rtcd.variance, get16x16var)
(x->src.y_buffer, x->src.y_stride,
x->e_mbd.predictor, 16, (unsigned int *)(&sse), &sum);
if (sse < x->encode_breakout)
{
sse2 = VP8_UVSSE(x, IF_RTCD(&cpi->rtcd.variance));
// Check u and v to make sure skip is ok
int sse2 = 0;
if (sse2 * 2 < x->encode_breakout)
// add dc check
if (abs(sum) < (cpi->common.Y2dequant[0][0] << 2))
{
x->skip = 1;
distortion2 = sse;
rate2 = 500;
disable_skip = 1;
this_rd = RDFUNC(x->rdmult, x->rddiv, rate2,
distortion2, cpi->target_bits_per_mb);
sse2 = VP8_UVSSE(x, IF_RTCD(&cpi->rtcd.variance));
break;
if (sse2 * 2 < x->encode_breakout)
{
x->skip = 1;
distortion2 = sse;
rate2 = 500;
disable_skip = 1;
this_rd = RDFUNC(x->rdmult, x->rddiv, rate2,
distortion2, cpi->target_bits_per_mb);
break;
}
}
}
}
//intermodecost[mode_index] = vp8_cost_mv_ref(this_mode, mdcounts); // Experimental debug code
// Add in the Mv/mode cost
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment