Commit 0930dde2 authored by Ronald S. Bultje's avatar Ronald S. Bultje
Browse files

Fix overflows in dual prediction mode selection.

Change-Id: I265ad46e01a307bca21e6223725e4055f5e08648
parent 46f9ad2c
......@@ -1348,6 +1348,7 @@ void vp8_encode_frame(VP8_COMP *cpi)
{
int frame_type, pred_type;
int redo = 0;
int single_diff, dual_diff, hybrid_diff;
/*
* This code does a single RD pass over the whole frame assuming
......@@ -1384,27 +1385,27 @@ void vp8_encode_frame(VP8_COMP *cpi)
cpi->common.dual_pred_mode = pred_type;
encode_frame_internal(cpi);
cpi->rd_single_diff /= cpi->common.MBs;
cpi->rd_prediction_type_threshes[frame_type][0] += cpi->rd_single_diff;
single_diff = cpi->rd_single_diff / cpi->common.MBs;
cpi->rd_prediction_type_threshes[frame_type][0] += single_diff;
cpi->rd_prediction_type_threshes[frame_type][0] >>= 1;
cpi->rd_dual_diff /= cpi->common.MBs;
cpi->rd_prediction_type_threshes[frame_type][1] += cpi->rd_dual_diff;
dual_diff = cpi->rd_dual_diff / cpi->common.MBs;
cpi->rd_prediction_type_threshes[frame_type][1] += dual_diff;
cpi->rd_prediction_type_threshes[frame_type][1] >>= 1;
cpi->rd_hybrid_diff /= cpi->common.MBs;
cpi->rd_prediction_type_threshes[frame_type][2] += cpi->rd_hybrid_diff;
hybrid_diff = cpi->rd_hybrid_diff / cpi->common.MBs;
cpi->rd_prediction_type_threshes[frame_type][2] += hybrid_diff;
cpi->rd_prediction_type_threshes[frame_type][2] >>= 1;
/* FIXME make "100" (the threshold at which to re-encode the
* current frame) a commandline option. */
if (cpi->common.dual_pred_mode == SINGLE_PREDICTION_ONLY &&
(cpi->rd_dual_diff >= 100 || cpi->rd_hybrid_diff >= 100))
(dual_diff >= 100 || hybrid_diff >= 100))
{
redo = 1;
cpi->common.dual_pred_mode = cpi->rd_dual_diff > cpi->rd_hybrid_diff ?
DUAL_PREDICTION_ONLY : HYBRID_PREDICTION;
}
else if (cpi->common.dual_pred_mode == DUAL_PREDICTION_ONLY &&
(cpi->rd_single_diff >= 100 || cpi->rd_hybrid_diff >= 100))
(single_diff >= 100 || hybrid_diff >= 100))
{
redo = 1;
cpi->common.dual_pred_mode = cpi->rd_single_diff > cpi->rd_hybrid_diff ?
......@@ -1430,7 +1431,7 @@ void vp8_encode_frame(VP8_COMP *cpi)
{
cpi->common.dual_pred_mode = DUAL_PREDICTION_ONLY;
}
else if (cpi->rd_single_diff >= 100 || cpi->rd_dual_diff >= 100)
else if (single_diff >= 100 || dual_diff >= 100)
{
redo = 1;
cpi->common.dual_pred_mode = cpi->rd_single_diff > cpi->rd_dual_diff ?
......
......@@ -353,7 +353,7 @@ typedef struct VP8_COMP
int rd_thresh_mult[MAX_MODES];
int rd_baseline_thresh[MAX_MODES];
int rd_threshes[MAX_MODES];
int rd_single_diff, rd_dual_diff, rd_hybrid_diff;
int64_t rd_single_diff, rd_dual_diff, rd_hybrid_diff;
int rd_prediction_type_threshes[4][NB_PREDICTION_TYPES];
int dual_pred_count[DUAL_PRED_CONTEXTS];
int single_pred_count[DUAL_PRED_CONTEXTS];
......
......@@ -3142,9 +3142,18 @@ void vp8_rd_pick_inter_mode(VP8_COMP *cpi, MACROBLOCK *x, int recon_yoffset, int
rd_update_mvcount(cpi, x, &frame_best_ref_mv[xd->mode_info_context->mbmi.ref_frame]);
*best_single_rd_diff = best_rd - best_single_rd;
*best_dual_rd_diff = best_rd - best_dual_rd;
*best_hybrid_rd_diff = best_rd - best_hybrid_rd;
if (best_single_rd == INT_MAX)
*best_single_rd_diff = INT_MIN;
else
*best_single_rd_diff = best_rd - best_single_rd;
if (best_dual_rd == INT_MAX)
*best_dual_rd_diff = INT_MIN;
else
*best_dual_rd_diff = best_rd - best_dual_rd;
if (best_hybrid_rd == INT_MAX)
*best_hybrid_rd_diff = INT_MIN;
else
*best_hybrid_rd_diff = best_rd - best_hybrid_rd;
}
void vp8_rd_pick_intra_mode(VP8_COMP *cpi, MACROBLOCK *x, int *rate_)
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment