Commit 6a632e20 authored by Jingning Han's avatar Jingning Han

Prevent out-of-boundary mem access in ext-inter

This resolves the assertion failure issue when ext-inter is turned
on.

BUG=aomedia:463

Change-Id: I8b5075d7c1c76c4fb222b750ac797f645adee6b2
parent 99ffce6a
......@@ -126,7 +126,9 @@ static struct av1_token intra_filter_encodings[INTRA_FILTERS];
#endif // CONFIG_EXT_INTRA
#if CONFIG_EXT_INTER
static struct av1_token interintra_mode_encodings[INTERINTRA_MODES];
#if CONFIG_WEDGE || CONFIG_COMPOUND_SEGMENT
static struct av1_token compound_type_encodings[COMPOUND_TYPES];
#endif // CONFIG_WEDGE || CONFIG_COMPOUND_SEGMENT
#endif // CONFIG_EXT_INTER
#if CONFIG_MOTION_VAR || CONFIG_WARPED_MOTION
static struct av1_token motion_mode_encodings[MOTION_MODES];
......@@ -177,7 +179,9 @@ void av1_encode_token_init(void) {
#endif // CONFIG_EXT_INTRA && CONFIG_INTRA_INTERP
#if CONFIG_EXT_INTER
av1_tokens_from_tree(interintra_mode_encodings, av1_interintra_mode_tree);
#if CONFIG_WEDGE || CONFIG_COMPOUND_SEGMENT
av1_tokens_from_tree(compound_type_encodings, av1_compound_type_tree);
#endif // CONFIG_WEDGE || CONFIG_COMPOUND_SEGMENT
#endif // CONFIG_EXT_INTER
#if CONFIG_MOTION_VAR || CONFIG_WARPED_MOTION
av1_tokens_from_tree(motion_mode_encodings, av1_motion_mode_tree);
......
......@@ -8765,7 +8765,7 @@ static int64_t handle_inter_mode(
int best_tmp_rate_mv = rate_mv;
int tmp_skip_txfm_sb;
int64_t tmp_skip_sse_sb;
int compound_type_cost[COMPOUND_TYPES];
int compound_type_cost[COMPOUND_TYPES] = { 0 };
uint8_t pred0[2 * MAX_SB_SQUARE];
uint8_t pred1[2 * MAX_SB_SQUARE];
uint8_t *preds0[1] = { pred0 };
......@@ -8778,10 +8778,10 @@ static int64_t handle_inter_mode(
best_mv[0].as_int = cur_mv[0].as_int;
best_mv[1].as_int = cur_mv[1].as_int;
memset(&best_compound_data, 0, sizeof(INTERINTER_COMPOUND_DATA));
av1_cost_tokens(compound_type_cost, cm->fc->compound_type_prob[bsize],
av1_compound_type_tree);
if (masked_compound_used) {
av1_cost_tokens(compound_type_cost, cm->fc->compound_type_prob[bsize],
av1_compound_type_tree);
// get inter predictors to use for masked compound modes
av1_build_inter_predictors_for_planes_single_buf(
xd, bsize, 0, 0, mi_row, mi_col, 0, preds0, strides);
......@@ -8794,6 +8794,7 @@ static int64_t handle_inter_mode(
tmp_rate_mv = rate_mv;
best_rd_cur = INT64_MAX;
mbmi->interinter_compound_data.type = cur_type;
rs2 = av1_cost_literal(get_interinter_compound_type_bits(
bsize, mbmi->interinter_compound_data.type)) +
(masked_compound_used
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment