Commit 77d197e6 authored by Geza Lore's avatar Geza Lore
Browse files

Fix segfault with --cpu-used >= 3 and ext-refs.

With ext-ref enabled, it is possible that when trying to encode the
first true ALTREF frame after a keyframe, the previous ALTREF frame
(alias for the keyframe) is the same as one of the new LAST{2,3,4}
reference frames, and hence cpi->ref_frame_flags will have the ALTREF
bit clear, as computed by get_ref_frame_flags in encoder.c.

sf->alt_ref_search_fp forces the previous ALTREF frame to
be used as the only possible  reference when encoding a new ALTREF
frame, but due to cpi->ref_frame_flags, some buffers will not be
initialized (see rdopt.c:7689 yv12_mb), leading to a segfault.

get_ref_frame_flags in encoder.c has been changed to prefer to keep
the  LAST frame, then the ALTREF frame, then any of the LAST{2,3,4}
frames and then the GOLDEN frame in that order of preference in case
any of them are the same. This avoids the segfault and behaves the
same for the baseline.

Change-Id: I4da1991667614009da5d3061a6316c0d5dbc6c0c
parent 436a6cc4
......@@ -4098,14 +4098,14 @@ static int get_ref_frame_flags(const VP10_COMP *cpi) {
const int last2_is_last =
map[cpi->lst_fb_idxes[1]] == map[cpi->lst_fb_idxes[0]];
const int gld_is_last2 = map[cpi->gld_fb_idx] == map[cpi->lst_fb_idxes[1]];
const int alt_is_last2 = map[cpi->alt_fb_idx] == map[cpi->lst_fb_idxes[1]];
const int last2_is_alt = map[cpi->alt_fb_idx] == map[cpi->lst_fb_idxes[1]];
const int last3_is_last =
map[cpi->lst_fb_idxes[2]] == map[cpi->lst_fb_idxes[0]];
const int last3_is_last2 =
map[cpi->lst_fb_idxes[2]] == map[cpi->lst_fb_idxes[1]];
const int gld_is_last3 = map[cpi->gld_fb_idx] == map[cpi->lst_fb_idxes[2]];
const int alt_is_last3 = map[cpi->alt_fb_idx] == map[cpi->lst_fb_idxes[2]];
const int last3_is_alt = map[cpi->alt_fb_idx] == map[cpi->lst_fb_idxes[2]];
const int last4_is_last =
map[cpi->lst_fb_idxes[3]] == map[cpi->lst_fb_idxes[0]];
......@@ -4114,21 +4114,16 @@ static int get_ref_frame_flags(const VP10_COMP *cpi) {
const int last4_is_last3 =
map[cpi->lst_fb_idxes[3]] == map[cpi->lst_fb_idxes[2]];
const int gld_is_last4 = map[cpi->gld_fb_idx] == map[cpi->lst_fb_idxes[3]];
const int alt_is_last4 = map[cpi->alt_fb_idx] == map[cpi->lst_fb_idxes[3]];
const int last4_is_alt = map[cpi->alt_fb_idx] == map[cpi->lst_fb_idxes[3]];
#else
const int gld_is_last = map[cpi->gld_fb_idx] == map[cpi->lst_fb_idx];
const int alt_is_last = map[cpi->alt_fb_idx] == map[cpi->lst_fb_idx];
#endif // CONFIG_EXT_REFS
const int gld_is_alt = map[cpi->gld_fb_idx] == map[cpi->alt_fb_idx];
int flags = VP9_ALT_FLAG | VP9_GOLD_FLAG | VP9_LAST_FLAG;
#if CONFIG_EXT_REFS
flags |= VP9_LAST2_FLAG;
flags |= VP9_LAST3_FLAG;
flags |= VP9_LAST4_FLAG;
#endif // CONFIG_EXT_REFS
int flags = VP9_REFFRAME_ALL;
if (gld_is_last)
if (gld_is_alt || gld_is_last)
flags &= ~VP9_GOLD_FLAG;
if (cpi->rc.frames_till_gf_update_due == INT_MAX)
......@@ -4137,24 +4132,18 @@ static int get_ref_frame_flags(const VP10_COMP *cpi) {
if (alt_is_last)
flags &= ~VP9_ALT_FLAG;
if (gld_is_alt)
flags &= ~VP9_ALT_FLAG;
#if CONFIG_EXT_REFS
if (last4_is_last || last4_is_last2 || last4_is_last3)
if (last4_is_alt || last4_is_last || last4_is_last2 || last4_is_last3)
flags &= ~VP9_LAST4_FLAG;
if (last3_is_last || last3_is_last2)
if (last3_is_alt || last3_is_last || last3_is_last2)
flags &= ~VP9_LAST3_FLAG;
if (last2_is_last)
if (last2_is_alt || last2_is_last)
flags &= ~VP9_LAST2_FLAG;
if (gld_is_last4 || gld_is_last3 || gld_is_last2)
flags &= ~VP9_GOLD_FLAG;
if (alt_is_last4 || alt_is_last3 || alt_is_last2)
flags &= ~VP9_ALT_FLAG;
#endif // CONFIG_EXT_REFS
return flags;
......
......@@ -7785,6 +7785,7 @@ void vp10_rd_pick_inter_mode_sb(VP10_COMP *cpi,
if (cpi->rc.is_src_frame_alt_ref) {
if (sf->alt_ref_search_fp) {
assert(cpi->ref_frame_flags & flag_list[ALTREF_FRAME]);
mode_skip_mask[ALTREF_FRAME] = 0;
ref_frame_skip_mask[0] = ~(1 << ALTREF_FRAME);
ref_frame_skip_mask[1] = SECOND_REF_FRAME_MASK;
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment