Commit c2217f22 authored by Jingning Han's avatar Jingning Han

Avoid mv cost check for invalid mv values

The compound inter prediction could potentially run with initial
motion vectors of invalid value and check the mv_cost, which triggers
overheap read. This commit resolves this issue by forcing a motion
vector value check for compound inter modes of both superblock and
sub8x8 block sizes.

Change-Id: I4f4fc19ce83c8272782bc382f12c82a3f03212fc
parent 333345cd
......@@ -1872,12 +1872,14 @@ static void rd_check_segment_txsize(VP9_COMP *cpi, MACROBLOCK *x,
mi_buf_restore(x, orig_src, orig_pre);
}
if (has_second_rf && this_mode == NEWMV &&
mbmi->interp_filter == EIGHTTAP) {
if (has_second_rf) {
if (seg_mvs[i][mbmi->ref_frame[1]].as_int == INVALID_MV ||
seg_mvs[i][mbmi->ref_frame[0]].as_int == INVALID_MV)
continue;
}
if (has_second_rf && this_mode == NEWMV &&
mbmi->interp_filter == EIGHTTAP) {
// adjust src pointers
mi_buf_shift(x, i);
if (cpi->sf.comp_inter_joint_search_thresh <= bsize) {
......@@ -2661,6 +2663,12 @@ static int64_t handle_inter_mode(VP9_COMP *cpi, MACROBLOCK *x,
int orig_dst_stride[MAX_MB_PLANE];
int rs = 0;
if (is_comp_pred) {
if (frame_mv[refs[0]].as_int == INVALID_MV ||
frame_mv[refs[1]].as_int == INVALID_MV)
return INT64_MAX;
}
if (this_mode == NEWMV) {
int rate_mv;
if (is_comp_pred) {
......@@ -2679,9 +2687,6 @@ static int64_t handle_inter_mode(VP9_COMP *cpi, MACROBLOCK *x,
&mbmi->ref_mvs[refs[1]][0].as_mv,
x->nmvjointcost, x->mvcost, MV_COST_WEIGHT);
}
if (frame_mv[refs[0]].as_int == INVALID_MV ||
frame_mv[refs[1]].as_int == INVALID_MV)
return INT64_MAX;
*rate2 += rate_mv;
} else {
int_mv tmp_mv;
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment