Commit ea284d62 authored by Yaowu Xu's avatar Yaowu Xu
Browse files

added checks to prevent rate/distortion overflow

At speed 2, due to the threshold scheme used, it is possible the rate
and distortion assigned with INT_MAX value. The patch added checking
to prevent the INT_MAX value is used in further calculation of RD
scores. The patch also changed the assertion in rd_use_partition() to
be mirror similar assertion in rd_pick_partition().

Change-Id: Idb52c543cc1e10abdf6e6a5d6e9cb535a42214dc
parent f00a237a
......@@ -1248,9 +1248,6 @@ static void rd_use_partition(VP9_COMP *cpi, MODE_INFO *m, TOKENEXTRA **tp,
case PARTITION_NONE:
pick_sb_modes(cpi, mi_row, mi_col, &last_part_rate, &last_part_dist,
bsize, get_block_context(x, bsize), INT64_MAX);
set_partition_seg_context(cm, xd, mi_row, mi_col);
pl = partition_plane_context(xd, bsize);
last_part_rate += x->partition_cost[pl][PARTITION_NONE];
break;
case PARTITION_HORZ:
*(get_sb_index(xd, subsize)) = 0;
......@@ -1267,9 +1264,6 @@ static void rd_use_partition(VP9_COMP *cpi, MODE_INFO *m, TOKENEXTRA **tp,
last_part_rate += rt;
last_part_dist += dt;
}
set_partition_seg_context(cm, xd, mi_row, mi_col);
pl = partition_plane_context(xd, bsize);
last_part_rate += x->partition_cost[pl][PARTITION_HORZ];
break;
case PARTITION_VERT:
*(get_sb_index(xd, subsize)) = 0;
......@@ -1286,9 +1280,6 @@ static void rd_use_partition(VP9_COMP *cpi, MODE_INFO *m, TOKENEXTRA **tp,
last_part_rate += rt;
last_part_dist += dt;
}
set_partition_seg_context(cm, xd, mi_row, mi_col);
pl = partition_plane_context(xd, bsize);
last_part_rate += x->partition_cost[pl][PARTITION_VERT];
break;
case PARTITION_SPLIT:
// Split partition.
......@@ -1308,16 +1299,23 @@ static void rd_use_partition(VP9_COMP *cpi, MODE_INFO *m, TOKENEXTRA **tp,
rd_use_partition(cpi, m + jj * bss * mis + ii * bss, tp, mi_row + y_idx,
mi_col + x_idx, subsize, &rt, &dt, i != 3);
if (rt == INT_MAX || dt == INT_MAX) {
last_part_rate = INT_MAX;
last_part_dist = INT_MAX;
break;
}
last_part_rate += rt;
last_part_dist += dt;
}
set_partition_seg_context(cm, xd, mi_row, mi_col);
pl = partition_plane_context(xd, bsize);
last_part_rate += x->partition_cost[pl][PARTITION_SPLIT];
break;
default:
assert(0);
}
set_partition_seg_context(cm, xd, mi_row, mi_col);
pl = partition_plane_context(xd, bsize);
if (last_part_rate < INT_MAX)
last_part_rate += x->partition_cost[pl][partition];
if (cpi->sf.adjust_partitioning_from_last_frame
&& partition != PARTITION_SPLIT && bsize > BLOCK_SIZE_SB8X8
&& (mi_row + ms < cm->mi_rows || mi_row + (ms >> 1) == cm->mi_rows)
......@@ -1392,10 +1390,12 @@ static void rd_use_partition(VP9_COMP *cpi, MODE_INFO *m, TOKENEXTRA **tp,
// We must have chosen a partitioning and encoding or we'll fail later on.
// No other opportunities for success.
assert(chosen_rate < INT_MAX && chosen_dist < INT_MAX);
if ( bsize == BLOCK_SIZE_SB64X64)
assert(chosen_rate < INT_MAX && chosen_dist < INT_MAX);
if (do_recon)
encode_sb(cpi, tp, mi_row, mi_col, bsize == BLOCK_SIZE_SB64X64, bsize);
*rate = chosen_rate;
*dist = chosen_dist;
}
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment