Commit 215084b3 authored by James Zern's avatar James Zern
Browse files

vp8: clamp probability values

fixes out of bounds reads on vp8_prob_cost table.
present since:
217591fd Added rate-targeted temporal scalability
v0.9.7-p1-71-g217591fd

Change-Id: I9194b773098a381f1e3f67bd3307f05df27c24fd
parent 7d8fc26c
......@@ -980,6 +980,12 @@ void vp8_calc_ref_frame_costs(int *ref_frame_cost,
int prob_garf
)
{
assert(prob_intra >= 0);
assert(prob_intra <= 255);
assert(prob_last >= 0);
assert(prob_last <= 255);
assert(prob_garf >= 0);
assert(prob_garf <= 255);
ref_frame_cost[INTRA_FRAME] = vp8_cost_zero(prob_intra);
ref_frame_cost[LAST_FRAME] = vp8_cost_one(prob_intra)
+ vp8_cost_zero(prob_last);
......
......@@ -2816,6 +2816,8 @@ static void update_rd_ref_frame_probs(VP8_COMP *cpi)
if (cpi->common.refresh_alt_ref_frame)
{
cpi->prob_intra_coded += 40;
if (cpi->prob_intra_coded > 255)
cpi->prob_intra_coded = 255;
cpi->prob_last_coded = 200;
cpi->prob_gf_coded = 1;
}
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment