Commit b1fcd175 authored by Alex Converse's avatar Alex Converse
Browse files

Fix unsigned overflow in rd_variance_adjustment.

Found with clang -fsanitize=integer

Change-Id: I2538e7483cb2d5f06bceecbd3326bdd88bfecfa1
parent f3f6b6fe
......@@ -3017,9 +3017,9 @@ static void rd_variance_adjustment(VP10_COMP *cpi,
? (source_variance - recon_variance)
: (recon_variance - source_variance);
var_error = (200 * source_variance * recon_variance) /
((source_variance * source_variance) +
(recon_variance * recon_variance));
var_error = ((int64_t)200 * source_variance * recon_variance) /
(((int64_t)source_variance * source_variance) +
((int64_t)recon_variance * recon_variance));
var_error = 100 - var_error;
}
......
......@@ -2831,9 +2831,9 @@ static void rd_variance_adjustment(VP9_COMP *cpi,
? (source_variance - recon_variance)
: (recon_variance - source_variance);
var_error = (200 * source_variance * recon_variance) /
((source_variance * source_variance) +
(recon_variance * recon_variance));
var_error = ((int64_t)200 * source_variance * recon_variance) /
(((int64_t)source_variance * source_variance) +
((int64_t)recon_variance * recon_variance));
var_error = 100 - var_error;
}
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment