Commit 07e33f4e authored by Sebastien Alaiwan's avatar Sebastien Alaiwan Committed by sebastien alaiwan

Fix integer overflow.

Found using the undefined behaviour sanitizer (-fsanitize=undefined).

Change-Id: I5534a97c0e38f76b38c6a53b42759b0b676e360a
parent 8c636c12
......@@ -127,8 +127,9 @@ void av1_caq_select_segment(const AV1_COMP *cpi, MACROBLOCK *mb, BLOCK_SIZE bs,
} else {
// Rate depends on fraction of a SB64 in frame (xmis * ymis / bw * bh).
// It is converted to bits * 256 units.
const int target_rate = (cpi->rc.sb64_target_rate * xmis * ymis * 256) /
(cm->mib_size * cm->mib_size);
const int64_t num = (int64_t)cpi->rc.sb64_target_rate * xmis * ymis * 256;
const int denom = cm->mib_size * cm->mib_size;
const int target_rate = (int)(num / denom);
double logvar;
double low_var_thresh;
const int aq_strength = get_aq_c_strength(cm->base_qindex, cm->bit_depth);
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment