Unverified Commit 90f50929 authored by Kyle Siefring's avatar Kyle Siefring Committed by GitHub

Add explicit wrapping behaviour to txfm butterfly (#768)

parent 730c23c5
......@@ -136,8 +136,15 @@ fn get_rect_tx_log_ratio(col: usize, row: usize) -> i8 {
// performs half a butterfly
#[inline]
fn half_btf(w0: i32, in0: i32, w1: i32, in1: i32, bit: usize) -> i32 {
let result = (w0 * in0) + (w1 * in1);
round_shift(result, bit)
// Ensure defined behaviour for when w0*in0 + w1*in1 is negative and
// overflows, but w0*in0 + w1*in1 + rounding isn't.
let result = (w0 * in0).wrapping_add(w1 * in1);
// Implement a version of round_shift with wrapping
if bit <= 0 {
result
} else {
result.wrapping_add(1 << (bit - 1)) >> bit
}
}
#[inline]
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment