Unverified Commit 48e5330e authored by Yushin Cho's avatar Yushin Cho Committed by GitHub

Fix missing inv sqrt2 term in rect inv tx (#775)

parent 2e4055a6
......@@ -1518,6 +1518,7 @@ trait InvTxfm2D: Dim {
bd: usize
) where T: Pixel, i32: AsPrimitive<T> {
let buffer = &mut [0i32; 64 * 64][..Self::W * Self::H];
let rect_type = get_rect_tx_log_ratio(Self::W, Self::H);
let tx_types_1d = get_1d_tx_types(tx_type)
.expect("TxType not supported by rust txfm code.");
// perform inv txfm on every row
......@@ -1528,7 +1529,11 @@ trait InvTxfm2D: Dim {
{
let mut temp_in: [i32; 64] = [0; 64];
for (raw, clamped) in input_slice.iter().zip(temp_in.iter_mut()) {
*clamped = clamp_value(*raw, range);
let mut val = *raw;
if rect_type.abs() == 1 {
val = round_shift(*raw * INV_SQRT2, SQRT2_BITS);
}
*clamped = clamp_value(val, range);
}
txfm_fn(&temp_in, buffer_slice, range);
}
......
......@@ -23,7 +23,8 @@ mod forward;
mod inverse;
static SQRT2_BITS: usize = 12;
static SQRT2: i32 = 5793;
static SQRT2: i32 = 5793; // 2^12 * sqrt(2)
static INV_SQRT2: i32 = 2896; // 2^12 / sqrt(2)
static COS_BIT_MIN: usize = 10;
......@@ -111,23 +112,23 @@ fn sinpi_arr(n: usize) -> &'static [i32; 5] {
/// Utility function that returns the log of the ratio of the col and row sizes.
#[inline]
fn get_rect_tx_log_ratio(col: usize, row: usize) -> i8 {
pub fn get_rect_tx_log_ratio(col: usize, row: usize) -> i8 {
if col == row {
return 0;
return 0
}
if col > row {
if col == row * 2 {
return 1;
return 1
}
if col == row * 4 {
return 2;
return 2
}
}
if row == col * 2 {
return -1;
return -1
}
if row == col * 4 {
return -2;
return -2
}
panic!("Unsupported transform size");
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment