Unverified Commit 2e4055a6 authored by Yushin Cho's avatar Yushin Cho Committed by GitHub

Fix that number of rows is used for vertical 1d tx in inverse tx (#774)

Number of columns sholud be used.
parent 33c298d3
......@@ -1536,7 +1536,7 @@ trait InvTxfm2D: Dim {
// perform inv txfm on every col
let range = cmp::max(bd + 6, 16);
let txfm_fn = INV_TXFM_FNS[tx_types_1d.0 as usize][Self::H.ilog() - 3];
for c in 0..Self::H {
for c in 0..Self::W {
let mut temp_in: [i32; 64] = [0; 64];
let mut temp_out: [i32; 64] = [0; 64];
for (raw, clamped) in
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment