From 69a16433c209d00443e7c495427a3039d9f573cb Mon Sep 17 00:00:00 2001
From: "Nathan E. Egge"
Date: Wed, 18 Oct 2017 12:50:28 -0400
Subject: [PATCH] Fix a bug in the DAALA_TX 16-point DST functions.
The OD_FDST_16() and OD_IDST_16() macros were written for use in the
OD_FDCT_32_ASYM macro which took asymmetrically scaled input and
after running an asymmetric butterfly step, passed it through to
the 16-point Type-II DCT and 16-point Type-IV DST.
Because the DST implementations were never tested as stand alone
transforms, some of the signs from the butterfly step ended up inside
the DST macros.
These extra operations will be addressed in a follow up patch.
Change-Id: I32f54a4bb70cd8fad4ae5646cfa4f5b14a0f969b
---
av1/common/daala_tx.c | 32 ++++++++++++++++----------------
1 file changed, 16 insertions(+), 16 deletions(-)
diff --git a/av1/common/daala_tx.c b/av1/common/daala_tx.c
index e5b2372e3..1630359a5 100644
--- a/av1/common/daala_tx.c
+++ b/av1/common/daala_tx.c
@@ -3433,21 +3433,21 @@ void od_bin_fdst16(od_coeff y[16], const od_coeff *x, int xstride) {
sf = x[0*xstride];
OD_FDST_16(s0, s8, s4, sc, s2, sa, s6, se, s1, s9, s5, sd, s3, sb, s7, sf);
y[0] = (od_coeff)sf;
- y[1] = (od_coeff)se;
+ y[1] = (od_coeff)-se;
y[2] = (od_coeff)sd;
- y[3] = (od_coeff)sc;
+ y[3] = (od_coeff)-sc;
y[4] = (od_coeff)sb;
- y[5] = (od_coeff)sa;
+ y[5] = (od_coeff)-sa;
y[6] = (od_coeff)s9;
- y[7] = (od_coeff)s8;
+ y[7] = (od_coeff)-s8;
y[8] = (od_coeff)s7;
- y[9] = (od_coeff)s6;
+ y[9] = (od_coeff)-s6;
y[10] = (od_coeff)s5;
- y[11] = (od_coeff)s4;
+ y[11] = (od_coeff)-s4;
y[12] = (od_coeff)s3;
- y[13] = (od_coeff)s2;
+ y[13] = (od_coeff)-s2;
y[14] = (od_coeff)s1;
- y[15] = (od_coeff)s0;
+ y[15] = (od_coeff)-s0;
}
void od_bin_idst16(od_coeff *x, int xstride, const od_coeff y[16]) {
@@ -3467,21 +3467,21 @@ void od_bin_idst16(od_coeff *x, int xstride, const od_coeff y[16]) {
int sd;
int se;
int sf;
- s0 = y[15];
+ s0 = -y[15];
s8 = y[14];
- s4 = y[13];
+ s4 = -y[13];
sc = y[12];
- s2 = y[11];
+ s2 = -y[11];
sa = y[10];
- s6 = y[9];
+ s6 = -y[9];
se = y[8];
- s1 = y[7];
+ s1 = -y[7];
s9 = y[6];
- s5 = y[5];
+ s5 = -y[5];
sd = y[4];
- s3 = y[3];
+ s3 = -y[3];
sb = y[2];
- s7 = y[1];
+ s7 = -y[1];
sf = y[0];
OD_IDST_16(s0, s8, s4, sc, s2, sa, s6, se, s1, s9, s5, sd, s3, sb, s7, sf);
x[0*xstride] = (od_coeff)sf;
--
2.22.2