Commit 57f6bfd0 authored by Monty Montgomery's avatar Monty Montgomery Committed by Christopher Montgomery
Browse files

Enable configurable fixed-depth coefficients in Daala TX

This patch turns on the fixed-depth TX code in the Daala toplevel TX

A REGRESSION IS EXPECTED as this is temporarily dropping Daala TX back
to Q3, which is reduced operating precision over current master.

subset-1:
monty-rest-of-stack-RDO-s1@2017-11-13T14:39:17.093Z ->
 monty-rest-of-stack-Q3-s1@2017-11-13T14:39:52.160Z

  PSNR | PSNR Cb | PSNR Cr | PSNR HVS |   SSIM | MS SSIM | CIEDE 2000
0.0927 |     N/A | -0.0001 |   0.1390 | 0.0871 |  0.0835 |     0.0826

objective-1-fast --limit=4:
monty-rest-of-stack-RDO-o1f4@2017-11-13T14:38:57.951Z ->
 monty-rest-of-stack-Q3-o1f4@2017-11-13T14:39:32.205Z

  PSNR | PSNR Cb | PSNR Cr | PSNR HVS |   SSIM | MS SSIM | CIEDE 2000
0.0981 |  0.2341 |  0.4784 |   0.1215 | 0.0761 |  0.1144 |     0.1444

Change-Id: Ibbe17226dd47980da632814422d6201c9fc6fa36
parent 4a05a58c
......@@ -16,6 +16,7 @@
#include "./aom_config.h"
#include "aom_dsp/txfm_common.h"
#include "av1/common/odintrin.h"
#include "aom_ports/mem.h"
#ifdef __cplusplus
......@@ -29,13 +30,10 @@ static INLINE tran_high_t dct_const_round_shift(tran_high_t input) {
static INLINE tran_high_t check_range(tran_high_t input, int bd) {
#if CONFIG_DAALA_TX
// Daala TX coeffs cover a different range from AV1 TX
// Temporary: this will be changing shortly upon moving to a fixed
// coeff depth!
// - 8 bit: signed 18 bit integer
// - 10 bit: signed 20 bit integer
// - 12 bit: signed 22 bit integer
const int32_t int_max = (1 << (9 + bd)) - 1;
// all depths: 18 bit integer
const int32_t int_max = (1 << (TX_COEFF_DEPTH + 6)) - 1;
const int32_t int_min = -int_max - 1;
(void)bd;
#else
// AV1 TX case
// - 8 bit: signed 16 bit integer
......
......@@ -67,8 +67,8 @@ void daala_inv_txfm_add(const tran_low_t *input_coeffs, void *output_pixels,
txfm_param->eob, px_depth);
} else {
// General TX case
// Q3 coeff Q4 TX compatability mode
const int downshift = 4;
const int downshift = TX_COEFF_DEPTH - px_depth;
assert(downshift >= 0);
assert(sizeof(tran_low_t) == sizeof(od_coeff));
assert(sizeof(tran_low_t) >= 4);
......@@ -93,16 +93,6 @@ void daala_inv_txfm_add(const tran_low_t *input_coeffs, void *output_pixels,
assert(col_tx);
assert(row_tx);
// This is temporary while we're testing against existing
// behavior (preshift up one).
// Remove before flight
od_coeff tmp[MAX_TX_SQUARE];
int upshift = 1;
for (r = 0; r < rows; ++r)
for (c = 0; c < cols; ++c)
tmp[r * cols + c] = input_coeffs[r * cols + c] << upshift;
input_coeffs = tmp;
// Inverse-transform rows
for (r = 0; r < rows; ++r) {
// The output addressing transposes
......
......@@ -59,9 +59,7 @@ void daala_fwd_txfm(const int16_t *input_pixels, tran_low_t *output_coeffs,
av1_fwht4x4(input_pixels, output_coeffs, input_stride);
} else {
// General TX case
// up 4, down 1 compatability mode
const int upshift = 4;
const int upshift = TX_COEFF_DEPTH - txfm_param->bd;
assert(upshift >= 0);
assert(sizeof(tran_low_t) == sizeof(od_coeff));
assert(sizeof(tran_low_t) >= 4);
......@@ -110,14 +108,6 @@ void daala_fwd_txfm(const int16_t *input_pixels, tran_low_t *output_coeffs,
else
row_tx(output_coeffs + r * cols, output_coeffs + r * cols, 1);
}
// This is temporary while we're testing against existing
// behavior (preshift up 4, then downshift by one)
int downshift = 1;
for (r = 0; r < rows; ++r)
for (c = 0; c < cols; ++c)
output_coeffs[r * cols + c] =
ROUND_POWER_OF_TWO_SIGNED(output_coeffs[r * cols + c], downshift);
}
}
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment