Commit a26262c3 authored by Monty Montgomery's avatar Monty Montgomery Committed by Christopher Montgomery

Eliminate tx_size dependant shifts for Daala TX

short-circuit av1_get_tx_scale to always return zero when
CONFIG_DAALA_TX, and remove it from the actual Daala TX toplevel

This has potential overflow consequences for any metrics computation
based on pixels; as such, also force use of the high-bitdepth path in
each of these case.

subset-1:
monty-rest-of-stack-baseline-s1@2017-11-13T00:39:03.881Z ->
monty-rest-of-stack-noshift-s1@2017-11-13T14:37:42.541Z

   PSNR | PSNR Cb | PSNR Cr | PSNR HVS |    SSIM | MS SSIM | CIEDE 2000
-0.0030 | -0.0523 |  0.2656 |  -0.0239 | -0.0033 | -0.0029 |     0.0067

objective-1-fast --limit=4:
monty-rest-of-stack-baseline-o1f4@2017-11-13T00:37:06.999Z ->
monty-rest-of-stack-noshift-o1f4@2017-11-13T14:37:16.992Z

   PSNR | PSNR Cb | PSNR Cr | PSNR HVS |    SSIM | MS SSIM | CIEDE 2000
-0.0264 |  0.2303 |  0.0822 |  -0.0109 | -0.0395 | -0.0709 |     0.0538

Change-Id: I57da71861f105dc7a404fa75a75bde573855ef79
parent 0b7756b7
...@@ -27,12 +27,24 @@ static INLINE tran_high_t dct_const_round_shift(tran_high_t input) { ...@@ -27,12 +27,24 @@ static INLINE tran_high_t dct_const_round_shift(tran_high_t input) {
} }
static INLINE tran_high_t check_range(tran_high_t input, int bd) { static INLINE tran_high_t check_range(tran_high_t input, int bd) {
const int32_t int_max = (1 << (7 + bd)) - 1; #if CONFIG_DAALA_TX
// Daala TX coeffs cover a different range from AV1 TX
// Temporary: this will be changing shortly upon moving to a fixed
// coeff depth!
// - 8 bit: signed 18 bit integer
// - 10 bit: signed 20 bit integer
// - 12 bit: signed 22 bit integer
const int32_t int_max = (1 << (9 + bd)) - 1;
const int32_t int_min = -int_max - 1; const int32_t int_min = -int_max - 1;
#if CONFIG_COEFFICIENT_RANGE_CHECKING #else
// AV1 TX case
// - 8 bit: signed 16 bit integer // - 8 bit: signed 16 bit integer
// - 10 bit: signed 18 bit integer // - 10 bit: signed 18 bit integer
// - 12 bit: signed 20 bit integer // - 12 bit: signed 20 bit integer
const int32_t int_max = (1 << (7 + bd)) - 1;
const int32_t int_min = -int_max - 1;
#endif
#if CONFIG_COEFFICIENT_RANGE_CHECKING
assert(int_min <= input); assert(int_min <= input);
assert(input <= int_max); assert(input <= int_max);
#endif // CONFIG_COEFFICIENT_RANGE_CHECKING #endif // CONFIG_COEFFICIENT_RANGE_CHECKING
......
...@@ -14,12 +14,10 @@ ...@@ -14,12 +14,10 @@
#include "./aom_dsp_rtcd.h" #include "./aom_dsp_rtcd.h"
#include "av1/common/daala_tx.h" #include "av1/common/daala_tx.h"
#include "av1/common/daala_inv_txfm.h" #include "av1/common/daala_inv_txfm.h"
#include "av1/common/idct.h"
#if CONFIG_DAALA_TX #if CONFIG_DAALA_TX
// Temporary while we still need av1_get_tx_scale() for testing
#include "av1/common/idct.h"
// Complete Daala TX map, sans lossless which is special cased // Complete Daala TX map, sans lossless which is special cased
typedef void (*daala_itx)(od_coeff *, int, const od_coeff[]); typedef void (*daala_itx)(od_coeff *, int, const od_coeff[]);
...@@ -69,7 +67,7 @@ void daala_inv_txfm_add(const tran_low_t *input_coeffs, void *output_pixels, ...@@ -69,7 +67,7 @@ void daala_inv_txfm_add(const tran_low_t *input_coeffs, void *output_pixels,
txfm_param->eob, px_depth); txfm_param->eob, px_depth);
} else { } else {
// General TX case // General TX case
// Q3 coeff Q4 TX compatability mode, with av1_get_tx_scale // Q3 coeff Q4 TX compatability mode
const int downshift = 4; const int downshift = 4;
assert(sizeof(tran_low_t) == sizeof(od_coeff)); assert(sizeof(tran_low_t) == sizeof(od_coeff));
assert(sizeof(tran_low_t) >= 4); assert(sizeof(tran_low_t) >= 4);
...@@ -96,10 +94,10 @@ void daala_inv_txfm_add(const tran_low_t *input_coeffs, void *output_pixels, ...@@ -96,10 +94,10 @@ void daala_inv_txfm_add(const tran_low_t *input_coeffs, void *output_pixels,
assert(row_tx); assert(row_tx);
// This is temporary while we're testing against existing // This is temporary while we're testing against existing
// behavior (preshift up one plus av1_get_tx_scale). // behavior (preshift up one).
// Remove before flight // Remove before flight
od_coeff tmp[MAX_TX_SQUARE]; od_coeff tmp[MAX_TX_SQUARE];
int upshift = 1 + av1_get_tx_scale(tx_size); int upshift = 1;
for (r = 0; r < rows; ++r) for (r = 0; r < rows; ++r)
for (c = 0; c < cols; ++c) for (c = 0; c < cols; ++c)
tmp[r * cols + c] = input_coeffs[r * cols + c] << upshift; tmp[r * cols + c] = input_coeffs[r * cols + c] << upshift;
......
...@@ -28,8 +28,13 @@ ...@@ -28,8 +28,13 @@
#endif #endif
int av1_get_tx_scale(const TX_SIZE tx_size) { int av1_get_tx_scale(const TX_SIZE tx_size) {
#if CONFIG_DAALA_TX
(void)tx_size;
return 0;
#else
const int pels = tx_size_2d[tx_size]; const int pels = tx_size_2d[tx_size];
return (pels > 256) + (pels > 1024) + (pels > 4096); return (pels > 256) + (pels > 1024) + (pels > 4096);
#endif
} }
#if !CONFIG_DAALA_TX #if !CONFIG_DAALA_TX
......
...@@ -17,9 +17,6 @@ ...@@ -17,9 +17,6 @@
#if CONFIG_DAALA_TX #if CONFIG_DAALA_TX
// Temporary while we still need av1_get_tx_scale() for testing
#include "av1/common/idct.h"
// Complete Daala TX map, sans lossless which is special cased // Complete Daala TX map, sans lossless which is special cased
typedef void (*daala_ftx)(od_coeff[], const od_coeff *, int); typedef void (*daala_ftx)(od_coeff[], const od_coeff *, int);
...@@ -62,7 +59,7 @@ void daala_fwd_txfm(const int16_t *input_pixels, tran_low_t *output_coeffs, ...@@ -62,7 +59,7 @@ void daala_fwd_txfm(const int16_t *input_pixels, tran_low_t *output_coeffs,
av1_fwht4x4(input_pixels, output_coeffs, input_stride); av1_fwht4x4(input_pixels, output_coeffs, input_stride);
} else { } else {
// General TX case // General TX case
// up 4, down 1 compatability mode with av1_get_tx_scale // up 4, down 1 compatability mode
const int upshift = 4; const int upshift = 4;
assert(upshift >= 0); assert(upshift >= 0);
...@@ -115,8 +112,8 @@ void daala_fwd_txfm(const int16_t *input_pixels, tran_low_t *output_coeffs, ...@@ -115,8 +112,8 @@ void daala_fwd_txfm(const int16_t *input_pixels, tran_low_t *output_coeffs,
} }
// This is temporary while we're testing against existing // This is temporary while we're testing against existing
// behavior (preshift up 4, then downshift by one plus av1_get_tx_scale) // behavior (preshift up 4, then downshift by one)
int downshift = 1 + av1_get_tx_scale(tx_size); int downshift = 1;
for (r = 0; r < rows; ++r) for (r = 0; r < rows; ++r)
for (c = 0; c < cols; ++c) for (c = 0; c < cols; ++c)
output_coeffs[r * cols + c] = output_coeffs[r * cols + c] =
......
...@@ -570,8 +570,13 @@ void av1_xform_quant(const AV1_COMMON *cm, MACROBLOCK *x, int plane, int block, ...@@ -570,8 +570,13 @@ void av1_xform_quant(const AV1_COMMON *cm, MACROBLOCK *x, int plane, int block,
if (xform_quant_idx != AV1_XFORM_QUANT_SKIP_QUANT) { if (xform_quant_idx != AV1_XFORM_QUANT_SKIP_QUANT) {
if (LIKELY(!x->skip_block)) { if (LIKELY(!x->skip_block)) {
#if CONFIG_DAALA_TX
quant_func_list[xform_quant_idx][1](coeff, tx2d_size, p, qcoeff, dqcoeff,
eob, scan_order, &qparam);
#else
quant_func_list[xform_quant_idx][txfm_param.is_hbd]( quant_func_list[xform_quant_idx][txfm_param.is_hbd](
coeff, tx2d_size, p, qcoeff, dqcoeff, eob, scan_order, &qparam); coeff, tx2d_size, p, qcoeff, dqcoeff, eob, scan_order, &qparam);
#endif
} else { } else {
av1_quantize_skip(tx2d_size, qcoeff, dqcoeff, eob); av1_quantize_skip(tx2d_size, qcoeff, dqcoeff, eob);
} }
......
...@@ -1930,6 +1930,10 @@ void av1_dist_block(const AV1_COMP *cpi, MACROBLOCK *x, int plane, ...@@ -1930,6 +1930,10 @@ void av1_dist_block(const AV1_COMP *cpi, MACROBLOCK *x, int plane,
tran_low_t *const coeff = BLOCK_OFFSET(p->coeff, block); tran_low_t *const coeff = BLOCK_OFFSET(p->coeff, block);
tran_low_t *const dqcoeff = BLOCK_OFFSET(pd->dqcoeff, block); tran_low_t *const dqcoeff = BLOCK_OFFSET(pd->dqcoeff, block);
#if CONFIG_DAALA_TX
*out_dist = av1_highbd_block_error(coeff, dqcoeff, buffer_length, &this_sse,
xd->bd);
#else
#if CONFIG_HIGHBITDEPTH #if CONFIG_HIGHBITDEPTH
if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH)
*out_dist = av1_highbd_block_error(coeff, dqcoeff, buffer_length, *out_dist = av1_highbd_block_error(coeff, dqcoeff, buffer_length,
...@@ -1937,6 +1941,7 @@ void av1_dist_block(const AV1_COMP *cpi, MACROBLOCK *x, int plane, ...@@ -1937,6 +1941,7 @@ void av1_dist_block(const AV1_COMP *cpi, MACROBLOCK *x, int plane,
else else
#endif #endif
*out_dist = av1_block_error(coeff, dqcoeff, buffer_length, &this_sse); *out_dist = av1_block_error(coeff, dqcoeff, buffer_length, &this_sse);
#endif
*out_dist = RIGHT_SIGNED_SHIFT(*out_dist, shift); *out_dist = RIGHT_SIGNED_SHIFT(*out_dist, shift);
*out_sse = RIGHT_SIGNED_SHIFT(this_sse, shift); *out_sse = RIGHT_SIGNED_SHIFT(this_sse, shift);
...@@ -2107,6 +2112,10 @@ static void block_rd_txfm(int plane, int block, int blk_row, int blk_col, ...@@ -2107,6 +2112,10 @@ static void block_rd_txfm(int plane, int block, int blk_row, int blk_col,
const int buffer_length = tx_size_2d[tx_size]; const int buffer_length = tx_size_2d[tx_size];
int64_t tmp_dist; int64_t tmp_dist;
int64_t tmp; int64_t tmp;
#if CONFIG_DAALA_TX
tmp_dist =
av1_highbd_block_error(coeff, dqcoeff, buffer_length, &tmp, xd->bd);
#else
#if CONFIG_HIGHBITDEPTH #if CONFIG_HIGHBITDEPTH
if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH)
tmp_dist = tmp_dist =
...@@ -2114,6 +2123,7 @@ static void block_rd_txfm(int plane, int block, int blk_row, int blk_col, ...@@ -2114,6 +2123,7 @@ static void block_rd_txfm(int plane, int block, int blk_row, int blk_col,
else else
#endif #endif
tmp_dist = av1_block_error(coeff, dqcoeff, buffer_length, &tmp); tmp_dist = av1_block_error(coeff, dqcoeff, buffer_length, &tmp);
#endif
tmp_dist = RIGHT_SIGNED_SHIFT(tmp_dist, shift); tmp_dist = RIGHT_SIGNED_SHIFT(tmp_dist, shift);
if ( if (
...@@ -3706,6 +3716,10 @@ void av1_tx_block_rd_b(const AV1_COMP *cpi, MACROBLOCK *x, TX_SIZE tx_size, ...@@ -3706,6 +3716,10 @@ void av1_tx_block_rd_b(const AV1_COMP *cpi, MACROBLOCK *x, TX_SIZE tx_size,
x->tune_metric != AOM_TUNE_PSNR; x->tune_metric != AOM_TUNE_PSNR;
#endif // CONFIG_DIST_8X8 #endif // CONFIG_DIST_8X8
#if CONFIG_DAALA_TX
tmp_dist =
av1_highbd_block_error(coeff, dqcoeff, buffer_length, &tmp_sse, xd->bd);
#else
#if CONFIG_HIGHBITDEPTH #if CONFIG_HIGHBITDEPTH
if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH)
tmp_dist = tmp_dist =
...@@ -3713,6 +3727,7 @@ void av1_tx_block_rd_b(const AV1_COMP *cpi, MACROBLOCK *x, TX_SIZE tx_size, ...@@ -3713,6 +3727,7 @@ void av1_tx_block_rd_b(const AV1_COMP *cpi, MACROBLOCK *x, TX_SIZE tx_size,
else else
#endif #endif
tmp_dist = av1_block_error(coeff, dqcoeff, buffer_length, &tmp_sse); tmp_dist = av1_block_error(coeff, dqcoeff, buffer_length, &tmp_sse);
#endif
tmp_dist = RIGHT_SIGNED_SHIFT(tmp_dist, shift); tmp_dist = RIGHT_SIGNED_SHIFT(tmp_dist, shift);
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment