Commit b02d2f39 authored by Debargha Mukherjee's avatar Debargha Mukherjee

Fix rd scales for transforms larger than 32x32

Change-Id: I1ddec0cf3513e2bd7568393e5ed5d52c25014ab4
parent 1e2fb103
...@@ -54,6 +54,10 @@ ...@@ -54,6 +54,10 @@
(((value) < 0) ? -ROUND_POWER_OF_TWO_64(-(value), (n)) \ (((value) < 0) ? -ROUND_POWER_OF_TWO_64(-(value), (n)) \
: ROUND_POWER_OF_TWO_64((value), (n))) : ROUND_POWER_OF_TWO_64((value), (n)))
/* shift right or left depending on sign of n */
#define RIGHT_SIGNED_SHIFT(value, n) \
((n) < 0 ? ((value) << (-(n))) : ((value) >> (n)))
#define ALIGN_POWER_OF_TWO(value, n) \ #define ALIGN_POWER_OF_TWO(value, n) \
(((value) + ((1 << (n)) - 1)) & ~((1 << (n)) - 1)) (((value) + ((1 << (n)) - 1)) & ~((1 << (n)) - 1))
......
...@@ -1752,25 +1752,22 @@ void av1_dist_block(const AV1_COMP *cpi, MACROBLOCK *x, int plane, ...@@ -1752,25 +1752,22 @@ void av1_dist_block(const AV1_COMP *cpi, MACROBLOCK *x, int plane,
#if CONFIG_HIGHBITDEPTH #if CONFIG_HIGHBITDEPTH
const int bd = (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) ? xd->bd : 8; const int bd = (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) ? xd->bd : 8;
*out_dist = av1_highbd_block_error2_c(coeff, dqcoeff, ref_coeff, *out_dist = av1_highbd_block_error2_c(coeff, dqcoeff, ref_coeff,
buffer_length, &this_sse, bd) >> buffer_length, &this_sse, bd);
shift;
#else #else
*out_dist = av1_block_error2_c(coeff, dqcoeff, ref_coeff, buffer_length, *out_dist =
&this_sse) >> av1_block_error2_c(coeff, dqcoeff, ref_coeff, buffer_length, &this_sse);
shift;
#endif // CONFIG_HIGHBITDEPTH #endif // CONFIG_HIGHBITDEPTH
#else // !CONFIG_PVQ #else // !CONFIG_PVQ
#if CONFIG_HIGHBITDEPTH #if CONFIG_HIGHBITDEPTH
if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH)
*out_dist = av1_highbd_block_error(coeff, dqcoeff, buffer_length, *out_dist = av1_highbd_block_error(coeff, dqcoeff, buffer_length,
&this_sse, xd->bd) >> &this_sse, xd->bd);
shift;
else else
#endif #endif
*out_dist = *out_dist = av1_block_error(coeff, dqcoeff, buffer_length, &this_sse);
av1_block_error(coeff, dqcoeff, buffer_length, &this_sse) >> shift;
#endif // CONFIG_PVQ #endif // CONFIG_PVQ
*out_sse = this_sse >> shift; *out_dist = RIGHT_SIGNED_SHIFT(*out_dist, shift);
*out_sse = RIGHT_SIGNED_SHIFT(this_sse, shift);
} else { } else {
const BLOCK_SIZE tx_bsize = txsize_to_bsize[tx_size]; const BLOCK_SIZE tx_bsize = txsize_to_bsize[tx_size];
#if !CONFIG_PVQ || CONFIG_DIST_8X8 #if !CONFIG_PVQ || CONFIG_DIST_8X8
...@@ -1951,11 +1948,11 @@ static void block_rd_txfm(int plane, int block, int blk_row, int blk_col, ...@@ -1951,11 +1948,11 @@ static void block_rd_txfm(int plane, int block, int blk_row, int blk_col,
#if CONFIG_HIGHBITDEPTH #if CONFIG_HIGHBITDEPTH
if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH)
tmp_dist = tmp_dist =
av1_highbd_block_error(coeff, dqcoeff, buffer_length, &tmp, xd->bd) >> av1_highbd_block_error(coeff, dqcoeff, buffer_length, &tmp, xd->bd);
shift;
else else
#endif #endif
tmp_dist = av1_block_error(coeff, dqcoeff, buffer_length, &tmp) >> shift; tmp_dist = av1_block_error(coeff, dqcoeff, buffer_length, &tmp);
tmp_dist = RIGHT_SIGNED_SHIFT(tmp_dist, shift);
if ( if (
#if CONFIG_DIST_8X8 #if CONFIG_DIST_8X8
...@@ -4442,13 +4439,13 @@ void av1_tx_block_rd_b(const AV1_COMP *cpi, MACROBLOCK *x, TX_SIZE tx_size, ...@@ -4442,13 +4439,13 @@ void av1_tx_block_rd_b(const AV1_COMP *cpi, MACROBLOCK *x, TX_SIZE tx_size,
#if CONFIG_HIGHBITDEPTH #if CONFIG_HIGHBITDEPTH
if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH)
tmp_dist = av1_highbd_block_error(coeff, dqcoeff, buffer_length, &tmp_sse, tmp_dist =
xd->bd) >> av1_highbd_block_error(coeff, dqcoeff, buffer_length, &tmp_sse, xd->bd);
shift;
else else
#endif #endif
tmp_dist = tmp_dist = av1_block_error(coeff, dqcoeff, buffer_length, &tmp_sse);
av1_block_error(coeff, dqcoeff, buffer_length, &tmp_sse) >> shift;
tmp_dist = RIGHT_SIGNED_SHIFT(tmp_dist, shift);
#if CONFIG_MRC_TX #if CONFIG_MRC_TX
if (tx_type == MRC_DCT && !xd->mi[0]->mbmi.valid_mrc_mask) { if (tx_type == MRC_DCT && !xd->mi[0]->mbmi.valid_mrc_mask) {
...@@ -4465,8 +4462,8 @@ void av1_tx_block_rd_b(const AV1_COMP *cpi, MACROBLOCK *x, TX_SIZE tx_size, ...@@ -4465,8 +4462,8 @@ void av1_tx_block_rd_b(const AV1_COMP *cpi, MACROBLOCK *x, TX_SIZE tx_size,
a, l, 1); a, l, 1);
} else { } else {
rd_stats->rate += rd_stats->zero_rate; rd_stats->rate += rd_stats->zero_rate;
rd_stats->dist += tmp_sse >> shift; rd_stats->dist += RIGHT_SIGNED_SHIFT(tmp_sse, shift);
rd_stats->sse += tmp_sse >> shift; rd_stats->sse += RIGHT_SIGNED_SHIFT(tmp_sse, shift);
rd_stats->skip = 1; rd_stats->skip = 1;
rd_stats->invalid_rate = 1; rd_stats->invalid_rate = 1;
return; return;
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment