Commit e93acb2d authored by Timothy B. Terriberry's avatar Timothy B. Terriberry Committed by Tim Terriberry

Scale PVQ input to OD_COEFF_SHIFT resolution.

This ensures we operate at the same precision that Daala uses, which matters
when activity masking is enabled, because of the gain companding.

Metrics from Patchset 4 (which had slightly incorrect rounding):

With activity masking (5 frames only):
av1_pvq_AM_ref_5f@2017-02-07T03:37:53.702Z -> av1_pvq_AM_derf_fix2_coeff_scaling_5f@2017-02-07T00:12:24.427Z

    PSNR |  PSNR Cb |  PSNR Cr | PSNR HVS |   SSIM | MS SSIM | CIEDE 2000
  0.6653 | -12.3177 | -12.1858 |   0.3350 | 4.1013 |  2.0964 |    -4.0539

In particular for Netflix_Crosswalk_1920x1080_60fps_8bit_420_60f.y4m
 -5.0589 | -22.3077 | -21.2188 |  -7.0389 | -3.3715 |-5.7794 |   -13.1891

I.e., it fixes the large regression with AM on this sequence, and
 substantially improves chroma (at a lesser cost to other metrics).

Without activity masking (5 frames only):
av1_pvq_ref_5f@2017-02-07T03:52:51.279Z -> av1_pvq_derf_fix2_coeff_scaling_5f@2017-02-07T00:12:48.873Z

    PSNR | PSNR Cb | PSNR Cr | PSNR HVS |   SSIM | MS SSIM | CIEDE 2000
  0.0989 | -0.0322 | -0.0464 |   0.1883 | 0.0795 |  0.0579 |     0.0923

Change-Id: I46b808b7c8e4733465f8bebc8336dfd5b75783ec
parent 7a40600c
...@@ -346,7 +346,9 @@ static int av1_pvq_decode_helper(od_dec_ctx *dec, int16_t *ref_coeff, ...@@ -346,7 +346,9 @@ static int av1_pvq_decode_helper(od_dec_ctx *dec, int16_t *ref_coeff,
int off; int off;
const int is_keyframe = 0; const int is_keyframe = 0;
const int has_dc_skip = 1; const int has_dc_skip = 1;
int quant_shift = get_tx_scale(bs); /*TODO(tterribe): Handle CONFIG_AOM_HIGHBITDEPTH.*/
int coeff_shift = 3 - get_tx_scale(bs);
int rounding_mask;
// DC quantizer for PVQ // DC quantizer for PVQ
int pvq_dc_quant; int pvq_dc_quant;
int lossless = (quant[0] == 0); int lossless = (quant[0] == 0);
...@@ -363,40 +365,47 @@ static int av1_pvq_decode_helper(od_dec_ctx *dec, int16_t *ref_coeff, ...@@ -363,40 +365,47 @@ static int av1_pvq_decode_helper(od_dec_ctx *dec, int16_t *ref_coeff,
od_raster_to_coding_order(ref_coeff_pvq, blk_size, tx_type, ref_coeff, od_raster_to_coding_order(ref_coeff_pvq, blk_size, tx_type, ref_coeff,
blk_size); blk_size);
assert(OD_COEFF_SHIFT >= 3);
if (lossless) if (lossless)
pvq_dc_quant = 1; pvq_dc_quant = 1;
else { else {
if (use_activity_masking) if (use_activity_masking)
pvq_dc_quant = OD_MAXI( pvq_dc_quant = OD_MAXI(
1, (quant[0] >> quant_shift) * 1, (quant[0] << (OD_COEFF_SHIFT - 3)) *
dec->state.pvq_qm_q4[pli][od_qm_get_index(bs, 0)] >> dec->state.pvq_qm_q4[pli][od_qm_get_index(bs, 0)] >>
4); 4);
else else
pvq_dc_quant = OD_MAXI(1, quant[0] >> quant_shift); pvq_dc_quant = OD_MAXI(1, quant[0] << (OD_COEFF_SHIFT - 3));
} }
off = od_qm_offset(bs, xdec); off = od_qm_offset(bs, xdec);
// copy int16 inputs to int32 // copy int16 inputs to int32
for (i = 0; i < blk_size * blk_size; i++) ref_int32[i] = ref_coeff_pvq[i]; for (i = 0; i < blk_size * blk_size; i++) {
ref_int32[i] = ref_coeff_pvq[i] << (OD_COEFF_SHIFT - coeff_shift);
}
od_pvq_decode(dec, ref_int32, out_int32, (int)quant[1] >> quant_shift, pli, od_pvq_decode(dec, ref_int32, out_int32, quant[1] << (OD_COEFF_SHIFT - 3),
bs, OD_PVQ_BETA[use_activity_masking][pli][bs], pli, bs, OD_PVQ_BETA[use_activity_masking][pli][bs],
OD_ROBUST_STREAM, is_keyframe, &flags, ac_dc_coded, OD_ROBUST_STREAM, is_keyframe, &flags, ac_dc_coded,
dec->state.qm + off, dec->state.qm_inv + off); dec->state.qm + off, dec->state.qm_inv + off);
// copy int32 result back to int16 if (!has_dc_skip || out_int32[0]) {
for (i = 0; i < blk_size * blk_size; i++) dqcoeff_pvq[i] = out_int32[i]; out_int32[0] =
if (!has_dc_skip || dqcoeff_pvq[0]) {
dqcoeff_pvq[0] =
has_dc_skip + generic_decode(dec->r, &dec->state.adapt.model_dc[pli], has_dc_skip + generic_decode(dec->r, &dec->state.adapt.model_dc[pli],
-1, &dec->state.adapt.ex_dc[pli][bs][0], 2, -1, &dec->state.adapt.ex_dc[pli][bs][0], 2,
"dc:mag"); "dc:mag");
if (dqcoeff_pvq[0]) if (out_int32[0]) out_int32[0] *= aom_read_bit(dec->r, "dc:sign") ? -1 : 1;
dqcoeff_pvq[0] *= aom_read_bit(dec->r, "dc:sign") ? -1 : 1; }
out_int32[0] = out_int32[0] * pvq_dc_quant + ref_int32[0];
// copy int32 result back to int16
assert(OD_COEFF_SHIFT > coeff_shift);
rounding_mask = (1 << (OD_COEFF_SHIFT - coeff_shift - 1)) - 1;
for (i = 0; i < blk_size * blk_size; i++) {
dqcoeff_pvq[i] = (out_int32[i] + (out_int32[i] < 0) + rounding_mask) >>
(OD_COEFF_SHIFT - coeff_shift);
} }
dqcoeff_pvq[0] = dqcoeff_pvq[0] * pvq_dc_quant + ref_coeff_pvq[0];
od_coding_order_to_raster(dqcoeff, blk_size, tx_type, dqcoeff_pvq, blk_size); od_coding_order_to_raster(dqcoeff, blk_size, tx_type, dqcoeff_pvq, blk_size);
......
...@@ -1129,7 +1129,9 @@ PVQ_SKIP_TYPE av1_pvq_encode_helper( ...@@ -1129,7 +1129,9 @@ PVQ_SKIP_TYPE av1_pvq_encode_helper(
int tx_size, TX_TYPE tx_type, int *rate, int speed, PVQ_INFO *pvq_info) { int tx_size, TX_TYPE tx_type, int *rate, int speed, PVQ_INFO *pvq_info) {
const int tx_blk_size = tx_size_wide[tx_size]; const int tx_blk_size = tx_size_wide[tx_size];
PVQ_SKIP_TYPE ac_dc_coded; PVQ_SKIP_TYPE ac_dc_coded;
int quant_shift = get_tx_scale(tx_size); /*TODO(tterribe): Handle CONFIG_AOM_HIGHBITDEPTH.*/
int coeff_shift = 3 - get_tx_scale(tx_size);
int rounding_mask;
int pvq_dc_quant; int pvq_dc_quant;
int use_activity_masking = daala_enc->use_activity_masking; int use_activity_masking = daala_enc->use_activity_masking;
int tell; int tell;
...@@ -1148,15 +1150,16 @@ PVQ_SKIP_TYPE av1_pvq_encode_helper( ...@@ -1148,15 +1150,16 @@ PVQ_SKIP_TYPE av1_pvq_encode_helper(
DECLARE_ALIGNED(16, int32_t, ref_int32[OD_TXSIZE_MAX * OD_TXSIZE_MAX]); DECLARE_ALIGNED(16, int32_t, ref_int32[OD_TXSIZE_MAX * OD_TXSIZE_MAX]);
DECLARE_ALIGNED(16, int32_t, out_int32[OD_TXSIZE_MAX * OD_TXSIZE_MAX]); DECLARE_ALIGNED(16, int32_t, out_int32[OD_TXSIZE_MAX * OD_TXSIZE_MAX]);
assert(OD_COEFF_SHIFT >= 3);
// DC quantizer for PVQ // DC quantizer for PVQ
if (use_activity_masking) if (use_activity_masking)
pvq_dc_quant = OD_MAXI( pvq_dc_quant = OD_MAXI(
1, (quant[0] >> quant_shift) * 1, (quant[0] << (OD_COEFF_SHIFT - 3)) *
daala_enc->state.pvq_qm_q4[plane] daala_enc->state.pvq_qm_q4[plane]
[od_qm_get_index(tx_size, 0)] >> [od_qm_get_index(tx_size, 0)] >>
4); 4);
else else
pvq_dc_quant = OD_MAXI(1, quant[0] >> quant_shift); pvq_dc_quant = OD_MAXI(1, quant[0] << (OD_COEFF_SHIFT - 3));
*eob = 0; *eob = 0;
...@@ -1174,8 +1177,8 @@ PVQ_SKIP_TYPE av1_pvq_encode_helper( ...@@ -1174,8 +1177,8 @@ PVQ_SKIP_TYPE av1_pvq_encode_helper(
// copy int16 inputs to int32 // copy int16 inputs to int32
for (i = 0; i < tx_blk_size * tx_blk_size; i++) { for (i = 0; i < tx_blk_size * tx_blk_size; i++) {
ref_int32[i] = ref_coeff_pvq[i]; ref_int32[i] = ref_coeff_pvq[i] << (OD_COEFF_SHIFT - coeff_shift);
in_int32[i] = coeff_pvq[i]; in_int32[i] = coeff_pvq[i] << (OD_COEFF_SHIFT - coeff_shift);
} }
#if PVQ_CHROMA_RD #if PVQ_CHROMA_RD
...@@ -1192,8 +1195,8 @@ PVQ_SKIP_TYPE av1_pvq_encode_helper( ...@@ -1192,8 +1195,8 @@ PVQ_SKIP_TYPE av1_pvq_encode_helper(
ac_dc_coded = od_pvq_encode( ac_dc_coded = od_pvq_encode(
daala_enc, ref_int32, in_int32, out_int32, daala_enc, ref_int32, in_int32, out_int32,
(int)quant[0] >> quant_shift, // scale/quantizer quant[0] << (OD_COEFF_SHIFT - 3), // scale/quantizer
(int)quant[1] >> quant_shift, // scale/quantizer quant[1] << (OD_COEFF_SHIFT - 3), // scale/quantizer
plane, tx_size, OD_PVQ_BETA[use_activity_masking][plane][tx_size], plane, tx_size, OD_PVQ_BETA[use_activity_masking][plane][tx_size],
OD_ROBUST_STREAM, OD_ROBUST_STREAM,
0, // is_keyframe, 0, // is_keyframe,
...@@ -1220,7 +1223,12 @@ PVQ_SKIP_TYPE av1_pvq_encode_helper( ...@@ -1220,7 +1223,12 @@ PVQ_SKIP_TYPE av1_pvq_encode_helper(
out_int32[0] += ref_int32[0]; out_int32[0] += ref_int32[0];
// copy int32 result back to int16 // copy int32 result back to int16
for (i = 0; i < tx_blk_size * tx_blk_size; i++) dqcoeff_pvq[i] = out_int32[i]; assert(OD_COEFF_SHIFT > coeff_shift);
rounding_mask = (1 << (OD_COEFF_SHIFT - coeff_shift - 1)) - 1;
for (i = 0; i < tx_blk_size * tx_blk_size; i++) {
dqcoeff_pvq[i] = (out_int32[i] + (out_int32[i] < 0) + rounding_mask) >>
(OD_COEFF_SHIFT - coeff_shift);
}
// Back to original coefficient order // Back to original coefficient order
od_coding_order_to_raster(dqcoeff, tx_blk_size, tx_type, dqcoeff_pvq, od_coding_order_to_raster(dqcoeff, tx_blk_size, tx_type, dqcoeff_pvq,
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment