Commit 60f2a229 authored by Monty Montgomery's avatar Monty Montgomery Committed by Christopher Montgomery

Add Daala TX fixed-coeff-depth capability to quantization

This patch completes the work to add fixed-depth TX domain support to
the quantization and dequantization code.  At present, it is active but
configured to behave identically to current AV1 master as RDO and TX
have not yet been updated to also support this functionality.

subset-1:
monty-rest-of-stack-noshift-s1@2017-11-13T14:37:42.541Z ->
 monty-rest-of-stack-quant-s1@2017-11-13T14:38:43.774Z

  PSNR | PSNR Cb | PSNR Cr | PSNR HVS |   SSIM | MS SSIM | CIEDE 2000
0.0000 |  0.0000 |  0.0000 |   0.0000 | 0.0000 |  0.0000 |     0.0000

objective-1-fast --limit=4:
monty-rest-of-stack-noshift-o1f4@2017-11-13T14:37:16.992Z ->
 monty-rest-of-stack-quant-o1f4@2017-11-13T14:38:28.828Z

  PSNR | PSNR Cb | PSNR Cr | PSNR HVS |   SSIM | MS SSIM | CIEDE 2000
0.0000 |  0.0000 |  0.0000 |   0.0000 | 0.0000 |  0.0000 |     0.0000

Change-Id: I3773a1fc128136c9fea227f4b547576a8aa6efa3
parent 605d63f3
......@@ -65,7 +65,11 @@ extern "C" {
/**The maximum number of color planes allowed in a single frame.*/
# define OD_NPLANES_MAX (3)
# define OD_COEFF_SHIFT (4)
/* Native coefficient 'bitdepth'; TX is scaled up by (TX_COEFF_DEPTH-bitdepth)
such that the real coefficient depth precision is always TX_CEOFF_DEPTH
regardless of bitdepth or transform size.
*/
# define TX_COEFF_DEPTH (11)
# define OD_DISABLE_CFL (1)
# define OD_DISABLE_FILTER (1)
......
This diff is collapsed.
......@@ -37,9 +37,11 @@ extern "C" {
struct AV1Common;
int16_t av1_dc_quant(int qindex, int delta, aom_bit_depth_t bit_depth);
int16_t av1_ac_quant(int qindex, int delta, aom_bit_depth_t bit_depth);
int16_t av1_qindex_from_ac(int ac, aom_bit_depth_t bit_depth);
int16_t av1_dc_quant_Q3(int qindex, int delta, aom_bit_depth_t bit_depth);
int16_t av1_ac_quant_Q3(int qindex, int delta, aom_bit_depth_t bit_depth);
int16_t av1_dc_quant_QTX(int qindex, int delta, aom_bit_depth_t bit_depth);
int16_t av1_ac_quant_QTX(int qindex, int delta, aom_bit_depth_t bit_depth);
int16_t av1_qindex_from_ac_Q3(int ac_Q3, aom_bit_depth_t bit_depth);
int av1_get_qindex(const struct segmentation *seg, int segment_id,
#if CONFIG_Q_SEGMENTATION
......
......@@ -394,14 +394,6 @@ static void decode_mbmi_block(AV1Decoder *const pbi, MACROBLOCKD *const xd,
aom_merge_corrupted_flag(&xd->corrupted, reader_corrupted_flag);
}
// Converts a Q3 quantizer lookup from static configuration to the
// actual TX scaling in use
static int dequant_Q3_to_QTX(int q3, int bd) {
// Right now, TX scale in use is still Q3
(void)bd;
return q3;
}
static void decode_token_and_recon_block(AV1Decoder *const pbi,
MACROBLOCKD *const xd, int mi_row,
int mi_col, aom_reader *r,
......@@ -439,12 +431,10 @@ static void decode_token_and_recon_block(AV1Decoder *const pbi,
: (j == 1 ? cm->u_dc_delta_q : cm->v_dc_delta_q);
const int ac_delta_q =
j == 0 ? 0 : (j == 1 ? cm->u_ac_delta_q : cm->v_ac_delta_q);
xd->plane[j].seg_dequant_QTX[i][0] = dequant_Q3_to_QTX(
av1_dc_quant(current_qindex, dc_delta_q, cm->bit_depth),
cm->bit_depth);
xd->plane[j].seg_dequant_QTX[i][1] = dequant_Q3_to_QTX(
av1_ac_quant(current_qindex, ac_delta_q, cm->bit_depth),
cm->bit_depth);
xd->plane[j].seg_dequant_QTX[i][0] =
av1_dc_quant_QTX(current_qindex, dc_delta_q, cm->bit_depth);
xd->plane[j].seg_dequant_QTX[i][1] =
av1_ac_quant_QTX(current_qindex, ac_delta_q, cm->bit_depth);
}
}
}
......@@ -1329,18 +1319,17 @@ static void setup_segmentation_dequant(AV1_COMMON *const cm) {
#else
const int qindex = av1_get_qindex(&cm->seg, i, cm->base_qindex);
#endif
cm->y_dequant_QTX[i][0] = dequant_Q3_to_QTX(
av1_dc_quant(qindex, cm->y_dc_delta_q, cm->bit_depth), cm->bit_depth);
cm->y_dequant_QTX[i][1] = dequant_Q3_to_QTX(
av1_ac_quant(qindex, 0, cm->bit_depth), cm->bit_depth);
cm->u_dequant_QTX[i][0] = dequant_Q3_to_QTX(
av1_dc_quant(qindex, cm->u_dc_delta_q, cm->bit_depth), cm->bit_depth);
cm->u_dequant_QTX[i][1] = dequant_Q3_to_QTX(
av1_ac_quant(qindex, cm->u_ac_delta_q, cm->bit_depth), cm->bit_depth);
cm->v_dequant_QTX[i][0] = dequant_Q3_to_QTX(
av1_dc_quant(qindex, cm->v_dc_delta_q, cm->bit_depth), cm->bit_depth);
cm->v_dequant_QTX[i][1] = dequant_Q3_to_QTX(
av1_ac_quant(qindex, cm->v_ac_delta_q, cm->bit_depth), cm->bit_depth);
cm->y_dequant_QTX[i][0] =
av1_dc_quant_QTX(qindex, cm->y_dc_delta_q, cm->bit_depth);
cm->y_dequant_QTX[i][1] = av1_ac_quant_QTX(qindex, 0, cm->bit_depth);
cm->u_dequant_QTX[i][0] =
av1_dc_quant_QTX(qindex, cm->u_dc_delta_q, cm->bit_depth);
cm->u_dequant_QTX[i][1] =
av1_ac_quant_QTX(qindex, cm->u_ac_delta_q, cm->bit_depth);
cm->v_dequant_QTX[i][0] =
av1_dc_quant_QTX(qindex, cm->v_dc_delta_q, cm->bit_depth);
cm->v_dequant_QTX[i][1] =
av1_ac_quant_QTX(qindex, cm->v_ac_delta_q, cm->bit_depth);
#if CONFIG_AOM_QM
const int lossless = qindex == 0 && cm->y_dc_delta_q == 0 &&
cm->u_dc_delta_q == 0 && cm->u_ac_delta_q == 0 &&
......
......@@ -43,7 +43,7 @@ static const double aq_c_var_thresholds[AQ_C_STRENGTHS][AQ_C_SEGMENTS] = {
static int get_aq_c_strength(int q_index, aom_bit_depth_t bit_depth) {
// Approximate base quatizer (truncated to int)
const int base_quant = av1_ac_quant(q_index, 0, bit_depth) / 4;
const int base_quant = av1_ac_quant_Q3(q_index, 0, bit_depth) / 4;
return (base_quant > 10) + (base_quant > 25);
}
......
......@@ -1516,7 +1516,7 @@ static void invert_quant(int16_t *quant, int16_t *shift, int d) {
}
static int get_qzbin_factor(int q, aom_bit_depth_t bit_depth) {
const int quant = av1_dc_quant(q, 0, bit_depth);
const int quant = av1_dc_quant_Q3(q, 0, bit_depth);
#if CONFIG_HIGHBITDEPTH
switch (bit_depth) {
case AOM_BITS_8: return q == 0 ? 64 : (quant < 148 ? 84 : 80);
......@@ -1545,10 +1545,11 @@ void av1_build_quantizer(aom_bit_depth_t bit_depth, int y_dc_delta_q,
for (i = 0; i < 2; ++i) {
int qrounding_factor_fp = 64;
// y quantizer setup with original coeff shift of Q3
quant_Q3 = i == 0 ? av1_dc_quant(q, y_dc_delta_q, bit_depth)
: av1_ac_quant(q, 0, bit_depth);
// y quantizer with TX scale; right now, it's still Q3 as above;
quant_QTX = quant_Q3;
quant_Q3 = i == 0 ? av1_dc_quant_Q3(q, y_dc_delta_q, bit_depth)
: av1_ac_quant_Q3(q, 0, bit_depth);
// y quantizer with TX scale
quant_QTX = i == 0 ? av1_dc_quant_QTX(q, y_dc_delta_q, bit_depth)
: av1_ac_quant_QTX(q, 0, bit_depth);
invert_quant(&quants->y_quant[q][i], &quants->y_quant_shift[q][i],
quant_QTX);
quants->y_quant_fp[q][i] = (1 << 16) / quant_QTX;
......@@ -1559,10 +1560,11 @@ void av1_build_quantizer(aom_bit_depth_t bit_depth, int y_dc_delta_q,
deq->y_dequant_Q3[q][i] = quant_Q3;
// u quantizer setup with original coeff shift of Q3
quant_Q3 = i == 0 ? av1_dc_quant(q, u_dc_delta_q, bit_depth)
: av1_ac_quant(q, u_ac_delta_q, bit_depth);
// u quantizer with TX scale; right now, it's still Q3 as above;
quant_QTX = quant_Q3;
quant_Q3 = i == 0 ? av1_dc_quant_Q3(q, u_dc_delta_q, bit_depth)
: av1_ac_quant_Q3(q, u_ac_delta_q, bit_depth);
// u quantizer with TX scale
quant_QTX = i == 0 ? av1_dc_quant_QTX(q, u_dc_delta_q, bit_depth)
: av1_ac_quant_QTX(q, u_ac_delta_q, bit_depth);
invert_quant(&quants->u_quant[q][i], &quants->u_quant_shift[q][i],
quant_QTX);
quants->u_quant_fp[q][i] = (1 << 16) / quant_QTX;
......@@ -1573,10 +1575,11 @@ void av1_build_quantizer(aom_bit_depth_t bit_depth, int y_dc_delta_q,
deq->u_dequant_Q3[q][i] = quant_Q3;
// v quantizer setup with original coeff shift of Q3
quant_Q3 = i == 0 ? av1_dc_quant(q, v_dc_delta_q, bit_depth)
: av1_ac_quant(q, v_ac_delta_q, bit_depth);
// v quantizer with TX scale; right now, it's still Q3 as above;
quant_QTX = quant_Q3;
quant_Q3 = i == 0 ? av1_dc_quant_Q3(q, v_dc_delta_q, bit_depth)
: av1_ac_quant_Q3(q, v_ac_delta_q, bit_depth);
// v quantizer with TX scale
quant_QTX = i == 0 ? av1_dc_quant_QTX(q, v_dc_delta_q, bit_depth)
: av1_ac_quant_QTX(q, v_ac_delta_q, bit_depth);
invert_quant(&quants->v_quant[q][i], &quants->v_quant_shift[q][i],
quant_QTX);
quants->v_quant_fp[q][i] = (1 << 16) / quant_QTX;
......
......@@ -329,7 +329,7 @@ void av1_cdef_search(YV12_BUFFER_CONFIG *frame, const YV12_BUFFER_CONFIG *ref,
int chroma_cdef = xd->plane[1].subsampling_x == xd->plane[1].subsampling_y &&
xd->plane[2].subsampling_x == xd->plane[2].subsampling_y;
quantizer =
av1_ac_quant(cm->base_qindex, 0, cm->bit_depth) >> (cm->bit_depth - 8);
av1_ac_quant_Q3(cm->base_qindex, 0, cm->bit_depth) >> (cm->bit_depth - 8);
lambda = .12 * quantizer * quantizer / 256.;
av1_setup_dst_planes(xd->plane, cm->sb_size, frame, 0, 0);
......
......@@ -414,7 +414,7 @@ void av1_pick_filter_level(const YV12_BUFFER_CONFIG *sd, AV1_COMP *cpi,
} else if (method >= LPF_PICK_FROM_Q) {
const int min_filter_level = 0;
const int max_filter_level = av1_get_max_filter_level(cpi);
const int q = av1_ac_quant(cm->base_qindex, 0, cm->bit_depth);
const int q = av1_ac_quant_Q3(cm->base_qindex, 0, cm->bit_depth);
// These values were determined by linear fitting the result of the
// searched level for 8 bit depth:
// Keyframes: filt_guess = q * 0.06699 - 1.60817
......
......@@ -157,15 +157,15 @@ double av1_convert_qindex_to_q(int qindex, aom_bit_depth_t bit_depth) {
// Convert the index to a real Q value (scaled down to match old Q values)
#if CONFIG_HIGHBITDEPTH
switch (bit_depth) {
case AOM_BITS_8: return av1_ac_quant(qindex, 0, bit_depth) / 4.0;
case AOM_BITS_10: return av1_ac_quant(qindex, 0, bit_depth) / 16.0;
case AOM_BITS_12: return av1_ac_quant(qindex, 0, bit_depth) / 64.0;
case AOM_BITS_8: return av1_ac_quant_Q3(qindex, 0, bit_depth) / 4.0;
case AOM_BITS_10: return av1_ac_quant_Q3(qindex, 0, bit_depth) / 16.0;
case AOM_BITS_12: return av1_ac_quant_Q3(qindex, 0, bit_depth) / 64.0;
default:
assert(0 && "bit_depth should be AOM_BITS_8, AOM_BITS_10 or AOM_BITS_12");
return -1.0;
}
#else
return av1_ac_quant(qindex, 0, bit_depth) / 4.0;
return av1_ac_quant_Q3(qindex, 0, bit_depth) / 4.0;
#endif
}
......
......@@ -367,25 +367,47 @@ static void od_enc_rc_reset(od_rc_state *rc) {
/*All of these initial scale/exp values are from Theora, and have not yet
been adapted to Daala, so they're certainly wrong.
The B-frame values especially are simply copies of the P-frame values.*/
/*XXXXX: This constant initialization, apart from tuning, is very
likely also scaled incorrectly.
In Theora, where these constants come from, (bits/pixel) ==
scale*((q_Q2/4)^-(exp/64))
This can be derived from looking at the update formula in
od_enc_rc_update_state().
I.e., we have a quantizer normalized to Q0 for 8-bit pixel values,
which we exponentiate. To get the same behavior here, we need to
continue normalizing the quantizer the same way. Otherwise we'll have
to exponentiate any scaling baked into the quantizer as well (which
seems complicated and unnecessary).
If we have normalized the quantizer before exponentiation, then
the initializers for log_scale[] should not depend on bit depth or
coefficient depth in any way.
This is now restored to the initialization as is was in Theora,
and should be revistited/rederived/corrected for AV1.*/
if (ibpp < 1) {
rc->exp[OD_I_FRAME] = 59;
rc->log_scale[OD_I_FRAME] = od_blog64(1997) - OD_Q57(OD_COEFF_SHIFT);
rc->log_scale[OD_I_FRAME] = od_blog64(1997) - OD_Q57(8);
} else if (ibpp < 2) {
rc->exp[OD_I_FRAME] = 55;
rc->log_scale[OD_I_FRAME] = od_blog64(1604) - OD_Q57(OD_COEFF_SHIFT);
rc->log_scale[OD_I_FRAME] = od_blog64(1604) - OD_Q57(8);
} else {
rc->exp[OD_I_FRAME] = 48;
rc->log_scale[OD_I_FRAME] = od_blog64(834) - OD_Q57(OD_COEFF_SHIFT);
rc->log_scale[OD_I_FRAME] = od_blog64(834) - OD_Q57(8);
}
if (ibpp < 4) {
rc->exp[OD_P_FRAME] = 100;
rc->log_scale[OD_P_FRAME] = od_blog64(2249) - OD_Q57(OD_COEFF_SHIFT);
rc->log_scale[OD_P_FRAME] = od_blog64(2249) - OD_Q57(8);
} else if (ibpp < 8) {
rc->exp[OD_P_FRAME] = 95;
rc->log_scale[OD_P_FRAME] = od_blog64(1751) - OD_Q57(OD_COEFF_SHIFT);
rc->log_scale[OD_P_FRAME] = od_blog64(1751) - OD_Q57(8);
} else {
rc->exp[OD_P_FRAME] = 73;
rc->log_scale[OD_P_FRAME] = od_blog64(1260) - OD_Q57(OD_COEFF_SHIFT);
rc->log_scale[OD_P_FRAME] = od_blog64(1260) - OD_Q57(8);
}
/*Golden P-frames both use the same log_scale and exp modeling
values as regular P-frames and the same scale follower.
......@@ -810,7 +832,11 @@ int od_enc_rc_select_quantizers_and_lambdas(od_rc_state *rc,
calculation, that needs to be modulated as well.
Calculate what is, effectively, a fractional coded quantizer. */
/*Get the log2 quantizer in Q57 (normalized for coefficient shift).*/
log_quantizer = od_blog64(rc->base_quantizer) - OD_Q57(OD_COEFF_SHIFT);
/*XXXXX: See the above XXXX comment in rate control
initialization; the scaling on the log-quantizer calculation
should be the same as in quantizer scale initialization, but
OD_Q57(8) is possibly the incorrect value. */
log_quantizer = od_blog64(rc->base_quantizer) - OD_Q57(8);
/*log_quantizer to Q21.*/
log_quantizer >>= 36;
/*scale log quantizer, result is Q33.*/
......@@ -824,7 +850,7 @@ int od_enc_rc_select_quantizers_and_lambdas(od_rc_state *rc,
/*Back to log2 quantizer in Q57.*/
log_quantizer = (log_quantizer - OD_LOG_QUANTIZER_OFFSET_Q45) *
OD_LOG_QUANTIZER_EXP_Q12 +
OD_Q57(OD_COEFF_SHIFT);
OD_Q57(8);
/*Convert Q57 log2 quantizer to unclamped linear target quantizer value.*/
rc->target_quantizer = od_bexp64(log_quantizer);
}
......@@ -881,7 +907,11 @@ int od_enc_rc_select_quantizers_and_lambdas(od_rc_state *rc,
for (i = 0; i < OD_FRAME_NSUBTYPES; i++) {
/*Modulate base quantizer by frame type.*/
/*Get the log2 quantizer in Q57 (normalized for coefficient shift).*/
log_quantizer = log_base_quantizer - OD_Q57(OD_COEFF_SHIFT);
/*XXXXX: See the above XXXX comment in rate control
initialization; the scaling on the log-quantizer calculation
should be the same as in quantizer scale initialization, but
OD_Q57(8) is possibly the incorrect value. */
log_quantizer = log_base_quantizer - OD_Q57(8);
/*log_quantizer to Q21.*/
log_quantizer >>= 36;
/*scale log quantizer, result is Q33.*/
......@@ -895,7 +925,7 @@ int od_enc_rc_select_quantizers_and_lambdas(od_rc_state *rc,
/*Back to log2 quantizer in Q57.*/
log_quantizer = (log_quantizer - OD_LOG_QUANTIZER_OFFSET_Q45) *
OD_LOG_QUANTIZER_EXP_Q12 +
OD_Q57(OD_COEFF_SHIFT);
OD_Q57(8);
/*Clamp modulated quantizer values.*/
log_quantizer = OD_CLAMPI(od_blog64(lossy_quantizer_min), log_quantizer,
od_blog64(lossy_quantizer_max));
......@@ -924,7 +954,11 @@ int od_enc_rc_select_quantizers_and_lambdas(od_rc_state *rc,
/*Modulate chosen base quantizer to produce target quantizer.*/
log_quantizer = od_blog64(base_quantizer);
/*Get the log2 quantizer in Q57 (normalized for coefficient shift).*/
log_quantizer -= OD_Q57(OD_COEFF_SHIFT);
/*XXXXX: See the above XXXX comment in rate control
initialization; the scaling on the log-quantizer calculation
should be the same as in quantizer scale initialization, but
OD_Q57(8) is possibly the incorrect value. */
log_quantizer -= OD_Q57(8);
/*log_quantizer to Q21.*/
log_quantizer >>= 36;
/*scale log quantizer, result is Q33.*/
......@@ -938,7 +972,7 @@ int od_enc_rc_select_quantizers_and_lambdas(od_rc_state *rc,
/*Back to log2 quantizer in Q57.*/
log_quantizer = (log_quantizer - OD_LOG_QUANTIZER_OFFSET_Q45) *
OD_LOG_QUANTIZER_EXP_Q12 +
OD_Q57(OD_COEFF_SHIFT);
OD_Q57(8);
/*Clamp modulated quantizer values.*/
log_quantizer = OD_CLAMPI(od_blog64(lossy_quantizer_min), log_quantizer,
od_blog64(lossy_quantizer_max));
......@@ -1023,7 +1057,18 @@ int od_enc_rc_select_quantizers_and_lambdas(od_rc_state *rc,
}
*bottom_idx = lossy_quantizer_min;
*top_idx = lossy_quantizer_max;
rc->target_quantizer = av1_qindex_from_ac(
/*XXXXXX: the store back to rc->target_quantizer just seems
wrong. target_quantizer is used as an actual linear quantizer
(like base_quantizer, I think it should be scaled the same way as
a Q0 quantizer for 8-bit inputs). But av1_qindex_from_ac*()
returns a quantizer index, which is completely incomparable.
Passing rc->target_quantizer directly to av1_qindex_from_ac_Q3 is
also probably incorrect. If we move to storing a value scaled the
same way as a Q0 quantizer for 8-bit inputs, then it should just
be rc->target_quantizer << (TX_COEFF_DEPTH - 8) for DAALA_TX, and
something depending on the bit depth for !DAALA_TX. */
rc->target_quantizer = av1_qindex_from_ac_Q3(
OD_CLAMPI(lossy_quantizer_min, rc->target_quantizer, lossy_quantizer_max),
rc->bit_depth);
return rc->target_quantizer;
......
......@@ -338,7 +338,7 @@ static const int rd_frame_type_factor[FRAME_UPDATE_TYPES] = {
};
int av1_compute_rd_mult(const AV1_COMP *cpi, int qindex) {
const int64_t q = av1_dc_quant(qindex, 0, cpi->common.bit_depth);
const int64_t q = av1_dc_quant_Q3(qindex, 0, cpi->common.bit_depth);
#if CONFIG_HIGHBITDEPTH
int64_t rdmult = 0;
switch (cpi->common.bit_depth) {
......@@ -368,16 +368,16 @@ static int compute_rd_thresh_factor(int qindex, aom_bit_depth_t bit_depth) {
double q;
#if CONFIG_HIGHBITDEPTH
switch (bit_depth) {
case AOM_BITS_8: q = av1_dc_quant(qindex, 0, AOM_BITS_8) / 4.0; break;
case AOM_BITS_10: q = av1_dc_quant(qindex, 0, AOM_BITS_10) / 16.0; break;
case AOM_BITS_12: q = av1_dc_quant(qindex, 0, AOM_BITS_12) / 64.0; break;
case AOM_BITS_8: q = av1_dc_quant_Q3(qindex, 0, AOM_BITS_8) / 4.0; break;
case AOM_BITS_10: q = av1_dc_quant_Q3(qindex, 0, AOM_BITS_10) / 16.0; break;
case AOM_BITS_12: q = av1_dc_quant_Q3(qindex, 0, AOM_BITS_12) / 64.0; break;
default:
assert(0 && "bit_depth should be AOM_BITS_8, AOM_BITS_10 or AOM_BITS_12");
return -1;
}
#else
(void)bit_depth;
q = av1_dc_quant(qindex, 0, AOM_BITS_8) / 4.0;
q = av1_dc_quant_Q3(qindex, 0, AOM_BITS_8) / 4.0;
#endif // CONFIG_HIGHBITDEPTH
// TODO(debargha): Adjust the function below.
return AOMMAX((int)(pow(q, RD_THRESH_POW) * 5.12), 8);
......@@ -1295,7 +1295,7 @@ void av1_update_rd_thresh_fact(const AV1_COMMON *const cm,
int av1_get_intra_cost_penalty(int qindex, int qdelta,
aom_bit_depth_t bit_depth) {
const int q = av1_dc_quant(qindex, qdelta, bit_depth);
const int q = av1_dc_quant_Q3(qindex, qdelta, bit_depth);
#if CONFIG_HIGHBITDEPTH
switch (bit_depth) {
case AOM_BITS_8: return 20 * q;
......
......@@ -4674,8 +4674,9 @@ static int predict_skip_flag(const MACROBLOCK *x, BLOCK_SIZE bsize) {
av1_fwd_txfm(p->src_diff, DCT_coefs, bw, &param);
#endif // CONFIG_TXMG
uint32_t dc = (uint32_t)av1_dc_quant(x->qindex, 0, xd->bd);
uint32_t ac = (uint32_t)av1_ac_quant(x->qindex, 0, xd->bd);
// Operating on TX domain, not pixels; we want the QTX quantizers
uint32_t dc = (uint32_t)av1_dc_quant_QTX(x->qindex, 0, xd->bd);
uint32_t ac = (uint32_t)av1_ac_quant_QTX(x->qindex, 0, xd->bd);
uint32_t max_quantized_coef = (100 * (uint32_t)abs(DCT_coefs[0])) / dc;
for (int i = 1; i < bw * bh; i++) {
uint32_t cur_quantized_coef = (100 * (uint32_t)abs(DCT_coefs[i])) / ac;
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment