Commit 6b56e99c authored by Sarah Parker's avatar Sarah Parker

Remove bands from new-quant profiles

Rather than having a set of parameters for each of the 6
COEF_BANDS, we have 1 for DC and 1 for AC coefficients.
No change in performance since all of the bands had the
same parameters.

Change-Id: I3665e7c1b21f117be776f371d87d64b097715735
parent 8a3d80eb
......@@ -157,18 +157,18 @@ if (aom_config("CONFIG_DAALA_TX") ne "yes") {
}
if (aom_config("CONFIG_NEW_QUANT") eq "yes") {
add_proto qw/void quantize_nuq/, "const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, const int16_t *dequant_ptr, const cuml_bins_type_nuq *cuml_bins_ptr, const dequant_val_type_nuq *dequant_val, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, uint16_t *eob_ptr, const int16_t *scan, const uint8_t *band";
add_proto qw/void quantize_nuq/, "const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, const int16_t *dequant_ptr, const cuml_bins_type_nuq *cuml_bins_ptr, const dequant_val_type_nuq *dequant_val, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, uint16_t *eob_ptr, const int16_t *scan";
add_proto qw/void quantize_fp_nuq/, "const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *quant_ptr, const int16_t *dequant_ptr, const cuml_bins_type_nuq *cuml_bins_ptr, const dequant_val_type_nuq *dequant_val, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, uint16_t *eob_ptr, const int16_t *scan, const uint8_t *band";
add_proto qw/void quantize_fp_nuq/, "const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *quant_ptr, const int16_t *dequant_ptr, const cuml_bins_type_nuq *cuml_bins_ptr, const dequant_val_type_nuq *dequant_val, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, uint16_t *eob_ptr, const int16_t *scan";
add_proto qw/void quantize_32x32_nuq/, "const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, const int16_t *dequant_ptr, const cuml_bins_type_nuq *cuml_bins_ptr, const dequant_val_type_nuq *dequant_val, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, uint16_t *eob_ptr, const int16_t *scan, const uint8_t *band";
add_proto qw/void quantize_32x32_nuq/, "const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, const int16_t *dequant_ptr, const cuml_bins_type_nuq *cuml_bins_ptr, const dequant_val_type_nuq *dequant_val, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, uint16_t *eob_ptr, const int16_t *scan";
add_proto qw/void quantize_32x32_fp_nuq/, "const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *quant_ptr, const int16_t *dequant_ptr, const cuml_bins_type_nuq *cuml_bins_ptr, const dequant_val_type_nuq *dequant_val, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, uint16_t *eob_ptr, const int16_t *scan, const uint8_t *band";
add_proto qw/void quantize_32x32_fp_nuq/, "const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *quant_ptr, const int16_t *dequant_ptr, const cuml_bins_type_nuq *cuml_bins_ptr, const dequant_val_type_nuq *dequant_val, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, uint16_t *eob_ptr, const int16_t *scan";
if (aom_config("CONFIG_TX64X64") eq "yes") {
add_proto qw/void quantize_64x64_nuq/, "const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, const int16_t *dequant_ptr, const cuml_bins_type_nuq *cuml_bins_ptr, const dequant_val_type_nuq *dequant_val, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, uint16_t *eob_ptr, const int16_t *scan, const uint8_t *band";
add_proto qw/void quantize_64x64_nuq/, "const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, const int16_t *dequant_ptr, const cuml_bins_type_nuq *cuml_bins_ptr, const dequant_val_type_nuq *dequant_val, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, uint16_t *eob_ptr, const int16_t *scan";
add_proto qw/void quantize_64x64_fp_nuq/, "const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *quant_ptr, const int16_t *dequant_ptr, const cuml_bins_type_nuq *cuml_bins_ptr, const dequant_val_type_nuq *dequant_val, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, uint16_t *eob_ptr, const int16_t *scan, const uint8_t *band";
add_proto qw/void quantize_64x64_fp_nuq/, "const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *quant_ptr, const int16_t *dequant_ptr, const cuml_bins_type_nuq *cuml_bins_ptr, const dequant_val_type_nuq *dequant_val, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, uint16_t *eob_ptr, const int16_t *scan";
}
}
......@@ -445,18 +445,18 @@ if (aom_config("CONFIG_AV1_ENCODER") eq "yes") {
# ENCODEMB INVOKE
if (aom_config("CONFIG_NEW_QUANT") eq "yes") {
add_proto qw/void highbd_quantize_nuq/, "const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, const int16_t *dequant_ptr, const cuml_bins_type_nuq *cuml_bins_ptr, const dequant_val_type_nuq *dequant_val, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, uint16_t *eob_ptr, const int16_t *scan, const uint8_t *band";
add_proto qw/void highbd_quantize_nuq/, "const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, const int16_t *dequant_ptr, const cuml_bins_type_nuq *cuml_bins_ptr, const dequant_val_type_nuq *dequant_val, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, uint16_t *eob_ptr, const int16_t *scan";
add_proto qw/void highbd_quantize_fp_nuq/, "const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *quant_ptr, const int16_t *dequant_ptr, const cuml_bins_type_nuq *cuml_bins_ptr, const dequant_val_type_nuq *dequant_val, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, uint16_t *eob_ptr, const int16_t *scan, const uint8_t *band";
add_proto qw/void highbd_quantize_fp_nuq/, "const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *quant_ptr, const int16_t *dequant_ptr, const cuml_bins_type_nuq *cuml_bins_ptr, const dequant_val_type_nuq *dequant_val, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, uint16_t *eob_ptr, const int16_t *scan";
add_proto qw/void highbd_quantize_32x32_nuq/, "const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, const int16_t *dequant_ptr, const cuml_bins_type_nuq *cuml_bins_ptr, const dequant_val_type_nuq *dequant_val, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, uint16_t *eob_ptr, const int16_t *scan, const uint8_t *band";
add_proto qw/void highbd_quantize_32x32_nuq/, "const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, const int16_t *dequant_ptr, const cuml_bins_type_nuq *cuml_bins_ptr, const dequant_val_type_nuq *dequant_val, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, uint16_t *eob_ptr, const int16_t *scan";
add_proto qw/void highbd_quantize_32x32_fp_nuq/, "const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *quant_ptr, const int16_t *dequant_ptr, const cuml_bins_type_nuq *cuml_bins_ptr, const dequant_val_type_nuq *dequant_val, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, uint16_t *eob_ptr, const int16_t *scan, const uint8_t *band";
add_proto qw/void highbd_quantize_32x32_fp_nuq/, "const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *quant_ptr, const int16_t *dequant_ptr, const cuml_bins_type_nuq *cuml_bins_ptr, const dequant_val_type_nuq *dequant_val, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, uint16_t *eob_ptr, const int16_t *scan";
if (aom_config("CONFIG_TX64X64") eq "yes") {
add_proto qw/void highbd_quantize_64x64_nuq/, "const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, const int16_t *dequant_ptr, const cuml_bins_type_nuq *cuml_bins_ptr, const dequant_val_type_nuq *dequant_val, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, uint16_t *eob_ptr, const int16_t *scan, const uint8_t *band";
add_proto qw/void highbd_quantize_64x64_nuq/, "const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, const int16_t *dequant_ptr, const cuml_bins_type_nuq *cuml_bins_ptr, const dequant_val_type_nuq *dequant_val, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, uint16_t *eob_ptr, const int16_t *scan";
add_proto qw/void highbd_quantize_64x64_fp_nuq/, "const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *quant_ptr, const int16_t *dequant_ptr, const cuml_bins_type_nuq *cuml_bins_ptr, const dequant_val_type_nuq *dequant_val, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, uint16_t *eob_ptr, const int16_t *scan, const uint8_t *band";
add_proto qw/void highbd_quantize_64x64_fp_nuq/, "const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *quant_ptr, const int16_t *dequant_ptr, const cuml_bins_type_nuq *cuml_bins_ptr, const dequant_val_type_nuq *dequant_val, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, uint16_t *eob_ptr, const int16_t *scan";
}
}
......
......@@ -498,8 +498,7 @@ typedef struct macroblockd_plane {
// shift/scale as TX.
int16_t seg_dequant_QTX[MAX_SEGMENTS][2];
#if CONFIG_NEW_QUANT
dequant_val_type_nuq seg_dequant_nuq_QTX[MAX_SEGMENTS][QUANT_PROFILES]
[COEF_BANDS];
dequant_val_type_nuq seg_dequant_nuq_QTX[MAX_SEGMENTS][QUANT_PROFILES][2];
#endif
uint8_t *color_index_map;
......
......@@ -352,12 +352,9 @@ typedef struct AV1Common {
int max_qmlevel;
#endif
#if CONFIG_NEW_QUANT
dequant_val_type_nuq y_dequant_nuq_QTX[MAX_SEGMENTS][QUANT_PROFILES]
[COEF_BANDS];
dequant_val_type_nuq u_dequant_nuq_QTX[MAX_SEGMENTS][QUANT_PROFILES]
[COEF_BANDS];
dequant_val_type_nuq v_dequant_nuq_QTX[MAX_SEGMENTS][QUANT_PROFILES]
[COEF_BANDS];
dequant_val_type_nuq y_dequant_nuq_QTX[MAX_SEGMENTS][QUANT_PROFILES][2];
dequant_val_type_nuq u_dequant_nuq_QTX[MAX_SEGMENTS][QUANT_PROFILES][2];
dequant_val_type_nuq v_dequant_nuq_QTX[MAX_SEGMENTS][QUANT_PROFILES][2];
#endif
/* We allocate a MODE_INFO struct for each macroblock, together with
......
......@@ -31,54 +31,38 @@ typedef struct {
uint8_t doff; // dequantization
} qprofile_type;
static const qprofile_type nuq[QUANT_PROFILES][COEF_BANDS] = {
static const qprofile_type nuq[QUANT_PROFILES][2] = {
{
// lossless
{ { 64, 128, 128 }, 0 }, // dc, band 0
{ { 64, 128, 128 }, 0 }, // band 1
{ { 64, 128, 128 }, 0 }, // band 2
{ { 64, 128, 128 }, 0 }, // band 3
{ { 64, 128, 128 }, 0 }, // band 4
{ { 64, 128, 128 }, 0 }, // band 5
{ { 64, 128, 128 }, 0 }, // dc
{ { 64, 128, 128 }, 0 }, // ac
},
{
{ { 64, 128, 128 }, 4 }, // dc, band 0
{ { 64, 128, 128 }, 4 }, // band 1
{ { 64, 128, 128 }, 4 }, // band 2
{ { 64, 128, 128 }, 4 }, // band 3
{ { 64, 128, 128 }, 4 }, // band 4
{ { 64, 128, 128 }, 4 } // band 5
{ { 64, 128, 128 }, 4 }, // dc
{ { 64, 128, 128 }, 4 }, // ac
},
{
{ { 64, 128, 128 }, 6 }, // dc, band 0
{ { 64, 128, 128 }, 6 }, // band 1
{ { 64, 128, 128 }, 6 }, // band 2
{ { 64, 128, 128 }, 6 }, // band 3
{ { 64, 128, 128 }, 6 }, // band 4
{ { 64, 128, 128 }, 6 } // band 5
{ { 64, 128, 128 }, 6 }, // dc
{ { 64, 128, 128 }, 6 }, // ac
},
{
{ { 64, 128, 128 }, 8 }, // dc, band 0
{ { 64, 128, 128 }, 8 }, // band 1
{ { 64, 128, 128 }, 8 }, // band 2
{ { 64, 128, 128 }, 8 }, // band 3
{ { 64, 128, 128 }, 8 }, // band 4
{ { 64, 128, 128 }, 8 } // band 5
{ { 64, 128, 128 }, 8 }, // dc
{ { 64, 128, 128 }, 8 }, // ac
}
};
static const uint8_t *get_nuq_knots(int band, int q_profile) {
return nuq[q_profile][band].knots;
static const uint8_t *get_nuq_knots(int is_ac_coeff, int q_profile) {
return nuq[q_profile][is_ac_coeff].knots;
}
static INLINE int16_t quant_to_doff_fixed(int band, int q_profile) {
return nuq[q_profile][band].doff;
static INLINE int16_t quant_to_doff_fixed(int is_ac_coeff, int q_profile) {
return nuq[q_profile][is_ac_coeff].doff;
}
// get cumulative bins
static INLINE void get_cuml_bins_nuq(int q, int band, tran_low_t *cuml_bins,
int q_profile) {
const uint8_t *knots = get_nuq_knots(band, q_profile);
static INLINE void get_cuml_bins_nuq(int q, int is_ac_coeff,
tran_low_t *cuml_bins, int q_profile) {
const uint8_t *knots = get_nuq_knots(is_ac_coeff, q_profile);
int16_t cuml_knots[NUQ_KNOTS];
int i;
cuml_knots[0] = knots[0];
......@@ -87,22 +71,22 @@ static INLINE void get_cuml_bins_nuq(int q, int band, tran_low_t *cuml_bins,
cuml_bins[i] = ROUND_POWER_OF_TWO(cuml_knots[i] * q, 7);
}
void av1_get_dequant_val_nuq(int q, int band, tran_low_t *dq,
void av1_get_dequant_val_nuq(int q, int is_ac_coeff, tran_low_t *dq,
tran_low_t *cuml_bins, int q_profile) {
const uint8_t *knots = get_nuq_knots(band, q_profile);
const uint8_t *knots = get_nuq_knots(is_ac_coeff, q_profile);
tran_low_t cuml_bins_[NUQ_KNOTS], *cuml_bins_ptr;
tran_low_t doff;
int i;
cuml_bins_ptr = (cuml_bins ? cuml_bins : cuml_bins_);
get_cuml_bins_nuq(q, band, cuml_bins_ptr, q_profile);
get_cuml_bins_nuq(q, is_ac_coeff, cuml_bins_ptr, q_profile);
dq[0] = 0;
for (i = 1; i < NUQ_KNOTS; ++i) {
doff = quant_to_doff_fixed(band, q_profile);
doff = quant_to_doff_fixed(is_ac_coeff, q_profile);
doff = ROUND_POWER_OF_TWO(doff * knots[i], 7);
dq[i] =
cuml_bins_ptr[i - 1] + ROUND_POWER_OF_TWO((knots[i] - doff * 2) * q, 8);
}
doff = quant_to_doff_fixed(band, q_profile);
doff = quant_to_doff_fixed(is_ac_coeff, q_profile);
dq[NUQ_KNOTS] =
cuml_bins_ptr[NUQ_KNOTS - 1] + ROUND_POWER_OF_TWO((64 - doff) * q, 7);
}
......
......@@ -85,7 +85,7 @@ static inline TX_SIZE get_qm_tx_size(TX_SIZE tx_size) {
typedef tran_low_t dequant_val_type_nuq[NUQ_KNOTS + 1];
typedef tran_low_t cuml_bins_type_nuq[NUQ_KNOTS];
void av1_get_dequant_val_nuq(int q, int band, tran_low_t *dq,
void av1_get_dequant_val_nuq(int q, int is_ac_coeff, tran_low_t *dq,
tran_low_t *cuml_bins, int dq_off_index);
tran_low_t av1_dequant_abscoeff_nuq(int v, int q, const tran_low_t *dq);
tran_low_t av1_dequant_coeff_nuq(int v, int q, const tran_low_t *dq);
......
......@@ -1303,7 +1303,8 @@ static void setup_segmentation_dequant(AV1_COMMON *const cm) {
#endif // CONFIG_AOM_QM
#if CONFIG_NEW_QUANT
for (int dq = 0; dq < QUANT_PROFILES; dq++) {
for (int b = 0; b < COEF_BANDS; ++b) {
// DC and AC coefs
for (int b = 0; b < 2; ++b) {
av1_get_dequant_val_nuq(cm->y_dequant_QTX[i][b != 0], b,
cm->y_dequant_nuq_QTX[i][dq][b], NULL, dq);
av1_get_dequant_val_nuq(cm->u_dequant_QTX[i][b != 0], b,
......
......@@ -145,7 +145,7 @@ static int decode_coefs(MACROBLOCKD *xd, PLANE_TYPE type, tran_low_t *dqcoeff,
int first_pos = (c == 0);
#if CONFIG_NEW_QUANT
dqv_val = &dq_val[band][0];
dqv_val = &dq_val[band != 0][0];
#endif // CONFIG_NEW_QUANT
comb_token = last_pos ? 2 * av1_read_record_bit(xd->counts, r, ACCT_STR) + 2
......@@ -186,7 +186,7 @@ static int decode_coefs(MACROBLOCKD *xd, PLANE_TYPE type, tran_low_t *dqcoeff,
token += av1_read_record_symbol(xd->counts, r, coef_tail_cdfs[band][ctx],
TAIL_TOKENS, ACCT_STR);
#if CONFIG_NEW_QUANT
dqv_val = &dq_val[band][0];
dqv_val = &dq_val[band != 0][0];
#endif // CONFIG_NEW_QUANT
*max_scan_line = AOMMAX(*max_scan_line, scan[c]);
......
This diff is collapsed.
......@@ -45,15 +45,12 @@ typedef void (*AV1_QUANT_FACADE)(const tran_low_t *coeff_ptr, intptr_t n_coeffs,
// All of its fields use the same coefficient shift/scaling at TX.
typedef struct {
#if CONFIG_NEW_QUANT
DECLARE_ALIGNED(
16, tran_low_t,
y_cuml_bins_nuq[QUANT_PROFILES][QINDEX_RANGE][COEF_BANDS][NUQ_KNOTS]);
DECLARE_ALIGNED(
16, tran_low_t,
u_cuml_bins_nuq[QUANT_PROFILES][QINDEX_RANGE][COEF_BANDS][NUQ_KNOTS]);
DECLARE_ALIGNED(
16, tran_low_t,
v_cuml_bins_nuq[QUANT_PROFILES][QINDEX_RANGE][COEF_BANDS][NUQ_KNOTS]);
DECLARE_ALIGNED(16, tran_low_t,
y_cuml_bins_nuq[QUANT_PROFILES][QINDEX_RANGE][2][NUQ_KNOTS]);
DECLARE_ALIGNED(16, tran_low_t,
u_cuml_bins_nuq[QUANT_PROFILES][QINDEX_RANGE][2][NUQ_KNOTS]);
DECLARE_ALIGNED(16, tran_low_t,
v_cuml_bins_nuq[QUANT_PROFILES][QINDEX_RANGE][2][NUQ_KNOTS]);
#endif // CONFIG_NEW_QUANT
// 0: dc 1: ac 2-8: ac repeated to SIMD width
DECLARE_ALIGNED(16, int16_t, y_quant[QINDEX_RANGE][8]);
......@@ -95,15 +92,12 @@ typedef struct {
DECLARE_ALIGNED(16, int16_t, u_dequant_Q3[QINDEX_RANGE][8]); // 8: SIMD width
DECLARE_ALIGNED(16, int16_t, v_dequant_Q3[QINDEX_RANGE][8]); // 8: SIMD width
#if CONFIG_NEW_QUANT
DECLARE_ALIGNED(
16, dequant_val_type_nuq,
y_dequant_val_nuq_QTX[QUANT_PROFILES][QINDEX_RANGE][COEF_BANDS]);
DECLARE_ALIGNED(
16, dequant_val_type_nuq,
u_dequant_val_nuq_QTX[QUANT_PROFILES][QINDEX_RANGE][COEF_BANDS]);
DECLARE_ALIGNED(
16, dequant_val_type_nuq,
v_dequant_val_nuq_QTX[QUANT_PROFILES][QINDEX_RANGE][COEF_BANDS]);
DECLARE_ALIGNED(16, dequant_val_type_nuq,
y_dequant_val_nuq_QTX[QUANT_PROFILES][QINDEX_RANGE][2]);
DECLARE_ALIGNED(16, dequant_val_type_nuq,
u_dequant_val_nuq_QTX[QUANT_PROFILES][QINDEX_RANGE][2]);
DECLARE_ALIGNED(16, dequant_val_type_nuq,
v_dequant_val_nuq_QTX[QUANT_PROFILES][QINDEX_RANGE][2]);
#endif // CONFIG_NEW_QUANT
} Dequants;
......
......@@ -259,15 +259,14 @@ static int optimize_b_greedy(const AV1_COMMON *cm, MACROBLOCK *mb, int plane,
if (x_a != 0) {
#if CONFIG_DAALA_TX
#if CONFIG_NEW_QUANT
dx = av1_dequant_coeff_nuq(x_a, dqv, dequant_val[band_translate[i]]) -
coeff[rc];
dx = av1_dequant_coeff_nuq(x_a, dqv, dequant_val[rc != 0]) - coeff[rc];
#else // CONFIG_NEW_QUANT
dx -= (dqv + sz) ^ sz;
#endif // CONFIG_NEW_QUANT
d2_a = ((int64_t)dx * dx + depth_round) >> depth_shift;
#else // CONFIG_DAALA_TX
#if CONFIG_NEW_QUANT
dx = av1_dequant_coeff_nuq(x_a, dqv, dequant_val[band_translate[i]]) -
dx = av1_dequant_coeff_nuq(x_a, dqv, dequant_val[rc != 0]) -
(coeff[rc] * (1 << shift));
dx >>= xd->bd - 8;
#else // CONFIG_NEW_QUANT
......@@ -349,16 +348,14 @@ static int optimize_b_greedy(const AV1_COMMON *cm, MACROBLOCK *mb, int plane,
if (x_a != 0) {
#if CONFIG_DAALA_TX
#if CONFIG_NEW_QUANT
dqc_a = av1_dequant_abscoeff_nuq(abs(x_a), dqv,
dequant_val[band_translate[i]]);
dqc_a = av1_dequant_abscoeff_nuq(abs(x_a), dqv, dequant_val[rc != 0]);
if (sz) dqc_a = -dqc_a;
#else
dqc_a = x_a * dqv;
#endif // CONFIG_NEW_QUANT
#else // CONFIG_DAALA_TX
#if CONFIG_NEW_QUANT
dqc_a = av1_dequant_abscoeff_nuq(abs(x_a), dqv,
dequant_val[band_translate[i]]);
dqc_a = av1_dequant_abscoeff_nuq(abs(x_a), dqv, dequant_val[rc != 0]);
dqc_a = shift ? ROUND_POWER_OF_TWO(dqc_a, shift) : dqc_a;
if (sz) dqc_a = -dqc_a;
#else
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment