Commit 125c0fca authored by Monty Montgomery's avatar Monty Montgomery Committed by Christopher Montgomery

Separate quantizers used for quantization from RDO

Generalize quantizer setup so that quantization and TX can use
different coefficient shifts/scalings without inpacting RDO lambda
generation.

This patch is documentaiton + a minor refactor setting up later
work; it causes no functional change.

monty-daalaTX-fulltest-Daalabaseline-o1f@2017-11-07T00:01:20.779Z ->
 monty-daalaTX-fulltest-DaalaRDO-o1f@2017-11-07T00:02:31.347Z

PSNR | PSNR Cb | PSNR Cr | PSNR HVS | SSIM | MS SSIM | CIEDE 2000
 N/A |  0.0000 |  0.0000 |      N/A |  N/A |     N/A |        N/A
(note-- the numbers above were collected using --cpu-used=3, which
 appears to be newly broken.)

monty-daalaTX-fulltest-Daalabaseline-o1f4@2017-11-07T05:59:16.553Z ->
 monty-daalaTX-fulltest-DaalaRDO-o1f4@2017-11-07T05:59:50.180Z

  PSNR | PSNR Cb | PSNR Cr | PSNR HVS |   SSIM | MS SSIM | CIEDE 2000
0.0000 |  0.0000 |  0.0000 |   0.0000 | 0.0000 |  0.0000 |     0.0000

monty-daalaTX-fulltest-Daalabaseline-s1@2017-11-07T00:01:46.582Z ->
 monty-daalaTX-fulltest-DaalaRDO-s1@2017-11-07T00:02:56.282Z

  PSNR | PSNR Cb | PSNR Cr | PSNR HVS |   SSIM | MS SSIM | CIEDE 2000
0.0000 |  0.0000 |  0.0000 |   0.0000 | 0.0000 |  0.0000 |     0.0000

Change-Id: Ia5a3c052eacd17184ca1b0fe6d032cfb9afdb77f
parent 848f7bc7
......@@ -521,10 +521,14 @@ typedef struct macroblockd_plane {
struct buf_2d pre[2];
ENTROPY_CONTEXT *above_context;
ENTROPY_CONTEXT *left_context;
int16_t seg_dequant[MAX_SEGMENTS][2];
// The dequantizers below are true dequntizers used only in the
// dequantization process. They have the same coefficient
// shift/scale as TX.
int16_t seg_dequant_QTX[MAX_SEGMENTS][2];
#if CONFIG_NEW_QUANT
dequant_val_type_nuq seg_dequant_nuq[MAX_SEGMENTS][QUANT_PROFILES]
[COEF_BANDS];
dequant_val_type_nuq seg_dequant_nuq_QTX[MAX_SEGMENTS][QUANT_PROFILES]
[COEF_BANDS];
#endif
uint8_t *color_index_map;
......@@ -539,11 +543,12 @@ typedef struct macroblockd_plane {
qm_val_t *seg_iqmatrix[MAX_SEGMENTS][TX_SIZES_ALL];
qm_val_t *seg_qmatrix[MAX_SEGMENTS][TX_SIZES_ALL];
#endif
// encoder
const int16_t *dequant;
#if CONFIG_NEW_QUANT
const dequant_val_type_nuq *dequant_val_nuq[QUANT_PROFILES];
#endif // CONFIG_NEW_QUANT
// the 'dequantizers' below are not literal dequantizer values.
// They're used by encoder RDO to generate ad-hoc lambda values.
// They use a hardwired Q3 coeff shift and do not necessarily match
// the TX scale in use.
const int16_t *dequant_Q3;
#if CONFIG_DIST_8X8
DECLARE_ALIGNED(16, int16_t, pred[MAX_SB_SQUARE]);
......
......@@ -300,8 +300,12 @@ typedef struct AV1Common {
int y_dc_delta_q;
int uv_dc_delta_q;
int uv_ac_delta_q;
int16_t y_dequant[MAX_SEGMENTS][2];
int16_t uv_dequant[MAX_SEGMENTS][2];
// The dequantizers below are true dequntizers used only in the
// dequantization process. They have the same coefficient
// shift/scale as TX.
int16_t y_dequant_QTX[MAX_SEGMENTS][2];
int16_t uv_dequant_QTX[MAX_SEGMENTS][2];
#if CONFIG_AOM_QM
// Global quant matrix tables
......@@ -320,8 +324,10 @@ typedef struct AV1Common {
int max_qmlevel;
#endif
#if CONFIG_NEW_QUANT
dequant_val_type_nuq y_dequant_nuq[MAX_SEGMENTS][QUANT_PROFILES][COEF_BANDS];
dequant_val_type_nuq uv_dequant_nuq[MAX_SEGMENTS][QUANT_PROFILES][COEF_BANDS];
dequant_val_type_nuq y_dequant_nuq_QTX[MAX_SEGMENTS][QUANT_PROFILES]
[COEF_BANDS];
dequant_val_type_nuq uv_dequant_nuq_QTX[MAX_SEGMENTS][QUANT_PROFILES]
[COEF_BANDS];
#endif
/* We allocate a MODE_INFO struct for each macroblock, together with
......@@ -678,24 +684,26 @@ static INLINE void av1_init_macroblockd(AV1_COMMON *cm, MACROBLOCKD *xd,
xd->plane[i].dqcoeff = dqcoeff;
xd->above_context[i] = cm->above_context[i];
if (xd->plane[i].plane_type == PLANE_TYPE_Y) {
memcpy(xd->plane[i].seg_dequant, cm->y_dequant, sizeof(cm->y_dequant));
memcpy(xd->plane[i].seg_dequant_QTX, cm->y_dequant_QTX,
sizeof(cm->y_dequant_QTX));
#if CONFIG_AOM_QM
memcpy(xd->plane[i].seg_iqmatrix, cm->y_iqmatrix, sizeof(cm->y_iqmatrix));
#endif
#if CONFIG_NEW_QUANT
memcpy(xd->plane[i].seg_dequant_nuq, cm->y_dequant_nuq,
sizeof(cm->y_dequant_nuq));
memcpy(xd->plane[i].seg_dequant_nuq_QTX, cm->y_dequant_nuq_QTX,
sizeof(cm->y_dequant_nuq_QTX));
#endif
} else {
memcpy(xd->plane[i].seg_dequant, cm->uv_dequant, sizeof(cm->uv_dequant));
memcpy(xd->plane[i].seg_dequant_QTX, cm->uv_dequant_QTX,
sizeof(cm->uv_dequant_QTX));
#if CONFIG_AOM_QM
memcpy(xd->plane[i].seg_iqmatrix, cm->uv_iqmatrix,
sizeof(cm->uv_iqmatrix));
#endif
#if CONFIG_NEW_QUANT
memcpy(xd->plane[i].seg_dequant_nuq, cm->uv_dequant_nuq,
sizeof(cm->uv_dequant_nuq));
memcpy(xd->plane[i].seg_dequant_nuq_QTX, cm->uv_dequant_nuq_QTX,
sizeof(cm->uv_dequant_nuq_QTX));
#endif
}
}
......
......@@ -457,6 +457,14 @@ static void decode_mbmi_block(AV1Decoder *const pbi, MACROBLOCKD *const xd,
aom_merge_corrupted_flag(&xd->corrupted, reader_corrupted_flag);
}
// Converts a Q3 quantizer lookup from static configuration to the
// actual TX scaling in use
static int dequant_Q3_to_QTX(int q3, int bd) {
// Right now, TX scale in use is still Q3
(void)bd;
return q3;
}
static void decode_token_and_recon_block(AV1Decoder *const pbi,
MACROBLOCKD *const xd, int mi_row,
int mi_col, aom_reader *r,
......@@ -488,11 +496,12 @@ static void decode_token_and_recon_block(AV1Decoder *const pbi,
for (j = 0; j < MAX_MB_PLANE; ++j) {
const int dc_delta_q = j == 0 ? cm->y_dc_delta_q : cm->uv_dc_delta_q;
const int ac_delta_q = j == 0 ? 0 : cm->uv_ac_delta_q;
xd->plane[j].seg_dequant[i][0] =
av1_dc_quant(current_qindex, dc_delta_q, cm->bit_depth);
xd->plane[j].seg_dequant[i][1] =
av1_ac_quant(current_qindex, ac_delta_q, cm->bit_depth);
xd->plane[j].seg_dequant_QTX[i][0] = dequant_Q3_to_QTX(
av1_dc_quant(current_qindex, dc_delta_q, cm->bit_depth),
cm->bit_depth);
xd->plane[j].seg_dequant_QTX[i][1] = dequant_Q3_to_QTX(
av1_ac_quant(current_qindex, ac_delta_q, cm->bit_depth),
cm->bit_depth);
}
}
}
......@@ -1359,12 +1368,14 @@ static void setup_segmentation_dequant(AV1_COMMON *const cm) {
const int max_segments = cm->seg.enabled ? MAX_SEGMENTS : 1;
for (int i = 0; i < max_segments; ++i) {
const int qindex = av1_get_qindex(&cm->seg, i, cm->base_qindex);
cm->y_dequant[i][0] = av1_dc_quant(qindex, cm->y_dc_delta_q, cm->bit_depth);
cm->y_dequant[i][1] = av1_ac_quant(qindex, 0, cm->bit_depth);
cm->uv_dequant[i][0] =
av1_dc_quant(qindex, cm->uv_dc_delta_q, cm->bit_depth);
cm->uv_dequant[i][1] =
av1_ac_quant(qindex, cm->uv_ac_delta_q, cm->bit_depth);
cm->y_dequant_QTX[i][0] = dequant_Q3_to_QTX(
av1_dc_quant(qindex, cm->y_dc_delta_q, cm->bit_depth), cm->bit_depth);
cm->y_dequant_QTX[i][1] = dequant_Q3_to_QTX(
av1_ac_quant(qindex, 0, cm->bit_depth), cm->bit_depth);
cm->uv_dequant_QTX[i][0] = dequant_Q3_to_QTX(
av1_dc_quant(qindex, cm->uv_dc_delta_q, cm->bit_depth), cm->bit_depth);
cm->uv_dequant_QTX[i][1] = dequant_Q3_to_QTX(
av1_ac_quant(qindex, cm->uv_ac_delta_q, cm->bit_depth), cm->bit_depth);
#if CONFIG_AOM_QM
const int lossless = qindex == 0 && cm->y_dc_delta_q == 0 &&
cm->uv_dc_delta_q == 0 && cm->uv_ac_delta_q == 0;
......@@ -1381,10 +1392,10 @@ static void setup_segmentation_dequant(AV1_COMMON *const cm) {
#if CONFIG_NEW_QUANT
for (int dq = 0; dq < QUANT_PROFILES; dq++) {
for (int b = 0; b < COEF_BANDS; ++b) {
av1_get_dequant_val_nuq(cm->y_dequant[i][b != 0], b,
cm->y_dequant_nuq[i][dq][b], NULL, dq);
av1_get_dequant_val_nuq(cm->uv_dequant[i][b != 0], b,
cm->uv_dequant_nuq[i][dq][b], NULL, dq);
av1_get_dequant_val_nuq(cm->y_dequant_QTX[i][b != 0], b,
cm->y_dequant_nuq_QTX[i][dq][b], NULL, dq);
av1_get_dequant_val_nuq(cm->uv_dequant_QTX[i][b != 0], b,
cm->uv_dequant_nuq_QTX[i][dq][b], NULL, dq);
}
}
#endif // CONFIG_NEW_QUANT
......
......@@ -65,7 +65,8 @@ uint8_t av1_read_coeffs_txb(const AV1_COMMON *const cm, MACROBLOCKD *xd,
const int seg_eob = tx_size_2d[tx_size];
int c = 0;
int update_eob = -1;
const int16_t *const dequant = xd->plane[plane].seg_dequant[mbmi->segment_id];
const int16_t *const dequant =
xd->plane[plane].seg_dequant_QTX[mbmi->segment_id];
const int shift = av1_get_tx_scale(tx_size);
const int bwl = b_width_log2_lookup[txsize_to_bsize[tx_size]] + 2;
const int height = tx_size_high[tx_size];
......
......@@ -356,7 +356,7 @@ int av1_decode_block_tokens(AV1_COMMON *cm, MACROBLOCKD *const xd, int plane,
TX_TYPE tx_type, int16_t *max_scan_line,
aom_reader *r, int seg_id) {
struct macroblockd_plane *const pd = &xd->plane[plane];
const int16_t *const dequant = pd->seg_dequant[seg_id];
const int16_t *const dequant = pd->seg_dequant_QTX[seg_id];
const int ctx =
get_entropy_context(tx_size, pd->above_context + x, pd->left_context + y);
#if CONFIG_NEW_QUANT
......@@ -372,7 +372,7 @@ int av1_decode_block_tokens(AV1_COMMON *cm, MACROBLOCKD *const xd, int plane,
const int eob =
decode_coefs(xd, pd->plane_type, pd->dqcoeff, tx_size, tx_type, dequant,
#if CONFIG_NEW_QUANT
pd->seg_dequant_nuq[seg_id][dq],
pd->seg_dequant_nuq_QTX[seg_id][dq],
#else
#if CONFIG_AOM_QM
pd->seg_iqmatrix[seg_id],
......
......@@ -61,8 +61,8 @@ int ifd_inspect(insp_frame_data *fd, void *decoder) {
int i, j;
for (i = 0; i < MAX_SEGMENTS; i++) {
for (j = 0; j < 2; j++) {
fd->y_dequant[i][j] = cm->y_dequant[i][j];
fd->uv_dequant[i][j] = cm->uv_dequant[i][j];
fd->y_dequant[i][j] = cm->y_dequant_QTX[i][j];
fd->uv_dequant[i][j] = cm->uv_dequant_QTX[i][j];
}
}
for (j = 0; j < cm->mi_rows; j++) {
......
......@@ -569,7 +569,6 @@ void av1_quantize_fp_64x64_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs,
void av1_quantize_fp_facade(const tran_low_t *coeff_ptr, intptr_t n_coeffs,
const MACROBLOCK_PLANE *p, tran_low_t *qcoeff_ptr,
const MACROBLOCKD_PLANE *pd,
tran_low_t *dqcoeff_ptr, uint16_t *eob_ptr,
const SCAN_ORDER *sc, const QUANT_PARAM *qparam) {
// obsolete skip_block
......@@ -578,10 +577,11 @@ void av1_quantize_fp_facade(const tran_low_t *coeff_ptr, intptr_t n_coeffs,
const qm_val_t *qm_ptr = qparam->qmatrix;
const qm_val_t *iqm_ptr = qparam->iqmatrix;
if (qm_ptr != NULL && iqm_ptr != NULL) {
quantize_fp_helper_c(coeff_ptr, n_coeffs, skip_block, p->zbin, p->round_fp,
p->quant_fp, p->quant_shift, qcoeff_ptr, dqcoeff_ptr,
pd->dequant, eob_ptr, sc->scan, sc->iscan, qm_ptr,
iqm_ptr, qparam->log_scale);
quantize_fp_helper_c(coeff_ptr, n_coeffs, skip_block, p->zbin_QTX,
p->round_fp_QTX, p->quant_fp_QTX, p->quant_shift_QTX,
qcoeff_ptr, dqcoeff_ptr, p->dequant_QTX, eob_ptr,
sc->scan, sc->iscan, qm_ptr, iqm_ptr,
qparam->log_scale);
} else {
#endif
switch (qparam->log_scale) {
......@@ -589,28 +589,30 @@ void av1_quantize_fp_facade(const tran_low_t *coeff_ptr, intptr_t n_coeffs,
if (n_coeffs < 16) {
// TODO(jingning): Need SIMD implementation for smaller block size
// quantization.
quantize_fp_helper_c(
coeff_ptr, n_coeffs, skip_block, p->zbin, p->round_fp,
p->quant_fp, p->quant_shift, qcoeff_ptr, dqcoeff_ptr, pd->dequant,
eob_ptr, sc->scan, sc->iscan, NULL, NULL, qparam->log_scale);
quantize_fp_helper_c(coeff_ptr, n_coeffs, skip_block, p->zbin_QTX,
p->round_fp_QTX, p->quant_fp_QTX,
p->quant_shift_QTX, qcoeff_ptr, dqcoeff_ptr,
p->dequant_QTX, eob_ptr, sc->scan, sc->iscan,
NULL, NULL, qparam->log_scale);
} else {
av1_quantize_fp(coeff_ptr, n_coeffs, skip_block, p->zbin, p->round_fp,
p->quant_fp, p->quant_shift, qcoeff_ptr, dqcoeff_ptr,
pd->dequant, eob_ptr, sc->scan, sc->iscan);
av1_quantize_fp(coeff_ptr, n_coeffs, skip_block, p->zbin_QTX,
p->round_fp_QTX, p->quant_fp_QTX, p->quant_shift_QTX,
qcoeff_ptr, dqcoeff_ptr, p->dequant_QTX, eob_ptr,
sc->scan, sc->iscan);
}
break;
case 1:
av1_quantize_fp_32x32(coeff_ptr, n_coeffs, skip_block, p->zbin,
p->round_fp, p->quant_fp, p->quant_shift,
qcoeff_ptr, dqcoeff_ptr, pd->dequant, eob_ptr,
sc->scan, sc->iscan);
av1_quantize_fp_32x32(coeff_ptr, n_coeffs, skip_block, p->zbin_QTX,
p->round_fp_QTX, p->quant_fp_QTX,
p->quant_shift_QTX, qcoeff_ptr, dqcoeff_ptr,
p->dequant_QTX, eob_ptr, sc->scan, sc->iscan);
break;
#if CONFIG_TX64X64
case 2:
av1_quantize_fp_64x64(coeff_ptr, n_coeffs, skip_block, p->zbin,
p->round_fp, p->quant_fp, p->quant_shift,
qcoeff_ptr, dqcoeff_ptr, pd->dequant, eob_ptr,
sc->scan, sc->iscan);
av1_quantize_fp_64x64(coeff_ptr, n_coeffs, skip_block, p->zbin_QTX,
p->round_fp_QTX, p->quant_fp_QTX,
p->quant_shift_QTX, qcoeff_ptr, dqcoeff_ptr,
p->dequant_QTX, eob_ptr, sc->scan, sc->iscan);
break;
#endif // CONFIG_TX64X64
default: assert(0);
......@@ -622,38 +624,41 @@ void av1_quantize_fp_facade(const tran_low_t *coeff_ptr, intptr_t n_coeffs,
void av1_quantize_b_facade(const tran_low_t *coeff_ptr, intptr_t n_coeffs,
const MACROBLOCK_PLANE *p, tran_low_t *qcoeff_ptr,
const MACROBLOCKD_PLANE *pd, tran_low_t *dqcoeff_ptr,
uint16_t *eob_ptr, const SCAN_ORDER *sc,
const QUANT_PARAM *qparam) {
tran_low_t *dqcoeff_ptr, uint16_t *eob_ptr,
const SCAN_ORDER *sc, const QUANT_PARAM *qparam) {
// obsolete skip_block
const int skip_block = 0;
#if CONFIG_AOM_QM
const qm_val_t *qm_ptr = qparam->qmatrix;
const qm_val_t *iqm_ptr = qparam->iqmatrix;
if (qm_ptr != NULL && iqm_ptr != NULL) {
quantize_b_helper_c(coeff_ptr, n_coeffs, skip_block, p->zbin, p->round,
p->quant, p->quant_shift, qcoeff_ptr, dqcoeff_ptr,
pd->dequant, eob_ptr, sc->scan, sc->iscan, qm_ptr,
iqm_ptr, qparam->log_scale);
quantize_b_helper_c(coeff_ptr, n_coeffs, skip_block, p->zbin_QTX,
p->round_QTX, p->quant_QTX, p->quant_shift_QTX,
qcoeff_ptr, dqcoeff_ptr, p->dequant_QTX, eob_ptr,
sc->scan, sc->iscan, qm_ptr, iqm_ptr,
qparam->log_scale);
} else {
#endif // CONFIG_AOM_QM
switch (qparam->log_scale) {
case 0:
aom_quantize_b(coeff_ptr, n_coeffs, skip_block, p->zbin, p->round,
p->quant, p->quant_shift, qcoeff_ptr, dqcoeff_ptr,
pd->dequant, eob_ptr, sc->scan, sc->iscan);
aom_quantize_b(coeff_ptr, n_coeffs, skip_block, p->zbin_QTX,
p->round_QTX, p->quant_QTX, p->quant_shift_QTX,
qcoeff_ptr, dqcoeff_ptr, p->dequant_QTX, eob_ptr,
sc->scan, sc->iscan);
break;
case 1:
aom_quantize_b_32x32(coeff_ptr, n_coeffs, skip_block, p->zbin, p->round,
p->quant, p->quant_shift, qcoeff_ptr, dqcoeff_ptr,
pd->dequant, eob_ptr, sc->scan, sc->iscan);
aom_quantize_b_32x32(coeff_ptr, n_coeffs, skip_block, p->zbin_QTX,
p->round_QTX, p->quant_QTX, p->quant_shift_QTX,
qcoeff_ptr, dqcoeff_ptr, p->dequant_QTX, eob_ptr,
sc->scan, sc->iscan);
break;
#if CONFIG_TX64X64
case 2:
aom_quantize_b_64x64(coeff_ptr, n_coeffs, skip_block, p->zbin, p->round,
p->quant, p->quant_shift, qcoeff_ptr, dqcoeff_ptr,
pd->dequant, eob_ptr, sc->scan, sc->iscan);
aom_quantize_b_64x64(coeff_ptr, n_coeffs, skip_block, p->zbin_QTX,
p->round_QTX, p->quant_QTX, p->quant_shift_QTX,
qcoeff_ptr, dqcoeff_ptr, p->dequant_QTX, eob_ptr,
sc->scan, sc->iscan);
break;
#endif // CONFIG_TX64X64
default: assert(0);
......@@ -697,7 +702,6 @@ static void quantize_dc(const tran_low_t *coeff_ptr, int n_coeffs,
void av1_quantize_dc_facade(const tran_low_t *coeff_ptr, intptr_t n_coeffs,
const MACROBLOCK_PLANE *p, tran_low_t *qcoeff_ptr,
const MACROBLOCKD_PLANE *pd,
tran_low_t *dqcoeff_ptr, uint16_t *eob_ptr,
const SCAN_ORDER *sc, const QUANT_PARAM *qparam) {
// obsolete skip_block
......@@ -711,18 +715,16 @@ void av1_quantize_dc_facade(const tran_low_t *coeff_ptr, intptr_t n_coeffs,
const qm_val_t *qm_ptr = NULL;
const qm_val_t *iqm_ptr = NULL;
#endif
quantize_dc(coeff_ptr, (int)n_coeffs, skip_block, p->round, p->quant_fp[0],
qcoeff_ptr, dqcoeff_ptr, pd->dequant[0], eob_ptr, qm_ptr, iqm_ptr,
qparam->log_scale);
quantize_dc(coeff_ptr, (int)n_coeffs, skip_block, p->round_QTX,
p->quant_fp_QTX[0], qcoeff_ptr, dqcoeff_ptr, p->dequant_QTX[0],
eob_ptr, qm_ptr, iqm_ptr, qparam->log_scale);
}
#if CONFIG_NEW_QUANT
void av1_quantize_b_nuq_facade(const tran_low_t *coeff_ptr, intptr_t n_coeffs,
const MACROBLOCK_PLANE *p,
tran_low_t *qcoeff_ptr,
const MACROBLOCKD_PLANE *pd,
tran_low_t *dqcoeff_ptr, uint16_t *eob_ptr,
const SCAN_ORDER *sc,
tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr,
uint16_t *eob_ptr, const SCAN_ORDER *sc,
const QUANT_PARAM *qparam) {
// obsolete skip_block
const int skip_block = 0;
......@@ -731,26 +733,26 @@ void av1_quantize_b_nuq_facade(const tran_low_t *coeff_ptr, intptr_t n_coeffs,
switch (qparam->log_scale) {
case 0:
quantize_nuq(coeff_ptr, n_coeffs, skip_block, p->quant, p->quant_shift,
pd->dequant,
quantize_nuq(coeff_ptr, n_coeffs, skip_block, p->quant_QTX,
p->quant_shift_QTX, p->dequant_QTX,
(const cuml_bins_type_nuq *)p->cuml_bins_nuq[dq],
(const dequant_val_type_nuq *)pd->dequant_val_nuq[dq],
(const dequant_val_type_nuq *)p->dequant_val_nuq_QTX[dq],
qcoeff_ptr, dqcoeff_ptr, eob_ptr, sc->scan, band);
break;
case 1:
quantize_32x32_nuq(coeff_ptr, n_coeffs, skip_block, p->quant,
p->quant_shift, pd->dequant,
(const cuml_bins_type_nuq *)p->cuml_bins_nuq[dq],
(const dequant_val_type_nuq *)pd->dequant_val_nuq[dq],
qcoeff_ptr, dqcoeff_ptr, eob_ptr, sc->scan, band);
quantize_32x32_nuq(
coeff_ptr, n_coeffs, skip_block, p->quant_QTX, p->quant_shift_QTX,
p->dequant_QTX, (const cuml_bins_type_nuq *)p->cuml_bins_nuq[dq],
(const dequant_val_type_nuq *)p->dequant_val_nuq_QTX[dq], qcoeff_ptr,
dqcoeff_ptr, eob_ptr, sc->scan, band);
break;
#if CONFIG_TX64X64
case 2:
quantize_64x64_nuq(coeff_ptr, n_coeffs, skip_block, p->quant,
p->quant_shift, pd->dequant,
(const cuml_bins_type_nuq *)p->cuml_bins_nuq[dq],
(const dequant_val_type_nuq *)pd->dequant_val_nuq[dq],
qcoeff_ptr, dqcoeff_ptr, eob_ptr, sc->scan, band);
quantize_64x64_nuq(
coeff_ptr, n_coeffs, skip_block, p->quant_QTX, p->quant_shift_QTX,
p->dequant_QTX, (const cuml_bins_type_nuq *)p->cuml_bins_nuq[dq],
(const dequant_val_type_nuq *)p->dequant_val_nuq_QTX[dq], qcoeff_ptr,
dqcoeff_ptr, eob_ptr, sc->scan, band);
break;
#endif // CONFIG_TX64X64
default: assert(0);
......@@ -759,10 +761,8 @@ void av1_quantize_b_nuq_facade(const tran_low_t *coeff_ptr, intptr_t n_coeffs,
void av1_quantize_fp_nuq_facade(const tran_low_t *coeff_ptr, intptr_t n_coeffs,
const MACROBLOCK_PLANE *p,
tran_low_t *qcoeff_ptr,
const MACROBLOCKD_PLANE *pd,
tran_low_t *dqcoeff_ptr, uint16_t *eob_ptr,
const SCAN_ORDER *sc,
tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr,
uint16_t *eob_ptr, const SCAN_ORDER *sc,
const QUANT_PARAM *qparam) {
// obsolete skip_block
const int skip_block = 0;
......@@ -771,24 +771,25 @@ void av1_quantize_fp_nuq_facade(const tran_low_t *coeff_ptr, intptr_t n_coeffs,
switch (qparam->log_scale) {
case 0:
quantize_fp_nuq(coeff_ptr, n_coeffs, skip_block, p->quant_fp, pd->dequant,
quantize_fp_nuq(coeff_ptr, n_coeffs, skip_block, p->quant_fp_QTX,
p->dequant_QTX,
(const cuml_bins_type_nuq *)p->cuml_bins_nuq[dq],
(const dequant_val_type_nuq *)pd->dequant_val_nuq[dq],
(const dequant_val_type_nuq *)p->dequant_val_nuq_QTX[dq],
qcoeff_ptr, dqcoeff_ptr, eob_ptr, sc->scan, band);
break;
case 1:
quantize_32x32_fp_nuq(
coeff_ptr, n_coeffs, skip_block, p->quant_fp, pd->dequant,
coeff_ptr, n_coeffs, skip_block, p->quant_fp_QTX, p->dequant_QTX,
(const cuml_bins_type_nuq *)p->cuml_bins_nuq[dq],
(const dequant_val_type_nuq *)pd->dequant_val_nuq[dq], qcoeff_ptr,
(const dequant_val_type_nuq *)p->dequant_val_nuq_QTX[dq], qcoeff_ptr,
dqcoeff_ptr, eob_ptr, sc->scan, band);
break;
#if CONFIG_TX64X64
case 2:
quantize_64x64_fp_nuq(
coeff_ptr, n_coeffs, skip_block, p->quant_fp, pd->dequant,
coeff_ptr, n_coeffs, skip_block, p->quant_fp_QTX, p->dequant_QTX,
(const cuml_bins_type_nuq *)p->cuml_bins_nuq[dq],
(const dequant_val_type_nuq *)pd->dequant_val_nuq[dq], qcoeff_ptr,
(const dequant_val_type_nuq *)p->dequant_val_nuq_QTX[dq], qcoeff_ptr,
dqcoeff_ptr, eob_ptr, sc->scan, band);
break;
#endif // CONFIG_TX64X64
......@@ -798,10 +799,8 @@ void av1_quantize_fp_nuq_facade(const tran_low_t *coeff_ptr, intptr_t n_coeffs,
void av1_quantize_dc_nuq_facade(const tran_low_t *coeff_ptr, intptr_t n_coeffs,
const MACROBLOCK_PLANE *p,
tran_low_t *qcoeff_ptr,
const MACROBLOCKD_PLANE *pd,
tran_low_t *dqcoeff_ptr, uint16_t *eob_ptr,
const SCAN_ORDER *sc,
tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr,
uint16_t *eob_ptr, const SCAN_ORDER *sc,
const QUANT_PARAM *qparam) {
// obsolete skip_block
const int skip_block = 0;
......@@ -810,23 +809,23 @@ void av1_quantize_dc_nuq_facade(const tran_low_t *coeff_ptr, intptr_t n_coeffs,
switch (qparam->log_scale) {
case 0:
quantize_dc_fp_nuq(coeff_ptr, n_coeffs, skip_block, p->quant_fp[0],
pd->dequant[0], p->cuml_bins_nuq[dq][0],
pd->dequant_val_nuq[dq][0], qcoeff_ptr, dqcoeff_ptr,
quantize_dc_fp_nuq(coeff_ptr, n_coeffs, skip_block, p->quant_fp_QTX[0],
p->dequant_QTX[0], p->cuml_bins_nuq[dq][0],
p->dequant_val_nuq_QTX[dq][0], qcoeff_ptr, dqcoeff_ptr,
eob_ptr);
break;
case 1:
quantize_dc_32x32_fp_nuq(coeff_ptr, n_coeffs, skip_block, p->quant_fp[0],
pd->dequant[0], p->cuml_bins_nuq[dq][0],
pd->dequant_val_nuq[dq][0], qcoeff_ptr,
dqcoeff_ptr, eob_ptr);
quantize_dc_32x32_fp_nuq(
coeff_ptr, n_coeffs, skip_block, p->quant_fp_QTX[0],
p->dequant_QTX[0], p->cuml_bins_nuq[dq][0],
p->dequant_val_nuq_QTX[dq][0], qcoeff_ptr, dqcoeff_ptr, eob_ptr);
break;
#if CONFIG_TX64X64
case 2:
quantize_dc_64x64_fp_nuq(coeff_ptr, n_coeffs, skip_block, p->quant_fp[0],
pd->dequant[0], p->cuml_bins_nuq[dq][0],
pd->dequant_val_nuq[dq][0], qcoeff_ptr,
dqcoeff_ptr, eob_ptr);
quantize_dc_64x64_fp_nuq(
coeff_ptr, n_coeffs, skip_block, p->quant_fp_QTX[0],
p->dequant_QTX[0], p->cuml_bins_nuq[dq][0],
p->dequant_val_nuq_QTX[dq][0], qcoeff_ptr, dqcoeff_ptr, eob_ptr);
break;
#endif // CONFIG_TX64X64
default: assert(0);
......@@ -837,7 +836,6 @@ void av1_quantize_dc_nuq_facade(const tran_low_t *coeff_ptr, intptr_t n_coeffs,
void av1_highbd_quantize_fp_facade(const tran_low_t *coeff_ptr,
intptr_t n_coeffs, const MACROBLOCK_PLANE *p,
tran_low_t *qcoeff_ptr,
const MACROBLOCKD_PLANE *pd,
tran_low_t *dqcoeff_ptr, uint16_t *eob_ptr,
const SCAN_ORDER *sc,
const QUANT_PARAM *qparam) {
......@@ -847,27 +845,28 @@ void av1_highbd_quantize_fp_facade(const tran_low_t *coeff_ptr,
const qm_val_t *qm_ptr = qparam->qmatrix;
const qm_val_t *iqm_ptr = qparam->iqmatrix;
if (qm_ptr != NULL && iqm_ptr != NULL) {
highbd_quantize_fp_helper_c(
coeff_ptr, n_coeffs, skip_block, p->zbin, p->round_fp, p->quant_fp,
p->quant_shift, qcoeff_ptr, dqcoeff_ptr, pd->dequant, eob_ptr, sc->scan,
sc->iscan, qm_ptr, iqm_ptr, qparam->log_scale);
highbd_quantize_fp_helper_c(coeff_ptr, n_coeffs, skip_block, p->zbin_QTX,
p->round_fp_QTX, p->quant_fp_QTX,
p->quant_shift_QTX, qcoeff_ptr, dqcoeff_ptr,
p->dequant_QTX, eob_ptr, sc->scan, sc->iscan,
qm_ptr, iqm_ptr, qparam->log_scale);
} else {
#endif // CONFIG_AOM_QM
if (n_coeffs < 16) {
// TODO(jingning): Need SIMD implementation for smaller block size
// quantization.
av1_highbd_quantize_fp_c(coeff_ptr, n_coeffs, skip_block, p->zbin,
p->round_fp, p->quant_fp, p->quant_shift,
qcoeff_ptr, dqcoeff_ptr, pd->dequant, eob_ptr,
sc->scan, sc->iscan, qparam->log_scale);
av1_highbd_quantize_fp_c(
coeff_ptr, n_coeffs, skip_block, p->zbin_QTX, p->round_fp_QTX,
p->quant_fp_QTX, p->quant_shift_QTX, qcoeff_ptr, dqcoeff_ptr,
p->dequant_QTX, eob_ptr, sc->scan, sc->iscan, qparam->log_scale);
return;
}
av1_highbd_quantize_fp(coeff_ptr, n_coeffs, skip_block, p->zbin,
p->round_fp, p->quant_fp, p->quant_shift, qcoeff_ptr,
dqcoeff_ptr, pd->dequant, eob_ptr, sc->scan,
sc->iscan, qparam->log_scale);
av1_highbd_quantize_fp(coeff_ptr, n_coeffs, skip_block, p->zbin_QTX,
p->round_fp_QTX, p->quant_fp_QTX, p->quant_shift_QTX,
qcoeff_ptr, dqcoeff_ptr, p->dequant_QTX, eob_ptr,
sc->scan, sc->iscan, qparam->log_scale);
#if CONFIG_AOM_QM
}
#endif
......@@ -876,7 +875,6 @@ void av1_highbd_quantize_fp_facade(const tran_low_t *coeff_ptr,
void av1_highbd_quantize_b_facade(const tran_low_t *coeff_ptr,
intptr_t n_coeffs, const MACROBLOCK_PLANE *p,
tran_low_t *qcoeff_ptr,
const MACROBLOCKD_PLANE *pd,
tran_low_t *dqcoeff_ptr, uint16_t *eob_ptr,
const SCAN_ORDER *sc,
const QUANT_PARAM *qparam) {
......@@ -886,41 +884,42 @@ void av1_highbd_quantize_b_facade(const tran_low_t *coeff_ptr,
const qm_val_t *qm_ptr = qparam->qmatrix;
const qm_val_t *iqm_ptr = qparam->iqmatrix;
if (qm_ptr != NULL && iqm_ptr != NULL) {
highbd_quantize_b_helper_c(coeff_ptr, n_coeffs, skip_block, p->zbin,
p->round, p->quant, p->quant_shift, qcoeff_ptr,
dqcoeff_ptr, pd->dequant, eob_ptr, sc->scan,
sc->iscan, qm_ptr, iqm_ptr, qparam->log_scale);
highbd_quantize_b_helper_c(coeff_ptr, n_coeffs, skip_block, p->zbin_QTX,
p->round_QTX, p->quant_QTX, p->quant_shift_QTX,
qcoeff_ptr, dqcoeff_ptr, p->dequant_QTX, eob_ptr,
sc->scan, sc->iscan, qm_ptr, iqm_ptr,
qparam->log_scale);
} else {
#endif // CONFIG_AOM_QM
switch (qparam->log_scale) {
case 0:
if (LIKELY(n_coeffs >= 8)) {
aom_highbd_quantize_b(coeff_ptr, n_coeffs, skip_block, p->zbin,
p->round, p->quant, p->quant_shift, qcoeff_ptr,
dqcoeff_ptr, pd->dequant, eob_ptr, sc->scan,
sc->iscan);
aom_highbd_quantize_b(coeff_ptr, n_coeffs, skip_block, p->zbin_QTX,
p->round_QTX, p->quant_QTX, p->quant_shift_QTX,
qcoeff_ptr, dqcoeff_ptr, p->dequant_QTX,
eob_ptr, sc->scan, sc->iscan);
} else {
// TODO(luoyi): Need SIMD (e.g. sse2) for smaller block size
// quantization
aom_highbd_quantize_b_c(coeff_ptr, n_coeffs, skip_block, p->zbin,
p->round, p->quant, p->quant_shift,
qcoeff_ptr, dqcoeff_ptr, pd->dequant, eob_ptr,
sc->scan, sc->iscan);
aom_highbd_quantize_b_c(coeff_ptr, n_coeffs, skip_block, p->zbin_QTX,
p->round_QTX, p->quant_QTX,
p->quant_shift_QTX, qcoeff_ptr, dqcoeff_ptr,
p->dequant_QTX, eob_ptr, sc->scan, sc->iscan);
}
break;
case 1:
aom_highbd_quantize_b_32x32(coeff_ptr, n_coeffs, skip_block, p->zbin,
p->round, p->quant, p->quant_shift,
qcoeff_ptr, dqcoeff_ptr, pd->dequant,
eob_ptr, sc->scan, sc->iscan);
aom_highbd_quantize_b_32x32(
coeff_ptr, n_coeffs, skip_block, p->zbin_QTX, p->round_QTX,
p->quant_QTX, p->quant_shift_QTX, qcoeff_ptr, dqcoeff_ptr,
p->dequant_QTX, eob_ptr, sc->scan, sc->iscan);
break;
#if CONFIG_TX64X64
case 2:
aom_highbd_quantize_b_64x64(coeff_ptr, n_coeffs, skip_block, p->zbin,
p->round, p->quant, p->quant_shift,
qcoeff_ptr, dqcoeff_ptr, pd->dequant,
eob_ptr, sc->scan, sc->iscan);
aom_highbd_quantize_b_64x64(
coeff_ptr, n_coeffs, skip_block, p->zbin_QTX, p->round_QTX,
p->quant_QTX, p->quant_shift_QTX, qcoeff_ptr, dqcoeff_ptr,
p->dequant_QTX, eob_ptr, sc->scan, sc->iscan);
break;
#endif // CONFIG_TX64X64
default: assert(0);
......@@ -963,7 +962,6 @@ static INLINE void highbd_quantize_dc(
void av1_highbd_quantize_dc_facade(const tran_low_t *coeff_ptr,
intptr_t n_coeffs, const MACROBLOCK_PLANE *p,
tran_low_t *qcoeff_ptr,
const MACROBLOCKD_PLANE *pd,
tran_low_t *dqcoeff_ptr, uint16_t *eob_ptr,
const SCAN_ORDER *sc,
const QUANT_PARAM *qparam) {
......@@ -979,9 +977,10 @@ void av1_highbd_quantize_dc_facade(const tran_low_t *coeff_ptr,
(void)sc;
highbd_quantize_dc(coeff_ptr, (int)n_coeffs, skip_block, p->round,
p->quant_fp[0], qcoeff_ptr, dqcoeff_ptr, pd->dequant[0],
eob_ptr, qm_ptr, iqm_ptr, qparam->log_scale);
highbd_quantize_dc(coeff_ptr, (int)n_coeffs, skip_block, p->round_QTX,
p->quant_fp_QTX[0], qcoeff_ptr, dqcoeff_ptr,
p->dequant_QTX[0], eob_ptr, qm_ptr, iqm_ptr,
qparam->log_scale);
}
#if CONFIG_NEW_QUANT
......@@ -1387,9 +1386,8 @@ void highbd_quantize_dc_64x64_fp_nuq(
void av1_highbd_quantize_b_nuq_facade(
const tran_low_t *coeff_ptr, intptr_t n_coeffs, const MACROBLOCK_PLANE *p,
tran_low_t *qcoeff_ptr, const MACROBLOCKD_PLANE *pd,
tran_low_t *dqcoeff_ptr, uint16_t *eob_ptr, const SCAN_ORDER *sc,
const QUANT_PARAM *qparam) {
tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, uint16_t *eob_ptr,
const SCAN_ORDER *sc, const QUANT_PARAM *qparam) {
// obsolete skip_block
const int skip_block = 0;
const uint8_t *band = get_band_translate(qparam->tx_size);
......@@ -1397,25 +1395,25 @@ void av1_highbd_quantize_b_nuq_facade(
switch (qparam->log_scale) {
case 0:
highbd_quantize_nuq(coeff_ptr, n_coeffs, skip_block, p->quant,
p->quant_shift, pd->dequant,
(const cuml_bins_type_nuq *)p->cuml_bins_nuq[dq],
(const dequant_val_type_nuq *)pd->dequant_val_nuq[dq],
qcoeff_ptr, dqcoeff_ptr, eob_ptr, sc->scan, band);
highbd_quantize_nuq(
coeff_ptr, n_coeffs, skip_block, p->quant_QTX, p->quant_shift_QTX,
p->dequant_QTX, (const cuml_bins_type_nuq *)p->cuml_bins_nuq[dq],
(const dequant_val_type_nuq *)p->dequant_val_nuq_QTX[dq], qcoeff_ptr,
dqcoeff_ptr, eob_ptr, sc->scan, band);
break;
case 1:
highbd_quantize_32x32_nuq(
coeff_ptr, n_coeffs, skip_block, p->quant, p->quant_shift,
pd->dequant, (const cuml_bins_type_nuq *)p->cuml_bins_nuq[dq],
(const dequant_val_type_nuq *)pd->dequant_val_nuq[dq], qcoeff_ptr,
coeff_ptr, n_coeffs, skip_block, p->quant_QTX, p->quant_shift_QTX,
p->dequant_QTX, (const cuml_bins_type_nuq *)p->cuml_bins_nuq[dq],
(const dequant_val_type_nuq *)p->dequant_val_nuq_QTX[dq], qcoeff_ptr,