quantize.c 17.2 KB
Newer Older
Jingning Han's avatar
Jingning Han committed
1
/*
2
 * Copyright (c) 2016, Alliance for Open Media. All rights reserved
Jingning Han's avatar
Jingning Han committed
3
 *
4 5 6 7 8 9
 * This source code is subject to the terms of the BSD 2 Clause License and
 * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
 * was not distributed with this source code in the LICENSE file, you can
 * obtain it at www.aomedia.org/license/software. If the Alliance for Open
 * Media Patent License 1.0 was not distributed with this source code in the
 * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
Jingning Han's avatar
Jingning Han committed
10 11
 */

12
#include <math.h>
Adrian Grange's avatar
Adrian Grange committed
13 14
#include "./aom_dsp_rtcd.h"
#include "aom_mem/aom_mem.h"
Yaowu Xu's avatar
Yaowu Xu committed
15
#include "aom_ports/mem.h"
Jingning Han's avatar
Jingning Han committed
16

Yaowu Xu's avatar
Yaowu Xu committed
17 18
#include "av1/common/quant_common.h"
#include "av1/common/seg_common.h"
Jingning Han's avatar
Jingning Han committed
19

Yaowu Xu's avatar
Yaowu Xu committed
20 21 22
#include "av1/encoder/encoder.h"
#include "av1/encoder/quantize.h"
#include "av1/encoder/rd.h"
Jingning Han's avatar
Jingning Han committed
23

24
void av1_quantize_fp_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs,
clang-format's avatar
clang-format committed
25 26 27 28 29 30
                       int skip_block, const int16_t *zbin_ptr,
                       const int16_t *round_ptr, const int16_t *quant_ptr,
                       const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr,
                       tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr,
                       uint16_t *eob_ptr, const int16_t *scan,
                       const int16_t *iscan
31
#if CONFIG_AOM_QM
clang-format's avatar
clang-format committed
32 33
                       ,
                       const qm_val_t *qm_ptr, const qm_val_t *iqm_ptr
34
#endif
clang-format's avatar
clang-format committed
35
                       ) {
Jingning Han's avatar
Jingning Han committed
36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51
  int i, eob = -1;
  // TODO(jingning) Decide the need of these arguments after the
  // quantization process is completed.
  (void)zbin_ptr;
  (void)quant_shift_ptr;
  (void)iscan;

  memset(qcoeff_ptr, 0, n_coeffs * sizeof(*qcoeff_ptr));
  memset(dqcoeff_ptr, 0, n_coeffs * sizeof(*dqcoeff_ptr));

  if (!skip_block) {
    // Quantization pass: All coefficients with index >= zero_flag are
    // skippable. Note: zero_flag can be zero.
    for (i = 0; i < n_coeffs; i++) {
      const int rc = scan[i];
      const int coeff = coeff_ptr[rc];
52 53 54 55 56 57 58
#if CONFIG_AOM_QM
      const qm_val_t wt = qm_ptr[rc];
      const qm_val_t iwt = iqm_ptr[rc];
      const int dequant =
          (dequant_ptr[rc != 0] * iwt + (1 << (AOM_QM_BITS - 1))) >>
          AOM_QM_BITS;
#endif
Jingning Han's avatar
Jingning Han committed
59 60 61
      const int coeff_sign = (coeff >> 31);
      const int abs_coeff = (coeff ^ coeff_sign) - coeff_sign;

62 63 64
      int64_t tmp = clamp(abs_coeff + round_ptr[rc != 0], INT16_MIN, INT16_MAX);
      int tmp32;
#if CONFIG_AOM_QM
Yaowu Xu's avatar
Yaowu Xu committed
65
      tmp32 = (int)((tmp * wt * quant_ptr[rc != 0]) >> (16 + AOM_QM_BITS));
66 67 68
      qcoeff_ptr[rc] = (tmp32 ^ coeff_sign) - coeff_sign;
      dqcoeff_ptr[rc] = qcoeff_ptr[rc] * dequant;
#else
Yaowu Xu's avatar
Yaowu Xu committed
69
      tmp32 = (int)((tmp * quant_ptr[rc != 0]) >> 16);
70
      qcoeff_ptr[rc] = (tmp32 ^ coeff_sign) - coeff_sign;
Jingning Han's avatar
Jingning Han committed
71
      dqcoeff_ptr[rc] = qcoeff_ptr[rc] * dequant_ptr[rc != 0];
72
#endif
Jingning Han's avatar
Jingning Han committed
73

74
      if (tmp32) eob = i;
Jingning Han's avatar
Jingning Han committed
75 76 77 78 79
    }
  }
  *eob_ptr = eob + 1;
}

80 81
#if CONFIG_AOM_HIGHBITDEPTH
void av1_highbd_quantize_fp_c(const tran_low_t *coeff_ptr, intptr_t count,
clang-format's avatar
clang-format committed
82 83 84 85 86 87 88
                              int skip_block, const int16_t *zbin_ptr,
                              const int16_t *round_ptr,
                              const int16_t *quant_ptr,
                              const int16_t *quant_shift_ptr,
                              tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr,
                              const int16_t *dequant_ptr, uint16_t *eob_ptr,
                              const int16_t *scan, const int16_t *iscan
89
#if CONFIG_AOM_QM
clang-format's avatar
clang-format committed
90 91
                              ,
                              const qm_val_t *qm_ptr, const qm_val_t *iqm_ptr
92
#endif
clang-format's avatar
clang-format committed
93
                              ) {
Jingning Han's avatar
Jingning Han committed
94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110
  int i;
  int eob = -1;
  // TODO(jingning) Decide the need of these arguments after the
  // quantization process is completed.
  (void)zbin_ptr;
  (void)quant_shift_ptr;
  (void)iscan;

  memset(qcoeff_ptr, 0, count * sizeof(*qcoeff_ptr));
  memset(dqcoeff_ptr, 0, count * sizeof(*dqcoeff_ptr));

  if (!skip_block) {
    // Quantization pass: All coefficients with index >= zero_flag are
    // skippable. Note: zero_flag can be zero.
    for (i = 0; i < count; i++) {
      const int rc = scan[i];
      const int coeff = coeff_ptr[rc];
111 112 113 114 115 116 117
#if CONFIG_AOM_QM
      const qm_val_t wt = qm_ptr[rc];
      const qm_val_t iwt = iqm_ptr[rc];
      const int dequant =
          (dequant_ptr[rc != 0] * iwt + (1 << (AOM_QM_BITS - 1))) >>
          AOM_QM_BITS;
#endif
Jingning Han's avatar
Jingning Han committed
118 119 120
      const int coeff_sign = (coeff >> 31);
      const int abs_coeff = (coeff ^ coeff_sign) - coeff_sign;
      const int64_t tmp = abs_coeff + round_ptr[rc != 0];
121 122 123 124 125 126
#if CONFIG_AOM_QM
      const uint32_t abs_qcoeff =
          (uint32_t)((tmp * quant_ptr[rc != 0] * wt) >> (16 + AOM_QM_BITS));
      qcoeff_ptr[rc] = (tran_low_t)((abs_qcoeff ^ coeff_sign) - coeff_sign);
      dqcoeff_ptr[rc] = qcoeff_ptr[rc] * dequant;
#else
Jingning Han's avatar
Jingning Han committed
127 128 129
      const uint32_t abs_qcoeff = (uint32_t)((tmp * quant_ptr[rc != 0]) >> 16);
      qcoeff_ptr[rc] = (tran_low_t)((abs_qcoeff ^ coeff_sign) - coeff_sign);
      dqcoeff_ptr[rc] = qcoeff_ptr[rc] * dequant_ptr[rc != 0];
130
#endif
clang-format's avatar
clang-format committed
131
      if (abs_qcoeff) eob = i;
Jingning Han's avatar
Jingning Han committed
132 133 134 135 136 137 138 139
    }
  }
  *eob_ptr = eob + 1;
}
#endif

// TODO(jingning) Refactor this file and combine functions with similar
// operations.
140
void av1_quantize_fp_32x32_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs,
clang-format's avatar
clang-format committed
141 142 143 144 145 146
                             int skip_block, const int16_t *zbin_ptr,
                             const int16_t *round_ptr, const int16_t *quant_ptr,
                             const int16_t *quant_shift_ptr,
                             tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr,
                             const int16_t *dequant_ptr, uint16_t *eob_ptr,
                             const int16_t *scan, const int16_t *iscan
147
#if CONFIG_AOM_QM
clang-format's avatar
clang-format committed
148 149
                             ,
                             const qm_val_t *qm_ptr, const qm_val_t *iqm_ptr
150
#endif
clang-format's avatar
clang-format committed
151
                             ) {
Jingning Han's avatar
Jingning Han committed
152 153 154 155 156 157 158 159 160 161 162 163
  int i, eob = -1;
  (void)zbin_ptr;
  (void)quant_shift_ptr;
  (void)iscan;

  memset(qcoeff_ptr, 0, n_coeffs * sizeof(*qcoeff_ptr));
  memset(dqcoeff_ptr, 0, n_coeffs * sizeof(*dqcoeff_ptr));

  if (!skip_block) {
    for (i = 0; i < n_coeffs; i++) {
      const int rc = scan[i];
      const int coeff = coeff_ptr[rc];
164 165 166 167 168 169 170 171
#if CONFIG_AOM_QM
      const qm_val_t wt = qm_ptr[rc];
      const qm_val_t iwt = iqm_ptr[rc];
      const int dequant =
          (dequant_ptr[rc != 0] * iwt + (1 << (AOM_QM_BITS - 1))) >>
          AOM_QM_BITS;
      int64_t tmp = 0;
#endif
Jingning Han's avatar
Jingning Han committed
172
      const int coeff_sign = (coeff >> 31);
173
      int tmp32 = 0;
Jingning Han's avatar
Jingning Han committed
174 175
      int abs_coeff = (coeff ^ coeff_sign) - coeff_sign;

176 177 178
#if CONFIG_AOM_QM
      if (abs_coeff * wt >= (dequant_ptr[rc != 0] << (AOM_QM_BITS - 2))) {
#else
Jingning Han's avatar
Jingning Han committed
179
      if (abs_coeff >= (dequant_ptr[rc != 0] >> 2)) {
180
#endif
Jingning Han's avatar
Jingning Han committed
181 182
        abs_coeff += ROUND_POWER_OF_TWO(round_ptr[rc != 0], 1);
        abs_coeff = clamp(abs_coeff, INT16_MIN, INT16_MAX);
183 184 185 186 187 188 189 190 191 192
#if CONFIG_AOM_QM
        tmp = abs_coeff * wt;
        tmp32 = (int)(tmp * quant_ptr[rc != 0]) >> (AOM_QM_BITS + 15);
        qcoeff_ptr[rc] = (tmp32 ^ coeff_sign) - coeff_sign;
        dqcoeff_ptr[rc] = (qcoeff_ptr[rc] * dequant) / 2;
#else
        tmp32 = (abs_coeff * quant_ptr[rc != 0]) >> 15;
        qcoeff_ptr[rc] = (tmp32 ^ coeff_sign) - coeff_sign;
        dqcoeff_ptr[rc] = (qcoeff_ptr[rc] * dequant_ptr[rc != 0]) / 2;
#endif
Jingning Han's avatar
Jingning Han committed
193 194
      }

195
      if (tmp32) eob = i;
Jingning Han's avatar
Jingning Han committed
196 197 198 199 200
    }
  }
  *eob_ptr = eob + 1;
}

201 202
#if CONFIG_AOM_HIGHBITDEPTH
void av1_highbd_quantize_fp_32x32_c(
clang-format's avatar
clang-format committed
203 204 205 206
    const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block,
    const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr,
    const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr,
    tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr,
207 208 209 210 211 212
    const int16_t *scan, const int16_t *iscan
#if CONFIG_AOM_QM
    ,
    const qm_val_t *qm_ptr, const qm_val_t *iqm_ptr
#endif
    ) {
Jingning Han's avatar
Jingning Han committed
213 214 215 216 217 218 219 220 221 222 223 224 225
  int i, eob = -1;
  (void)zbin_ptr;
  (void)quant_shift_ptr;
  (void)iscan;

  memset(qcoeff_ptr, 0, n_coeffs * sizeof(*qcoeff_ptr));
  memset(dqcoeff_ptr, 0, n_coeffs * sizeof(*dqcoeff_ptr));

  if (!skip_block) {
    for (i = 0; i < n_coeffs; i++) {
      uint32_t abs_qcoeff = 0;
      const int rc = scan[i];
      const int coeff = coeff_ptr[rc];
226 227 228 229 230 231 232
#if CONFIG_AOM_QM
      const qm_val_t wt = qm_ptr[rc];
      const qm_val_t iwt = iqm_ptr[rc];
      const int dequant =
          (dequant_ptr[rc != 0] * iwt + (1 << (AOM_QM_BITS - 1))) >>
          AOM_QM_BITS;
#endif
Jingning Han's avatar
Jingning Han committed
233 234
      const int coeff_sign = (coeff >> 31);
      const int abs_coeff = (coeff ^ coeff_sign) - coeff_sign;
235 236 237
#if CONFIG_AOM_QM
      if (abs_coeff * wt >= (dequant_ptr[rc != 0] << (AOM_QM_BITS - 2))) {
#else
Jingning Han's avatar
Jingning Han committed
238
      if (abs_coeff >= (dequant_ptr[rc != 0] >> 2)) {
239
#endif
clang-format's avatar
clang-format committed
240 241
        const int64_t tmp =
            abs_coeff + ROUND_POWER_OF_TWO(round_ptr[rc != 0], 1);
242 243 244 245 246 247
#if CONFIG_AOM_QM
        abs_qcoeff =
            (uint32_t)((tmp * wt * quant_ptr[rc != 0]) >> (AOM_QM_BITS + 15));
        qcoeff_ptr[rc] = (tran_low_t)((abs_qcoeff ^ coeff_sign) - coeff_sign);
        dqcoeff_ptr[rc] = (qcoeff_ptr[rc] * dequant) / 2;
#else
clang-format's avatar
clang-format committed
248
        abs_qcoeff = (uint32_t)((tmp * quant_ptr[rc != 0]) >> 15);
Jingning Han's avatar
Jingning Han committed
249
        qcoeff_ptr[rc] = (tran_low_t)((abs_qcoeff ^ coeff_sign) - coeff_sign);
250 251
        dqcoeff_ptr[rc] = (qcoeff_ptr[rc] * dequant_ptr[rc != 0]) / 2;
#endif
Jingning Han's avatar
Jingning Han committed
252 253
      }

clang-format's avatar
clang-format committed
254
      if (abs_qcoeff) eob = i;
Jingning Han's avatar
Jingning Han committed
255 256 257 258 259 260
    }
  }
  *eob_ptr = eob + 1;
}
#endif

261
void av1_regular_quantize_b_4x4(MACROBLOCK *x, int plane, int block,
clang-format's avatar
clang-format committed
262
                                const int16_t *scan, const int16_t *iscan) {
Jingning Han's avatar
Jingning Han committed
263 264 265
  MACROBLOCKD *const xd = &x->e_mbd;
  struct macroblock_plane *p = &x->plane[plane];
  struct macroblockd_plane *pd = &xd->plane[plane];
266 267 268 269 270 271
#if CONFIG_AOM_QM
  int seg_id = xd->mi[0]->mbmi.segment_id;
  int is_intra = is_inter_block(&xd->mi[0]->mbmi);
  const qm_val_t *qmatrix = pd->seg_qmatrix[seg_id][is_intra][0];
  const qm_val_t *iqmatrix = pd->seg_iqmatrix[seg_id][is_intra][0];
#endif
Jingning Han's avatar
Jingning Han committed
272

273
#if CONFIG_AOM_HIGHBITDEPTH
Jingning Han's avatar
Jingning Han committed
274
  if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) {
Adrian Grange's avatar
Adrian Grange committed
275
    aom_highbd_quantize_b(BLOCK_OFFSET(p->coeff, block), 16, x->skip_block,
Jingning Han's avatar
Jingning Han committed
276 277
                          p->zbin, p->round, p->quant, p->quant_shift,
                          BLOCK_OFFSET(p->qcoeff, block),
clang-format's avatar
clang-format committed
278
                          BLOCK_OFFSET(pd->dqcoeff, block), pd->dequant,
279
#if !CONFIG_AOM_QM
clang-format's avatar
clang-format committed
280
                          &p->eobs[block], scan, iscan);
281
#else
clang-format's avatar
clang-format committed
282
                          &p->eobs[block], scan, iscan, qmatrix, iqmatrix);
283
#endif
Jingning Han's avatar
Jingning Han committed
284 285 286
    return;
  }
#endif
Adrian Grange's avatar
Adrian Grange committed
287
  aom_quantize_b(BLOCK_OFFSET(p->coeff, block), 16, x->skip_block, p->zbin,
clang-format's avatar
clang-format committed
288
                 p->round, p->quant, p->quant_shift,
Jingning Han's avatar
Jingning Han committed
289
                 BLOCK_OFFSET(p->qcoeff, block),
clang-format's avatar
clang-format committed
290
                 BLOCK_OFFSET(pd->dqcoeff, block), pd->dequant, &p->eobs[block],
291
#if !CONFIG_AOM_QM
clang-format's avatar
clang-format committed
292
                 scan, iscan);
293
#else
clang-format's avatar
clang-format committed
294
                 scan, iscan, qmatrix, iqmatrix);
295
#endif
Jingning Han's avatar
Jingning Han committed
296 297 298
}

static void invert_quant(int16_t *quant, int16_t *shift, int d) {
Yaowu Xu's avatar
Yaowu Xu committed
299 300
  uint32_t t;
  int l, m;
Jingning Han's avatar
Jingning Han committed
301
  t = d;
clang-format's avatar
clang-format committed
302
  for (l = 0; t > 1; l++) t >>= 1;
Yaowu Xu's avatar
Yaowu Xu committed
303 304
  m = 1 + (1 << (16 + l)) / d;
  *quant = (int16_t)(m - (1 << 16));
Jingning Han's avatar
Jingning Han committed
305 306 307
  *shift = 1 << (16 - l);
}

Adrian Grange's avatar
Adrian Grange committed
308
static int get_qzbin_factor(int q, aom_bit_depth_t bit_depth) {
309 310
  const int quant = av1_dc_quant(q, 0, bit_depth);
#if CONFIG_AOM_HIGHBITDEPTH
Jingning Han's avatar
Jingning Han committed
311
  switch (bit_depth) {
clang-format's avatar
clang-format committed
312 313 314
    case AOM_BITS_8: return q == 0 ? 64 : (quant < 148 ? 84 : 80);
    case AOM_BITS_10: return q == 0 ? 64 : (quant < 592 ? 84 : 80);
    case AOM_BITS_12: return q == 0 ? 64 : (quant < 2368 ? 84 : 80);
Jingning Han's avatar
Jingning Han committed
315
    default:
Adrian Grange's avatar
Adrian Grange committed
316
      assert(0 && "bit_depth should be AOM_BITS_8, AOM_BITS_10 or AOM_BITS_12");
Jingning Han's avatar
Jingning Han committed
317 318 319
      return -1;
  }
#else
clang-format's avatar
clang-format committed
320
  (void)bit_depth;
Jingning Han's avatar
Jingning Han committed
321 322 323 324
  return q == 0 ? 64 : (quant < 148 ? 84 : 80);
#endif
}

325 326
void av1_init_quantizer(AV1_COMP *cpi) {
  AV1_COMMON *const cm = &cpi->common;
Jingning Han's avatar
Jingning Han committed
327 328 329 330 331 332 333 334 335
  QUANTS *const quants = &cpi->quants;
  int i, q, quant;

  for (q = 0; q < QINDEX_RANGE; q++) {
    const int qzbin_factor = get_qzbin_factor(q, cm->bit_depth);
    const int qrounding_factor = q == 0 ? 64 : 48;

    for (i = 0; i < 2; ++i) {
      int qrounding_factor_fp = i == 0 ? 48 : 42;
clang-format's avatar
clang-format committed
336
      if (q == 0) qrounding_factor_fp = 64;
Jingning Han's avatar
Jingning Han committed
337 338

      // y
339 340
      quant = i == 0 ? av1_dc_quant(q, cm->y_dc_delta_q, cm->bit_depth)
                     : av1_ac_quant(q, 0, cm->bit_depth);
Jingning Han's avatar
Jingning Han committed
341 342 343 344 345 346 347 348
      invert_quant(&quants->y_quant[q][i], &quants->y_quant_shift[q][i], quant);
      quants->y_quant_fp[q][i] = (1 << 16) / quant;
      quants->y_round_fp[q][i] = (qrounding_factor_fp * quant) >> 7;
      quants->y_zbin[q][i] = ROUND_POWER_OF_TWO(qzbin_factor * quant, 7);
      quants->y_round[q][i] = (qrounding_factor * quant) >> 7;
      cpi->y_dequant[q][i] = quant;

      // uv
349 350
      quant = i == 0 ? av1_dc_quant(q, cm->uv_dc_delta_q, cm->bit_depth)
                     : av1_ac_quant(q, cm->uv_ac_delta_q, cm->bit_depth);
clang-format's avatar
clang-format committed
351 352
      invert_quant(&quants->uv_quant[q][i], &quants->uv_quant_shift[q][i],
                   quant);
Jingning Han's avatar
Jingning Han committed
353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379
      quants->uv_quant_fp[q][i] = (1 << 16) / quant;
      quants->uv_round_fp[q][i] = (qrounding_factor_fp * quant) >> 7;
      quants->uv_zbin[q][i] = ROUND_POWER_OF_TWO(qzbin_factor * quant, 7);
      quants->uv_round[q][i] = (qrounding_factor * quant) >> 7;
      cpi->uv_dequant[q][i] = quant;
    }

    for (i = 2; i < 8; i++) {
      quants->y_quant[q][i] = quants->y_quant[q][1];
      quants->y_quant_fp[q][i] = quants->y_quant_fp[q][1];
      quants->y_round_fp[q][i] = quants->y_round_fp[q][1];
      quants->y_quant_shift[q][i] = quants->y_quant_shift[q][1];
      quants->y_zbin[q][i] = quants->y_zbin[q][1];
      quants->y_round[q][i] = quants->y_round[q][1];
      cpi->y_dequant[q][i] = cpi->y_dequant[q][1];

      quants->uv_quant[q][i] = quants->uv_quant[q][1];
      quants->uv_quant_fp[q][i] = quants->uv_quant_fp[q][1];
      quants->uv_round_fp[q][i] = quants->uv_round_fp[q][1];
      quants->uv_quant_shift[q][i] = quants->uv_quant_shift[q][1];
      quants->uv_zbin[q][i] = quants->uv_zbin[q][1];
      quants->uv_round[q][i] = quants->uv_round[q][1];
      cpi->uv_dequant[q][i] = cpi->uv_dequant[q][1];
    }
  }
}

380
void av1_init_plane_quantizers(const AV1_COMP *cpi, MACROBLOCK *x) {
381
  const AV1_COMMON *const cm = &cpi->common;
Jingning Han's avatar
Jingning Han committed
382
  MACROBLOCKD *const xd = &x->e_mbd;
383
  const QUANTS *const quants = &cpi->quants;
Jingning Han's avatar
Jingning Han committed
384
  const int segment_id = xd->mi[0]->mbmi.segment_id;
385 386
  const int qindex = av1_get_qindex(&cm->seg, segment_id, cm->base_qindex);
  const int rdmult = av1_compute_rd_mult(cpi, qindex + cm->y_dc_delta_q);
Jingning Han's avatar
Jingning Han committed
387
  int i;
388
#if CONFIG_AOM_QM
389 390
  int minqm = cm->min_qmlevel;
  int maxqm = cm->max_qmlevel;
391
  // Quant matrix only depends on the base QP so there is only one set per frame
392
  int qmlevel = (xd->lossless[segment_id] || cm->using_qmatrix == 0)
393 394
                    ? NUM_QM_LEVELS - 1
                    : aom_get_qmlevel(cm->base_qindex, minqm, maxqm);
395
#endif
Jingning Han's avatar
Jingning Han committed
396 397 398 399 400 401 402 403

  // Y
  x->plane[0].quant = quants->y_quant[qindex];
  x->plane[0].quant_fp = quants->y_quant_fp[qindex];
  x->plane[0].round_fp = quants->y_round_fp[qindex];
  x->plane[0].quant_shift = quants->y_quant_shift[qindex];
  x->plane[0].zbin = quants->y_zbin[qindex];
  x->plane[0].round = quants->y_round[qindex];
404 405 406 407 408 409
#if CONFIG_AOM_QM
  memcpy(&xd->plane[0].seg_qmatrix[segment_id], cm->gqmatrix[qmlevel][0],
         sizeof(cm->gqmatrix[qmlevel][0]));
  memcpy(&xd->plane[0].seg_iqmatrix[segment_id], cm->giqmatrix[qmlevel][0],
         sizeof(cm->giqmatrix[qmlevel][0]));
#endif
Jingning Han's avatar
Jingning Han committed
410 411 412 413 414 415 416 417 418 419 420 421 422
  xd->plane[0].dequant = cpi->y_dequant[qindex];

  x->plane[0].quant_thred[0] = x->plane[0].zbin[0] * x->plane[0].zbin[0];
  x->plane[0].quant_thred[1] = x->plane[0].zbin[1] * x->plane[0].zbin[1];

  // UV
  for (i = 1; i < 3; i++) {
    x->plane[i].quant = quants->uv_quant[qindex];
    x->plane[i].quant_fp = quants->uv_quant_fp[qindex];
    x->plane[i].round_fp = quants->uv_round_fp[qindex];
    x->plane[i].quant_shift = quants->uv_quant_shift[qindex];
    x->plane[i].zbin = quants->uv_zbin[qindex];
    x->plane[i].round = quants->uv_round[qindex];
423 424 425 426 427 428
#if CONFIG_AOM_QM
    memcpy(&xd->plane[i].seg_qmatrix[segment_id], cm->gqmatrix[qmlevel][1],
           sizeof(cm->gqmatrix[qmlevel][1]));
    memcpy(&xd->plane[i].seg_iqmatrix[segment_id], cm->giqmatrix[qmlevel][1],
           sizeof(cm->giqmatrix[qmlevel][1]));
#endif
Jingning Han's avatar
Jingning Han committed
429 430 431 432 433 434 435 436 437
    xd->plane[i].dequant = cpi->uv_dequant[qindex];

    x->plane[i].quant_thred[0] = x->plane[i].zbin[0] * x->plane[i].zbin[0];
    x->plane[i].quant_thred[1] = x->plane[i].zbin[1] * x->plane[i].zbin[1];
  }

  x->skip_block = segfeature_active(&cm->seg, segment_id, SEG_LVL_SKIP);
  x->q_index = qindex;

438
  set_error_per_bit(x, rdmult);
Jingning Han's avatar
Jingning Han committed
439

440
  av1_initialize_me_consts(cpi, x, x->q_index);
Jingning Han's avatar
Jingning Han committed
441 442
}

443 444
void av1_frame_init_quantizer(AV1_COMP *cpi) {
  av1_init_plane_quantizers(cpi, &cpi->td.mb);
Jingning Han's avatar
Jingning Han committed
445 446
}

447 448
void av1_set_quantizer(AV1_COMMON *cm, int q) {
  // quantizer has to be reinitialized with av1_init_quantizer() if any
Jingning Han's avatar
Jingning Han committed
449 450 451 452 453 454 455 456 457 458
  // delta_q changes.
  cm->base_qindex = q;
  cm->y_dc_delta_q = 0;
  cm->uv_dc_delta_q = 0;
  cm->uv_ac_delta_q = 0;
}

// Table that converts 0-63 Q-range values passed in outside to the Qindex
// range used internally.
static const int quantizer_to_qindex[] = {
clang-format's avatar
clang-format committed
459 460 461 462 463
  0,   4,   8,   12,  16,  20,  24,  28,  32,  36,  40,  44,  48,
  52,  56,  60,  64,  68,  72,  76,  80,  84,  88,  92,  96,  100,
  104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 144, 148, 152,
  156, 160, 164, 168, 172, 176, 180, 184, 188, 192, 196, 200, 204,
  208, 212, 216, 220, 224, 228, 232, 236, 240, 244, 249, 255,
Jingning Han's avatar
Jingning Han committed
464 465
};

466
int av1_quantizer_to_qindex(int quantizer) {
Jingning Han's avatar
Jingning Han committed
467 468 469
  return quantizer_to_qindex[quantizer];
}

470
int av1_qindex_to_quantizer(int qindex) {
Jingning Han's avatar
Jingning Han committed
471 472 473
  int quantizer;

  for (quantizer = 0; quantizer < 64; ++quantizer)
clang-format's avatar
clang-format committed
474
    if (quantizer_to_qindex[quantizer] >= qindex) return quantizer;
Jingning Han's avatar
Jingning Han committed
475 476 477

  return 63;
}