encodemb.c 49.4 KB
Newer Older
Jingning Han's avatar
Jingning Han committed
1
/*
2
 * Copyright (c) 2016, Alliance for Open Media. All rights reserved
Jingning Han's avatar
Jingning Han committed
3
 *
4 5 6 7 8 9
 * This source code is subject to the terms of the BSD 2 Clause License and
 * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
 * was not distributed with this source code in the LICENSE file, you can
 * obtain it at www.aomedia.org/license/software. If the Alliance for Open
 * Media Patent License 1.0 was not distributed with this source code in the
 * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
Jingning Han's avatar
Jingning Han committed
10 11
 */

Yaowu Xu's avatar
Yaowu Xu committed
12
#include "./av1_rtcd.h"
Adrian Grange's avatar
Adrian Grange committed
13 14
#include "./aom_config.h"
#include "./aom_dsp_rtcd.h"
Jingning Han's avatar
Jingning Han committed
15

Yaowu Xu's avatar
Yaowu Xu committed
16
#include "aom_dsp/quantize.h"
Adrian Grange's avatar
Adrian Grange committed
17
#include "aom_mem/aom_mem.h"
Yaowu Xu's avatar
Yaowu Xu committed
18
#include "aom_ports/mem.h"
Jingning Han's avatar
Jingning Han committed
19

Yaowu Xu's avatar
Yaowu Xu committed
20 21 22 23
#include "av1/common/idct.h"
#include "av1/common/reconinter.h"
#include "av1/common/reconintra.h"
#include "av1/common/scan.h"
Jingning Han's avatar
Jingning Han committed
24

Yaowu Xu's avatar
Yaowu Xu committed
25
#include "av1/encoder/encodemb.h"
26
#include "av1/encoder/hybrid_fwd_txfm.h"
Yaowu Xu's avatar
Yaowu Xu committed
27 28
#include "av1/encoder/rd.h"
#include "av1/encoder/tokenize.h"
Jingning Han's avatar
Jingning Han committed
29

30 31 32 33 34 35
#if CONFIG_PVQ
#include "av1/encoder/encint.h"
#include "av1/common/partition.h"
#include "av1/encoder/pvq_encoder.h"
#endif

Jingning Han's avatar
Jingning Han committed
36 37 38 39 40
struct optimize_ctx {
  ENTROPY_CONTEXT ta[MAX_MB_PLANE][16];
  ENTROPY_CONTEXT tl[MAX_MB_PLANE][16];
};

41
void av1_subtract_plane(MACROBLOCK *x, BLOCK_SIZE bsize, int plane) {
Jingning Han's avatar
Jingning Han committed
42 43 44 45 46 47
  struct macroblock_plane *const p = &x->plane[plane];
  const struct macroblockd_plane *const pd = &x->e_mbd.plane[plane];
  const BLOCK_SIZE plane_bsize = get_plane_block_size(bsize, pd);
  const int bw = 4 * num_4x4_blocks_wide_lookup[plane_bsize];
  const int bh = 4 * num_4x4_blocks_high_lookup[plane_bsize];

48
#if CONFIG_AOM_HIGHBITDEPTH
Jingning Han's avatar
Jingning Han committed
49
  if (x->e_mbd.cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) {
Adrian Grange's avatar
Adrian Grange committed
50
    aom_highbd_subtract_block(bh, bw, p->src_diff, bw, p->src.buf,
Jingning Han's avatar
Jingning Han committed
51 52 53 54
                              p->src.stride, pd->dst.buf, pd->dst.stride,
                              x->e_mbd.bd);
    return;
  }
55
#endif  // CONFIG_AOM_HIGHBITDEPTH
Adrian Grange's avatar
Adrian Grange committed
56
  aom_subtract_block(bh, bw, p->src_diff, bw, p->src.buf, p->src.stride,
Jingning Han's avatar
Jingning Han committed
57 58 59
                     pd->dst.buf, pd->dst.stride);
}

60
#define RDTRUNC(RM, DM, R, D)                        \
Adrian Grange's avatar
Adrian Grange committed
61 62
  (((1 << (AV1_PROB_COST_SHIFT - 1)) + (R) * (RM)) & \
   ((1 << AV1_PROB_COST_SHIFT) - 1))
Jingning Han's avatar
Jingning Han committed
63

64
typedef struct av1_token_state {
clang-format's avatar
clang-format committed
65 66 67 68 69
  int rate;
  int error;
  int next;
  int16_t token;
  short qc;
70
} av1_token_state;
Jingning Han's avatar
Jingning Han committed
71

72
#if !CONFIG_PVQ
Jingning Han's avatar
Jingning Han committed
73 74 75
// TODO(jimbankoski): experiment to find optimal RD numbers.
static const int plane_rd_mult[PLANE_TYPES] = { 4, 2 };

clang-format's avatar
clang-format committed
76 77 78 79 80 81 82 83 84
#define UPDATE_RD_COST()                                \
  {                                                     \
    rd_cost0 = RDCOST(rdmult, rddiv, rate0, error0);    \
    rd_cost1 = RDCOST(rdmult, rddiv, rate1, error1);    \
    if (rd_cost0 == rd_cost1) {                         \
      rd_cost0 = RDTRUNC(rdmult, rddiv, rate0, error0); \
      rd_cost1 = RDTRUNC(rdmult, rddiv, rate1, error1); \
    }                                                   \
  }
Jingning Han's avatar
Jingning Han committed
85 86 87

// This function is a place holder for now but may ultimately need
// to scan previous tokens to work out the correct context.
clang-format's avatar
clang-format committed
88 89
static int trellis_get_coeff_context(const int16_t *scan, const int16_t *nb,
                                     int idx, int token, uint8_t *token_cache) {
Jingning Han's avatar
Jingning Han committed
90
  int bak = token_cache[scan[idx]], pt;
91
  token_cache[scan[idx]] = av1_pt_energy_class[token];
Jingning Han's avatar
Jingning Han committed
92 93 94 95 96
  pt = get_coef_context(nb, token_cache, idx + 1);
  token_cache[scan[idx]] = bak;
  return pt;
}

97 98
static int optimize_b(const AV1_COMMON *const cm, MACROBLOCK *mb, int plane,
                      int block, TX_SIZE tx_size, int ctx) {
Jingning Han's avatar
Jingning Han committed
99 100 101 102
  MACROBLOCKD *const xd = &mb->e_mbd;
  struct macroblock_plane *const p = &mb->plane[plane];
  struct macroblockd_plane *const pd = &xd->plane[plane];
  const int ref = is_inter_block(&xd->mi[0]->mbmi);
103
  av1_token_state tokens[1025][2];
Jingning Han's avatar
Jingning Han committed
104 105 106 107 108 109 110
  unsigned best_index[1025][2];
  uint8_t token_cache[1024];
  const tran_low_t *const coeff = BLOCK_OFFSET(mb->plane[plane].coeff, block);
  tran_low_t *const qcoeff = BLOCK_OFFSET(p->qcoeff, block);
  tran_low_t *const dqcoeff = BLOCK_OFFSET(pd->dqcoeff, block);
  const int eob = p->eobs[block];
  const PLANE_TYPE type = pd->plane_type;
111
  const int default_eob = 1 << (tx_size_1d_log2[tx_size] * 2);
Jingning Han's avatar
Jingning Han committed
112
  const int mul = 1 + (tx_size == TX_32X32);
113 114 115 116 117
#if CONFIG_AOM_QM
  int seg_id = xd->mi[0]->mbmi.segment_id;
  int is_intra = !is_inter_block(&xd->mi[0]->mbmi);
  const qm_val_t *iqmatrix = pd->seg_iqmatrix[seg_id][is_intra][tx_size];
#endif
Jingning Han's avatar
Jingning Han committed
118 119
  const int16_t *dequant_ptr = pd->dequant;
  const uint8_t *const band_translate = get_band_translate(tx_size);
hui su's avatar
hui su committed
120
  TX_TYPE tx_type = get_tx_type(type, xd, block);
Angie Chiang's avatar
Angie Chiang committed
121
  const SCAN_ORDER *const scan_order = get_scan(cm, tx_size, tx_type);
122 123
  const int16_t *const scan = scan_order->scan;
  const int16_t *const nb = scan_order->neighbors;
Jingning Han's avatar
Jingning Han committed
124 125 126 127 128 129 130
  int next = eob, sz = 0;
  int64_t rdmult = mb->rdmult * plane_rd_mult[type], rddiv = mb->rddiv;
  int64_t rd_cost0, rd_cost1;
  int rate0, rate1, error0, error1;
  int16_t t0, t1;
  EXTRABIT e0;
  int best, band, pt, i, final_eob;
131 132
#if CONFIG_AOM_HIGHBITDEPTH
  const int *cat6_high_cost = av1_get_high_cost_table(xd->bd);
Jingning Han's avatar
Jingning Han committed
133
#else
134
  const int *cat6_high_cost = av1_get_high_cost_table(8);
Jingning Han's avatar
Jingning Han committed
135 136 137 138 139 140
#endif

  assert((!type && !plane) || (type && plane));
  assert(eob <= default_eob);

  /* Now set up a Viterbi trellis to evaluate alternative roundings. */
clang-format's avatar
clang-format committed
141
  if (!ref) rdmult = (rdmult * 9) >> 4;
Jingning Han's avatar
Jingning Han committed
142 143 144 145 146 147 148 149 150 151

  /* Initialize the sentinel node of the trellis. */
  tokens[eob][0].rate = 0;
  tokens[eob][0].error = 0;
  tokens[eob][0].next = default_eob;
  tokens[eob][0].token = EOB_TOKEN;
  tokens[eob][0].qc = 0;
  tokens[eob][1] = tokens[eob][0];

  for (i = 0; i < eob; i++)
clang-format's avatar
clang-format committed
152
    token_cache[scan[i]] = av1_pt_energy_class[av1_get_token(qcoeff[scan[i]])];
Jingning Han's avatar
Jingning Han committed
153 154 155

  for (i = eob; i-- > 0;) {
    int base_bits, d2, dx;
156

Jingning Han's avatar
Jingning Han committed
157
    const int rc = scan[i];
158 159 160
#if CONFIG_AOM_QM
    int iwt = iqmatrix[rc];
#endif
Jingning Han's avatar
Jingning Han committed
161 162 163 164 165 166 167 168 169
    int x = qcoeff[rc];
    /* Only add a trellis state for non-zero coefficients. */
    if (x) {
      int shortcut = 0;
      error0 = tokens[next][0].error;
      error1 = tokens[next][1].error;
      /* Evaluate the first possibility for this state. */
      rate0 = tokens[next][0].rate;
      rate1 = tokens[next][1].rate;
170
      av1_get_token_extra(x, &t0, &e0);
Jingning Han's avatar
Jingning Han committed
171 172 173 174
      /* Consider both possible successor states. */
      if (next < default_eob) {
        band = band_translate[i + 1];
        pt = trellis_get_coeff_context(scan, nb, i, t0, token_cache);
clang-format's avatar
clang-format committed
175 176 177 178
        rate0 += mb->token_costs[tx_size][type][ref][band][0][pt]
                                [tokens[next][0].token];
        rate1 += mb->token_costs[tx_size][type][ref][band][0][pt]
                                [tokens[next][1].token];
Jingning Han's avatar
Jingning Han committed
179 180 181 182
      }
      UPDATE_RD_COST();
      /* And pick the best. */
      best = rd_cost1 < rd_cost0;
183
      base_bits = av1_get_cost(t0, e0, cat6_high_cost);
Jingning Han's avatar
Jingning Han committed
184
      dx = mul * (dqcoeff[rc] - coeff[rc]);
185
#if CONFIG_AOM_HIGHBITDEPTH
Jingning Han's avatar
Jingning Han committed
186 187 188
      if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) {
        dx >>= xd->bd - 8;
      }
189
#endif  // CONFIG_AOM_HIGHBITDEPTH
Jingning Han's avatar
Jingning Han committed
190 191 192 193 194 195 196 197 198 199 200 201
      d2 = dx * dx;
      tokens[i][0].rate = base_bits + (best ? rate1 : rate0);
      tokens[i][0].error = d2 + (best ? error1 : error0);
      tokens[i][0].next = next;
      tokens[i][0].token = t0;
      tokens[i][0].qc = x;
      best_index[i][0] = best;

      /* Evaluate the second possibility for this state. */
      rate0 = tokens[next][0].rate;
      rate1 = tokens[next][1].rate;

202 203 204 205 206 207
#if CONFIG_AOM_QM
      if ((abs(x) * dequant_ptr[rc != 0] * iwt >
           ((abs(coeff[rc]) * mul) << AOM_QM_BITS)) &&
          (abs(x) * dequant_ptr[rc != 0] * iwt <
           ((abs(coeff[rc]) * mul + dequant_ptr[rc != 0]) << AOM_QM_BITS)))
#else
Jingning Han's avatar
Jingning Han committed
208
      if ((abs(x) * dequant_ptr[rc != 0] > abs(coeff[rc]) * mul) &&
clang-format's avatar
clang-format committed
209 210
          (abs(x) * dequant_ptr[rc != 0] <
           abs(coeff[rc]) * mul + dequant_ptr[rc != 0]))
211
#endif
Jingning Han's avatar
Jingning Han committed
212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229
        shortcut = 1;
      else
        shortcut = 0;

      if (shortcut) {
        sz = -(x < 0);
        x -= 2 * sz + 1;
      }

      /* Consider both possible successor states. */
      if (!x) {
        /* If we reduced this coefficient to zero, check to see if
         *  we need to move the EOB back here.
         */
        t0 = tokens[next][0].token == EOB_TOKEN ? EOB_TOKEN : ZERO_TOKEN;
        t1 = tokens[next][1].token == EOB_TOKEN ? EOB_TOKEN : ZERO_TOKEN;
        e0 = 0;
      } else {
230
        av1_get_token_extra(x, &t0, &e0);
Jingning Han's avatar
Jingning Han committed
231 232 233 234 235 236
        t1 = t0;
      }
      if (next < default_eob) {
        band = band_translate[i + 1];
        if (t0 != EOB_TOKEN) {
          pt = trellis_get_coeff_context(scan, nb, i, t0, token_cache);
clang-format's avatar
clang-format committed
237 238
          rate0 += mb->token_costs[tx_size][type][ref][band][!x][pt]
                                  [tokens[next][0].token];
Jingning Han's avatar
Jingning Han committed
239 240 241
        }
        if (t1 != EOB_TOKEN) {
          pt = trellis_get_coeff_context(scan, nb, i, t1, token_cache);
clang-format's avatar
clang-format committed
242 243
          rate1 += mb->token_costs[tx_size][type][ref][band][!x][pt]
                                  [tokens[next][1].token];
Jingning Han's avatar
Jingning Han committed
244 245 246 247 248 249
        }
      }

      UPDATE_RD_COST();
      /* And pick the best. */
      best = rd_cost1 < rd_cost0;
250
      base_bits = av1_get_cost(t0, e0, cat6_high_cost);
Jingning Han's avatar
Jingning Han committed
251 252

      if (shortcut) {
253
#if CONFIG_AOM_HIGHBITDEPTH
Jingning Han's avatar
Jingning Han committed
254 255 256 257 258 259 260
        if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) {
          dx -= ((dequant_ptr[rc != 0] >> (xd->bd - 8)) + sz) ^ sz;
        } else {
          dx -= (dequant_ptr[rc != 0] + sz) ^ sz;
        }
#else
        dx -= (dequant_ptr[rc != 0] + sz) ^ sz;
261
#endif  // CONFIG_AOM_HIGHBITDEPTH
Jingning Han's avatar
Jingning Han committed
262 263
        d2 = dx * dx;
      }
264

Jingning Han's avatar
Jingning Han committed
265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308
      tokens[i][1].rate = base_bits + (best ? rate1 : rate0);
      tokens[i][1].error = d2 + (best ? error1 : error0);
      tokens[i][1].next = next;
      tokens[i][1].token = best ? t1 : t0;
      tokens[i][1].qc = x;
      best_index[i][1] = best;
      /* Finally, make this the new head of the trellis. */
      next = i;
    } else {
      /* There's no choice to make for a zero coefficient, so we don't
       *  add a new trellis node, but we do need to update the costs.
       */
      band = band_translate[i + 1];
      t0 = tokens[next][0].token;
      t1 = tokens[next][1].token;
      /* Update the cost of each path if we're past the EOB token. */
      if (t0 != EOB_TOKEN) {
        tokens[next][0].rate +=
            mb->token_costs[tx_size][type][ref][band][1][0][t0];
        tokens[next][0].token = ZERO_TOKEN;
      }
      if (t1 != EOB_TOKEN) {
        tokens[next][1].rate +=
            mb->token_costs[tx_size][type][ref][band][1][0][t1];
        tokens[next][1].token = ZERO_TOKEN;
      }
      best_index[i][0] = best_index[i][1] = 0;
      /* Don't update next, because we didn't add a new node. */
    }
  }

  /* Now pick the best path through the whole trellis. */
  band = band_translate[i + 1];
  rate0 = tokens[next][0].rate;
  rate1 = tokens[next][1].rate;
  error0 = tokens[next][0].error;
  error1 = tokens[next][1].error;
  t0 = tokens[next][0].token;
  t1 = tokens[next][1].token;
  rate0 += mb->token_costs[tx_size][type][ref][band][0][ctx][t0];
  rate1 += mb->token_costs[tx_size][type][ref][band][0][ctx][t1];
  UPDATE_RD_COST();
  best = rd_cost1 < rd_cost0;
  final_eob = -1;
309 310
  memset(qcoeff, 0, sizeof(*qcoeff) * default_eob);
  memset(dqcoeff, 0, sizeof(*dqcoeff) * default_eob);
Jingning Han's avatar
Jingning Han committed
311 312 313
  for (i = next; i < eob; i = next) {
    const int x = tokens[i][best].qc;
    const int rc = scan[i];
314 315 316 317 318
#if CONFIG_AOM_QM
    const int iwt = iqmatrix[rc];
    const int dequant =
        (dequant_ptr[rc != 0] * iwt + (1 << (AOM_QM_BITS - 1))) >> AOM_QM_BITS;
#endif
Jingning Han's avatar
Jingning Han committed
319 320 321 322 323
    if (x) {
      final_eob = i;
    }

    qcoeff[rc] = x;
324 325 326
#if CONFIG_AOM_QM
    dqcoeff[rc] = (x * dequant) / mul;
#else
Jingning Han's avatar
Jingning Han committed
327
    dqcoeff[rc] = (x * dequant_ptr[rc != 0]) / mul;
328
#endif
Jingning Han's avatar
Jingning Han committed
329 330 331 332 333 334 335 336 337

    next = tokens[i][best].next;
    best = best_index[i][best];
  }
  final_eob++;

  mb->plane[plane].eobs[block] = final_eob;
  return final_eob;
}
338
#endif
Jingning Han's avatar
Jingning Han committed
339

340 341
// TODO(sarahparker) refactor fwd quant functions to use fwd_txfm fns in
// hybrid_fwd_txfm.c
342 343 344
void av1_xform_quant_fp(const AV1_COMMON *const cm, MACROBLOCK *x, int plane,
                        int block, int blk_row, int blk_col,
                        BLOCK_SIZE plane_bsize, TX_SIZE tx_size) {
Jingning Han's avatar
Jingning Han committed
345
  MACROBLOCKD *const xd = &x->e_mbd;
346
#if !CONFIG_PVQ
Jingning Han's avatar
Jingning Han committed
347 348
  const struct macroblock_plane *const p = &x->plane[plane];
  const struct macroblockd_plane *const pd = &xd->plane[plane];
349 350 351 352
#else
  struct macroblock_plane *const p = &x->plane[plane];
  struct macroblockd_plane *const pd = &xd->plane[plane];
#endif
hui su's avatar
hui su committed
353 354
  PLANE_TYPE plane_type = (plane == 0) ? PLANE_TYPE_Y : PLANE_TYPE_UV;
  TX_TYPE tx_type = get_tx_type(plane_type, xd, block);
Angie Chiang's avatar
Angie Chiang committed
355
  const SCAN_ORDER *const scan_order = get_scan(cm, tx_size, tx_type);
Jingning Han's avatar
Jingning Han committed
356 357 358 359 360
  tran_low_t *const coeff = BLOCK_OFFSET(p->coeff, block);
  tran_low_t *const qcoeff = BLOCK_OFFSET(p->qcoeff, block);
  tran_low_t *const dqcoeff = BLOCK_OFFSET(pd->dqcoeff, block);
  uint16_t *const eob = &p->eobs[block];
  const int diff_stride = 4 * num_4x4_blocks_wide_lookup[plane_bsize];
361
  int seg_id = xd->mi[0]->mbmi.segment_id;
362
#if CONFIG_AOM_QM
363 364 365 366
  int is_intra = !is_inter_block(&xd->mi[0]->mbmi);
  const qm_val_t *qmatrix = pd->seg_qmatrix[seg_id][is_intra][tx_size];
  const qm_val_t *iqmatrix = pd->seg_iqmatrix[seg_id][is_intra][tx_size];
#endif
367
#if !CONFIG_PVQ
Jingning Han's avatar
Jingning Han committed
368
  const int16_t *src_diff;
369
  (void)cm;
370 371 372 373 374 375 376 377 378 379

  /*
    FWD_TXFM_PARAM fwd_txfm_param;
    fwd_txfm_param.tx_type = tx_type;
    fwd_txfm_param.tx_size = tx_size;
    fwd_txfm_param.fwd_txfm_opt = FWD_TXFM_OPT_NORMAL;
    fwd_txfm_param.rd_transform = x->use_lp32x32fdct;
    fwd_txfm_param.lossless = xd->lossless[xd->mi[0]->mbmi.segment_id];
  */

380
  src_diff = &p->src_diff[4 * (blk_row * diff_stride + blk_col)];
381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414
#else
  MB_MODE_INFO *mbmi = &xd->mi[0]->mbmi;
  tran_low_t *ref_coeff = BLOCK_OFFSET(pd->pvq_ref_coeff, block);
  uint8_t *src, *dst;
  int16_t *src_int16, *pred;
  const int src_stride = p->src.stride;
  const int dst_stride = pd->dst.stride;
  int tx_blk_size;
  int i, j;
  int skip = 1;
  PVQ_INFO *pvq_info = NULL;

  (void)scan_order;
  (void)qcoeff;

  if (x->pvq_coded) {
    assert(block < MAX_PVQ_BLOCKS_IN_SB);
    pvq_info = &x->pvq[block][plane];
  }
  dst = &pd->dst.buf[4 * (blk_row * dst_stride + blk_col)];
  src = &p->src.buf[4 * (blk_row * src_stride + blk_col)];
  src_int16 = &p->src_int16[4 * (blk_row * diff_stride + blk_col)];
  pred = &pd->pred[4 * (blk_row * diff_stride + blk_col)];
  // transform block size in pixels
  tx_blk_size = tx_size_1d[tx_size];

  // copy uint8 orig and predicted block to int16 buffer
  // in order to use existing VP10 transform functions
  for (j = 0; j < tx_blk_size; j++)
    for (i = 0; i < tx_blk_size; i++) {
      src_int16[diff_stride * j + i] = src[src_stride * j + i];
      pred[diff_stride * j + i] = dst[dst_stride * j + i];
    }
#endif
Jingning Han's avatar
Jingning Han committed
415

416
#if CONFIG_AOM_HIGHBITDEPTH
Jingning Han's avatar
Jingning Han committed
417 418 419 420
  if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) {
    switch (tx_size) {
      case TX_32X32:
        highbd_fdct32x32(x->use_lp32x32fdct, src_diff, coeff, diff_stride);
clang-format's avatar
clang-format committed
421 422 423
        av1_highbd_quantize_fp_32x32(
            coeff, 1024, x->skip_block, p->zbin, p->round_fp, p->quant_fp,
            p->quant_shift, qcoeff, dqcoeff, pd->dequant, eob, scan_order->scan,
424
#if !CONFIG_AOM_QM
clang-format's avatar
clang-format committed
425
            scan_order->iscan);
426
#else
clang-format's avatar
clang-format committed
427
            scan_order->iscan, qmatrix, iqmatrix);
428
#endif
Jingning Han's avatar
Jingning Han committed
429 430
        break;
      case TX_16X16:
Adrian Grange's avatar
Adrian Grange committed
431
        aom_highbd_fdct16x16(src_diff, coeff, diff_stride);
432
        av1_highbd_quantize_fp(coeff, 256, x->skip_block, p->zbin, p->round_fp,
clang-format's avatar
clang-format committed
433 434
                               p->quant_fp, p->quant_shift, qcoeff, dqcoeff,
                               pd->dequant, eob, scan_order->scan,
435
#if !CONFIG_AOM_QM
clang-format's avatar
clang-format committed
436
                               scan_order->iscan);
437
#else
clang-format's avatar
clang-format committed
438
                               scan_order->iscan, qmatrix, iqmatrix);
439
#endif
Jingning Han's avatar
Jingning Han committed
440 441
        break;
      case TX_8X8:
Adrian Grange's avatar
Adrian Grange committed
442
        aom_highbd_fdct8x8(src_diff, coeff, diff_stride);
443
        av1_highbd_quantize_fp(coeff, 64, x->skip_block, p->zbin, p->round_fp,
clang-format's avatar
clang-format committed
444 445
                               p->quant_fp, p->quant_shift, qcoeff, dqcoeff,
                               pd->dequant, eob, scan_order->scan,
446
#if !CONFIG_AOM_QM
clang-format's avatar
clang-format committed
447
                               scan_order->iscan);
448
#else
clang-format's avatar
clang-format committed
449
                               scan_order->iscan, qmatrix, iqmatrix);
450
#endif
Jingning Han's avatar
Jingning Han committed
451 452
        break;
      case TX_4X4:
453
        if (xd->lossless[seg_id]) {
454
          av1_highbd_fwht4x4(src_diff, coeff, diff_stride);
455
        } else {
Adrian Grange's avatar
Adrian Grange committed
456
          aom_highbd_fdct4x4(src_diff, coeff, diff_stride);
457
        }
458
        av1_highbd_quantize_fp(coeff, 16, x->skip_block, p->zbin, p->round_fp,
clang-format's avatar
clang-format committed
459 460
                               p->quant_fp, p->quant_shift, qcoeff, dqcoeff,
                               pd->dequant, eob, scan_order->scan,
461
#if !CONFIG_AOM_QM
clang-format's avatar
clang-format committed
462
                               scan_order->iscan);
463
#else
clang-format's avatar
clang-format committed
464
                               scan_order->iscan, qmatrix, iqmatrix);
465
#endif
Jingning Han's avatar
Jingning Han committed
466
        break;
clang-format's avatar
clang-format committed
467
      default: assert(0);
Jingning Han's avatar
Jingning Han committed
468 469 470
    }
    return;
  }
471
#endif  // CONFIG_AOM_HIGHBITDEPTH
Jingning Han's avatar
Jingning Han committed
472

473
#if !CONFIG_PVQ
Jingning Han's avatar
Jingning Han committed
474 475 476
  switch (tx_size) {
    case TX_32X32:
      fdct32x32(x->use_lp32x32fdct, src_diff, coeff, diff_stride);
477
      av1_quantize_fp_32x32(coeff, 1024, x->skip_block, p->zbin, p->round_fp,
clang-format's avatar
clang-format committed
478 479
                            p->quant_fp, p->quant_shift, qcoeff, dqcoeff,
                            pd->dequant, eob, scan_order->scan,
480
#if !CONFIG_AOM_QM
clang-format's avatar
clang-format committed
481
                            scan_order->iscan);
482
#else
clang-format's avatar
clang-format committed
483
                            scan_order->iscan, qmatrix, iqmatrix);
484
#endif
Jingning Han's avatar
Jingning Han committed
485 486
      break;
    case TX_16X16:
Adrian Grange's avatar
Adrian Grange committed
487
      aom_fdct16x16(src_diff, coeff, diff_stride);
488
      av1_quantize_fp(coeff, 256, x->skip_block, p->zbin, p->round_fp,
clang-format's avatar
clang-format committed
489 490
                      p->quant_fp, p->quant_shift, qcoeff, dqcoeff, pd->dequant,
                      eob, scan_order->scan,
491
#if !CONFIG_AOM_QM
clang-format's avatar
clang-format committed
492
                      scan_order->iscan);
493
#else
clang-format's avatar
clang-format committed
494
                      scan_order->iscan, qmatrix, iqmatrix);
495
#endif
Jingning Han's avatar
Jingning Han committed
496 497
      break;
    case TX_8X8:
498
      av1_fdct8x8_quant(src_diff, diff_stride, coeff, 64, x->skip_block,
clang-format's avatar
clang-format committed
499 500
                        p->zbin, p->round_fp, p->quant_fp, p->quant_shift,
                        qcoeff, dqcoeff, pd->dequant, eob, scan_order->scan,
501
#if !CONFIG_AOM_QM
clang-format's avatar
clang-format committed
502
                        scan_order->iscan);
503
#else
clang-format's avatar
clang-format committed
504
                        scan_order->iscan, qmatrix, iqmatrix);
505
#endif
Jingning Han's avatar
Jingning Han committed
506 507
      break;
    case TX_4X4:
508
      if (xd->lossless[seg_id]) {
509
        av1_fwht4x4(src_diff, coeff, diff_stride);
510
      } else {
Adrian Grange's avatar
Adrian Grange committed
511
        aom_fdct4x4(src_diff, coeff, diff_stride);
512
      }
513
      av1_quantize_fp(coeff, 16, x->skip_block, p->zbin, p->round_fp,
clang-format's avatar
clang-format committed
514 515
                      p->quant_fp, p->quant_shift, qcoeff, dqcoeff, pd->dequant,
                      eob, scan_order->scan,
516
#if !CONFIG_AOM_QM
clang-format's avatar
clang-format committed
517
                      scan_order->iscan);
518
#else
clang-format's avatar
clang-format committed
519
                      scan_order->iscan, qmatrix, iqmatrix);
520 521
#endif
      break;
clang-format's avatar
clang-format committed
522
    default: assert(0); break;
Jingning Han's avatar
Jingning Han committed
523
  }
524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572
#else   // #if !CONFIG_PVQ
  switch (tx_size) {
    case TX_32X32:
      // NOTE: Using x->use_lp32x32fdct == 1 will makes enc and dec mismatched,
      // because decoder always uses x->use_lp32x32fdct == 0,
      // forward transform of predicted image.
      fdct32x32(0, pred, ref_coeff, diff_stride);
      // forward transform of original image.
      fdct32x32(0, src_int16, coeff, diff_stride);
      break;
    case TX_16X16:
      aom_fdct16x16(pred, ref_coeff, diff_stride);
      aom_fdct16x16(src_int16, coeff, diff_stride);
      break;
    case TX_8X8:
      aom_fdct8x8(pred, ref_coeff, diff_stride);
      aom_fdct8x8(src_int16, coeff, diff_stride);
      break;
    case TX_4X4:
      if (xd->lossless[seg_id]) {
        av1_fwht4x4(pred, ref_coeff, diff_stride);
        av1_fwht4x4(src_int16, coeff, diff_stride);
      } else {
        aom_fdct4x4(pred, ref_coeff, diff_stride);
        aom_fdct4x4(src_int16, coeff, diff_stride);
      }
      break;
    default: assert(0); break;
  }

  // PVQ for inter mode block
  if (!x->skip_block)
    skip = av1_pvq_encode_helper(&x->daala_enc,
                                 coeff,        // target original vector
                                 ref_coeff,    // reference vector
                                 dqcoeff,      // de-quantized vector
                                 eob,          // End of Block marker
                                 pd->dequant,  // aom's quantizers
                                 plane,        // image plane
                                 tx_size,      // block size in log_2 - 2
                                 tx_type,
                                 &x->rate,  // rate measured
                                 x->pvq_speed,
                                 pvq_info);  // PVQ info for a block

  x->pvq_skip[plane] = skip;

  if (!skip) mbmi->skip = 0;
#endif  // #if !CONFIG_PVQ
Jingning Han's avatar
Jingning Han committed
573 574
}

575 576 577
void av1_xform_quant(const AV1_COMMON *const cm, MACROBLOCK *x, int plane,
                     int block, int blk_row, int blk_col,
                     BLOCK_SIZE plane_bsize, TX_SIZE tx_size) {
Jingning Han's avatar
Jingning Han committed
578
  MACROBLOCKD *const xd = &x->e_mbd;
579
#if !CONFIG_PVQ
Jingning Han's avatar
Jingning Han committed
580 581
  const struct macroblock_plane *const p = &x->plane[plane];
  const struct macroblockd_plane *const pd = &xd->plane[plane];
582 583 584 585
#else
  struct macroblock_plane *const p = &x->plane[plane];
  struct macroblockd_plane *const pd = &xd->plane[plane];
#endif
hui su's avatar
hui su committed
586 587
  PLANE_TYPE plane_type = (plane == 0) ? PLANE_TYPE_Y : PLANE_TYPE_UV;
  TX_TYPE tx_type = get_tx_type(plane_type, xd, block);
Angie Chiang's avatar
Angie Chiang committed
588
  const SCAN_ORDER *const scan_order = get_scan(cm, tx_size, tx_type);
Jingning Han's avatar
Jingning Han committed
589 590 591 592 593
  tran_low_t *const coeff = BLOCK_OFFSET(p->coeff, block);
  tran_low_t *const qcoeff = BLOCK_OFFSET(p->qcoeff, block);
  tran_low_t *const dqcoeff = BLOCK_OFFSET(pd->dqcoeff, block);
  uint16_t *const eob = &p->eobs[block];
  const int diff_stride = 4 * num_4x4_blocks_wide_lookup[plane_bsize];
594
  int seg_id = xd->mi[0]->mbmi.segment_id;
595 596
  FWD_TXFM_PARAM fwd_txfm_param;

597 598 599 600 601
#if CONFIG_AOM_QM
  int is_intra = !is_inter_block(&xd->mi[0]->mbmi);
  const qm_val_t *qmatrix = pd->seg_qmatrix[seg_id][is_intra][tx_size];
  const qm_val_t *iqmatrix = pd->seg_iqmatrix[seg_id][is_intra][tx_size];
#endif
602 603

#if !CONFIG_PVQ
Jingning Han's avatar
Jingning Han committed
604
  const int16_t *src_diff;
605

606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642
  src_diff = &p->src_diff[4 * (blk_row * diff_stride + blk_col)];
#else
  MB_MODE_INFO *mbmi = &xd->mi[0]->mbmi;
  tran_low_t *ref_coeff = BLOCK_OFFSET(pd->pvq_ref_coeff, block);
  uint8_t *src, *dst;
  int16_t *src_int16, *pred;
  const int src_stride = p->src.stride;
  const int dst_stride = pd->dst.stride;
  int tx_blk_size;
  int i, j;
  int skip = 1;
  PVQ_INFO *pvq_info = NULL;

  (void)scan_order;
  (void)qcoeff;

  if (x->pvq_coded) {
    assert(block < MAX_PVQ_BLOCKS_IN_SB);
    pvq_info = &x->pvq[block][plane];
  }
  dst = &pd->dst.buf[4 * (blk_row * dst_stride + blk_col)];
  src = &p->src.buf[4 * (blk_row * src_stride + blk_col)];
  src_int16 = &p->src_int16[4 * (blk_row * diff_stride + blk_col)];
  pred = &pd->pred[4 * (blk_row * diff_stride + blk_col)];

  // transform block size in pixels
  tx_blk_size = tx_size_1d[tx_size];

  // copy uint8 orig and predicted block to int16 buffer
  // in order to use existing VP10 transform functions
  for (j = 0; j < tx_blk_size; j++)
    for (i = 0; i < tx_blk_size; i++) {
      src_int16[diff_stride * j + i] = src[src_stride * j + i];
      pred[diff_stride * j + i] = dst[dst_stride * j + i];
    }
#endif

643 644 645 646 647 648
  fwd_txfm_param.tx_type = tx_type;
  fwd_txfm_param.tx_size = tx_size;
  fwd_txfm_param.fwd_txfm_opt = FWD_TXFM_OPT_NORMAL;
  fwd_txfm_param.rd_transform = x->use_lp32x32fdct;
  fwd_txfm_param.lossless = xd->lossless[seg_id];

649
#if CONFIG_AOM_HIGHBITDEPTH
Jingning Han's avatar
Jingning Han committed
650
  if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) {
651
    highbd_fwd_txfm(src_diff, coeff, diff_stride, &fwd_txfm_param);
clang-format's avatar
clang-format committed
652
    switch (tx_size) {
Jingning Han's avatar
Jingning Han committed
653
      case TX_32X32:
Adrian Grange's avatar
Adrian Grange committed
654
        aom_highbd_quantize_b_32x32(coeff, 1024, x->skip_block, p->zbin,
Jingning Han's avatar
Jingning Han committed
655
                                    p->round, p->quant, p->quant_shift, qcoeff,
clang-format's avatar
clang-format committed
656
                                    dqcoeff, pd->dequant, eob, scan_order->scan,
657
#if !CONFIG_AOM_QM
clang-format's avatar
clang-format committed
658
                                    scan_order->iscan);
659 660 661
#else
                                    scan_order->iscan, qmatrix, iqmatrix);
#endif
Jingning Han's avatar
Jingning Han committed
662 663
        break;
      case TX_16X16:
Adrian Grange's avatar
Adrian Grange committed
664
        aom_highbd_quantize_b(coeff, 256, x->skip_block, p->zbin, p->round,
Jingning Han's avatar
Jingning Han committed
665
                              p->quant, p->quant_shift, qcoeff, dqcoeff,
clang-format's avatar
clang-format committed
666
                              pd->dequant, eob, scan_order->scan,
667
#if !CONFIG_AOM_QM
clang-format's avatar
clang-format committed
668
                              scan_order->iscan);
669 670 671
#else
                              scan_order->iscan, qmatrix, iqmatrix);
#endif
Jingning Han's avatar
Jingning Han committed
672 673
        break;
      case TX_8X8:
Adrian Grange's avatar
Adrian Grange committed
674
        aom_highbd_quantize_b(coeff, 64, x->skip_block, p->zbin, p->round,
Jingning Han's avatar
Jingning Han committed
675
                              p->quant, p->quant_shift, qcoeff, dqcoeff,
clang-format's avatar
clang-format committed
676
                              pd->dequant, eob, scan_order->scan,
677
#if !CONFIG_AOM_QM
clang-format's avatar
clang-format committed
678
                              scan_order->iscan);
679 680 681
#else
                              scan_order->iscan, qmatrix, iqmatrix);
#endif
Jingning Han's avatar
Jingning Han committed
682 683
        break;
      case TX_4X4:
Adrian Grange's avatar
Adrian Grange committed
684
        aom_highbd_quantize_b(coeff, 16, x->skip_block, p->zbin, p->round,
Jingning Han's avatar
Jingning Han committed
685
                              p->quant, p->quant_shift, qcoeff, dqcoeff,
clang-format's avatar
clang-format committed
686
                              pd->dequant, eob, scan_order->scan,
687
#if !CONFIG_AOM_QM
clang-format's avatar
clang-format committed
688
                              scan_order->iscan);
689 690 691
#else
                              scan_order->iscan, qmatrix, iqmatrix);
#endif
Jingning Han's avatar
Jingning Han committed
692
        break;
clang-format's avatar
clang-format committed
693
      default: assert(0);
Jingning Han's avatar
Jingning Han committed
694 695 696
    }
    return;
  }
697
#endif  // CONFIG_AOM_HIGHBITDEPTH
Jingning Han's avatar
Jingning Han committed
698

699
#if !CONFIG_PVQ
700
  fwd_txfm(src_diff, coeff, diff_stride, &fwd_txfm_param);
Jingning Han's avatar
Jingning Han committed
701 702
  switch (tx_size) {
    case TX_32X32:
Adrian Grange's avatar
Adrian Grange committed
703
      aom_quantize_b_32x32(coeff, 1024, x->skip_block, p->zbin, p->round,
Jingning Han's avatar
Jingning Han committed
704 705
                           p->quant, p->quant_shift, qcoeff, dqcoeff,
                           pd->dequant, eob, scan_order->scan,
706
#if !CONFIG_AOM_QM
Jingning Han's avatar
Jingning Han committed
707
                           scan_order->iscan);
708 709 710
#else
                           scan_order->iscan, qmatrix, iqmatrix);
#endif
Jingning Han's avatar
Jingning Han committed
711 712
      break;
    case TX_16X16:
Adrian Grange's avatar
Adrian Grange committed
713
      aom_quantize_b(coeff, 256, x->skip_block, p->zbin, p->round, p->quant,
clang-format's avatar
clang-format committed
714
                     p->quant_shift, qcoeff, dqcoeff, pd->dequant, eob,
715 716 717 718 719 720
                     scan_order->scan,
#if !CONFIG_AOM_QM
                     scan_order->iscan);
#else
                     scan_order->iscan, qmatrix, iqmatrix);
#endif
Jingning Han's avatar
Jingning Han committed
721 722
      break;
    case TX_8X8:
Adrian Grange's avatar
Adrian Grange committed
723
      aom_quantize_b(coeff, 64, x->skip_block, p->zbin, p->round, p->quant,
clang-format's avatar
clang-format committed
724
                     p->quant_shift, qcoeff, dqcoeff, pd->dequant, eob,
725 726 727 728 729 730
                     scan_order->scan,
#if !CONFIG_AOM_QM
                     scan_order->iscan);
#else
                     scan_order->iscan, qmatrix, iqmatrix);
#endif
Jingning Han's avatar
Jingning Han committed
731 732
      break;
    case TX_4X4:
Adrian Grange's avatar
Adrian Grange committed
733
      aom_quantize_b(coeff, 16, x->skip_block, p->zbin, p->round, p->quant,
clang-format's avatar
clang-format committed
734
                     p->quant_shift, qcoeff, dqcoeff, pd->dequant, eob,
735 736 737 738 739 740 741
                     scan_order->scan,
#if !CONFIG_AOM_QM
                     scan_order->iscan);
#else
                     scan_order->iscan, qmatrix, iqmatrix);
#endif
      break;
clang-format's avatar
clang-format committed
742
    default: assert(0); break;
Jingning Han's avatar
Jingning Han committed
743
  }
744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768
#else   // #if !CONFIG_PVQ
  fwd_txfm_param.rd_transform = 0;

  fwd_txfm(src_int16, coeff, diff_stride, &fwd_txfm_param);
  fwd_txfm(pred, ref_coeff, diff_stride, &fwd_txfm_param);

  // PVQ for inter mode block
  if (!x->skip_block)
    skip = av1_pvq_encode_helper(&x->daala_enc,
                                 coeff,        // target original vector
                                 ref_coeff,    // reference vector
                                 dqcoeff,      // de-quantized vector
                                 eob,          // End of Block marker
                                 pd->dequant,  // aom's quantizers
                                 plane,        // image plane
                                 tx_size,      // block size in log_2 - 2
                                 tx_type,
                                 &x->rate,  // rate measured
                                 x->pvq_speed,
                                 pvq_info);  // PVQ info for a block

  x->pvq_skip[plane] = skip;

  if (!skip) mbmi->skip = 0;
#endif  // #if !CONFIG_PVQ
Jingning Han's avatar
Jingning Han committed
769 770
}

771
static void encode_block(int plane, int block, int blk_row, int blk_col,
clang-format's avatar
clang-format committed
772
                         BLOCK_SIZE plane_bsize, TX_SIZE tx_size, void *arg) {
Jingning Han's avatar
Jingning Han committed
773
  struct encode_b_args *const args = arg;
774
  const AV1_COMMON *const cm = args->cm;
Jingning Han's avatar
Jingning Han committed
775 776 777 778 779 780 781 782
  MACROBLOCK *const x = args->x;
  MACROBLOCKD *const xd = &x->e_mbd;
  struct optimize_ctx *const ctx = args->ctx;
  struct macroblock_plane *const p = &x->plane[plane];
  struct macroblockd_plane *const pd = &xd->plane[plane];
  tran_low_t *const dqcoeff = BLOCK_OFFSET(pd->dqcoeff, block);
  uint8_t *dst;
  ENTROPY_CONTEXT *a, *l;
hui su's avatar
hui su committed
783
  TX_TYPE tx_type = get_tx_type(pd->plane_type, xd, block);
784 785 786 787
#if CONFIG_PVQ
  int tx_blk_size;
  int i, j;
#endif
788 789 790
  dst = &pd->dst.buf[4 * blk_row * pd->dst.stride + 4 * blk_col];
  a = &ctx->ta[plane][blk_col];
  l = &ctx->tl[plane][blk_row];
Jingning Han's avatar
Jingning Han committed
791

792
  if (x->quant_fp) {
793 794
    av1_xform_quant_fp(cm, x, plane, block, blk_row, blk_col, plane_bsize,
                       tx_size);
795
  } else {
796 797
    av1_xform_quant(cm, x, plane, block, blk_row, blk_col, plane_bsize,
                    tx_size);
Jingning Han's avatar
Jingning Han committed
798 799
  }

800
#if !CONFIG_PVQ
801
  if (x->optimize) {
802
    const int combined_ctx = combine_entropy_contexts(*a, *l);
803
    *a = *l = optimize_b(cm, x, plane, block, tx_size, combined_ctx) > 0;
Jingning Han's avatar
Jingning Han committed
804 805 806 807
  } else {
    *a = *l = p->eobs[block] > 0;
  }

clang-format's avatar
clang-format committed
808
  if (p->eobs[block]) *(args->skip) = 0;
Jingning Han's avatar
Jingning Han committed
809

clang-format's avatar
clang-format committed
810
  if (p->eobs[block] == 0) return;
811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828
#else
  *a = *l = !x->pvq_skip[plane];

  if (!x->pvq_skip[plane]) *(args->skip) = 0;

  if (x->pvq_skip[plane]) return;

  // transform block size in pixels
  tx_blk_size = tx_size_1d[tx_size];

  // Since av1 does not have separate function which does inverse transform
  // but av1_inv_txfm_add_*x*() also does addition of predicted image to
  // inverse transformed image,
  // pass blank dummy image to av1_inv_txfm_add_*x*(), i.e. set dst as zeros
  for (j = 0; j < tx_blk_size; j++)
    for (i = 0; i < tx_blk_size; i++) dst[j * pd->dst.stride + i] = 0;
#endif

829
#if CONFIG_AOM_HIGHBITDEPTH
Jingning Han's avatar
Jingning Han committed
830 831 832
  if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) {
    switch (tx_size) {
      case TX_32X32:
833
        av1_highbd_inv_txfm_add_32x32(dqcoeff, dst, pd->dst.stride,
clang-format's avatar
clang-format committed
834
                                      p->eobs[block], xd->bd, tx_type);
Jingning Han's avatar
Jingning Han committed
835 836
        break;
      case TX_16X16:
837
        av1_highbd_inv_txfm_add_16x16(dqcoeff, dst, pd->dst.stride,
clang-format's avatar
clang-format committed
838
                                      p->eobs[block], xd->bd, tx_type);
Jingning Han's avatar
Jingning Han committed
839 840
        break;
      case TX_8X8:
841
        av1_highbd_inv_txfm_add_8x8(dqcoeff, dst, pd->dst.stride,
clang-format's avatar
clang-format committed
842
                                    p->eobs[block], xd->bd, tx_type);
Jingning Han's avatar
Jingning Han committed
843 844
        break;
      case TX_4X4:
845
        // this is like av1_short_idct4x4 but has a special case around eob<=1
Jingning Han's avatar
Jingning Han committed
846 847
        // which is significant (not just an optimization) for the lossless
        // case.
848
        av1_highbd_inv_txfm_add_4x4(dqcoeff, dst, pd->dst.stride,
clang-format's avatar
clang-format committed
849 850
                                    p->eobs[block], xd->bd, tx_type,
                                    xd->lossless[xd->mi[0]->mbmi.segment_id]);
851
        break;
clang-format's avatar
clang-format committed
852
      default: assert(0 && "Invalid transform size"); break;
Jingning Han's avatar
Jingning Han committed
853
    }
hui su's avatar
hui su committed
854

Jingning Han's avatar
Jingning Han committed
855 856
    return;
  }
857
#endif  // CONFIG_AOM_HIGHBITDEPTH
Jingning Han's avatar
Jingning Han committed
858 859
  switch (tx_size) {
    case TX_32X32:
860
      av1_inv_txfm_add_32x32(dqcoeff, dst, pd->dst.stride, p->eobs[block],
clang-format's avatar
clang-format committed
861
                             tx_type);
Jingning Han's avatar
Jingning Han committed
862 863
      break;
    case TX_16X16:
864
      av1_inv_txfm_add_16x16(dqcoeff, dst, pd->dst.stride, p->eobs[block],
clang-format's avatar
clang-format committed
865
                             tx_type);
Jingning Han's avatar
Jingning Han committed
866 867
      break;
    case TX_8X8:
868
      av1_inv_txfm_add_8x8(dqcoeff, dst, pd->dst.stride, p->eobs[block],
clang-format's avatar
clang-format committed
869
                           tx_type);
Jingning Han's avatar
Jingning Han committed
870 871
      break;
    case TX_4X4:
872
      // this is like av1_short_idct4x4 but has a special case around eob<=1
Jingning Han's avatar
Jingning Han committed
873 874
      // which is significant (not just an optimization) for the lossless
      // case.
875
      av1_inv_txfm_add_4x4(dqcoeff, dst, pd->dst.stride, p->eobs[block],
clang-format's avatar
clang-format committed
876
                           tx_type, xd->lossless[xd->mi[0]->mbmi.segment_id]);
877
      break;
clang-format's avatar
clang-format committed
878
    default: assert(0 && "Invalid transform size"); break;
Jingning Han's avatar
Jingning Han committed
879 880 881
  }
}

882 883 884 885 886
typedef struct encode_block_pass1_args {
  AV1_COMMON *cm;
  MACROBLOCK *x;
} encode_block_pass1_args;

887
static void encode_block_pass1(int plane, int block, int blk_row, int blk_col,
clang-format's avatar
clang-format committed
888 889
                               BLOCK_SIZE plane_bsize, TX_SIZE tx_size,
                               void *arg) {
890 891 892
  encode_block_pass1_args *args = (encode_block_pass1_args *)arg;
  AV1_COMMON *cm = args->cm;
  MACROBLOCK *const x = args->x;
Jingning Han's avatar
Jingning Han committed
893 894 895 896 897
  MACROBLOCKD *const xd = &x->e_mbd;
  struct macroblock_plane *const p = &x->plane[plane];
  struct macroblockd_plane *const pd = &xd->plane[plane];
  tran_low_t *const dqcoeff = BLOCK_OFFSET(pd->dqcoeff, block);
  uint8_t *dst;
898
  dst = &pd->dst.buf[4 * blk_row * pd->dst.stride + 4 * blk_col];
Jingning Han's avatar
Jingning Han committed
899

900
  av1_xform_quant(cm, x, plane, block, blk_row, blk_col, plane_bsize, tx_size);
Jingning Han's avatar
Jingning Han committed
901

902
#if !CONFIG_PVQ
Jingning Han's avatar
Jingning Han committed
903
  if (p->eobs[block] > 0) {
904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923
#else
  if (!x->pvq_skip[plane]) {
#endif

#if CONFIG_PVQ
    {
      int tx_blk_size;
      int i, j;
      // transform block size in pixels
      tx_blk_size = tx_size_1d[tx_size];

      // Since av1 does not have separate function which does inverse transform
      // but av1_inv_txfm_add_*x*() also does addition of predicted image to
      // inverse transformed image,
      // pass blank dummy image to av1_inv_txfm_add_*x*(), i.e. set dst as zeros
      for (j = 0; j < tx_blk_size; j++)
        for (i = 0; i < tx_blk_size; i++) dst[j * pd->dst.stride + i] = 0;
    }
#endif

924
#if CONFIG_AOM_HIGHBITDEPTH
Jingning Han's avatar
Jingning Han committed
925
    if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) {
926
      if (xd->lossless[0]) {
927
        av1_highbd_iwht4x4_add(dqcoeff, dst, pd->dst.stride, p->eobs[block],
clang-format's avatar
clang-format committed
928
                               xd->bd);