encodemb.c 45.7 KB
Newer Older
Jingning Han's avatar
Jingning Han committed
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18
/*
 *  Copyright (c) 2010 The WebM project authors. All Rights Reserved.
 *
 *  Use of this source code is governed by a BSD-style license
 *  that can be found in the LICENSE file in the root of the source
 *  tree. An additional intellectual property rights grant can be found
 *  in the file PATENTS.  All contributing project authors may
 *  be found in the AUTHORS file in the root of the source tree.
 */

#include "./vp10_rtcd.h"
#include "./vpx_config.h"
#include "./vpx_dsp_rtcd.h"

#include "vpx_dsp/quantize.h"
#include "vpx_mem/vpx_mem.h"
#include "vpx_ports/mem.h"

19 20 21 22
#include "vp10/common/idct.h"
#include "vp10/common/reconinter.h"
#include "vp10/common/reconintra.h"
#include "vp10/common/scan.h"
Jingning Han's avatar
Jingning Han committed
23

24 25 26
#include "vp10/encoder/encodemb.h"
#include "vp10/encoder/rd.h"
#include "vp10/encoder/tokenize.h"
Jingning Han's avatar
Jingning Han committed
27 28 29 30 31 32 33 34 35 36 37 38 39

struct optimize_ctx {
  ENTROPY_CONTEXT ta[MAX_MB_PLANE][16];
  ENTROPY_CONTEXT tl[MAX_MB_PLANE][16];
};

void vp10_subtract_plane(MACROBLOCK *x, BLOCK_SIZE bsize, int plane) {
  struct macroblock_plane *const p = &x->plane[plane];
  const struct macroblockd_plane *const pd = &x->e_mbd.plane[plane];
  const BLOCK_SIZE plane_bsize = get_plane_block_size(bsize, pd);
  const int bw = 4 * num_4x4_blocks_wide_lookup[plane_bsize];
  const int bh = 4 * num_4x4_blocks_high_lookup[plane_bsize];

40
#if CONFIG_VPX_HIGHBITDEPTH
Jingning Han's avatar
Jingning Han committed
41 42 43 44 45 46
  if (x->e_mbd.cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) {
    vpx_highbd_subtract_block(bh, bw, p->src_diff, bw, p->src.buf,
                              p->src.stride, pd->dst.buf, pd->dst.stride,
                              x->e_mbd.bd);
    return;
  }
47
#endif  // CONFIG_VPX_HIGHBITDEPTH
Jingning Han's avatar
Jingning Han committed
48 49 50 51 52 53 54
  vpx_subtract_block(bh, bw, p->src_diff, bw, p->src.buf, p->src.stride,
                     pd->dst.buf, pd->dst.stride);
}

#define RDTRUNC(RM, DM, R, D) ((128 + (R) * (RM)) & 0xFF)

typedef struct vp10_token_state {
clang-format's avatar
clang-format committed
55 56 57 58 59
  int rate;
  int error;
  int next;
  int16_t token;
  short qc;
Jingning Han's avatar
Jingning Han committed
60 61 62 63 64
} vp10_token_state;

// TODO(jimbankoski): experiment to find optimal RD numbers.
static const int plane_rd_mult[PLANE_TYPES] = { 4, 2 };

clang-format's avatar
clang-format committed
65 66 67 68 69 70 71 72 73
#define UPDATE_RD_COST()                                \
  {                                                     \
    rd_cost0 = RDCOST(rdmult, rddiv, rate0, error0);    \
    rd_cost1 = RDCOST(rdmult, rddiv, rate1, error1);    \
    if (rd_cost0 == rd_cost1) {                         \
      rd_cost0 = RDTRUNC(rdmult, rddiv, rate0, error0); \
      rd_cost1 = RDTRUNC(rdmult, rddiv, rate1, error1); \
    }                                                   \
  }
Jingning Han's avatar
Jingning Han committed
74 75 76

// This function is a place holder for now but may ultimately need
// to scan previous tokens to work out the correct context.
clang-format's avatar
clang-format committed
77 78
static int trellis_get_coeff_context(const int16_t *scan, const int16_t *nb,
                                     int idx, int token, uint8_t *token_cache) {
Jingning Han's avatar
Jingning Han committed
79 80 81 82 83 84 85
  int bak = token_cache[scan[idx]], pt;
  token_cache[scan[idx]] = vp10_pt_energy_class[token];
  pt = get_coef_context(nb, token_cache, idx + 1);
  token_cache[scan[idx]] = bak;
  return pt;
}

clang-format's avatar
clang-format committed
86 87
static int optimize_b(MACROBLOCK *mb, int plane, int block, TX_SIZE tx_size,
                      int ctx) {
Jingning Han's avatar
Jingning Han committed
88 89 90 91 92 93 94 95 96 97 98 99 100 101
  MACROBLOCKD *const xd = &mb->e_mbd;
  struct macroblock_plane *const p = &mb->plane[plane];
  struct macroblockd_plane *const pd = &xd->plane[plane];
  const int ref = is_inter_block(&xd->mi[0]->mbmi);
  vp10_token_state tokens[1025][2];
  unsigned best_index[1025][2];
  uint8_t token_cache[1024];
  const tran_low_t *const coeff = BLOCK_OFFSET(mb->plane[plane].coeff, block);
  tran_low_t *const qcoeff = BLOCK_OFFSET(p->qcoeff, block);
  tran_low_t *const dqcoeff = BLOCK_OFFSET(pd->dqcoeff, block);
  const int eob = p->eobs[block];
  const PLANE_TYPE type = pd->plane_type;
  const int default_eob = 16 << (tx_size << 1);
  const int mul = 1 + (tx_size == TX_32X32);
102 103 104 105 106
#if CONFIG_AOM_QM
  int seg_id = xd->mi[0]->mbmi.segment_id;
  int is_intra = !is_inter_block(&xd->mi[0]->mbmi);
  const qm_val_t *iqmatrix = pd->seg_iqmatrix[seg_id][is_intra][tx_size];
#endif
Jingning Han's avatar
Jingning Han committed
107 108
  const int16_t *dequant_ptr = pd->dequant;
  const uint8_t *const band_translate = get_band_translate(tx_size);
hui su's avatar
hui su committed
109 110
  TX_TYPE tx_type = get_tx_type(type, xd, block);
  const scan_order *const so = get_scan(tx_size, tx_type);
Jingning Han's avatar
Jingning Han committed
111 112 113 114 115 116 117 118 119
  const int16_t *const scan = so->scan;
  const int16_t *const nb = so->neighbors;
  int next = eob, sz = 0;
  int64_t rdmult = mb->rdmult * plane_rd_mult[type], rddiv = mb->rddiv;
  int64_t rd_cost0, rd_cost1;
  int rate0, rate1, error0, error1;
  int16_t t0, t1;
  EXTRABIT e0;
  int best, band, pt, i, final_eob;
120
#if CONFIG_VPX_HIGHBITDEPTH
Jingning Han's avatar
Jingning Han committed
121 122 123 124 125 126 127 128 129
  const int16_t *cat6_high_cost = vp10_get_high_cost_table(xd->bd);
#else
  const int16_t *cat6_high_cost = vp10_get_high_cost_table(8);
#endif

  assert((!type && !plane) || (type && plane));
  assert(eob <= default_eob);

  /* Now set up a Viterbi trellis to evaluate alternative roundings. */
clang-format's avatar
clang-format committed
130
  if (!ref) rdmult = (rdmult * 9) >> 4;
Jingning Han's avatar
Jingning Han committed
131 132 133 134 135 136 137 138 139 140 141 142 143 144 145

  /* Initialize the sentinel node of the trellis. */
  tokens[eob][0].rate = 0;
  tokens[eob][0].error = 0;
  tokens[eob][0].next = default_eob;
  tokens[eob][0].token = EOB_TOKEN;
  tokens[eob][0].qc = 0;
  tokens[eob][1] = tokens[eob][0];

  for (i = 0; i < eob; i++)
    token_cache[scan[i]] =
        vp10_pt_energy_class[vp10_get_token(qcoeff[scan[i]])];

  for (i = eob; i-- > 0;) {
    int base_bits, d2, dx;
146

Jingning Han's avatar
Jingning Han committed
147
    const int rc = scan[i];
148 149 150
#if CONFIG_AOM_QM
    int iwt = iqmatrix[rc];
#endif
Jingning Han's avatar
Jingning Han committed
151 152 153 154 155 156 157 158 159 160 161 162 163 164
    int x = qcoeff[rc];
    /* Only add a trellis state for non-zero coefficients. */
    if (x) {
      int shortcut = 0;
      error0 = tokens[next][0].error;
      error1 = tokens[next][1].error;
      /* Evaluate the first possibility for this state. */
      rate0 = tokens[next][0].rate;
      rate1 = tokens[next][1].rate;
      vp10_get_token_extra(x, &t0, &e0);
      /* Consider both possible successor states. */
      if (next < default_eob) {
        band = band_translate[i + 1];
        pt = trellis_get_coeff_context(scan, nb, i, t0, token_cache);
clang-format's avatar
clang-format committed
165 166 167 168 169 170
        rate0 +=
            mb->token_costs[tx_size][type][ref][band][0][pt][tokens[next][0]
                                                                 .token];
        rate1 +=
            mb->token_costs[tx_size][type][ref][band][0][pt][tokens[next][1]
                                                                 .token];
Jingning Han's avatar
Jingning Han committed
171 172 173 174 175 176
      }
      UPDATE_RD_COST();
      /* And pick the best. */
      best = rd_cost1 < rd_cost0;
      base_bits = vp10_get_cost(t0, e0, cat6_high_cost);
      dx = mul * (dqcoeff[rc] - coeff[rc]);
177
#if CONFIG_VPX_HIGHBITDEPTH
Jingning Han's avatar
Jingning Han committed
178 179 180
      if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) {
        dx >>= xd->bd - 8;
      }
181
#endif  // CONFIG_VPX_HIGHBITDEPTH
Jingning Han's avatar
Jingning Han committed
182 183 184 185 186 187 188 189 190 191 192 193
      d2 = dx * dx;
      tokens[i][0].rate = base_bits + (best ? rate1 : rate0);
      tokens[i][0].error = d2 + (best ? error1 : error0);
      tokens[i][0].next = next;
      tokens[i][0].token = t0;
      tokens[i][0].qc = x;
      best_index[i][0] = best;

      /* Evaluate the second possibility for this state. */
      rate0 = tokens[next][0].rate;
      rate1 = tokens[next][1].rate;

194 195 196 197 198 199
#if CONFIG_AOM_QM
      if ((abs(x) * dequant_ptr[rc != 0] * iwt >
           ((abs(coeff[rc]) * mul) << AOM_QM_BITS)) &&
          (abs(x) * dequant_ptr[rc != 0] * iwt <
           ((abs(coeff[rc]) * mul + dequant_ptr[rc != 0]) << AOM_QM_BITS)))
#else
Jingning Han's avatar
Jingning Han committed
200
      if ((abs(x) * dequant_ptr[rc != 0] > abs(coeff[rc]) * mul) &&
clang-format's avatar
clang-format committed
201 202
          (abs(x) * dequant_ptr[rc != 0] <
           abs(coeff[rc]) * mul + dequant_ptr[rc != 0]))
203
#endif
Jingning Han's avatar
Jingning Han committed
204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228
        shortcut = 1;
      else
        shortcut = 0;

      if (shortcut) {
        sz = -(x < 0);
        x -= 2 * sz + 1;
      }

      /* Consider both possible successor states. */
      if (!x) {
        /* If we reduced this coefficient to zero, check to see if
         *  we need to move the EOB back here.
         */
        t0 = tokens[next][0].token == EOB_TOKEN ? EOB_TOKEN : ZERO_TOKEN;
        t1 = tokens[next][1].token == EOB_TOKEN ? EOB_TOKEN : ZERO_TOKEN;
        e0 = 0;
      } else {
        vp10_get_token_extra(x, &t0, &e0);
        t1 = t0;
      }
      if (next < default_eob) {
        band = band_translate[i + 1];
        if (t0 != EOB_TOKEN) {
          pt = trellis_get_coeff_context(scan, nb, i, t0, token_cache);
clang-format's avatar
clang-format committed
229 230 231
          rate0 +=
              mb->token_costs[tx_size][type][ref][band][!x][pt][tokens[next][0]
                                                                    .token];
Jingning Han's avatar
Jingning Han committed
232 233 234
        }
        if (t1 != EOB_TOKEN) {
          pt = trellis_get_coeff_context(scan, nb, i, t1, token_cache);
clang-format's avatar
clang-format committed
235 236 237
          rate1 +=
              mb->token_costs[tx_size][type][ref][band][!x][pt][tokens[next][1]
                                                                    .token];
Jingning Han's avatar
Jingning Han committed
238 239 240 241 242 243 244 245 246
        }
      }

      UPDATE_RD_COST();
      /* And pick the best. */
      best = rd_cost1 < rd_cost0;
      base_bits = vp10_get_cost(t0, e0, cat6_high_cost);

      if (shortcut) {
247
#if CONFIG_VPX_HIGHBITDEPTH
Jingning Han's avatar
Jingning Han committed
248 249 250 251 252 253 254
        if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) {
          dx -= ((dequant_ptr[rc != 0] >> (xd->bd - 8)) + sz) ^ sz;
        } else {
          dx -= (dequant_ptr[rc != 0] + sz) ^ sz;
        }
#else
        dx -= (dequant_ptr[rc != 0] + sz) ^ sz;
255
#endif  // CONFIG_VPX_HIGHBITDEPTH
Jingning Han's avatar
Jingning Han committed
256 257
        d2 = dx * dx;
      }
258

Jingning Han's avatar
Jingning Han committed
259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307
      tokens[i][1].rate = base_bits + (best ? rate1 : rate0);
      tokens[i][1].error = d2 + (best ? error1 : error0);
      tokens[i][1].next = next;
      tokens[i][1].token = best ? t1 : t0;
      tokens[i][1].qc = x;
      best_index[i][1] = best;
      /* Finally, make this the new head of the trellis. */
      next = i;
    } else {
      /* There's no choice to make for a zero coefficient, so we don't
       *  add a new trellis node, but we do need to update the costs.
       */
      band = band_translate[i + 1];
      t0 = tokens[next][0].token;
      t1 = tokens[next][1].token;
      /* Update the cost of each path if we're past the EOB token. */
      if (t0 != EOB_TOKEN) {
        tokens[next][0].rate +=
            mb->token_costs[tx_size][type][ref][band][1][0][t0];
        tokens[next][0].token = ZERO_TOKEN;
      }
      if (t1 != EOB_TOKEN) {
        tokens[next][1].rate +=
            mb->token_costs[tx_size][type][ref][band][1][0][t1];
        tokens[next][1].token = ZERO_TOKEN;
      }
      best_index[i][0] = best_index[i][1] = 0;
      /* Don't update next, because we didn't add a new node. */
    }
  }

  /* Now pick the best path through the whole trellis. */
  band = band_translate[i + 1];
  rate0 = tokens[next][0].rate;
  rate1 = tokens[next][1].rate;
  error0 = tokens[next][0].error;
  error1 = tokens[next][1].error;
  t0 = tokens[next][0].token;
  t1 = tokens[next][1].token;
  rate0 += mb->token_costs[tx_size][type][ref][band][0][ctx][t0];
  rate1 += mb->token_costs[tx_size][type][ref][band][0][ctx][t1];
  UPDATE_RD_COST();
  best = rd_cost1 < rd_cost0;
  final_eob = -1;
  memset(qcoeff, 0, sizeof(*qcoeff) * (16 << (tx_size * 2)));
  memset(dqcoeff, 0, sizeof(*dqcoeff) * (16 << (tx_size * 2)));
  for (i = next; i < eob; i = next) {
    const int x = tokens[i][best].qc;
    const int rc = scan[i];
308 309 310 311 312
#if CONFIG_AOM_QM
    const int iwt = iqmatrix[rc];
    const int dequant =
        (dequant_ptr[rc != 0] * iwt + (1 << (AOM_QM_BITS - 1))) >> AOM_QM_BITS;
#endif
Jingning Han's avatar
Jingning Han committed
313 314 315 316 317
    if (x) {
      final_eob = i;
    }

    qcoeff[rc] = x;
318 319 320
#if CONFIG_AOM_QM
    dqcoeff[rc] = (x * dequant) / mul;
#else
Jingning Han's avatar
Jingning Han committed
321
    dqcoeff[rc] = (x * dequant_ptr[rc != 0]) / mul;
322
#endif
Jingning Han's avatar
Jingning Han committed
323 324 325 326 327 328 329 330 331 332

    next = tokens[i][best].next;
    best = best_index[i][best];
  }
  final_eob++;

  mb->plane[plane].eobs[block] = final_eob;
  return final_eob;
}

clang-format's avatar
clang-format committed
333 334
static INLINE void fdct32x32(int rd_transform, const int16_t *src,
                             tran_low_t *dst, int src_stride) {
Jingning Han's avatar
Jingning Han committed
335 336 337 338 339 340
  if (rd_transform)
    vpx_fdct32x32_rd(src, dst, src_stride);
  else
    vpx_fdct32x32(src, dst, src_stride);
}

341
#if CONFIG_VPX_HIGHBITDEPTH
Jingning Han's avatar
Jingning Han committed
342 343 344 345 346 347 348
static INLINE void highbd_fdct32x32(int rd_transform, const int16_t *src,
                                    tran_low_t *dst, int src_stride) {
  if (rd_transform)
    vpx_highbd_fdct32x32_rd(src, dst, src_stride);
  else
    vpx_highbd_fdct32x32(src, dst, src_stride);
}
349
#endif  // CONFIG_VPX_HIGHBITDEPTH
Jingning Han's avatar
Jingning Han committed
350

351 352 353 354 355 356
void vp10_fwd_txfm_4x4(const int16_t *src_diff, tran_low_t *coeff,
                       int diff_stride, TX_TYPE tx_type, int lossless) {
  if (lossless) {
    vp10_fwht4x4(src_diff, coeff, diff_stride);
  } else {
    switch (tx_type) {
357 358 359
      case DCT_DCT:
        vpx_fdct4x4(src_diff, coeff, diff_stride);
        break;
360 361
      case ADST_DCT:
      case DCT_ADST:
362 363 364 365 366 367
      case ADST_ADST:
        vp10_fht4x4(src_diff, coeff, diff_stride, tx_type);
        break;
      default:
        assert(0);
        break;
368 369 370 371 372 373 374 375 376 377
    }
  }
}

static void fwd_txfm_8x8(const int16_t *src_diff, tran_low_t *coeff,
                         int diff_stride, TX_TYPE tx_type) {
  switch (tx_type) {
    case DCT_DCT:
    case ADST_DCT:
    case DCT_ADST:
378 379 380 381 382 383
    case ADST_ADST:
      vp10_fht8x8(src_diff, coeff, diff_stride, tx_type);
      break;
    default:
      assert(0);
      break;
384 385 386 387 388 389 390 391 392
  }
}

static void fwd_txfm_16x16(const int16_t *src_diff, tran_low_t *coeff,
                           int diff_stride, TX_TYPE tx_type) {
  switch (tx_type) {
    case DCT_DCT:
    case ADST_DCT:
    case DCT_ADST:
393 394 395 396 397 398
    case ADST_ADST:
      vp10_fht16x16(src_diff, coeff, diff_stride, tx_type);
      break;
    default:
      assert(0);
      break;
399 400 401 402 403 404 405
  }
}

static void fwd_txfm_32x32(int rd_transform, const int16_t *src_diff,
                           tran_low_t *coeff, int diff_stride,
                           TX_TYPE tx_type) {
  switch (tx_type) {
406 407 408
    case DCT_DCT:
      fdct32x32(rd_transform, src_diff, coeff, diff_stride);
      break;
409 410
    case ADST_DCT:
    case DCT_ADST:
411 412 413 414 415 416
    case ADST_ADST:
      assert(0);
      break;
    default:
      assert(0);
      break;
417 418 419
  }
}

420
#if CONFIG_VPX_HIGHBITDEPTH
421 422 423 424 425 426 427
void vp10_highbd_fwd_txfm_4x4(const int16_t *src_diff, tran_low_t *coeff,
                              int diff_stride, TX_TYPE tx_type, int lossless) {
  if (lossless) {
    assert(tx_type == DCT_DCT);
    vp10_highbd_fwht4x4(src_diff, coeff, diff_stride);
  } else {
    switch (tx_type) {
428 429 430
      case DCT_DCT:
        vpx_highbd_fdct4x4(src_diff, coeff, diff_stride);
        break;
431 432 433 434 435
      case ADST_DCT:
      case DCT_ADST:
      case ADST_ADST:
        vp10_highbd_fht4x4(src_diff, coeff, diff_stride, tx_type);
        break;
436 437 438
      default:
        assert(0);
        break;
439 440 441 442 443
    }
  }
}

static void highbd_fwd_txfm_8x8(const int16_t *src_diff, tran_low_t *coeff,
clang-format's avatar
clang-format committed
444
                                int diff_stride, TX_TYPE tx_type) {
445
  switch (tx_type) {
446 447 448
    case DCT_DCT:
      vpx_highbd_fdct8x8(src_diff, coeff, diff_stride);
      break;
449 450 451 452 453
    case ADST_DCT:
    case DCT_ADST:
    case ADST_ADST:
      vp10_highbd_fht8x8(src_diff, coeff, diff_stride, tx_type);
      break;
454 455 456
    default:
      assert(0);
      break;
457 458 459 460
  }
}

static void highbd_fwd_txfm_16x16(const int16_t *src_diff, tran_low_t *coeff,
clang-format's avatar
clang-format committed
461
                                  int diff_stride, TX_TYPE tx_type) {
462
  switch (tx_type) {
463 464 465
    case DCT_DCT:
      vpx_highbd_fdct16x16(src_diff, coeff, diff_stride);
      break;
466 467 468 469 470
    case ADST_DCT:
    case DCT_ADST:
    case ADST_ADST:
      vp10_highbd_fht16x16(src_diff, coeff, diff_stride, tx_type);
      break;
471 472 473
    default:
      assert(0);
      break;
474 475 476 477 478 479 480 481 482 483 484 485
  }
}

static void highbd_fwd_txfm_32x32(int rd_transform, const int16_t *src_diff,
                                  tran_low_t *coeff, int diff_stride,
                                  TX_TYPE tx_type) {
  switch (tx_type) {
    case DCT_DCT:
      highbd_fdct32x32(rd_transform, src_diff, coeff, diff_stride);
      break;
    case ADST_DCT:
    case DCT_ADST:
486 487 488 489 490 491
    case ADST_ADST:
      assert(0);
      break;
    default:
      assert(0);
      break;
492 493
  }
}
494
#endif  // CONFIG_VPX_HIGHBITDEPTH
495

clang-format's avatar
clang-format committed
496 497
void vp10_xform_quant_fp(MACROBLOCK *x, int plane, int block, int blk_row,
                         int blk_col, BLOCK_SIZE plane_bsize, TX_SIZE tx_size) {
Jingning Han's avatar
Jingning Han committed
498 499 500
  MACROBLOCKD *const xd = &x->e_mbd;
  const struct macroblock_plane *const p = &x->plane[plane];
  const struct macroblockd_plane *const pd = &xd->plane[plane];
hui su's avatar
hui su committed
501 502 503
  PLANE_TYPE plane_type = (plane == 0) ? PLANE_TYPE_Y : PLANE_TYPE_UV;
  TX_TYPE tx_type = get_tx_type(plane_type, xd, block);
  const scan_order *const scan_order = get_scan(tx_size, tx_type);
Jingning Han's avatar
Jingning Han committed
504 505 506 507 508
  tran_low_t *const coeff = BLOCK_OFFSET(p->coeff, block);
  tran_low_t *const qcoeff = BLOCK_OFFSET(p->qcoeff, block);
  tran_low_t *const dqcoeff = BLOCK_OFFSET(pd->dqcoeff, block);
  uint16_t *const eob = &p->eobs[block];
  const int diff_stride = 4 * num_4x4_blocks_wide_lookup[plane_bsize];
509 510 511 512 513 514
#if CONFIG_AOM_QM
  int seg_id = xd->mi[0]->mbmi.segment_id;
  int is_intra = !is_inter_block(&xd->mi[0]->mbmi);
  const qm_val_t *qmatrix = pd->seg_qmatrix[seg_id][is_intra][tx_size];
  const qm_val_t *iqmatrix = pd->seg_iqmatrix[seg_id][is_intra][tx_size];
#endif
Jingning Han's avatar
Jingning Han committed
515
  const int16_t *src_diff;
516
  src_diff = &p->src_diff[4 * (blk_row * diff_stride + blk_col)];
Jingning Han's avatar
Jingning Han committed
517

518
#if CONFIG_VPX_HIGHBITDEPTH
Jingning Han's avatar
Jingning Han committed
519 520 521 522 523
  if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) {
    switch (tx_size) {
      case TX_32X32:
        highbd_fdct32x32(x->use_lp32x32fdct, src_diff, coeff, diff_stride);
        vp10_highbd_quantize_fp_32x32(coeff, 1024, x->skip_block, p->zbin,
clang-format's avatar
clang-format committed
524 525
                                      p->round_fp, p->quant_fp, p->quant_shift,
                                      qcoeff, dqcoeff, pd->dequant, eob,
526 527 528 529 530 531
                                      scan_order->scan,
#if !CONFIG_AOM_QM
                                      scan_order->iscan);
#else
                                      scan_order->iscan, qmatrix, iqmatrix);
#endif
Jingning Han's avatar
Jingning Han committed
532 533 534 535
        break;
      case TX_16X16:
        vpx_highbd_fdct16x16(src_diff, coeff, diff_stride);
        vp10_highbd_quantize_fp(coeff, 256, x->skip_block, p->zbin, p->round_fp,
clang-format's avatar
clang-format committed
536 537
                                p->quant_fp, p->quant_shift, qcoeff, dqcoeff,
                                pd->dequant, eob, scan_order->scan,
538
#if !CONFIG_AOM_QM
clang-format's avatar
clang-format committed
539
                                scan_order->iscan);
540 541 542
#else
                                scan_order->iscan, qmatrix, iqmatrix);
#endif
Jingning Han's avatar
Jingning Han committed
543 544 545 546
        break;
      case TX_8X8:
        vpx_highbd_fdct8x8(src_diff, coeff, diff_stride);
        vp10_highbd_quantize_fp(coeff, 64, x->skip_block, p->zbin, p->round_fp,
clang-format's avatar
clang-format committed
547 548
                                p->quant_fp, p->quant_shift, qcoeff, dqcoeff,
                                pd->dequant, eob, scan_order->scan,
549
#if !CONFIG_AOM_QM
clang-format's avatar
clang-format committed
550
                                scan_order->iscan);
551 552 553
#else
                                scan_order->iscan, qmatrix, iqmatrix);
#endif
Jingning Han's avatar
Jingning Han committed
554 555
        break;
      case TX_4X4:
556
        if (xd->lossless[xd->mi[0]->mbmi.segment_id]) {
557 558 559 560
          vp10_highbd_fwht4x4(src_diff, coeff, diff_stride);
        } else {
          vpx_highbd_fdct4x4(src_diff, coeff, diff_stride);
        }
Jingning Han's avatar
Jingning Han committed
561
        vp10_highbd_quantize_fp(coeff, 16, x->skip_block, p->zbin, p->round_fp,
clang-format's avatar
clang-format committed
562 563
                                p->quant_fp, p->quant_shift, qcoeff, dqcoeff,
                                pd->dequant, eob, scan_order->scan,
564
#if !CONFIG_AOM_QM
clang-format's avatar
clang-format committed
565
                                scan_order->iscan);
566 567 568
#else
                                scan_order->iscan, qmatrix, iqmatrix);
#endif
Jingning Han's avatar
Jingning Han committed
569
        break;
570 571
      default:
        assert(0);
Jingning Han's avatar
Jingning Han committed
572 573 574
    }
    return;
  }
575
#endif  // CONFIG_VPX_HIGHBITDEPTH
Jingning Han's avatar
Jingning Han committed
576 577 578 579 580

  switch (tx_size) {
    case TX_32X32:
      fdct32x32(x->use_lp32x32fdct, src_diff, coeff, diff_stride);
      vp10_quantize_fp_32x32(coeff, 1024, x->skip_block, p->zbin, p->round_fp,
clang-format's avatar
clang-format committed
581 582
                             p->quant_fp, p->quant_shift, qcoeff, dqcoeff,
                             pd->dequant, eob, scan_order->scan,
583
#if !CONFIG_AOM_QM
clang-format's avatar
clang-format committed
584
                             scan_order->iscan);
585 586 587
#else
                             scan_order->iscan, qmatrix, iqmatrix);
#endif
Jingning Han's avatar
Jingning Han committed
588 589 590 591
      break;
    case TX_16X16:
      vpx_fdct16x16(src_diff, coeff, diff_stride);
      vp10_quantize_fp(coeff, 256, x->skip_block, p->zbin, p->round_fp,
clang-format's avatar
clang-format committed
592
                       p->quant_fp, p->quant_shift, qcoeff, dqcoeff,
593 594 595 596 597 598
                       pd->dequant, eob, scan_order->scan,
#if !CONFIG_AOM_QM
                       scan_order->iscan);
#else
                       scan_order->iscan, qmatrix, iqmatrix);
#endif
Jingning Han's avatar
Jingning Han committed
599 600
      break;
    case TX_8X8:
clang-format's avatar
clang-format committed
601 602 603
      vp10_fdct8x8_quant(src_diff, diff_stride, coeff, 64, x->skip_block,
                         p->zbin, p->round_fp, p->quant_fp, p->quant_shift,
                         qcoeff, dqcoeff, pd->dequant, eob, scan_order->scan,
604
#if !CONFIG_AOM_QM
clang-format's avatar
clang-format committed
605
                         scan_order->iscan);
606 607 608
#else
                         scan_order->iscan, qmatrix, iqmatrix);
#endif
Jingning Han's avatar
Jingning Han committed
609 610
      break;
    case TX_4X4:
611
      if (xd->lossless[xd->mi[0]->mbmi.segment_id]) {
612 613 614 615
        vp10_fwht4x4(src_diff, coeff, diff_stride);
      } else {
        vpx_fdct4x4(src_diff, coeff, diff_stride);
      }
Jingning Han's avatar
Jingning Han committed
616
      vp10_quantize_fp(coeff, 16, x->skip_block, p->zbin, p->round_fp,
clang-format's avatar
clang-format committed
617
                       p->quant_fp, p->quant_shift, qcoeff, dqcoeff,
618 619 620 621 622 623 624 625 626
                       pd->dequant, eob, scan_order->scan,
#if !CONFIG_AOM_QM
                       scan_order->iscan);
#else
                       scan_order->iscan, qmatrix, iqmatrix);
#endif
      break;
    default:
      assert(0);
Jingning Han's avatar
Jingning Han committed
627 628 629 630
      break;
  }
}

clang-format's avatar
clang-format committed
631 632
void vp10_xform_quant_dc(MACROBLOCK *x, int plane, int block, int blk_row,
                         int blk_col, BLOCK_SIZE plane_bsize, TX_SIZE tx_size) {
Jingning Han's avatar
Jingning Han committed
633 634 635 636 637 638 639 640
  MACROBLOCKD *const xd = &x->e_mbd;
  const struct macroblock_plane *const p = &x->plane[plane];
  const struct macroblockd_plane *const pd = &xd->plane[plane];
  tran_low_t *const coeff = BLOCK_OFFSET(p->coeff, block);
  tran_low_t *const qcoeff = BLOCK_OFFSET(p->qcoeff, block);
  tran_low_t *const dqcoeff = BLOCK_OFFSET(pd->dqcoeff, block);
  uint16_t *const eob = &p->eobs[block];
  const int diff_stride = 4 * num_4x4_blocks_wide_lookup[plane_bsize];
641 642 643 644 645 646
  int seg_id = xd->mi[0]->mbmi.segment_id;
#if CONFIG_AOM_QM
  int is_intra = !is_inter_block(&xd->mi[0]->mbmi);
  const qm_val_t *qmatrix = pd->seg_qmatrix[seg_id][is_intra][tx_size];
  const qm_val_t *iqmatrix = pd->seg_iqmatrix[seg_id][is_intra][tx_size];
#endif
Jingning Han's avatar
Jingning Han committed
647
  const int16_t *src_diff;
648
  src_diff = &p->src_diff[4 * (blk_row * diff_stride + blk_col)];
Jingning Han's avatar
Jingning Han committed
649

650
#if CONFIG_VPX_HIGHBITDEPTH
Jingning Han's avatar
Jingning Han committed
651 652 653 654 655 656
  if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) {
    switch (tx_size) {
      case TX_32X32:
        vpx_highbd_fdct32x32_1(src_diff, coeff, diff_stride);
        vpx_highbd_quantize_dc_32x32(coeff, x->skip_block, p->round,
                                     p->quant_fp[0], qcoeff, dqcoeff,
657 658 659 660 661 662
                                     pd->dequant[0],
#if !CONFIG_AOM_QM
                                     eob);
#else
                                     eob, qmatrix, iqmatrix);
#endif
Jingning Han's avatar
Jingning Han committed
663 664 665 666
        break;
      case TX_16X16:
        vpx_highbd_fdct16x16_1(src_diff, coeff, diff_stride);
        vpx_highbd_quantize_dc(coeff, 256, x->skip_block, p->round,
clang-format's avatar
clang-format committed
667
                               p->quant_fp[0], qcoeff, dqcoeff, pd->dequant[0],
668
#if !CONFIG_AOM_QM
clang-format's avatar
clang-format committed
669
                               eob);
670 671 672
#else
                               eob, qmatrix, iqmatrix);
#endif
Jingning Han's avatar
Jingning Han committed
673 674 675 676
        break;
      case TX_8X8:
        vpx_highbd_fdct8x8_1(src_diff, coeff, diff_stride);
        vpx_highbd_quantize_dc(coeff, 64, x->skip_block, p->round,
clang-format's avatar
clang-format committed
677
                               p->quant_fp[0], qcoeff, dqcoeff, pd->dequant[0],
678
#if !CONFIG_AOM_QM
clang-format's avatar
clang-format committed
679
                               eob);
680 681 682
#else
                               eob, qmatrix, iqmatrix);
#endif
Jingning Han's avatar
Jingning Han committed
683 684
        break;
      case TX_4X4:
685
        if (xd->lossless[seg_id]) {
686 687 688 689
          vp10_highbd_fwht4x4(src_diff, coeff, diff_stride);
        } else {
          vpx_highbd_fdct4x4(src_diff, coeff, diff_stride);
        }
Jingning Han's avatar
Jingning Han committed
690
        vpx_highbd_quantize_dc(coeff, 16, x->skip_block, p->round,
clang-format's avatar
clang-format committed
691
                               p->quant_fp[0], qcoeff, dqcoeff, pd->dequant[0],
692
#if !CONFIG_AOM_QM
clang-format's avatar
clang-format committed
693
                               eob);
694 695 696
#else
                               eob, qmatrix, iqmatrix);
#endif
Jingning Han's avatar
Jingning Han committed
697
        break;
698 699
      default:
        assert(0);
Jingning Han's avatar
Jingning Han committed
700 701 702
    }
    return;
  }
703
#endif  // CONFIG_VPX_HIGHBITDEPTH
Jingning Han's avatar
Jingning Han committed
704 705 706 707

  switch (tx_size) {
    case TX_32X32:
      vpx_fdct32x32_1(src_diff, coeff, diff_stride);
clang-format's avatar
clang-format committed
708
      vpx_quantize_dc_32x32(coeff, x->skip_block, p->round, p->quant_fp[0],
709 710 711 712 713 714
                            qcoeff, dqcoeff, pd->dequant[0],
#if !CONFIG_AOM_QM
                            eob);
#else
                            eob, qmatrix, iqmatrix);
#endif
Jingning Han's avatar
Jingning Han committed
715 716 717
      break;
    case TX_16X16:
      vpx_fdct16x16_1(src_diff, coeff, diff_stride);
clang-format's avatar
clang-format committed
718
      vpx_quantize_dc(coeff, 256, x->skip_block, p->round, p->quant_fp[0],
719 720 721 722 723 724
                      qcoeff, dqcoeff, pd->dequant[0],
#if !CONFIG_AOM_QM
                      eob);
#else
                      eob, qmatrix, iqmatrix);
#endif
Jingning Han's avatar
Jingning Han committed
725 726 727
      break;
    case TX_8X8:
      vpx_fdct8x8_1(src_diff, coeff, diff_stride);
clang-format's avatar
clang-format committed
728
      vpx_quantize_dc(coeff, 64, x->skip_block, p->round, p->quant_fp[0],
729 730 731 732 733 734
                      qcoeff, dqcoeff, pd->dequant[0],
#if !CONFIG_AOM_QM
                      eob);
#else
                      eob, qmatrix, iqmatrix);
#endif
Jingning Han's avatar
Jingning Han committed
735 736
      break;
    case TX_4X4:
737
      if (xd->lossless[seg_id]) {
738 739 740 741
        vp10_fwht4x4(src_diff, coeff, diff_stride);
      } else {
        vpx_fdct4x4(src_diff, coeff, diff_stride);
      }
clang-format's avatar
clang-format committed
742
      vpx_quantize_dc(coeff, 16, x->skip_block, p->round, p->quant_fp[0],
743 744 745 746 747 748 749 750 751
                      qcoeff, dqcoeff, pd->dequant[0],
#if !CONFIG_AOM_QM
                      eob);
#else
                      eob, qmatrix, iqmatrix);
#endif
      break;
    default:
      assert(0);
Jingning Han's avatar
Jingning Han committed
752 753 754 755
      break;
  }
}

clang-format's avatar
clang-format committed
756 757
void vp10_xform_quant(MACROBLOCK *x, int plane, int block, int blk_row,
                      int blk_col, BLOCK_SIZE plane_bsize, TX_SIZE tx_size) {
Jingning Han's avatar
Jingning Han committed
758 759 760
  MACROBLOCKD *const xd = &x->e_mbd;
  const struct macroblock_plane *const p = &x->plane[plane];
  const struct macroblockd_plane *const pd = &xd->plane[plane];
hui su's avatar
hui su committed
761 762 763
  PLANE_TYPE plane_type = (plane == 0) ? PLANE_TYPE_Y : PLANE_TYPE_UV;
  TX_TYPE tx_type = get_tx_type(plane_type, xd, block);
  const scan_order *const scan_order = get_scan(tx_size, tx_type);
Jingning Han's avatar
Jingning Han committed
764 765 766 767 768
  tran_low_t *const coeff = BLOCK_OFFSET(p->coeff, block);
  tran_low_t *const qcoeff = BLOCK_OFFSET(p->qcoeff, block);
  tran_low_t *const dqcoeff = BLOCK_OFFSET(pd->dqcoeff, block);
  uint16_t *const eob = &p->eobs[block];
  const int diff_stride = 4 * num_4x4_blocks_wide_lookup[plane_bsize];
769 770 771 772 773 774
  int seg_id = xd->mi[0]->mbmi.segment_id;
#if CONFIG_AOM_QM
  int is_intra = !is_inter_block(&xd->mi[0]->mbmi);
  const qm_val_t *qmatrix = pd->seg_qmatrix[seg_id][is_intra][tx_size];
  const qm_val_t *iqmatrix = pd->seg_iqmatrix[seg_id][is_intra][tx_size];
#endif
Jingning Han's avatar
Jingning Han committed
775
  const int16_t *src_diff;
776
  src_diff = &p->src_diff[4 * (blk_row * diff_stride + blk_col)];
Jingning Han's avatar
Jingning Han committed
777

778
#if CONFIG_VPX_HIGHBITDEPTH
Jingning Han's avatar
Jingning Han committed
779
  if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) {
clang-format's avatar
clang-format committed
780
    switch (tx_size) {
Jingning Han's avatar
Jingning Han committed
781
      case TX_32X32:
hui su's avatar
hui su committed
782
        highbd_fwd_txfm_32x32(x->use_lp32x32fdct, src_diff, coeff, diff_stride,
clang-format's avatar
clang-format committed
783
                              tx_type);
Jingning Han's avatar
Jingning Han committed
784 785
        vpx_highbd_quantize_b_32x32(coeff, 1024, x->skip_block, p->zbin,
                                    p->round, p->quant, p->quant_shift, qcoeff,
clang-format's avatar
clang-format committed
786
                                    dqcoeff, pd->dequant, eob, scan_order->scan,
787
#if !CONFIG_AOM_QM
clang-format's avatar
clang-format committed
788
                                    scan_order->iscan);
789 790 791
#else
                                    scan_order->iscan, qmatrix, iqmatrix);
#endif
Jingning Han's avatar
Jingning Han committed
792 793
        break;
      case TX_16X16:
hui su's avatar
hui su committed
794
        highbd_fwd_txfm_16x16(src_diff, coeff, diff_stride, tx_type);
Jingning Han's avatar
Jingning Han committed
795 796
        vpx_highbd_quantize_b(coeff, 256, x->skip_block, p->zbin, p->round,
                              p->quant, p->quant_shift, qcoeff, dqcoeff,
clang-format's avatar
clang-format committed
797
                              pd->dequant, eob, scan_order->scan,
798
#if !CONFIG_AOM_QM
clang-format's avatar
clang-format committed
799
                              scan_order->iscan);
800 801 802
#else
                              scan_order->iscan, qmatrix, iqmatrix);
#endif
Jingning Han's avatar
Jingning Han committed
803 804
        break;
      case TX_8X8:
hui su's avatar
hui su committed
805
        highbd_fwd_txfm_8x8(src_diff, coeff, diff_stride, tx_type);
Jingning Han's avatar
Jingning Han committed
806 807
        vpx_highbd_quantize_b(coeff, 64, x->skip_block, p->zbin, p->round,
                              p->quant, p->quant_shift, qcoeff, dqcoeff,
clang-format's avatar
clang-format committed
808
                              pd->dequant, eob, scan_order->scan,
809
#if !CONFIG_AOM_QM
clang-format's avatar
clang-format committed
810
                              scan_order->iscan);
811 812 813
#else
                              scan_order->iscan, qmatrix, iqmatrix);
#endif
Jingning Han's avatar
Jingning Han committed
814 815
        break;
      case TX_4X4:
hui su's avatar
hui su committed
816
        vp10_highbd_fwd_txfm_4x4(src_diff, coeff, diff_stride, tx_type,
817
                                 xd->lossless[seg_id]);
Jingning Han's avatar
Jingning Han committed
818 819
        vpx_highbd_quantize_b(coeff, 16, x->skip_block, p->zbin, p->round,
                              p->quant, p->quant_shift, qcoeff, dqcoeff,
clang-format's avatar
clang-format committed
820
                              pd->dequant, eob, scan_order->scan,
821
#if !CONFIG_AOM_QM
clang-format's avatar
clang-format committed
822
                              scan_order->iscan);
823 824 825
#else
                              scan_order->iscan, qmatrix, iqmatrix);
#endif
Jingning Han's avatar
Jingning Han committed
826
        break;
827 828
      default:
        assert(0);
Jingning Han's avatar
Jingning Han committed
829 830 831
    }
    return;
  }
832
#endif  // CONFIG_VPX_HIGHBITDEPTH
Jingning Han's avatar
Jingning Han committed
833 834 835

  switch (tx_size) {
    case TX_32X32:
hui su's avatar
hui su committed
836
      fwd_txfm_32x32(x->use_lp32x32fdct, src_diff, coeff, diff_stride, tx_type);
Jingning Han's avatar
Jingning Han committed
837 838 839
      vpx_quantize_b_32x32(coeff, 1024, x->skip_block, p->zbin, p->round,
                           p->quant, p->quant_shift, qcoeff, dqcoeff,
                           pd->dequant, eob, scan_order->scan,
840
#if !CONFIG_AOM_QM
Jingning Han's avatar
Jingning Han committed
841
                           scan_order->iscan);
842 843 844
#else
                           scan_order->iscan, qmatrix, iqmatrix);
#endif
Jingning Han's avatar
Jingning Han committed
845 846
      break;
    case TX_16X16:
hui su's avatar
hui su committed
847
      fwd_txfm_16x16(src_diff, coeff, diff_stride, tx_type);
clang-format's avatar
clang-format committed
848 849
      vpx_quantize_b(coeff, 256, x->skip_block, p->zbin, p->round, p->quant,
                     p->quant_shift, qcoeff, dqcoeff, pd->dequant, eob,
850 851 852 853 854 855
                     scan_order->scan,
#if !CONFIG_AOM_QM
                     scan_order->iscan);
#else
                     scan_order->iscan, qmatrix, iqmatrix);
#endif
Jingning Han's avatar
Jingning Han committed
856 857
      break;
    case TX_8X8:
hui su's avatar
hui su committed
858
      fwd_txfm_8x8(src_diff, coeff, diff_stride, tx_type);
clang-format's avatar
clang-format committed
859 860
      vpx_quantize_b(coeff, 64, x->skip_block, p->zbin, p->round, p->quant,
                     p->quant_shift, qcoeff, dqcoeff, pd->dequant, eob,
861 862 863 864 865 866
                     scan_order->scan,
#if !CONFIG_AOM_QM
                     scan_order->iscan);
#else
                     scan_order->iscan, qmatrix, iqmatrix);
#endif
Jingning Han's avatar
Jingning Han committed
867 868
      break;
    case TX_4X4:
869
      vp10_fwd_txfm_4x4(src_diff, coeff, diff_stride, tx_type,
870
                        xd->lossless[seg_id]);
clang-format's avatar
clang-format committed
871 872
      vpx_quantize_b(coeff, 16, x->skip_block, p->zbin, p->round, p->quant,
                     p->quant_shift, qcoeff, dqcoeff, pd->dequant, eob,
873 874 875 876 877 878 879 880 881
                     scan_order->scan,
#if !CONFIG_AOM_QM
                     scan_order->iscan);
#else
                     scan_order->iscan, qmatrix, iqmatrix);
#endif
      break;
    default:
      assert(0);
Jingning Han's avatar
Jingning Han committed
882 883 884 885
      break;
  }
}

886
static void encode_block(int plane, int block, int blk_row, int blk_col,
clang-format's avatar
clang-format committed
887
                         BLOCK_SIZE plane_bsize, TX_SIZE tx_size, void *arg) {
Jingning Han's avatar
Jingning Han committed
888 889 890 891 892 893 894 895 896
  struct encode_b_args *const args = arg;
  MACROBLOCK *const x = args->x;
  MACROBLOCKD *const xd = &x->e_mbd;
  struct optimize_ctx *const ctx = args->ctx;
  struct macroblock_plane *const p = &x->plane[plane];
  struct macroblockd_plane *const pd = &xd->plane[plane];
  tran_low_t *const dqcoeff = BLOCK_OFFSET(pd->dqcoeff, block);
  uint8_t *dst;
  ENTROPY_CONTEXT *a, *l;
hui su's avatar
hui su committed
897
  TX_TYPE tx_type = get_tx_type(pd->plane_type, xd, block);
898 899 900
  dst = &pd->dst.buf[4 * blk_row * pd->dst.stride + 4 * blk_col];
  a = &ctx->ta[plane][blk_col];
  l = &ctx->tl[plane][blk_row];
Jingning Han's avatar
Jingning Han committed
901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918

  // TODO(jingning): per transformed block zero forcing only enabled for
  // luma component. will integrate chroma components as well.
  if (x->zcoeff_blk[tx_size][block] && plane == 0) {
    p->eobs[block] = 0;
    *a = *l = 0;
    return;
  }

  if (!x->skip_recode) {
    if (x->quant_fp) {
      // Encoding process for rtc mode
      if (x->skip_txfm[0] == SKIP_TXFM_AC_DC && plane == 0) {
        // skip forward transform
        p->eobs[block] = 0;
        *a = *l = 0;
        return;
      } else {
clang-format's avatar
clang-format committed
919 920
        vp10_xform_quant_fp(x, plane, block, blk_row, blk_col, plane_bsize,
                            tx_size);
Jingning Han's avatar
Jingning Han committed
921 922 923 924 925 926
      }
    } else {
      if (max_txsize_lookup[plane_bsize] == tx_size) {
        int txfm_blk_index = (plane << 2) + (block >> (tx_size << 1));
        if (x->skip_txfm[txfm_blk_index] == SKIP_TXFM_NONE) {
          // full forward transform and quantization
clang-format's avatar
clang-format committed
927 928
          vp10_xform_quant(x, plane, block, blk_row, blk_col, plane_bsize,
                           tx_size);
Jingning Han's avatar
Jingning Han committed
929 930
        } else if (x->skip_txfm[txfm_blk_index] == SKIP_TXFM_AC_ONLY) {
          // fast path forward transform and quantization
clang-format's avatar
clang-format committed
931 932
          vp10_xform_quant_dc(x, plane, block, blk_row, blk_col, plane_bsize,
                              tx_size);
Jingning Han's avatar
Jingning Han committed
933 934 935 936 937 938 939
        } else {
          // skip forward transform
          p->eobs[block] = 0;
          *a = *l = 0;
          return;
        }
      } else {
clang-format's avatar
clang-format committed
940 941
        vp10_xform_quant(x, plane, block, blk_row, blk_col, plane_bsize,
                         tx_size);
Jingning Han's avatar
Jingning Han committed
942 943 944 945 946 947 948 949 950 951 952
      }
    }
  }

  if (x->optimize && (!x->skip_recode || !x->skip_optimize)) {
    const int ctx = combine_entropy_contexts(*a, *l);
    *a = *l = optimize_b(x, plane, block, tx_size, ctx) > 0;
  } else {
    *a = *l = p->eobs[block] > 0;
  }

clang-format's avatar
clang-format committed
953
  if (p->eobs[block]) *(args->skip) = 0;
Jingning Han's avatar
Jingning Han committed
954

clang-format's avatar
clang-format committed
955
  if (p->eobs[block] == 0) return;
956
#if CONFIG_VPX_HIGHBITDEPTH
Jingning Han's avatar
Jingning Han committed
957 958 959
  if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) {
    switch (tx_size) {
      case TX_32X32:
hui su's avatar
hui su committed
960 961
        vp10_highbd_inv_txfm_add_32x32(dqcoeff, dst, pd->dst.stride,
                                       p->eobs[block], xd->bd, tx_type);
Jingning Han's avatar
Jingning Han committed
962 963
        break;
      case TX_16X16:
hui su's avatar
hui su committed
964 965
        vp10_highbd_inv_txfm_add_16x16(dqcoeff, dst, pd->dst.stride,
                                       p->eobs[block], xd->bd, tx_type);
Jingning Han's avatar
Jingning Han committed
966 967
        break;
      case TX_8X8:
hui su's avatar
hui su committed
968 969
        vp10_highbd_inv_txfm_add_8x8(dqcoeff, dst, pd->dst.stride,
                                     p->eobs[block], xd->bd, tx_type);
Jingning Han's avatar
Jingning Han committed
970 971 972 973 974
        break;
      case TX_4X4:
        // this is like vp10_short_idct4x4 but has a special case around eob<=1
        // which is significant (not just an optimization) for the lossless
        // case.
hui su's avatar
hui su committed
975 976
        vp10_highbd_inv_txfm_add_4x4(dqcoeff, dst, pd->dst.stride,
                                     p->eobs[block], xd->bd, tx_type,
977
                                     xd->lossless[xd->mi[0]->mbmi.segment_id]);
Jingning Han's avatar
Jingning Han committed
978
        break;
979 980 981
      default:
        assert(0 && "Invalid transform size");
        break;
Jingning Han's avatar
Jingning Han committed
982
    }
hui su's avatar
hui su committed
983

Jingning Han's avatar
Jingning Han committed
984 985
    return;
  }
986
#endif  // CONFIG_VPX_HIGHBITDEPTH
Jingning Han's avatar
Jingning Han committed
987 988 989

  switch (tx_size) {
    case TX_32X32:
hui su's avatar
hui su committed
990 991
      vp10_inv_txfm_add_32x32(dqcoeff, dst, pd->dst.stride, p->eobs[block],
                              tx_type);
Jingning Han's avatar
Jingning Han committed
992 993
      break;
    case TX_16X16:
hui su's avatar
hui su committed
994 995
      vp10_inv_txfm_add_16x16(dqcoeff, dst, pd->dst.stride, p->eobs[block],
                              tx_type);
Jingning Han's avatar
Jingning Han committed
996 997
      break;
    case TX_8X8:
hui su's avatar
hui su committed
998 999
      vp10_inv_txfm_add_8x8(dqcoeff, dst, pd->dst.stride, p->eobs[block],
                            tx_type);
Jingning Han's avatar
Jingning Han committed
1000 1001 1002 1003 1004
      break;
    case TX_4X4:
      // this is like vp10_short_idct4x4 but has a special case around eob<=1
      // which is significant (not just an optimization) for the lossless
      // case.
hui su's avatar
hui su committed
1005
      vp10_inv_txfm_add_4x4(dqcoeff, dst, pd->dst.stride, p->eobs[block],
1006
                            tx_type, xd->lossless[xd->mi[0]->mbmi.segment_id]);
Jingning Han's avatar
Jingning Han committed
1007
      break;
1008 1009 1010
    default:
      assert(0 && "Invalid transform size");
      break;
Jingning Han's avatar
Jingning Han committed
1011 1012 1013
  }
}

1014
static void encode_block_pass1(int plane, int block, int blk_row, int blk_col,
clang-format's avatar
clang-format committed
1015 1016
                               BLOCK_SIZE plane_bsize, TX_SIZE tx_size,
                               void *arg) {
Jingning Han's avatar
Jingning Han committed
1017 1018 1019 1020 1021 1022
  MACROBLOCK *const x = (MACROBLOCK *)arg;
  MACROBLOCKD *const xd = &x->e_mbd;
  struct macroblock_plane *const p = &x->plane[plane];
  struct macroblockd_plane *const pd = &xd->plane[plane];
  tran_low_t *const dqcoeff = BLOCK_OFFSET(pd->dqcoeff, block);
  uint8_t *dst;
1023
  dst = &pd->dst.buf[4 * blk_row * pd->dst.stride + 4 * blk_col];
Jingning Han's avatar
Jingning Han committed
1024

1025
  vp10_xform_quant(x, plane, block, blk_row, blk_col, plane_bsize, tx_size);
Jingning Han's avatar
Jingning Han committed
1026 1027

  if (p->eobs[block] > 0) {
1028
#if CONFIG_VPX_HIGHBITDEPTH
Jingning Han's avatar
Jingning Han committed
1029
    if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) {
1030
      if (xd->lossless[0]) {
clang-format's avatar
clang-format committed
1031 1032
        vp10_highbd_iwht4x4_add(dqcoeff, dst, pd->dst.stride, p->eobs[block],
                                xd->bd);
1033
      } else {
clang-format's avatar
clang-format committed
1034 1035
        vp10_highbd_idct4x4_add(dqcoeff, dst, pd->dst.stride, p->eobs[block],
                                xd->bd);
1036 1037
      }
      return;
Jingning Han's avatar
Jingning Han committed
1038
    }
1039
#endif  // CONFIG_VPX_HIGHBITDEPTH
1040
    if (xd->lossless[0]) {
1041 1042 1043 1044
      vp10_iwht4x4_add(dqcoeff, dst, pd->dst.stride, p->eobs[block]);
    } else {
      vp10_idct4x4_add(dqcoeff, dst, pd->dst.stride, p->eobs[block]);
    }
Jingning Han's avatar
Jingning Han committed
1045 1046 1047 1048 1049 1050
  }
}

void vp10_encode_sby_pass1(MACROBLOCK *x, BLOCK_SIZE bsize) {
  vp10_subtract_plane(x, bsize, 0);
  vp10_foreach_transformed_block_in_plane(&x->e_mbd, bsize, 0,
clang-format's avatar
clang-format committed
1051
                                          encode_block_pass1, x);
Jingning Han's avatar
Jingning Han committed
1052 1053 1054 1055 1056 1057
}

void vp10_encode_sb(MACROBLOCK *x, BLOCK_SIZE bsize) {
  MACROBLOCKD *const xd = &x->e_mbd;
  struct optimize_ctx ctx;
  MB_MODE_INFO *mbmi = &xd->mi[0]->mbmi;
clang-format's avatar
clang-format committed
1058
  struct encode_b_args arg = { x, &ctx, &mbmi->skip };
Jingning Han's avatar
Jingning Han committed
1059 1060 1061 1062
  int plane;

  mbmi->skip = 1;

clang-format's avatar
clang-format committed
1063
  if (x->skip) return;
Jingning Han's avatar
Jingning Han committed
1064 1065

  for (plane = 0; plane < MAX_MB_PLANE; ++plane) {
clang-format's avatar
clang-format committed
1066
    if (!x->skip_recode) vp10_subtract_plane(x, bsize, plane);
Jingning Han's avatar
Jingning Han committed
1067 1068

    if (x->optimize && (!x->skip_recode || !x->skip_optimize)) {
clang-format's avatar
clang-format committed
1069
      const struct macroblockd_plane *const pd = &xd->plane[plane];
Jingning Han's avatar
Jingning Han committed
1070
      const TX_SIZE tx_size = plane ? get_uv_tx_size(mbmi, pd) : mbmi->tx_size;
clang-format's avatar
clang-format committed
1071 1072
      vp10_get_entropy_contexts(bsize, tx_size, pd, ctx.ta[plane],
                                ctx.tl[plane]);
Jingning Han's avatar
Jingning Han committed
1073 1074 1075
    }

    vp10_foreach_transformed_block_in_plane(xd, bsize, plane, encode_block,
clang-format's avatar
clang-format committed
1076
                                            &arg);
Jingning Han's avatar
Jingning Han committed
1077 1078 1079
  }
}

1080
void vp10_encode_block_intra(int plane, int block, int blk_row, int blk_col,
clang-format's avatar
clang-format committed
1081 1082 1083
                             BLOCK_SIZE plane_bsize, TX_SIZE tx_size,
                             void *arg) {
  struct encode_b_args *const args = arg;
Jingning Han's avatar
Jingning Han committed
1084 1085 1086 1087 1088 1089 1090 1091
  MACROBLOCK *const x = args->x;
  MACROBLOCKD *const xd = &x->e_mbd;
  MB_MODE_INFO *mbmi = &xd->mi[0]->mbmi;
  struct macroblock_plane *const p = &x->plane[plane];
  struct macroblockd_plane *const pd = &xd->plane[plane];
  tran_low_t *coeff = BLOCK_OFFSET(p->coeff, block);
  tran_low_t *qcoeff = BLOCK_OFFSET(p->qcoeff, block);
  tran_low_t *dqcoeff = BLOCK_OFFSET(pd->dqcoeff, block);
hui su's avatar
hui su committed
1092 1093 1094
  PLANE_TYPE plane_type = (plane == 0) ? PLANE_TYPE_Y : PLANE_TYPE_UV;
  TX_TYPE tx_type = get_tx_type(plane_type, xd, block);
  const scan_order *const scan_order = get_scan(tx_size, tx_type);
Jingning Han's avatar