encodemb.c 50.1 KB
Newer Older
Jingning Han's avatar
Jingning Han committed
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19
/*
 *  Copyright (c) 2010 The WebM project authors. All Rights Reserved.
 *
 *  Use of this source code is governed by a BSD-style license
 *  that can be found in the LICENSE file in the root of the source
 *  tree. An additional intellectual property rights grant can be found
 *  in the file PATENTS.  All contributing project authors may
 *  be found in the AUTHORS file in the root of the source tree.
 */


#include "./vp10_rtcd.h"
#include "./vpx_config.h"
#include "./vpx_dsp_rtcd.h"

#include "vpx_dsp/quantize.h"
#include "vpx_mem/vpx_mem.h"
#include "vpx_ports/mem.h"

20 21 22 23
#include "vp10/common/idct.h"
#include "vp10/common/reconinter.h"
#include "vp10/common/reconintra.h"
#include "vp10/common/scan.h"
Jingning Han's avatar
Jingning Han committed
24

25
#include "vp10/encoder/encodemb.h"
Angie Chiang's avatar
Angie Chiang committed
26
#include "vp10/encoder/hybrid_fwd_txfm.h"
Angie Chiang's avatar
Angie Chiang committed
27
#include "vp10/encoder/quantize.h"
28 29
#include "vp10/encoder/rd.h"
#include "vp10/encoder/tokenize.h"
Jingning Han's avatar
Jingning Han committed
30 31

struct optimize_ctx {
32 33
  ENTROPY_CONTEXT ta[MAX_MB_PLANE][2 * MAX_MIB_SIZE];
  ENTROPY_CONTEXT tl[MAX_MB_PLANE][2 * MAX_MIB_SIZE];
Jingning Han's avatar
Jingning Han committed
34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59
};

void vp10_subtract_plane(MACROBLOCK *x, BLOCK_SIZE bsize, int plane) {
  struct macroblock_plane *const p = &x->plane[plane];
  const struct macroblockd_plane *const pd = &x->e_mbd.plane[plane];
  const BLOCK_SIZE plane_bsize = get_plane_block_size(bsize, pd);
  const int bw = 4 * num_4x4_blocks_wide_lookup[plane_bsize];
  const int bh = 4 * num_4x4_blocks_high_lookup[plane_bsize];

#if CONFIG_VP9_HIGHBITDEPTH
  if (x->e_mbd.cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) {
    vpx_highbd_subtract_block(bh, bw, p->src_diff, bw, p->src.buf,
                              p->src.stride, pd->dst.buf, pd->dst.stride,
                              x->e_mbd.bd);
    return;
  }
#endif  // CONFIG_VP9_HIGHBITDEPTH
  vpx_subtract_block(bh, bw, p->src_diff, bw, p->src.buf, p->src.stride,
                     pd->dst.buf, pd->dst.stride);
}

typedef struct vp10_token_state {
  int           rate;
  int           error;
  int           next;
  int16_t       token;
60
  tran_low_t    qc;
61
  tran_low_t    dqc;
Jingning Han's avatar
Jingning Han committed
62 63
} vp10_token_state;

64 65 66 67
// These numbers are empirically obtained.
static const int plane_rd_mult[REF_TYPES][PLANE_TYPES] = {
    {10, 6}, {8, 5},
};
Jingning Han's avatar
Jingning Han committed
68 69 70 71 72 73 74

#define UPDATE_RD_COST()\
{\
  rd_cost0 = RDCOST(rdmult, rddiv, rate0, error0);\
  rd_cost1 = RDCOST(rdmult, rddiv, rate1, error1);\
}

75 76 77 78 79 80 81 82 83 84 85 86 87 88
static const int16_t band_count_table[TX_SIZES][8] = {
  { 1, 2, 3, 4,  3,   16 - 13, 0 },
  { 1, 2, 3, 4, 11,   64 - 21, 0 },
  { 1, 2, 3, 4, 11,  256 - 21, 0 },
  { 1, 2, 3, 4, 11, 1024 - 21, 0 },
};

static const int16_t band_cum_count_table[TX_SIZES][8] = {
  { 0, 1, 3, 6, 10, 13, 16, 0 },
  { 0, 1, 3, 6, 10, 21, 64, 0 },
  { 0, 1, 3, 6, 10, 21, 256, 0 },
  { 0, 1, 3, 6, 10, 21, 1024, 0 },
};

89 90
int vp10_optimize_b(MACROBLOCK *mb, int plane, int block,
                    TX_SIZE tx_size, int ctx) {
Jingning Han's avatar
Jingning Han committed
91 92 93 94
  MACROBLOCKD *const xd = &mb->e_mbd;
  struct macroblock_plane *const p = &mb->plane[plane];
  struct macroblockd_plane *const pd = &xd->plane[plane];
  const int ref = is_inter_block(&xd->mi[0]->mbmi);
95 96
  vp10_token_state tokens[MAX_TX_SQUARE + 1][2];
  unsigned best_index[MAX_TX_SQUARE + 1][2];
97
  uint8_t token_cache[MAX_TX_SQUARE];
Jingning Han's avatar
Jingning Han committed
98 99 100 101 102 103
  const tran_low_t *const coeff = BLOCK_OFFSET(mb->plane[plane].coeff, block);
  tran_low_t *const qcoeff = BLOCK_OFFSET(p->qcoeff, block);
  tran_low_t *const dqcoeff = BLOCK_OFFSET(pd->dqcoeff, block);
  const int eob = p->eobs[block];
  const PLANE_TYPE type = pd->plane_type;
  const int default_eob = 16 << (tx_size << 1);
104 105
  const int16_t* const dequant_ptr = pd->dequant;
  const uint8_t* const band_translate = get_band_translate(tx_size);
hui su's avatar
hui su committed
106
  TX_TYPE tx_type = get_tx_type(type, xd, block, tx_size);
107
  const scan_order* const so =
108
      get_scan(tx_size, tx_type, is_inter_block(&xd->mi[0]->mbmi));
109 110
  const int16_t* const scan = so->scan;
  const int16_t* const nb = so->neighbors;
111 112 113
#if CONFIG_NEW_QUANT
  const dequant_val_type_nuq *dequant_val = pd->dequant_val_nuq;
#endif  // CONFIG_NEW_QUANT
114 115
  const int shift = get_tx_scale(xd, tx_type, tx_size);
  const int dq_step[2] = { dequant_ptr[0] >> shift, dequant_ptr[1] >> shift };
Jingning Han's avatar
Jingning Han committed
116
  int next = eob, sz = 0;
hui su's avatar
hui su committed
117 118
  const int64_t rdmult = (mb->rdmult * plane_rd_mult[ref][type]) >> 1;
  const int64_t rddiv = mb->rddiv;
Jingning Han's avatar
Jingning Han committed
119 120 121
  int64_t rd_cost0, rd_cost1;
  int rate0, rate1, error0, error1;
  int16_t t0, t1;
122 123 124
  int best, band = (eob < default_eob) ?
      band_translate[eob] : band_translate[eob - 1];
  int pt, i, final_eob;
Jingning Han's avatar
Jingning Han committed
125
#if CONFIG_VP9_HIGHBITDEPTH
126
  const int *cat6_high_cost = vp10_get_high_cost_table(xd->bd);
Jingning Han's avatar
Jingning Han committed
127
#else
128
  const int *cat6_high_cost = vp10_get_high_cost_table(8);
Jingning Han's avatar
Jingning Han committed
129
#endif
130 131
  unsigned int (*token_costs)[2][COEFF_CONTEXTS][ENTROPY_TOKENS] =
                   mb->token_costs[tx_size][type][ref];
132 133
  const int16_t *band_counts = &band_count_table[tx_size][band];
  int16_t band_left = eob - band_cum_count_table[tx_size][band] + 1;
134 135
  int shortcut = 0;
  int next_shortcut = 0;
136 137

  token_costs += band;
Jingning Han's avatar
Jingning Han committed
138 139 140

  assert((!type && !plane) || (type && plane));
  assert(eob <= default_eob);
141

Jingning Han's avatar
Jingning Han committed
142 143 144 145 146 147 148 149 150
  /* Now set up a Viterbi trellis to evaluate alternative roundings. */
  /* Initialize the sentinel node of the trellis. */
  tokens[eob][0].rate = 0;
  tokens[eob][0].error = 0;
  tokens[eob][0].next = default_eob;
  tokens[eob][0].token = EOB_TOKEN;
  tokens[eob][0].qc = 0;
  tokens[eob][1] = tokens[eob][0];

151 152
  for (i = 0; i < eob; i++) {
    const int rc = scan[i];
153 154 155
    tokens[i][0].rate = vp10_get_token_cost(qcoeff[rc], &t0, cat6_high_cost);
    tokens[i][0].token = t0;
    token_cache[rc] = vp10_pt_energy_class[t0];
156
  }
Jingning Han's avatar
Jingning Han committed
157 158 159 160 161

  for (i = eob; i-- > 0;) {
    int base_bits, d2, dx;
    const int rc = scan[i];
    int x = qcoeff[rc];
162
    next_shortcut = shortcut;
163

Jingning Han's avatar
Jingning Han committed
164 165 166 167 168 169 170
    /* Only add a trellis state for non-zero coefficients. */
    if (x) {
      error0 = tokens[next][0].error;
      error1 = tokens[next][1].error;
      /* Evaluate the first possibility for this state. */
      rate0 = tokens[next][0].rate;
      rate1 = tokens[next][1].rate;
171

172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187
      if (next_shortcut) {
        /* Consider both possible successor states. */
        if (next < default_eob) {
          pt = get_coef_context(nb, token_cache, i + 1);
          rate0 += (*token_costs)[0][pt][tokens[next][0].token];
          rate1 += (*token_costs)[0][pt][tokens[next][1].token];
        }
        UPDATE_RD_COST();
        /* And pick the best. */
        best = rd_cost1 < rd_cost0;
      } else {
        if (next < default_eob) {
          pt = get_coef_context(nb, token_cache, i + 1);
          rate0 += (*token_costs)[0][pt][tokens[next][0].token];
        }
        best = 0;
Jingning Han's avatar
Jingning Han committed
188
      }
189 190

      dx = (dqcoeff[rc] - coeff[rc]) * (1 << shift);
Jingning Han's avatar
Jingning Han committed
191 192 193 194 195 196
#if CONFIG_VP9_HIGHBITDEPTH
      if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) {
        dx >>= xd->bd - 8;
      }
#endif  // CONFIG_VP9_HIGHBITDEPTH
      d2 = dx * dx;
197
      tokens[i][0].rate += (best ? rate1 : rate0);
Jingning Han's avatar
Jingning Han committed
198 199 200
      tokens[i][0].error = d2 + (best ? error1 : error0);
      tokens[i][0].next = next;
      tokens[i][0].qc = x;
201
      tokens[i][0].dqc = dqcoeff[rc];
Jingning Han's avatar
Jingning Han committed
202 203 204 205 206 207
      best_index[i][0] = best;

      /* Evaluate the second possibility for this state. */
      rate0 = tokens[next][0].rate;
      rate1 = tokens[next][1].rate;

208 209 210 211
#if CONFIG_NEW_QUANT
      shortcut = (
          (dequant_abscoeff_nuq(
              abs(x), dequant_ptr[rc != 0],
212
              dequant_val[band_translate[i]]) > (abs(coeff[rc]) << shift)) &&
213 214
          (dequant_abscoeff_nuq(
              abs(x) - 1, dequant_ptr[rc != 0],
215
              dequant_val[band_translate[i]]) < (abs(coeff[rc]) << shift)));
216
#else   // CONFIG_NEW_QUANT
217 218
      if ((abs(x) * dequant_ptr[rc != 0] > (abs(coeff[rc]) << shift)) &&
          (abs(x) * dequant_ptr[rc != 0] < (abs(coeff[rc]) << shift) +
Jingning Han's avatar
Jingning Han committed
219 220 221 222
                                               dequant_ptr[rc != 0]))
        shortcut = 1;
      else
        shortcut = 0;
223
#endif   // CONFIG_NEW_QUANT
Jingning Han's avatar
Jingning Han committed
224 225 226 227

      if (shortcut) {
        sz = -(x < 0);
        x -= 2 * sz + 1;
228 229 230 231
      } else {
        tokens[i][1] = tokens[i][0];
        best_index[i][1] = best_index[i][0];
        next = i;
232 233 234 235 236 237

        if (!(--band_left)) {
          --band_counts;
          band_left = *band_counts;
          --token_costs;
        }
238
        continue;
Jingning Han's avatar
Jingning Han committed
239 240 241 242 243 244 245 246 247
      }

      /* Consider both possible successor states. */
      if (!x) {
        /* If we reduced this coefficient to zero, check to see if
         *  we need to move the EOB back here.
         */
        t0 = tokens[next][0].token == EOB_TOKEN ? EOB_TOKEN : ZERO_TOKEN;
        t1 = tokens[next][1].token == EOB_TOKEN ? EOB_TOKEN : ZERO_TOKEN;
248
        base_bits = 0;
Jingning Han's avatar
Jingning Han committed
249
      } else {
250
        base_bits = vp10_get_token_cost(x, &t0, cat6_high_cost);
Jingning Han's avatar
Jingning Han committed
251 252
        t1 = t0;
      }
253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273

      if (next_shortcut) {
        if (next < default_eob) {
          if (t0 != EOB_TOKEN) {
            token_cache[rc] = vp10_pt_energy_class[t0];
            pt = get_coef_context(nb, token_cache, i + 1);
            rate0 += (*token_costs)[!x][pt][tokens[next][0].token];
          }
          if (t1 != EOB_TOKEN) {
            token_cache[rc] = vp10_pt_energy_class[t1];
            pt = get_coef_context(nb, token_cache, i + 1);
            rate1 += (*token_costs)[!x][pt][tokens[next][1].token];
          }
        }

        UPDATE_RD_COST();
        /* And pick the best. */
        best = rd_cost1 < rd_cost0;
      } else {
        // The two states in next stage are identical.
        if (next < default_eob && t0 != EOB_TOKEN) {
Jingning Han's avatar
Jingning Han committed
274 275
          token_cache[rc] = vp10_pt_energy_class[t0];
          pt = get_coef_context(nb, token_cache, i + 1);
276
          rate0 += (*token_costs)[!x][pt][tokens[next][0].token];
Jingning Han's avatar
Jingning Han committed
277
        }
278
        best = 0;
Jingning Han's avatar
Jingning Han committed
279 280 281
      }

      if (shortcut) {
282 283 284
#if CONFIG_NEW_QUANT
        dx = dequant_coeff_nuq(
            x, dequant_ptr[rc != 0],
285
            dequant_val[band_translate[i]]) - (coeff[rc] << shift);
286 287 288 289 290 291
#if CONFIG_VP9_HIGHBITDEPTH
        if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) {
          dx >>= xd->bd - 8;
        }
#endif  // CONFIG_VP9_HIGHBITDEPTH
#else   // CONFIG_NEW_QUANT
Jingning Han's avatar
Jingning Han committed
292 293 294 295 296 297 298 299 300
#if CONFIG_VP9_HIGHBITDEPTH
        if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) {
          dx -= ((dequant_ptr[rc != 0] >> (xd->bd - 8)) + sz) ^ sz;
        } else {
          dx -= (dequant_ptr[rc != 0] + sz) ^ sz;
        }
#else
        dx -= (dequant_ptr[rc != 0] + sz) ^ sz;
#endif  // CONFIG_VP9_HIGHBITDEPTH
301
#endif  // CONFIG_NEW_QUANT
Jingning Han's avatar
Jingning Han committed
302 303 304 305 306 307 308
        d2 = dx * dx;
      }
      tokens[i][1].rate = base_bits + (best ? rate1 : rate0);
      tokens[i][1].error = d2 + (best ? error1 : error0);
      tokens[i][1].next = next;
      tokens[i][1].token = best ? t1 : t0;
      tokens[i][1].qc = x;
309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326

      if (x) {
        tran_low_t offset = dq_step[rc != 0];
        // The 32x32 transform coefficient uses half quantization step size.
        // Account for the rounding difference in the dequantized coefficeint
        // value when the quantization index is dropped from an even number
        // to an odd number.
        if (shift & x)
          offset += (dequant_ptr[rc != 0] & 0x01);

        if (sz == 0)
          tokens[i][1].dqc = dqcoeff[rc] - offset;
        else
          tokens[i][1].dqc = dqcoeff[rc] + offset;
      } else {
        tokens[i][1].dqc = 0;
      }

Jingning Han's avatar
Jingning Han committed
327 328 329 330 331 332 333 334 335
      best_index[i][1] = best;
      /* Finally, make this the new head of the trellis. */
      next = i;
    } else {
      /* There's no choice to make for a zero coefficient, so we don't
       *  add a new trellis node, but we do need to update the costs.
       */
      t0 = tokens[next][0].token;
      t1 = tokens[next][1].token;
336
      pt = get_coef_context(nb, token_cache, i + 1);
Jingning Han's avatar
Jingning Han committed
337 338
      /* Update the cost of each path if we're past the EOB token. */
      if (t0 != EOB_TOKEN) {
339
        tokens[next][0].rate += (*token_costs)[1][pt][t0];
Jingning Han's avatar
Jingning Han committed
340 341 342
        tokens[next][0].token = ZERO_TOKEN;
      }
      if (t1 != EOB_TOKEN) {
343
        tokens[next][1].rate += (*token_costs)[1][pt][t1];
Jingning Han's avatar
Jingning Han committed
344 345 346
        tokens[next][1].token = ZERO_TOKEN;
      }
      best_index[i][0] = best_index[i][1] = 0;
347
      shortcut = (tokens[next][0].rate != tokens[next][1].rate);
Jingning Han's avatar
Jingning Han committed
348 349
      /* Don't update next, because we didn't add a new node. */
    }
350 351 352 353 354 355

    if (!(--band_left)) {
      --band_counts;
      band_left = *band_counts;
      --token_costs;
    }
Jingning Han's avatar
Jingning Han committed
356 357 358 359 360 361 362 363 364
  }

  /* Now pick the best path through the whole trellis. */
  rate0 = tokens[next][0].rate;
  rate1 = tokens[next][1].rate;
  error0 = tokens[next][0].error;
  error1 = tokens[next][1].error;
  t0 = tokens[next][0].token;
  t1 = tokens[next][1].token;
365 366
  rate0 += (*token_costs)[0][ctx][t0];
  rate1 += (*token_costs)[0][ctx][t1];
Jingning Han's avatar
Jingning Han committed
367 368
  UPDATE_RD_COST();
  best = rd_cost1 < rd_cost0;
369

Jingning Han's avatar
Jingning Han committed
370
  final_eob = -1;
371

Jingning Han's avatar
Jingning Han committed
372 373 374
  for (i = next; i < eob; i = next) {
    const int x = tokens[i][best].qc;
    const int rc = scan[i];
375 376

    if (x) final_eob = i;
Jingning Han's avatar
Jingning Han committed
377
    qcoeff[rc] = x;
378 379
    dqcoeff[rc] = tokens[i][best].dqc;

380 381 382 383 384 385
#if CONFIG_NEW_QUANT
    dqcoeff[rc] = dequant_abscoeff_nuq(abs(x), dequant_ptr[rc != 0],
                                       dequant_val[band_translate[i]]);
    if (shift) dqcoeff[rc] = ROUND_POWER_OF_TWO(dqcoeff[rc], shift);
    if (x < 0) dqcoeff[rc] = -dqcoeff[rc];
#endif  // CONFIG_NEW_QUANT
Jingning Han's avatar
Jingning Han committed
386 387 388 389 390 391 392

    next = tokens[i][best].next;
    best = best_index[i][best];
  }
  final_eob++;

  mb->plane[plane].eobs[block] = final_eob;
393
  assert(final_eob <= default_eob);
Jingning Han's avatar
Jingning Han committed
394 395 396
  return final_eob;
}

397
#if CONFIG_VP9_HIGHBITDEPTH
Angie Chiang's avatar
Angie Chiang committed
398 399
typedef enum QUANT_FUNC {
  QUANT_FUNC_LOWBD = 0,
400 401
  QUANT_FUNC_HIGHBD = 1,
  QUANT_FUNC_LAST = 2
Angie Chiang's avatar
Angie Chiang committed
402 403
} QUANT_FUNC;

404 405
static VP10_QUANT_FACADE
    quant_func_list[VP10_XFORM_QUANT_LAST][QUANT_FUNC_LAST] = {
406 407 408 409
        {vp10_quantize_fp_facade, vp10_highbd_quantize_fp_facade},
        {vp10_quantize_b_facade, vp10_highbd_quantize_b_facade},
        {vp10_quantize_dc_facade, vp10_highbd_quantize_dc_facade},
        {NULL, NULL}};
410

Angie Chiang's avatar
Angie Chiang committed
411 412 413
#else
typedef enum QUANT_FUNC {
  QUANT_FUNC_LOWBD = 0,
414
  QUANT_FUNC_LAST = 1
Angie Chiang's avatar
Angie Chiang committed
415
} QUANT_FUNC;
Angie Chiang's avatar
Angie Chiang committed
416

417 418
static VP10_QUANT_FACADE
    quant_func_list[VP10_XFORM_QUANT_LAST][QUANT_FUNC_LAST] = {
419 420 421 422
        {vp10_quantize_fp_facade},
        {vp10_quantize_b_facade},
        {vp10_quantize_dc_facade},
        {NULL}};
Angie Chiang's avatar
Angie Chiang committed
423
#endif
424

425 426 427
static FWD_TXFM_OPT fwd_txfm_opt_list[VP10_XFORM_QUANT_LAST] = {
    FWD_TXFM_OPT_NORMAL, FWD_TXFM_OPT_NORMAL, FWD_TXFM_OPT_DC,
    FWD_TXFM_OPT_NORMAL};
428

Angie Chiang's avatar
Angie Chiang committed
429 430 431
void vp10_xform_quant(MACROBLOCK *x, int plane, int block, int blk_row,
                      int blk_col, BLOCK_SIZE plane_bsize, TX_SIZE tx_size,
                      VP10_XFORM_QUANT xform_quant_idx) {
Jingning Han's avatar
Jingning Han committed
432 433 434
  MACROBLOCKD *const xd = &x->e_mbd;
  const struct macroblock_plane *const p = &x->plane[plane];
  const struct macroblockd_plane *const pd = &xd->plane[plane];
hui su's avatar
hui su committed
435
  PLANE_TYPE plane_type = (plane == 0) ? PLANE_TYPE_Y : PLANE_TYPE_UV;
hui su's avatar
hui su committed
436
  TX_TYPE tx_type = get_tx_type(plane_type, xd, block, tx_size);
437 438
  const scan_order *const scan_order =
      get_scan(tx_size, tx_type, is_inter_block(&xd->mi[0]->mbmi));
Jingning Han's avatar
Jingning Han committed
439 440 441 442 443 444
  tran_low_t *const coeff = BLOCK_OFFSET(p->coeff, block);
  tran_low_t *const qcoeff = BLOCK_OFFSET(p->qcoeff, block);
  tran_low_t *const dqcoeff = BLOCK_OFFSET(pd->dqcoeff, block);
  uint16_t *const eob = &p->eobs[block];
  const int diff_stride = 4 * num_4x4_blocks_wide_lookup[plane_bsize];
  const int16_t *src_diff;
Angie Chiang's avatar
Angie Chiang committed
445 446
  const int tx1d_size = get_tx1d_size(tx_size);
  const int tx2d_size = tx1d_size * tx1d_size;
Angie Chiang's avatar
Angie Chiang committed
447 448

  FWD_TXFM_PARAM fwd_txfm_param;
449 450 451
  QUANT_PARAM qparam;

  fwd_txfm_param.tx_type = tx_type;
Angie Chiang's avatar
Angie Chiang committed
452
  fwd_txfm_param.tx_size = tx_size;
453
  fwd_txfm_param.fwd_txfm_opt = fwd_txfm_opt_list[xform_quant_idx];
Angie Chiang's avatar
Angie Chiang committed
454 455 456
  fwd_txfm_param.rd_transform = x->use_lp32x32fdct;
  fwd_txfm_param.lossless = xd->lossless[xd->mi[0]->mbmi.segment_id];

457
  src_diff = &p->src_diff[4 * (blk_row * diff_stride + blk_col)];
Jingning Han's avatar
Jingning Han committed
458

459
  qparam.log_scale = get_tx_scale(xd, tx_type, tx_size);
Jingning Han's avatar
Jingning Han committed
460
#if CONFIG_VP9_HIGHBITDEPTH
461
  fwd_txfm_param.bd = xd->bd;
Jingning Han's avatar
Jingning Han committed
462
  if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) {
Angie Chiang's avatar
Angie Chiang committed
463
    highbd_fwd_txfm(src_diff, coeff, diff_stride, &fwd_txfm_param);
464 465 466 467
    if (xform_quant_idx != VP10_XFORM_QUANT_SKIP_QUANT) {
      if (x->skip_block) {
        vp10_quantize_skip(tx2d_size, qcoeff, dqcoeff, eob);
      } else {
468 469 470
        quant_func_list[xform_quant_idx][QUANT_FUNC_HIGHBD](
            coeff, tx2d_size, p, qcoeff, pd, dqcoeff, eob,
            scan_order, &qparam);
471 472 473 474 475 476 477 478
      }
    }
    return;
  }
#endif  // CONFIG_VP9_HIGHBITDEPTH

  fwd_txfm(src_diff, coeff, diff_stride, &fwd_txfm_param);
  if (xform_quant_idx != VP10_XFORM_QUANT_SKIP_QUANT) {
Angie Chiang's avatar
Angie Chiang committed
479 480 481
    if (x->skip_block) {
      vp10_quantize_skip(tx2d_size, qcoeff, dqcoeff, eob);
    } else {
482 483 484
      quant_func_list[xform_quant_idx][QUANT_FUNC_LOWBD](
          coeff, tx2d_size, p, qcoeff, pd, dqcoeff, eob,
          scan_order, &qparam);
Jingning Han's avatar
Jingning Han committed
485 486 487 488
    }
  }
}

489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951
#if CONFIG_NEW_QUANT
void vp10_xform_quant_nuq(MACROBLOCK *x, int plane, int block, int blk_row,
                          int blk_col, BLOCK_SIZE plane_bsize,
                          TX_SIZE tx_size) {
  MACROBLOCKD *const xd = &x->e_mbd;
  const struct macroblock_plane *const p = &x->plane[plane];
  const struct macroblockd_plane *const pd = &xd->plane[plane];
  PLANE_TYPE plane_type = (plane == 0) ? PLANE_TYPE_Y : PLANE_TYPE_UV;
  TX_TYPE tx_type = get_tx_type(plane_type, xd, block, tx_size);
  const scan_order *const scan_order =
      get_scan(tx_size, tx_type, is_inter_block(&xd->mi[0]->mbmi));
  tran_low_t *const coeff = BLOCK_OFFSET(p->coeff, block);
  tran_low_t *const qcoeff = BLOCK_OFFSET(p->qcoeff, block);
  tran_low_t *const dqcoeff = BLOCK_OFFSET(pd->dqcoeff, block);
  uint16_t *const eob = &p->eobs[block];
  const int diff_stride = 4 * num_4x4_blocks_wide_lookup[plane_bsize];
  const int16_t *src_diff;
  const uint8_t* band = get_band_translate(tx_size);

  FWD_TXFM_PARAM fwd_txfm_param;

  fwd_txfm_param.tx_type = tx_type;
  fwd_txfm_param.tx_size = tx_size;
  fwd_txfm_param.fwd_txfm_opt = fwd_txfm_opt_list[VP10_XFORM_QUANT_B];
  fwd_txfm_param.rd_transform = x->use_lp32x32fdct;
  fwd_txfm_param.lossless = xd->lossless[xd->mi[0]->mbmi.segment_id];

  src_diff = &p->src_diff[4 * (blk_row * diff_stride + blk_col)];

// TODO(sarahparker) add all of these new quant quantize functions
// to quant_func_list, just trying to get this expr to work for now
#if CONFIG_VP9_HIGHBITDEPTH
  fwd_txfm_param.bd = xd->bd;
  if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) {
    highbd_fwd_txfm(src_diff, coeff, diff_stride, &fwd_txfm_param);
    switch (tx_size) {
      case TX_32X32:
        highbd_quantize_32x32_nuq(coeff, 1024, x->skip_block,
                                  p->quant, p->quant_shift, pd->dequant,
                                  (const cuml_bins_type_nuq *)
                                      p->cuml_bins_nuq,
                                  (const dequant_val_type_nuq *)
                                      pd->dequant_val_nuq,
                                  qcoeff, dqcoeff, eob,
                                  scan_order->scan, band);
        break;
      case TX_16X16:
        highbd_quantize_nuq(coeff, 256, x->skip_block,
                            p->quant, p->quant_shift, pd->dequant,
                            (const cuml_bins_type_nuq *)p->cuml_bins_nuq,
                            (const dequant_val_type_nuq *)
                                pd->dequant_val_nuq,
                            qcoeff, dqcoeff, eob,
                            scan_order->scan, band);
        break;
      case TX_8X8:
        highbd_quantize_nuq(coeff, 64, x->skip_block,
                            p->quant, p->quant_shift, pd->dequant,
                            (const cuml_bins_type_nuq *)p->cuml_bins_nuq,
                            (const dequant_val_type_nuq *)
                                pd->dequant_val_nuq,
                            qcoeff, dqcoeff, eob,
                            scan_order->scan, band);
        break;
      case TX_4X4:
        highbd_quantize_nuq(coeff, 16, x->skip_block,
                            p->quant, p->quant_shift, pd->dequant,
                            (const cuml_bins_type_nuq *)p->cuml_bins_nuq,
                            (const dequant_val_type_nuq *)
                                pd->dequant_val_nuq,
                            qcoeff, dqcoeff, eob,
                            scan_order->scan, band);
        break;
      default:
        assert(0);
    }
    return;
  }
#endif  // CONFIG_VP9_HIGHBITDEPTH

  fwd_txfm(src_diff, coeff, diff_stride, &fwd_txfm_param);
  switch (tx_size) {
    case TX_32X32:
      quantize_32x32_nuq(coeff, 1024, x->skip_block,
                         p->quant, p->quant_shift, pd->dequant,
                         (const cuml_bins_type_nuq *)p->cuml_bins_nuq,
                         (const dequant_val_type_nuq *)
                         pd->dequant_val_nuq,
                         qcoeff, dqcoeff, eob,
                         scan_order->scan, band);
      break;
    case TX_16X16:
      quantize_nuq(coeff, 256, x->skip_block,
                   p->quant, p->quant_shift, pd->dequant,
                   (const cuml_bins_type_nuq *)p->cuml_bins_nuq,
                   (const dequant_val_type_nuq *)pd->dequant_val_nuq,
                   qcoeff, dqcoeff, eob,
                   scan_order->scan, band);
      break;
    case TX_8X8:
      quantize_nuq(coeff, 64, x->skip_block,
                   p->quant, p->quant_shift, pd->dequant,
                   (const cuml_bins_type_nuq *)p->cuml_bins_nuq,
                   (const dequant_val_type_nuq *)pd->dequant_val_nuq,
                   qcoeff, dqcoeff, eob,
                   scan_order->scan, band);
      break;
    case TX_4X4:
      quantize_nuq(coeff, 16, x->skip_block,
                   p->quant, p->quant_shift, pd->dequant,
                   (const cuml_bins_type_nuq *)p->cuml_bins_nuq,
                   (const dequant_val_type_nuq *)pd->dequant_val_nuq,
                   qcoeff, dqcoeff, eob,
                   scan_order->scan, band);
      break;
    default:
      assert(0);
      break;
  }
}

void vp10_xform_quant_fp_nuq(MACROBLOCK *x, int plane, int block, int blk_row,
                             int blk_col, BLOCK_SIZE plane_bsize,
                             TX_SIZE tx_size) {
  MACROBLOCKD *const xd = &x->e_mbd;
  const struct macroblock_plane *const p = &x->plane[plane];
  const struct macroblockd_plane *const pd = &xd->plane[plane];
  PLANE_TYPE plane_type = (plane == 0) ? PLANE_TYPE_Y : PLANE_TYPE_UV;
  TX_TYPE tx_type = get_tx_type(plane_type, xd, block, tx_size);
  const scan_order *const scan_order =
      get_scan(tx_size, tx_type, is_inter_block(&xd->mi[0]->mbmi));
  tran_low_t *const coeff = BLOCK_OFFSET(p->coeff, block);
  tran_low_t *const qcoeff = BLOCK_OFFSET(p->qcoeff, block);
  tran_low_t *const dqcoeff = BLOCK_OFFSET(pd->dqcoeff, block);
  uint16_t *const eob = &p->eobs[block];
  const int diff_stride = 4 * num_4x4_blocks_wide_lookup[plane_bsize];
  const int16_t *src_diff;
  const uint8_t* band = get_band_translate(tx_size);

  FWD_TXFM_PARAM fwd_txfm_param;

  fwd_txfm_param.tx_type = tx_type;
  fwd_txfm_param.tx_size = tx_size;
  fwd_txfm_param.fwd_txfm_opt = fwd_txfm_opt_list[VP10_XFORM_QUANT_FP];
  fwd_txfm_param.rd_transform = x->use_lp32x32fdct;
  fwd_txfm_param.lossless = xd->lossless[xd->mi[0]->mbmi.segment_id];

  src_diff = &p->src_diff[4 * (blk_row * diff_stride + blk_col)];

// TODO(sarahparker) add all of these new quant quantize functions
// to quant_func_list, just trying to get this expr to work for now
#if CONFIG_VP9_HIGHBITDEPTH
  fwd_txfm_param.bd = xd->bd;
  if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) {
    highbd_fwd_txfm(src_diff, coeff, diff_stride, &fwd_txfm_param);
    switch (tx_size) {
      case TX_32X32:
        highbd_quantize_32x32_fp_nuq(coeff, 1024, x->skip_block,
                                     p->quant_fp, pd->dequant,
                                     (const cuml_bins_type_nuq *)
                                         p->cuml_bins_nuq,
                                     (const dequant_val_type_nuq *)
                                         pd->dequant_val_nuq,
                                     qcoeff, dqcoeff, eob,
                                     scan_order->scan, band);
        break;
      case TX_16X16:
        highbd_quantize_fp_nuq(coeff, 256, x->skip_block,
                               p->quant_fp, pd->dequant,
                               (const cuml_bins_type_nuq *)
                                  p->cuml_bins_nuq,
                               (const dequant_val_type_nuq *)
                                   pd->dequant_val_nuq,
                               qcoeff, dqcoeff, eob,
                               scan_order->scan, band);
        break;
      case TX_8X8:
        highbd_quantize_fp_nuq(coeff, 64, x->skip_block,
                               p->quant_fp, pd->dequant,
                               (const cuml_bins_type_nuq *)
                                  p->cuml_bins_nuq,
                               (const dequant_val_type_nuq *)
                                   pd->dequant_val_nuq,
                               qcoeff, dqcoeff, eob,
                               scan_order->scan, band);
        break;
      case TX_4X4:
        highbd_quantize_fp_nuq(coeff, 16, x->skip_block,
                               p->quant_fp, pd->dequant,
                               (const cuml_bins_type_nuq *)
                                   p->cuml_bins_nuq,
                               (const dequant_val_type_nuq *)
                                   pd->dequant_val_nuq,
                               qcoeff, dqcoeff, eob,
                               scan_order->scan, band);
        break;
      default:
        assert(0);
    }
    return;
  }
#endif  // CONFIG_VP9_HIGHBITDEPTH

  fwd_txfm(src_diff, coeff, diff_stride, &fwd_txfm_param);
  switch (tx_size) {
    case TX_32X32:
      quantize_32x32_fp_nuq(coeff, 1024, x->skip_block,
                            p->quant_fp, pd->dequant,
                            (const cuml_bins_type_nuq *)
                                p->cuml_bins_nuq,
                            (const dequant_val_type_nuq *)
                                pd->dequant_val_nuq,
                            qcoeff, dqcoeff, eob,
                            scan_order->scan, band);
      break;
    case TX_16X16:
      quantize_fp_nuq(coeff, 256, x->skip_block,
                      p->quant_fp, pd->dequant,
                      (const cuml_bins_type_nuq *)
                          p->cuml_bins_nuq,
                      (const dequant_val_type_nuq *)
                          pd->dequant_val_nuq,
                      qcoeff, dqcoeff, eob,
                      scan_order->scan, band);
      break;
    case TX_8X8:
      quantize_fp_nuq(coeff, 64, x->skip_block,
                      p->quant_fp, pd->dequant,
                      (const cuml_bins_type_nuq *)
                          p->cuml_bins_nuq,
                      (const dequant_val_type_nuq *)
                          pd->dequant_val_nuq,
                      qcoeff, dqcoeff, eob,
                      scan_order->scan, band);
      break;
    case TX_4X4:
      quantize_fp_nuq(coeff, 16, x->skip_block,
                      p->quant_fp, pd->dequant,
                      (const cuml_bins_type_nuq *)
                          p->cuml_bins_nuq,
                      (const dequant_val_type_nuq *)
                          pd->dequant_val_nuq,
                      qcoeff, dqcoeff, eob,
                      scan_order->scan, band);
      break;
    default:
      assert(0);
      break;
  }
}

void vp10_xform_quant_dc_nuq(MACROBLOCK *x, int plane, int block, int blk_row,
                             int blk_col, BLOCK_SIZE plane_bsize,
                             TX_SIZE tx_size) {
  MACROBLOCKD *const xd = &x->e_mbd;
  const struct macroblock_plane *const p = &x->plane[plane];
  const struct macroblockd_plane *const pd = &xd->plane[plane];
  PLANE_TYPE plane_type = (plane == 0) ? PLANE_TYPE_Y : PLANE_TYPE_UV;
  TX_TYPE tx_type = get_tx_type(plane_type, xd, block, tx_size);
  tran_low_t *const coeff = BLOCK_OFFSET(p->coeff, block);
  tran_low_t *const qcoeff = BLOCK_OFFSET(p->qcoeff, block);
  tran_low_t *const dqcoeff = BLOCK_OFFSET(pd->dqcoeff, block);
  uint16_t *const eob = &p->eobs[block];
  const int diff_stride = 4 * num_4x4_blocks_wide_lookup[plane_bsize];
  const int16_t *src_diff;

  FWD_TXFM_PARAM fwd_txfm_param;

  fwd_txfm_param.tx_type = tx_type;
  fwd_txfm_param.tx_size = tx_size;
  fwd_txfm_param.fwd_txfm_opt = fwd_txfm_opt_list[VP10_XFORM_QUANT_DC];
  fwd_txfm_param.rd_transform = x->use_lp32x32fdct;
  fwd_txfm_param.lossless = xd->lossless[xd->mi[0]->mbmi.segment_id];

  src_diff = &p->src_diff[4 * (blk_row * diff_stride + blk_col)];

// TODO(sarahparker) add all of these new quant quantize functions
// to quant_func_list, just trying to get this expr to work for now
#if CONFIG_VP9_HIGHBITDEPTH
  fwd_txfm_param.bd = xd->bd;
  if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) {
    highbd_fwd_txfm(src_diff, coeff, diff_stride, &fwd_txfm_param);
    switch (tx_size) {
      case TX_32X32:
        highbd_quantize_dc_32x32_nuq(coeff, 1024, x->skip_block,
                                     p->quant[0], p->quant_shift[0],
                                     pd->dequant[0],
                                     p->cuml_bins_nuq[0],
                                     pd->dequant_val_nuq[0],
                                     qcoeff, dqcoeff, eob);
        break;
      case TX_16X16:
        highbd_quantize_dc_nuq(coeff, 256, x->skip_block,
                               p->quant[0], p->quant_shift[0],
                               pd->dequant[0],
                               p->cuml_bins_nuq[0],
                               pd->dequant_val_nuq[0],
                               qcoeff, dqcoeff, eob);
        break;
      case TX_8X8:
        highbd_quantize_dc_nuq(coeff, 64, x->skip_block,
                               p->quant[0], p->quant_shift[0],
                               pd->dequant[0],
                               p->cuml_bins_nuq[0],
                               pd->dequant_val_nuq[0],
                               qcoeff, dqcoeff, eob);
        break;
      case TX_4X4:
        highbd_quantize_dc_nuq(coeff, 16, x->skip_block,
                               p->quant[0], p->quant_shift[0],
                               pd->dequant[0],
                               p->cuml_bins_nuq[0],
                               pd->dequant_val_nuq[0],
                               qcoeff, dqcoeff, eob);
        break;
      default:
        assert(0);
    }
    return;
  }
#endif  // CONFIG_VP9_HIGHBITDEPTH

  fwd_txfm(src_diff, coeff, diff_stride, &fwd_txfm_param);
  switch (tx_size) {
    case TX_32X32:
      quantize_dc_32x32_nuq(coeff, 1024, x->skip_block,
                            p->quant[0], p->quant_shift[0], pd->dequant[0],
                            p->cuml_bins_nuq[0],
                            pd->dequant_val_nuq[0],
                            qcoeff, dqcoeff, eob);
      break;
    case TX_16X16:
      quantize_dc_nuq(coeff, 256, x->skip_block,
                      p->quant[0], p->quant_shift[0], pd->dequant[0],
                      p->cuml_bins_nuq[0],
                      pd->dequant_val_nuq[0],
                      qcoeff, dqcoeff, eob);
      break;
    case TX_8X8:
      quantize_dc_nuq(coeff, 64, x->skip_block,
                      p->quant[0], p->quant_shift[0], pd->dequant[0],
                      p->cuml_bins_nuq[0],
                      pd->dequant_val_nuq[0],
                      qcoeff, dqcoeff, eob);
      break;
    case TX_4X4:
      quantize_dc_nuq(coeff, 16, x->skip_block,
                      p->quant[0], p->quant_shift[0], pd->dequant[0],
                      p->cuml_bins_nuq[0],
                      pd->dequant_val_nuq[0],
                      qcoeff, dqcoeff, eob);
      break;
    default:
      assert(0);
      break;
  }
}

void vp10_xform_quant_dc_fp_nuq(MACROBLOCK *x, int plane, int block,
                                int blk_row, int blk_col,
                                BLOCK_SIZE plane_bsize, TX_SIZE tx_size) {
  MACROBLOCKD *const xd = &x->e_mbd;
  const struct macroblock_plane *const p = &x->plane[plane];
  const struct macroblockd_plane *const pd = &xd->plane[plane];
  PLANE_TYPE plane_type = (plane == 0) ? PLANE_TYPE_Y : PLANE_TYPE_UV;
  TX_TYPE tx_type = get_tx_type(plane_type, xd, block, tx_size);
  tran_low_t *const coeff = BLOCK_OFFSET(p->coeff, block);
  tran_low_t *const qcoeff = BLOCK_OFFSET(p->qcoeff, block);
  tran_low_t *const dqcoeff = BLOCK_OFFSET(pd->dqcoeff, block);
  uint16_t *const eob = &p->eobs[block];
  const int diff_stride = 4 * num_4x4_blocks_wide_lookup[plane_bsize];
  const int16_t *src_diff;

  FWD_TXFM_PARAM fwd_txfm_param;

  fwd_txfm_param.tx_type = tx_type;
  fwd_txfm_param.tx_size = tx_size;
  fwd_txfm_param.fwd_txfm_opt = fwd_txfm_opt_list[VP10_XFORM_QUANT_DC];
  fwd_txfm_param.rd_transform = x->use_lp32x32fdct;
  fwd_txfm_param.lossless = xd->lossless[xd->mi[0]->mbmi.segment_id];

  src_diff = &p->src_diff[4 * (blk_row * diff_stride + blk_col)];

// TODO(sarahparker) add all of these new quant quantize functions
// to quant_func_list, just trying to get this expr to work for now
#if CONFIG_VP9_HIGHBITDEPTH
  fwd_txfm_param.bd = xd->bd;
  if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) {
    highbd_fwd_txfm(src_diff, coeff, diff_stride, &fwd_txfm_param);
    switch (tx_size) {
      case TX_32X32:
        highbd_quantize_dc_32x32_fp_nuq(coeff, 1024, x->skip_block,
                                        p->quant_fp[0], pd->dequant[0],
                                        p->cuml_bins_nuq[0],
                                        pd->dequant_val_nuq[0],
                                        qcoeff, dqcoeff, eob);
        break;
      case TX_16X16:
        highbd_quantize_dc_fp_nuq(coeff, 256, x->skip_block,
                                  p->quant_fp[0], pd->dequant[0],
                                  p->cuml_bins_nuq[0],
                                  pd->dequant_val_nuq[0],
                                  qcoeff, dqcoeff, eob);
        break;
      case TX_8X8:
        highbd_quantize_dc_fp_nuq(coeff, 64, x->skip_block,
                                  p->quant_fp[0], pd->dequant[0],
                                  p->cuml_bins_nuq[0],
                                  pd->dequant_val_nuq[0],
                                  qcoeff, dqcoeff, eob);
        break;
      case TX_4X4:
        highbd_quantize_dc_fp_nuq(coeff, 16, x->skip_block,
                                  p->quant_fp[0], pd->dequant[0],
                                  p->cuml_bins_nuq[0],
                                  pd->dequant_val_nuq[0],
                                  qcoeff, dqcoeff, eob);
        break;
      default:
        assert(0);
    }
    return;
  }
#endif  // CONFIG_VP9_HIGHBITDEPTH

  fwd_txfm(src_diff, coeff, diff_stride, &fwd_txfm_param);
  switch (tx_size) {
    case TX_32X32:
      quantize_dc_32x32_fp_nuq(coeff, 1024, x->skip_block,
                               p->quant_fp[0], pd->dequant[0],
                               p->cuml_bins_nuq[0],
                               pd->dequant_val_nuq[0],
                               qcoeff, dqcoeff, eob);
      break;
    case TX_16X16:
      quantize_dc_fp_nuq(coeff, 256, x->skip_block,
                         p->quant_fp[0], pd->dequant[0],
                         p->cuml_bins_nuq[0],
                         pd->dequant_val_nuq[0],
                         qcoeff, dqcoeff, eob);

      break;
    case TX_8X8:
      quantize_dc_fp_nuq(coeff, 64, x->skip_block,
                         p->quant_fp[0], pd->dequant[0],
                         p->cuml_bins_nuq[0],
                         pd->dequant_val_nuq[0],
                         qcoeff, dqcoeff, eob);
      break;
    case TX_4X4:
      quantize_dc_fp_nuq(coeff, 16, x->skip_block,
                         p->quant_fp[0], pd->dequant[0],
                         p->cuml_bins_nuq[0],
                         pd->dequant_val_nuq[0],
                         qcoeff, dqcoeff, eob);
      break;
    default:
      assert(0);
      break;
  }
}
#endif  // CONFIG_NEW_QUANT

952 953
static void encode_block(int plane, int block, int blk_row, int blk_col,
                         BLOCK_SIZE plane_bsize,
Jingning Han's avatar
Jingning Han committed
954 955 956 957 958 959 960 961 962
                         TX_SIZE tx_size, void *arg) {
  struct encode_b_args *const args = arg;
  MACROBLOCK *const x = args->x;
  MACROBLOCKD *const xd = &x->e_mbd;
  struct macroblock_plane *const p = &x->plane[plane];
  struct macroblockd_plane *const pd = &xd->plane[plane];
  tran_low_t *const dqcoeff = BLOCK_OFFSET(pd->dqcoeff, block);
  uint8_t *dst;
  ENTROPY_CONTEXT *a, *l;
Angie Chiang's avatar
Angie Chiang committed
963
  INV_TXFM_PARAM inv_txfm_param;
964 965
#if CONFIG_VAR_TX
  int i;
966
  const int bwl = b_width_log2_lookup[plane_bsize];
967
#endif
968
  dst = &pd->dst.buf[4 * blk_row * pd->dst.stride + 4 * blk_col];
969 970
  a = &args->ta[blk_col];
  l = &args->tl[blk_row];
Jingning Han's avatar
Jingning Han committed
971 972 973

  // TODO(jingning): per transformed block zero forcing only enabled for
  // luma component. will integrate chroma components as well.
974 975 976 977 978 979 980
  // Turn this back on when the rate-distortion loop is synchronized with
  // the recursive transform block coding.
//  if (x->zcoeff_blk[tx_size][block] && plane == 0) {
//    p->eobs[block] = 0;
//    *a = *l = 0;
//    return;
//  }
Jingning Han's avatar
Jingning Han committed
981

Alex Converse's avatar
Alex Converse committed
982
#if CONFIG_VAR_TX
983 984 985
  // Assert not magic number (uninitialised).
  assert(x->blk_skip[plane][(blk_row << bwl) + blk_col] != 234);

986
  if (x->blk_skip[plane][(blk_row << bwl) + blk_col] == 0) {
987
#else
988
  {
989
#endif
Jingning Han's avatar
Jingning Han committed
990 991
    if (x->quant_fp) {
      // Encoding process for rtc mode
992
      if (x->skip_txfm[0][0] == SKIP_TXFM_AC_DC && plane == 0) {
Jingning Han's avatar
Jingning Han committed
993 994 995 996 997
        // skip forward transform
        p->eobs[block] = 0;
        *a = *l = 0;
        return;
      } else {
998 999 1000 1001
#if CONFIG_NEW_QUANT
        vp10_xform_quant_fp_nuq(x, plane, block, blk_row, blk_col, plane_bsize,
                                tx_size);
#else
Angie Chiang's avatar
Angie Chiang committed
1002 1003
        vp10_xform_quant(x, plane, block, blk_row, blk_col, plane_bsize,
                         tx_size, VP10_XFORM_QUANT_FP);
1004
#endif
Jingning Han's avatar
Jingning Han committed
1005 1006 1007
      }
    } else {
      if (max_txsize_lookup[plane_bsize] == tx_size) {
1008 1009
        int blk_index = (block >> (tx_size << 1));
        if (x->skip_txfm[plane][blk_index] == SKIP_TXFM_NONE) {
Jingning Han's avatar
Jingning Han committed
1010
          // full forward transform and quantization
1011 1012 1013 1014
#if CONFIG_NEW_QUANT
          vp10_xform_quant_nuq(x, plane, block, blk_row, blk_col, plane_bsize,
                               tx_size);
#else
Angie Chiang's avatar
Angie Chiang committed
1015
          vp10_xform_quant(x, plane, block, blk_row, blk_col, plane_bsize,
1016
                           tx_size, VP10_XFORM_QUANT_FP);
1017
#endif  // CONFIG_NEW_QUANT
1018
        } else if (x->skip_txfm[plane][blk_index] == SKIP_TXFM_AC_ONLY) {
Jingning Han's avatar
Jingning Han committed
1019
          // fast path forward transform and quantization
1020 1021 1022 1023
#if CONFIG_NEW_QUANT
          vp10_xform_quant_dc_nuq(x, plane, block, blk_row, blk_col,
                                  plane_bsize, tx_size);
#else
Angie Chiang's avatar
Angie Chiang committed
1024 1025
          vp10_xform_quant(x, plane, block, blk_row, blk_col, plane_bsize,
                           tx_size, VP10_XFORM_QUANT_DC);
1026
#endif  // CONFIG_NEW_QUANT
Jingning Han's avatar
Jingning Han committed
1027 1028 1029 1030
        } else {
          // skip forward transform
          p->eobs[block] = 0;
          *a = *l = 0;
1031
#if !CONFIG_VAR_TX
Jingning Han's avatar
Jingning Han committed
1032
          return;
1033
#endif
Jingning Han's avatar
Jingning Han committed
1034 1035
        }
      } else {
1036 1037 1038 1039
#if CONFIG_NEW_QUANT
        vp10_xform_quant_nuq(x, plane, block, blk_row, blk_col, plane_bsize,
                             tx_size);
#else
Angie Chiang's avatar
Angie Chiang committed
1040
        vp10_xform_quant(x, plane, block, blk_row, blk_col, plane_bsize,
1041
                         tx_size, VP10_XFORM_QUANT_FP);
1042
#endif  // CONFIG_NEW_QUANT
Jingning Han's avatar
Jingning Han committed
1043 1044 1045
      }
    }
  }
1046 1047
#if CONFIG_VAR_TX
  else {
1048
    p->eobs[block] = 0;
1049 1050
  }
#endif
Jingning Han's avatar
Jingning Han committed
1051

1052
  if (x->optimize && p->eobs[block]) {
1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075
    int ctx;
#if CONFIG_VAR_TX
    switch (tx_size) {
      case TX_4X4:
        break;
      case TX_8X8:
        a[0] = !!*(const uint16_t *)&a[0];
        l[0] = !!*(const uint16_t *)&l[0];
        break;
      case TX_16X16:
        a[0] = !!*(const uint32_t *)&a[0];
        l[0] = !!*(const uint32_t *)&l[0];
        break;
      case TX_32X32:
        a[0] = !!*(const uint64_t *)&a[0];
        l[0] = !!*(const uint64_t *)&l[0];
        break;
      default:
        assert(0 && "Invalid transform size.");
        break;
    }
#endif
    ctx = combine_entropy_contexts(*a, *l);
1076
    *a = *l = vp10_optimize_b(x, plane, block, tx_size, ctx) > 0;
Jingning Han's avatar
Jingning Han committed
1077 1078 1079 1080
  } else {
    *a = *l = p->eobs[block] > 0;
  }

1081 1082 1083 1084 1085 1086 1087
#if CONFIG_VAR_TX
  for (i = 0; i < (1 << tx_size); ++i) {
    a[i] = a[0];
    l[i] = l[0];
  }
#endif

Jingning Han's avatar
Jingning Han committed
1088 1089 1090
  if (p->eobs[block])
    *(args->skip) = 0;

1091
  if (p->eobs[block] == 0)
Jingning Han's avatar
Jingning Han committed
1092
    return;
Angie Chiang's avatar
Angie Chiang committed
1093 1094 1095 1096 1097 1098 1099

  // inverse transform parameters
  inv_txfm_param.tx_type = get_tx_type(pd->plane_type, xd, block, tx_size);
  inv_txfm_param.tx_size = tx_size;
  inv_txfm_param.eob = p->eobs[block];
  inv_txfm_param.lossless = xd->lossless[xd->mi[0]->mbmi.segment_id];

Jingning Han's avatar
Jingning Han committed
1100 1101
#if CONFIG_VP9_HIGHBITDEPTH
  if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) {
Angie Chiang's avatar
Angie Chiang committed
1102 1103
    inv_txfm_param.bd = xd->bd;
    highbd_inv_txfm_add(dqcoeff, dst, pd->dst.stride, &inv_txfm_param);
Jingning Han's avatar
Jingning Han committed
1104 1105 1106
    return;
  }
#endif  // CONFIG_VP9_HIGHBITDEPTH
Angie Chiang's avatar
Angie Chiang committed
1107
  inv_txfm_add(dqcoeff, dst, pd->dst.stride, &inv_txfm_param);
Jingning Han's avatar
Jingning Han committed
1108 1109
}

1110 1111 1112 1113 1114 1115 1116 1117
#if CONFIG_VAR_TX
static void encode_block_inter(int plane, int block, int blk_row, int blk_col,
                               BLOCK_SIZE plane_bsize, TX_SIZE tx_size,
                               void *arg) {
  struct encode_b_args *const args = arg;
  MACROBLOCK *const x = args->x;
  MACROBLOCKD *const xd = &x->e_mbd;
  MB_MODE_INFO *const mbmi = &xd->mi[0]->mbmi;
1118
  const BLOCK_SIZE bsize = txsize_to_bsize[tx_size];
1119
  const struct macroblockd_plane *const pd = &xd->plane[plane];
1120 1121 1122 1123 1124
  const int tx_row = blk_row >> (1 - pd->subsampling_y);
  const int tx_col = blk_col >> (1 - pd->subsampling_x);
  const TX_SIZE plane_tx_size = plane ?
      get_uv_tx_size_impl(mbmi->inter_tx_size[tx_row][tx_col], bsize, 0, 0) :
      mbmi->inter_tx_size[tx_row][tx_col];
1125 1126 1127 1128 1129 1130 1131 1132 1133 1134 1135 1136 1137 1138 1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161

  int max_blocks_high = num_4x4_blocks_high_lookup[plane_bsize];
  int max_blocks_wide = num_4x4_blocks_wide_lookup[plane_bsize];

  if (xd->mb_to_bottom_edge < 0)
    max_blocks_high += xd->mb_to_bottom_edge >> (5 + pd->subsampling_y);
  if (xd->mb_to_right_edge < 0)
    max_blocks_wide += xd->mb_to_right_edge >> (5 + pd->subsampling_x);

  if (blk_row >= max_blocks_high || blk_col >= max_blocks_wide)
    return;

  if (tx_size == plane_tx_size) {
    encode_block(plane, block, blk_row, blk_col, plane_bsize,
                 tx_size, arg);
  } else {
    int bsl = b_width_log2_lookup[bsize];
    int i;

    assert(bsl > 0);
    --bsl;

    for (i = 0; i < 4; ++i) {
      const int offsetr = blk_row + ((i >> 1) << bsl);
      const int offsetc = blk_col + ((i & 0x01) << bsl);
      int step = 1 << (2 * (tx_size - 1));

      if (offsetr >= max_blocks_high || offsetc >= max_blocks_wide)
        continue;

      encode_block_inter(plane, block + i * step, offsetr, offsetc,
                         plane_bsize, tx_size - 1, arg);
    }
  }
}
#endif

1162 1163
static void encode_block_pass1(int plane, int block, int blk_row, int blk_col,
                               BLOCK_SIZE plane_bsize,
Jingning Han's avatar
Jingning Han committed
1164 1165 1166 1167 1168 1169 1170
                               TX_SIZE tx_size, void *arg) {
  MACROBLOCK *const x = (MACROBLOCK *)arg;
  MACROBLOCKD *const xd = &x->e_mbd;
  struct macroblock_plane *const p = &x->plane[plane];
  struct macroblockd_plane *const pd = &xd->plane[plane];
  tran_low_t *const dqcoeff = BLOCK_OFFSET(pd->dqcoeff, block);
  uint8_t *dst;
1171
  dst = &pd->dst.buf[4 * blk_row * pd->dst.stride + 4 * blk_col];
Jingning Han's avatar
Jingning Han committed
1172

1173 1174 1175 1176
#if CONFIG_NEW_QUANT
  vp10_xform_quant_nuq(x, plane, block, blk_row, blk_col, plane_bsize,
                       tx_size);
#else
Angie Chiang's avatar
Angie Chiang committed
1177 1178
  vp10_xform_quant(x, plane, block, blk_row, blk_col, plane_bsize,
                   tx_size, VP10_XFORM_QUANT_B);
1179
#endif  // CONFIG_NEW_QUANT
Jingning Han's avatar
Jingning Han committed
1180 1181 1182 1183

  if (p->eobs[block] > 0) {
#if CONFIG_VP9_HIGHBITDEPTH
    if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) {
Yaowu Xu's avatar
Yaowu Xu committed
1184
      if (xd->lossless[xd->mi[0]->mbmi.segment_id]) {
1185 1186 1187 1188 1189 1190 1191
        vp10_highbd_iwht4x4_add(dqcoeff, dst, pd->dst.stride,
                                p->eobs[block], xd->bd);
      } else {
        vp10_highbd_idct4x4_add(dqcoeff, dst, pd->dst.stride,
                                p->eobs[block], xd->bd);
      }
      return;
Jingning Han's avatar
Jingning Han committed
1192 1193
    }
#endif  // CONFIG_VP9_HIGHBITDEPTH
Yaowu Xu's avatar
Yaowu Xu committed
1194
    if (xd->lossless[xd->mi[0]->mbmi.segment_id]) {
1195 1196 1197 1198
      vp10_iwht4x4_add(dqcoeff, dst, pd->dst.stride, p->eobs[block]);
    } else {
      vp10_idct4x4_add(dqcoeff, dst, pd->dst.stride, p->eobs[block]);
    }
Jingning Han's avatar
Jingning Han committed
1199 1200 1201 1202 1203 1204
  }
}

void vp10_encode_sby_pass1(MACROBLOCK *x, BLOCK_SIZE bsize) {
  vp10_subtract_plane(x, bsize, 0);
  vp10_foreach_transformed_block_in_plane(&x->e_mbd, bsize, 0,
1205
                                          encode_block_pass1, x);
Jingning Han's avatar
Jingning Han committed
1206 1207 1208 1209 1210 1211
}

void vp10_encode_sb(MACROBLOCK *x, BLOCK_SIZE bsize) {
  MACROBLOCKD *const xd = &x->e_mbd;
  struct