rdopt.c

/*
 *  Copyright (c) 2010 The WebM project authors. All Rights Reserved.
 *
 *  Use of this source code is governed by a BSD-style license
 *  that can be found in the LICENSE file in the root of the source
 *  tree. An additional intellectual property rights grant can be found
 *  in the file PATENTS.  All contributing project authors may
 *  be found in the AUTHORS file in the root of the source tree.
 */


#include <stdio.h>
#include <math.h>
#include <limits.h>
#include <assert.h>
#include "vp8/common/pragmas.h"

#include "tokenize.h"
#include "treewriter.h"
#include "onyx_int.h"
#include "modecosts.h"
#include "encodeintra.h"
#include "vp8/common/entropymode.h"
#include "vp8/common/reconinter.h"
#include "vp8/common/reconintra.h"
#include "vp8/common/reconintra4x4.h"
#include "vp8/common/findnearmv.h"
#include "vp8/common/quant_common.h"
#include "encodemb.h"
#include "quantize.h"
#include "vp8/common/idct.h"
#include "vp8/common/g_common.h"
#include "variance.h"
#include "mcomp.h"
#include "rdopt.h"
#include "ratectrl.h"
#include "vpx_mem/vpx_mem.h"
#include "dct.h"
#include "vp8/common/systemdependent.h"
#include "vp8/encoder/encodemv.h"

#include "vp8/common/seg_common.h"
#include "vp8/common/pred_common.h"
#include "vp8/common/entropy.h"

#if CONFIG_NEWBESTREFMV
#include "vp8/common/mvref_common.h"
#endif

#if CONFIG_RUNTIME_CPU_DETECT
#define IF_RTCD(x)  (x)
#else
#define IF_RTCD(x)  NULL
#endif

extern void vp8cx_mb_init_quantizer(VP8_COMP *cpi, MACROBLOCK *x);
extern void vp8_update_zbin_extra(VP8_COMP *cpi, MACROBLOCK *x);

#define MAXF(a,b)            (((a) > (b)) ? (a) : (b))

#define INVALID_MV 0x80008000

#if CONFIG_SWITCHABLE_INTERP
/* Factor to weigh the rate for switchable interp filters */
#define SWITCHABLE_INTERP_RATE_FACTOR 1
#endif

static const int auto_speed_thresh[17] = {
  1000,
  200,
  150,
  130,
  150,
  125,
  120,
  115,
  115,
  115,
  115,
  115,
  115,
  115,
  115,
  115,
  105
};

#if CONFIG_PRED_FILTER
const MODE_DEFINITION vp8_mode_order[MAX_MODES] = {
  {ZEROMV,    LAST_FRAME,   0,  0},
  {ZEROMV,    LAST_FRAME,   0,  1},
  {DC_PRED,   INTRA_FRAME,  0,  0},

  {NEARESTMV, LAST_FRAME,   0,  0},
  {NEARESTMV, LAST_FRAME,   0,  1},
  {NEARMV,    LAST_FRAME,   0,  0},
  {NEARMV,    LAST_FRAME,   0,  1},

  {ZEROMV,    GOLDEN_FRAME, 0,  0},
  {ZEROMV,    GOLDEN_FRAME, 0,  1},
  {NEARESTMV, GOLDEN_FRAME, 0,  0},
  {NEARESTMV, GOLDEN_FRAME, 0,  1},

  {ZEROMV,    ALTREF_FRAME, 0,  0},
  {ZEROMV,    ALTREF_FRAME, 0,  1},
  {NEARESTMV, ALTREF_FRAME, 0,  0},
  {NEARESTMV, ALTREF_FRAME, 0,  1},

  {NEARMV,    GOLDEN_FRAME, 0,  0},
  {NEARMV,    GOLDEN_FRAME, 0,  1},
  {NEARMV,    ALTREF_FRAME, 0,  0},
  {NEARMV,    ALTREF_FRAME, 0,  1},

  {V_PRED,    INTRA_FRAME,  0,  0},
  {H_PRED,    INTRA_FRAME,  0,  0},
  {D45_PRED,  INTRA_FRAME,  0,  0},
  {D135_PRED, INTRA_FRAME,  0,  0},
  {D117_PRED, INTRA_FRAME,  0,  0},
  {D153_PRED, INTRA_FRAME,  0,  0},
  {D27_PRED,  INTRA_FRAME,  0,  0},
  {D63_PRED,  INTRA_FRAME,  0,  0},

  {TM_PRED,   INTRA_FRAME,  0,  0},

  {NEWMV,     LAST_FRAME,   0,  0},
  {NEWMV,     LAST_FRAME,   0,  1},
  {NEWMV,     GOLDEN_FRAME, 0,  0},
  {NEWMV,     GOLDEN_FRAME, 0,  1},
  {NEWMV,     ALTREF_FRAME, 0,  0},
  {NEWMV,     ALTREF_FRAME, 0,  1},

  {SPLITMV,   LAST_FRAME,   0,  0},
  {SPLITMV,   GOLDEN_FRAME, 0,  0},
  {SPLITMV,   ALTREF_FRAME, 0,  0},

  {B_PRED,    INTRA_FRAME,  0,  0},
  {I8X8_PRED, INTRA_FRAME,  0,  0},

  /* compound prediction modes */
  {ZEROMV,    LAST_FRAME,   GOLDEN_FRAME, 0},
  {NEARESTMV, LAST_FRAME,   GOLDEN_FRAME, 0},
  {NEARMV,    LAST_FRAME,   GOLDEN_FRAME, 0},

  {ZEROMV,    ALTREF_FRAME, LAST_FRAME,   0},
  {NEARESTMV, ALTREF_FRAME, LAST_FRAME,   0},
  {NEARMV,    ALTREF_FRAME, LAST_FRAME,   0},

  {ZEROMV,    GOLDEN_FRAME, ALTREF_FRAME, 0},
  {NEARESTMV, GOLDEN_FRAME, ALTREF_FRAME, 0},
  {NEARMV,    GOLDEN_FRAME, ALTREF_FRAME, 0},

  {NEWMV,     LAST_FRAME,   GOLDEN_FRAME, 0},
  {NEWMV,     ALTREF_FRAME, LAST_FRAME,   0},
  {NEWMV,     GOLDEN_FRAME, ALTREF_FRAME, 0},

  {SPLITMV,   LAST_FRAME,   GOLDEN_FRAME, 0},
  {SPLITMV,   ALTREF_FRAME, LAST_FRAME,   0},
  {SPLITMV,   GOLDEN_FRAME, ALTREF_FRAME, 0}
};
#else
const MODE_DEFINITION vp8_mode_order[MAX_MODES] = {
  {ZEROMV,    LAST_FRAME,   0},
  {DC_PRED,   INTRA_FRAME,  0},

  {NEARESTMV, LAST_FRAME,   0},
  {NEARMV,    LAST_FRAME,   0},

  {ZEROMV,    GOLDEN_FRAME, 0},
  {NEARESTMV, GOLDEN_FRAME, 0},

  {ZEROMV,    ALTREF_FRAME, 0},
  {NEARESTMV, ALTREF_FRAME, 0},

  {NEARMV,    GOLDEN_FRAME, 0},
  {NEARMV,    ALTREF_FRAME, 0},

  {V_PRED,    INTRA_FRAME,  0},
  {H_PRED,    INTRA_FRAME,  0},
  {D45_PRED,  INTRA_FRAME,  0},
  {D135_PRED, INTRA_FRAME,  0},
  {D117_PRED, INTRA_FRAME,  0},
  {D153_PRED, INTRA_FRAME,  0},
  {D27_PRED,  INTRA_FRAME,  0},
  {D63_PRED,  INTRA_FRAME,  0},

  {TM_PRED,   INTRA_FRAME,  0},

  {NEWMV,     LAST_FRAME,   0},
  {NEWMV,     GOLDEN_FRAME, 0},
  {NEWMV,     ALTREF_FRAME, 0},

  {SPLITMV,   LAST_FRAME,   0},
  {SPLITMV,   GOLDEN_FRAME, 0},
  {SPLITMV,   ALTREF_FRAME, 0},

  {B_PRED,    INTRA_FRAME,  0},
  {I8X8_PRED, INTRA_FRAME,  0},

  /* compound prediction modes */
  {ZEROMV,    LAST_FRAME,   GOLDEN_FRAME},
  {NEARESTMV, LAST_FRAME,   GOLDEN_FRAME},
  {NEARMV,    LAST_FRAME,   GOLDEN_FRAME},

  {ZEROMV,    ALTREF_FRAME, LAST_FRAME},
  {NEARESTMV, ALTREF_FRAME, LAST_FRAME},
  {NEARMV,    ALTREF_FRAME, LAST_FRAME},

  {ZEROMV,    GOLDEN_FRAME, ALTREF_FRAME},
  {NEARESTMV, GOLDEN_FRAME, ALTREF_FRAME},
  {NEARMV,    GOLDEN_FRAME, ALTREF_FRAME},

  {NEWMV,     LAST_FRAME,   GOLDEN_FRAME},
  {NEWMV,     ALTREF_FRAME, LAST_FRAME  },
  {NEWMV,     GOLDEN_FRAME, ALTREF_FRAME},

  {SPLITMV,   LAST_FRAME,   GOLDEN_FRAME},
  {SPLITMV,   ALTREF_FRAME, LAST_FRAME  },
  {SPLITMV,   GOLDEN_FRAME, ALTREF_FRAME}
};
#endif

static void fill_token_costs(
  unsigned int (*c)[COEF_BANDS] [PREV_COEF_CONTEXTS] [MAX_ENTROPY_TOKENS],
  const vp8_prob(*p)[COEF_BANDS] [PREV_COEF_CONTEXTS] [ENTROPY_NODES],
  int block_type_counts) {
  int i, j, k;

  for (i = 0; i < block_type_counts; i++)
    for (j = 0; j < COEF_BANDS; j++)
      for (k = 0; k < PREV_COEF_CONTEXTS; k++) {
        if (k == 0 && ((j > 0 && i > 0) || (j > 1 && i == 0)))
          vp8_cost_tokens_skip((int *)(c [i][j][k]),
                               p [i][j][k],
                               vp8_coef_tree);
        else
          vp8_cost_tokens((int *)(c [i][j][k]),
                          p [i][j][k],
                          vp8_coef_tree);
      }
}


static int rd_iifactor [ 32 ] =  {    4,   4,   3,   2,   1,   0,   0,   0,
                                      0,   0,   0,   0,   0,   0,   0,   0,
                                      0,   0,   0,   0,   0,   0,   0,   0,
                                      0,   0,   0,   0,   0,   0,   0,   0,
                                 };

// 3* dc_qlookup[Q]*dc_qlookup[Q];

/* values are now correlated to quantizer */
static int sad_per_bit16lut[QINDEX_RANGE];
static int sad_per_bit4lut[QINDEX_RANGE];

void vp8_init_me_luts() {
  int i;

  // Initialize the sad lut tables using a formulaic calculation for now
  // This is to make it easier to resolve the impact of experimental changes
  // to the quantizer tables.
  for (i = 0; i < QINDEX_RANGE; i++) {
    sad_per_bit16lut[i] =
      (int)((0.0418 * vp8_convert_qindex_to_q(i)) + 2.4107);
    sad_per_bit4lut[i] = (int)((0.063 * vp8_convert_qindex_to_q(i)) + 2.742);
  }
}

int compute_rd_mult(int qindex) {
  int q;

  q = vp8_dc_quant(qindex, 0);
  return (11 * q * q) >> 6;
}

void vp8cx_initialize_me_consts(VP8_COMP *cpi, int QIndex) {
  cpi->mb.sadperbit16 =  sad_per_bit16lut[QIndex];
  cpi->mb.sadperbit4  =  sad_per_bit4lut[QIndex];
}


void vp8_initialize_rd_consts(VP8_COMP *cpi, int QIndex) {
  int q, i;

  vp8_clear_system_state();  // __asm emms;

  // Further tests required to see if optimum is different
  // for key frames, golden frames and arf frames.
  // if (cpi->common.refresh_golden_frame ||
  //     cpi->common.refresh_alt_ref_frame)
  QIndex = (QIndex < 0) ? 0 : ((QIndex > MAXQ) ? MAXQ : QIndex);

  cpi->RDMULT = compute_rd_mult(QIndex);

  // Extend rate multiplier along side quantizer zbin increases
  if (cpi->zbin_over_quant  > 0) {
    double oq_factor;

    // Experimental code using the same basic equation as used for Q above
    // The units of cpi->zbin_over_quant are 1/128 of Q bin size
    oq_factor = 1.0 + ((double)0.0015625 * cpi->zbin_over_quant);
    cpi->RDMULT = (int)((double)cpi->RDMULT * oq_factor * oq_factor);
  }

  if (cpi->pass == 2 && (cpi->common.frame_type != KEY_FRAME)) {
    if (cpi->twopass.next_iiratio > 31)
      cpi->RDMULT += (cpi->RDMULT * rd_iifactor[31]) >> 4;
    else
      cpi->RDMULT +=
        (cpi->RDMULT * rd_iifactor[cpi->twopass.next_iiratio]) >> 4;
  }

  if (cpi->RDMULT < 7)
    cpi->RDMULT = 7;

  cpi->mb.errorperbit = (cpi->RDMULT / 110);
  cpi->mb.errorperbit += (cpi->mb.errorperbit == 0);

  vp8_set_speed_features(cpi);

  q = (int)pow(vp8_dc_quant(QIndex, 0) >> 2, 1.25);
  q = q << 2;
  cpi->RDMULT = cpi->RDMULT << 4;

  if (q < 8)
    q = 8;

  if (cpi->RDMULT > 1000) {
    cpi->RDDIV = 1;
    cpi->RDMULT /= 100;

    for (i = 0; i < MAX_MODES; i++) {
      if (cpi->sf.thresh_mult[i] < INT_MAX) {
        cpi->rd_threshes[i] = cpi->sf.thresh_mult[i] * q / 100;
      } else {
        cpi->rd_threshes[i] = INT_MAX;
      }

      cpi->rd_baseline_thresh[i] = cpi->rd_threshes[i];
    }
  } else {
    cpi->RDDIV = 100;

    for (i = 0; i < MAX_MODES; i++) {
      if (cpi->sf.thresh_mult[i] < (INT_MAX / q)) {
        cpi->rd_threshes[i] = cpi->sf.thresh_mult[i] * q;
      } else {
        cpi->rd_threshes[i] = INT_MAX;
      }

      cpi->rd_baseline_thresh[i] = cpi->rd_threshes[i];
    }
  }

  fill_token_costs(
    cpi->mb.token_costs[TX_4X4],
    (const vp8_prob( *)[8][PREV_COEF_CONTEXTS][11]) cpi->common.fc.coef_probs,
    BLOCK_TYPES);
#if CONFIG_HYBRIDTRANSFORM
  fill_token_costs(
    cpi->mb.hybrid_token_costs[TX_4X4],
    (const vp8_prob( *)[8][PREV_COEF_CONTEXTS][11])
    cpi->common.fc.hybrid_coef_probs,
    BLOCK_TYPES);
#endif

  fill_token_costs(
    cpi->mb.token_costs[TX_8X8],
    (const vp8_prob( *)[8][PREV_COEF_CONTEXTS][11]) cpi->common.fc.coef_probs_8x8,
    BLOCK_TYPES_8X8);
#if CONFIG_HYBRIDTRANSFORM8X8
  fill_token_costs(
    cpi->mb.hybrid_token_costs[TX_8X8],
    (const vp8_prob( *)[8][PREV_COEF_CONTEXTS][11])
    cpi->common.fc.hybrid_coef_probs_8x8,
    BLOCK_TYPES_8X8);
#endif

  fill_token_costs(
    cpi->mb.token_costs[TX_16X16],
    (const vp8_prob(*)[8][PREV_COEF_CONTEXTS][11]) cpi->common.fc.coef_probs_16x16,
    BLOCK_TYPES_16X16);
#if CONFIG_HYBRIDTRANSFORM16X16
  fill_token_costs(
    cpi->mb.hybrid_token_costs[TX_16X16],
    (const vp8_prob(*)[8][PREV_COEF_CONTEXTS][11])
    cpi->common.fc.hybrid_coef_probs_16x16,
    BLOCK_TYPES_16X16);
#endif

  /*rough estimate for costing*/
  cpi->common.kf_ymode_probs_index = cpi->common.base_qindex >> 4;
  vp8_init_mode_costs(cpi);

  if (cpi->common.frame_type != KEY_FRAME)
  {
#if CONFIG_NEWMVENTROPY
    vp8_build_nmv_cost_table(
        cpi->mb.nmvjointcost,
        cpi->mb.e_mbd.allow_high_precision_mv ?
        cpi->mb.nmvcost_hp : cpi->mb.nmvcost,
        &cpi->common.fc.nmvc,
        cpi->mb.e_mbd.allow_high_precision_mv, 1, 1);
#endif
  }
}

void vp8_auto_select_speed(VP8_COMP *cpi) {
  int milliseconds_for_compress = (int)(1000000 / cpi->oxcf.frame_rate);

  milliseconds_for_compress = milliseconds_for_compress * (16 - cpi->oxcf.cpu_used) / 16;

#if 0

  if (0) {
    FILE *f;

    f = fopen("speed.stt", "a");
    fprintf(f, " %8ld %10ld %10ld %10ld\n",
            cpi->common.current_video_frame, cpi->Speed, milliseconds_for_compress, cpi->avg_pick_mode_time);
    fclose(f);
  }

#endif

  /*
  // this is done during parameter valid check
  if( cpi->oxcf.cpu_used > 16)
      cpi->oxcf.cpu_used = 16;
  if( cpi->oxcf.cpu_used < -16)
      cpi->oxcf.cpu_used = -16;
  */

  if (cpi->avg_pick_mode_time < milliseconds_for_compress &&
      (cpi->avg_encode_time - cpi->avg_pick_mode_time) <
      milliseconds_for_compress) {
    if (cpi->avg_pick_mode_time == 0) {
      cpi->Speed = 4;
    } else {
      if (milliseconds_for_compress * 100 < cpi->avg_encode_time * 95) {
        cpi->Speed          += 2;
        cpi->avg_pick_mode_time = 0;
        cpi->avg_encode_time = 0;

        if (cpi->Speed > 16) {
          cpi->Speed = 16;
        }
      }

      if (milliseconds_for_compress * 100 >
          cpi->avg_encode_time * auto_speed_thresh[cpi->Speed]) {
        cpi->Speed          -= 1;
        cpi->avg_pick_mode_time = 0;
        cpi->avg_encode_time = 0;

        // In real-time mode, cpi->speed is in [4, 16].
        if (cpi->Speed < 4) {      // if ( cpi->Speed < 0 )
          cpi->Speed = 4;        // cpi->Speed = 0;
        }
      }
    }
  } else {
    cpi->Speed += 4;

    if (cpi->Speed > 16)
      cpi->Speed = 16;


    cpi->avg_pick_mode_time = 0;
    cpi->avg_encode_time = 0;
  }
}

int vp8_block_error_c(short *coeff, short *dqcoeff, int block_size) {
  int i, error = 0;

  for (i = 0; i < block_size; i++) {
    int this_diff = coeff[i] - dqcoeff[i];
    error += this_diff * this_diff;
  }

  return error;
}

int vp8_mbblock_error_c(MACROBLOCK *mb, int dc) {
  BLOCK  *be;
  BLOCKD *bd;
  int i, j;
  int berror, error = 0;

  for (i = 0; i < 16; i++) {
    be = &mb->block[i];
    bd = &mb->e_mbd.block[i];

    berror = 0;

    for (j = dc; j < 16; j++) {
      int this_diff = be->coeff[j] - bd->dqcoeff[j];
      berror += this_diff * this_diff;
    }

    error += berror;
  }

  return error;
}

int vp8_mbuverror_c(MACROBLOCK *mb) {
  BLOCK  *be;
  BLOCKD *bd;

  int i, error = 0;

  for (i = 16; i < 24; i++) {
    be = &mb->block[i];
    bd = &mb->e_mbd.block[i];

    error += vp8_block_error_c(be->coeff, bd->dqcoeff, 16);
  }

  return error;
}

int VP8_UVSSE(MACROBLOCK *x, const vp8_variance_rtcd_vtable_t *rtcd) {
  unsigned char *uptr, *vptr;
  unsigned char *upred_ptr = (*(x->block[16].base_src) + x->block[16].src);
  unsigned char *vpred_ptr = (*(x->block[20].base_src) + x->block[20].src);
  int uv_stride = x->block[16].src_stride;

  unsigned int sse1 = 0;
  unsigned int sse2 = 0;
  int mv_row = x->e_mbd.mode_info_context->mbmi.mv[0].as_mv.row;
  int mv_col = x->e_mbd.mode_info_context->mbmi.mv[0].as_mv.col;
  int offset;
  int pre_stride = x->e_mbd.block[16].pre_stride;

  if (mv_row < 0)
    mv_row -= 1;
  else
    mv_row += 1;

  if (mv_col < 0)
    mv_col -= 1;
  else
    mv_col += 1;

  mv_row /= 2;
  mv_col /= 2;

  offset = (mv_row >> 3) * pre_stride + (mv_col >> 3);
  uptr = x->e_mbd.pre.u_buffer + offset;
  vptr = x->e_mbd.pre.v_buffer + offset;

  if ((mv_row | mv_col) & 7) {
    VARIANCE_INVOKE(rtcd, subpixvar8x8)(uptr, pre_stride,
                                        (mv_col & 7) << 1, (mv_row & 7) << 1, upred_ptr, uv_stride, &sse2);
    VARIANCE_INVOKE(rtcd, subpixvar8x8)(vptr, pre_stride,
                                        (mv_col & 7) << 1, (mv_row & 7) << 1, vpred_ptr, uv_stride, &sse1);
    sse2 += sse1;
  } else {
    VARIANCE_INVOKE(rtcd, var8x8)(uptr, pre_stride,
                                  upred_ptr, uv_stride, &sse2);
    VARIANCE_INVOKE(rtcd, var8x8)(vptr, pre_stride,
                                  vpred_ptr, uv_stride, &sse1);
    sse2 += sse1;
  }
  return sse2;

}

static int cost_coeffs_2x2(MACROBLOCK *mb,
                           BLOCKD *b, PLANE_TYPE type,
                           ENTROPY_CONTEXT *a, ENTROPY_CONTEXT *l) {
  int c = (type == PLANE_TYPE_Y_NO_DC); /* start at coef 0, unless Y with Y2 */
  int eob = b->eob;
  int pt;    /* surrounding block/prev coef predictor */
  int cost = 0;
  short *qcoeff_ptr = b->qcoeff;

  VP8_COMBINEENTROPYCONTEXTS(pt, *a, *l);
  assert(eob <= 4);

  for (; c < eob; c++) {
    int v = qcoeff_ptr[vp8_default_zig_zag1d[c]];
    int t = vp8_dct_value_tokens_ptr[v].Token;
    cost += mb->token_costs[TX_8X8][type][vp8_coef_bands[c]][pt][t];
    cost += vp8_dct_value_cost_ptr[v];
    pt = vp8_prev_token_class[t];
  }

  if (c < 4)
    cost += mb->token_costs[TX_8X8][type][vp8_coef_bands[c]]
            [pt] [DCT_EOB_TOKEN];

  pt = (c != !type); // is eob first coefficient;
  *a = *l = pt;
  return cost;
}

static int cost_coeffs(MACROBLOCK *mb, BLOCKD *b, PLANE_TYPE type,
                       ENTROPY_CONTEXT *a, ENTROPY_CONTEXT *l,
                       int tx_size) {
  const int eob = b->eob;
  int c = (type == PLANE_TYPE_Y_NO_DC); /* start at coef 0, unless Y with Y2 */
  int cost = 0, default_eob, seg_eob;
  int pt;                     /* surrounding block/prev coef predictor */
  int const *scan, *band;
  short *qcoeff_ptr = b->qcoeff;
  MACROBLOCKD *xd = &mb->e_mbd;
  MB_MODE_INFO *mbmi = &mb->e_mbd.mode_info_context->mbmi;
#if CONFIG_HYBRIDTRANSFORM || CONFIG_HYBRIDTRANSFORM8X8 || CONFIG_HYBRIDTRANSFORM16X16
  TX_TYPE tx_type = DCT_DCT;
#endif
  int segment_id = mbmi->segment_id;

  switch (tx_size) {
    case TX_4X4:
      scan = vp8_default_zig_zag1d;
      band = vp8_coef_bands;
      default_eob = 16;
#if CONFIG_HYBRIDTRANSFORM
      if (type == PLANE_TYPE_Y_WITH_DC) {
        tx_type = get_tx_type_4x4(xd, b);
        if (tx_type != DCT_DCT) {
          switch (tx_type) {
            case ADST_DCT:
              scan = vp8_row_scan;
              break;

            case DCT_ADST:
              scan = vp8_col_scan;
              break;

            default:
              scan = vp8_default_zig_zag1d;
              break;
          }
        }
      }
#endif

      break;
    case TX_8X8:
      scan = vp8_default_zig_zag1d_8x8;
      band = vp8_coef_bands_8x8;
      default_eob = 64;
#if CONFIG_HYBRIDTRANSFORM8X8
      if (type == PLANE_TYPE_Y_WITH_DC) {
        BLOCKD *bb;
        int ib = (b - xd->block);
        if (ib < 16) {
          ib = (ib & 8) + ((ib & 4) >> 1);
          bb = xd->block + ib;
          tx_type = get_tx_type_8x8(xd, bb);
        }
      }
#endif
      break;
    case TX_16X16:
      scan = vp8_default_zig_zag1d_16x16;
      band = vp8_coef_bands_16x16;
      default_eob = 256;
#if CONFIG_HYBRIDTRANSFORM16X16
      if (type == PLANE_TYPE_Y_WITH_DC) {
        tx_type = get_tx_type_16x16(xd, b);
      }
#endif
      break;
    default:
      break;
  }
  if (segfeature_active(&mb->e_mbd, segment_id, SEG_LVL_EOB))
    seg_eob = get_segdata(&mb->e_mbd, segment_id, SEG_LVL_EOB);
  else
    seg_eob = default_eob;

  VP8_COMBINEENTROPYCONTEXTS(pt, *a, *l);

#if CONFIG_HYBRIDTRANSFORM || CONFIG_HYBRIDTRANSFORM8X8 || CONFIG_HYBRIDTRANSFORM16X16
  if (tx_type != DCT_DCT) {
    for (; c < eob; c++) {
      int v = qcoeff_ptr[scan[c]];
      int t = vp8_dct_value_tokens_ptr[v].Token;
      cost += mb->hybrid_token_costs[tx_size][type][band[c]][pt][t];
      cost += vp8_dct_value_cost_ptr[v];
      pt = vp8_prev_token_class[t];
    }
    if (c < seg_eob)
      cost += mb->hybrid_token_costs[tx_size][type][band[c]]
          [pt][DCT_EOB_TOKEN];
  } else
#endif
  {
    for (; c < eob; c++) {
      int v = qcoeff_ptr[scan[c]];
      int t = vp8_dct_value_tokens_ptr[v].Token;
      cost += mb->token_costs[tx_size][type][band[c]][pt][t];
      cost += vp8_dct_value_cost_ptr[v];
      pt = vp8_prev_token_class[t];
    }
    if (c < seg_eob)
      cost += mb->token_costs[tx_size][type][band[c]]
          [pt][DCT_EOB_TOKEN];
  }

  pt = (c != !type); // is eob first coefficient;
  *a = *l = pt;
  return cost;
}

static int vp8_rdcost_mby(MACROBLOCK *mb) {
  int cost = 0;
  int b;
  MACROBLOCKD *xd = &mb->e_mbd;
  ENTROPY_CONTEXT_PLANES t_above, t_left;
  ENTROPY_CONTEXT *ta;
  ENTROPY_CONTEXT *tl;

  vpx_memcpy(&t_above, xd->above_context, sizeof(ENTROPY_CONTEXT_PLANES));
  vpx_memcpy(&t_left, xd->left_context, sizeof(ENTROPY_CONTEXT_PLANES));

  ta = (ENTROPY_CONTEXT *)&t_above;
  tl = (ENTROPY_CONTEXT *)&t_left;

  for (b = 0; b < 16; b++)
    cost += cost_coeffs(mb, xd->block + b, PLANE_TYPE_Y_NO_DC,
                        ta + vp8_block2above[b], tl + vp8_block2left[b],
                        TX_4X4);

  cost += cost_coeffs(mb, xd->block + 24, PLANE_TYPE_Y2,
                      ta + vp8_block2above[24], tl + vp8_block2left[24],
                      TX_4X4);

  return cost;
}

static void macro_block_yrd_4x4(MACROBLOCK *mb,
                                int *Rate,
                                int *Distortion,
                                const VP8_ENCODER_RTCD *rtcd,
                                int *skippable) {
  int b;
  MACROBLOCKD *const xd = &mb->e_mbd;
  BLOCK   *const mb_y2 = mb->block + 24;
  BLOCKD *const x_y2  = xd->block + 24;
  short *Y2DCPtr = mb_y2->src_diff;
  BLOCK *beptr;
  int d;

  ENCODEMB_INVOKE(&rtcd->encodemb, submby)(
    mb->src_diff,
    *(mb->block[0].base_src),
    xd->predictor,
    mb->block[0].src_stride);

  // Fdct and building the 2nd order block
  for (beptr = mb->block; beptr < mb->block + 16; beptr += 2) {
    mb->vp8_short_fdct8x4(beptr->src_diff, beptr->coeff, 32);
    *Y2DCPtr++ = beptr->coeff[0];
    *Y2DCPtr++ = beptr->coeff[16];
  }

  // 2nd order fdct
  mb->short_walsh4x4(mb_y2->src_diff, mb_y2->coeff, 8);

  // Quantization
  for (b = 0; b < 16; b++) {
    mb->quantize_b_4x4(&mb->block[b], &xd->block[b]);
  }

  // DC predication and Quantization of 2nd Order block
  mb->quantize_b_4x4(mb_y2, x_y2);

  // Distortion
  d = ENCODEMB_INVOKE(&rtcd->encodemb, mberr)(mb, 1);

  d += ENCODEMB_INVOKE(&rtcd->encodemb, berr)(mb_y2->coeff, x_y2->dqcoeff, 16);

  *Distortion = (d >> 2);
  // rate
  *Rate = vp8_rdcost_mby(mb);
  *skippable = mby_is_skippable_4x4(&mb->e_mbd, 1);
}

static int vp8_rdcost_mby_8x8(MACROBLOCK *mb, int backup) {
  int cost = 0;
  int b;
  MACROBLOCKD *xd = &mb->e_mbd;
  ENTROPY_CONTEXT_PLANES t_above, t_left;
  ENTROPY_CONTEXT *ta;
  ENTROPY_CONTEXT *tl;

  if (backup) {
    vpx_memcpy(&t_above,xd->above_context, sizeof(ENTROPY_CONTEXT_PLANES));
    vpx_memcpy(&t_left, xd->left_context, sizeof(ENTROPY_CONTEXT_PLANES));

    ta = (ENTROPY_CONTEXT *)&t_above;
    tl = (ENTROPY_CONTEXT *)&t_left;
  } else {
    ta = (ENTROPY_CONTEXT *)mb->e_mbd.above_context;
    tl = (ENTROPY_CONTEXT *)mb->e_mbd.left_context;
  }

  for (b = 0; b < 16; b += 4)
    cost += cost_coeffs(mb, xd->block + b, PLANE_TYPE_Y_NO_DC,
                        ta + vp8_block2above_8x8[b], tl + vp8_block2left_8x8[b],
                        TX_8X8);

  cost += cost_coeffs_2x2(mb, xd->block + 24, PLANE_TYPE_Y2,
                          ta + vp8_block2above[24], tl + vp8_block2left[24]);
  return cost;
}

static void macro_block_yrd_8x8(MACROBLOCK *mb,
                                int *Rate,
                                int *Distortion,
                                const VP8_ENCODER_RTCD *rtcd,
                                int *skippable) {
  MACROBLOCKD *const xd = &mb->e_mbd;
  BLOCK   *const mb_y2 = mb->block + 24;
  BLOCKD *const x_y2  = xd->block + 24;
  int d;

  ENCODEMB_INVOKE(&rtcd->encodemb, submby)(
    mb->src_diff,
    *(mb->block[0].base_src),
    xd->predictor,
    mb->block[0].src_stride);

  vp8_transform_mby_8x8(mb);
  vp8_quantize_mby_8x8(mb);

  /* remove 1st order dc to properly combine 1st/2nd order distortion */
  mb->coeff[0] = 0;
  mb->coeff[64] = 0;
  mb->coeff[128] = 0;
  mb->coeff[192] = 0;
  xd->dqcoeff[0] = 0;
  xd->dqcoeff[64] = 0;
  xd->dqcoeff[128] = 0;
  xd->dqcoeff[192] = 0;

  d = ENCODEMB_INVOKE(&rtcd->encodemb, mberr)(mb, 0);
  d += ENCODEMB_INVOKE(&rtcd->encodemb, berr)(mb_y2->coeff, x_y2->dqcoeff, 16);

  *Distortion = (d >> 2);
  // rate
  *Rate = vp8_rdcost_mby_8x8(mb, 1);
  *skippable = mby_is_skippable_8x8(&mb->e_mbd, 1);
}

static int vp8_rdcost_mby_16x16(MACROBLOCK *mb) {
  int cost;
  MACROBLOCKD *xd = &mb->e_mbd;
  ENTROPY_CONTEXT_PLANES t_above, t_left;
  ENTROPY_CONTEXT *ta, *tl;

  vpx_memcpy(&t_above, xd->above_context, sizeof(ENTROPY_CONTEXT_PLANES));
  vpx_memcpy(&t_left, xd->left_context, sizeof(ENTROPY_CONTEXT_PLANES));

  ta = (ENTROPY_CONTEXT *)&t_above;
  tl = (ENTROPY_CONTEXT *)&t_left;

  cost = cost_coeffs(mb, xd->block, PLANE_TYPE_Y_WITH_DC, ta, tl, TX_16X16);
  return cost;
}

static void macro_block_yrd_16x16(MACROBLOCK *mb, int *Rate, int *Distortion,
                                  const VP8_ENCODER_RTCD *rtcd, int *skippable) {
  int d;
  MACROBLOCKD *xd = &mb->e_mbd;
  BLOCKD *b  = &mb->e_mbd.block[0];
  BLOCK  *be = &mb->block[0];
#if CONFIG_HYBRIDTRANSFORM16X16
  TX_TYPE tx_type;
#endif

  ENCODEMB_INVOKE(&rtcd->encodemb, submby)(
    mb->src_diff,
    *(mb->block[0].base_src),
    mb->e_mbd.predictor,
    mb->block[0].src_stride);

#if CONFIG_HYBRIDTRANSFORM16X16
  tx_type = get_tx_type_16x16(xd, b);
  if (tx_type != DCT_DCT) {
    vp8_fht_c(be->src_diff, be->coeff, 32, tx_type, 16);
  } else
    vp8_transform_mby_16x16(mb);
#else
  vp8_transform_mby_16x16(mb);
#endif

  vp8_quantize_mby_16x16(mb);
#if CONFIG_HYBRIDTRANSFORM16X16
  // TODO(jingning) is it possible to quickly determine whether to force
  //                trailing coefficients to be zero, instead of running trellis
  //                optimization in the rate-distortion optimization loop?
  if (mb->e_mbd.mode_info_context->mbmi.mode < I8X8_PRED)
    vp8_optimize_mby_16x16(mb, rtcd);
#endif

  d = ENCODEMB_INVOKE(&rtcd->encodemb, mberr)(mb, 0);

  *Distortion = (d >> 2);
  // rate
  *Rate = vp8_rdcost_mby_16x16(mb);
  *skippable = mby_is_skippable_16x16(&mb->e_mbd);
}

static void macro_block_yrd(VP8_COMP *cpi, MACROBLOCK *x, int *rate,
                            int *distortion, int *skippable,
                            int64_t txfm_cache[NB_TXFM_MODES]) {
  VP8_COMMON *cm = &cpi->common;
  MB_MODE_INFO *mbmi = &x->e_mbd.mode_info_context->mbmi;

#if CONFIG_TX_SELECT

  MACROBLOCKD *xd = &x->e_mbd;
  int can_skip = cm->mb_no_coeff_skip;
  vp8_prob skip_prob = can_skip ? get_pred_prob(cm, xd, PRED_MBSKIP) : 128;
  int s0, s1;
  int r4x4, r4x4s, r8x8, r8x8s, d4x4, d8x8, s4x4, s8x8;
  int64_t rd4x4, rd8x8, rd4x4s, rd8x8s;
  int d16x16, r16x16, r16x16s, s16x16;
  int64_t rd16x16, rd16x16s;

  // FIXME don't do sub x3
  if (skip_prob == 0)
    skip_prob = 1;
  s0 = vp8_cost_bit(skip_prob, 0);
  s1 = vp8_cost_bit(skip_prob, 1);
  macro_block_yrd_16x16(x, &r16x16, &d16x16, IF_RTCD(&cpi->rtcd), &s16x16);
  if (can_skip) {
    if (s16x16) {
      rd16x16 = RDCOST(x->rdmult, x->rddiv, s1, d16x16);
    } else {
      rd16x16 = RDCOST(x->rdmult, x->rddiv, r16x16 + s0, d16x16);
    }
  } else {
    rd16x16 = RDCOST(x->rdmult, x->rddiv, r16x16, d16x16);
  }
  r16x16s = r16x16 + vp8_cost_one(cm->prob_tx[0]) + vp8_cost_one(cm->prob_tx[1]);
  if (can_skip) {
    if (s16x16) {
      rd16x16s = RDCOST(x->rdmult, x->rddiv, s1, d16x16);
    } else {
      rd16x16s = RDCOST(x->rdmult, x->rddiv, r16x16s + s0, d16x16);
    }
  } else {
    rd16x16s = RDCOST(x->rdmult, x->rddiv, r16x16s, d16x16);
  }
  macro_block_yrd_8x8(x, &r8x8, &d8x8, IF_RTCD(&cpi->rtcd), &s8x8);
  if (can_skip) {
    if (s8x8) {
      rd8x8 = RDCOST(x->rdmult, x->rddiv, s1, d8x8);
    } else {
      rd8x8 = RDCOST(x->rdmult, x->rddiv, r8x8 + s0, d8x8);
    }
  } else {
    rd8x8 = RDCOST(x->rdmult, x->rddiv, r8x8, d8x8);
  }
  r8x8s = r8x8 + vp8_cost_one(cm->prob_tx[0]);
  r8x8s += vp8_cost_zero(cm->prob_tx[1]);
  if (can_skip) {
    if (s8x8) {
      rd8x8s = RDCOST(x->rdmult, x->rddiv, s1, d8x8);
    } else {
      rd8x8s = RDCOST(x->rdmult, x->rddiv, r8x8s + s0, d8x8);
    }
  } else {
    rd8x8s = RDCOST(x->rdmult, x->rddiv, r8x8s, d8x8);
  }
  macro_block_yrd_4x4(x, &r4x4, &d4x4, IF_RTCD(&cpi->rtcd), &s4x4);
  if (can_skip) {
    if (s4x4) {
      rd4x4 = RDCOST(x->rdmult, x->rddiv, s1, d4x4);
    } else {
      rd4x4 = RDCOST(x->rdmult, x->rddiv, r4x4 + s0, d4x4);
    }
  } else {
    rd4x4 = RDCOST(x->rdmult, x->rddiv, r4x4, d4x4);
  }
  r4x4s = r4x4 + vp8_cost_zero(cm->prob_tx[0]);
  if (can_skip) {
    if (s4x4) {
      rd4x4s = RDCOST(x->rdmult, x->rddiv, s1, d4x4);
    } else {
      rd4x4s = RDCOST(x->rdmult, x->rddiv, r4x4s + s0, d4x4);
    }
  } else {
    rd4x4s = RDCOST(x->rdmult, x->rddiv, r4x4s, d4x4);
  }

  if ( cpi->common.txfm_mode == ALLOW_16X16 ||
      (cpi->common.txfm_mode == TX_MODE_SELECT &&
       rd16x16s < rd8x8s && rd16x16s < rd4x4s)) {
    mbmi->txfm_size = TX_16X16;
    *skippable = s16x16;
    *distortion = d16x16;
    *rate = (cpi->common.txfm_mode == ALLOW_16X16) ? r16x16 : r16x16s;