rdopt.c 78.5 KB
Newer Older
John Koleszar's avatar
John Koleszar committed
1
/*
2
 *  Copyright (c) 2010 The WebM project authors. All Rights Reserved.
John Koleszar's avatar
John Koleszar committed
3
 *
4
 *  Use of this source code is governed by a BSD-style license
5 6
 *  that can be found in the LICENSE file in the root of the source
 *  tree. An additional intellectual property rights grant can be found
7
 *  in the file PATENTS.  All contributing project authors may
8
 *  be found in the AUTHORS file in the root of the source tree.
John Koleszar's avatar
John Koleszar committed
9 10 11 12 13 14 15
 */


#include <stdio.h>
#include <math.h>
#include <limits.h>
#include <assert.h>
16 17
#include "vpx_config.h"
#include "vpx_rtcd.h"
John Koleszar's avatar
John Koleszar committed
18
#include "vp8/common/pragmas.h"
John Koleszar's avatar
John Koleszar committed
19 20 21 22 23
#include "tokenize.h"
#include "treewriter.h"
#include "onyx_int.h"
#include "modecosts.h"
#include "encodeintra.h"
John Koleszar's avatar
John Koleszar committed
24 25 26 27
#include "vp8/common/entropymode.h"
#include "vp8/common/reconinter.h"
#include "vp8/common/reconintra4x4.h"
#include "vp8/common/findnearmv.h"
28
#include "vp8/common/quant_common.h"
John Koleszar's avatar
John Koleszar committed
29 30 31 32
#include "encodemb.h"
#include "quantize.h"
#include "variance.h"
#include "mcomp.h"
Yunqing Wang's avatar
Yunqing Wang committed
33
#include "rdopt.h"
John Koleszar's avatar
John Koleszar committed
34
#include "vpx_mem/vpx_mem.h"
John Koleszar's avatar
John Koleszar committed
35
#include "vp8/common/systemdependent.h"
John Koleszar's avatar
John Koleszar committed
36

Scott LaVarnway's avatar
Scott LaVarnway committed
37 38
extern void vp8_update_zbin_extra(VP8_COMP *cpi, MACROBLOCK *x);

John Koleszar's avatar
John Koleszar committed
39 40
#define MAXF(a,b)            (((a) > (b)) ? (a) : (b))

41
static const int auto_speed_thresh[17] =
John Koleszar's avatar
John Koleszar committed
42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93
{
    1000,
    200,
    150,
    130,
    150,
    125,
    120,
    115,
    115,
    115,
    115,
    115,
    115,
    115,
    115,
    115,
    105
};

const MB_PREDICTION_MODE vp8_mode_order[MAX_MODES] =
{
    ZEROMV,
    DC_PRED,

    NEARESTMV,
    NEARMV,

    ZEROMV,
    NEARESTMV,

    ZEROMV,
    NEARESTMV,

    NEARMV,
    NEARMV,

    V_PRED,
    H_PRED,
    TM_PRED,

    NEWMV,
    NEWMV,
    NEWMV,

    SPLITMV,
    SPLITMV,
    SPLITMV,

    B_PRED,
};

94 95 96 97
/* This table determines the search order in reference frame priority order,
 * which may not necessarily match INTRA,LAST,GOLDEN,ARF
 */
const int vp8_ref_frame_order[MAX_MODES] =
John Koleszar's avatar
John Koleszar committed
98
{
99 100
    1,
    0,
John Koleszar's avatar
John Koleszar committed
101

102 103
    1,
    1,
John Koleszar's avatar
John Koleszar committed
104

105 106
    2,
    2,
John Koleszar's avatar
John Koleszar committed
107

108 109
    3,
    3,
John Koleszar's avatar
John Koleszar committed
110

111 112
    2,
    3,
John Koleszar's avatar
John Koleszar committed
113

114 115 116
    0,
    0,
    0,
John Koleszar's avatar
John Koleszar committed
117

118 119 120
    1,
    2,
    3,
John Koleszar's avatar
John Koleszar committed
121

122 123 124
    1,
    2,
    3,
John Koleszar's avatar
John Koleszar committed
125

126
    0,
John Koleszar's avatar
John Koleszar committed
127 128 129
};

static void fill_token_costs(
130 131
    unsigned int c      [BLOCK_TYPES] [COEF_BANDS] [PREV_COEF_CONTEXTS] [MAX_ENTROPY_TOKENS],
    const vp8_prob p    [BLOCK_TYPES] [COEF_BANDS] [PREV_COEF_CONTEXTS] [ENTROPY_NODES]
John Koleszar's avatar
John Koleszar committed
132 133 134 135 136 137 138 139 140 141 142 143 144
)
{
    int i, j, k;


    for (i = 0; i < BLOCK_TYPES; i++)
        for (j = 0; j < COEF_BANDS; j++)
            for (k = 0; k < PREV_COEF_CONTEXTS; k++)

                vp8_cost_tokens((int *)(c [i][j][k]), p [i][j][k], vp8_coef_tree);

}

Paul Wilkins's avatar
Paul Wilkins committed
145
static int rd_iifactor [ 32 ] =  {    4,   4,   3,   2,   1,   0,   0,   0,
John Koleszar's avatar
John Koleszar committed
146 147 148 149 150
                                      0,   0,   0,   0,   0,   0,   0,   0,
                                      0,   0,   0,   0,   0,   0,   0,   0,
                                      0,   0,   0,   0,   0,   0,   0,   0,
                                 };

151 152
/* values are now correlated to quantizer */
static int sad_per_bit16lut[QINDEX_RANGE] =
John Koleszar's avatar
John Koleszar committed
153
{
Yaowu Xu's avatar
Yaowu Xu committed
154 155 156 157 158 159
    2,  2,  2,  2,  2,  2,  2,  2,
    2,  2,  2,  2,  2,  2,  2,  2,
    3,  3,  3,  3,  3,  3,  3,  3,
    3,  3,  3,  3,  3,  3,  4,  4,
    4,  4,  4,  4,  4,  4,  4,  4,
    4,  4,  5,  5,  5,  5,  5,  5,
160
    5,  5,  5,  5,  5,  5,  6,  6,
Yaowu Xu's avatar
Yaowu Xu committed
161 162 163 164 165 166 167 168 169
    6,  6,  6,  6,  6,  6,  6,  6,
    6,  6,  7,  7,  7,  7,  7,  7,
    7,  7,  7,  7,  7,  7,  8,  8,
    8,  8,  8,  8,  8,  8,  8,  8,
    8,  8,  9,  9,  9,  9,  9,  9,
    9,  9,  9,  9,  9,  9,  10, 10,
    10, 10, 10, 10, 10, 10, 11, 11,
    11, 11, 11, 11, 12, 12, 12, 12,
    12, 12, 13, 13, 13, 13, 14, 14
John Koleszar's avatar
John Koleszar committed
170
};
171
static int sad_per_bit4lut[QINDEX_RANGE] =
John Koleszar's avatar
John Koleszar committed
172
{
Yaowu Xu's avatar
Yaowu Xu committed
173 174 175 176 177 178 179
    2,  2,  2,  2,  2,  2,  3,  3,
    3,  3,  3,  3,  3,  3,  3,  3,
    3,  3,  3,  3,  4,  4,  4,  4,
    4,  4,  4,  4,  4,  4,  5,  5,
    5,  5,  5,  5,  6,  6,  6,  6,
    6,  6,  6,  6,  6,  6,  6,  6,
    7,  7,  7,  7,  7,  7,  7,  7,
180
    7,  7,  7,  7,  7,  8,  8,  8,
Yaowu Xu's avatar
Yaowu Xu committed
181 182 183 184 185 186 187 188
    8,  8,  9,  9,  9,  9,  9,  9,
    10, 10, 10, 10, 10, 10, 10, 10,
    11, 11, 11, 11, 11, 11, 11, 11,
    12, 12, 12, 12, 12, 12, 12, 12,
    13, 13, 13, 13, 13, 13, 13, 14,
    14, 14, 14, 14, 15, 15, 15, 15,
    16, 16, 16, 16, 17, 17, 17, 18,
    18, 18, 19, 19, 19, 20, 20, 20,
John Koleszar's avatar
John Koleszar committed
189 190 191 192
};

void vp8cx_initialize_me_consts(VP8_COMP *cpi, int QIndex)
{
Yaowu Xu's avatar
Yaowu Xu committed
193 194
    cpi->mb.sadperbit16 =  sad_per_bit16lut[QIndex];
    cpi->mb.sadperbit4  =  sad_per_bit4lut[QIndex];
John Koleszar's avatar
John Koleszar committed
195 196 197 198 199 200
}

void vp8_initialize_rd_consts(VP8_COMP *cpi, int Qvalue)
{
    int q;
    int i;
201
    double capped_q = (Qvalue < 160) ? (double)Qvalue : 160.0;
202
    double rdconst = 2.70;
John Koleszar's avatar
John Koleszar committed
203 204 205

    vp8_clear_system_state();  //__asm emms;

206 207 208 209 210
    // Further tests required to see if optimum is different
    // for key frames, golden frames and arf frames.
    // if (cpi->common.refresh_golden_frame ||
    //     cpi->common.refresh_alt_ref_frame)
    cpi->RDMULT = (int)(rdconst * (capped_q * capped_q));
John Koleszar's avatar
John Koleszar committed
211

212 213 214 215 216 217 218 219 220 221 222 223
    // Extend rate multiplier along side quantizer zbin increases
    if (cpi->zbin_over_quant  > 0)
    {
        double oq_factor;
        double modq;

        // Experimental code using the same basic equation as used for Q above
        // The units of cpi->zbin_over_quant are 1/128 of Q bin size
        oq_factor = 1.0 + ((double)0.0015625 * cpi->zbin_over_quant);
        modq = (int)((double)capped_q * oq_factor);
        cpi->RDMULT = (int)(rdconst * (modq * modq));
    }
John Koleszar's avatar
John Koleszar committed
224

Paul Wilkins's avatar
Paul Wilkins committed
225
    if (cpi->pass == 2 && (cpi->common.frame_type != KEY_FRAME))
John Koleszar's avatar
John Koleszar committed
226
    {
227
        if (cpi->twopass.next_iiratio > 31)
Paul Wilkins's avatar
Paul Wilkins committed
228
            cpi->RDMULT += (cpi->RDMULT * rd_iifactor[31]) >> 4;
John Koleszar's avatar
John Koleszar committed
229
        else
230 231
            cpi->RDMULT +=
                (cpi->RDMULT * rd_iifactor[cpi->twopass.next_iiratio]) >> 4;
John Koleszar's avatar
John Koleszar committed
232 233
    }

234
    cpi->mb.errorperbit = (cpi->RDMULT / 110);
235 236
    cpi->mb.errorperbit += (cpi->mb.errorperbit==0);

John Koleszar's avatar
John Koleszar committed
237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281
    vp8_set_speed_features(cpi);

    q = (int)pow(Qvalue, 1.25);

    if (q < 8)
        q = 8;

    if (cpi->RDMULT > 1000)
    {
        cpi->RDDIV = 1;
        cpi->RDMULT /= 100;

        for (i = 0; i < MAX_MODES; i++)
        {
            if (cpi->sf.thresh_mult[i] < INT_MAX)
            {
                cpi->rd_threshes[i] = cpi->sf.thresh_mult[i] * q / 100;
            }
            else
            {
                cpi->rd_threshes[i] = INT_MAX;
            }

            cpi->rd_baseline_thresh[i] = cpi->rd_threshes[i];
        }
    }
    else
    {
        cpi->RDDIV = 100;

        for (i = 0; i < MAX_MODES; i++)
        {
            if (cpi->sf.thresh_mult[i] < (INT_MAX / q))
            {
                cpi->rd_threshes[i] = cpi->sf.thresh_mult[i] * q;
            }
            else
            {
                cpi->rd_threshes[i] = INT_MAX;
            }

            cpi->rd_baseline_thresh[i] = cpi->rd_threshes[i];
        }
    }

282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300
    {
      // build token cost array for the type of frame we have now
      FRAME_CONTEXT *l = &cpi->lfc_n;

      if(cpi->common.refresh_alt_ref_frame)
          l = &cpi->lfc_a;
      else if(cpi->common.refresh_golden_frame)
          l = &cpi->lfc_g;

      fill_token_costs(
          cpi->mb.token_costs,
          (const vp8_prob( *)[8][3][11]) l->coef_probs
      );
      /*
      fill_token_costs(
          cpi->mb.token_costs,
          (const vp8_prob( *)[8][3][11]) cpi->common.fc.coef_probs);
      */

John Koleszar's avatar
John Koleszar committed
301

302 303 304 305 306 307 308
      // TODO make these mode costs depend on last,alt or gold too.  (jbb)
      vp8_init_mode_costs(cpi);

      // TODO figure onnnnuut why making mv cost frame type dependent didn't help (jbb)
      //vp8_build_component_cost_table(cpi->mb.mvcost, (const MV_CONTEXT *) l->mvc, flags);

    }
John Koleszar's avatar
John Koleszar committed
309 310 311 312 313

}

void vp8_auto_select_speed(VP8_COMP *cpi)
{
314
    int milliseconds_for_compress = (int)(1000000 / cpi->frame_rate);
John Koleszar's avatar
John Koleszar committed
315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333

    milliseconds_for_compress = milliseconds_for_compress * (16 - cpi->oxcf.cpu_used) / 16;

#if 0

    if (0)
    {
        FILE *f;

        f = fopen("speed.stt", "a");
        fprintf(f, " %8ld %10ld %10ld %10ld\n",
                cpi->common.current_video_frame, cpi->Speed, milliseconds_for_compress, cpi->avg_pick_mode_time);
        fclose(f);
    }

#endif

    /*
    // this is done during parameter valid check
Johann's avatar
Johann committed
334 335 336 337
    if( cpi->oxcf.cpu_used > 16)
        cpi->oxcf.cpu_used = 16;
    if( cpi->oxcf.cpu_used < -16)
        cpi->oxcf.cpu_used = -16;
John Koleszar's avatar
John Koleszar committed
338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359
    */

    if (cpi->avg_pick_mode_time < milliseconds_for_compress && (cpi->avg_encode_time - cpi->avg_pick_mode_time) < milliseconds_for_compress)
    {
        if (cpi->avg_pick_mode_time == 0)
        {
            cpi->Speed = 4;
        }
        else
        {
            if (milliseconds_for_compress * 100 < cpi->avg_encode_time * 95)
            {
                cpi->Speed          += 2;
                cpi->avg_pick_mode_time = 0;
                cpi->avg_encode_time = 0;

                if (cpi->Speed > 16)
                {
                    cpi->Speed = 16;
                }
            }

360
            if (milliseconds_for_compress * 100 > cpi->avg_encode_time * auto_speed_thresh[cpi->Speed])
John Koleszar's avatar
John Koleszar committed
361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447
            {
                cpi->Speed          -= 1;
                cpi->avg_pick_mode_time = 0;
                cpi->avg_encode_time = 0;

                // In real-time mode, cpi->speed is in [4, 16].
                if (cpi->Speed < 4)        //if ( cpi->Speed < 0 )
                {
                    cpi->Speed = 4;        //cpi->Speed = 0;
                }
            }
        }
    }
    else
    {
        cpi->Speed += 4;

        if (cpi->Speed > 16)
            cpi->Speed = 16;


        cpi->avg_pick_mode_time = 0;
        cpi->avg_encode_time = 0;
    }
}

int vp8_block_error_c(short *coeff, short *dqcoeff)
{
    int i;
    int error = 0;

    for (i = 0; i < 16; i++)
    {
        int this_diff = coeff[i] - dqcoeff[i];
        error += this_diff * this_diff;
    }

    return error;
}

int vp8_mbblock_error_c(MACROBLOCK *mb, int dc)
{
    BLOCK  *be;
    BLOCKD *bd;
    int i, j;
    int berror, error = 0;

    for (i = 0; i < 16; i++)
    {
        be = &mb->block[i];
        bd = &mb->e_mbd.block[i];

        berror = 0;

        for (j = dc; j < 16; j++)
        {
            int this_diff = be->coeff[j] - bd->dqcoeff[j];
            berror += this_diff * this_diff;
        }

        error += berror;
    }

    return error;
}

int vp8_mbuverror_c(MACROBLOCK *mb)
{

    BLOCK  *be;
    BLOCKD *bd;


    int i;
    int error = 0;

    for (i = 16; i < 24; i++)
    {
        be = &mb->block[i];
        bd = &mb->e_mbd.block[i];

        error += vp8_block_error_c(be->coeff, bd->dqcoeff);
    }

    return error;
}

John Koleszar's avatar
John Koleszar committed
448
int VP8_UVSSE(MACROBLOCK *x)
John Koleszar's avatar
John Koleszar committed
449 450 451 452 453 454 455 456
{
    unsigned char *uptr, *vptr;
    unsigned char *upred_ptr = (*(x->block[16].base_src) + x->block[16].src);
    unsigned char *vpred_ptr = (*(x->block[20].base_src) + x->block[20].src);
    int uv_stride = x->block[16].src_stride;

    unsigned int sse1 = 0;
    unsigned int sse2 = 0;
457 458
    int mv_row = x->e_mbd.mode_info_context->mbmi.mv.as_mv.row;
    int mv_col = x->e_mbd.mode_info_context->mbmi.mv.as_mv.col;
John Koleszar's avatar
John Koleszar committed
459
    int offset;
Scott LaVarnway's avatar
Scott LaVarnway committed
460
    int pre_stride = x->e_mbd.pre.uv_stride;
John Koleszar's avatar
John Koleszar committed
461

462 463 464 465 466 467 468 469 470 471 472 473
    if (mv_row < 0)
        mv_row -= 1;
    else
        mv_row += 1;

    if (mv_col < 0)
        mv_col -= 1;
    else
        mv_col += 1;

    mv_row /= 2;
    mv_col /= 2;
John Koleszar's avatar
John Koleszar committed
474 475 476 477 478 479 480

    offset = (mv_row >> 3) * pre_stride + (mv_col >> 3);
    uptr = x->e_mbd.pre.u_buffer + offset;
    vptr = x->e_mbd.pre.v_buffer + offset;

    if ((mv_row | mv_col) & 7)
    {
John Koleszar's avatar
John Koleszar committed
481
        vp8_sub_pixel_variance8x8(uptr, pre_stride,
482
            mv_col & 7, mv_row & 7, upred_ptr, uv_stride, &sse2);
John Koleszar's avatar
John Koleszar committed
483
        vp8_sub_pixel_variance8x8(vptr, pre_stride,
484
            mv_col & 7, mv_row & 7, vpred_ptr, uv_stride, &sse1);
John Koleszar's avatar
John Koleszar committed
485 486 487 488
        sse2 += sse1;
    }
    else
    {
John Koleszar's avatar
John Koleszar committed
489
        vp8_variance8x8(uptr, pre_stride,
490
            upred_ptr, uv_stride, &sse2);
John Koleszar's avatar
John Koleszar committed
491
        vp8_variance8x8(vptr, pre_stride,
492
            vpred_ptr, uv_stride, &sse1);
John Koleszar's avatar
John Koleszar committed
493 494 495 496 497 498 499 500 501
        sse2 += sse1;
    }
    return sse2;

}

static int cost_coeffs(MACROBLOCK *mb, BLOCKD *b, int type, ENTROPY_CONTEXT *a, ENTROPY_CONTEXT *l)
{
    int c = !type;              /* start at coef 0, unless Y with Y2 */
502
    int eob = (int)(*b->eob);
John Koleszar's avatar
John Koleszar committed
503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530
    int pt ;    /* surrounding block/prev coef predictor */
    int cost = 0;
    short *qcoeff_ptr = b->qcoeff;

    VP8_COMBINEENTROPYCONTEXTS(pt, *a, *l);

# define QC( I)  ( qcoeff_ptr [vp8_default_zig_zag1d[I]] )

    for (; c < eob; c++)
    {
        int v = QC(c);
        int t = vp8_dct_value_tokens_ptr[v].Token;
        cost += mb->token_costs [type] [vp8_coef_bands[c]] [pt] [t];
        cost += vp8_dct_value_cost_ptr[v];
        pt = vp8_prev_token_class[t];
    }

# undef QC

    if (c < 16)
        cost += mb->token_costs [type] [vp8_coef_bands[c]] [pt] [DCT_EOB_TOKEN];

    pt = (c != !type); // is eob first coefficient;
    *a = *l = pt;

    return cost;
}

Scott LaVarnway's avatar
Scott LaVarnway committed
531
static int vp8_rdcost_mby(MACROBLOCK *mb)
John Koleszar's avatar
John Koleszar committed
532 533 534 535
{
    int cost = 0;
    int b;
    MACROBLOCKD *x = &mb->e_mbd;
536 537 538 539 540 541
    ENTROPY_CONTEXT_PLANES t_above, t_left;
    ENTROPY_CONTEXT *ta;
    ENTROPY_CONTEXT *tl;

    vpx_memcpy(&t_above, mb->e_mbd.above_context, sizeof(ENTROPY_CONTEXT_PLANES));
    vpx_memcpy(&t_left, mb->e_mbd.left_context, sizeof(ENTROPY_CONTEXT_PLANES));
John Koleszar's avatar
John Koleszar committed
542

543 544
    ta = (ENTROPY_CONTEXT *)&t_above;
    tl = (ENTROPY_CONTEXT *)&t_left;
John Koleszar's avatar
John Koleszar committed
545 546

    for (b = 0; b < 16; b++)
Scott LaVarnway's avatar
Scott LaVarnway committed
547
        cost += cost_coeffs(mb, x->block + b, PLANE_TYPE_Y_NO_DC,
548
                    ta + vp8_block2above[b], tl + vp8_block2left[b]);
John Koleszar's avatar
John Koleszar committed
549

Scott LaVarnway's avatar
Scott LaVarnway committed
550
    cost += cost_coeffs(mb, x->block + 24, PLANE_TYPE_Y2,
Scott LaVarnway's avatar
Scott LaVarnway committed
551
                ta + vp8_block2above[24], tl + vp8_block2left[24]);
John Koleszar's avatar
John Koleszar committed
552 553 554 555

    return cost;
}

556 557
static void macro_block_yrd( MACROBLOCK *mb,
                             int *Rate,
558
                             int *Distortion)
559 560 561 562 563 564 565 566 567
{
    int b;
    MACROBLOCKD *const x = &mb->e_mbd;
    BLOCK   *const mb_y2 = mb->block + 24;
    BLOCKD *const x_y2  = x->block + 24;
    short *Y2DCPtr = mb_y2->src_diff;
    BLOCK *beptr;
    int d;

568
    vp8_subtract_mby( mb->src_diff, *(mb->block[0].base_src),
569
        mb->block[0].src_stride,  mb->e_mbd.predictor, 16);
570 571 572 573

    // Fdct and building the 2nd order block
    for (beptr = mb->block; beptr < mb->block + 16; beptr += 2)
    {
John Koleszar's avatar
John Koleszar committed
574
        mb->short_fdct8x4(beptr->src_diff, beptr->coeff, 32);
575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591
        *Y2DCPtr++ = beptr->coeff[0];
        *Y2DCPtr++ = beptr->coeff[16];
    }

    // 2nd order fdct
    mb->short_walsh4x4(mb_y2->src_diff, mb_y2->coeff, 8);

    // Quantization
    for (b = 0; b < 16; b++)
    {
        mb->quantize_b(&mb->block[b], &mb->e_mbd.block[b]);
    }

    // DC predication and Quantization of 2nd Order block
    mb->quantize_b(mb_y2, x_y2);

    // Distortion
592 593
    d = vp8_mbblock_error(mb, 1) << 2;
    d += vp8_block_error(mb_y2->coeff, x_y2->dqcoeff);
594 595 596 597 598 599

    *Distortion = (d >> 4);

    // rate
    *Rate = vp8_rdcost_mby(mb);
}
John Koleszar's avatar
John Koleszar committed
600

601
static void copy_predictor(unsigned char *dst, const unsigned char *predictor)
602
{
603 604 605 606 607 608
    const unsigned int *p = (const unsigned int *)predictor;
    unsigned int *d = (unsigned int *)dst;
    d[0] = p[0];
    d[4] = p[4];
    d[8] = p[8];
    d[12] = p[12];
609
}
610
static int rd_pick_intra4x4block(
John Koleszar's avatar
John Koleszar committed
611 612 613 614 615
    VP8_COMP *cpi,
    MACROBLOCK *x,
    BLOCK *be,
    BLOCKD *b,
    B_PREDICTION_MODE *best_mode,
616
    unsigned int *bmode_costs,
John Koleszar's avatar
John Koleszar committed
617 618 619 620 621 622 623 624
    ENTROPY_CONTEXT *a,
    ENTROPY_CONTEXT *l,

    int *bestrate,
    int *bestratey,
    int *bestdistortion)
{
    B_PREDICTION_MODE mode;
625
    int best_rd = INT_MAX;
John Koleszar's avatar
John Koleszar committed
626 627 628 629 630
    int rate = 0;
    int distortion;

    ENTROPY_CONTEXT ta = *a, tempa = *a;
    ENTROPY_CONTEXT tl = *l, templ = *l;
631 632 633 634 635 636 637
    /*
     * The predictor buffer is a 2d buffer with a stride of 16.  Create
     * a temp buffer that meets the stride requirements, but we are only
     * interested in the left 4x4 block
     * */
    DECLARE_ALIGNED_ARRAY(16, unsigned char,  best_predictor, 16*4);
    DECLARE_ALIGNED_ARRAY(16, short, best_dqcoeff, 16);
Scott LaVarnway's avatar
Scott LaVarnway committed
638 639
    int dst_stride = x->e_mbd.dst.y_stride;
    unsigned char *base_dst = x->e_mbd.dst.y_buffer;
John Koleszar's avatar
John Koleszar committed
640 641 642 643 644 645

    for (mode = B_DC_PRED; mode <= B_HU_PRED; mode++)
    {
        int this_rd;
        int ratey;

646 647
        rate = bmode_costs[mode];

Scott LaVarnway's avatar
Scott LaVarnway committed
648 649
        vp8_intra4x4_predict(base_dst + b->offset, dst_stride, mode,
                             b->predictor, 16);
650
        vp8_subtract_b(be, b, 16);
John Koleszar's avatar
John Koleszar committed
651
        x->short_fdct4x4(be->src_diff, be->coeff, 32);
652
        x->quantize_b(be, b);
John Koleszar's avatar
John Koleszar committed
653 654 655 656

        tempa = ta;
        templ = tl;

Scott LaVarnway's avatar
Scott LaVarnway committed
657
        ratey = cost_coeffs(x, b, PLANE_TYPE_Y_WITH_DC, &tempa, &templ);
John Koleszar's avatar
John Koleszar committed
658
        rate += ratey;
659
        distortion = vp8_block_error(be->coeff, b->dqcoeff) >> 2;
John Koleszar's avatar
John Koleszar committed
660 661 662 663 664 665 666 667 668 669 670 671

        this_rd = RDCOST(x->rdmult, x->rddiv, rate, distortion);

        if (this_rd < best_rd)
        {
            *bestrate = rate;
            *bestratey = ratey;
            *bestdistortion = distortion;
            best_rd = this_rd;
            *best_mode = mode;
            *a = tempa;
            *l = templ;
672 673
            copy_predictor(best_predictor, b->predictor);
            vpx_memcpy(best_dqcoeff, b->dqcoeff, 32);
John Koleszar's avatar
John Koleszar committed
674 675
        }
    }
Scott LaVarnway's avatar
Scott LaVarnway committed
676
    b->bmi.as_mode = (B_PREDICTION_MODE)(*best_mode);
677

Scott LaVarnway's avatar
Scott LaVarnway committed
678 679
    vp8_short_idct4x4llm(best_dqcoeff, best_predictor, 16, base_dst + b->offset,
                         dst_stride);
John Koleszar's avatar
John Koleszar committed
680

681
    return best_rd;
John Koleszar's avatar
John Koleszar committed
682 683
}

684 685
static int rd_pick_intra4x4mby_modes(VP8_COMP *cpi, MACROBLOCK *mb, int *Rate,
                                     int *rate_y, int *Distortion, int best_rd)
John Koleszar's avatar
John Koleszar committed
686 687 688 689 690 691
{
    MACROBLOCKD *const xd = &mb->e_mbd;
    int i;
    int cost = mb->mbmode_cost [xd->frame_type] [B_PRED];
    int distortion = 0;
    int tot_rate_y = 0;
692
    int64_t total_rd = 0;
693 694 695
    ENTROPY_CONTEXT_PLANES t_above, t_left;
    ENTROPY_CONTEXT *ta;
    ENTROPY_CONTEXT *tl;
696
    unsigned int *bmode_costs;
697 698 699 700 701 702

    vpx_memcpy(&t_above, mb->e_mbd.above_context, sizeof(ENTROPY_CONTEXT_PLANES));
    vpx_memcpy(&t_left, mb->e_mbd.left_context, sizeof(ENTROPY_CONTEXT_PLANES));

    ta = (ENTROPY_CONTEXT *)&t_above;
    tl = (ENTROPY_CONTEXT *)&t_left;
John Koleszar's avatar
John Koleszar committed
703

Scott LaVarnway's avatar
Scott LaVarnway committed
704
    intra_prediction_down_copy(xd, xd->dst.y_buffer - xd->dst.y_stride + 16);
John Koleszar's avatar
John Koleszar committed
705

706 707
    bmode_costs = mb->inter_bmode_costs;

John Koleszar's avatar
John Koleszar committed
708 709 710 711 712 713 714
    for (i = 0; i < 16; i++)
    {
        MODE_INFO *const mic = xd->mode_info_context;
        const int mis = xd->mode_info_stride;
        B_PREDICTION_MODE UNINITIALIZED_IS_SAFE(best_mode);
        int UNINITIALIZED_IS_SAFE(r), UNINITIALIZED_IS_SAFE(ry), UNINITIALIZED_IS_SAFE(d);

715 716
        if (mb->e_mbd.frame_type == KEY_FRAME)
        {
Scott LaVarnway's avatar
Scott LaVarnway committed
717 718
            const B_PREDICTION_MODE A = above_block_mode(mic, i, mis);
            const B_PREDICTION_MODE L = left_block_mode(mic, i);
719 720 721 722

            bmode_costs  = mb->bmode_costs[A][L];
        }

723
        total_rd += rd_pick_intra4x4block(
724
            cpi, mb, mb->block + i, xd->block + i, &best_mode, bmode_costs,
725 726
            ta + vp8_block2above[i],
            tl + vp8_block2left[i], &r, &ry, &d);
John Koleszar's avatar
John Koleszar committed
727 728 729 730

        cost += r;
        distortion += d;
        tot_rate_y += ry;
Scott LaVarnway's avatar
Scott LaVarnway committed
731 732

        mic->bmi[i].as_mode = best_mode;
733

734
        if(total_rd >= (int64_t)best_rd)
735
            break;
John Koleszar's avatar
John Koleszar committed
736 737
    }

738
    if(total_rd >= (int64_t)best_rd)
739
        return INT_MAX;
740

John Koleszar's avatar
John Koleszar committed
741 742 743 744 745 746
    *Rate = cost;
    *rate_y += tot_rate_y;
    *Distortion = distortion;

    return RDCOST(mb->rdmult, mb->rddiv, cost, distortion);
}
747 748 749 750 751 752 753


static int rd_pick_intra16x16mby_mode(VP8_COMP *cpi,
                                      MACROBLOCK *x,
                                      int *Rate,
                                      int *rate_y,
                                      int *Distortion)
John Koleszar's avatar
John Koleszar committed
754 755 756 757
{
    MB_PREDICTION_MODE mode;
    MB_PREDICTION_MODE UNINITIALIZED_IS_SAFE(mode_selected);
    int rate, ratey;
758
    int distortion;
John Koleszar's avatar
John Koleszar committed
759
    int best_rd = INT_MAX;
760
    int this_rd;
John Koleszar's avatar
John Koleszar committed
761 762 763 764

    //Y Search for 16x16 intra prediction mode
    for (mode = DC_PRED; mode <= TM_PRED; mode++)
    {
765
        x->e_mbd.mode_info_context->mbmi.mode = mode;
John Koleszar's avatar
John Koleszar committed
766

John Koleszar's avatar
John Koleszar committed
767
        vp8_build_intra_predictors_mby
768
            (&x->e_mbd);
John Koleszar's avatar
John Koleszar committed
769

770
        macro_block_yrd(x, &ratey, &distortion);
771 772
        rate = ratey + x->mbmode_cost[x->e_mbd.frame_type]
                                     [x->e_mbd.mode_info_context->mbmi.mode];
John Koleszar's avatar
John Koleszar committed
773 774 775 776 777 778 779 780 781

        this_rd = RDCOST(x->rdmult, x->rddiv, rate, distortion);

        if (this_rd < best_rd)
        {
            mode_selected = mode;
            best_rd = this_rd;
            *Rate = rate;
            *rate_y = ratey;
782
            *Distortion = distortion;
John Koleszar's avatar
John Koleszar committed
783 784 785
        }
    }

786
    x->e_mbd.mode_info_context->mbmi.mode = mode_selected;
John Koleszar's avatar
John Koleszar committed
787 788 789 790 791 792 793 794
    return best_rd;
}

static int rd_cost_mbuv(MACROBLOCK *mb)
{
    int b;
    int cost = 0;
    MACROBLOCKD *x = &mb->e_mbd;
795 796 797 798 799 800
    ENTROPY_CONTEXT_PLANES t_above, t_left;
    ENTROPY_CONTEXT *ta;
    ENTROPY_CONTEXT *tl;

    vpx_memcpy(&t_above, mb->e_mbd.above_context, sizeof(ENTROPY_CONTEXT_PLANES));
    vpx_memcpy(&t_left, mb->e_mbd.left_context, sizeof(ENTROPY_CONTEXT_PLANES));
John Koleszar's avatar
John Koleszar committed
801

802 803
    ta = (ENTROPY_CONTEXT *)&t_above;
    tl = (ENTROPY_CONTEXT *)&t_left;
John Koleszar's avatar
John Koleszar committed
804

Scott LaVarnway's avatar
Scott LaVarnway committed
805 806
    for (b = 16; b < 24; b++)
        cost += cost_coeffs(mb, x->block + b, PLANE_TYPE_UV,
807
                    ta + vp8_block2above[b], tl + vp8_block2left[b]);
John Koleszar's avatar
John Koleszar committed
808 809 810 811 812

    return cost;
}


813 814
static int rd_inter16x16_uv(VP8_COMP *cpi, MACROBLOCK *x, int *rate,
                            int *distortion, int fullpixel)
John Koleszar's avatar
John Koleszar committed
815
{
816
    vp8_build_inter16x16_predictors_mbuv(&x->e_mbd);
817
    vp8_subtract_mbuv(x->src_diff,
818 819
        x->src.u_buffer, x->src.v_buffer, x->src.uv_stride,
        &x->e_mbd.predictor[256], &x->e_mbd.predictor[320], 8);
John Koleszar's avatar
John Koleszar committed
820

821 822 823 824
    vp8_transform_mbuv(x);
    vp8_quantize_mbuv(x);

    *rate       = rd_cost_mbuv(x);
825
    *distortion = vp8_mbuverror(x) / 4;
826 827 828 829 830 831 832 833

    return RDCOST(x->rdmult, x->rddiv, *rate, *distortion);
}

static int rd_inter4x4_uv(VP8_COMP *cpi, MACROBLOCK *x, int *rate,
                          int *distortion, int fullpixel)
{
    vp8_build_inter4x4_predictors_mbuv(&x->e_mbd);
834
    vp8_subtract_mbuv(x->src_diff,
835 836
        x->src.u_buffer, x->src.v_buffer, x->src.uv_stride,
        &x->e_mbd.predictor[256], &x->e_mbd.predictor[320], 8);
837 838 839

    vp8_transform_mbuv(x);
    vp8_quantize_mbuv(x);
John Koleszar's avatar
John Koleszar committed
840 841

    *rate       = rd_cost_mbuv(x);
842
    *distortion = vp8_mbuverror(x) / 4;
John Koleszar's avatar
John Koleszar committed
843

844
    return RDCOST(x->rdmult, x->rddiv, *rate, *distortion);
John Koleszar's avatar
John Koleszar committed
845 846
}

847
static void rd_pick_intra_mbuv_mode(VP8_COMP *cpi, MACROBLOCK *x, int *rate, int *rate_tokenonly, int *distortion)
John Koleszar's avatar
John Koleszar committed
848 849 850 851 852 853 854 855 856 857 858 859 860
{
    MB_PREDICTION_MODE mode;
    MB_PREDICTION_MODE UNINITIALIZED_IS_SAFE(mode_selected);
    int best_rd = INT_MAX;
    int UNINITIALIZED_IS_SAFE(d), UNINITIALIZED_IS_SAFE(r);
    int rate_to;

    for (mode = DC_PRED; mode <= TM_PRED; mode++)
    {
        int rate;
        int distortion;
        int this_rd;

861
        x->e_mbd.mode_info_context->mbmi.uv_mode = mode;
John Koleszar's avatar
John Koleszar committed
862
        vp8_build_intra_predictors_mbuv
863
                     (&x->e_mbd);
864
        vp8_subtract_mbuv(x->src_diff,
865 866
                      x->src.u_buffer, x->src.v_buffer, x->src.uv_stride,
                      &x->e_mbd.predictor[256], &x->e_mbd.predictor[320], 8);
867 868
        vp8_transform_mbuv(x);
        vp8_quantize_mbuv(x);
John Koleszar's avatar
John Koleszar committed
869 870

        rate_to = rd_cost_mbuv(x);
871
        rate = rate_to + x->intra_uv_mode_cost[x->e_mbd.frame_type][x->e_mbd.mode_info_context->mbmi.uv_mode];
John Koleszar's avatar
John Koleszar committed
872

873
        distortion = vp8_mbuverror(x) / 4;
John Koleszar's avatar
John Koleszar committed
874

875
        this_rd = RDCOST(x->rdmult, x->rddiv, rate, distortion);
John Koleszar's avatar
John Koleszar committed
876 877 878 879 880 881 882 883 884 885 886 887 888 889

        if (this_rd < best_rd)
        {
            best_rd = this_rd;
            d = distortion;
            r = rate;
            *rate_tokenonly = rate_to;
            mode_selected = mode;
        }
    }

    *rate = r;
    *distortion = d;

890
    x->e_mbd.mode_info_context->mbmi.uv_mode = mode_selected;
John Koleszar's avatar
John Koleszar committed
891 892 893 894 895 896 897
}

int vp8_cost_mv_ref(MB_PREDICTION_MODE m, const int near_mv_ref_ct[4])
{
    vp8_prob p [VP8_MVREFS-1];
    assert(NEARESTMV <= m  &&  m <= SPLITMV);
    vp8_mv_ref_probs(p, near_mv_ref_ct);
898 899
    return vp8_cost_token(vp8_mv_ref_tree, p,
                          vp8_mv_ref_encoding_array - NEARESTMV + m);
John Koleszar's avatar
John Koleszar committed
900 901
}

Scott LaVarnway's avatar
Scott LaVarnway committed
902
void vp8_set_mbmode_and_mvs(MACROBLOCK *x, MB_PREDICTION_MODE mb, int_mv *mv)
John Koleszar's avatar
John Koleszar committed
903
{
904
    x->e_mbd.mode_info_context->mbmi.mode = mb;
Scott LaVarnway's avatar
Scott LaVarnway committed
905
    x->e_mbd.mode_info_context->mbmi.mv.as_int = mv->as_int;
John Koleszar's avatar
John Koleszar committed
906 907 908 909 910 911
}

static int labels2mode(
    MACROBLOCK *x,
    int const *labelings, int which_label,
    B_PREDICTION_MODE this_mode,
Scott LaVarnway's avatar
Scott LaVarnway committed
912
    int_mv *this_mv, int_mv *best_ref_mv,
John Koleszar's avatar
John Koleszar committed
913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952
    int *mvcost[2]
)
{
    MACROBLOCKD *const xd = & x->e_mbd;
    MODE_INFO *const mic = xd->mode_info_context;
    const int mis = xd->mode_info_stride;

    int cost = 0;
    int thismvcost = 0;

    /* We have to be careful retrieving previously-encoded motion vectors.
       Ones from this macroblock have to be pulled from the BLOCKD array
       as they have not yet made it to the bmi array in our MB_MODE_INFO. */

    int i = 0;

    do
    {
        BLOCKD *const d = xd->block + i;
        const int row = i >> 2,  col = i & 3;

        B_PREDICTION_MODE m;

        if (labelings[i] != which_label)
            continue;

        if (col  &&  labelings[i] == labelings[i-1])
            m = LEFT4X4;
        else if (row  &&  labelings[i] == labelings[i-4])
            m = ABOVE4X4;
        else
        {
            // the only time we should do costing for new motion vector or mode
            // is when we are on a new label  (jbb May 08, 2007)
            switch (m = this_mode)
            {
            case NEW4X4 :
                thismvcost  = vp8_mv_bit_cost(this_mv, best_ref_mv, mvcost, 102);
                break;
            case LEFT4X4:
Scott LaVarnway's avatar
Scott LaVarnway committed
953
                this_mv->as_int = col ? d[-1].bmi.mv.as_int : left_block_mv(mic, i);
John Koleszar's avatar
John Koleszar committed
954 955
                break;
            case ABOVE4X4:
Scott LaVarnway's avatar
Scott LaVarnway committed
956
                this_mv->as_int = row ? d[-4].bmi.mv.as_int : above_block_mv(mic, i, mis);
John Koleszar's avatar
John Koleszar committed
957 958
                break;
            case ZERO4X4:
Scott LaVarnway's avatar
Scott LaVarnway committed
959
                this_mv->as_int = 0;
John Koleszar's avatar
John Koleszar committed
960 961 962 963 964 965 966
                break;
            default:
                break;
            }

            if (m == ABOVE4X4)  // replace above with left if same
            {
Scott LaVarnway's avatar
Scott LaVarnway committed
967
                int_mv left_mv;
Scott LaVarnway's avatar
Scott LaVarnway committed
968

Scott LaVarnway's avatar
Scott LaVarnway committed
969
                left_mv.as_int = col ? d[-1].bmi.mv.as_int :
Scott LaVarnway's avatar
Scott LaVarnway committed
970
                                        left_block_mv(mic, i);
John Koleszar's avatar
John Koleszar committed
971

Scott LaVarnway's avatar
Scott LaVarnway committed
972
                if (left_mv.as_int == this_mv->as_int)
John Koleszar's avatar
John Koleszar committed
973 974 975 976 977 978
                    m = LEFT4X4;
            }

            cost = x->inter_bmode_costs[ m];
        }

Scott LaVarnway's avatar
Scott LaVarnway committed
979
        d->bmi.mv.as_int = this_mv->as_int;
John Koleszar's avatar
John Koleszar committed
980

981 982 983
        x->partition_info->bmi[i].mode = m;
        x->partition_info->bmi[i].mv.as_int = this_mv->as_int;

John Koleszar's avatar
John Koleszar committed
984 985 986 987 988 989 990
    }
    while (++i < 16);

    cost += thismvcost ;
    return cost;
}

991 992 993
static int rdcost_mbsegment_y(MACROBLOCK *mb, const int *labels,
                              int which_label, ENTROPY_CONTEXT *ta,
                              ENTROPY_CONTEXT *tl)
John Koleszar's avatar
John Koleszar committed
994 995 996 997 998 999 1000
{
    int cost = 0;
    int b;
    MACROBLOCKD *x = &mb->e_mbd;

    for (b = 0; b < 16; b++)
        if (labels[ b] == which_label)
Scott LaVarnway's avatar
Scott LaVarnway committed
1001
            cost += cost_coeffs(mb, x->block + b, PLANE_TYPE_Y_WITH_DC,
1002 1003
                                ta + vp8_block2above[b],
                                tl + vp8_block2left[b]);
John Koleszar's avatar
John Koleszar committed
1004 1005 1006 1007

    return cost;

}
1008
static unsigned int vp8_encode_inter_mb_segment(MACROBLOCK *x, int const *labels, int which_label)
John Koleszar's avatar
John Koleszar committed
1009 1010 1011
{
    int i;
    unsigned int distortion = 0;
Scott LaVarnway's avatar
Scott LaVarnway committed
1012 1013 1014
    int pre_stride = x->e_mbd.pre.y_stride;
    unsigned char *base_pre = x->e_mbd.pre.y_buffer;

John Koleszar's avatar
John Koleszar committed
1015 1016 1017 1018 1019 1020 1021 1022

    for (i = 0; i < 16; i++)
    {
        if (labels[i] == which_label)
        {
            BLOCKD *bd = &x->e_mbd.block[i];
            BLOCK *be = &x->block[i];

Scott LaVarnway's avatar
Scott LaVarnway committed
1023
            vp8_build_inter_predictors_b(bd, 16, base_pre, pre_stride, x->e_mbd.subpixel_predict);
1024
            vp8_subtract_b(be, bd, 16);
John Koleszar's avatar
John Koleszar committed
1025
            x->short_fdct4x4(be->src_diff, be->coeff, 32);
John Koleszar's avatar
John Koleszar committed
1026 1027 1028

            // set to 0 no way to account for 2nd order DC so discount
            //be->coeff[0] = 0;
1029
            x->quantize_b(be, bd);
John Koleszar's avatar
John Koleszar committed
1030

1031
            distortion += vp8_block_error(be->coeff, bd->dqcoeff);
John Koleszar's avatar
John Koleszar committed
1032 1033 1034 1035 1036 1037
        }
    }

    return distortion;
}

1038 1039 1040 1041 1042

static const unsigned int segmentation_to_sseshift[4] = {3, 3, 2, 0};


typedef struct
John Koleszar's avatar
John Koleszar committed
1043
{
Scott LaVarnway's avatar
Scott LaVarnway committed
1044 1045
  int_mv *ref_mv;
  int_mv mvp;
1046 1047 1048 1049 1050 1051 1052

  int segment_rd;
  int segment_num;
  int r;
  int d;
  int segment_yrate;
  B_PREDICTION_MODE modes[16];
1053
  int_mv mvs[16];
1054 1055 1056 1057 1058
  unsigned char eobs[16];

  int mvthresh;
  int *mdcounts;

Scott LaVarnway's avatar
Scott LaVarnway committed
1059
  int_mv sv_mvp[4];     // save 4 mvp from 8x8
1060 1061
  int sv_istep[2];  // save 2 initial step_param for 16x8/8x16

1062 1063 1064
} BEST_SEG_INFO;


1065 1066
static void rd_check_segment(VP8_COMP *cpi, MACROBLOCK *x,
                             BEST_SEG_INFO *bsi, unsigned int segmentation)
1067 1068
{
    int i;
John Koleszar's avatar
John Koleszar committed
1069 1070 1071
    int const *labels;
    int br = 0;
    int bd = 0;
1072
    B_PREDICTION_MODE this_mode;
John Koleszar's avatar
John Koleszar committed
1073

1074

1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090
    int label_count;
    int this_segment_rd = 0;
    int label_mv_thresh;
    int rate = 0;
    int sbr = 0;
    int sbd = 0;
    int segmentyrate = 0;

    vp8_variance_fn_ptr_t *v_fn_ptr;

    ENTROPY_CONTEXT_PLANES t_above, t_left;
    ENTROPY_CONTEXT *ta;
    ENTROPY_CONTEXT *tl;
    ENTROPY_CONTEXT_PLANES t_above_b, t_left_b;
    ENTROPY_CONTEXT *ta_b;
    ENTROPY_CONTEXT *tl_b;
John Koleszar's avatar
John Koleszar committed
1091

1092 1093
    vpx_memcpy(&t_above, x->e_mbd.above_context, sizeof(ENTROPY_CONTEXT_PLANES));
    vpx_memcpy(&t_left, x->e_mbd.left_context, sizeof(ENTROPY_CONTEXT_PLANES));
John Koleszar's avatar
John Koleszar committed
1094

1095 1096 1097 1098 1099 1100 1101
    ta = (ENTROPY_CONTEXT *)&t_above;
    tl = (ENTROPY_CONTEXT *)&t_left;
    ta_b = (ENTROPY_CONTEXT *)&t_above_b;
    tl_b = (ENTROPY_CONTEXT *)&t_left_b;

    br = 0;
    bd = 0;
1102

1103 1104 1105
    v_fn_ptr = &cpi->fn_ptr[segmentation];
    labels = vp8_mbsplits[segmentation];
    label_count = vp8_mbsplit_count[segmentation];
1106

1107 1108 1109 1110 1111 1112 1113 1114 1115
    // 64 makes this threshold really big effectively
    // making it so that we very rarely check mvs on
    // segments.   setting this to 1 would make mv thresh
    // roughly equal to what it is for macroblocks
    label_mv_thresh = 1 * bsi->mvthresh / label_count ;

    // Segmentation method overheads
    rate = vp8_cost_token(vp8_mbsplit_tree, vp8_mbsplit_probs, vp8_mbsplit_encodings + segmentation);
    rate += vp8_cost_mv_ref(SPLITMV, bsi->mdcounts);
1116
    this_segment_rd += RDCOST(x->rdmult, x->rddiv, rate, 0);
1117 1118 1119
    br += rate;

    for (i = 0; i < label_count; i++)
John Koleszar's avatar
John Koleszar committed
1120
    {
Scott LaVarnway's avatar
Scott LaVarnway committed
1121
        int_mv mode_mv[B_MODE_COUNT];
1122 1123 1124
        int best_label_rd = INT_MAX;
        B_PREDICTION_MODE mode_selected = ZERO4X4;
        int bestlabelyrate = 0;
John Koleszar's avatar
John Koleszar committed
1125

1126 1127 1128 1129 1130 1131 1132 1133 1134
        // search for the best motion vector on this segment
        for (this_mode = LEFT4X4; this_mode <= NEW4X4 ; this_mode ++)
        {
            int this_rd;
            int distortion;
            int labelyrate;
            ENTROPY_CONTEXT_PLANES t_above_s, t_left_s;
            ENTROPY_CONTEXT *ta_s;
            ENTROPY_CONTEXT *tl_s;
John Koleszar's avatar
John Koleszar committed
1135

1136 1137
            vpx_memcpy(&t_above_s, &t_above, sizeof(ENTROPY_CONTEXT_PLANES));
            vpx_memcpy(&t_left_s, &t_left, sizeof(ENTROPY_CONTEXT_PLANES));
John Koleszar's avatar
John Koleszar committed
1138

1139 1140
            ta_s = (ENTROPY_CONTEXT *)&t_above_s;
            tl_s = (ENTROPY_CONTEXT *)&t_left_s;
John Koleszar's avatar
John Koleszar committed
1141

1142
            if (this_mode == NEW4X4)
John Koleszar's avatar
John Koleszar committed
1143
            {
1144
                int sseshift;
John Koleszar's avatar
John Koleszar committed
1145
                int num00;
1146
                int step_param = 0;
1147
                int further_steps;
1148 1149 1150
                int n;
                int thissme;
                int bestsme = INT_MAX;
Scott LaVarnway's avatar
Scott LaVarnway committed