Commit bb61327b by Paul Wilkins Committed by Gerrit Code Review

### Merge "vp9_rdopt.c: Use int64_t for dist_sum and rate_sum in model_rd_for_sb()."

parents f6ab614b a0e49557
 ... @@ -431,7 +431,9 @@ static void model_rd_for_sb(VP9_COMP *cpi, BLOCK_SIZE bsize, ... @@ -431,7 +431,9 @@ static void model_rd_for_sb(VP9_COMP *cpi, BLOCK_SIZE bsize, // Note our transform coeffs are 8 times an orthogonal transform. // Note our transform coeffs are 8 times an orthogonal transform. // Hence quantizer step is also 8 times. To get effective quantizer // Hence quantizer step is also 8 times. To get effective quantizer // we need to divide by 8 before sending to modeling function. // we need to divide by 8 before sending to modeling function. int i, rate_sum = 0, dist_sum = 0; int i; int64_t rate_sum = 0; int64_t dist_sum = 0; int ref = xd->mi_8x8[0]->mbmi.ref_frame[0]; int ref = xd->mi_8x8[0]->mbmi.ref_frame[0]; unsigned int sse; unsigned int sse; ... @@ -448,13 +450,13 @@ static void model_rd_for_sb(VP9_COMP *cpi, BLOCK_SIZE bsize, ... @@ -448,13 +450,13 @@ static void model_rd_for_sb(VP9_COMP *cpi, BLOCK_SIZE bsize, // Fast approximate the modelling function. // Fast approximate the modelling function. if (cpi->speed > 4) { if (cpi->speed > 4) { int rate; int64_t rate; int64_t dist; int64_t dist; int64_t square_error = sse; int64_t square_error = sse; int quantizer = (pd->dequant[1] >> 3); int quantizer = (pd->dequant[1] >> 3); if ( quantizer < 120) if (quantizer < 120) rate = (square_error * (280-quantizer) )>> 8; rate = (square_error * (280 - quantizer)) >> 8; else else rate = 0; rate = 0; dist = (square_error * quantizer) >> 8; dist = (square_error * quantizer) >> 8; ... @@ -466,12 +468,12 @@ static void model_rd_for_sb(VP9_COMP *cpi, BLOCK_SIZE bsize, ... @@ -466,12 +468,12 @@ static void model_rd_for_sb(VP9_COMP *cpi, BLOCK_SIZE bsize, model_rd_from_var_lapndz(sse, 1 << num_pels_log2_lookup[bs], model_rd_from_var_lapndz(sse, 1 << num_pels_log2_lookup[bs], pd->dequant[1] >> 3, &rate, &dist); pd->dequant[1] >> 3, &rate, &dist); rate_sum += rate; rate_sum += rate; dist_sum += (int)dist; dist_sum += dist; } } } } *out_rate_sum = rate_sum; *out_rate_sum = (int)rate_sum; *out_dist_sum = (int64_t)dist_sum << 4; *out_dist_sum = dist_sum << 4; } } static void model_rd_for_sb_y_tx(VP9_COMP *cpi, BLOCK_SIZE bsize, static void model_rd_for_sb_y_tx(VP9_COMP *cpi, BLOCK_SIZE bsize, ... ...
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!