Commit 81d16e3f authored by Yaowu Xu's avatar Yaowu Xu
Browse files

fixed an issue with 8x8 token cost in trellisquant

changed the token cost for 8x8 transformed macroblock used in trellisquant
from those derived from 4x4 transform coefficient distribution to those
derived from 8x8 transform coefficient distribution. Test results show
this fix help 8x8 transform based compression consistently on cif and hd
sets:

http://www.corp.google.com/~yaowu/no_crawl/t8x8/cif_cost8x8only.html
(avg psnr:.14% glb psnr: .17% ssim: .20%)
http://www.corp.google.com/~yaowu/no_crawl/t8x8/hd_cost8x8only.html
(avg psnr:.17% glb psnr: .18% ssim: .58%)

Note: To test the effect of this change, 8x8 transform was forced to be used
only on 16x16 predicted macroblocks on inter frames, the effect would be
bigger had all macroblocks been forcd to use 8x8 transform.

Change-Id: If9b7868b75357c66541f511e5ee78e4d2d4929a4
parent aebb16bf
...@@ -118,7 +118,14 @@ typedef struct ...@@ -118,7 +118,14 @@ typedef struct
unsigned char *active_ptr; unsigned char *active_ptr;
MV_CONTEXT *mvc; MV_CONTEXT *mvc;
unsigned int token_costs[BLOCK_TYPES] [COEF_BANDS] [PREV_COEF_CONTEXTS] [MAX_ENTROPY_TOKENS]; unsigned int token_costs[BLOCK_TYPES] [COEF_BANDS]
[PREV_COEF_CONTEXTS][MAX_ENTROPY_TOKENS];
#if CONFIG_T8X8
unsigned int token_costs_8x8[BLOCK_TYPES] [COEF_BANDS]
[PREV_COEF_CONTEXTS] [MAX_ENTROPY_TOKENS];
#endif
int optimize; int optimize;
int q_index; int q_index;
......
...@@ -907,9 +907,9 @@ void optimize_b_8x8(MACROBLOCK *mb, int i, int type, ...@@ -907,9 +907,9 @@ void optimize_b_8x8(MACROBLOCK *mb, int i, int type,
band = vp8_coef_bands_8x8[i + 1]; band = vp8_coef_bands_8x8[i + 1];
pt = vp8_prev_token_class[t0]; pt = vp8_prev_token_class[t0];
rate0 += rate0 +=
mb->token_costs[type][band][pt][tokens[next][0].token]; mb->token_costs_8x8[type][band][pt][tokens[next][0].token];
rate1 += rate1 +=
mb->token_costs[type][band][pt][tokens[next][1].token]; mb->token_costs_8x8[type][band][pt][tokens[next][1].token];
} }
rd_cost0 = RDCOST_8x8(rdmult, rddiv, rate0, error0); rd_cost0 = RDCOST_8x8(rdmult, rddiv, rate0, error0);
rd_cost1 = RDCOST_8x8(rdmult, rddiv, rate1, error1); rd_cost1 = RDCOST_8x8(rdmult, rddiv, rate1, error1);
...@@ -966,13 +966,13 @@ void optimize_b_8x8(MACROBLOCK *mb, int i, int type, ...@@ -966,13 +966,13 @@ void optimize_b_8x8(MACROBLOCK *mb, int i, int type,
if(t0!=DCT_EOB_TOKEN) if(t0!=DCT_EOB_TOKEN)
{ {
pt = vp8_prev_token_class[t0]; pt = vp8_prev_token_class[t0];
rate0 += mb->token_costs[type][band][pt][ rate0 += mb->token_costs_8x8[type][band][pt][
tokens[next][0].token]; tokens[next][0].token];
} }
if(t1!=DCT_EOB_TOKEN) if(t1!=DCT_EOB_TOKEN)
{ {
pt = vp8_prev_token_class[t1]; pt = vp8_prev_token_class[t1];
rate1 += mb->token_costs[type][band][pt][ rate1 += mb->token_costs_8x8[type][band][pt][
tokens[next][1].token]; tokens[next][1].token];
} }
} }
...@@ -1013,12 +1013,12 @@ void optimize_b_8x8(MACROBLOCK *mb, int i, int type, ...@@ -1013,12 +1013,12 @@ void optimize_b_8x8(MACROBLOCK *mb, int i, int type,
/* Update the cost of each path if we're past the EOB token. */ /* Update the cost of each path if we're past the EOB token. */
if (t0 != DCT_EOB_TOKEN) if (t0 != DCT_EOB_TOKEN)
{ {
tokens[next][0].rate += mb->token_costs[type][band][0][t0]; tokens[next][0].rate += mb->token_costs_8x8[type][band][0][t0];
tokens[next][0].token = ZERO_TOKEN; tokens[next][0].token = ZERO_TOKEN;
} }
if (t1 != DCT_EOB_TOKEN) if (t1 != DCT_EOB_TOKEN)
{ {
tokens[next][1].rate += mb->token_costs[type][band][0][t1]; tokens[next][1].rate += mb->token_costs_8x8[type][band][0][t1];
tokens[next][1].token = ZERO_TOKEN; tokens[next][1].token = ZERO_TOKEN;
} }
/* Don't update next, because we didn't add a new node. */ /* Don't update next, because we didn't add a new node. */
...@@ -1034,8 +1034,8 @@ void optimize_b_8x8(MACROBLOCK *mb, int i, int type, ...@@ -1034,8 +1034,8 @@ void optimize_b_8x8(MACROBLOCK *mb, int i, int type,
error1 = tokens[next][1].error; error1 = tokens[next][1].error;
t0 = tokens[next][0].token; t0 = tokens[next][0].token;
t1 = tokens[next][1].token; t1 = tokens[next][1].token;
rate0 += mb->token_costs[type][band][pt][t0]; rate0 += mb->token_costs_8x8[type][band][pt][t0];
rate1 += mb->token_costs[type][band][pt][t1]; rate1 += mb->token_costs_8x8[type][band][pt][t1];
rd_cost0 = RDCOST_8x8(rdmult, rddiv, rate0, error0); rd_cost0 = RDCOST_8x8(rdmult, rddiv, rate0, error0);
rd_cost1 = RDCOST_8x8(rdmult, rddiv, rate1, error1); rd_cost1 = RDCOST_8x8(rdmult, rddiv, rate1, error1);
if (rd_cost0 == rd_cost1) if (rd_cost0 == rd_cost1)
......
...@@ -215,7 +215,6 @@ static void fill_token_costs( ...@@ -215,7 +215,6 @@ static void fill_token_costs(
for (i = 0; i < BLOCK_TYPES; i++) for (i = 0; i < BLOCK_TYPES; i++)
for (j = 0; j < COEF_BANDS; j++) for (j = 0; j < COEF_BANDS; j++)
for (k = 0; k < PREV_COEF_CONTEXTS; k++) for (k = 0; k < PREV_COEF_CONTEXTS; k++)
vp8_cost_tokens((int *)(c [i][j][k]), p [i][j][k], vp8_coef_tree); vp8_cost_tokens((int *)(c [i][j][k]), p [i][j][k], vp8_coef_tree);
} }
...@@ -363,6 +362,12 @@ void vp8_initialize_rd_consts(VP8_COMP *cpi, int QIndex) ...@@ -363,6 +362,12 @@ void vp8_initialize_rd_consts(VP8_COMP *cpi, int QIndex)
(const vp8_prob( *)[8][3][11]) cpi->common.fc.coef_probs (const vp8_prob( *)[8][3][11]) cpi->common.fc.coef_probs
); );
#if CONFIG_T8X8
fill_token_costs(
cpi->mb.token_costs_8x8,
(const vp8_prob( *)[8][3][11]) cpi->common.fc.coef_probs_8x8
);
#endif
#if CONFIG_QIMODE #if CONFIG_QIMODE
//rough estimate for costing //rough estimate for costing
cpi->common.kf_ymode_probs_index = cpi->common.base_qindex>>4; cpi->common.kf_ymode_probs_index = cpi->common.base_qindex>>4;
...@@ -666,6 +671,7 @@ static void macro_block_yrd( MACROBLOCK *mb, ...@@ -666,6 +671,7 @@ static void macro_block_yrd( MACROBLOCK *mb,
*Rate = vp8_rdcost_mby(mb); *Rate = vp8_rdcost_mby(mb);
} }
static void copy_predictor(unsigned char *dst, const unsigned char *predictor) static void copy_predictor(unsigned char *dst, const unsigned char *predictor)
{ {
const unsigned int *p = (const unsigned int *)predictor; const unsigned int *p = (const unsigned int *)predictor;
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment