Commit dd8600f5 authored by Jingning Han's avatar Jingning Han

Support rd model in txk sel search

Make the per transform block kernel selection process unified with
the rate distortion model used in preliminary mode search. This
makes the txk-sel model search space same as baseline.

Change-Id: I82a2d94e88a03c88154582575ced500197f8a409
parent 206d22f2
...@@ -164,6 +164,8 @@ struct macroblock { ...@@ -164,6 +164,8 @@ struct macroblock {
// Save the transform RD search info. // Save the transform RD search info.
TX_RD_RECORD tx_rd_record; TX_RD_RECORD tx_rd_record;
int rd_model;
// Also save RD info on the TX size search level for square TX sizes. // Also save RD info on the TX size search level for square TX sizes.
TX_SIZE_RD_RECORD TX_SIZE_RD_RECORD
tx_size_rd_record_8X8[(MAX_MIB_SIZE >> 1) * (MAX_MIB_SIZE >> 1)]; tx_size_rd_record_8X8[(MAX_MIB_SIZE >> 1) * (MAX_MIB_SIZE >> 1)];
......
...@@ -2365,7 +2365,7 @@ int64_t av1_search_txk_type(const AV1_COMP *cpi, MACROBLOCK *x, int plane, ...@@ -2365,7 +2365,7 @@ int64_t av1_search_txk_type(const AV1_COMP *cpi, MACROBLOCK *x, int plane,
MACROBLOCKD *xd = &x->e_mbd; MACROBLOCKD *xd = &x->e_mbd;
MB_MODE_INFO *mbmi = &xd->mi[0]->mbmi; MB_MODE_INFO *mbmi = &xd->mi[0]->mbmi;
TX_TYPE txk_start = DCT_DCT; TX_TYPE txk_start = DCT_DCT;
TX_TYPE txk_end = TX_TYPES - 1; TX_TYPE txk_end = x->rd_model ? DCT_DCT : TX_TYPES - 1;
TX_TYPE best_tx_type = txk_start; TX_TYPE best_tx_type = txk_start;
int64_t best_rd = INT64_MAX; int64_t best_rd = INT64_MAX;
uint8_t best_txb_ctx = 0; uint8_t best_txb_ctx = 0;
......
...@@ -2257,8 +2257,10 @@ static int64_t estimate_yrd_for_sb(const AV1_COMP *const cpi, BLOCK_SIZE bs, ...@@ -2257,8 +2257,10 @@ static int64_t estimate_yrd_for_sb(const AV1_COMP *const cpi, BLOCK_SIZE bs,
MACROBLOCK *x, int *r, int64_t *d, int *s, MACROBLOCK *x, int *r, int64_t *d, int *s,
int64_t *sse, int64_t ref_best_rd) { int64_t *sse, int64_t ref_best_rd) {
RD_STATS rd_stats; RD_STATS rd_stats;
x->rd_model = 1;
int64_t rd = txfm_yrd(cpi, x, &rd_stats, ref_best_rd, bs, DCT_DCT, int64_t rd = txfm_yrd(cpi, x, &rd_stats, ref_best_rd, bs, DCT_DCT,
max_txsize_lookup[bs]); max_txsize_lookup[bs]);
x->rd_model = 0;
*r = rd_stats.rate; *r = rd_stats.rate;
*d = rd_stats.dist; *d = rd_stats.dist;
*s = rd_stats.skip; *s = rd_stats.skip;
...@@ -2434,7 +2436,10 @@ static void choose_tx_size_type_from_rd(const AV1_COMP *const cpi, ...@@ -2434,7 +2436,10 @@ static void choose_tx_size_type_from_rd(const AV1_COMP *const cpi,
for (tx_type = tx_start; tx_type < tx_end; ++tx_type) { for (tx_type = tx_start; tx_type < tx_end; ++tx_type) {
RD_STATS this_rd_stats; RD_STATS this_rd_stats;
if (skip_txfm_search(cpi, x, bs, tx_type, n, prune)) continue; if (skip_txfm_search(cpi, x, bs, tx_type, n, prune)) continue;
if (mbmi->ref_mv_idx > 0) x->rd_model = 1;
rd = txfm_yrd(cpi, x, &this_rd_stats, ref_best_rd, bs, tx_type, n); rd = txfm_yrd(cpi, x, &this_rd_stats, ref_best_rd, bs, tx_type, n);
x->rd_model = 0;
// Early termination in transform size search. // Early termination in transform size search.
if (cpi->sf.tx_size_search_breakout && if (cpi->sf.tx_size_search_breakout &&
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment