Commit 994dba25 authored by Yue Chen's avatar Yue Chen

Clean ups and bug-fixes for filter_intra

Clean up: clean up unused entropy related arrays
Bug-fixes: resolve compiling errors when it is enabled with
entropy_stats and ext_skip

Change-Id: I2893e1b5c09e847fb4677f6e62709464208b76f8
parent 3ebb0d00
......@@ -1485,9 +1485,8 @@ static const aom_prob default_comp_group_idx_probs[COMP_GROUP_IDX_CONTEXTS] = {
#endif // CONFIG_JNT_COMP
#if CONFIG_FILTER_INTRA
static const aom_cdf_prob default_filter_intra_mode_cdf[2][CDF_SIZE(
FILTER_INTRA_MODES)] = { { AOM_CDF5(14259, 17304, 20463, 29377) },
{ AOM_CDF5(32768, 32768, 32768, 32768) } };
static const aom_cdf_prob default_filter_intra_mode_cdf[CDF_SIZE(
FILTER_INTRA_MODES)] = { AOM_CDF5(14259, 17304, 20463, 29377) };
static const aom_cdf_prob default_filter_intra_cdfs[TX_SIZES_ALL][CDF_SIZE(2)] =
{
......
......@@ -195,7 +195,7 @@ typedef struct frame_contexts {
struct segmentation_probs seg;
#if CONFIG_FILTER_INTRA
aom_cdf_prob filter_intra_cdfs[TX_SIZES_ALL][CDF_SIZE(2)];
aom_cdf_prob filter_intra_mode_cdf[PLANE_TYPES][CDF_SIZE(FILTER_INTRA_MODES)];
aom_cdf_prob filter_intra_mode_cdf[CDF_SIZE(FILTER_INTRA_MODES)];
#endif // CONFIG_FILTER_INTRA
#if CONFIG_LOOP_RESTORATION
aom_cdf_prob switchable_restore_cdf[CDF_SIZE(RESTORE_SWITCHABLE_TYPES)];
......@@ -346,10 +346,8 @@ typedef struct FRAME_COUNTS {
#endif // CONFIG_ENTROPY_STATS
struct seg_counts seg;
#if CONFIG_FILTER_INTRA
unsigned int filter_intra_mode[PLANE_TYPES][FILTER_INTRA_MODES];
unsigned int filter_intra_mode[FILTER_INTRA_MODES];
unsigned int filter_intra_tx[TX_SIZES_ALL][2];
unsigned int filter_intra_mode_ctx[KF_MODE_CONTEXTS][KF_MODE_CONTEXTS]
[FILTER_INTRA_MODES];
#endif // CONFIG_FILTER_INTRA
} FRAME_COUNTS;
......
......@@ -1270,6 +1270,10 @@ static void filter_intra_predictor(uint8_t *dst, ptrdiff_t stride,
assert(bw <= 32 && bh <= 32);
// The initialization is just for silencing Jenkins static analysis warnings
for (r = 0; r < bh + 1; ++r)
memset(buffer[r], 0, (bw + 1) * sizeof(buffer[0][0]));
for (r = 0; r < bh; ++r) buffer[r + 1][0] = (int)left[r];
for (c = 0; c < bw + 1; ++c) buffer[0][c] = (int)above[c - 1];
......@@ -1370,6 +1374,10 @@ static void highbd_filter_intra_predictor(uint16_t *dst, ptrdiff_t stride,
assert(bw <= 32 && bh <= 32);
// The initialization is just for silencing Jenkins static analysis warnings
for (r = 0; r < bh + 1; ++r)
memset(buffer[r], 0, (bw + 1) * sizeof(buffer[0][0]));
for (r = 0; r < bh; ++r) buffer[r + 1][0] = (int)left[r];
for (c = 0; c < bw + 1; ++c) buffer[0][c] = (int)above[c - 1];
......
......@@ -875,9 +875,8 @@ static void read_filter_intra_mode_info(MACROBLOCKD *const xd, aom_reader *r) {
filter_intra_mode_info->use_filter_intra = aom_read_symbol(
r, xd->tile_ctx->filter_intra_cdfs[mbmi->tx_size], 2, ACCT_STR);
if (filter_intra_mode_info->use_filter_intra) {
filter_intra_mode_info->filter_intra_mode =
aom_read_symbol(r, xd->tile_ctx->filter_intra_mode_cdf[0],
FILTER_INTRA_MODES, ACCT_STR);
filter_intra_mode_info->filter_intra_mode = aom_read_symbol(
r, xd->tile_ctx->filter_intra_mode_cdf, FILTER_INTRA_MODES, ACCT_STR);
}
}
}
......
......@@ -818,7 +818,7 @@ static void write_filter_intra_mode_info(const MACROBLOCKD *xd,
if (mbmi->filter_intra_mode_info.use_filter_intra) {
const FILTER_INTRA_MODE mode =
mbmi->filter_intra_mode_info.filter_intra_mode;
aom_write_symbol(w, mode, xd->tile_ctx->filter_intra_mode_cdf[0],
aom_write_symbol(w, mode, xd->tile_ctx->filter_intra_mode_cdf,
FILTER_INTRA_MODES);
}
}
......
......@@ -297,7 +297,7 @@ struct macroblock {
int y_mode_costs[INTRA_MODES][INTRA_MODES][INTRA_MODES];
#if CONFIG_FILTER_INTRA
int filter_intra_cost[TX_SIZES_ALL][2];
int filter_intra_mode_cost[PLANE_TYPES][FILTER_INTRA_MODES];
int filter_intra_mode_cost[FILTER_INTRA_MODES];
#endif
int switchable_interp_costs[SWITCHABLE_FILTER_CONTEXTS][SWITCHABLE_FILTERS];
#if CONFIG_EXT_PARTITION_TYPES
......
......@@ -4156,24 +4156,15 @@ static void sum_intra_stats(FRAME_COUNTS *counts, MACROBLOCKD *xd,
const int use_filter_intra_mode =
mbmi->filter_intra_mode_info.use_filter_intra;
#if CONFIG_ENTROPY_STATS
++counts->filter_intra_tx[mbmi->tx_size][use_filter_intra_mode];
if (use_filter_intra_mode) {
const PREDICTION_MODE above = av1_above_block_mode(above_mi);
const PREDICTION_MODE left = av1_left_block_mode(left_mi);
#if CONFIG_KF_CTX
int above_ctx = intra_mode_context[above];
int left_ctx = intra_mode_context[left];
#endif
++counts->filter_intra_mode[0][mbmi->filter_intra_mode_info
.filter_intra_mode[0]];
++counts->filter_intra_mode_ctx[above_ctx][left_ctx]
[mbmi->filter_intra_mode_info
.filter_intra_mode[0]];
++counts
->filter_intra_mode[mbmi->filter_intra_mode_info.filter_intra_mode];
}
++counts->filter_intra_tx[mbmi->tx_size][use_filter_intra_mode];
#endif // CONFIG_ENTROPY_STATS
if (allow_update_cdf) {
if (use_filter_intra_mode)
update_cdf(fc->filter_intra_mode_cdf[0],
update_cdf(fc->filter_intra_mode_cdf,
mbmi->filter_intra_mode_info.filter_intra_mode,
FILTER_INTRA_MODES);
update_cdf(fc->filter_intra_cdfs[mbmi->tx_size], use_filter_intra_mode,
......
......@@ -119,8 +119,8 @@ void av1_fill_mode_rates(AV1_COMMON *const cm, MACROBLOCK *x,
NULL);
#if CONFIG_FILTER_INTRA
av1_cost_tokens_from_cdf(x->filter_intra_mode_cost[0],
fc->filter_intra_mode_cdf[0], NULL);
av1_cost_tokens_from_cdf(x->filter_intra_mode_cost, fc->filter_intra_mode_cdf,
NULL);
for (i = 0; i < TX_SIZES_ALL; ++i)
av1_cost_tokens_from_cdf(x->filter_intra_cost[i], fc->filter_intra_cdfs[i],
NULL);
......
......@@ -2759,7 +2759,7 @@ static int64_t intra_model_yrd(const AV1_COMP *const cpi, MACROBLOCK *const x,
if (mbmi->filter_intra_mode_info.use_filter_intra) {
const int mode = mbmi->filter_intra_mode_info.filter_intra_mode;
mode_cost += x->filter_intra_cost[mbmi->tx_size][1] +
x->filter_intra_mode_cost[0][mode];
x->filter_intra_mode_cost[mode];
} else {
mode_cost += x->filter_intra_cost[mbmi->tx_size][0];
}
......@@ -3071,7 +3071,7 @@ static int rd_pick_filter_intra_sby(const AV1_COMP *const cpi, MACROBLOCK *x,
if (tokenonly_rd_stats.rate == INT_MAX) continue;
this_rate = tokenonly_rd_stats.rate +
x->filter_intra_cost[mbmi->tx_size][1] +
x->filter_intra_mode_cost[0][mode] + mode_cost;
x->filter_intra_mode_cost[mode] + mode_cost;
this_rd = RDCOST(x->rdmult, this_rate, tokenonly_rd_stats.dist);
if (this_rd < *best_rd) {
......@@ -9169,8 +9169,7 @@ static void estimate_skip_mode_rdcost(
}
#if CONFIG_FILTER_INTRA
mbmi->filter_intra_mode_info.use_filter_intra_mode[0] = 0;
mbmi->filter_intra_mode_info.use_filter_intra_mode[1] = 0;
mbmi->filter_intra_mode_info.use_filter_intra = 0;
#endif // CONFIG_FILTER_INTRA
mbmi->interintra_mode = (INTERINTRA_MODE)(II_DC_PRED - 1);
#if CONFIG_JNT_COMP
......@@ -9848,10 +9847,9 @@ void av1_rd_pick_inter_mode_sb(const AV1_COMP *cpi, TileDataEnc *tile_data,
super_block_yrd(cpi, x, &rd_stats_y_fi, bsize, best_rd);
if (rd_stats_y_fi.rate == INT_MAX) continue;
this_rate_tmp = rd_stats_y_fi.rate +
x->filter_intra_cost[mbmi->tx_size][1] +
x->filter_intra_mode_cost[0][fi_mode] +
intra_mode_cost[mbmi->mode];
this_rate_tmp =
rd_stats_y_fi.rate + x->filter_intra_cost[mbmi->tx_size][1] +
x->filter_intra_mode_cost[fi_mode] + intra_mode_cost[mbmi->mode];
this_rd_tmp = RDCOST(x->rdmult, this_rate_tmp, rd_stats_y_fi.dist);
if (this_rd_tmp < best_rd_tmp) {
......@@ -9953,8 +9951,8 @@ void av1_rd_pick_inter_mode_sb(const AV1_COMP *cpi, TileDataEnc *tile_data,
x->filter_intra_cost[mbmi->tx_size]
[mbmi->filter_intra_mode_info.use_filter_intra];
if (mbmi->filter_intra_mode_info.use_filter_intra) {
rate2 += x->filter_intra_mode_cost[0][mbmi->filter_intra_mode_info
.filter_intra_mode];
rate2 += x->filter_intra_mode_cost[mbmi->filter_intra_mode_info
.filter_intra_mode];
}
}
#endif // CONFIG_FILTER_INTRA
......@@ -10714,8 +10712,7 @@ PALETTE_EXIT:
best_mbmode.interintra_mode = (INTERINTRA_MODE)(II_DC_PRED - 1);
#if CONFIG_FILTER_INTRA
best_mbmode.filter_intra_mode_info.use_filter_intra_mode[0] = 0;
best_mbmode.filter_intra_mode_info.use_filter_intra_mode[1] = 0;
best_mbmode.filter_intra_mode_info.use_filter_intra = 0;
#endif // CONFIG_FILTER_INTRA
set_default_interp_filters(&best_mbmode, cm->interp_filter);
......
......@@ -591,34 +591,19 @@ int main(int argc, const char **argv) {
"static const aom_cdf_prob default_intrabc_cdf[CDF_SIZE(2)]");
#endif
/* Transform type */
// TODO(yuec): different trees are used depending on selected ext tx set
/* supertx experiment */
/* filter_intra experiment */
#if CONFIG_FILTER_INTRA
cts_each_dim[0] = PLANE_TYPES;
cts_each_dim[1] = FILTER_INTRA_MODES;
cts_each_dim[0] = FILTER_INTRA_MODES;
optimize_cdf_table(
&fc.filter_intra_mode[0][0], probsfile, 2, cts_each_dim,
&fc.filter_intra_mode[0], probsfile, 1, cts_each_dim,
"static const aom_cdf_prob "
"default_filter_intra_mode_cdf[2][CDF_SIZE(FILTER_INTRA_MODES)]");
"default_filter_intra_mode_cdf[CDF_SIZE(FILTER_INTRA_MODES)]");
cts_each_dim[0] = TX_SIZES_ALL;
cts_each_dim[1] = 2;
optimize_cdf_table(&fc.filter_intra_tx[0][0], probsfile, 2, cts_each_dim,
"static const aom_cdf_prob "
"default_filter_intra_cdfs[TX_SIZES_ALL][CDF_SIZE(2)]");
cts_each_dim[0] = KF_MODE_CONTEXTS;
cts_each_dim[1] = KF_MODE_CONTEXTS;
cts_each_dim[2] = FILTER_INTRA_MODES;
optimize_cdf_table(&fc.filter_intra_mode_ctx[0][0][0], probsfile, 3,
cts_each_dim,
"static const aom_cdf_prob "
"default_filter_intra_mode_cdf[KF_MODE_CONTEXTS][KF_MODE_"
"CONTEXTS][CDF_SIZE(FILTER_INTRA_MODES)]");
#endif
#if CONFIG_LV_MAP
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment