Commit 92245c87 authored by Yaowu Xu's avatar Yaowu Xu

Avoid out of bound array access

Change-Id: I4066561b769cf2bd4af515c9d351f609c08e3076
parent a7d6f442
......@@ -4898,7 +4898,11 @@ static void select_tx_type_yrd(const AV1_COMP *cpi, MACROBLOCK *x,
if (this_rd_stats.rate != INT_MAX) {
const int ext_tx_set = get_ext_tx_set(
mbmi->min_tx_size, bsize, is_inter, cm->reduced_tx_set_used);
const TX_SIZE square_tx_size = txsize_sqr_map[mbmi->min_tx_size];
TX_SIZE square_tx_size = txsize_sqr_map[mbmi->min_tx_size];
#if CONFIG_TX64X64
// For TX_64X64, use TX_32X32 to lookup cost
if (square_tx_size == TX_64X64) square_tx_size = TX_32X32;
#endif // CONFIG_TX64X64
this_rd_stats.rate +=
x->inter_tx_type_costs[ext_tx_set][square_tx_size][mbmi->tx_type];
this_rd_stats.rate -=
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment