Unverified Commit ecb01d41 authored by Yushin Cho's avatar Yushin Cho Committed by GitHub

Fix get_tx_set() and get_txsize_entropy_ctx() for rectangular tx_sizes (#745)

Also fix uv_tx_type decision for intra mode block.
parent d0f3a3c6
......@@ -508,11 +508,11 @@ pub fn has_chroma(
pub fn get_tx_set(
tx_size: TxSize, is_inter: bool, use_reduced_set: bool
) -> TxSet {
let tx_size_sqr_up = tx_size.sqr_up();
let tx_size_sqr = tx_size.sqr();
if tx_size_sqr > TxSize::TX_32X32 {
if tx_size.width() >= 64 || tx_size.height() >= 64 {
TxSet::TX_SET_DCTONLY
} else if tx_size_sqr_up == TxSize::TX_32X32 {
} else if tx_size_sqr == TxSize::TX_32X32 {
if is_inter {
TxSet::TX_SET_DCT_IDTX
} else {
......@@ -2992,7 +2992,7 @@ impl ContextWriter {
}
pub fn get_txsize_entropy_ctx(&mut self, tx_size: TxSize) -> usize {
(tx_size.sqr() as usize + tx_size.sqr() as usize + 1) >> 1
(tx_size.sqr() as usize + tx_size.sqr_up() as usize + 1) >> 1
}
pub fn txb_init_levels(
......
......@@ -1714,7 +1714,7 @@ pub fn write_tx_blocks(fi: &FrameInvariants, fs: &mut FrameState,
}
if bw_uv > 0 && bh_uv > 0 {
let uv_tx_type = if uv_tx_size == TxSize::TX_32X32 {
let uv_tx_type = if uv_tx_size.width() >= 32 || uv_tx_size.height() >= 32 {
TxType::DCT_DCT
} else {
uv_intra_mode_to_tx_type_context(chroma_mode)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment