Commit 6fb6ab1b authored by Yushin Cho's avatar Yushin Cho Committed by Yushin Cho

Revise rdo_tx_size_type() for tx_size decision

Also add asserts to check the # of tx block for chroma is only one.
Later, this asserts become not valid if partition size is > 64x64
and chroma format is not 420.
parent d6e08b5f
......@@ -143,7 +143,7 @@ pub static max_txsize_rect_lookup: [TxSize; BlockSize::BLOCK_SIZES_ALL] = [
TX_16X64, TX_64X16
];
static sub_tx_size_map: [TxSize; TxSize::TX_SIZES_ALL] = [
pub static sub_tx_size_map: [TxSize; TxSize::TX_SIZES_ALL] = [
TX_4X4, // TX_4X4
TX_4X4, // TX_8X8
TX_8X8, // TX_16X16
......
......@@ -1409,6 +1409,13 @@ pub fn write_tx_blocks<T: Pixel>(
}
if bw_uv > 0 && bh_uv > 0 {
// TODO: Disable these asserts temporarilly, since chroma_sampling_422_aom and chroma_sampling_444_aom
// tests seems trigerring them as well, which should not
// TODO: Not valid if partition > 64x64 && chroma != 420
/*if xdec == 1 && ydec == 1 {
assert!(bw_uv == 1, "bw_uv = {}, bh_uv = {}", bw_uv, bh_uv);
assert!(bh_uv == 1, "bw_uv = {}, bh_uv = {}", bw_uv, bh_uv);
}*/
let uv_tx_type = if uv_tx_size.width() >= 32 || uv_tx_size.height() >= 32 {
TxType::DCT_DCT
} else {
......@@ -1487,6 +1494,13 @@ pub fn write_tx_tree<T: Pixel>(
let plane_bsize = get_plane_block_size(bsize, xdec, ydec);
if bw_uv > 0 && bh_uv > 0 {
// TODO: Disable these asserts temporarilly, since chroma_sampling_422_aom and chroma_sampling_444_aom
// tests seems trigerring them as well, which should not
// TODO: Not valid if partition > 64x64 && chroma != 420
/*if xdec == 1 && ydec == 1 {
debug_assert!(bw_uv == 1, "bw_uv = {}, bh_uv = {}", bw_uv, bh_uv);
debug_assert!(bh_uv == 1, "bw_uv = {}, bh_uv = {}", bw_uv, bh_uv);
}*/
let uv_tx_type = if has_coeff {tx_type} else {TxType::DCT_DCT}; // if inter mode, uv_tx_type == tx_type
for p in 1..3 {
......
......@@ -363,27 +363,14 @@ pub fn rdo_tx_size_type<T: Pixel>(
cw: &mut ContextWriter, bsize: BlockSize, bo: BlockOffset,
luma_mode: PredictionMode, ref_frames: [usize; 2], mvs: [MotionVector; 2], skip: bool
) -> (TxSize, TxType) {
// these rules follow TX_MODE_LARGEST
let tx_size = {
use self::BlockSize::*;
use self::TxSize::*;
match bsize {
BLOCK_4X4 => TX_4X4,
BLOCK_8X8 => TX_8X8,
BLOCK_16X16 => TX_16X16,
BLOCK_4X8 => TX_4X8,
BLOCK_8X4 => TX_8X4,
BLOCK_8X16 => TX_8X16,
BLOCK_16X8 => TX_16X8,
BLOCK_16X32 => TX_16X32,
BLOCK_32X16 => TX_32X16,
BLOCK_32X32 => TX_32X32,
BLOCK_32X64 => TX_32X64,
BLOCK_64X32 => TX_64X32,
BLOCK_64X64 => TX_64X64,
_ => unimplemented!()
}
use crate::context::max_txsize_rect_lookup;
let tx_size = if !fi.tx_mode_select || !luma_mode.is_intra() {
max_txsize_rect_lookup[bsize as usize]
} else {
sub_tx_size_map[max_txsize_rect_lookup[bsize as usize] as usize] // 1 level partition
};
debug_assert!(tx_size.width_log2() <= bsize.width_log2());
debug_assert!(tx_size.height_log2() <= bsize.height_log2());
// Luma plane transform type decision
let is_inter = !luma_mode.is_intra();
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment