Commit 0deb6377 authored by Yushin Cho's avatar Yushin Cho

Fix rec pixels not computed when tx is smaller than block

The fix applies when tx rdo is turned on.
parent bb8d5690
...@@ -930,6 +930,9 @@ pub fn encode_tx_block<T: Pixel>( ...@@ -930,6 +930,9 @@ pub fn encode_tx_block<T: Pixel>(
let area = Area::BlockStartingAt { bo: tile_bo }; let area = Area::BlockStartingAt { bo: tile_bo };
assert!(tx_size.sqr() <= TxSize::TX_32X32 || tx_type == TxType::DCT_DCT); assert!(tx_size.sqr() <= TxSize::TX_32X32 || tx_type == TxType::DCT_DCT);
debug_assert!(p != 0 || !mode.is_intra() || tx_size.block_size() == plane_bsize || !for_rdo_use,
"mode.is_intra()={:#?}, plane={:#?}, tx_size.block_size()={:#?}, plane_bsize={:#?}, for_rdo_use={:#?}",
mode.is_intra(), p, tx_size.block_size(), plane_bsize, for_rdo_use);
if mode.is_intra() { if mode.is_intra() {
let bit_depth = fi.sequence.bit_depth; let bit_depth = fi.sequence.bit_depth;
......
...@@ -608,7 +608,7 @@ pub fn rdo_mode_decision<T: Pixel>( ...@@ -608,7 +608,7 @@ pub fn rdo_mode_decision<T: Pixel>(
} }
// TODO(yushin): luma and chroma would have different decision based on chroma format // TODO(yushin): luma and chroma would have different decision based on chroma format
let needs_rec = luma_mode_is_intra && tx_size < bsize.tx_size(); let needs_rec = luma_mode_is_intra && tx_size.block_size() < bsize;
encode_block_a(&fi.sequence, ts, cw, wr, bsize, tile_bo, skip); encode_block_a(&fi.sequence, ts, cw, wr, bsize, tile_bo, skip);
let tx_dist = let tx_dist =
...@@ -974,6 +974,7 @@ pub fn rdo_tx_type_decision<T: Pixel>( ...@@ -974,6 +974,7 @@ pub fn rdo_tx_type_decision<T: Pixel>(
} else { } else {
RDOType::PixelDistRealRate RDOType::PixelDistRealRate
}; };
let for_rdo_use = tx_size.block_size() == bsize;
for &tx_type in tx_types { for &tx_type in tx_types {
// Skip unsupported transform types // Skip unsupported transform types
...@@ -989,7 +990,7 @@ pub fn rdo_tx_type_decision<T: Pixel>( ...@@ -989,7 +990,7 @@ pub fn rdo_tx_type_decision<T: Pixel>(
let tell = wr.tell_frac(); let tell = wr.tell_frac();
let tx_dist = if is_inter { let tx_dist = if is_inter {
write_tx_tree( write_tx_tree(
fi, ts, cw, wr, mode, tile_bo, bsize, tx_size, tx_type, false, true, rdo_type, true fi, ts, cw, wr, mode, tile_bo, bsize, tx_size, tx_type, false, true, rdo_type, for_rdo_use
) )
} else { } else {
write_tx_blocks( write_tx_blocks(
...@@ -1007,7 +1008,7 @@ pub fn rdo_tx_type_decision<T: Pixel>( ...@@ -1007,7 +1008,7 @@ pub fn rdo_tx_type_decision<T: Pixel>(
CFLParams::default(), // Unused. CFLParams::default(), // Unused.
true, true,
rdo_type, rdo_type,
true for_rdo_use
) )
}; };
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment