Commit 904cfa2d authored by Monty's avatar Monty Committed by Yushin Cho

Fix deblocking code to deal with luma TX partitioning

Subpartitioning the luma TX to sub-partition size broke the Chroma deblocking code.
This patch implements proper luma/chroma tx size determination in the deblocking code.
parent c51dc9f6
......@@ -1261,10 +1261,10 @@ pub struct Block {
pub mv: [MotionVector; 2],
pub neighbors_ref_counts: [usize; TOTAL_REFS_PER_FRAME],
pub cdef_index: u8,
pub bsize: BlockSize,
pub n4_w: usize, /* block width in the unit of mode_info */
pub n4_h: usize, /* block height in the unit of mode_info */
pub tx_w: usize, /* transform width in the unit of mode_info */
pub tx_h: usize, /* transform height in the unit of mode_info */
pub txsize: TxSize,
// The block-level deblock_deltas are left-shifted by
// fi.deblock.block_delta_shift and added to the frame-configured
// deltas
......@@ -1282,10 +1282,10 @@ impl Block {
mv: [ MotionVector::default(); 2],
neighbors_ref_counts: [0; TOTAL_REFS_PER_FRAME],
cdef_index: 0,
bsize: BLOCK_64X64,
n4_w: BLOCK_64X64.width_mi(),
n4_h: BLOCK_64X64.height_mi(),
tx_w: TX_64X64.width_mi(),
tx_h: TX_64X64.height_mi(),
txsize: TX_64X64,
deblock_deltas: [0, 0, 0, 0],
segmentation_idx: 0,
}
......@@ -1583,13 +1583,11 @@ impl BlockContext {
pub fn set_block_size(&mut self, bo: BlockOffset, bsize: BlockSize) {
let n4_w = bsize.width_mi();
let n4_h = bsize.height_mi();
self.for_each(bo, bsize, |block| { block.n4_w = n4_w; block.n4_h = n4_h } );
self.for_each(bo, bsize, |block| { block.bsize = bsize; block.n4_w = n4_w; block.n4_h = n4_h } );
}
pub fn set_tx_size(&mut self, bo: BlockOffset, bsize: BlockSize, txsize: TxSize) {
let tx_w = txsize.width_mi();
let tx_h = txsize.height_mi();
self.for_each(bo, bsize, |block| { block.tx_w = tx_w; block.tx_h = tx_h } );
pub fn set_tx_size(&mut self, bo: BlockOffset, bsize: BlockSize, tx_size: TxSize) {
self.for_each(bo, bsize, |block| { block.txsize = tx_size } );
}
pub fn get_mode(&mut self, bo: BlockOffset) -> PredictionMode {
......
......@@ -126,15 +126,19 @@ fn deblock_size<T: Pixel>(
{
0
} else {
let (tx_size, prev_tx_size) = if vertical {
(cmp::max(block.tx_w >> xdec, 1), cmp::max(prev_block.tx_w >> xdec, 1))
let (txsize, prev_txsize) = if pli==0 {
(block.txsize, prev_block.txsize)
} else {
(cmp::max(block.tx_h >> ydec, 1), cmp::max(prev_block.tx_h >> ydec, 1))
(block.bsize.largest_uv_tx_size(xdec, ydec), prev_block.bsize.largest_uv_tx_size(xdec, ydec))
};
let (tx_n, prev_tx_n) = if vertical {
(cmp::max(txsize.width_mi(), 1), cmp::max(prev_txsize.width_mi(), 1))
} else {
(cmp::max(txsize.height_mi(), 1), cmp::max(prev_txsize.height_mi(), 1))
};
cmp::min(
if pli == 0 { 14 } else { 6 },
cmp::min(tx_size, prev_tx_size) << MI_SIZE_LOG2
cmp::min(tx_n, prev_tx_n) << MI_SIZE_LOG2
)
}
}
......@@ -1023,10 +1027,11 @@ fn sse_size14<T: Pixel>(
fn filter_v_edge<T: Pixel>(
deblock: &DeblockState, bc: &BlockContext, bo: BlockOffset, p: &mut Plane<T>,
pli: usize, bd: usize
pli: usize, bd: usize, xdec: usize, ydec: usize
) {
let block = bc.at(bo);
let tx_edge = bo.x & (block.tx_w - 1) == 0;
let txsize = if pli==0 { block.txsize } else { block.bsize.largest_uv_tx_size(xdec, ydec) };
let tx_edge = bo.x >> xdec & (txsize.width_mi() - 1) == 0;
if tx_edge {
let prev_block = deblock_left(bc, bo, p);
let block_edge = bo.x & (block.n4_w - 1) == 0;
......@@ -1060,10 +1065,11 @@ fn filter_v_edge<T: Pixel>(
fn sse_v_edge<T: Pixel>(
bc: &BlockContext, bo: BlockOffset, rec_plane: &Plane<T>, src_plane: &Plane<T>,
tally: &mut [i64; MAX_LOOP_FILTER + 2], pli: usize, bd: usize
tally: &mut [i64; MAX_LOOP_FILTER + 2], pli: usize, bd: usize, xdec: usize, ydec: usize
) {
let block = bc.at(bo);
let tx_edge = bo.x & (block.tx_w - 1) == 0;
let txsize = if pli==0 { block.txsize } else { block.bsize.largest_uv_tx_size(xdec, ydec) };
let tx_edge = bo.x >> xdec & (txsize.width_mi() - 1) == 0;
if tx_edge {
let prev_block = deblock_left(bc, bo, rec_plane);
let block_edge = bo.x & (block.n4_w - 1) == 0;
......@@ -1126,10 +1132,11 @@ fn sse_v_edge<T: Pixel>(
fn filter_h_edge<T: Pixel>(
deblock: &DeblockState, bc: &BlockContext, bo: BlockOffset, p: &mut Plane<T>,
pli: usize, bd: usize
pli: usize, bd: usize, xdec: usize, ydec: usize
) {
let block = bc.at(bo);
let tx_edge = bo.y & (block.tx_h - 1) == 0;
let txsize = if pli==0 { block.txsize } else { block.bsize.largest_uv_tx_size(xdec, ydec) };
let tx_edge = bo.y >> ydec & (txsize.height_mi() - 1) == 0;
if tx_edge {
let prev_block = deblock_up(bc, bo, p);
let block_edge = bo.y & (block.n4_h - 1) == 0;
......@@ -1163,10 +1170,11 @@ fn filter_h_edge<T: Pixel>(
fn sse_h_edge<T: Pixel>(
bc: &BlockContext, bo: BlockOffset, rec_plane: &Plane<T>, src_plane: &Plane<T>,
tally: &mut [i64; MAX_LOOP_FILTER + 2], pli: usize, bd: usize
tally: &mut [i64; MAX_LOOP_FILTER + 2], pli: usize, bd: usize, xdec: usize, ydec: usize
) {
let block = bc.at(bo);
let tx_edge = bo.y & (block.tx_h - 1) == 0;
let txsize = if pli==0 { block.txsize } else { block.bsize.largest_uv_tx_size(xdec, ydec) };
let tx_edge = bo.y >> ydec & (txsize.height_mi() - 1) == 0;
if tx_edge {
let prev_block = deblock_up(bc, bo, rec_plane);
let block_edge = bo.y & (block.n4_h - 1) == 0;
......@@ -1256,7 +1264,7 @@ pub fn deblock_plane<T: Pixel>(
// edge). Unroll to avoid corner-cases.
if bc.rows > 0 {
for x in (1 << xdec..bc.cols).step_by(1 << xdec) {
filter_v_edge(deblock, bc, BlockOffset { x, y: 0 }, p, pli, bd);
filter_v_edge(deblock, bc, BlockOffset { x, y: 0 }, p, pli, bd, xdec, ydec);
}
if bc.rows > 1 << ydec {
for x in (1 << xdec..bc.cols).step_by(1 << xdec) {
......@@ -1266,7 +1274,9 @@ pub fn deblock_plane<T: Pixel>(
BlockOffset { x, y: 1 << ydec },
p,
pli,
bd
bd,
xdec,
ydec
);
}
}
......@@ -1277,19 +1287,21 @@ pub fn deblock_plane<T: Pixel>(
for y in ((2 << ydec)..bc.rows).step_by(1 << ydec) {
// Check for vertical edge at first MI block boundary on this row
if 1 << xdec < bc.cols {
filter_v_edge(deblock, bc, BlockOffset { x: 1 << xdec, y }, p, pli, bd);
filter_v_edge(deblock, bc, BlockOffset { x: 1 << xdec, y }, p, pli, bd, xdec, ydec);
}
// run the rest of the row with both vertical and horizontal edge filtering.
// Horizontal lags vertical edge by one row and two columns.
for x in (2 << xdec..bc.cols).step_by(1 << xdec) {
filter_v_edge(deblock, bc, BlockOffset { x, y }, p, pli, bd);
filter_v_edge(deblock, bc, BlockOffset { x, y }, p, pli, bd, xdec, ydec);
filter_h_edge(
deblock,
bc,
BlockOffset { x: x - (2 << xdec), y: y - (1 << ydec) },
p,
pli,
bd
bd,
xdec,
ydec
);
}
// ..and the last two horizontal edges for the row
......@@ -1300,7 +1312,9 @@ pub fn deblock_plane<T: Pixel>(
BlockOffset { x: bc.cols - (2 << xdec), y: y - (1 << ydec) },
p,
pli,
bd
bd,
xdec,
ydec
);
if bc.cols - (1 << xdec) > 0 {
filter_h_edge(
......@@ -1309,7 +1323,9 @@ pub fn deblock_plane<T: Pixel>(
BlockOffset { x: bc.cols - (1 << xdec), y: y - (1 << ydec) },
p,
pli,
bd
bd,
xdec,
ydec
);
}
}
......@@ -1324,7 +1340,9 @@ pub fn deblock_plane<T: Pixel>(
BlockOffset { x, y: bc.rows - (1 << ydec) },
p,
pli,
bd
bd,
xdec,
ydec
);
}
}
......@@ -1341,7 +1359,7 @@ fn sse_plane<T: Pixel>(
// No horizontal edge filtering along top of frame
for x in (1 << xdec..bc.cols).step_by(1 << xdec) {
sse_v_edge(bc, BlockOffset { x, y: 0 }, rec, src, v_sse, pli, bd);
sse_v_edge(bc, BlockOffset { x, y: 0 }, rec, src, v_sse, pli, bd, xdec, ydec);
}
// Unlike actual filtering, we're counting horizontal and vertical
......@@ -1349,10 +1367,10 @@ fn sse_plane<T: Pixel>(
// behind vertical.
for y in (1 << ydec..bc.rows).step_by(1 << ydec) {
// No vertical filtering along left edge of frame
sse_h_edge(bc, BlockOffset { x: 0, y }, rec, src, h_sse, pli, bd);
sse_h_edge(bc, BlockOffset { x: 0, y }, rec, src, h_sse, pli, bd, xdec, ydec);
for x in (1 << xdec..bc.cols).step_by(1 << xdec) {
sse_v_edge(bc, BlockOffset { x, y }, rec, src, v_sse, pli, bd);
sse_h_edge(bc, BlockOffset { x, y }, rec, src, h_sse, pli, bd);
sse_v_edge(bc, BlockOffset { x, y }, rec, src, v_sse, pli, bd, xdec, ydec);
sse_h_edge(bc, BlockOffset { x, y }, rec, src, h_sse, pli, bd, xdec, ydec);
}
}
}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment