Unverified Commit 76357db0 authored by fbossen's avatar fbossen Committed by GitHub

Add support for multiple reference frames (#567)

* Last two frames are used as references and are labeled as LAST
and ALTREF.

* Add rules to include MV with different reference index in MV stack

* Make 2nd reference frame more distant from current frame

* Use correct reference frame in chroma motion compensation

* Enable multiple reference frames only at slower speed settings

* Add unequal quantizer assignment to temporally predicted frames
parent b0743cd7
......@@ -9,7 +9,7 @@
use encoder::*;
use context::CDFContext;
use partition::LAST_FRAME;
use partition::*;
use std::collections::VecDeque;
use std::fmt;
......@@ -122,10 +122,12 @@ impl Context {
FrameType::INTER
};
let slot_idx = frame_number_in_segment % REF_FRAMES as u64;
self.fi.refresh_frame_flags = if self.fi.frame_type == FrameType::KEY {
ALL_REF_FRAMES_MASK
} else {
1
1 << slot_idx
};
self.fi.intra_only = self.fi.frame_type == FrameType::KEY
......@@ -133,19 +135,47 @@ impl Context {
// self.fi.use_prev_frame_mvs =
// !(self.fi.intra_only || self.fi.error_resilient);
let use_multiple_ref_frames = self.fi.config.speed <= 2;
let log_boost_frequency = if use_multiple_ref_frames {
2 // Higher quality frame every 4 frames
} else {
0 // No boosting with single reference frame
};
assert!(log_boost_frequency >= 0 && log_boost_frequency <= 2);
let boost_frequency = 1 << log_boost_frequency;
self.fi.base_q_idx = if self.fi.frame_type == FrameType::KEY {
let q_boost = 15;
self.fi.config.quantizer.max(1 + q_boost).min(255 + q_boost) - q_boost
} else {
} else if slot_idx & (boost_frequency - 1) == 0 {
self.fi.config.quantizer.max(1).min(255)
} else {
let q_drop = 15;
self.fi.config.quantizer.min(255 - q_drop) + q_drop
} as u8;
let first_ref_frame = LAST_FRAME;
let second_ref_frame = if use_multiple_ref_frames {
ALTREF_FRAME
} else {
NONE_FRAME
};
self.fi.primary_ref_frame = if self.fi.intra_only || self.fi.error_resilient {
PRIMARY_REF_NONE
} else {
(LAST_FRAME - LAST_FRAME) as u32
(first_ref_frame - LAST_FRAME) as u32
};
for i in 0..INTER_REFS_PER_FRAME {
self.fi.ref_frames[i] = if i == second_ref_frame - LAST_FRAME {
(REF_FRAMES + slot_idx as usize - 2) & boost_frequency as usize
} else {
(REF_FRAMES + slot_idx as usize - 1) & (REF_FRAMES - 1)
};
}
let data = encode_frame(&mut self.seq, &mut self.fi, &mut fs);
let number = self.fi.number as usize;
......
......@@ -1928,7 +1928,16 @@ impl ContextWriter {
cmp::max(col_offset, -(mi_col as isize))
}
fn find_matching_mv(&self, blk: &Block, mv_stack: &mut Vec<CandidateMV>, weight: u32) -> bool {
fn find_matching_mv(&self, blk: &Block, mv_stack: &mut Vec<CandidateMV>) -> bool {
for mv_cand in mv_stack {
if blk.mv[0].row == mv_cand.this_mv.row && blk.mv[0].col == mv_cand.this_mv.col {
return true;
}
}
false
}
fn find_matching_mv_and_update_weight(&self, blk: &Block, mv_stack: &mut Vec<CandidateMV>, weight: u32) -> bool {
for mut mv_cand in mv_stack {
if blk.mv[0].row == mv_cand.this_mv.row && blk.mv[0].col == mv_cand.this_mv.col {
mv_cand.weight += weight;
......@@ -1945,7 +1954,7 @@ impl ContextWriter {
}
if blk.ref_frames[0] == ref_frame {
let found_match = self.find_matching_mv(blk, mv_stack, weight);
let found_match = self.find_matching_mv_and_update_weight(blk, mv_stack, weight);
if !found_match && mv_stack.len() < MAX_REF_MV_STACK_SIZE {
let mv_cand = CandidateMV {
......@@ -1967,6 +1976,21 @@ impl ContextWriter {
}
}
fn add_extra_mv_candidate(&self, blk: &Block, mv_stack: &mut Vec<CandidateMV>) {
for cand_list in 0..2 {
if blk.ref_frames[cand_list] > INTRA_FRAME {
if !self.find_matching_mv(blk, mv_stack) {
let mv_cand = CandidateMV {
this_mv: blk.mv[0],
comp_mv: blk.mv[1],
weight: 2
};
mv_stack.push(mv_cand);
}
}
}
}
fn scan_row_mbmi(&mut self, bo: &BlockOffset, row_offset: isize, max_row_offs: isize,
processed_rows: &mut isize, ref_frame: usize,
mv_stack: &mut Vec<CandidateMV>, newmv_count: &mut usize, bsize: BlockSize) -> bool {
......@@ -2193,9 +2217,39 @@ impl ContextWriter {
/* TODO: Find nearest match and assign nearest and near mvs */
// Sort MV stack according to weight
// 7.10.2.11 Sort MV stack according to weight
mv_stack.sort_by(|a, b| b.weight.cmp(&a.weight));
if mv_stack.len() < 2 {
// 7.10.2.12 Extra search process
let w4 = bsize.width_mi().min(16).min(self.bc.cols - bo.x);
let h4 = bsize.height_mi().min(16).min(self.bc.rows - bo.y);
let num4x4 = w4.min(h4);
let passes = if up_avail { 0 } else { 1 } .. if left_avail { 2 } else { 1 };
for pass in passes {
let mut idx = 0;
while idx < num4x4 && mv_stack.len() < 2 {
let rbo = if pass == 0 {
bo.with_offset(idx as isize, -1)
} else {
bo.with_offset(-1, idx as isize)
};
let blk = &self.bc.at(&rbo);
self.add_extra_mv_candidate(blk, mv_stack);
idx += if pass == 0 {
blk.n4_w
} else {
blk.n4_h
};
}
}
}
/* TODO: Handle single reference frame extension */
// clamp mvs
......@@ -2231,6 +2285,10 @@ impl ContextWriter {
/* TODO: Set the zeromv ref to 0 */
}
if ref_frame <= INTRA_FRAME {
return 0;
}
let mode_context = self.setup_mvref_list(bo, ref_frame, mv_stack, bsize, is_sec_rect);
mode_context
}
......@@ -2360,7 +2418,6 @@ impl ContextWriter {
pub fn write_ref_frames(&mut self, w: &mut dyn Writer, bo: &BlockOffset) {
let rf = self.bc.at(bo).ref_frames;
assert!(rf[0] == LAST_FRAME);
/* TODO: Handle multiple references */
......
......@@ -856,7 +856,7 @@ impl<'a> UncompressedHeader for BitWriter<'a, BE> {
}
}
for i in 0..7 {
for i in 0..INTER_REFS_PER_FRAME {
if !frame_refs_short_signaling {
self.write(REF_FRAMES_LOG2 as u32, fi.ref_frames[i] as u8)?;
}
......@@ -1301,14 +1301,17 @@ pub fn motion_compensate(fi: &FrameInvariants, fs: &mut FrameState, cw: &mut Con
assert!(xdec == 1 && ydec == 1);
// TODO: these are only valid for 4:2:0
let mv0 = &cw.bc.at(&bo.with_offset(-1,-1)).mv[0];
let rf0 = cw.bc.at(&bo.with_offset(-1,-1)).ref_frames[0];
let mv1 = &cw.bc.at(&bo.with_offset(0,-1)).mv[0];
let rf1 = cw.bc.at(&bo.with_offset(0,-1)).ref_frames[0];
let po1 = PlaneOffset { x: po.x+2, y: po.y };
let mv2 = &cw.bc.at(&bo.with_offset(-1,0)).mv[0];
let rf2 = cw.bc.at(&bo.with_offset(-1,0)).ref_frames[0];
let po2 = PlaneOffset { x: po.x, y: po.y+2 };
let po3 = PlaneOffset { x: po.x+2, y: po.y+2 };
luma_mode.predict_inter(fi, p, &po, &mut rec.mut_slice(&po), 2, 2, ref_frame, mv0, bit_depth);
luma_mode.predict_inter(fi, p, &po1, &mut rec.mut_slice(&po1), 2, 2, ref_frame, mv1, bit_depth);
luma_mode.predict_inter(fi, p, &po2, &mut rec.mut_slice(&po2), 2, 2, ref_frame, mv2, bit_depth);
luma_mode.predict_inter(fi, p, &po, &mut rec.mut_slice(&po), 2, 2, rf0, mv0, bit_depth);
luma_mode.predict_inter(fi, p, &po1, &mut rec.mut_slice(&po1), 2, 2, rf1, mv1, bit_depth);
luma_mode.predict_inter(fi, p, &po2, &mut rec.mut_slice(&po2), 2, 2, rf2, mv2, bit_depth);
luma_mode.predict_inter(fi, p, &po3, &mut rec.mut_slice(&po3), 2, 2, ref_frame, &mv, bit_depth);
}
} else {
......@@ -1349,6 +1352,9 @@ pub fn encode_block_b(seq: &Sequence, fi: &FrameInvariants, fs: &mut FrameState,
}
cw.bc.set_block_size(bo, bsize);
cw.bc.set_mode(bo, bsize, luma_mode);
cw.bc.set_ref_frame(bo, bsize, ref_frame);
cw.bc.set_motion_vector(bo, bsize, mv);
//write_q_deltas();
if cw.bc.code_deltas && fs.deblock.block_deltas_enabled && (bsize < sb_size || !skip) {
cw.write_block_deblock_deltas(w, bo, fs.deblock.block_delta_multi);
......@@ -1359,8 +1365,6 @@ pub fn encode_block_b(seq: &Sequence, fi: &FrameInvariants, fs: &mut FrameState,
cw.write_is_inter(w, bo, is_inter);
if is_inter {
cw.fill_neighbours_ref_counts(bo);
cw.bc.set_ref_frame(bo, bsize, ref_frame);
cw.bc.set_motion_vector(bo, bsize, mv);
cw.write_ref_frames(w, bo);
//let mode_context = if bo.x == 0 && bo.y == 0 { 0 } else if bo.x ==0 || bo.y == 0 { 51 } else { 85 };
......
......@@ -14,7 +14,7 @@ use self::BlockSize::*;
use self::TxSize::*;
use encoder::FrameInvariants;
pub const NONE_FRAME: isize = -1;
pub const NONE_FRAME: usize = 8;
pub const INTRA_FRAME: usize = 0;
pub const LAST_FRAME: usize = 1;
pub const LAST2_FRAME: usize = 2;
......@@ -911,7 +911,6 @@ impl PredictionMode {
ref_frame: usize, mv: &MotionVector, bit_depth: usize
) {
assert!(!self.is_intra());
assert!(ref_frame == LAST_FRAME);
match fi.rec_buffer.frames[fi.ref_frames[ref_frame - LAST_FRAME]] {
Some(ref rec) => {
......
......@@ -302,25 +302,46 @@ pub fn rdo_mode_decision(
RAV1E_INTRA_MODES_MINIMAL
};
let mut mode_set: Vec<PredictionMode> = Vec::new();
let mut mv_stack = Vec::new();
let mode_context =
cw.find_mvrefs(bo, LAST_FRAME, &mut mv_stack, bsize, false);
let mut ref_frame_set = Vec::new();
let mut ref_slot_set = Vec::new();
if fi.frame_type == FrameType::INTER {
mode_set.extend_from_slice(RAV1E_INTER_MODES_MINIMAL);
if fi.config.speed <= 2 {
if mv_stack.len() >= 3 {
mode_set.push(PredictionMode::NEAR1MV);
for i in LAST_FRAME..NONE_FRAME {
if !ref_slot_set.contains(&fi.ref_frames[i - LAST_FRAME]) {
ref_frame_set.push(i);
ref_slot_set.push(fi.ref_frames[i - LAST_FRAME]);
}
if mv_stack.len() >= 4 {
mode_set.push(PredictionMode::NEAR2MV);
}
assert!(ref_frame_set.len() != 0);
}
let mut mode_set: Vec<(PredictionMode, usize)> = Vec::new();
let mut mv_stacks = Vec::new();
let mut mode_contexts = Vec::new();
for (i, &ref_frame) in ref_frame_set.iter().enumerate() {
let mut mvs: Vec<CandidateMV> = Vec::new();
mode_contexts.push(cw.find_mvrefs(bo, ref_frame, &mut mvs, bsize, false));
if fi.frame_type == FrameType::INTER {
for &x in RAV1E_INTER_MODES_MINIMAL {
mode_set.push((x, i));
}
if fi.config.speed <= 2 {
if mvs.len() >= 3 {
mode_set.push((PredictionMode::NEAR1MV, i));
}
if mvs.len() >= 4 {
mode_set.push((PredictionMode::NEAR2MV, i));
}
}
}
mv_stacks.push(mvs);
}
let luma_rdo = |luma_mode: PredictionMode, fs: &mut FrameState, cw: &mut ContextWriter, best: &mut EncodingSettings,
mv: MotionVector, ref_frame: usize, mode_set_chroma: &[PredictionMode], luma_mode_is_intra: bool| {
mv: MotionVector, ref_frame: usize, mode_set_chroma: &[PredictionMode], luma_mode_is_intra: bool,
mode_context: usize, mv_stack: &Vec<CandidateMV>| {
let (tx_size, tx_type) = rdo_tx_size_type(
seq, fi, fs, cw, bsize, bo, luma_mode, ref_frame, mv, false,
);
......@@ -350,7 +371,7 @@ pub fn rdo_mode_decision(
tx_size,
tx_type,
mode_context,
&mv_stack
mv_stack
);
let cost = wr.tell_frac() - tell;
......@@ -388,28 +409,33 @@ pub fn rdo_mode_decision(
};
};
mode_set.iter().for_each(|&luma_mode| {
let ref_frame = LAST_FRAME;
if fi.frame_type != FrameType::INTER {
assert!(mode_set.len() == 0);
}
mode_set.iter().for_each(|&(luma_mode, i)| {
let mv = match luma_mode {
PredictionMode::NEWMV => motion_estimation(fi, fs, bsize, bo, ref_frame, pmv),
PredictionMode::NEARESTMV => if mv_stack.len() > 0 {
mv_stack[0].this_mv
PredictionMode::NEWMV => motion_estimation(fi, fs, bsize, bo, ref_frame_set[i], pmv),
PredictionMode::NEARESTMV => if mv_stacks[i].len() > 0 {
mv_stacks[i][0].this_mv
} else {
MotionVector { row: 0, col: 0 }
},
PredictionMode::NEAR0MV => if mv_stack.len() > 1 {
mv_stack[1].this_mv
PredictionMode::NEAR0MV => if mv_stacks[i].len() > 1 {
mv_stacks[i][1].this_mv
} else {
MotionVector { row: 0, col: 0 }
},
PredictionMode::NEAR1MV | PredictionMode::NEAR2MV =>
mv_stack[luma_mode as usize - PredictionMode::NEAR0MV as usize + 1].this_mv,
mv_stacks[i][luma_mode as usize - PredictionMode::NEAR0MV as usize + 1].this_mv,
_ => MotionVector { row: 0, col: 0 }
};
let mode_set_chroma = vec![luma_mode];
luma_rdo(luma_mode, fs, cw, &mut best, mv, ref_frame, &mode_set_chroma, false);
luma_rdo(luma_mode, fs, cw, &mut best, mv, ref_frame_set[i], &mode_set_chroma, false,
mode_contexts[i], &mv_stacks[i]);
});
if !best.skip {
intra_mode_set.iter().for_each(|&luma_mode| {
let mv = MotionVector { row: 0, col: 0 };
......@@ -417,7 +443,8 @@ pub fn rdo_mode_decision(
if is_chroma_block && luma_mode != PredictionMode::DC_PRED {
mode_set_chroma.push(PredictionMode::DC_PRED);
}
luma_rdo(luma_mode, fs, cw, &mut best, mv, INTRA_FRAME, &mode_set_chroma, true);
luma_rdo(luma_mode, fs, cw, &mut best, mv, INTRA_FRAME, &mode_set_chroma, true,
0, &Vec::new());
});
}
......@@ -464,8 +491,8 @@ pub fn rdo_mode_decision(
cfl,
best.tx_size,
best.tx_type,
mode_context,
&mv_stack
0,
&Vec::new()
);
let cost = wr.tell_frac() - tell;
......@@ -492,6 +519,7 @@ pub fn rdo_mode_decision(
}
cw.bc.set_mode(bo, bsize, best.mode_luma);
cw.bc.set_ref_frame(bo, bsize, best.ref_frame);
cw.bc.set_motion_vector(bo, bsize, best.mv);
assert!(best.rd >= 0_f64);
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment