Commit eeacc4c0 authored by Yue Chen's avatar Yue Chen

Bug fix: determine tx_mode based on lossless mode of all segments

When segment feature is on, frame level cm->tx_mode can be set to
ONLY_4X4 only if all segments are lossless. Otherwise will cause
bugs when xd->lossless[i] is 0 and xd->lossless[0] is 1.
Also fix the condition of coding tx_type, which should be on when
the qindex of current segment is > 0.

BUG=aomedia:106
BUG=aomedia:104

Change-Id: Ic076083bb78b3b99a6f7d17ec82ee402c64bcc52
parent f7159223
......@@ -125,10 +125,27 @@ static int decode_unsigned_max(struct aom_read_bit_buffer *rb, int max) {
return data > max ? max : data;
}
static TX_MODE read_tx_mode(struct aom_read_bit_buffer *rb) {
static TX_MODE read_tx_mode(AV1_COMMON *cm, MACROBLOCKD *xd,
struct aom_read_bit_buffer *rb) {
int i, all_lossless = 1;
#if CONFIG_TX64X64
TX_MODE tx_mode =
aom_rb_read_bit(rb) ? TX_MODE_SELECT : aom_rb_read_literal(rb, 2);
TX_MODE tx_mode;
#endif
if (cm->seg.enabled) {
for (i = 0; i < MAX_SEGMENTS; ++i) {
if (!xd->lossless[i]) {
all_lossless = 0;
break;
}
}
} else {
all_lossless = xd->lossless[0];
}
if (all_lossless) return ONLY_4X4;
#if CONFIG_TX64X64
tx_mode = aom_rb_read_bit(rb) ? TX_MODE_SELECT : aom_rb_read_literal(rb, 2);
if (tx_mode == ALLOW_32X32) tx_mode += aom_rb_read_bit(rb);
return tx_mode;
#else
......@@ -4228,8 +4245,7 @@ static size_t read_uncompressed_header(AV1Decoder *pbi,
}
setup_segmentation_dequant(cm);
cm->tx_mode =
(!cm->seg.enabled && xd->lossless[0]) ? ONLY_4X4 : read_tx_mode(rb);
cm->tx_mode = read_tx_mode(cm, xd, rb);
cm->reference_mode = read_frame_reference_mode(cm, rb);
read_tile_info(pbi, rb);
......
......@@ -700,7 +700,9 @@ static void read_tx_type(const AV1_COMMON *const cm, MACROBLOCKD *xd,
#if CONFIG_EXT_TX
const TX_SIZE square_tx_size = txsize_sqr_map[tx_size];
if (get_ext_tx_types(tx_size, mbmi->sb_type, inter_block) > 1 &&
cm->base_qindex > 0 && !mbmi->skip &&
((!cm->seg.enabled && cm->base_qindex > 0) ||
(cm->seg.enabled && xd->qindex[mbmi->segment_id] > 0)) &&
!mbmi->skip &&
#if CONFIG_SUPERTX
!supertx_enabled &&
#endif // CONFIG_SUPERTX
......@@ -731,12 +733,17 @@ static void read_tx_type(const AV1_COMMON *const cm, MACROBLOCKD *xd,
mbmi->tx_type = DCT_DCT;
}
#else
if (tx_size < TX_32X32 && cm->base_qindex > 0 && !mbmi->skip &&
if (tx_size < TX_32X32 &&
((!cm->seg.enabled && cm->base_qindex > 0) ||
(cm->seg.enabled && xd->qindex[mbmi->segment_id] > 0)) &&
!mbmi->skip &&
#if CONFIG_SUPERTX
!supertx_enabled &&
#endif // CONFIG_SUPERTX
!segfeature_active(&cm->seg, mbmi->segment_id, SEG_LVL_SKIP)) {
FRAME_COUNTS *counts = xd->counts;
if (inter_block) {
#if CONFIG_EC_MULTISYMBOL
mbmi->tx_type = av1_ext_tx_inv[aom_read_symbol(
......
......@@ -1158,7 +1158,7 @@ static void write_palette_mode_info(const AV1_COMMON *cm, const MACROBLOCKD *xd,
}
#endif // CONFIG_PALETTE
static void write_tx_type(const AV1_COMMON *const cm,
static void write_tx_type(const AV1_COMMON *const cm, const MACROBLOCKD *xd,
const MB_MODE_INFO *const mbmi,
#if CONFIG_SUPERTX
const int supertx_enabled,
......@@ -1174,7 +1174,9 @@ static void write_tx_type(const AV1_COMMON *const cm,
#if CONFIG_EXT_TX
const TX_SIZE square_tx_size = txsize_sqr_map[tx_size];
const BLOCK_SIZE bsize = mbmi->sb_type;
if (get_ext_tx_types(tx_size, bsize, is_inter) > 1 && cm->base_qindex > 0 &&
if (get_ext_tx_types(tx_size, bsize, is_inter) > 1 &&
((!cm->seg.enabled && cm->base_qindex > 0) ||
(cm->seg.enabled && xd->qindex[mbmi->segment_id] > 0)) &&
!mbmi->skip &&
#if CONFIG_SUPERTX
!supertx_enabled &&
......@@ -1199,7 +1201,10 @@ static void write_tx_type(const AV1_COMMON *const cm,
}
}
#else
if (tx_size < TX_32X32 && cm->base_qindex > 0 && !mbmi->skip &&
if (tx_size < TX_32X32 &&
((!cm->seg.enabled && cm->base_qindex > 0) ||
(cm->seg.enabled && xd->qindex[mbmi->segment_id] > 0)) &&
!mbmi->skip &&
#if CONFIG_SUPERTX
!supertx_enabled &&
#endif // CONFIG_SUPERTX
......@@ -1643,7 +1648,7 @@ static void pack_inter_mode_mvs(AV1_COMP *cpi, const MODE_INFO *mi,
#endif // CONFIG_DUAL_FILTE || CONFIG_WARPED_MOTION
}
write_tx_type(cm, mbmi,
write_tx_type(cm, xd, mbmi,
#if CONFIG_SUPERTX
supertx_enabled,
#endif
......@@ -1738,7 +1743,7 @@ static void write_mb_modes_kf(AV1_COMMON *cm, const MACROBLOCKD *xd,
if (bsize >= BLOCK_8X8) write_filter_intra_mode_info(cm, mbmi, w);
#endif // CONFIG_FILTER_INTRA
write_tx_type(cm, mbmi,
write_tx_type(cm, xd, mbmi,
#if CONFIG_SUPERTX
0,
#endif
......@@ -3442,16 +3447,33 @@ static void update_seg_probs(AV1_COMP *cpi, aom_writer *w) {
}
#endif
static void write_tx_mode(TX_MODE mode, struct aom_write_bit_buffer *wb) {
static void write_tx_mode(AV1_COMMON *cm, MACROBLOCKD *xd, TX_MODE *mode,
struct aom_write_bit_buffer *wb) {
int i, all_lossless = 1;
if (cm->seg.enabled) {
for (i = 0; i < MAX_SEGMENTS; ++i) {
if (!xd->lossless[i]) {
all_lossless = 0;
break;
}
}
} else {
all_lossless = xd->lossless[0];
}
if (all_lossless) {
*mode = ONLY_4X4;
return;
}
#if CONFIG_TX64X64
aom_wb_write_bit(wb, mode == TX_MODE_SELECT);
if (mode != TX_MODE_SELECT) {
aom_wb_write_literal(wb, AOMMIN(mode, ALLOW_32X32), 2);
if (mode >= ALLOW_32X32) aom_wb_write_bit(wb, mode == ALLOW_64X64);
aom_wb_write_bit(wb, *mode == TX_MODE_SELECT);
if (*mode != TX_MODE_SELECT) {
aom_wb_write_literal(wb, AOMMIN(*mode, ALLOW_32X32), 2);
if (*mode >= ALLOW_32X32) aom_wb_write_bit(wb, *mode == ALLOW_64X64);
}
#else
aom_wb_write_bit(wb, mode == TX_MODE_SELECT);
if (mode != TX_MODE_SELECT) aom_wb_write_literal(wb, mode, 2);
aom_wb_write_bit(wb, *mode == TX_MODE_SELECT);
if (*mode != TX_MODE_SELECT) aom_wb_write_literal(wb, *mode, 2);
#endif // CONFIG_TX64X64
}
......@@ -4243,10 +4265,7 @@ static void write_uncompressed_header(AV1_COMP *cpi,
}
#endif
if (!cm->seg.enabled && xd->lossless[0])
cm->tx_mode = ONLY_4X4;
else
write_tx_mode(cm->tx_mode, wb);
write_tx_mode(cm, xd, &cm->tx_mode, wb);
if (cpi->allow_comp_inter_inter) {
const int use_hybrid_pred = cm->reference_mode == REFERENCE_MODE_SELECT;
......
......@@ -4727,7 +4727,19 @@ static MV_REFERENCE_FRAME get_frame_type(const AV1_COMP *cpi) {
}
static TX_MODE select_tx_mode(const AV1_COMP *cpi, MACROBLOCKD *const xd) {
if (xd->lossless[0]) return ONLY_4X4;
int i, all_lossless = 1;
if (cpi->common.seg.enabled) {
for (i = 0; i < MAX_SEGMENTS; ++i) {
if (!xd->lossless[i]) {
all_lossless = 0;
break;
}
}
} else {
all_lossless = xd->lossless[0];
}
if (all_lossless) return ONLY_4X4;
if (cpi->sf.tx_size_search_method == USE_LARGESTALL)
return ALLOW_32X32 + CONFIG_TX64X64;
else if (cpi->sf.tx_size_search_method == USE_FULL_RD ||
......@@ -5002,6 +5014,7 @@ static void encode_frame_internal(AV1_COMP *cpi) {
: cm->base_qindex;
xd->lossless[i] = qindex == 0 && cm->y_dc_delta_q == 0 &&
cm->uv_dc_delta_q == 0 && cm->uv_ac_delta_q == 0;
xd->qindex[i] = qindex;
}
if (!cm->seg.enabled && xd->lossless[0]) x->optimize = 0;
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment