From 1c1161f1956e7c5e0fd53b00e4fb939f69242ddf Mon Sep 17 00:00:00 2001 From: Steinar Midtskogen Date: Fri, 8 Sep 2017 15:03:51 +0200 Subject: [PATCH] CDEF: Do not filter chroma if subsampling_x != subsampling_y Since CDEF looks uses the luma direction for chroma, CDEF would have to change significantly to support formats like 4:2:2. The limited use of such formats does not justify the complexity to support this, so the simple solution is to mandate that the chroma planes aren't filtered if subsampling_x != subsampling_y. Most of the visual gain is in luma, anyway. This also means that the chroma strengths and chroma skip condition shall not be sent if subsampling_x != subsampling_y. BUG=aomedia:720 Change-Id: I35c184a6fe0908ae0fee1e74494b6904fa9a3c82 --- av1/common/cdef.c | 1 + av1/decoder/decodeframe.c | 4 +++- av1/encoder/bitstream.c | 3 ++- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/av1/common/cdef.c b/av1/common/cdef.c index dab070781..397a14845 100644 --- a/av1/common/cdef.c +++ b/av1/common/cdef.c @@ -189,6 +189,7 @@ void av1_cdef_frame(YV12_BUFFER_CONFIG *frame, AV1_COMMON *cm, ydec[pli] = xd->plane[pli].subsampling_y; mi_wide_l2[pli] = MI_SIZE_LOG2 - xd->plane[pli].subsampling_x; mi_high_l2[pli] = MI_SIZE_LOG2 - xd->plane[pli].subsampling_y; + if (xdec[pli] != ydec[pli]) nplanes = 1; } stride = (cm->mi_cols << MI_SIZE_LOG2) + 2 * CDEF_HBORDER; for (pli = 0; pli < nplanes; pli++) { diff --git a/av1/decoder/decodeframe.c b/av1/decoder/decodeframe.c index dd18053a9..026866f69 100644 --- a/av1/decoder/decodeframe.c +++ b/av1/decoder/decodeframe.c @@ -2857,7 +2857,9 @@ static void setup_cdef(AV1_COMMON *cm, struct aom_read_bit_buffer *rb) { cm->nb_cdef_strengths = 1 << cm->cdef_bits; for (i = 0; i < cm->nb_cdef_strengths; i++) { cm->cdef_strengths[i] = aom_rb_read_literal(rb, CDEF_STRENGTH_BITS); - cm->cdef_uv_strengths[i] = aom_rb_read_literal(rb, CDEF_STRENGTH_BITS); + cm->cdef_uv_strengths[i] = cm->subsampling_x == cm->subsampling_y + ? aom_rb_read_literal(rb, CDEF_STRENGTH_BITS) + : 0; } } #endif // CONFIG_CDEF diff --git a/av1/encoder/bitstream.c b/av1/encoder/bitstream.c index 21e3c190b..cfa6d0704 100644 --- a/av1/encoder/bitstream.c +++ b/av1/encoder/bitstream.c @@ -3407,7 +3407,8 @@ static void encode_cdef(const AV1_COMMON *cm, struct aom_write_bit_buffer *wb) { aom_wb_write_literal(wb, cm->cdef_bits, 2); for (i = 0; i < cm->nb_cdef_strengths; i++) { aom_wb_write_literal(wb, cm->cdef_strengths[i], CDEF_STRENGTH_BITS); - aom_wb_write_literal(wb, cm->cdef_uv_strengths[i], CDEF_STRENGTH_BITS); + if (cm->subsampling_x == cm->subsampling_y) + aom_wb_write_literal(wb, cm->cdef_uv_strengths[i], CDEF_STRENGTH_BITS); } } #endif -- GitLab