Commit ef1b74c7 authored by Steinar Midtskogen's avatar Steinar Midtskogen

Make the CDEF RDO handle 4:2:2 properly

This fixes an assert:

av1/common/cdef_block.c:561: cdef_filter_fb: Assertion `bsize ==
BLOCK_8X8 || bsize == BLOCK_4X4' failed.

The RDO simply assigned a strength of 0 in the 4:2:2 case and called
cdef_filter_fb(), but cdef_filter_fb() will complain about 4:2:2 even
if the strength is 0.

The fix assigns a chroma mse of 0 when the the subsampling is
different for x and y rather than to call the filter.  This is faster
also.  The mse isn't really 0, but calculating the actual chroma mse
doesn't change result.

BUG=aomedia:881

Change-Id: I6154e21ddcca30e51baf805684dace10459c3350
parent 146a060a
......@@ -423,35 +423,40 @@ void av1_cdef_search(YV12_BUFFER_CONFIG *frame, const YV12_BUFFER_CONFIG *ref,
int xsize = (nhb << mi_wide_l2[pli]) +
CDEF_HBORDER * (fbc != nhfb - 1) + xoff;
sec_strength = gi % CDEF_SEC_STRENGTHS;
if (pli && !chroma_cdef) {
curr_mse = 0;
} else {
#if CONFIG_CDEF_SINGLEPASS
copy_sb16_16(&in[(-yoff * CDEF_BSTRIDE - xoff)], CDEF_BSTRIDE,
src[pli],
(fbr * MI_SIZE_64X64 << mi_high_l2[pli]) - yoff,
(fbc * MI_SIZE_64X64 << mi_wide_l2[pli]) - xoff,
stride[pli], ysize, xsize);
cdef_filter_fb(NULL, tmp_dst, CDEF_BSTRIDE, in, xdec[pli], ydec[pli],
dir, &dirinit, var, pli, dlist, cdef_count, threshold,
sec_strength + (sec_strength == 3), pri_damping,
sec_damping, coeff_shift);
#else
if (sec_strength == 0)
copy_sb16_16(&in[(-yoff * CDEF_BSTRIDE - xoff)], CDEF_BSTRIDE,
src[pli],
(fbr * MI_SIZE_64X64 << mi_high_l2[pli]) - yoff,
(fbc * MI_SIZE_64X64 << mi_wide_l2[pli]) - xoff,
stride[pli], ysize, xsize);
cdef_filter_fb(sec_strength ? NULL : (uint8_t *)in, CDEF_BSTRIDE,
tmp_dst, in, xdec[pli], ydec[pli], dir, &dirinit, var,
pli, dlist, cdef_count, threshold,
sec_strength + (sec_strength == 3), sec_damping,
pri_damping, coeff_shift, sec_strength != 0, 1);
cdef_filter_fb(NULL, tmp_dst, CDEF_BSTRIDE, in, xdec[pli],
ydec[pli], dir, &dirinit, var, pli, dlist,
cdef_count, threshold,
sec_strength + (sec_strength == 3), pri_damping,
sec_damping, coeff_shift);
#else
if (sec_strength == 0)
copy_sb16_16(&in[(-yoff * CDEF_BSTRIDE - xoff)], CDEF_BSTRIDE,
src[pli],
(fbr * MI_SIZE_64X64 << mi_high_l2[pli]) - yoff,
(fbc * MI_SIZE_64X64 << mi_wide_l2[pli]) - xoff,
stride[pli], ysize, xsize);
cdef_filter_fb(sec_strength ? NULL : (uint8_t *)in, CDEF_BSTRIDE,
tmp_dst, in, xdec[pli], ydec[pli], dir, &dirinit,
var, pli, dlist, cdef_count, threshold,
sec_strength + (sec_strength == 3), sec_damping,
pri_damping, coeff_shift, sec_strength != 0, 1);
#endif
curr_mse = compute_cdef_dist(
ref_coeff[pli] +
(fbr * MI_SIZE_64X64 << mi_high_l2[pli]) * stride[pli] +
(fbc * MI_SIZE_64X64 << mi_wide_l2[pli]),
stride[pli], tmp_dst, dlist, cdef_count, bsize[pli], coeff_shift,
pli);
curr_mse = compute_cdef_dist(
ref_coeff[pli] +
(fbr * MI_SIZE_64X64 << mi_high_l2[pli]) * stride[pli] +
(fbc * MI_SIZE_64X64 << mi_wide_l2[pli]),
stride[pli], tmp_dst, dlist, cdef_count, bsize[pli],
coeff_shift, pli);
}
if (pli < 2)
mse[pli][sb_count][gi] = curr_mse;
else
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment