Commit c0f6bf25 authored by Yushin Cho's avatar Yushin Cho

Another fix of daala-dist for cb4x4

Daala-dist replaces the luma distortion of sub8x8 partitions with
its own distortion thus requires to split the luma distortion only.
Doing so, there has been a bug that INT_MAX64 value comes
when the sub8x8 parition is skipped. This happened because the existing
code does not initialize the rd_stats_y or tmp_rd_stats_y, i.e. rd_stat struct
for luma only in several places.

Change-Id: If229b53bb7a6cff0b8751138a32b1dcf02665624
parent abd4343a
......@@ -3551,6 +3551,11 @@ static void rd_pick_partition(const AV1_COMP *const cpi, ThreadData *td,
&this_rdc, best_rdc.rdcost - sum_rdc.rdcost, pc_tree->split[idx]);
#endif // CONFIG_SUPERTX
#if CONFIG_DAALA_DIST && CONFIG_CB4X4
if (bsize == BLOCK_8X8 && this_rdc.rate != INT_MAX) {
assert(this_rdc.dist_y < INT64_MAX);
}
#endif
if (this_rdc.rate == INT_MAX) {
sum_rdc.rdcost = INT64_MAX;
#if CONFIG_SUPERTX
......@@ -3565,7 +3570,10 @@ static void rd_pick_partition(const AV1_COMP *const cpi, ThreadData *td,
sum_rate_nocoef += this_rate_nocoef;
#endif // CONFIG_SUPERTX
#if CONFIG_DAALA_DIST && CONFIG_CB4X4
if (bsize == BLOCK_8X8) {
assert(this_rdc.dist_y < INT64_MAX);
sum_rdc.dist_y += this_rdc.dist_y;
}
#endif
}
}
......@@ -3581,6 +3589,7 @@ static void rd_pick_partition(const AV1_COMP *const cpi, ThreadData *td,
src_stride, x->decoded_8x8, 8, 8, 8, 1,
use_activity_masking, x->qindex)
<< 4;
assert(sum_rdc.dist_y < INT64_MAX);
sum_rdc.dist = sum_rdc.dist - sum_rdc.dist_y + daala_dist;
sum_rdc.rdcost =
RDCOST(x->rdmult, x->rddiv, sum_rdc.rate, sum_rdc.dist);
......@@ -4068,6 +4077,12 @@ static void rd_pick_partition(const AV1_COMP *const cpi, ThreadData *td,
// checks occur in some sub function and thus are used...
(void)best_rd;
*rd_cost = best_rdc;
#if CONFIG_DAALA_DIST && CONFIG_CB4X4
if (bsize <= BLOCK_8X8 && rd_cost->rate != INT_MAX) {
assert(rd_cost->dist_y < INT64_MAX);
}
#endif // CONFIG_DAALA_DIST && CONFIG_CB4X4
#if CONFIG_SUPERTX
*rate_nocoef = best_rate_nocoef;
#endif // CONFIG_SUPERTX
......
......@@ -9498,6 +9498,19 @@ void av1_rd_pick_inter_mode_sb(const AV1_COMP *cpi, TileDataEnc *tile_data,
{
RD_STATS rd_stats, rd_stats_y, rd_stats_uv;
av1_init_rd_stats(&rd_stats);
#if CONFIG_DAALA_DIST && CONFIG_CB4X4
// While av1 master uses rd_stats_y.rate through out the codebase,
// which is set when handle_inter_moden is called, the daala-dist code
// in rd_pick_partition() for cb4x4 and sub8x8 blocks need to know
// .dist_y which comes from rd_stats_y.dist and rd_stats_y.sse.
// The problem is rd_stats_y.dist and rd_stats_y.sse are sometimes not
// initialized when rd_stats.skip = 1,
// then instead rd_stats.dist and rd_stats.sse have the
// combined luma and chroma dist and sse.
// This can be seen inside motion_mode_rd(), which is called by
// handle_inter_mode().
if (bsize < BLOCK_8X8) av1_init_rd_stats(&rd_stats_y);
#endif
rd_stats.rate = rate2;
// Point to variables that are maintained between loop iterations
......@@ -9517,7 +9530,14 @@ void av1_rd_pick_inter_mode_sb(const AV1_COMP *cpi, TileDataEnc *tile_data,
rate_y = rd_stats_y.rate;
rate_uv = rd_stats_uv.rate;
#if CONFIG_DAALA_DIST && CONFIG_CB4X4
if (bsize < BLOCK_8X8) distortion2_y = rd_stats_y.dist;
if (bsize < BLOCK_8X8) {
if (rd_stats_y.rate != INT_MAX) {
assert(rd_stats_y.sse < INT64_MAX);
assert(rd_stats_y.dist < INT64_MAX);
}
total_sse_y = rd_stats_y.sse;
distortion2_y = rd_stats_y.dist;
}
#endif
}
......@@ -9585,6 +9605,7 @@ void av1_rd_pick_inter_mode_sb(const AV1_COMP *cpi, TileDataEnc *tile_data,
RD_STATS tmp_rd_stats, tmp_rd_stats_y, tmp_rd_stats_uv;
av1_invalid_rd_stats(&tmp_rd_stats);
x->skip = 0;
mbmi->ref_mv_idx = 1 + ref_idx;
......@@ -9655,7 +9676,15 @@ void av1_rd_pick_inter_mode_sb(const AV1_COMP *cpi, TileDataEnc *tile_data,
frame_mv[NEARMV][ref_frame] = cur_mv;
av1_init_rd_stats(&tmp_rd_stats);
#if CONFIG_DAALA_DIST && CONFIG_CB4X4
// With the same reason as 'rd_stats_y' passed to above
// handle_inter_mode(), tmp_rd_stats_y.dist and
// tmp_rd_stats_y.sse are sometimes not initialized, esp. when
// tmp_rd_stats.skip = 1 and tmp_rd_stats.dist and .sse
// represent combined luma and chroma .dist and .sse,
// we should initialized tmp_rd_stats_y.
if (bsize < BLOCK_8X8) av1_init_rd_stats(&tmp_rd_stats_y);
#endif
// Point to variables that are not maintained between iterations
args.single_newmv = dummy_single_newmv;
#if CONFIG_EXT_INTER
......@@ -9730,6 +9759,10 @@ void av1_rd_pick_inter_mode_sb(const AV1_COMP *cpi, TileDataEnc *tile_data,
backup_skip = x->skip;
#if CONFIG_DAALA_DIST && CONFIG_CB4X4
if (bsize < BLOCK_8X8) {
if (tmp_rd_stats_y.rate != INT_MAX) {
assert(tmp_rd_stats_y.sse < INT64_MAX);
assert(tmp_rd_stats_y.dist < INT64_MAX);
}
total_sse_y = tmp_rd_stats_y.sse;
distortion2_y = tmp_rd_stats_y.dist;
}
......@@ -9808,7 +9841,10 @@ void av1_rd_pick_inter_mode_sb(const AV1_COMP *cpi, TileDataEnc *tile_data,
rate_y = 0;
rate_uv = 0;
#if CONFIG_DAALA_DIST && CONFIG_CB4X4
if (bsize < BLOCK_8X8) distortion2_y = total_sse_y;
if (bsize < BLOCK_8X8) {
assert(total_sse_y < INT64_MAX);
distortion2_y = total_sse_y;
}
#endif
}
} else {
......@@ -9829,6 +9865,12 @@ void av1_rd_pick_inter_mode_sb(const AV1_COMP *cpi, TileDataEnc *tile_data,
#endif // CONFIG_MOTION_VAR || CONFIG_WARPED_MOTION
}
#if CONFIG_DAALA_DIST && CONFIG_CB4X4
if ((bsize < BLOCK_8X8) && (rate2 != INT_MAX)) {
assert(distortion2_y < INT64_MAX);
}
#endif
if (ref_frame == INTRA_FRAME) {
// Keep record of best intra rd
if (this_rd < best_intra_rd) {
......@@ -9900,7 +9942,10 @@ void av1_rd_pick_inter_mode_sb(const AV1_COMP *cpi, TileDataEnc *tile_data,
this_skip2 || skippable);
best_rate_uv = rate_uv;
#if CONFIG_DAALA_DIST && CONFIG_CB4X4
if (bsize < BLOCK_8X8) rd_cost->dist_y = distortion2_y;
if (bsize < BLOCK_8X8) {
assert(distortion2_y < INT64_MAX);
rd_cost->dist_y = distortion2_y;
}
#endif
#if CONFIG_VAR_TX
for (i = 0; i < MAX_MB_PLANE; ++i)
......@@ -9909,7 +9954,11 @@ void av1_rd_pick_inter_mode_sb(const AV1_COMP *cpi, TileDataEnc *tile_data,
#endif // CONFIG_VAR_TX
}
}
#if CONFIG_DAALA_DIST && CONFIG_CB4X4
if ((bsize < BLOCK_8X8) && (rd_cost->rate != INT_MAX)) {
assert(rd_cost->dist_y < INT64_MAX);
}
#endif
/* keep record of best compound/single-only prediction */
if (!disable_skip && ref_frame != INTRA_FRAME) {
int64_t single_rd, hybrid_rd, single_rate, hybrid_rate;
......@@ -10029,15 +10078,25 @@ void av1_rd_pick_inter_mode_sb(const AV1_COMP *cpi, TileDataEnc *tile_data,
rd_cost->rate +=
(rd_stats_y.rate + rd_stats_uv.rate - best_rate_y - best_rate_uv);
rd_cost->dist = rd_stats_y.dist + rd_stats_uv.dist;
#if CONFIG_DAALA_DIST && CONFIG_CB4X4
if (bsize < BLOCK_8X8) rd_cost->dist_y = rd_stats_y.dist;
#endif
rd_cost->rdcost =
RDCOST(x->rdmult, x->rddiv, rd_cost->rate, rd_cost->dist);
best_skip2 = skip_blk;
#if CONFIG_DAALA_DIST && CONFIG_CB4X4
if (bsize < BLOCK_8X8) {
assert(rd_cost->rate != INT_MAX);
assert(rd_cost->dist_y < INT64_MAX);
rd_cost->dist_y = rd_stats_y.dist;
}
#endif
}
}
#if CONFIG_DAALA_DIST && CONFIG_CB4X4
if ((bsize < BLOCK_8X8) && (rd_cost->rate != INT_MAX)) {
assert(rd_cost->dist_y < INT64_MAX);
}
#endif
#if CONFIG_PALETTE
// Only try palette mode when the best mode so far is an intra mode.
if (try_palette && !is_inter_mode(best_mbmode.mode)) {
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment