Adaptive txfm size selection depending on residual sse/variance

This commit enables an adaptive transform size selection method
for speed -6. It uses largest transform size when the sse is more
than 4 times of variance, i.e., most energy is compacted in the
DC coefficient. Otherwise, use the default TX_8X8. It improves
the compression efficiency for rtc set of speed -6 by 0.8%, no
speed change observed.

Change-Id: Ie6ed1e728ff7bf88ebe940a60811361cdd19969c

vp9/encoder/vp9_encodeframe.c

@@ -2390,7 +2390,7 @@ static TX_MODE select_tx_mode(const VP9_COMP *cpi) {
                  rd_opt->tx_select_threshes[frame_type][TX_MODE_SELECT] ?
                      ALLOW_32X32 : TX_MODE_SELECT;
     } else if (cpi->sf.tx_size_search_method == USE_TX_8X8) {
-      return ALLOW_8X8;
+      return TX_MODE_SELECT;
     } else {
       unsigned int total = 0;
       int i;

vp9/encoder/vp9_pickmode.c

@@ -252,6 +252,17 @@ static void model_rd_for_sb_y(VP9_COMP *cpi, BLOCK_SIZE bsize,
   else
     x->skip_txfm = 0;
 
+  if (cpi->common.tx_mode == TX_MODE_SELECT) {
+    if (sse > (var << 2))
+      xd->mi[0]->mbmi.tx_size = MIN(max_txsize_lookup[bsize],
+                          tx_mode_to_biggest_tx_size[cpi->common.tx_mode]);
+    else
+      xd->mi[0]->mbmi.tx_size = TX_8X8;
+  } else {
+    xd->mi[0]->mbmi.tx_size = MIN(max_txsize_lookup[bsize],
+                         tx_mode_to_biggest_tx_size[cpi->common.tx_mode]);
+  }
+
   vp9_model_rd_from_var_lapndz(sse - var, 1 << num_pels_log2_lookup[bsize],
                                dc_quant >> 3, &rate, &dist);
   *out_rate_sum = rate >> 1;
@@ -293,6 +304,8 @@ int64_t vp9_pick_inter_mode(VP9_COMP *cpi, MACROBLOCK *x,
   struct macroblockd_plane *const pd = &xd->plane[0];
   PREDICTION_MODE this_mode, best_mode = ZEROMV;
   MV_REFERENCE_FRAME ref_frame, best_ref_frame = LAST_FRAME;
+  TX_SIZE best_tx_size = MIN(max_txsize_lookup[bsize],
+                             tx_mode_to_biggest_tx_size[cpi->common.tx_mode]);
   INTERP_FILTER best_pred_filter = EIGHTTAP;
   int_mv frame_mv[MB_MODE_COUNT][MAX_REF_FRAMES];
   struct buf_2d yv12_mb[4][MAX_MB_PLANE];
@@ -486,6 +499,7 @@ int64_t vp9_pick_inter_mode(VP9_COMP *cpi, MACROBLOCK *x,
         int64_t pf_dist[3];
         unsigned int pf_var[3];
         unsigned int pf_sse[3];
+        TX_SIZE pf_tx_size[3];
         int64_t best_cost = INT64_MAX;
         INTERP_FILTER best_filter = SWITCHABLE, filter;
         PRED_BUFFER *current_pred = this_mode_pred;
@@ -499,6 +513,7 @@ int64_t vp9_pick_inter_mode(VP9_COMP *cpi, MACROBLOCK *x,
           cost = RDCOST(x->rdmult, x->rddiv,
                         vp9_get_switchable_rate(cpi) + pf_rate[filter],
                         pf_dist[filter]);
+          pf_tx_size[filter] = mbmi->tx_size;
           if (cost < best_cost) {
             best_filter = filter;
             best_cost = cost;
@@ -523,6 +538,7 @@ int64_t vp9_pick_inter_mode(VP9_COMP *cpi, MACROBLOCK *x,
           free_pred_buffer(current_pred);
 
         mbmi->interp_filter = best_filter;
+        mbmi->tx_size = pf_tx_size[mbmi->interp_filter];
         rate = pf_rate[mbmi->interp_filter];
         dist = pf_dist[mbmi->interp_filter];
         var_y = pf_var[mbmi->interp_filter];
@@ -624,6 +640,7 @@ int64_t vp9_pick_inter_mode(VP9_COMP *cpi, MACROBLOCK *x,
         *returndistortion = dist;
         best_mode = this_mode;
         best_pred_filter = mbmi->interp_filter;
+        best_tx_size = mbmi->tx_size;
         best_ref_frame = ref_frame;
         skip_txfm = x->skip_txfm;
 
@@ -657,10 +674,11 @@ int64_t vp9_pick_inter_mode(VP9_COMP *cpi, MACROBLOCK *x,
                       bw, bh);
   }
 
-  mbmi->mode = best_mode;
+  mbmi->mode          = best_mode;
   mbmi->interp_filter = best_pred_filter;
-  mbmi->ref_frame[0] = best_ref_frame;
-  mbmi->mv[0].as_int = frame_mv[best_mode][best_ref_frame].as_int;
+  mbmi->tx_size       = best_tx_size;
+  mbmi->ref_frame[0]  = best_ref_frame;
+  mbmi->mv[0].as_int  = frame_mv[best_mode][best_ref_frame].as_int;
   xd->mi[0]->bmi[0].as_mv[0].as_int = mbmi->mv[0].as_int;
   x->skip_txfm = skip_txfm;
 
@@ -669,7 +687,6 @@ int64_t vp9_pick_inter_mode(VP9_COMP *cpi, MACROBLOCK *x,
   if (!x->skip && best_rd > inter_mode_thresh &&
       bsize <= cpi->sf.max_intra_bsize) {
     int i, j;
-    const int step   = 1 << mbmi->tx_size;
     const int width  = num_4x4_blocks_wide_lookup[bsize];
     const int height = num_4x4_blocks_high_lookup[bsize];
 
@@ -679,6 +696,10 @@ int64_t vp9_pick_inter_mode(VP9_COMP *cpi, MACROBLOCK *x,
     const int src_stride = p->src.stride;
     int block_idx = 0;
 
+    TX_SIZE tmp_tx_size = MIN(max_txsize_lookup[bsize],
+                              tx_mode_to_biggest_tx_size[cpi->common.tx_mode]);
+    const int step = 1 << tmp_tx_size;
+
     for (this_mode = DC_PRED; this_mode <= DC_PRED; ++this_mode) {
       if (cpi->sf.reuse_inter_pred_sby) {
         pd->dst.buf = tmp[0].data;
@@ -688,7 +709,7 @@ int64_t vp9_pick_inter_mode(VP9_COMP *cpi, MACROBLOCK *x,
       for (j = 0; j < height; j += step) {
         for (i = 0; i < width; i += step) {
           vp9_predict_intra_block(xd, block_idx, b_width_log2(bsize),
-                                  mbmi->tx_size, this_mode,
+                                  tmp_tx_size, this_mode,
                                   &p->src.buf[4 * (j * dst_stride + i)],
                                   src_stride,
                                   &pd->dst.buf[4 * (j * dst_stride + i)],
@@ -715,6 +736,7 @@ int64_t vp9_pick_inter_mode(VP9_COMP *cpi, MACROBLOCK *x,
         *returnrate = rate;
         *returndistortion = dist;
         mbmi->mode = this_mode;
+        mbmi->tx_size = tmp_tx_size;
         mbmi->ref_frame[0] = INTRA_FRAME;
         mbmi->uv_mode = this_mode;
         mbmi->mv[0].as_int = INVALID_MV;

vp9/encoder/vp9_speed_features.c

@@ -272,7 +272,8 @@ static void set_rt_speed_feature(VP9_COMP *cpi, SPEED_FEATURES *sf,
     sf->search_type_check_frequency = 50;
     sf->source_var_thresh = 360;
 
-    sf->tx_size_search_method = USE_TX_8X8;
+    sf->tx_size_search_method = (cm->frame_type == KEY_FRAME) ?
+        USE_LARGESTALL : USE_TX_8X8;
     sf->max_intra_bsize = BLOCK_8X8;
 
     // This feature is only enabled when partition search is disabled.