Fix rd_filter_cache update in sub8x8 RD loop

Properly handle the rd_filter_cache update, when early termination
or skip prediction filter type check is triggered.

Change-Id: Ie7b9a75fed3358f45ffd15817f2b36670c14eb2d

vp9/encoder/vp9_rdopt.c

@@ -3876,6 +3876,7 @@ int64_t vp9_rd_pick_inter_mode_sub8x8(VP9_COMP *cpi, MACROBLOCK *x,
     int this_skip2 = 0;
     int64_t total_sse = INT_MAX;
     int early_term = 0;
+    int64_t mask_rd = 0;
 
     for (i = 0; i < TX_MODES; ++i)
       tx_cache[i] = INT64_MAX;
@@ -4069,21 +4070,20 @@ int64_t vp9_rd_pick_inter_mode_sub8x8(VP9_COMP *cpi, MACROBLOCK *x,
           cpi->rd_thresh_sub8x8[segment_id][bsize][THR_GOLD] : this_rd_thresh;
       xd->mi_8x8[0]->mbmi.tx_size = TX_4X4;
 
-      cpi->rd_filter_cache[SWITCHABLE_FILTERS] = INT64_MAX;
+      for (i = 0; i < SWITCHABLE_FILTER_CONTEXTS; ++i)
+        cpi->rd_filter_cache[i] = INT64_MAX;
+
       if (cm->mcomp_filter_type != BILINEAR) {
         tmp_best_filter = EIGHTTAP;
         if (x->source_variance <
             cpi->sf.disable_filter_search_var_thresh) {
           tmp_best_filter = EIGHTTAP;
-          vp9_zero(cpi->rd_filter_cache);
         } else if (cpi->sf.adaptive_pred_filter_type == 1 &&
                    ctx->pred_filter_type < SWITCHABLE) {
           tmp_best_filter = ctx->pred_filter_type;
-          vp9_zero(cpi->rd_filter_cache);
         } else if (cpi->sf.adaptive_pred_filter_type == 2) {
           tmp_best_filter = ctx->pred_filter_type < SWITCHABLE ?
                               ctx->pred_filter_type : 0;
-          vp9_zero(cpi->rd_filter_cache);
         } else {
           for (switchable_filter_index = 0;
                switchable_filter_index < SWITCHABLE_FILTERS;
@@ -4105,7 +4105,6 @@ int64_t vp9_rd_pick_inter_mode_sub8x8(VP9_COMP *cpi, MACROBLOCK *x,
 
             if (tmp_rd == INT64_MAX)
               continue;
-            cpi->rd_filter_cache[switchable_filter_index] = tmp_rd;
             rs = get_switchable_rate(x);
             rs_rd = RDCOST(x->rdmult, x->rddiv, rs, 0);
             cpi->rd_filter_cache[SWITCHABLE_FILTERS] =
@@ -4114,6 +4113,9 @@ int64_t vp9_rd_pick_inter_mode_sub8x8(VP9_COMP *cpi, MACROBLOCK *x,
             if (cm->mcomp_filter_type == SWITCHABLE)
               tmp_rd += rs_rd;
 
+            cpi->rd_filter_cache[switchable_filter_index] = tmp_rd;
+            mask_rd = MAX(tmp_rd, mask_rd);
+
             newbest = (tmp_rd < tmp_best_rd);
             if (newbest) {
               tmp_best_filter = mbmi->interp_filter;
@@ -4361,16 +4363,17 @@ int64_t vp9_rd_pick_inter_mode_sub8x8(VP9_COMP *cpi, MACROBLOCK *x,
         cm->mcomp_filter_type != BILINEAR) {
       int64_t ref = cpi->rd_filter_cache[cm->mcomp_filter_type == SWITCHABLE ?
                               SWITCHABLE_FILTERS : cm->mcomp_filter_type];
+      int64_t adj_rd;
+
       for (i = 0; i < SWITCHABLE_FILTER_CONTEXTS; i++) {
-        int64_t adj_rd;
-        // In cases of poor prediction, filter_cache[] can contain really big
-        // values, which actually are bigger than this_rd itself. This can
-        // cause negative best_filter_rd[] values, which is obviously silly.
-        // Therefore, if filter_cache < ref, we do an adjusted calculation.
-        if (cpi->rd_filter_cache[i] >= ref)
-          adj_rd = this_rd + cpi->rd_filter_cache[i] - ref;
-        else  // FIXME(rbultje) do this for comppred also
-          adj_rd = this_rd - (ref - cpi->rd_filter_cache[i]) * this_rd / ref;
+        if (ref == INT64_MAX)
+          adj_rd = 0;
+        else if (cpi->rd_filter_cache[i] == INT64_MAX)
+          adj_rd = mask_rd - ref + 10;
+        else
+          adj_rd = cpi->rd_filter_cache[i] - ref;
+
+        adj_rd += this_rd;
         best_filter_rd[i] = MIN(best_filter_rd[i], adj_rd);
       }
     }