Merge "Enforce intra_y_mode_mask for inter rd."

vp9/encoder/vp9_rdopt.c

@@ -3142,6 +3142,8 @@ int64_t vp9_rd_pick_inter_mode_sb(VP9_COMP *cpi, MACROBLOCK *x,
   const int *const rd_threshes = cpi->rd_threshes[segment_id][bsize];
   const int *const rd_thresh_freq_fact = cpi->rd_thresh_freq_fact[bsize];
   const int mode_search_skip_flags = cpi->sf.mode_search_skip_flags;
+  const int intra_y_mode_mask =
+      cpi->sf.intra_y_mode_mask[max_txsize_lookup[bsize]];
 
   x->skip_encode = cpi->sf.skip_encode_frame && x->q_index < QIDX_SKIP_THRESH;
 
@@ -3305,24 +3307,28 @@ int64_t vp9_rd_pick_inter_mode_sb(VP9_COMP *cpi, MACROBLOCK *x,
         mode_excluded = cm->reference_mode == COMPOUND_REFERENCE;
     }
 
-    if (ref_frame == INTRA_FRAME && this_mode != DC_PRED) {
-      // Disable intra modes other than DC_PRED for blocks with low variance
-      // Threshold for intra skipping based on source variance
-      // TODO(debargha): Specialize the threshold for super block sizes
-      const unsigned int skip_intra_var_thresh = 64;
-      if ((mode_search_skip_flags & FLAG_SKIP_INTRA_LOWVAR) &&
-          x->source_variance < skip_intra_var_thresh)
+    if (ref_frame == INTRA_FRAME) {
+      if (!(intra_y_mode_mask & (1 << this_mode)))
         continue;
-      // Only search the oblique modes if the best so far is
-      // one of the neighboring directional modes
-      if ((mode_search_skip_flags & FLAG_SKIP_INTRA_BESTINTER) &&
-          (this_mode >= D45_PRED && this_mode <= TM_PRED)) {
-        if (vp9_mode_order[best_mode_index].ref_frame[0] > INTRA_FRAME)
+      if (this_mode != DC_PRED) {
+        // Disable intra modes other than DC_PRED for blocks with low variance
+        // Threshold for intra skipping based on source variance
+        // TODO(debargha): Specialize the threshold for super block sizes
+        const unsigned int skip_intra_var_thresh = 64;
+        if ((mode_search_skip_flags & FLAG_SKIP_INTRA_LOWVAR) &&
+            x->source_variance < skip_intra_var_thresh)
           continue;
-      }
-      if (mode_search_skip_flags & FLAG_SKIP_INTRA_DIRMISMATCH) {
-        if (conditional_skipintra(this_mode, best_intra_mode))
+        // Only search the oblique modes if the best so far is
+        // one of the neighboring directional modes
+        if ((mode_search_skip_flags & FLAG_SKIP_INTRA_BESTINTER) &&
+            (this_mode >= D45_PRED && this_mode <= TM_PRED)) {
+          if (vp9_mode_order[best_mode_index].ref_frame[0] > INTRA_FRAME)
             continue;
+        }
+        if (mode_search_skip_flags & FLAG_SKIP_INTRA_DIRMISMATCH) {
+          if (conditional_skipintra(this_mode, best_intra_mode))
+              continue;
+        }
       }
     } else {
       // if we're near/nearest and mv == 0,0, compare to zeromv