Problem of over smoothing with intra modes.

In some cases intra modes in inter frames give
an over smoothed appearance. Especially with
noisy but flat content.

Also in some cases there were problems with key
frame sizing again with very flat but noisy content.

These are temporary changes to help alleviate the
visual problems but will almost certainly hurt metric
results especially at the very low data rate end.

Change-Id: I11549179a19277ffc283d9788bc70168f2a8bdc9

vp9/encoder/vp9_firstpass.c

@@ -41,9 +41,10 @@
 #define RMAX       128.0
 #define GF_RMAX    96.0
 #define ERR_DIVISOR   150.0
+#define MIN_DECAY_FACTOR 0.1
 
-#define KF_MB_INTRA_MIN 300
-#define GF_MB_INTRA_MIN 200
+#define KF_MB_INTRA_MIN 150
+#define GF_MB_INTRA_MIN 100
 
 #define DOUBLE_DIVIDE_CHECK(X) ((X)<0?(X)-.000001:(X)+.000001)
 
@@ -1405,10 +1406,9 @@ static int calc_arf_boost(
     // Cumulative effect of prediction quality decay
     if (!flash_detected) {
       decay_accumulator =
-        decay_accumulator *
-        get_prediction_decay_rate(cpi, &this_frame);
-      decay_accumulator =
-        decay_accumulator < 0.1 ? 0.1 : decay_accumulator;
+        decay_accumulator * get_prediction_decay_rate(cpi, &this_frame);
+      decay_accumulator = decay_accumulator < MIN_DECAY_FACTOR
+                          ? MIN_DECAY_FACTOR : decay_accumulator;
     }
 
     boost_score += (decay_accumulator *
@@ -1443,10 +1443,9 @@ static int calc_arf_boost(
     // Cumulative effect of prediction quality decay
     if (!flash_detected) {
       decay_accumulator =
-        decay_accumulator *
-        get_prediction_decay_rate(cpi, &this_frame);
-      decay_accumulator =
-        decay_accumulator < 0.1 ? 0.1 : decay_accumulator;
+        decay_accumulator * get_prediction_decay_rate(cpi, &this_frame);
+      decay_accumulator = decay_accumulator < MIN_DECAY_FACTOR
+                          ? MIN_DECAY_FACTOR : decay_accumulator;
     }
 
     boost_score += (decay_accumulator *
@@ -1632,7 +1631,7 @@ static void define_gf_group(VP9_COMP *cpi, FIRSTPASS_STATS *this_frame) {
         ((mv_ratio_accumulator > 100.0) ||
          (abs_mv_in_out_accumulator > 3.0) ||
          (mv_in_out_accumulator < -2.0) ||
-         ((boost_score - old_boost_score) < 12.5))
+         ((boost_score - old_boost_score) < IIFACTOR))
       )) {
       boost_score = old_boost_score;
       break;
@@ -2393,7 +2392,8 @@ static void find_next_key_frame(VP9_COMP *cpi, FIRSTPASS_STATS *this_frame) {
     if (!detect_flash(cpi, 0)) {
       loop_decay_rate = get_prediction_decay_rate(cpi, &next_frame);
       decay_accumulator = decay_accumulator * loop_decay_rate;
-      decay_accumulator = decay_accumulator < 0.1 ? 0.1 : decay_accumulator;
+      decay_accumulator = decay_accumulator < MIN_DECAY_FACTOR
+                            ? MIN_DECAY_FACTOR : decay_accumulator;
     }
 
     boost_score += (decay_accumulator * r);
@@ -2433,14 +2433,11 @@ static void find_next_key_frame(VP9_COMP *cpi, FIRSTPASS_STATS *this_frame) {
     int allocation_chunks;
     int alt_kf_bits;
 
-    if (kf_boost < 300) {
-      kf_boost += (cpi->twopass.frames_to_key * 3);
-      if (kf_boost > 300)
-        kf_boost = 300;
-    }
+    if (kf_boost < (cpi->twopass.frames_to_key * 5))
+      kf_boost = (cpi->twopass.frames_to_key * 5);
 
-    if (kf_boost < 250)                                                      // Min KF boost
-      kf_boost = 250;
+    if (kf_boost < 300) // Min KF boost
+      kf_boost = 300;
 
     // Make a note of baseline boost and the zero motion
     // accumulator value for use elsewhere.

vp9/encoder/vp9_rdopt.c

@@ -1312,7 +1312,6 @@ static int64_t rd_pick_intra16x16mby_mode(VP9_COMP *cpi,
 
       this_rd = RDCOST(x->rdmult, x->rddiv, rate, distortion);
 
-
       if (this_rd < best_rd) {
         mode_selected = mode;
         txfm_size = mbmi->txfm_size;
@@ -1546,6 +1545,7 @@ static int64_t rd_pick_intra8x8mby_modes(VP9_COMP *cpi, MACROBLOCK *mb,
     mic->bmi[ib].as_mode.second = best_second_mode;
 #endif
   }
+
   *Rate = cost;
   *rate_y = tot_rate_y;
   *Distortion = distortion;
@@ -3385,6 +3385,9 @@ static void rd_pick_inter_mode(VP9_COMP *cpi, MACROBLOCK *x,
   unsigned int ref_costs[MAX_REF_FRAMES];
   int_mv seg_mvs[NB_PARTITIONINGS][16 /* n_blocks */][MAX_REF_FRAMES - 1];
 
+  int intra_cost_penalty = 20 * vp9_dc_quant(cpi->common.base_qindex,
+                                             cpi->common.y1dc_delta_q);
+
   vpx_memset(mode8x8, 0, sizeof(mode8x8));
   vpx_memset(&frame_mv, 0, sizeof(frame_mv));
   vpx_memset(&best_mbmode, 0, sizeof(best_mbmode));
@@ -3580,16 +3583,17 @@ static void rd_pick_inter_mode(VP9_COMP *cpi, MACROBLOCK *x,
     if (!mbmi->ref_frame) {
       switch (this_mode) {
         default:
-        case DC_PRED:
         case V_PRED:
         case H_PRED:
-        case TM_PRED:
         case D45_PRED:
         case D135_PRED:
         case D117_PRED:
         case D153_PRED:
         case D27_PRED:
         case D63_PRED:
+          rate2 += intra_cost_penalty;
+        case DC_PRED:
+        case TM_PRED:
           mbmi->ref_frame = INTRA_FRAME;
           // FIXME compound intra prediction
           vp9_build_intra_predictors_mby(&x->e_mbd);
@@ -3623,6 +3627,7 @@ static void rd_pick_inter_mode(VP9_COMP *cpi, MACROBLOCK *x,
 #endif
                                              0);
           rate2 += rate;
+          rate2 += intra_cost_penalty;
           distortion2 += distortion;
 
           if (tmp_rd < best_yrd) {
@@ -3715,6 +3720,7 @@ static void rd_pick_inter_mode(VP9_COMP *cpi, MACROBLOCK *x,
           }
 
           rate2 += rate;
+          rate2 += intra_cost_penalty;
           distortion2 += distortion;
 
           /* TODO: uv rate maybe over-estimated here since there is UV intra