Reset rate cost value in rd mode search

When early termination is triggered, properly reset the rate cost
to invalid value to avoid potential ioc issue.

Change-Id: I3444390be2e49a34bb02cf8a74c33d5dbd96d88d

vp9/encoder/vp9_rdopt.c

@@ -3394,6 +3394,7 @@ void vp9_rd_pick_inter_mode_sb(VP9_COMP *cpi, MACROBLOCK *x,
   }
 
   if (best_mode_index < 0 || best_rd >= best_rd_so_far) {
+    rd_cost->rate = INT_MAX;
     rd_cost->rdcost = INT64_MAX;
     return;
   }
@@ -3564,6 +3565,7 @@ void vp9_rd_pick_inter_mode_sb_seg_skip(VP9_COMP *cpi, MACROBLOCK *x,
   rd_cost->rdcost = this_rd;
 
   if (this_rd >= best_rd_so_far) {
+    rd_cost->rate = INT_MAX;
     rd_cost->rdcost = INT64_MAX;
     return;
   }
@@ -4115,6 +4117,7 @@ void vp9_rd_pick_inter_mode_sub8x8(VP9_COMP *cpi, MACROBLOCK *x,
   }
 
   if (best_rd >= best_rd_so_far) {
+    rd_cost->rate = INT_MAX;
     rd_cost->rdcost = INT64_MAX;
     return;
   }