Fix aq_complexity tx_size reset

BUG=aomedia:1065

Change-Id: I0951a276865a5d810eb04bbb5251ed5c1b417ca4

av1/encoder/aq_complexity.c

@@ -123,8 +123,9 @@ void av1_caq_select_segment(const AV1_COMP *cpi, MACROBLOCK *mb, BLOCK_SIZE bs,
     segment = DEFAULT_AQ2_SEG;
   } else {
     // Rate depends on fraction of a SB64 in frame (xmis * ymis / bw * bh).
-    // It is converted to bits * 256 units.
-    const int64_t num = (int64_t)cpi->rc.sb64_target_rate * xmis * ymis * 256;
+    // It is converted to bits << AV1_PROB_COST_SHIFT units.
+    const int64_t num = (int64_t)(cpi->rc.sb64_target_rate * xmis * ymis)
+                        << AV1_PROB_COST_SHIFT;
     const int denom = cm->mib_size * cm->mib_size;
     const int target_rate = (int)(num / denom);
     double logvar;

av1/encoder/encodeframe.c

@@ -427,6 +427,14 @@ static void reset_tx_size(MACROBLOCKD *xd, MB_MODE_INFO *mbmi,
   } else if (tx_mode != TX_MODE_SELECT) {
     mbmi->tx_size =
         tx_size_from_tx_mode(mbmi->sb_type, tx_mode, is_inter_block(mbmi));
+  } else {
+    BLOCK_SIZE bsize = mbmi->sb_type;
+    TX_SIZE max_rect_txsize = get_max_rect_tx_size(bsize, is_inter_block(mbmi));
+    TX_SIZE min_tx_size =
+        (TX_SIZE)AOMMAX((int)TX_4X4,
+                        txsize_sqr_map[max_rect_txsize] - MAX_TX_DEPTH +
+                            is_rect_tx(max_rect_txsize));
+    mbmi->tx_size = (TX_SIZE)AOMMAX(mbmi->tx_size, min_tx_size);
   }
 }