A bug fix for var-tx

Fixes a crash with supertx, ext-tx and rect-tx

Change-Id: I6b5f4cfd6e209558541a791be685b55156aa0138

av1/encoder/encodeframe.c

@@ -1304,10 +1304,13 @@ static void update_state_supertx(AV1_COMP *cpi, ThreadData *td,
 #if CONFIG_VAR_TX
   {
     const TX_SIZE mtx = mbmi->tx_size;
+    const int num_4x4_blocks_wide = num_4x4_blocks_wide_txsize_lookup[mtx] >> 1;
+    const int num_4x4_blocks_high = num_4x4_blocks_high_txsize_lookup[mtx] >> 1;
     int idy, idx;
-    for (idy = 0; idy < (1 << mtx) / 2; ++idy)
-      for (idx = 0; idx < (1 << mtx) / 2; ++idx)
-        mbmi->inter_tx_size[idy][idx] = mbmi->tx_size;
+    mbmi->inter_tx_size[0][0] = mtx;
+    for (idy = 0; idy < num_4x4_blocks_high; ++idy)
+      for (idx = 0; idx < num_4x4_blocks_wide; ++idx)
+        mbmi->inter_tx_size[idy][idx] = mtx;
   }
 #endif  // CONFIG_VAR_TX
   // Turn motion variation off for supertx

av1/encoder/rdopt.c

@@ -3321,7 +3321,7 @@ static void select_tx_type_yrd(const AV1_COMP *cpi, MACROBLOCK *x, int *rate,
   TX_TYPE tx_type, best_tx_type = DCT_DCT;
   const int is_inter = is_inter_block(mbmi);
   TX_SIZE best_tx_size[MAX_MIB_SIZE][MAX_MIB_SIZE];
-  TX_SIZE best_tx = TX_SIZES;
+  TX_SIZE best_tx = max_txsize_lookup[bsize];
   uint8_t best_blk_skip[MAX_MIB_SIZE * MAX_MIB_SIZE * 4];
   const int n4 = 1 << (num_pels_log2_lookup[bsize] - 4);
   int idx, idy;
@@ -7707,6 +7707,7 @@ static int64_t handle_inter_mode(
     } else {
       x->skip = 1;
       *disable_skip = 1;
+      mbmi->tx_size = tx_size_from_tx_mode(bsize, cm->tx_mode, 1);
 
 // The cost of skip bit needs to be added.
 #if CONFIG_OBMC || CONFIG_WARPED_MOTION