Avoid out of bound array access

Change-Id: I4066561b769cf2bd4af515c9d351f609c08e3076

av1/encoder/rdopt.c

@@ -4898,7 +4898,11 @@ static void select_tx_type_yrd(const AV1_COMP *cpi, MACROBLOCK *x,
       if (this_rd_stats.rate != INT_MAX) {
         const int ext_tx_set = get_ext_tx_set(
             mbmi->min_tx_size, bsize, is_inter, cm->reduced_tx_set_used);
-        const TX_SIZE square_tx_size = txsize_sqr_map[mbmi->min_tx_size];
+        TX_SIZE square_tx_size = txsize_sqr_map[mbmi->min_tx_size];
+#if CONFIG_TX64X64
+        // For TX_64X64, use TX_32X32 to lookup cost
+        if (square_tx_size == TX_64X64) square_tx_size = TX_32X32;
+#endif  // CONFIG_TX64X64
         this_rd_stats.rate +=
             x->inter_tx_type_costs[ext_tx_set][square_tx_size][mbmi->tx_type];
         this_rd_stats.rate -=