Fix for supertx with rect-tx

Change-Id: I0cc3523a8992f889f8dd203449ceb55f2a422324

vp10/common/blockd.h

@@ -418,8 +418,9 @@ static const TX_TYPE intra_mode_to_tx_type_context[INTRA_MODES] = {
 
 #if CONFIG_SUPERTX
 static INLINE int supertx_enabled(const MB_MODE_INFO *mbmi) {
-  return (int)mbmi->tx_size > VPXMIN(b_width_log2_lookup[mbmi->sb_type],
-                                     b_height_log2_lookup[mbmi->sb_type]);
+  return (int)txsize_sqr_map[mbmi->tx_size] >
+         VPXMIN(b_width_log2_lookup[mbmi->sb_type],
+                b_height_log2_lookup[mbmi->sb_type]);
 }
 #endif  // CONFIG_SUPERTX
 
@@ -733,8 +734,8 @@ static INLINE TX_SIZE get_uv_tx_size(const MB_MODE_INFO *mbmi,
                                      const struct macroblockd_plane *pd) {
 #if CONFIG_SUPERTX
   if (supertx_enabled(mbmi))
-    return uvsupertx_size_lookup[mbmi->tx_size][pd->subsampling_x]
-                                [pd->subsampling_y];
+    return uvsupertx_size_lookup[txsize_sqr_map[mbmi->tx_size]]
+                                [pd->subsampling_x][pd->subsampling_y];
 #endif  // CONFIG_SUPERTX
   return get_uv_tx_size_impl(mbmi->tx_size, mbmi->sb_type, pd->subsampling_x,
                              pd->subsampling_y);

vp10/encoder/rd.c

@@ -593,7 +593,7 @@ static void get_entropy_contexts_plane(
       for (i = 0; i < num_4x4_h; i += 8)
         t_left[i] = !!*(const uint64_t *)&left[i];
       break;
-#if CONFIG_EXT_TX
+#if CONFIG_EXT_TX && CONFIG_RECT_TX
     case TX_4X8:
       memcpy(t_above, above, sizeof(ENTROPY_CONTEXT) * num_4x4_w);
       for (i = 0; i < num_4x4_h; i += 2)
@@ -628,7 +628,7 @@ static void get_entropy_contexts_plane(
       for (i = 0; i < num_4x4_h; i += 4)
         t_left[i] = !!*(const uint32_t *)&left[i];
       break;
-#endif  // CONFIG_EXT_TX
+#endif  // CONFIG_EXT_TX && CONFIG_RECT_TX
     default: assert(0 && "Invalid transform size."); break;
   }
 }