Refactor transform size coding

Introduce the transform block partition depth macro definition.

Change-Id: I218dc77a77c8e967da4d270d4ec0d7691b712a5f

av1/common/entropymode.c

@@ -863,8 +863,8 @@ const aom_tree_index av1_tx_size_tree[TX_SIZES - 1][TREE_SIZE(TX_SIZES)] = {
   },
 };
 
-static const aom_prob default_tx_size_prob[TX_SIZES - 1][TX_SIZE_CONTEXTS]
-                                          [TX_SIZES - 1] = {
+static const aom_prob default_tx_size_prob[MAX_TX_DEPTH][TX_SIZE_CONTEXTS]
+                                          [MAX_TX_DEPTH] = {
                                             {
                                                 // Max tx_size is 8X8
                                                 { 100 },
@@ -1593,7 +1593,7 @@ void av1_adapt_intra_frame_probs(AV1_COMMON *cm) {
   const FRAME_COUNTS *counts = &cm->counts;
 
   if (cm->tx_mode == TX_MODE_SELECT) {
-    for (i = 0; i < TX_SIZES - 1; ++i) {
+    for (i = 0; i < MAX_TX_DEPTH; ++i) {
       for (j = 0; j < TX_SIZE_CONTEXTS; ++j)
         aom_tree_merge_probs(av1_tx_size_tree[i], pre_fc->tx_size_probs[i][j],
                              counts->tx_size[i][j], fc->tx_size_probs[i][j]);

av1/common/entropymode.h

@@ -125,7 +125,7 @@ typedef struct frame_contexts {
 #else
   aom_prob comp_ref_prob[REF_CONTEXTS][COMP_REFS - 1];
 #endif  // CONFIG_EXT_REFS
-  aom_prob tx_size_probs[TX_SIZES - 1][TX_SIZE_CONTEXTS][TX_SIZES - 1];
+  aom_prob tx_size_probs[MAX_TX_DEPTH][TX_SIZE_CONTEXTS][MAX_TX_DEPTH];
 #if CONFIG_VAR_TX
   aom_prob txfm_partition_prob[TXFM_PARTITION_CONTEXTS];
 #if CONFIG_EXT_TX && CONFIG_RECT_TX
@@ -237,7 +237,7 @@ typedef struct FRAME_COUNTS {
   // to use forward updates for the coeff probs, and as such it does not really
   // belong into this structure.
   unsigned int tx_size_totals[TX_SIZES];
-  unsigned int tx_size[TX_SIZES - 1][TX_SIZE_CONTEXTS][TX_SIZES];
+  unsigned int tx_size[MAX_TX_DEPTH][TX_SIZE_CONTEXTS][TX_SIZES];
 #if CONFIG_VAR_TX
   unsigned int txfm_partition[TXFM_PARTITION_CONTEXTS][2];
 #if CONFIG_EXT_TX && CONFIG_RECT_TX

av1/common/enums.h

@@ -148,6 +148,8 @@ typedef enum ATTRIBUTE_PACKED {
   TX_INVALID = 255          // Invalid transform size
 } TX_SIZE;
 
+#define MAX_TX_DEPTH (TX_32X32 - TX_4X4)
+
 #define MAX_TX_SIZE_LOG2 5
 #define MAX_TX_SIZE (1 << MAX_TX_SIZE_LOG2)
 #define MIN_TX_SIZE_LOG2 2

av1/decoder/decodeframe.c

@@ -3661,7 +3661,7 @@ static int read_compressed_header(AV1Decoder *pbi, const uint8_t *data,
 #endif
 
   if (cm->tx_mode == TX_MODE_SELECT) {
-    for (i = 0; i < TX_SIZES - 1; ++i)
+    for (i = 0; i < MAX_TX_DEPTH; ++i)
       for (j = 0; j < TX_SIZE_CONTEXTS; ++j)
         for (k = 0; k < i + 1; ++k)
           av1_diff_update_prob(&r, &fc->tx_size_probs[i][j][k], ACCT_STR);

av1/encoder/bitstream.c

@@ -74,7 +74,7 @@ static struct av1_token palette_size_encodings[PALETTE_MAX_SIZE - 1];
 static struct av1_token palette_color_encodings[PALETTE_MAX_SIZE - 1]
                                                [PALETTE_MAX_SIZE];
 #endif  // CONFIG_PALETTE
-static const struct av1_token tx_size_encodings[TX_SIZES - 1][TX_SIZES] = {
+static const struct av1_token tx_size_encodings[MAX_TX_DEPTH][TX_SIZES] = {
   { { 0, 1 }, { 1, 1 } },                      // Max tx_size is 8X8
   { { 0, 1 }, { 2, 2 }, { 3, 2 } },            // Max tx_size is 16X16
   { { 0, 1 }, { 2, 2 }, { 6, 3 }, { 7, 3 } },  // Max tx_size is 32X32
@@ -2904,7 +2904,7 @@ static void update_txfm_probs(AV1_COMMON *cm, aom_writer *w,
                               FRAME_COUNTS *counts) {
   if (cm->tx_mode == TX_MODE_SELECT) {
     int i, j;
-    for (i = 0; i < TX_SIZES - 1; ++i)
+    for (i = 0; i < MAX_TX_DEPTH; ++i)
       for (j = 0; j < TX_SIZE_CONTEXTS; ++j)
         prob_diff_update(av1_tx_size_tree[i], cm->fc->tx_size_probs[i][j],
                          counts->tx_size[i][j], i + 2, w);

av1/encoder/rd.c

@@ -110,7 +110,7 @@ static void fill_mode_costs(AV1_COMP *cpi) {
   }
 #endif  // CONFIG_PALETTE
 
-  for (i = 0; i < TX_SIZES - 1; ++i)
+  for (i = 0; i < MAX_TX_DEPTH; ++i)
     for (j = 0; j < TX_SIZE_CONTEXTS; ++j)
       av1_cost_tokens(cpi->tx_size_cost[i][j], fc->tx_size_probs[i][j],
                       av1_tx_size_tree[i]);