Add default 64x64 scan orders and entropy models

Adds 64x64 default scan orders
Adds 64x64 coefficient entropy models (just copy the 32x32 ones)
Entropy context updates for 64x64 transform
Various misc. changes to support 64x64 transforms

Change-Id: I2c0bc4ba540886dd196e87a78b205407ad3e866b

av1/common/entropy.h

@@ -154,7 +154,8 @@ void av1_partial_adapt_probs(struct AV1Common *cm, int mi_row, int mi_col);
 // This macro is currently unused but may be used by certain implementations
 #define MAXBAND_INDEX 21
 
-DECLARE_ALIGNED(16, extern const uint8_t, av1_coefband_trans_8x8plus[1024]);
+DECLARE_ALIGNED(16, extern const uint8_t,
+                av1_coefband_trans_8x8plus[MAX_TX_SQUARE]);
 #if CONFIG_EXT_TX
 DECLARE_ALIGNED(16, extern const uint8_t, av1_coefband_trans_4x8_8x4[32]);
 #endif  // CONFIG_EXT_TX
@@ -258,6 +259,12 @@ static INLINE int get_entropy_context(TX_SIZE tx_size, const ENTROPY_CONTEXT *a,
       above_ec = !!*(const uint64_t *)a;
       left_ec = !!*(const uint64_t *)l;
       break;
+#if CONFIG_TX64X64
+    case TX_64X64:
+      above_ec = !!(*(const uint64_t *)a | *(const uint64_t *)(a + 8));
+      left_ec = !!(*(const uint64_t *)l | *(const uint64_t *)(l + 8));
+      break;
+#endif  // CONFIG_TX64X64
     default: assert(0 && "Invalid transform size."); break;
   }
   return combine_entropy_contexts(above_ec, left_ec);

av1/common/enums.h

@@ -157,7 +157,7 @@ typedef enum ATTRIBUTE_PACKED {
   TX_INVALID = 255    // Invalid transform size
 } TX_SIZE;
 
-#define MAX_TX_DEPTH (TX_32X32 - TX_4X4)
+#define MAX_TX_DEPTH (TX_SIZES - 1 - TX_4X4)
 
 #define MAX_TX_SIZE_LOG2 (5 + CONFIG_TX64X64)
 #define MAX_TX_SIZE (1 << MAX_TX_SIZE_LOG2)
@@ -419,7 +419,7 @@ typedef enum {
 #define REF_CONTEXTS 5
 
 #if CONFIG_VAR_TX
-#define TXFM_PARTITION_CONTEXTS 16
+#define TXFM_PARTITION_CONTEXTS ((TX_SIZES - TX_8X8) * 6 - 2)
 typedef uint8_t TXFM_CONTEXT;
 #endif
 

av1/common/onyxc_int.h

@@ -810,24 +810,13 @@ static INLINE int txfm_partition_context(TXFM_CONTEXT *above_ctx,
   const int above = *above_ctx < txw;
   const int left = *left_ctx < txh;
   TX_SIZE max_tx_size = max_txsize_lookup[bsize];
-  int category = 15;
-
-  if (max_tx_size == TX_32X32) {
-    if (tx_size == TX_32X32)
-      category = 0;
-    else
-      category = 1;
-  } else if (max_tx_size == TX_16X16) {
-    if (tx_size == TX_16X16)
-      category = 2;
-    else
-      category = 3;
-  } else if (max_tx_size == TX_8X8) {
-    category = 4;
-  }
-
-  if (category == 15) return category;
+  int category = TXFM_PARTITION_CONTEXTS - 1;
 
+  if (max_tx_size >= TX_8X8) {
+    category = (tx_size != max_tx_size && max_tx_size > TX_8X8) +
+               (TX_SIZES - 1 - tx_size) * 2;
+  }
+  if (category == TXFM_PARTITION_CONTEXTS - 1) return category;
   return category * 3 + above + left;
 }
 #endif

av1/decoder/decodeframe.c

@@ -318,7 +318,7 @@ static int av1_pvq_decode_helper(od_dec_ctx *dec, int16_t *ref_coeff,
   int off;
   const int is_keyframe = 0;
   const int has_dc_skip = 1;
-  int quant_shift = bs == TX_32X32 ? 1 : 0;
+  int quant_shift = get_tx_scale(bs);
   // DC quantizer for PVQ
   int pvq_dc_quant;
   int lossless = (quant[0] == 0);

av1/encoder/encodeframe.c

@@ -4400,7 +4400,7 @@ static void encode_rd_sb_row(AV1_COMP *cpi, ThreadData *td,
       TX_SIZE t;
       SUBFRAME_STATS *subframe_stats = &cpi->subframe_stats;
 
-      for (t = TX_4X4; t <= TX_32X32; ++t)
+      for (t = TX_4X4; t < TX_SIZES; ++t)
         av1_full_to_model_counts(cpi->td.counts->coef[t],
                                  cpi->td.rd_counts.coef_counts[t]);
       av1_partial_adapt_probs(cm, mi_row, mi_col);

av1/encoder/encodemb.c

@@ -1431,7 +1431,7 @@ int av1_pvq_encode_helper(daala_enc_ctx *daala_enc, tran_low_t *const coeff,
   // int pvq_dc_quant = OD_MAXI(1,
   //  quant * daala_enc->state.pvq_qm_q4[plane][od_qm_get_index(tx_size, 0)] >>
   //  4);
-  int quant_shift = tx_size == TX_32X32 ? 1 : 0;
+  int quant_shift = get_tx_scale(tx_size);
   // DC quantizer for PVQ
   int pvq_dc_quant = OD_MAXI(1, quant[0] >> quant_shift);
   int tell;

av1/encoder/encoder.c

@@ -4740,7 +4740,7 @@ static void encode_frame_to_data_rate(AV1_COMP *cpi, size_t *size,
 
   av1_update_reference_frames(cpi);
 
-  for (t = TX_4X4; t <= TX_32X32; t++)
+  for (t = TX_4X4; t < TX_SIZES; t++)
     av1_full_to_model_counts(cpi->td.counts->coef[t],
                              cpi->td.rd_counts.coef_counts[t]);
 

av1/encoder/rd.c

@@ -159,7 +159,7 @@ void av1_fill_token_costs(av1_coeff_cost *c,
                           av1_coeff_probs_model (*p)[PLANE_TYPES]) {
   int i, j, k, l;
   TX_SIZE t;
-  for (t = TX_4X4; t <= TX_32X32; ++t)
+  for (t = TX_4X4; t < TX_SIZES; ++t)
     for (i = 0; i < PLANE_TYPES; ++i)
       for (j = 0; j < REF_TYPES; ++j)
         for (k = 0; k < COEF_BANDS; ++k)
@@ -571,6 +571,16 @@ static void get_entropy_contexts_plane(
       for (i = 0; i < num_4x4_h; i += 8)
         t_left[i] = !!*(const uint64_t *)&left[i];
       break;
+#if CONFIG_TX64X64
+    case TX_64X64:
+      for (i = 0; i < num_4x4_w; i += 16)
+        t_above[i] =
+            !!(*(const uint64_t *)&above[i] | *(const uint64_t *)&above[i + 8]);
+      for (i = 0; i < num_4x4_h; i += 16)
+        t_left[i] = !!(*(const uint64_t *)&left[i] |
+                       *(const uint64_t *)&left[i + 8];
+      break;
+#endif  // CONFIG_TX64X64
     case TX_4X8:
       memcpy(t_above, above, sizeof(ENTROPY_CONTEXT) * num_4x4_w);
       for (i = 0; i < num_4x4_h; i += 2)

av1/encoder/rdopt.c

@@ -1460,7 +1460,7 @@ static int64_t choose_tx_size_fix_type(const AV1_COMP *const cpi, BLOCK_SIZE bs,
     evaluate_rect_tx = is_rect_tx_allowed(xd, mbmi);
 #endif  // CONFIG_EXT_TX && CONFIG_RECT_TX
     start_tx = max_tx_size;
-    end_tx = (max_tx_size == TX_32X32) ? TX_8X8 : TX_4X4;
+    end_tx = (max_tx_size >= TX_32X32) ? TX_8X8 : TX_4X4;
   } else {
     const TX_SIZE chosen_tx_size =
         tx_size_from_tx_mode(bs, cm->tx_mode, is_inter);
@@ -1504,7 +1504,7 @@ static int64_t choose_tx_size_fix_type(const AV1_COMP *const cpi, BLOCK_SIZE bs,
     if (is_inter && x->use_default_inter_tx_type &&
         tx_type != get_default_tx_type(0, xd, 0, n))
       continue;
-    if (max_tx_size == TX_32X32 && n == TX_4X4) continue;
+    if (max_tx_size >= TX_32X32 && n == TX_4X4) continue;
 #if CONFIG_EXT_TX
     ext_tx_set = get_ext_tx_set(n, bs, is_inter);
     if (is_inter) {

av1/encoder/speed_features.c

@@ -162,6 +162,10 @@ static void set_good_speed_feature(AV1_COMP *cpi, AV1_COMMON *cm,
     sf->adaptive_pred_interp_filter = 1;
 
     sf->recode_loop = ALLOW_RECODE_KFARFGF;
+#if CONFIG_TX64X64
+    sf->intra_y_mode_mask[TX_64X64] = INTRA_DC_H_V;
+    sf->intra_uv_mode_mask[TX_64X64] = INTRA_DC_H_V;
+#endif  // CONFIG_TX64X64
     sf->intra_y_mode_mask[TX_32X32] = INTRA_DC_H_V;
     sf->intra_uv_mode_mask[TX_32X32] = INTRA_DC_H_V;
     sf->intra_y_mode_mask[TX_16X16] = INTRA_DC_H_V;
@@ -211,6 +215,10 @@ static void set_good_speed_feature(AV1_COMP *cpi, AV1_COMMON *cm,
     sf->recode_loop = ALLOW_RECODE_KFMAXBW;
     sf->adaptive_rd_thresh = 3;
     sf->mode_skip_start = 6;
+#if CONFIG_TX64X64
+    sf->intra_y_mode_mask[TX_64X64] = INTRA_DC;
+    sf->intra_uv_mode_mask[TX_64X64] = INTRA_DC;
+#endif  // CONFIG_TX64X64
     sf->intra_y_mode_mask[TX_32X32] = INTRA_DC;
     sf->intra_uv_mode_mask[TX_32X32] = INTRA_DC;
     sf->adaptive_interp_filter_search = 1;
@@ -309,6 +317,10 @@ static void set_rt_speed_feature(AV1_COMP *cpi, SPEED_FEATURES *sf, int speed,
     sf->adaptive_pred_interp_filter = 1;
     sf->mv.auto_mv_step_size = 1;
     sf->adaptive_rd_thresh = 2;
+#if CONFIG_TX64X64
+    sf->intra_y_mode_mask[TX_64X64] = INTRA_DC_H_V;
+    sf->intra_uv_mode_mask[TX_64X64] = INTRA_DC_H_V;
+#endif  // CONFIG_TX64X64
     sf->intra_y_mode_mask[TX_32X32] = INTRA_DC_H_V;
     sf->intra_uv_mode_mask[TX_32X32] = INTRA_DC_H_V;
     sf->intra_uv_mode_mask[TX_16X16] = INTRA_DC_H_V;
@@ -357,6 +369,9 @@ static void set_rt_speed_feature(AV1_COMP *cpi, SPEED_FEATURES *sf, int speed,
       sf->intra_y_mode_mask[i] = INTRA_DC_H_V;
       sf->intra_uv_mode_mask[i] = INTRA_DC;
     }
+#if CONFIG_TX64X64
+    sf->intra_y_mode_mask[TX_64X64] = INTRA_DC;
+#endif  // CONFIG_TX64X64
     sf->intra_y_mode_mask[TX_32X32] = INTRA_DC;
     sf->frame_parameter_update = 0;
     sf->mv.search_method = FAST_HEX;