Commit fe0fb1d9 authored by Timothy B. Terriberry's avatar Timothy B. Terriberry Committed by Tim Terriberry

cb4x4: Fix indexing of tx_size counts.

Create named constants for the minimum TX_SIZE used in the luma
plane, as well as the minimum allowed TX_SIZE for which we have to
explicitly code a selection (i.e., one larger than the minimum luma
TX_SIZE).

Then, use these constants consistently with tx_size_to_depth() to
index the tx_size counts in FRAME_COUNTS, and also consistently use
TX_SIZE named constants to index the tx_size_implied counts.

Failing to index these counts correctly can, e.g., cause tx_mode to
be chosen incorrectly.

Change-Id: I706a62a33e2282e67c97a68bade87fb8023ec13b
parent 2af277a4
......@@ -1096,11 +1096,11 @@ static INLINE TX_TYPE get_tx_type(PLANE_TYPE plane_type, const MACROBLOCKD *xd,
void av1_setup_block_planes(MACROBLOCKD *xd, int ss_x, int ss_y);
static INLINE int tx_size_to_depth(TX_SIZE tx_size) {
return (int)(tx_size - TX_4X4);
return (int)(tx_size - TX_SIZE_LUMA_MIN);
}
static INLINE TX_SIZE depth_to_tx_size(int depth) {
return (TX_SIZE)(depth + TX_4X4);
return (TX_SIZE)(depth + TX_SIZE_LUMA_MIN);
}
static INLINE TX_SIZE get_uv_tx_size(const MB_MODE_INFO *mbmi,
......
......@@ -401,7 +401,7 @@ typedef struct FRAME_COUNTS {
// to use forward updates for the coeff probs, and as such it does not really
// belong into this structure.
unsigned int tx_size_totals[TX_SIZES];
unsigned int tx_size[MAX_TX_DEPTH][TX_SIZE_CONTEXTS][TX_SIZES];
unsigned int tx_size[MAX_TX_DEPTH][TX_SIZE_CONTEXTS][MAX_TX_DEPTH + 1];
#if CONFIG_EXT_TX && CONFIG_RECT_TX && CONFIG_RECT_TX_EXT
unsigned int quarter_tx_size[2];
#endif // CONFIG_EXT_TX && CONFIG_RECT_TX && CONFIG_RECT_TX_EXT
......
......@@ -175,7 +175,12 @@ typedef enum ATTRIBUTE_PACKED {
TX_INVALID = 255 // Invalid transform size
} TX_SIZE;
#define MAX_TX_DEPTH (TX_SIZES - 1 - TX_4X4)
#define TX_SIZE_LUMA_MIN (TX_4X4)
/* We don't need to code a transform size unless the allowed size is at least
one more than the minimum. */
#define TX_SIZE_CTX_MIN (TX_SIZE_LUMA_MIN + 1)
#define MAX_TX_DEPTH (TX_SIZES - TX_SIZE_CTX_MIN)
#define MAX_TX_SIZE_LOG2 (5 + CONFIG_TX64X64)
#define MAX_TX_SIZE (1 << MAX_TX_SIZE_LOG2)
......
......@@ -231,7 +231,9 @@ static void update_tx_counts(AV1_COMMON *cm, MACROBLOCKD *xd,
if (blk_row >= max_blocks_high || blk_col >= max_blocks_wide) return;
if (tx_size == plane_tx_size) {
++xd->counts->tx_size[max_tx_size - TX_8X8][ctx][tx_size];
int depth;
depth = tx_size_to_depth(tx_size);
++xd->counts->tx_size[max_tx_size - TX_SIZE_CTX_MIN][ctx][depth];
mbmi->tx_size = tx_size;
} else {
int bsl = b_width_log2_lookup[bsize];
......
......@@ -5528,41 +5528,50 @@ void av1_encode_frame(AV1_COMP *cpi) {
int count32x32_32x32p = 0, count32x32_lp = 0;
int count64x64_64x64p = 0;
for (i = 0; i < TX_SIZE_CONTEXTS; ++i) {
int depth;
// counts->tx_size[max_depth][context_idx][this_depth_level]
count4x4 += counts->tx_size[0][i][0];
count4x4 += counts->tx_size[1][i][0];
count4x4 += counts->tx_size[2][i][0];
count4x4 += counts->tx_size[3][i][0];
count8x8_8x8p += counts->tx_size[0][i][1];
count8x8_lp += counts->tx_size[1][i][1];
count8x8_lp += counts->tx_size[2][i][1];
count8x8_lp += counts->tx_size[3][i][1];
count16x16_16x16p += counts->tx_size[1][i][2];
count16x16_lp += counts->tx_size[2][i][2];
count16x16_lp += counts->tx_size[3][i][2];
count32x32_32x32p += counts->tx_size[2][i][3];
count32x32_lp += counts->tx_size[3][i][3];
count64x64_64x64p += counts->tx_size[3][i][4];
depth = tx_size_to_depth(TX_4X4);
count4x4 += counts->tx_size[TX_8X8 - TX_SIZE_CTX_MIN][i][depth];
count4x4 += counts->tx_size[TX_16X16 - TX_SIZE_CTX_MIN][i][depth];
count4x4 += counts->tx_size[TX_32X32 - TX_SIZE_CTX_MIN][i][depth];
count4x4 += counts->tx_size[TX_64X64 - TX_SIZE_CTX_MIN][i][depth];
depth = tx_size_to_depth(TX_8X8);
count8x8_8x8p += counts->tx_size[TX_8X8 - TX_SIZE_CTX_MIN][i][depth];
count8x8_lp += counts->tx_size[TX_16X16 - TX_SIZE_CTX_MIN][i][depth];
count8x8_lp += counts->tx_size[TX_32X32 - TX_SIZE_CTX_MIN][i][depth];
count8x8_lp += counts->tx_size[TX_64X64 - TX_SIZE_CTX_MIN][i][depth];
depth = tx_size_to_depth(TX_16X16);
count16x16_16x16p +=
counts->tx_size[TX_16X16 - TX_SIZE_CTX_MIN][i][depth];
count16x16_lp += counts->tx_size[TX_32X32 - TX_SIZE_CTX_MIN][i][depth];
count16x16_lp += counts->tx_size[TX_64X64 - TX_SIZE_CTX_MIN][i][depth];
depth = tx_size_to_depth(TX_32X32);
count32x32_32x32p +=
counts->tx_size[TX_32X32 - TX_SIZE_CTX_MIN][i][depth];
count32x32_lp += counts->tx_size[TX_64X64 - TX_SIZE_CTX_MIN][i][depth];
depth = tx_size_to_depth(TX_64X64);
count64x64_64x64p +=
counts->tx_size[TX_64X64 - TX_SIZE_CTX_MIN][i][depth];
}
#if CONFIG_EXT_TX && CONFIG_RECT_TX
count4x4 += counts->tx_size_implied[0][TX_4X4];
count4x4 += counts->tx_size_implied[1][TX_4X4];
count4x4 += counts->tx_size_implied[2][TX_4X4];
count4x4 += counts->tx_size_implied[3][TX_4X4];
count8x8_8x8p += counts->tx_size_implied[1][TX_8X8];
count8x8_lp += counts->tx_size_implied[2][TX_8X8];
count8x8_lp += counts->tx_size_implied[3][TX_8X8];
count8x8_lp += counts->tx_size_implied[4][TX_8X8];
count16x16_16x16p += counts->tx_size_implied[2][TX_16X16];
count16x16_lp += counts->tx_size_implied[3][TX_16X16];
count16x16_lp += counts->tx_size_implied[4][TX_16X16];
count32x32_32x32p += counts->tx_size_implied[3][TX_32X32];
count32x32_lp += counts->tx_size_implied[4][TX_32X32];
count64x64_64x64p += counts->tx_size_implied[4][TX_64X64];
count4x4 += counts->tx_size_implied[TX_4X4][TX_4X4];
count4x4 += counts->tx_size_implied[TX_8X8][TX_4X4];
count4x4 += counts->tx_size_implied[TX_16X16][TX_4X4];
count4x4 += counts->tx_size_implied[TX_32X32][TX_4X4];
count8x8_8x8p += counts->tx_size_implied[TX_8X8][TX_8X8];
count8x8_lp += counts->tx_size_implied[TX_16X16][TX_8X8];
count8x8_lp += counts->tx_size_implied[TX_32X32][TX_8X8];
count8x8_lp += counts->tx_size_implied[TX_64X64][TX_8X8];
count16x16_16x16p += counts->tx_size_implied[TX_16X16][TX_16X16];
count16x16_lp += counts->tx_size_implied[TX_32X32][TX_16X16];
count16x16_lp += counts->tx_size_implied[TX_64X64][TX_16X16];
count32x32_32x32p += counts->tx_size_implied[TX_32X32][TX_32X32];
count32x32_lp += counts->tx_size_implied[TX_64X64][TX_32X32];
count64x64_64x64p += counts->tx_size_implied[TX_64X64][TX_64X64];
#endif // CONFIG_EXT_TX && CONFIG_RECT_TX
if (count4x4 == 0 && count16x16_lp == 0 && count16x16_16x16p == 0 &&
count32x32_lp == 0 && count32x32_32x32p == 0 &&
......@@ -5614,30 +5623,37 @@ void av1_encode_frame(AV1_COMP *cpi) {
int count16x16_16x16p = 0, count16x16_lp = 0;
int count32x32 = 0;
for (i = 0; i < TX_SIZE_CONTEXTS; ++i) {
int depth;
// counts->tx_size[max_depth][context_idx][this_depth_level]
count4x4 += counts->tx_size[0][i][0];
count4x4 += counts->tx_size[1][i][0];
count4x4 += counts->tx_size[2][i][0];
count8x8_8x8p += counts->tx_size[0][i][1];
count8x8_lp += counts->tx_size[1][i][1];
count8x8_lp += counts->tx_size[2][i][1];
count16x16_16x16p += counts->tx_size[1][i][2];
count16x16_lp += counts->tx_size[2][i][2];
count32x32 += counts->tx_size[2][i][3];
depth = tx_size_to_depth(TX_4X4);
count4x4 += counts->tx_size[TX_8X8 - TX_SIZE_CTX_MIN][i][depth];
count4x4 += counts->tx_size[TX_16X16 - TX_SIZE_CTX_MIN][i][depth];
count4x4 += counts->tx_size[TX_32X32 - TX_SIZE_CTX_MIN][i][depth];
depth = tx_size_to_depth(TX_8X8);
count8x8_8x8p += counts->tx_size[TX_8X8 - TX_SIZE_CTX_MIN][i][depth];
count8x8_lp += counts->tx_size[TX_16X16 - TX_SIZE_CTX_MIN][i][depth];
count8x8_lp += counts->tx_size[TX_32X32 - TX_SIZE_CTX_MIN][i][depth];
depth = tx_size_to_depth(TX_16X16);
count16x16_16x16p +=
counts->tx_size[TX_16X16 - TX_SIZE_CTX_MIN][i][depth];
count16x16_lp += counts->tx_size[TX_32X32 - TX_SIZE_CTX_MIN][i][depth];
depth = tx_size_to_depth(TX_32X32);
count32x32 += counts->tx_size[TX_32X32 - TX_SIZE_CTX_MIN][i][depth];
}
#if CONFIG_EXT_TX && CONFIG_RECT_TX
count4x4 += counts->tx_size_implied[0][TX_4X4];
count4x4 += counts->tx_size_implied[1][TX_4X4];
count4x4 += counts->tx_size_implied[2][TX_4X4];
count4x4 += counts->tx_size_implied[3][TX_4X4];
count8x8_8x8p += counts->tx_size_implied[1][TX_8X8];
count8x8_lp += counts->tx_size_implied[2][TX_8X8];
count8x8_lp += counts->tx_size_implied[3][TX_8X8];
count16x16_lp += counts->tx_size_implied[3][TX_16X16];
count16x16_16x16p += counts->tx_size_implied[2][TX_16X16];
count32x32 += counts->tx_size_implied[3][TX_32X32];
count4x4 += counts->tx_size_implied[TX_4X4][TX_4X4];
count4x4 += counts->tx_size_implied[TX_8X8][TX_4X4];
count4x4 += counts->tx_size_implied[TX_16X16][TX_4X4];
count4x4 += counts->tx_size_implied[TX_32X32][TX_4X4];
count8x8_8x8p += counts->tx_size_implied[TX_8X8][TX_8X8];
count8x8_lp += counts->tx_size_implied[TX_16X16][TX_8X8];
count8x8_lp += counts->tx_size_implied[TX_32X32][TX_8X8];
count16x16_16x16p += counts->tx_size_implied[TX_16X16][TX_16X16];
count16x16_lp += counts->tx_size_implied[TX_32X32][TX_16X16];
count32x32 += counts->tx_size_implied[TX_32X32][TX_32X32];
#endif // CONFIG_EXT_TX && CONFIG_RECT_TX
if (count4x4 == 0 && count16x16_lp == 0 && count16x16_16x16p == 0 &&
#if CONFIG_SUPERTX
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment