Commit feb925fe authored by Urvang Joshi's avatar Urvang Joshi

Enable rectangular transforms for Intra also.

These are under EXT_TX + RECT_TX experiment combo.

Results
=======

Derf Set:
--------
All Intra frames: 1.8% avg improvement (and 1.78% BD-rate improvement)
Video: 0.230% avg improvement (and 0.262% BD-rate improvement)

Objective-1-fast set
--------------------
Video: 0.52 PSNR improvement

Change-Id: I1893465929858e38419f327752dc61c19b96b997
parent 60a055bd
......@@ -688,7 +688,7 @@ static INLINE int is_rect_tx_allowed_bsize(BLOCK_SIZE bsize) {
static INLINE int is_rect_tx_allowed(const MACROBLOCKD *xd,
const MB_MODE_INFO *mbmi) {
return is_inter_block(mbmi) && is_rect_tx_allowed_bsize(mbmi->sb_type) &&
return is_rect_tx_allowed_bsize(mbmi->sb_type) &&
!xd->lossless[mbmi->segment_id];
}
......@@ -699,40 +699,33 @@ static INLINE int is_rect_tx(TX_SIZE tx_size) { return tx_size >= TX_SIZES; }
static INLINE TX_SIZE tx_size_from_tx_mode(BLOCK_SIZE bsize, TX_MODE tx_mode,
int is_inter) {
const TX_SIZE largest_tx_size = tx_mode_to_biggest_tx_size[tx_mode];
#if CONFIG_VAR_TX || (CONFIG_EXT_TX && CONFIG_RECT_TX)
const TX_SIZE max_rect_tx_size = max_txsize_rect_lookup[bsize];
#else
const TX_SIZE max_tx_size = max_txsize_lookup[bsize];
#endif // CONFIG_VAR_TX || (CONFIG_EXT_TX && CONFIG_RECT_TX)
(void)is_inter;
#if CONFIG_VAR_TX
const TX_SIZE max_tx_size = max_txsize_rect_lookup[bsize];
#if CONFIG_CB4X4
if (!is_inter || bsize == BLOCK_4X4)
if (bsize == BLOCK_4X4)
return AOMMIN(max_txsize_lookup[bsize], largest_tx_size);
#else
if (!is_inter || bsize < BLOCK_8X8)
if (bsize < BLOCK_8X8)
return AOMMIN(max_txsize_lookup[bsize], largest_tx_size);
#endif
if (txsize_sqr_map[max_tx_size] <= largest_tx_size)
return max_tx_size;
if (txsize_sqr_map[max_rect_tx_size] <= largest_tx_size)
return max_rect_tx_size;
else
return largest_tx_size;
#else
const TX_SIZE max_tx_size = max_txsize_lookup[bsize];
#endif
#if CONFIG_EXT_TX && CONFIG_RECT_TX
if (!is_inter) {
return AOMMIN(max_tx_size, largest_tx_size);
} else {
const TX_SIZE max_rect_tx_size = max_txsize_rect_lookup[bsize];
#elif CONFIG_EXT_TX && CONFIG_RECT_TX
if (txsize_sqr_up_map[max_rect_tx_size] <= largest_tx_size) {
return max_rect_tx_size;
} else {
return largest_tx_size;
}
}
#else
(void)is_inter;
return AOMMIN(max_tx_size, largest_tx_size);
#endif // CONFIG_EXT_TX && CONFIG_RECT_TX
#endif // CONFIG_VAR_TX
}
#if CONFIG_FILTER_INTRA
......
......@@ -487,7 +487,10 @@ static const TX_SIZE max_txsize_rect_lookup[BLOCK_SIZES] = {
#endif // CONFIG_TX64X64
};
// Same as "max_txsize_lookup[bsize] - TX_8X8", invalid for bsize < 8X8
#if CONFIG_EXT_TX && CONFIG_RECT_TX
// Same as "max_txsize_lookup[bsize] - TX_8X8", except for rectangular
// block which may use a rectangular transform, in which case it is
// "(max_txsize_lookup[bsize] + 1) - TX_8X8", invalid for bsize < 8X8
static const int32_t intra_tx_size_cat_lookup[BLOCK_SIZES] = {
#if CONFIG_CB4X4
// 2X2, 2X4, 4X2,
......@@ -498,9 +501,9 @@ static const int32_t intra_tx_size_cat_lookup[BLOCK_SIZES] = {
// 4X8, 8X4, 8X8
INT32_MIN, INT32_MIN, TX_8X8 - TX_8X8,
// 8X16, 16X8, 16X16
TX_8X8 - TX_8X8, TX_8X8 - TX_8X8, TX_16X16 - TX_8X8,
TX_16X16 - TX_8X8, TX_16X16 - TX_8X8, TX_16X16 - TX_8X8,
// 16X32, 32X16, 32X32
TX_16X16 - TX_8X8, TX_16X16 - TX_8X8, TX_32X32 - TX_8X8,
TX_32X32 - TX_8X8, TX_32X32 - TX_8X8, TX_32X32 - TX_8X8,
// 32X64, 64X32,
TX_32X32 - TX_8X8, TX_32X32 - TX_8X8,
#if CONFIG_TX64X64
......@@ -519,12 +522,9 @@ static const int32_t intra_tx_size_cat_lookup[BLOCK_SIZES] = {
#endif // CONFIG_EXT_PARTITION
#endif // CONFIG_TX64X64
};
#if CONFIG_EXT_TX && CONFIG_RECT_TX
// Same as "max_txsize_lookup[bsize] - TX_8X8", except for rectangular
// block which may use a rectangular transform, in which case it is
// "(max_txsize_lookup[bsize] + 1) - TX_8X8", invalid for bsize < 8X8
static const int32_t inter_tx_size_cat_lookup[BLOCK_SIZES] = {
#else
// Same as "max_txsize_lookup[bsize] - TX_8X8", invalid for bsize < 8X8
static const int32_t intra_tx_size_cat_lookup[BLOCK_SIZES] = {
#if CONFIG_CB4X4
// 2X2, 2X4, 4X2,
INT32_MIN, INT32_MIN, INT32_MIN,
......@@ -534,9 +534,9 @@ static const int32_t inter_tx_size_cat_lookup[BLOCK_SIZES] = {
// 4X8, 8X4, 8X8
INT32_MIN, INT32_MIN, TX_8X8 - TX_8X8,
// 8X16, 16X8, 16X16
TX_16X16 - TX_8X8, TX_16X16 - TX_8X8, TX_16X16 - TX_8X8,
TX_8X8 - TX_8X8, TX_8X8 - TX_8X8, TX_16X16 - TX_8X8,
// 16X32, 32X16, 32X32
TX_32X32 - TX_8X8, TX_32X32 - TX_8X8, TX_32X32 - TX_8X8,
TX_16X16 - TX_8X8, TX_16X16 - TX_8X8, TX_32X32 - TX_8X8,
// 32X64, 64X32,
TX_32X32 - TX_8X8, TX_32X32 - TX_8X8,
#if CONFIG_TX64X64
......@@ -555,10 +555,10 @@ static const int32_t inter_tx_size_cat_lookup[BLOCK_SIZES] = {
#endif // CONFIG_EXT_PARTITION
#endif // CONFIG_TX64X64
};
#else
#define inter_tx_size_cat_lookup intra_tx_size_cat_lookup
#endif // CONFIG_EXT_TX && CONFIG_RECT_TX
#define inter_tx_size_cat_lookup intra_tx_size_cat_lookup
/* clang-format on */
static const TX_SIZE sub_tx_size_map[TX_SIZES_ALL] = {
......
......@@ -3534,6 +3534,9 @@ static void adapt_coef_probs(AV1_COMMON *cm, TX_SIZE tx_size,
(const unsigned int(*)[REF_TYPES][COEF_BANDS]
[COEFF_CONTEXTS])cm->counts.eob_branch[tx_size];
int i, j, k, l, m;
#if CONFIG_RECT_TX
assert(!is_rect_tx(tx_size));
#endif // CONFIG_RECT_TX
for (i = 0; i < PLANE_TYPES; ++i)
for (j = 0; j < REF_TYPES; ++j)
......
......@@ -2526,6 +2526,7 @@ static void combine_interintra_highbd(
}
#endif // CONFIG_AOM_HIGHBITDEPTH
// TODO(urvang/davidbarker): Refactor with av1_predict_intra_block().
static void build_intra_predictors_for_interintra(MACROBLOCKD *xd, uint8_t *ref,
int ref_stride, uint8_t *dst,
int dst_stride,
......
......@@ -1387,6 +1387,8 @@ static void build_intra_predictors_high(
filter_intra_mode_info->filter_intra_mode[plane != 0];
#endif // CONFIG_FILTER_INTRA
int base = 128 << (xd->bd - 8);
assert(tx_size_wide[tx_size] == tx_size_high[tx_size]);
// 127 127 127 .. 127 127 127 127 127 127
// 129 A B .. Y Z
// 129 C D .. W X
......@@ -1552,6 +1554,7 @@ static void build_intra_predictors(const MACROBLOCKD *xd, const uint8_t *ref,
const FILTER_INTRA_MODE filter_intra_mode =
filter_intra_mode_info->filter_intra_mode[plane != 0];
#endif // CONFIG_FILTER_INTRA
assert(tx_size_wide[tx_size] == tx_size_high[tx_size]);
// 127 127 127 .. 127 127 127 127 127 127
// 129 A B .. Y Z
......@@ -1687,11 +1690,11 @@ static void build_intra_predictors(const MACROBLOCKD *xd, const uint8_t *ref,
}
}
void av1_predict_intra_block(const MACROBLOCKD *xd, int wpx, int hpx,
static void predict_square_intra_block(const MACROBLOCKD *xd, int wpx, int hpx,
TX_SIZE tx_size, PREDICTION_MODE mode,
const uint8_t *ref, int ref_stride, uint8_t *dst,
int dst_stride, int col_off, int row_off,
int plane) {
const uint8_t *ref, int ref_stride,
uint8_t *dst, int dst_stride,
int col_off, int row_off, int plane) {
const BLOCK_SIZE bsize = xd->mi[0]->mbmi.sb_type;
const struct macroblockd_plane *const pd = &xd->plane[plane];
const int txw = tx_size_wide_unit[tx_size];
......@@ -1725,6 +1728,7 @@ void av1_predict_intra_block(const MACROBLOCKD *xd, int wpx, int hpx,
tx_size, row_off, col_off, pd->subsampling_x);
const int have_bottom = av1_has_bottom(bsize, mi_row, mi_col, yd > 0, tx_size,
row_off, col_off, pd->subsampling_y);
assert(txwpx == txhpx);
#if CONFIG_PALETTE
if (xd->mi[0]->mbmi.palette_mode_info.palette_size[plane != 0] > 0) {
......@@ -1782,6 +1786,142 @@ void av1_predict_intra_block(const MACROBLOCKD *xd, int wpx, int hpx,
plane);
}
void av1_predict_intra_block(const MACROBLOCKD *xd, int wpx, int hpx,
TX_SIZE tx_size, PREDICTION_MODE mode,
const uint8_t *ref, int ref_stride, uint8_t *dst,
int dst_stride, int col_off, int row_off,
int plane) {
const int tx_width = tx_size_wide[tx_size];
const int tx_height = tx_size_high[tx_size];
if (tx_width == tx_height) {
predict_square_intra_block(xd, wpx, hpx, tx_size, mode, ref, ref_stride,
dst, dst_stride, col_off, row_off, plane);
} else {
#if CONFIG_EXT_TX && CONFIG_RECT_TX
#if CONFIG_AOM_HIGHBITDEPTH
uint16_t tmp16[MAX_SB_SIZE];
#endif
uint8_t tmp[MAX_SB_SIZE];
const TX_SIZE sub_tx_size = txsize_sqr_map[tx_size];
assert(sub_tx_size < TX_SIZES);
assert((tx_width == wpx && tx_height == hpx) ||
(tx_width == (wpx >> 1) && tx_height == hpx) ||
(tx_width == wpx && tx_height == (hpx >> 1)));
if (tx_width < tx_height) {
assert(tx_height == (tx_width << 1));
// Predict the top square sub-block.
predict_square_intra_block(xd, wpx, hpx, sub_tx_size, mode, ref,
ref_stride, dst, dst_stride, col_off, row_off,
plane);
{
const int half_tx_height = tx_height >> 1;
const int half_txh_unit = tx_size_high_unit[tx_size] >> 1;
// Cast away const to modify 'ref' temporarily; will be restored later.
uint8_t *src_2 = (uint8_t *)ref + half_tx_height * ref_stride;
uint8_t *dst_2 = dst + half_tx_height * dst_stride;
const int row_off_2 = row_off + half_txh_unit;
// Save the last row of top square sub-block as 'above' row for bottom
// square sub-block.
if (src_2 != dst_2 || ref_stride != dst_stride) {
#if CONFIG_AOM_HIGHBITDEPTH
if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) {
uint16_t *src_2_16 = CONVERT_TO_SHORTPTR(src_2);
uint16_t *dst_2_16 = CONVERT_TO_SHORTPTR(dst_2);
memcpy(tmp16, src_2_16 - ref_stride, tx_width * sizeof(*src_2_16));
memcpy(src_2_16 - ref_stride, dst_2_16 - dst_stride,
tx_width * sizeof(*src_2_16));
} else {
#endif // CONFIG_AOM_HIGHBITDEPTH
memcpy(tmp, src_2 - ref_stride, tx_width * sizeof(*src_2));
memcpy(src_2 - ref_stride, dst_2 - dst_stride,
tx_width * sizeof(*src_2));
#if CONFIG_AOM_HIGHBITDEPTH
}
#endif // CONFIG_AOM_HIGHBITDEPTH
}
// Predict the bottom square sub-block.
predict_square_intra_block(xd, wpx, hpx, sub_tx_size, mode, src_2,
ref_stride, dst_2, dst_stride, col_off,
row_off_2, plane);
// Restore the last row of top square sub-block.
if (src_2 != dst_2 || ref_stride != dst_stride) {
#if CONFIG_AOM_HIGHBITDEPTH
if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) {
uint16_t *src_2_16 = CONVERT_TO_SHORTPTR(src_2);
memcpy(src_2_16 - ref_stride, tmp16, tx_width * sizeof(*src_2_16));
} else {
#endif // CONFIG_AOM_HIGHBITDEPTH
memcpy(src_2 - ref_stride, tmp, tx_width * sizeof(*src_2));
#if CONFIG_AOM_HIGHBITDEPTH
}
#endif // CONFIG_AOM_HIGHBITDEPTH
}
}
} else { // tx_width > tx_height
assert(tx_width == (tx_height << 1));
// Predict the left square sub-block
predict_square_intra_block(xd, wpx, hpx, sub_tx_size, mode, ref,
ref_stride, dst, dst_stride, col_off, row_off,
plane);
{
int i;
const int half_tx_width = tx_width >> 1;
const int half_txw_unit = tx_size_wide_unit[tx_size] >> 1;
// Cast away const to modify 'ref' temporarily; will be restored later.
uint8_t *src_2 = (uint8_t *)ref + half_tx_width;
uint8_t *dst_2 = dst + half_tx_width;
const int col_off_2 = col_off + half_txw_unit;
// Save the last column of left square sub-block as 'left' column for
// right square sub-block.
if (src_2 != dst_2 || ref_stride != dst_stride) {
#if CONFIG_AOM_HIGHBITDEPTH
if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) {
uint16_t *src_2_16 = CONVERT_TO_SHORTPTR(src_2);
uint16_t *dst_2_16 = CONVERT_TO_SHORTPTR(dst_2);
for (i = 0; i < tx_height; ++i) {
tmp16[i] = src_2_16[i * ref_stride - 1];
src_2_16[i * ref_stride - 1] = dst_2_16[i * dst_stride - 1];
}
} else {
#endif // CONFIG_AOM_HIGHBITDEPTH
for (i = 0; i < tx_height; ++i) {
tmp[i] = src_2[i * ref_stride - 1];
src_2[i * ref_stride - 1] = dst_2[i * dst_stride - 1];
}
#if CONFIG_AOM_HIGHBITDEPTH
}
#endif // CONFIG_AOM_HIGHBITDEPTH
}
// Predict the right square sub-block.
predict_square_intra_block(xd, wpx, hpx, sub_tx_size, mode, src_2,
ref_stride, dst_2, dst_stride, col_off_2,
row_off, plane);
// Restore the last column of left square sub-block.
if (src_2 != dst_2 || ref_stride != dst_stride) {
#if CONFIG_AOM_HIGHBITDEPTH
if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) {
uint16_t *src_2_16 = CONVERT_TO_SHORTPTR(src_2);
for (i = 0; i < tx_height; ++i) {
src_2_16[i * ref_stride - 1] = tmp16[i];
}
} else {
#endif // CONFIG_AOM_HIGHBITDEPTH
for (i = 0; i < tx_height; ++i) {
src_2[i * ref_stride - 1] = tmp[i];
}
#if CONFIG_AOM_HIGHBITDEPTH
}
#endif // CONFIG_AOM_HIGHBITDEPTH
}
}
}
#else
assert(0);
#endif // CONFIG_EXT_TX && CONFIG_RECT_TX
}
}
void av1_init_intra_predictors(void) {
once(av1_init_intra_predictors_internal);
}
This diff is collapsed.
......@@ -26,7 +26,7 @@ extern "C" {
#define MAX_NEIGHBORS 2
extern const SCAN_ORDER av1_default_scan_orders[TX_SIZES];
extern const SCAN_ORDER av1_intra_scan_orders[TX_SIZES][TX_TYPES];
extern const SCAN_ORDER av1_intra_scan_orders[TX_SIZES_ALL][TX_TYPES];
extern const SCAN_ORDER av1_inter_scan_orders[TX_SIZES_ALL][TX_TYPES];
#if CONFIG_ADAPT_SCAN
......
......@@ -466,7 +466,7 @@ static void predict_and_reconstruct_intra_block(AV1_COMMON *cm,
PREDICTION_MODE mode = (plane == 0) ? mbmi->mode : mbmi->uv_mode;
PLANE_TYPE plane_type = (plane == 0) ? PLANE_TYPE_Y : PLANE_TYPE_UV;
uint8_t *dst;
int block_idx = (row << 1) + col;
const int block_idx = (row << 1) + col;
#if CONFIG_PVQ
(void)cm;
(void)r;
......@@ -475,7 +475,7 @@ static void predict_and_reconstruct_intra_block(AV1_COMMON *cm,
#if !CONFIG_CB4X4
if (mbmi->sb_type < BLOCK_8X8)
if (plane == 0) mode = xd->mi[0]->bmi[(row << 1) + col].as_mode;
if (plane == 0) mode = xd->mi[0]->bmi[block_idx].as_mode;
#endif
av1_predict_intra_block(xd, pd->width, pd->height, tx_size, mode, dst,
......
......@@ -384,45 +384,32 @@ static TX_SIZE read_selected_tx_size(AV1_COMMON *cm, MACROBLOCKD *xd,
int tx_size_cat, aom_reader *r) {
FRAME_COUNTS *counts = xd->counts;
const int ctx = get_tx_size_context(xd);
int depth = aom_read_tree(r, av1_tx_size_tree[tx_size_cat],
const int depth =
aom_read_tree(r, av1_tx_size_tree[tx_size_cat],
cm->fc->tx_size_probs[tx_size_cat][ctx], ACCT_STR);
TX_SIZE tx_size = depth_to_tx_size(depth);
const TX_SIZE tx_size = depth_to_tx_size(depth);
#if CONFIG_RECT_TX
assert(!is_rect_tx(tx_size));
#endif // CONFIG_RECT_TX
if (counts) ++counts->tx_size[tx_size_cat][ctx][depth];
return tx_size;
}
static TX_SIZE read_tx_size_intra(AV1_COMMON *cm, MACROBLOCKD *xd,
aom_reader *r) {
TX_MODE tx_mode = cm->tx_mode;
BLOCK_SIZE bsize = xd->mi[0]->mbmi.sb_type;
if (xd->lossless[xd->mi[0]->mbmi.segment_id]) return TX_4X4;
if (bsize >= BLOCK_8X8) {
if (tx_mode == TX_MODE_SELECT) {
const TX_SIZE tx_size =
read_selected_tx_size(cm, xd, intra_tx_size_cat_lookup[bsize], r);
assert(tx_size <= max_txsize_lookup[bsize]);
return tx_size;
} else {
return tx_size_from_tx_mode(bsize, cm->tx_mode, 0);
}
} else {
return TX_4X4;
}
}
static TX_SIZE read_tx_size_inter(AV1_COMMON *cm, MACROBLOCKD *xd,
int allow_select, aom_reader *r) {
TX_MODE tx_mode = cm->tx_mode;
BLOCK_SIZE bsize = xd->mi[0]->mbmi.sb_type;
static TX_SIZE read_tx_size(AV1_COMMON *cm, MACROBLOCKD *xd, int is_inter,
int allow_select_inter, aom_reader *r) {
const TX_MODE tx_mode = cm->tx_mode;
const BLOCK_SIZE bsize = xd->mi[0]->mbmi.sb_type;
if (xd->lossless[xd->mi[0]->mbmi.segment_id]) return TX_4X4;
#if CONFIG_CB4X4 && CONFIG_VAR_TX
if (bsize > BLOCK_4X4) {
#else
if (bsize >= BLOCK_8X8) {
#endif
if (allow_select && tx_mode == TX_MODE_SELECT) {
#endif // CONFIG_CB4X4 && CONFIG_VAR_TX
if ((!is_inter || allow_select_inter) && tx_mode == TX_MODE_SELECT) {
const int32_t tx_size_cat = is_inter ? inter_tx_size_cat_lookup[bsize]
: intra_tx_size_cat_lookup[bsize];
const TX_SIZE coded_tx_size =
read_selected_tx_size(cm, xd, inter_tx_size_cat_lookup[bsize], r);
read_selected_tx_size(cm, xd, tx_size_cat, r);
#if CONFIG_EXT_TX && CONFIG_RECT_TX
if (coded_tx_size > max_txsize_lookup[bsize]) {
assert(coded_tx_size == max_txsize_lookup[bsize] + 1);
......@@ -433,7 +420,7 @@ static TX_SIZE read_tx_size_inter(AV1_COMMON *cm, MACROBLOCKD *xd,
#endif // CONFIG_EXT_TX && CONFIG_RECT_TX
return coded_tx_size;
} else {
return tx_size_from_tx_mode(bsize, cm->tx_mode, 1);
return tx_size_from_tx_mode(bsize, tx_mode, is_inter);
}
} else {
#if CONFIG_EXT_TX && CONFIG_RECT_TX
......@@ -441,7 +428,7 @@ static TX_SIZE read_tx_size_inter(AV1_COMMON *cm, MACROBLOCKD *xd,
return max_txsize_rect_lookup[bsize];
#else
return TX_4X4;
#endif
#endif // CONFIG_EXT_TX && CONFIG_RECT_TX
}
}
......@@ -711,6 +698,7 @@ static void read_tx_type(const AV1_COMMON *const cm, MACROBLOCKD *xd,
#endif
if (!FIXED_TX_TYPE) {
#if CONFIG_EXT_TX
const TX_SIZE square_tx_size = txsize_sqr_map[tx_size];
if (get_ext_tx_types(tx_size, mbmi->sb_type, inter_block) > 1 &&
cm->base_qindex > 0 && !mbmi->skip &&
#if CONFIG_SUPERTX
......@@ -724,19 +712,19 @@ static void read_tx_type(const AV1_COMMON *const cm, MACROBLOCKD *xd,
if (eset > 0) {
mbmi->tx_type = aom_read_tree(
r, av1_ext_tx_inter_tree[eset],
cm->fc->inter_ext_tx_prob[eset][txsize_sqr_map[tx_size]],
ACCT_STR);
cm->fc->inter_ext_tx_prob[eset][square_tx_size], ACCT_STR);
if (counts)
++counts->inter_ext_tx[eset][txsize_sqr_map[tx_size]]
[mbmi->tx_type];
++counts->inter_ext_tx[eset][square_tx_size][mbmi->tx_type];
}
} else if (ALLOW_INTRA_EXT_TX) {
if (eset > 0) {
mbmi->tx_type = aom_read_tree(
r, av1_ext_tx_intra_tree[eset],
cm->fc->intra_ext_tx_prob[eset][tx_size][mbmi->mode], ACCT_STR);
cm->fc->intra_ext_tx_prob[eset][square_tx_size][mbmi->mode],
ACCT_STR);
if (counts)
++counts->intra_ext_tx[eset][tx_size][mbmi->mode][mbmi->tx_type];
++counts->intra_ext_tx[eset][square_tx_size][mbmi->mode]
[mbmi->tx_type];
}
}
} else {
......@@ -807,7 +795,7 @@ static void read_intra_frame_mode_info(AV1_COMMON *const cm,
}
#endif
mbmi->tx_size = read_tx_size_intra(cm, xd, r);
mbmi->tx_size = read_tx_size(cm, xd, 0, 1, r);
mbmi->ref_frame[0] = INTRA_FRAME;
mbmi->ref_frame[1] = NONE;
......@@ -1967,10 +1955,7 @@ static void read_inter_frame_mode_info(AV1Decoder *const pbi,
read_tx_size_vartx(cm, xd, mbmi, xd->counts, max_tx_size,
height != width, idy, idx, r);
} else {
if (inter_block)
mbmi->tx_size = read_tx_size_inter(cm, xd, !mbmi->skip, r);
else
mbmi->tx_size = read_tx_size_intra(cm, xd, r);
mbmi->tx_size = read_tx_size(cm, xd, inter_block, !mbmi->skip, r);
if (inter_block) {
const int width = block_size_wide[bsize] >> tx_size_wide_log2[0];
......@@ -1984,10 +1969,7 @@ static void read_inter_frame_mode_info(AV1Decoder *const pbi,
set_txfm_ctxs(mbmi->tx_size, xd->n8_w, xd->n8_h, mbmi->skip, xd);
}
#else
if (inter_block)
mbmi->tx_size = read_tx_size_inter(cm, xd, !mbmi->skip, r);
else
mbmi->tx_size = read_tx_size_intra(cm, xd, r);
mbmi->tx_size = read_tx_size(cm, xd, inter_block, !mbmi->skip, r);
#endif // CONFIG_VAR_TX
#if CONFIG_SUPERTX
}
......
......@@ -1169,9 +1169,10 @@ static void write_tx_type(const AV1_COMMON *const cm,
const TX_SIZE tx_size = is_inter ? mbmi->min_tx_size : mbmi->tx_size;
#else
const TX_SIZE tx_size = mbmi->tx_size;
#endif
#endif // CONFIG_VAR_TX
if (!FIXED_TX_TYPE) {
#if CONFIG_EXT_TX
const TX_SIZE square_tx_size = txsize_sqr_map[tx_size];
const BLOCK_SIZE bsize = mbmi->sb_type;
if (get_ext_tx_types(tx_size, bsize, is_inter) > 1 && cm->base_qindex > 0 &&
!mbmi->skip &&
......@@ -1182,18 +1183,21 @@ static void write_tx_type(const AV1_COMMON *const cm,
int eset = get_ext_tx_set(tx_size, bsize, is_inter);
if (is_inter) {
assert(ext_tx_used_inter[eset][mbmi->tx_type]);
if (eset > 0)
av1_write_token(
w, av1_ext_tx_inter_tree[eset],
cm->fc->inter_ext_tx_prob[eset][txsize_sqr_map[tx_size]],
if (eset > 0) {
av1_write_token(w, av1_ext_tx_inter_tree[eset],
cm->fc->inter_ext_tx_prob[eset][square_tx_size],
&ext_tx_inter_encodings[eset][mbmi->tx_type]);
}
} else if (ALLOW_INTRA_EXT_TX) {
if (eset > 0)
av1_write_token(w, av1_ext_tx_intra_tree[eset],
cm->fc->intra_ext_tx_prob[eset][tx_size][mbmi->mode],
assert(ext_tx_used_intra[eset][mbmi->tx_type]);
if (eset > 0) {
av1_write_token(
w, av1_ext_tx_intra_tree[eset],
cm->fc->intra_ext_tx_prob[eset][square_tx_size][mbmi->mode],
&ext_tx_intra_encodings[eset][mbmi->tx_type]);
}
}
}
#else
if (tx_size < TX_32X32 && cm->base_qindex > 0 && !mbmi->skip &&
#if CONFIG_SUPERTX
......@@ -2641,6 +2645,9 @@ static void build_tree_distribution(AV1_COMP *cpi, TX_SIZE tx_size,
unsigned int(*eob_branch_ct)[REF_TYPES][COEF_BANDS][COEFF_CONTEXTS] =
cpi->common.counts.eob_branch[tx_size];
int i, j, k, l, m;
#if CONFIG_RECT_TX
assert(!is_rect_tx(tx_size));
#endif // CONFIG_RECT_TX
for (i = 0; i < PLANE_TYPES; ++i) {
for (j = 0; j < REF_TYPES; ++j) {
......@@ -2679,6 +2686,9 @@ static void update_coef_probs_common(aom_writer *const bc, AV1_COMP *cpi,
#else
const int probwt = 1;
#endif
#if CONFIG_RECT_TX
assert(!is_rect_tx(tx_size));
#endif // CONFIG_RECT_TX
switch (cpi->sf.use_fast_coef_updates) {
case TWO_LOOP: {
......
......@@ -204,6 +204,27 @@ struct macroblock {
#endif
};
// Converts block_index for given transform size to index of the block in raster
// order.
static inline int av1_block_index_to_raster_order(TX_SIZE tx_size,
int block_idx) {
// For transform size 4x8, the possible block_idx values are 0 & 2, because
// block_idx values are incremented in steps of size 'tx_width_unit x
// tx_height_unit'. But, for this transform size, block_idx = 2 corresponds to
// block number 1 in raster order, inside an 8x8 MI block.
// For any other transform size, the two indices are equivalent.
return (tx_size == TX_4X8 && block_idx == 2) ? 1 : block_idx;
}
// Inverse of above function.
// Note: only implemented for transform sizes 4x4, 4x8 and 8x4 right now.
static inline int av1_raster_order_to_block_index(TX_SIZE tx_size,
int raster_order) {
assert(tx_size == TX_4X4 || tx_size == TX_4X8 || tx_size == TX_8X4);
// We ensure that block indices are 0 & 2 if tx size is 4x8 or 8x4.
return (tx_size == TX_4X4) ? raster_order : (raster_order > 0) ? 2 : 0;
}
#ifdef __cplusplus
} // extern "C"
#endif
......
......@@ -5581,13 +5581,17 @@ static void encode_superblock(const AV1_COMP *const cpi, ThreadData *td,
} else {
intra_tx_size = tx_size_from_tx_mode(bsize, cm->tx_mode, 1);
}
} else {
#if CONFIG_EXT_TX && CONFIG_RECT_TX
intra_tx_size = tx_size;
#else
intra_tx_size = (bsize >= BLOCK_8X8) ? tx_size : TX_4X4;
#endif // CONFIG_EXT_TX && CONFIG_RECT_TX
}
#if CONFIG_EXT_TX && CONFIG_RECT_TX
++td->counts->tx_size_implied[max_txsize_lookup[bsize]]
[txsize_sqr_up_map[tx_size]];
#endif // CONFIG_EXT_TX && CONFIG_RECT_TX
} else {
intra_tx_size = (bsize >= BLOCK_8X8) ? tx_size : TX_4X4;
}
for (j = 0; j < mi_height; j++)
for (i = 0; i < mi_width; i++)
......@@ -5613,7 +5617,8 @@ static void encode_superblock(const AV1_COMP *const cpi, ThreadData *td,
++td->counts->inter_ext_tx[eset][txsize_sqr_map[tx_size]]
[mbmi->tx_type];
} else {
++td->counts->intra_ext_tx[eset][tx_size][mbmi->mode][mbmi->tx_type];
++td->counts->intra_ext_tx[eset][txsize_sqr_map[tx_size]][mbmi->mode]
[mbmi->tx_type];
}
}
}
......
......@@ -101,7 +101,7 @@ int av1_optimize_b(const AV1_COMMON *cm, MACROBLOCK *mb, int plane, int block,
av1_token_state tokens[MAX_TX_SQUARE + 1][2];
unsigned best_index[MAX_TX_SQUARE + 1][2];