Commit 16c64e33 authored by David Barker's avatar David Barker

Miscellaneous fixes for var-tx

Lots of small bug fixes, mainly around the transform size coding:

* The loop filter was accidentally using the non-subsampled
  block size for the V plane, due to comparing a plane index
  (0, 1, or 2) against PLANE_TYPE_UV (== 1)

* We allowed an initial update of the transform partition probabilities
  even on frames where we know they will never be used
  (because tx_mode != TX_MODE_SELECT).
  Further, these probabilities would not be reverted at the end
  of the frame, leading to the probability delta persisting across frames.

  Change this to behave more like the non-var-tx transform size coding,
  where probability deltas are only coded for frames with
  tx_mode == TX_MODE_SELECT, and the deltas only apply for one frame.

* Fix decoder for the case where the video as a whole isn't lossless,
  and we have tx_mode == TX_MODE_SELECT, but the current segment
  *is* lossless.
  Note that the encoder already does the right thing in this case.

* Don't allow the transform splitting to recurse "below" 4x4.
  This is really just a refactor, but means we can increase the
  maximum depth when subdividing rectangular transforms if we
  want to, whereas the previous code would have needed special cases
  for 4x8 and 8x4 transforms.

* Finally, when we hit the maximum splitting depth, don't update
  the counts as if we had coded a 'no split' symbol.

Change-Id: Iaebdacc9de81d2e93d3c49241e719bbc02e32682
parent 09b5b168
...@@ -2739,7 +2739,7 @@ static TX_SIZE av1_get_transform_size(const MODE_INFO *const mi, ...@@ -2739,7 +2739,7 @@ static TX_SIZE av1_get_transform_size(const MODE_INFO *const mi,
const uint32_t scale_horz, const uint32_t scale_horz,
const uint32_t scale_vert) { const uint32_t scale_vert) {
const MB_MODE_INFO *mbmi = &mi->mbmi; const MB_MODE_INFO *mbmi = &mi->mbmi;
TX_SIZE tx_size = (plane == PLANE_TYPE_Y) TX_SIZE tx_size = (plane == AOM_PLANE_Y)
? mbmi->tx_size ? mbmi->tx_size
: av1_get_uv_tx_size(mbmi, plane_ptr); : av1_get_uv_tx_size(mbmi, plane_ptr);
assert(tx_size < TX_SIZES_ALL); assert(tx_size < TX_SIZES_ALL);
...@@ -2774,9 +2774,9 @@ static TX_SIZE av1_get_transform_size(const MODE_INFO *const mi, ...@@ -2774,9 +2774,9 @@ static TX_SIZE av1_get_transform_size(const MODE_INFO *const mi,
assert(mb_tx_size < TX_SIZES_ALL); assert(mb_tx_size < TX_SIZES_ALL);
tx_size = (plane == PLANE_TYPE_UV) tx_size = (plane == AOM_PLANE_Y)
? uv_txsize_lookup[bsize][mb_tx_size][0][0] ? mb_tx_size
: mb_tx_size; : uv_txsize_lookup[bsize][mb_tx_size][0][0];
assert(tx_size < TX_SIZES_ALL); assert(tx_size < TX_SIZES_ALL);
} }
#else #else
......
...@@ -4926,8 +4926,9 @@ static int read_compressed_header(AV1Decoder *pbi, const uint8_t *data, ...@@ -4926,8 +4926,9 @@ static int read_compressed_header(AV1Decoder *pbi, const uint8_t *data,
#if !CONFIG_NEW_MULTISYMBOL #if !CONFIG_NEW_MULTISYMBOL
#if CONFIG_VAR_TX #if CONFIG_VAR_TX
for (i = 0; i < TXFM_PARTITION_CONTEXTS; ++i) if (cm->tx_mode == TX_MODE_SELECT)
av1_diff_update_prob(&r, &fc->txfm_partition_prob[i], ACCT_STR); for (i = 0; i < TXFM_PARTITION_CONTEXTS; ++i)
av1_diff_update_prob(&r, &fc->txfm_partition_prob[i], ACCT_STR);
#endif // CONFIG_VAR_TX #endif // CONFIG_VAR_TX
for (i = 0; i < SKIP_CONTEXTS; ++i) for (i = 0; i < SKIP_CONTEXTS; ++i)
av1_diff_update_prob(&r, &fc->skip_probs[i], ACCT_STR); av1_diff_update_prob(&r, &fc->skip_probs[i], ACCT_STR);
......
...@@ -428,6 +428,7 @@ static void read_tx_size_vartx(AV1_COMMON *cm, MACROBLOCKD *xd, ...@@ -428,6 +428,7 @@ static void read_tx_size_vartx(AV1_COMMON *cm, MACROBLOCKD *xd,
[MAX_MIB_SIZE] = [MAX_MIB_SIZE] =
(TX_SIZE(*)[MAX_MIB_SIZE]) & mbmi->inter_tx_size[tx_row][tx_col]; (TX_SIZE(*)[MAX_MIB_SIZE]) & mbmi->inter_tx_size[tx_row][tx_col];
if (blk_row >= max_blocks_high || blk_col >= max_blocks_wide) return; if (blk_row >= max_blocks_high || blk_col >= max_blocks_wide) return;
assert(tx_size > TX_4X4);
if (depth == MAX_VARTX_DEPTH) { if (depth == MAX_VARTX_DEPTH) {
int idx, idy; int idx, idy;
...@@ -437,7 +438,6 @@ static void read_tx_size_vartx(AV1_COMMON *cm, MACROBLOCKD *xd, ...@@ -437,7 +438,6 @@ static void read_tx_size_vartx(AV1_COMMON *cm, MACROBLOCKD *xd,
inter_tx_size[idy][idx] = tx_size; inter_tx_size[idy][idx] = tx_size;
mbmi->tx_size = tx_size; mbmi->tx_size = tx_size;
mbmi->min_tx_size = AOMMIN(mbmi->min_tx_size, get_min_tx_size(tx_size)); mbmi->min_tx_size = AOMMIN(mbmi->min_tx_size, get_min_tx_size(tx_size));
if (counts) ++counts->txfm_partition[ctx][0];
txfm_partition_update(xd->above_txfm_context + blk_col, txfm_partition_update(xd->above_txfm_context + blk_col,
xd->left_txfm_context + blk_row, tx_size, tx_size); xd->left_txfm_context + blk_row, tx_size, tx_size);
return; return;
...@@ -456,7 +456,7 @@ static void read_tx_size_vartx(AV1_COMMON *cm, MACROBLOCKD *xd, ...@@ -456,7 +456,7 @@ static void read_tx_size_vartx(AV1_COMMON *cm, MACROBLOCKD *xd,
if (counts) ++counts->txfm_partition[ctx][1]; if (counts) ++counts->txfm_partition[ctx][1];
if (tx_size == TX_8X8) { if (sub_txs == TX_4X4) {
int idx, idy; int idx, idy;
inter_tx_size[0][0] = sub_txs; inter_tx_size[0][0] = sub_txs;
for (idy = 0; idy < tx_size_high_unit[tx_size] / 2; ++idy) for (idy = 0; idy < tx_size_high_unit[tx_size] / 2; ++idy)
...@@ -2861,7 +2861,7 @@ static void read_inter_frame_mode_info(AV1Decoder *const pbi, ...@@ -2861,7 +2861,7 @@ static void read_inter_frame_mode_info(AV1Decoder *const pbi,
#else #else
bsize >= BLOCK_8X8 && bsize >= BLOCK_8X8 &&
#endif #endif
!mbmi->skip && inter_block) { !mbmi->skip && inter_block && !xd->lossless[mbmi->segment_id]) {
const TX_SIZE max_tx_size = max_txsize_rect_lookup[bsize]; const TX_SIZE max_tx_size = max_txsize_rect_lookup[bsize];
const int bh = tx_size_high_unit[max_tx_size]; const int bh = tx_size_high_unit[max_tx_size];
const int bw = tx_size_wide_unit[max_tx_size]; const int bw = tx_size_wide_unit[max_tx_size];
......
...@@ -376,7 +376,7 @@ static void write_tx_size_vartx(const AV1_COMMON *cm, MACROBLOCKD *xd, ...@@ -376,7 +376,7 @@ static void write_tx_size_vartx(const AV1_COMMON *cm, MACROBLOCKD *xd,
aom_write(w, 1, cm->fc->txfm_partition_prob[ctx]); aom_write(w, 1, cm->fc->txfm_partition_prob[ctx]);
#endif #endif
if (tx_size == TX_8X8) { if (sub_txs == TX_4X4) {
txfm_partition_update(xd->above_txfm_context + blk_col, txfm_partition_update(xd->above_txfm_context + blk_col,
xd->left_txfm_context + blk_row, sub_txs, tx_size); xd->left_txfm_context + blk_row, sub_txs, tx_size);
return; return;
...@@ -401,7 +401,7 @@ static void update_txfm_partition_probs(AV1_COMMON *cm, aom_writer *w, ...@@ -401,7 +401,7 @@ static void update_txfm_partition_probs(AV1_COMMON *cm, aom_writer *w,
counts->txfm_partition[k], probwt); counts->txfm_partition[k], probwt);
} }
#endif // CONFIG_NEW_MULTISYMBOL #endif // CONFIG_NEW_MULTISYMBOL
#endif #endif // CONFIG_VAR_TX
static void write_selected_tx_size(const AV1_COMMON *cm, const MACROBLOCKD *xd, static void write_selected_tx_size(const AV1_COMMON *cm, const MACROBLOCKD *xd,
aom_writer *w) { aom_writer *w) {
...@@ -4701,7 +4701,8 @@ static uint32_t write_compressed_header(AV1_COMP *cpi, uint8_t *data) { ...@@ -4701,7 +4701,8 @@ static uint32_t write_compressed_header(AV1_COMP *cpi, uint8_t *data) {
#endif // CONFIG_LV_MAP #endif // CONFIG_LV_MAP
#if CONFIG_VAR_TX && !CONFIG_NEW_MULTISYMBOL #if CONFIG_VAR_TX && !CONFIG_NEW_MULTISYMBOL
update_txfm_partition_probs(cm, header_bc, counts, probwt); if (cm->tx_mode == TX_MODE_SELECT)
update_txfm_partition_probs(cm, header_bc, counts, probwt);
#endif #endif
#if !CONFIG_NEW_MULTISYMBOL #if !CONFIG_NEW_MULTISYMBOL
......
...@@ -5806,6 +5806,18 @@ static void update_txfm_count(MACROBLOCK *x, MACROBLOCKD *xd, ...@@ -5806,6 +5806,18 @@ static void update_txfm_count(MACROBLOCK *x, MACROBLOCKD *xd,
const TX_SIZE plane_tx_size = mbmi->inter_tx_size[tx_row][tx_col]; const TX_SIZE plane_tx_size = mbmi->inter_tx_size[tx_row][tx_col];
if (blk_row >= max_blocks_high || blk_col >= max_blocks_wide) return; if (blk_row >= max_blocks_high || blk_col >= max_blocks_wide) return;
assert(tx_size > TX_4X4);
if (depth == MAX_VARTX_DEPTH) {
// Don't add to counts in this case
#if CONFIG_RECT_TX_EXT
if (tx_size == plane_tx_size)
#endif
mbmi->tx_size = tx_size;
txfm_partition_update(xd->above_txfm_context + blk_col,
xd->left_txfm_context + blk_row, tx_size, tx_size);
return;
}
#if CONFIG_RECT_TX_EXT #if CONFIG_RECT_TX_EXT
if (tx_size == plane_tx_size || if (tx_size == plane_tx_size ||
...@@ -5829,7 +5841,7 @@ static void update_txfm_count(MACROBLOCK *x, MACROBLOCKD *xd, ...@@ -5829,7 +5841,7 @@ static void update_txfm_count(MACROBLOCK *x, MACROBLOCKD *xd,
++counts->txfm_partition[ctx][1]; ++counts->txfm_partition[ctx][1];
++x->txb_split_count; ++x->txb_split_count;
if (tx_size == TX_8X8) { if (sub_txs == TX_4X4) {
mbmi->inter_tx_size[tx_row][tx_col] = TX_4X4; mbmi->inter_tx_size[tx_row][tx_col] = TX_4X4;
mbmi->tx_size = TX_4X4; mbmi->tx_size = TX_4X4;
txfm_partition_update(xd->above_txfm_context + blk_col, txfm_partition_update(xd->above_txfm_context + blk_col,
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment