Commit 16c64e33 authored by David Barker's avatar David Barker

Miscellaneous fixes for var-tx

Lots of small bug fixes, mainly around the transform size coding:

* The loop filter was accidentally using the non-subsampled
  block size for the V plane, due to comparing a plane index
  (0, 1, or 2) against PLANE_TYPE_UV (== 1)

* We allowed an initial update of the transform partition probabilities
  even on frames where we know they will never be used
  (because tx_mode != TX_MODE_SELECT).
  Further, these probabilities would not be reverted at the end
  of the frame, leading to the probability delta persisting across frames.

  Change this to behave more like the non-var-tx transform size coding,
  where probability deltas are only coded for frames with
  tx_mode == TX_MODE_SELECT, and the deltas only apply for one frame.

* Fix decoder for the case where the video as a whole isn't lossless,
  and we have tx_mode == TX_MODE_SELECT, but the current segment
  *is* lossless.
  Note that the encoder already does the right thing in this case.

* Don't allow the transform splitting to recurse "below" 4x4.
  This is really just a refactor, but means we can increase the
  maximum depth when subdividing rectangular transforms if we
  want to, whereas the previous code would have needed special cases
  for 4x8 and 8x4 transforms.

* Finally, when we hit the maximum splitting depth, don't update
  the counts as if we had coded a 'no split' symbol.

Change-Id: Iaebdacc9de81d2e93d3c49241e719bbc02e32682
parent 09b5b168
......@@ -2739,7 +2739,7 @@ static TX_SIZE av1_get_transform_size(const MODE_INFO *const mi,
const uint32_t scale_horz,
const uint32_t scale_vert) {
const MB_MODE_INFO *mbmi = &mi->mbmi;
TX_SIZE tx_size = (plane == PLANE_TYPE_Y)
TX_SIZE tx_size = (plane == AOM_PLANE_Y)
? mbmi->tx_size
: av1_get_uv_tx_size(mbmi, plane_ptr);
assert(tx_size < TX_SIZES_ALL);
......@@ -2774,9 +2774,9 @@ static TX_SIZE av1_get_transform_size(const MODE_INFO *const mi,
assert(mb_tx_size < TX_SIZES_ALL);
tx_size = (plane == PLANE_TYPE_UV)
? uv_txsize_lookup[bsize][mb_tx_size][0][0]
: mb_tx_size;
tx_size = (plane == AOM_PLANE_Y)
? mb_tx_size
: uv_txsize_lookup[bsize][mb_tx_size][0][0];
assert(tx_size < TX_SIZES_ALL);
}
#else
......
......@@ -4926,6 +4926,7 @@ static int read_compressed_header(AV1Decoder *pbi, const uint8_t *data,
#if !CONFIG_NEW_MULTISYMBOL
#if CONFIG_VAR_TX
if (cm->tx_mode == TX_MODE_SELECT)
for (i = 0; i < TXFM_PARTITION_CONTEXTS; ++i)
av1_diff_update_prob(&r, &fc->txfm_partition_prob[i], ACCT_STR);
#endif // CONFIG_VAR_TX
......
......@@ -428,6 +428,7 @@ static void read_tx_size_vartx(AV1_COMMON *cm, MACROBLOCKD *xd,
[MAX_MIB_SIZE] =
(TX_SIZE(*)[MAX_MIB_SIZE]) & mbmi->inter_tx_size[tx_row][tx_col];
if (blk_row >= max_blocks_high || blk_col >= max_blocks_wide) return;
assert(tx_size > TX_4X4);
if (depth == MAX_VARTX_DEPTH) {
int idx, idy;
......@@ -437,7 +438,6 @@ static void read_tx_size_vartx(AV1_COMMON *cm, MACROBLOCKD *xd,
inter_tx_size[idy][idx] = tx_size;
mbmi->tx_size = tx_size;
mbmi->min_tx_size = AOMMIN(mbmi->min_tx_size, get_min_tx_size(tx_size));
if (counts) ++counts->txfm_partition[ctx][0];
txfm_partition_update(xd->above_txfm_context + blk_col,
xd->left_txfm_context + blk_row, tx_size, tx_size);
return;
......@@ -456,7 +456,7 @@ static void read_tx_size_vartx(AV1_COMMON *cm, MACROBLOCKD *xd,
if (counts) ++counts->txfm_partition[ctx][1];
if (tx_size == TX_8X8) {
if (sub_txs == TX_4X4) {
int idx, idy;
inter_tx_size[0][0] = sub_txs;
for (idy = 0; idy < tx_size_high_unit[tx_size] / 2; ++idy)
......@@ -2861,7 +2861,7 @@ static void read_inter_frame_mode_info(AV1Decoder *const pbi,
#else
bsize >= BLOCK_8X8 &&
#endif
!mbmi->skip && inter_block) {
!mbmi->skip && inter_block && !xd->lossless[mbmi->segment_id]) {
const TX_SIZE max_tx_size = max_txsize_rect_lookup[bsize];
const int bh = tx_size_high_unit[max_tx_size];
const int bw = tx_size_wide_unit[max_tx_size];
......
......@@ -376,7 +376,7 @@ static void write_tx_size_vartx(const AV1_COMMON *cm, MACROBLOCKD *xd,
aom_write(w, 1, cm->fc->txfm_partition_prob[ctx]);
#endif
if (tx_size == TX_8X8) {
if (sub_txs == TX_4X4) {
txfm_partition_update(xd->above_txfm_context + blk_col,
xd->left_txfm_context + blk_row, sub_txs, tx_size);
return;
......@@ -401,7 +401,7 @@ static void update_txfm_partition_probs(AV1_COMMON *cm, aom_writer *w,
counts->txfm_partition[k], probwt);
}
#endif // CONFIG_NEW_MULTISYMBOL
#endif
#endif // CONFIG_VAR_TX
static void write_selected_tx_size(const AV1_COMMON *cm, const MACROBLOCKD *xd,
aom_writer *w) {
......@@ -4701,6 +4701,7 @@ static uint32_t write_compressed_header(AV1_COMP *cpi, uint8_t *data) {
#endif // CONFIG_LV_MAP
#if CONFIG_VAR_TX && !CONFIG_NEW_MULTISYMBOL
if (cm->tx_mode == TX_MODE_SELECT)
update_txfm_partition_probs(cm, header_bc, counts, probwt);
#endif
......
......@@ -5806,6 +5806,18 @@ static void update_txfm_count(MACROBLOCK *x, MACROBLOCKD *xd,
const TX_SIZE plane_tx_size = mbmi->inter_tx_size[tx_row][tx_col];
if (blk_row >= max_blocks_high || blk_col >= max_blocks_wide) return;
assert(tx_size > TX_4X4);
if (depth == MAX_VARTX_DEPTH) {
// Don't add to counts in this case
#if CONFIG_RECT_TX_EXT
if (tx_size == plane_tx_size)
#endif
mbmi->tx_size = tx_size;
txfm_partition_update(xd->above_txfm_context + blk_col,
xd->left_txfm_context + blk_row, tx_size, tx_size);
return;
}
#if CONFIG_RECT_TX_EXT
if (tx_size == plane_tx_size ||
......@@ -5829,7 +5841,7 @@ static void update_txfm_count(MACROBLOCK *x, MACROBLOCKD *xd,
++counts->txfm_partition[ctx][1];
++x->txb_split_count;
if (tx_size == TX_8X8) {
if (sub_txs == TX_4X4) {
mbmi->inter_tx_size[tx_row][tx_col] = TX_4X4;
mbmi->tx_size = TX_4X4;
txfm_partition_update(xd->above_txfm_context + blk_col,
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment