Commit f1cf7178 authored by Luc Trudeau's avatar Luc Trudeau

[CFL] Use MAX_NUM_TXB_SQUARE instead of MAX_NUM_TXB

MAX_NUM_TXB represents the maximum number of transform blocks in 1
dimension. CfL requires the maximum number of transform blocks in 2
dimensions. As such, we now use MAX_NUM_TXB_SQUARE instead of the
erroneous MAX_NUM_TXB for the average buffer.

There's an assert guarding for overrun on the average buffer.
This fix stops assertions on 4:4:4 sequences. This patch does
not alter AWCY results as AWCY only covers 4:2:0 sequences (for
which MAX_NUM_TXB turns out to be sufficient). 

Change-Id: I628db0131f60abc2d06cbbe3fe3dc40e28894ce3
parent 03b7e7d5
......@@ -714,7 +714,7 @@ typedef struct cfl_ctx {
// * Max error will be 1/16th.
// Note: 3 is chosen so that y_averages fits in 15 bits when 12 bit input is
// used
int y_averages_q3[MAX_NUM_TXB];
int y_averages_q3[MAX_NUM_TXB_SQUARE];
int y_averages_stride;
int are_parameters_computed;
......
......@@ -214,6 +214,7 @@ static void cfl_compute_averages(CFL_CTX *cfl, TX_SIZE tx_size) {
}
t_y_pix += MAX_SB_SIZE;
}
assert(a < MAX_NUM_TXB_SQUARE);
averages_q3[a++] =
((sum << 3) + (1 << (num_pel_log2 - 1))) >> num_pel_log2;
......@@ -226,7 +227,6 @@ static void cfl_compute_averages(CFL_CTX *cfl, TX_SIZE tx_size) {
}
cfl->y_averages_stride = stride;
assert(a <= MAX_NUM_TXB);
}
static INLINE int cfl_idx_to_alpha(int alpha_idx, int joint_sign,
......
......@@ -206,6 +206,7 @@ typedef enum ATTRIBUTE_PACKED {
#define MAX_TX_BLOCKS_IN_MAX_SB (1 << MAX_TX_BLOCKS_IN_MAX_SB_LOG2)
#define MAX_NUM_TXB (1 << (MAX_SB_SIZE_LOG2 - MIN_TX_SIZE_LOG2))
#define MAX_NUM_TXB_SQUARE (MAX_NUM_TXB * MAX_NUM_TXB)
#if CONFIG_NCOBMC_ADAPT_WEIGHT
typedef enum ATTRIBUTE_PACKED {
......
......@@ -5832,7 +5832,7 @@ static int rd_pick_intra_angle_sbuv(const AV1_COMP *const cpi, MACROBLOCK *x,
#if CONFIG_CFL
static int64_t cfl_alpha_dist(const uint8_t *y_pix, int y_stride,
const int y_averages_q3[MAX_NUM_TXB],
const int y_averages_q3[MAX_NUM_TXB_SQUARE],
const uint8_t *src, int src_stride, int width,
int height, TX_SIZE tx_size, int dc_pred,
int alpha_q3, int64_t *dist_neg_out) {
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment