Commit f1cf7178 authored by Luc Trudeau's avatar Luc Trudeau

[CFL] Use MAX_NUM_TXB_SQUARE instead of MAX_NUM_TXB

MAX_NUM_TXB represents the maximum number of transform blocks in 1
dimension. CfL requires the maximum number of transform blocks in 2
dimensions. As such, we now use MAX_NUM_TXB_SQUARE instead of the
erroneous MAX_NUM_TXB for the average buffer.

There's an assert guarding for overrun on the average buffer.
This fix stops assertions on 4:4:4 sequences. This patch does
not alter AWCY results as AWCY only covers 4:2:0 sequences (for
which MAX_NUM_TXB turns out to be sufficient). 

Change-Id: I628db0131f60abc2d06cbbe3fe3dc40e28894ce3
parent 03b7e7d5
...@@ -714,7 +714,7 @@ typedef struct cfl_ctx { ...@@ -714,7 +714,7 @@ typedef struct cfl_ctx {
// * Max error will be 1/16th. // * Max error will be 1/16th.
// Note: 3 is chosen so that y_averages fits in 15 bits when 12 bit input is // Note: 3 is chosen so that y_averages fits in 15 bits when 12 bit input is
// used // used
int y_averages_q3[MAX_NUM_TXB]; int y_averages_q3[MAX_NUM_TXB_SQUARE];
int y_averages_stride; int y_averages_stride;
int are_parameters_computed; int are_parameters_computed;
......
...@@ -214,6 +214,7 @@ static void cfl_compute_averages(CFL_CTX *cfl, TX_SIZE tx_size) { ...@@ -214,6 +214,7 @@ static void cfl_compute_averages(CFL_CTX *cfl, TX_SIZE tx_size) {
} }
t_y_pix += MAX_SB_SIZE; t_y_pix += MAX_SB_SIZE;
} }
assert(a < MAX_NUM_TXB_SQUARE);
averages_q3[a++] = averages_q3[a++] =
((sum << 3) + (1 << (num_pel_log2 - 1))) >> num_pel_log2; ((sum << 3) + (1 << (num_pel_log2 - 1))) >> num_pel_log2;
...@@ -226,7 +227,6 @@ static void cfl_compute_averages(CFL_CTX *cfl, TX_SIZE tx_size) { ...@@ -226,7 +227,6 @@ static void cfl_compute_averages(CFL_CTX *cfl, TX_SIZE tx_size) {
} }
cfl->y_averages_stride = stride; cfl->y_averages_stride = stride;
assert(a <= MAX_NUM_TXB);
} }
static INLINE int cfl_idx_to_alpha(int alpha_idx, int joint_sign, static INLINE int cfl_idx_to_alpha(int alpha_idx, int joint_sign,
......
...@@ -206,6 +206,7 @@ typedef enum ATTRIBUTE_PACKED { ...@@ -206,6 +206,7 @@ typedef enum ATTRIBUTE_PACKED {
#define MAX_TX_BLOCKS_IN_MAX_SB (1 << MAX_TX_BLOCKS_IN_MAX_SB_LOG2) #define MAX_TX_BLOCKS_IN_MAX_SB (1 << MAX_TX_BLOCKS_IN_MAX_SB_LOG2)
#define MAX_NUM_TXB (1 << (MAX_SB_SIZE_LOG2 - MIN_TX_SIZE_LOG2)) #define MAX_NUM_TXB (1 << (MAX_SB_SIZE_LOG2 - MIN_TX_SIZE_LOG2))
#define MAX_NUM_TXB_SQUARE (MAX_NUM_TXB * MAX_NUM_TXB)
#if CONFIG_NCOBMC_ADAPT_WEIGHT #if CONFIG_NCOBMC_ADAPT_WEIGHT
typedef enum ATTRIBUTE_PACKED { typedef enum ATTRIBUTE_PACKED {
......
...@@ -5832,7 +5832,7 @@ static int rd_pick_intra_angle_sbuv(const AV1_COMP *const cpi, MACROBLOCK *x, ...@@ -5832,7 +5832,7 @@ static int rd_pick_intra_angle_sbuv(const AV1_COMP *const cpi, MACROBLOCK *x,
#if CONFIG_CFL #if CONFIG_CFL
static int64_t cfl_alpha_dist(const uint8_t *y_pix, int y_stride, static int64_t cfl_alpha_dist(const uint8_t *y_pix, int y_stride,
const int y_averages_q3[MAX_NUM_TXB], const int y_averages_q3[MAX_NUM_TXB_SQUARE],
const uint8_t *src, int src_stride, int width, const uint8_t *src, int src_stride, int width,
int height, TX_SIZE tx_size, int dc_pred, int height, TX_SIZE tx_size, int dc_pred,
int alpha_q3, int64_t *dist_neg_out) { int alpha_q3, int64_t *dist_neg_out) {
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment