Commit ba2d817a authored by Jingning Han's avatar Jingning Han

Constrain counter range in adaptive scan

Limit the maximum transform block count to be 256 per adaptive
scan order model.

Change-Id: If6ae054d4427b784f05dd944747b6249b86f401b
parent ad4ac8a8
......@@ -5508,7 +5508,10 @@ static void update_scan_count(int16_t *scan, int max_scan,
void av1_update_scan_count_facade(AV1_COMMON *cm, FRAME_COUNTS *counts,
TX_SIZE tx_size, TX_TYPE tx_type,
const tran_low_t *dqcoeffs, int max_scan) {
if (cm->use_adapt_scan && do_adapt_scan(tx_size, tx_type)) {
if (cm->use_adapt_scan && do_adapt_scan(tx_size, tx_type) && max_scan) {
#if SUB_REGION_COUNT
if (counts->txb_count[tx_size][tx_type] >= UINT8_MAX) return;
#endif
int16_t *scan = get_adapt_scan(cm->fc, tx_size, tx_type);
uint32_t *non_zero_count = get_non_zero_counts(counts, tx_size, tx_type);
update_scan_count(scan, max_scan, dqcoeffs, non_zero_count);
......
......@@ -33,6 +33,7 @@ extern const SCAN_ORDER av1_inter_scan_orders[TX_SIZES_ALL][TX_TYPES];
#define USE_2X2_PROB 0
#define CACHE_SCAN_PROB 0
#define REDUCED_SET 1
#define SUB_REGION_COUNT 1
#define USE_TOPOLOGICAL_SORT 0
#define USE_LIMIT_SCAN_DISTANCE 0
void av1_update_scan_count_facade(AV1_COMMON *cm, FRAME_COUNTS *counts,
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment