Commit 93c3f3db authored by Jingning Han's avatar Jingning Han

Support rectangular tx_size in level-map optimize_b

Validate the provided coefficient location with respect to the
height and width of the transform block size.

Change-Id: Id4f10052141fd914f5aea5ae4202cf35d3e63867
parent 341d79e1
......@@ -868,7 +868,7 @@ int try_level_down(int coeff_idx, const TxbCache *txb_cache,
const int nb_scan_idx = iscan[nb_coeff_idx];
if (nb_scan_idx < eob && nb_row >= 0 && nb_col >= 0 &&
nb_row < txb_info->stride && nb_col < txb_info->stride) {
nb_row < txb_info->height && nb_col < txb_info->stride) {
const int cost_diff = try_neighbor_level_down_nz(
nb_coeff_idx, coeff_idx, txb_cache, txb_probs, txb_info);
if (cost_map)
......@@ -886,7 +886,7 @@ int try_level_down(int coeff_idx, const TxbCache *txb_cache,
const int nb_coeff_idx = nb_row * txb_info->stride + nb_col;
const int nb_scan_idx = iscan[nb_coeff_idx];
if (nb_scan_idx < eob && nb_row >= 0 && nb_col >= 0 &&
nb_row < txb_info->stride && nb_col < txb_info->stride) {
nb_row < txb_info->height && nb_col < txb_info->stride) {
const int cost_diff = try_neighbor_level_down_base(
nb_coeff_idx, coeff_idx, txb_cache, txb_probs, txb_info);
if (cost_map)
......@@ -904,7 +904,7 @@ int try_level_down(int coeff_idx, const TxbCache *txb_cache,
const int nb_coeff_idx = nb_row * txb_info->stride + nb_col;
const int nb_scan_idx = iscan[nb_coeff_idx];
if (nb_scan_idx < eob && nb_row >= 0 && nb_col >= 0 &&
nb_row < txb_info->stride && nb_col < txb_info->stride) {
nb_row < txb_info->height && nb_col < txb_info->stride) {
const int cost_diff = try_neighbor_level_down_br(
nb_coeff_idx, coeff_idx, txb_cache, txb_probs, txb_info);
if (cost_map)
......@@ -1032,7 +1032,7 @@ void update_level_down(int coeff_idx, TxbCache *txb_cache, TxbInfo *txb_info) {
const int nb_coeff_idx = nb_row * txb_info->stride + nb_col;
const int nb_scan_idx = iscan[nb_coeff_idx];
if (nb_scan_idx < eob && nb_row >= 0 && nb_col >= 0 &&
nb_row < txb_info->stride && nb_col < txb_info->stride) {
nb_row < txb_info->height && nb_col < txb_info->stride) {
const int scan_idx = iscan[coeff_idx];
if (scan_idx < nb_scan_idx) {
const int level = 1;
......@@ -1060,7 +1060,7 @@ void update_level_down(int coeff_idx, TxbCache *txb_cache, TxbInfo *txb_info) {
if (!has_base(nb_coeff, 0)) continue;
const int nb_scan_idx = iscan[nb_coeff_idx];
if (nb_scan_idx < eob && nb_row >= 0 && nb_col >= 0 &&
nb_row < txb_info->stride && nb_col < txb_info->stride) {
nb_row < txb_info->height && nb_col < txb_info->stride) {
if (row >= nb_row && col >= nb_col)
update_mag_arr(txb_cache->base_mag_arr[nb_coeff_idx], abs_qc);
const int mag =
......@@ -1093,7 +1093,7 @@ void update_level_down(int coeff_idx, TxbCache *txb_cache, TxbInfo *txb_info) {
const tran_low_t nb_coeff = txb_info->qcoeff[nb_coeff_idx];
if (!has_br(nb_coeff)) continue;
if (nb_scan_idx < eob && nb_row >= 0 && nb_col >= 0 &&
nb_row < txb_info->stride && nb_col < txb_info->stride) {
nb_row < txb_info->height && nb_col < txb_info->stride) {
const int level = 1 + NUM_BASE_LEVELS;
if (abs_qc == level) {
txb_cache->br_count_arr[nb_coeff_idx] -= 1;
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment