Commit 248f0557 authored by Angie Chiang's avatar Angie Chiang

Fix txfm_stage_range_check

Only check cos_bit range if cos_bit is not NULL

Change-Id: I286fc056812b20242cc962a8b008af7093d05b1d
parent 4b29ea86
......@@ -367,18 +367,14 @@ void txfm_stage_range_check(const int8_t *stage_range, int stage_num,
int high_range) {
for (int i = 0; i < stage_num; ++i) {
EXPECT_LE(stage_range[i], low_range);
if (cos_bit != NULL) {
ASSERT_LE(stage_range[i] + cos_bit[i], high_range) << "stage = " << i;
}
}
for (int i = 0; i < stage_num - 1; ++i) {
// make sure there is no overflow while doing half_btf()
EXPECT_LE(stage_range[i] + cos_bit[i], high_range);
EXPECT_LE(stage_range[i + 1] + cos_bit[i], high_range);
if (stage_range[i] + cos_bit[i] > high_range) {
std::cout << i;
assert(0);
}
if (stage_range[i + 1] + cos_bit[i] > high_range) {
std::cout << i;
assert(0);
if (cos_bit != NULL) {
ASSERT_LE(stage_range[i + 1] + cos_bit[i], high_range) << "stage = " << i;
}
}
}
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment