From a3028ddf55d0892c0336ec1051f31c39038895aa Mon Sep 17 00:00:00 2001
From: Yaowu Xu <yaowu@google.com>
Date: Wed, 18 May 2016 09:06:52 -0700
Subject: [PATCH] change to use correct type
This commit changes to use uint32_t for cost (always non-negative),
and promote to int64_t before calculation of the savings.
This fixes an integer overflow.
Change-Id: I71c2580d188cc79d2d8069241d0353cf331b5c83
---
av1/encoder/subexp.c | 9 +++++----
1 file changed, 5 insertions(+), 4 deletions(-)
diff --git a/av1/encoder/subexp.c b/av1/encoder/subexp.c
index 74a3ca073d..e9eeb34c02 100644
--- a/av1/encoder/subexp.c
+++ b/av1/encoder/subexp.c
@@ -123,15 +123,16 @@ void av1_write_prob_diff_update(aom_writer *w, aom_prob newp, aom_prob oldp) {
int av1_prob_diff_update_savings_search(const unsigned int *ct, aom_prob oldp,
aom_prob *bestp, aom_prob upd) {
- const int old_b = cost_branch256(ct, oldp);
+ const uint32_t old_b = cost_branch256(ct, oldp);
int bestsavings = 0;
aom_prob newp, bestnewp = oldp;
const int step = *bestp > oldp ? -1 : 1;
for (newp = *bestp; newp != oldp; newp += step) {
- const int new_b = cost_branch256(ct, newp);
- const int update_b = prob_diff_update_cost(newp, oldp) + av1_cost_upd256;
- const int savings = old_b - new_b - update_b;
+ const uint32_t new_b = cost_branch256(ct, newp);
+ const uint32_t update_b =
+ prob_diff_update_cost(newp, oldp) + av1_cost_upd256;
+ const int savings = (int)((int64_t)old_b - new_b - update_b);
if (savings > bestsavings) {
bestsavings = savings;
bestnewp = newp;
--
GitLab