Commit 5f8361a7 authored by James Zern's avatar James Zern

get_prob(): rationalize int types

promote the unsigned int calculation to uint64_t rather than int64_t for
type consistency

cherry-picked from libvpx:
47d6f16a0 get_prob(): rationalize int types

Change-Id: Ic34dee1dc707d9faf6a3ae250bfe39b60bef3438
parent 073c6f33
......@@ -58,7 +58,7 @@ typedef const aom_tree_index aom_tree[];
static INLINE aom_prob get_prob(unsigned int num, unsigned int den) {
assert(den != 0);
{
const int p = (int)(((int64_t)num * 256 + (den >> 1)) / den);
const int p = (int)(((uint64_t)num * 256 + (den >> 1)) / den);
// (p > 255) ? 255 : (p < 1) ? 1 : p;
const int clipped_prob = p | ((255 - p) >> 23) | (p == 0);
return (aom_prob)clipped_prob;
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment