Commit 6de4a77d authored by Angie Chiang's avatar Angie Chiang
Browse files

Remove double operation from tx_size selection

This CL fix the bug
rdopt.c:1687: choose_tx_size_from_rd: Assertion
`mbmi->tx_type == DCT_DCT' failed

It is caused by
1) mms register access before double operation
2) different compiler behaviors
code:
  int64_t a = INT64_MAX;
  double b = 1. * INT64_MAX;
  printf("a < b: %d\n", a < b);
result:
  a < b: 0

code:
  --target=x86-linux-gcc
  int64_t a = INT64_MAX;
  double b = 1. * INT64_MAX;
  printf("a < b: %d\n", a < b);
result:
  a < b: 1

I remove the double operation and test it with EXT_TX experiment.
The psnr change is around 0.05%, which is considered as noise level.

Change-Id: If8935c70c8603617fcfa8571accd30ccdda786a0
parent 716f0ea3
......@@ -81,12 +81,6 @@
#define MIN_EARLY_TERM_INDEX 3
#define NEW_MV_DISCOUNT_FACTOR 8
#if CONFIG_EXT_TX
const double ext_tx_th = 0.99;
#else
const double ext_tx_th = 0.99;
#endif
const double ADST_FLIP_SVM[8] = {-6.6623, -2.8062, -3.2531, 3.1671, // vert
-7.7051, -3.2234, -3.6193, 3.4533}; // horz
......@@ -1567,7 +1561,7 @@ static void choose_largest_tx_size(VP10_COMP *cpi, MACROBLOCK *x,
if (is_inter_block(mbmi) && !xd->lossless[mbmi->segment_id] && !s)
this_rd = VPXMIN(this_rd, RDCOST(x->rdmult, x->rddiv, s1, psse));
if (this_rd < ((best_tx_type == DCT_DCT) ? ext_tx_th : 1) * best_rd) {
if (this_rd < best_rd) {
best_rd = this_rd;
best_tx_type = mbmi->tx_type;
}
......@@ -1602,7 +1596,7 @@ static void choose_largest_tx_size(VP10_COMP *cpi, MACROBLOCK *x,
if (is_inter && !xd->lossless[mbmi->segment_id] && !s)
this_rd = VPXMIN(this_rd, RDCOST(x->rdmult, x->rddiv, s1, psse));
if (this_rd < ((best_tx_type == DCT_DCT) ? ext_tx_th : 1) * best_rd) {
if (this_rd < best_rd) {
best_rd = this_rd;
best_tx_type = mbmi->tx_type;
}
......@@ -1665,7 +1659,7 @@ static void choose_tx_size_from_rd(VP10_COMP *cpi, MACROBLOCK *x,
for (tx_type = DCT_DCT; tx_type < TX_TYPES; ++tx_type) {
rd = choose_tx_size_fix_type(cpi, x, &r, &d, &s, &sse, ref_best_rd, bs,
tx_type, prune);
if (rd < (is_inter && best_tx_type == DCT_DCT ? ext_tx_th : 1) * best_rd) {
if (rd < best_rd) {
best_rd = rd;
*distortion = d;
*rate = r;
......@@ -3287,7 +3281,7 @@ static void select_tx_type_yrd(const VP10_COMP *cpi, MACROBLOCK *x,
rd = select_tx_size_fix_type(cpi, x, &this_rate, &this_dist, &this_skip,
&this_sse, bsize, ref_best_rd, tx_type);
if (rd < (is_inter && best_tx_type == DCT_DCT ? ext_tx_th : 1) * best_rd) {
if (rd < best_rd) {
best_rd = rd;
*distortion = this_dist;
*rate = this_rate;
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment