Correct scale factor for TX_16X64 and TX_64X16.

BUG=aomedia:1114

Change-Id: I7fbeb4c2da996801b945304e182403ec325f95bc

av1/encoder/av1_fwd_txfm2d.c

@@ -84,10 +84,9 @@ static INLINE void fwd_txfm2d_c(const int16_t *input, int32_t *output,
   int rect_type2_shift = 0;
   if (rect_type == 2 || rect_type == -2) {
     const int txfm_size_max = AOMMAX(txfm_size_col, txfm_size_row);
-    // For 64x16 / 16x64 shift 3 bits, for 32x8 / 8x32 shift 2 bits, for
-    // 16x4 / 4x16 shift by 1 bit.
-    rect_type2_shift =
-        (txfm_size_max == 64 ? 3 : (txfm_size_max == 32 ? 2 : 1));
+    // For 64x16 / 16x64 / 32x8 / 8x32 shift 2 bits, and
+    // For 16x4 / 4x16 shift by 1 bit.
+    rect_type2_shift = (txfm_size_max >= 32) ? 2 : 1;
   }
   // Take the shift from the larger dimension in the rectangular case.
   const int8_t *shift = (txfm_size_col > txfm_size_row) ? cfg->row_cfg->shift

test/av1_fwd_txfm2d_test.cc

@@ -162,7 +162,7 @@ vector<AV1FwdTxfm2dParam> GetTxfm2dParamList() {
       param_list.push_back(AV1FwdTxfm2dParam(tx_type, TX_32X64, 136, 7));
       param_list.push_back(AV1FwdTxfm2dParam(tx_type, TX_64X32, 136, 7));
       param_list.push_back(AV1FwdTxfm2dParam(tx_type, TX_16X64, 16, 1.6));
-      param_list.push_back(AV1FwdTxfm2dParam(tx_type, TX_64X16, 16, 1.6));
+      param_list.push_back(AV1FwdTxfm2dParam(tx_type, TX_64X16, 20, 2.0));
     }
 #endif  // CONFIG_TX64X64
   }

test/av1_inv_txfm2d_test.cc

@@ -167,8 +167,8 @@ vector<AV1InvTxfm2dParam> GetInvTxfm2dParamList() {
       param_list.push_back(AV1InvTxfm2dParam(tx_type, TX_64X64, 3, 0.2));
       param_list.push_back(AV1InvTxfm2dParam(tx_type, TX_32X64, 3, 0.3));
       param_list.push_back(AV1InvTxfm2dParam(tx_type, TX_64X32, 3, 0.31));
-      param_list.push_back(AV1InvTxfm2dParam(tx_type, TX_16X64, 2, 0.11));
-      param_list.push_back(AV1InvTxfm2dParam(tx_type, TX_64X16, 2, 0.11));
+      param_list.push_back(AV1InvTxfm2dParam(tx_type, TX_16X64, 2, 0.16));
+      param_list.push_back(AV1InvTxfm2dParam(tx_type, TX_64X16, 2, 0.16));
     }
 #endif  // CONFIG_TX64X64
   }

test/av1_txfm_test.cc

@@ -124,8 +124,7 @@ double get_amplification_factor(TX_TYPE tx_type, TX_SIZE tx_size) {
     amplify_factor *= pow(2, 0.5);
   } else if (abs(rect_type) == 2) {
     const int tx_max_dim = AOMMAX(tx_width, tx_height);
-    const int rect_type2_shift =
-        tx_max_dim == 64 ? 3 : tx_max_dim == 32 ? 2 : 1;
+    const int rect_type2_shift = (tx_max_dim >= 32) ? 2 : 1;
     amplify_factor *= pow(2, rect_type2_shift);
   }
   return amplify_factor;