Commit f3208f36 authored by Deb Mukherjee's avatar Deb Mukherjee

Some cleanups and fixes.

Separates the logic on transform type selection previously spread out
over a number of files into a separate function. Currently the tx_type
field in b_mode_info is not used, but still left in there to eventually
use for signaling the transform type in the bitstream.

Also, now for tx_type = DCT_DCT, the regular integer DCT is used, as
opposed to the floating point DCT used in conjuction with hybrid
transform.

Results change somewhat due to the transform change, but are within
reasonable limits. The hd/std-hd sets are slightly up, while derf/yt
are slightly down.

Change-Id: I5776840c2239ca2da31ca6cfd7fd1148dc5f9e0f
parent b44397eb
......@@ -151,14 +151,6 @@ typedef enum {
#define VP8_MVREFS (1 + SPLITMV - NEARESTMV)
#if CONFIG_HYBRIDTRANSFORM || CONFIG_HYBRIDTRANSFORM16X16
#define ACTIVE_HT 110 // quantization stepsize threshold
#endif
#if CONFIG_HYBRIDTRANSFORM16X16
#define ACTIVE_HT16 300
#endif
typedef enum {
B_DC_PRED, /* average of above and left pixels */
B_TM_PRED,
......@@ -182,50 +174,6 @@ typedef enum {
B_MODE_COUNT
} B_PREDICTION_MODE;
#if CONFIG_HYBRIDTRANSFORM8X8 || CONFIG_HYBRIDTRANSFORM16X16
// convert MB_PREDICTION_MODE to B_PREDICTION_MODE
static B_PREDICTION_MODE pred_mode_conv(MB_PREDICTION_MODE mode) {
B_PREDICTION_MODE b_mode;
switch (mode) {
case DC_PRED:
b_mode = B_DC_PRED;
break;
case V_PRED:
b_mode = B_VE_PRED;
break;
case H_PRED:
b_mode = B_HE_PRED;
break;
case TM_PRED:
b_mode = B_TM_PRED;
break;
case D45_PRED:
b_mode = B_LD_PRED;
break;
case D135_PRED:
b_mode = B_RD_PRED;
break;
case D117_PRED:
b_mode = B_VR_PRED;
break;
case D153_PRED:
b_mode = B_HD_PRED;
break;
case D27_PRED:
b_mode = B_VL_PRED;
break;
case D63_PRED:
b_mode = B_HU_PRED;
break;
default :
// for debug purpose, to be removed after full testing
assert(0);
break;
}
return b_mode;
}
#endif
#define VP8_BINTRAMODES (B_HU_PRED + 1) /* 10 */
#define VP8_SUBMVREFS (1 + NEW4X4 - LEFT4X4)
......@@ -438,68 +386,150 @@ typedef struct MacroBlockD {
int_mv ref_mv[MAX_MV_REFS];
#endif
#if CONFIG_HYBRIDTRANSFORM || CONFIG_HYBRIDTRANSFORM8X8 || CONFIG_HYBRIDTRANSFORM16X16
int q_index;
#endif
} MACROBLOCKD;
#if CONFIG_HYBRIDTRANSFORM || CONFIG_HYBRIDTRANSFORM16X16
#define ACTIVE_HT 110 // quantization stepsize threshold
#endif
#if CONFIG_HYBRIDTRANSFORM8X8
#define ACTIVE_HT8 300
#endif
#if CONFIG_HYBRIDTRANSFORM16X16
#define ACTIVE_HT16 300
#endif
#if CONFIG_HYBRIDTRANSFORM8X8 || CONFIG_HYBRIDTRANSFORM16X16
// convert MB_PREDICTION_MODE to B_PREDICTION_MODE
static B_PREDICTION_MODE pred_mode_conv(MB_PREDICTION_MODE mode) {
B_PREDICTION_MODE b_mode;
switch (mode) {
case DC_PRED:
b_mode = B_DC_PRED;
break;
case V_PRED:
b_mode = B_VE_PRED;
break;
case H_PRED:
b_mode = B_HE_PRED;
break;
case TM_PRED:
b_mode = B_TM_PRED;
break;
case D45_PRED:
b_mode = B_LD_PRED;
break;
case D135_PRED:
b_mode = B_RD_PRED;
break;
case D117_PRED:
b_mode = B_VR_PRED;
break;
case D153_PRED:
b_mode = B_HD_PRED;
break;
case D27_PRED:
b_mode = B_HU_PRED;
break;
case D63_PRED:
b_mode = B_VL_PRED;
break;
default :
// for debug purpose, to be removed after full testing
assert(0);
break;
}
return b_mode;
}
#endif
#if CONFIG_HYBRIDTRANSFORM || CONFIG_HYBRIDTRANSFORM8X8 || CONFIG_HYBRIDTRANSFORM16X16
// transform mapping
static void txfm_map(BLOCKD *b, B_PREDICTION_MODE bmode) {
static TX_TYPE txfm_map(B_PREDICTION_MODE bmode) {
// map transform type
TX_TYPE tx_type;
switch (bmode) {
case B_TM_PRED :
case B_RD_PRED :
b->bmi.as_mode.tx_type = ADST_ADST;
tx_type = ADST_ADST;
break;
case B_VE_PRED :
case B_VR_PRED :
b->bmi.as_mode.tx_type = ADST_DCT;
tx_type = ADST_DCT;
break;
case B_HE_PRED :
case B_HD_PRED :
case B_HU_PRED :
b->bmi.as_mode.tx_type = DCT_ADST;
tx_type = DCT_ADST;
break;
default :
b->bmi.as_mode.tx_type = DCT_DCT;
tx_type = DCT_DCT;
break;
}
return tx_type;
}
#endif
#if CONFIG_HYBRIDTRANSFORM
static TX_TYPE get_tx_type_4x4(const MACROBLOCKD *xd, const BLOCKD *b) {
TX_TYPE tx_type = DCT_DCT;
if (xd->mode_info_context->mbmi.mode == B_PRED &&
xd->q_index < ACTIVE_HT) {
tx_type = txfm_map(b->bmi.as_mode.first);
}
return tx_type;
}
#endif
#if CONFIG_HYBRIDTRANSFORM8X8
static TX_TYPE get_tx_type_8x8(const MACROBLOCKD *xd, const BLOCKD *b) {
TX_TYPE tx_type = DCT_DCT;
if (xd->mode_info_context->mbmi.mode == I8X8_PRED &&
xd->q_index < ACTIVE_HT8) {
tx_type = txfm_map(pred_mode_conv(b->bmi.as_mode.first));
}
return tx_type;
}
#endif
static TX_TYPE get_tx_type(MACROBLOCKD *xd, const BLOCKD *b) {
#if CONFIG_HYBRIDTRANSFORM16X16
static TX_TYPE get_tx_type_16x16(const MACROBLOCKD *xd, const BLOCKD *b) {
TX_TYPE tx_type = DCT_DCT;
if (xd->mode_info_context->mbmi.mode < I8X8_PRED &&
xd->q_index < ACTIVE_HT16) {
tx_type = txfm_map(pred_mode_conv(xd->mode_info_context->mbmi.mode));
}
return tx_type;
}
#endif
#if CONFIG_HYBRIDTRANSFORM || CONFIG_HYBRIDTRANSFORM8X8 || \
CONFIG_HYBRIDTRANSFORM16X16
static TX_TYPE get_tx_type(const MACROBLOCKD *xd, const BLOCKD *b) {
TX_TYPE tx_type = DCT_DCT;
int ib = (b - xd->block);
if (ib >= 16) return tx_type;
if (ib >= 16)
return tx_type;
#if CONFIG_HYBRIDTRANSFORM16X16
if (xd->mode_info_context->mbmi.txfm_size == TX_16X16) {
if (xd->mode_info_context->mbmi.mode < I8X8_PRED &&
xd->q_index < ACTIVE_HT16)
tx_type = b->bmi.as_mode.tx_type;
return tx_type;
tx_type = get_tx_type_16x16(xd, b);
}
#endif
#if CONFIG_HYBRIDTRANSFORM8X8
if (xd->mode_info_context->mbmi.txfm_size == TX_8X8) {
BLOCKD *bb;
ib = (ib & 8) + ((ib & 4) >> 1);
bb = xd->block + ib;
if (xd->mode_info_context->mbmi.mode == I8X8_PRED)
tx_type = bb->bmi.as_mode.tx_type;
return tx_type;
tx_type = get_tx_type_8x8(xd, &xd->block[ib]);
}
#endif
#if CONFIG_HYBRIDTRANSFORM
if (xd->mode_info_context->mbmi.txfm_size == TX_4X4) {
if (xd->mode_info_context->mbmi.mode == B_PRED &&
xd->q_index < ACTIVE_HT) {
tx_type = b->bmi.as_mode.tx_type;
}
return tx_type;
tx_type = get_tx_type_4x4(xd, b);
}
#endif
return tx_type;
......
......@@ -189,6 +189,7 @@ void vp8_ihtllm_c(short *input, short *output, int pitch,
// pointers to vertical and horizontal transforms
float *ptv, *pth;
assert(tx_type != DCT_DCT);
// load and convert residual array into floating-point
for(j = 0; j < tx_dim; j++) {
for(i = 0; i < tx_dim; i++) {
......
......@@ -1234,6 +1234,14 @@ static void read_mb_modes_mv(VP8D_COMP *pbi, MODE_INFO *mi, MB_MODE_INFO *mbmi,
#endif
do {
mi->bmi[j].as_mode.first = (B_PREDICTION_MODE)vp8_read_bmode(bc, pbi->common.fc.bmode_prob);
/*
{
int p;
for (p = 0; p < VP8_BINTRAMODES - 1; ++p)
printf(" %d", pbi->common.fc.bmode_prob[p]);
printf("\nbmode[%d][%d]: %d\n", pbi->common.current_video_frame, j, mi->bmi[j].as_mode.first);
}
*/
pbi->common.fc.bmode_counts[mi->bmi[j].as_mode.first]++;
#if CONFIG_COMP_INTRA_PRED
if (use_comp_pred) {
......
......@@ -115,17 +115,13 @@ void mb_init_dequantizer(VP8D_COMP *pbi, MACROBLOCKD *xd) {
}
} else
QIndex = pc->base_qindex;
xd->q_index = QIndex;
/* Set up the block level dequant pointers */
for (i = 0; i < 16; i++) {
xd->block[i].dequant = pc->Y1dequant[QIndex];
}
#if CONFIG_HYBRIDTRANSFORM || CONFIG_HYBRIDTRANSFORM8X8 || CONFIG_HYBRIDTRANSFORM16X16
xd->q_index = QIndex;
#endif
#if CONFIG_LOSSLESS
if (!QIndex) {
pbi->common.rtcd.idct.idct1 = vp8_short_inv_walsh4x4_1_x8_c;
......@@ -210,21 +206,16 @@ static void decode_macroblock(VP8D_COMP *pbi, MACROBLOCKD *xd,
int eobtotal = 0;
MB_PREDICTION_MODE mode;
int i;
int tx_type;
int tx_size;
#if CONFIG_HYBRIDTRANSFORM || CONFIG_HYBRIDTRANSFORM8X8 || \
CONFIG_HYBRIDTRANSFORM16X16
TX_TYPE tx_type;
#endif
#if CONFIG_SUPERBLOCKS
VP8_COMMON *pc = &pbi->common;
int orig_skip_flag = xd->mode_info_context->mbmi.mb_skip_coeff;
#endif
#if CONFIG_HYBRIDTRANSFORM || CONFIG_HYBRIDTRANSFORM16X16
int QIndex;
int active_ht;
#endif
#if CONFIG_HYBRIDTRANSFORM16X16
int active_ht16;
#endif
// re-initialize macroblock dequantizer before detokenization
if (xd->segmentation_enabled)
mb_init_dequantizer(pbi, xd);
......@@ -235,43 +226,9 @@ static void decode_macroblock(VP8D_COMP *pbi, MACROBLOCKD *xd,
}
#endif
tx_type = xd->mode_info_context->mbmi.txfm_size;
tx_size = xd->mode_info_context->mbmi.txfm_size;
mode = xd->mode_info_context->mbmi.mode;
#if CONFIG_HYBRIDTRANSFORM
// parse transform types for intra 4x4 mode
QIndex = xd->q_index;
active_ht = (QIndex < ACTIVE_HT);
if (mode == B_PRED) {
for (i = 0; i < 16; i++) {
BLOCKD *b = &xd->block[i];
int b_mode = xd->mode_info_context->bmi[i].as_mode.first;
if(active_ht)
txfm_map(b, b_mode);
} // loop over 4x4 blocks
}
#endif
#if CONFIG_HYBRIDTRANSFORM8X8
if (mode == I8X8_PRED) {
for (i = 0; i < 4; i++) {
int ib = vp8_i8x8_block[i];
BLOCKD *b = &xd->block[ib];
int i8x8mode = b->bmi.as_mode.first;
txfm_map(b, pred_mode_conv(i8x8mode));
}
}
#endif
#if CONFIG_HYBRIDTRANSFORM16X16
active_ht16 = (QIndex < ACTIVE_HT16);
if (mode < I8X8_PRED) {
BLOCKD *b = &xd->block[0];
if(active_ht16)
txfm_map(b, pred_mode_conv(mode));
}
#endif
if (xd->mode_info_context->mbmi.mb_skip_coeff) {
vp8_reset_mb_tokens_context(xd);
#if CONFIG_SUPERBLOCKS
......@@ -288,12 +245,13 @@ static void decode_macroblock(VP8D_COMP *pbi, MACROBLOCKD *xd,
xd->block[i].eob = 0;
xd->eobs[i] = 0;
}
if (tx_type == TX_16X16)
if (tx_size == TX_16X16) {
eobtotal = vp8_decode_mb_tokens_16x16(pbi, xd);
else if (tx_type == TX_8X8)
} else if (tx_size == TX_8X8) {
eobtotal = vp8_decode_mb_tokens_8x8(pbi, xd);
else
} else {
eobtotal = vp8_decode_mb_tokens(pbi, xd);
}
}
//mode = xd->mode_info_context->mbmi.mode;
......@@ -380,13 +338,17 @@ static void decode_macroblock(VP8D_COMP *pbi, MACROBLOCKD *xd,
if (xd->mode_info_context->mbmi.txfm_size == TX_8X8) {
#if CONFIG_HYBRIDTRANSFORM8X8
vp8_ht_dequant_idct_add_8x8_c(b->bmi.as_mode.tx_type,
q, dq, pre, dst, 16, stride);
q += 64;
tx_type = get_tx_type(xd, &xd->block[idx]);
if (tx_type != DCT_DCT) {
vp8_ht_dequant_idct_add_8x8_c(tx_type,
q, dq, pre, dst, 16, stride);
} else {
vp8_dequant_idct_add_8x8_c(q, dq, pre, dst, 16, stride);
}
#else
vp8_dequant_idct_add_8x8_c(q, dq, pre, dst, 16, stride);
q += 64;
#endif
q += 64;
} else {
for (j = 0; j < 4; j++) {
b = &xd->block[ib + iblock[j]];
......@@ -396,7 +358,7 @@ static void decode_macroblock(VP8D_COMP *pbi, MACROBLOCKD *xd,
}
b = &xd->block[16 + i];
vp8_intra_uv4x4_predict(b, i8x8mode, b->predictor);
vp8_intra_uv4x4_predict(b, i8x8mode, b->predictor);
DEQUANT_INVOKE(&pbi->dequant, idct_add)(b->qcoeff, b->dequant,
b->predictor,
*(b->base_dst) + b->dst, 8,
......@@ -425,16 +387,17 @@ static void decode_macroblock(VP8D_COMP *pbi, MACROBLOCKD *xd,
#endif
#if CONFIG_HYBRIDTRANSFORM
if(active_ht)
vp8_ht_dequant_idct_add_c( (TX_TYPE)b->bmi.as_mode.tx_type, b->qcoeff,
b->dequant, b->predictor,
*(b->base_dst) + b->dst, 16, b->dst_stride);
else
tx_type = get_tx_type(xd, b);
if (tx_type != DCT_DCT) {
vp8_ht_dequant_idct_add_c(tx_type, b->qcoeff,
b->dequant, b->predictor,
*(b->base_dst) + b->dst, 16, b->dst_stride);
} else {
vp8_dequant_idct_add_c(b->qcoeff, b->dequant, b->predictor,
*(b->base_dst) + b->dst, 16, b->dst_stride);
}
#else
if (xd->eobs[i] > 1)
{
if (xd->eobs[i] > 1) {
DEQUANT_INVOKE(&pbi->dequant, idct_add)
(b->qcoeff, b->dequant, b->predictor,
*(b->base_dst) + b->dst, 16, b->dst_stride);
......@@ -454,15 +417,12 @@ static void decode_macroblock(VP8D_COMP *pbi, MACROBLOCKD *xd,
} else {
BLOCKD *b = &xd->block[24];
if (tx_type == TX_16X16) {
if (tx_size == TX_16X16) {
#if CONFIG_HYBRIDTRANSFORM16X16
if (mode < I8X8_PRED && active_ht16) {
BLOCKD *bd = &xd->block[0];
TX_TYPE txfm;
txfm_map(bd, pred_mode_conv(mode));
txfm = bd->bmi.as_mode.tx_type;
vp8_ht_dequant_idct_add_16x16_c(txfm, xd->qcoeff,
BLOCKD *bd = &xd->block[0];
tx_type = get_tx_type(xd, bd);
if (tx_type != DCT_DCT) {
vp8_ht_dequant_idct_add_16x16_c(tx_type, xd->qcoeff,
xd->block[0].dequant, xd->predictor,
xd->dst.y_buffer, 16, xd->dst.y_stride);
} else {
......@@ -475,8 +435,7 @@ static void decode_macroblock(VP8D_COMP *pbi, MACROBLOCKD *xd,
xd->predictor, xd->dst.y_buffer,
16, xd->dst.y_stride);
#endif
}
else if (tx_type == TX_8X8) {
} else if (tx_size == TX_8X8) {
#if CONFIG_SUPERBLOCKS
void *orig = xd->mode_info_context;
int n, num = xd->mode_info_context->mbmi.encoded_as_sb ? 4 : 1;
......@@ -561,10 +520,9 @@ static void decode_macroblock(VP8D_COMP *pbi, MACROBLOCKD *xd,
#if CONFIG_SUPERBLOCKS
if (!xd->mode_info_context->mbmi.encoded_as_sb) {
#endif
if ((tx_type == TX_8X8 &&
if ((tx_size == TX_8X8 &&
xd->mode_info_context->mbmi.mode != I8X8_PRED)
|| tx_type == TX_16X16
)
|| tx_size == TX_16X16)
DEQUANT_INVOKE(&pbi->dequant, idct_add_uv_block_8x8) //
(xd->qcoeff + 16 * 16, xd->block[16].dequant,
xd->predictor + 16 * 16, xd->dst.u_buffer, xd->dst.v_buffer,
......
......@@ -578,8 +578,7 @@ int vp8_decode_mb_tokens_8x8(VP8D_COMP *pbi, MACROBLOCKD *xd) {
if (i == 16)
type = PLANE_TYPE_UV;
#if CONFIG_HYBRIDTRANSFORM8X8
if (type == PLANE_TYPE_Y_WITH_DC &&
xd->mode_info_context->mbmi.mode == I8X8_PRED) {
if (type == PLANE_TYPE_Y_WITH_DC) {
tx_type = get_tx_type(xd, xd->block + i);
}
#endif
......@@ -673,10 +672,7 @@ int vp8_decode_mb_tokens(VP8D_COMP *dx, MACROBLOCKD *xd) {
type = PLANE_TYPE_UV;
#if CONFIG_HYBRIDTRANSFORM
if (type == PLANE_TYPE_Y_WITH_DC)
tx_type = get_tx_type(xd, &xd->block[i]);
#endif
#if CONFIG_HYBRIDTRANSFORM
tx_type = get_tx_type(xd, &xd->block[i]);
switch(tx_type) {
case ADST_DCT :
scan = vp8_row_scan;
......
......@@ -996,6 +996,14 @@ static void pack_inter_mode_mvs(VP8_COMP *const cpi) {
#endif
write_bmode(w, m->bmi[j].as_mode.first,
pc->fc.bmode_prob);
/*
if (!cpi->dummy_packing) {
int p;
for (p = 0; p < VP8_BINTRAMODES - 1; ++p)
printf(" %d", pc->fc.bmode_prob[p]);
printf("\nbmode[%d][%d]: %d\n", pc->current_video_frame, j, m->bmi[j].as_mode.first);
}
*/
#if CONFIG_COMP_INTRA_PRED
if (uses_second) {
write_bmode(w, mode2, pc->fc.bmode_prob);
......
......@@ -170,7 +170,6 @@ typedef struct {
#endif
int optimize;
int q_index;
// Structure to hold context for each of the 4 MBs within a SB:
// when encoded as 4 independent MBs:
......
......@@ -419,6 +419,7 @@ void vp8_fht_c(short *input, short *output, int pitch,
// pointers to vertical and horizontal transforms
float *ptv, *pth;
assert(tx_type != DCT_DCT);
// load and convert residual array into floating-point
for(j = 0; j < tx_dim; j++) {
for(i = 0; i < tx_dim; i++) {
......
......@@ -1061,7 +1061,7 @@ static void encode_sb(VP8_COMP *cpi,
x->partition_info += 2;
xd->mode_info_context += 2;
xd->prev_mode_info_context += 2;
(*tp)->Token = EOSB_TOKEN;
(*tp)++;
if (mb_row < cm->mb_rows) cpi->tplist[mb_row].stop = *tp;
......@@ -1931,7 +1931,7 @@ void vp8cx_encode_intra_super_block(VP8_COMP *cpi,
update_sb_skip_coeff_state(cpi, x, ta, tl, tp, t, skip);
}
}
#endif
#endif /* CONFIG_SUPERBLOCKS */
void vp8cx_encode_intra_macro_block(VP8_COMP *cpi,
MACROBLOCK *x,
......@@ -1942,7 +1942,6 @@ void vp8cx_encode_intra_macro_block(VP8_COMP *cpi,
adjust_act_zbin(cpi, x);
vp8_update_zbin_extra(cpi, x);
}
if (mbmi->mode == I8X8_PRED) {
vp8_encode_intra8x8mby(IF_RTCD(&cpi->rtcd), x);
vp8_encode_intra8x8mbuv(IF_RTCD(&cpi->rtcd), x);
......
......@@ -57,6 +57,9 @@ void vp8_encode_intra4x4block(const VP8_ENCODER_RTCD *rtcd,
MACROBLOCK *x, int ib) {
BLOCKD *b = &x->e_mbd.block[ib];
BLOCK *be = &x->block[ib];
#if CONFIG_HYBRIDTRANSFORM
TX_TYPE tx_type;
#endif
#if CONFIG_COMP_INTRA_PRED
if (b->bmi.as_mode.second == (B_PREDICTION_MODE)(B_DC_PRED - 1)) {
......@@ -72,11 +75,11 @@ void vp8_encode_intra4x4block(const VP8_ENCODER_RTCD *rtcd,
ENCODEMB_INVOKE(&rtcd->encodemb, subb)(be, b, 16);
#if CONFIG_HYBRIDTRANSFORM
if (x->q_index < ACTIVE_HT) {
txfm_map(b, b->bmi.as_mode.first);
vp8_fht_c(be->src_diff, be->coeff, 32, b->bmi.as_mode.tx_type, 4);
vp8_ht_quantize_b_4x4(be, b);
vp8_ihtllm_c(b->dqcoeff, b->diff, 32, b->bmi.as_mode.tx_type, 4);
tx_type = get_tx_type(&x->e_mbd, b);
if (tx_type != DCT_DCT) {
vp8_fht_c(be->src_diff, be->coeff, 32, tx_type, 4);
vp8_ht_quantize_b_4x4(be, b, tx_type);
vp8_ihtllm_c(b->dqcoeff, b->diff, 32, tx_type, 4);
} else
#endif
{
......@@ -91,12 +94,6 @@ void vp8_encode_intra4x4block(const VP8_ENCODER_RTCD *rtcd,
void vp8_encode_intra4x4mby(const VP8_ENCODER_RTCD *rtcd, MACROBLOCK *mb) {
int i;
#if 0
MACROBLOCKD *xd = &mb->e_mbd;
// Intra modes requiring top-right MB reconstructed data have been disabled
vp8_intra_prediction_down_copy(xd);
#endif
for (i = 0; i < 16; i++)
vp8_encode_intra4x4block(rtcd, mb, i);
return;
......@@ -107,7 +104,7 @@ void vp8_encode_intra16x16mby(const VP8_ENCODER_RTCD *rtcd, MACROBLOCK *x) {
BLOCK *b = &x->block[0];
TX_SIZE tx_size = xd->mode_info_context->mbmi.txfm_size;
#if CONFIG_HYBRIDTRANSFORM16X16
TX_TYPE txfm_type = xd->mode_info_context->bmi[0].as_mode.tx_type;
TX_TYPE tx_type;
#endif
#if CONFIG_COMP_INTRA_PRED
......@@ -124,16 +121,14 @@ void vp8_encode_intra16x16mby(const VP8_ENCODER_RTCD *rtcd, MACROBLOCK *x) {
if (tx_size == TX_16X16) {
#if CONFIG_HYBRIDTRANSFORM16X16
if ((xd->mode_info_context->mbmi.mode < I8X8_PRED) &&
(x->q_index < ACTIVE_HT16)) {
BLOCKD *bd = &xd->block[0];
txfm_map(bd, pred_mode_conv(xd->mode_info_context->mbmi.mode));
txfm_type = bd->bmi.as_mode.tx_type;
vp8_fht_c(b->src_diff, b->coeff, 32, txfm_type, 16);
BLOCKD *bd = &xd->block[0];
tx_type = get_tx_type(xd, bd);
if (tx_type != DCT_DCT) {
vp8_fht_c(b->src_diff, b->coeff, 32, tx_type, 16);
vp8_quantize_mby_16x16(x);
if (x->optimize)
vp8_optimize_mby_16x16(x, rtcd);
vp8_ihtllm_c(bd->dqcoeff, bd->diff, 32, txfm_type, 16);
vp8_ihtllm_c(bd->dqcoeff, bd->diff, 32, tx_type, 16);
} else
#endif
{
......@@ -201,6 +196,9 @@ void vp8_encode_intra8x8(const VP8_ENCODER_RTCD *rtcd,
BLOCK *be = &x->block[ib];
const int iblock[4] = {0, 1, 4, 5};
int i;
#if CONFIG_HYBRIDTRANSFORM8X8
TX_TYPE tx_type;
#endif
#if CONFIG_COMP_INTRA_PRED
if (b->bmi.as_mode.second == (MB_PREDICTION_MODE)(DC_PRED - 1)) {
......@@ -220,16 +218,20 @@ void vp8_encode_intra8x8(const VP8_ENCODER_RTCD *rtcd,
vp8_subtract_4b_c(be, b, 16);
#if CONFIG_HYBRIDTRANSFORM8X8
txfm_map(b, pred_mode_conv(b->bmi.as_mode.first));
vp8_fht_c(be->src_diff, (x->block + idx)->coeff, 32,
b->bmi.as_mode.tx_type, 8);
x->quantize_b_8x8(x->block + idx, xd->block + idx);
vp8_ihtllm_c(xd->block[idx].dqcoeff, xd->block[ib].diff, 32,
b->bmi.as_mode.tx_type, 8);