diff --git a/aom_dsp/variance.c b/aom_dsp/variance.c
index cb5924a170d234b66e933d271e73faeccb7a0ece..a4c3616e73da775c43b921f31768a7b02a6930f3 100644
--- a/aom_dsp/variance.c
+++ b/aom_dsp/variance.c
@@ -718,23 +718,21 @@ void aom_highbd_upsampled_pred_c(uint16_t *comp_pred, int width, int height,
                                   CONVERT_TO_BYTEPTR(comp_pred), width, NULL,
                                   -1, kernel, 16, width, height, bd);
     } else {
-      DECLARE_ALIGNED(16, uint8_t,
-                      temp[((MAX_SB_SIZE * 2 + 16) + 16) * MAX_SB_SIZE]);
-      const uint16_t *ref;
+      DECLARE_ALIGNED(16, uint16_t,
+                      temp[((MAX_SB_SIZE + 16) + 16) * MAX_SB_SIZE]);
       const int16_t *kernel_x;
       const int16_t *kernel_y;
       int intermediate_height;
-      ref = CONVERT_TO_SHORTPTR(ref8);
       kernel_x = av1_get_interp_filter_subpel_kernel(filter, subpel_x_q3 << 1);
       kernel_y = av1_get_interp_filter_subpel_kernel(filter, subpel_y_q3 << 1);
       intermediate_height =
           (((height - 1) * 8 + subpel_y_q3) >> 3) + filter.taps;
       assert(intermediate_height <= (MAX_SB_SIZE * 2 + 16) + 16);
       /*Directly call C versions to allow this to work for small (2x2) sizes.*/
-      aom_highbd_convolve8_horiz_c(
-          CONVERT_TO_BYTEPTR(ref - ref_stride * ((filter.taps >> 1) - 1)),
-          ref_stride, CONVERT_TO_BYTEPTR(temp), MAX_SB_SIZE, kernel_x, 16, NULL,
-          -1, width, intermediate_height, bd);
+      aom_highbd_convolve8_horiz_c(ref8 - ref_stride * ((filter.taps >> 1) - 1),
+                                   ref_stride, CONVERT_TO_BYTEPTR(temp),
+                                   MAX_SB_SIZE, kernel_x, 16, NULL, -1, width,
+                                   intermediate_height, bd);
       aom_highbd_convolve8_vert_c(
           CONVERT_TO_BYTEPTR(temp + MAX_SB_SIZE * ((filter.taps >> 1) - 1)),
           MAX_SB_SIZE, CONVERT_TO_BYTEPTR(comp_pred), width, NULL, -1, kernel_y,
diff --git a/aom_dsp/x86/highbd_variance_sse2.c b/aom_dsp/x86/highbd_variance_sse2.c
index 033ebe6eb412c52f73d0ecd3f7af13ef2b390953..93923ffb080f46e9ce105c2be8b55c25a867a340 100644
--- a/aom_dsp/x86/highbd_variance_sse2.c
+++ b/aom_dsp/x86/highbd_variance_sse2.c
@@ -658,21 +658,19 @@ void aom_highbd_upsampled_pred_sse2(uint16_t *comp_pred, int width, int height,
                                 width, NULL, -1, kernel, 16, width, height, bd);
     } else {
       DECLARE_ALIGNED(16, uint16_t,
-                      temp[((MAX_SB_SIZE * 2 + 16) + 16) * MAX_SB_SIZE]);
-      const uint16_t *ref;
+                      temp[((MAX_SB_SIZE + 16) + 16) * MAX_SB_SIZE]);
       const int16_t *kernel_x;
       const int16_t *kernel_y;
       int intermediate_height;
-      ref = CONVERT_TO_SHORTPTR(ref8);
       kernel_x = av1_get_interp_filter_subpel_kernel(filter, subpel_x_q3 << 1);
       kernel_y = av1_get_interp_filter_subpel_kernel(filter, subpel_y_q3 << 1);
       intermediate_height =
           (((height - 1) * 8 + subpel_y_q3) >> 3) + filter.taps;
       assert(intermediate_height <= (MAX_SB_SIZE * 2 + 16) + 16);
-      aom_highbd_convolve8_horiz(
-          CONVERT_TO_BYTEPTR(ref - ref_stride * ((filter.taps >> 1) - 1)),
-          ref_stride, CONVERT_TO_BYTEPTR(temp), MAX_SB_SIZE, kernel_x, 16, NULL,
-          -1, width, intermediate_height, bd);
+      aom_highbd_convolve8_horiz(ref8 - ref_stride * ((filter.taps >> 1) - 1),
+                                 ref_stride, CONVERT_TO_BYTEPTR(temp),
+                                 MAX_SB_SIZE, kernel_x, 16, NULL, -1, width,
+                                 intermediate_height, bd);
       aom_highbd_convolve8_vert(
           CONVERT_TO_BYTEPTR(temp + MAX_SB_SIZE * ((filter.taps >> 1) - 1)),
           MAX_SB_SIZE, CONVERT_TO_BYTEPTR(comp_pred), width, NULL, -1, kernel_y,