diff --git a/vp10/encoder/encoder.c b/vp10/encoder/encoder.c
index 266c02dccd914900e178c264f81beae68b57a0fb..dfafb2a7530b4b2edf60218eb11d07c0f6cc0ef8 100644
--- a/vp10/encoder/encoder.c
+++ b/vp10/encoder/encoder.c
@@ -3260,7 +3260,8 @@ void vp10_update_reference_frames(VP10_COMP *cpi) {
         // NOTE: The ALT_REFs' are indexed reversely, and ALT0 refers to the
         //       farthest ALT_REF from the first frame in the gf group.
         int tmp = cpi->arf_map[0];
-        cpi->arf_map[0] = cpi->bwd_fb_idx;
+        cpi->arf_map[0] = cpi->alt_fb_idx;
+        cpi->alt_fb_idx = cpi->bwd_fb_idx;
         cpi->bwd_fb_idx = tmp;
       }
       ref_cnt_fb(pool->frame_bufs, &cm->ref_frame_map[cpi->bwd_fb_idx],
@@ -3309,7 +3310,14 @@ void vp10_update_reference_frames(VP10_COMP *cpi) {
     //      v                v                v
     // lst_fb_idxes[2], lst_fb_idxes[0], lst_fb_idxes[1]
     int ref_frame;
-
+    if (cpi->rc.is_bwd_ref_frame && cpi->num_extra_arfs) {
+      // We have swapped the virtual indices to use ALT0 as BWD_REF
+      // and we need to swap them back.
+      int tmp = cpi->arf_map[0];
+      cpi->arf_map[0] = cpi->alt_fb_idx;
+      cpi->alt_fb_idx = cpi->bwd_fb_idx;
+      cpi->bwd_fb_idx = tmp;
+    }
     if (cm->frame_type == KEY_FRAME) {
       for (ref_frame = 0; ref_frame < LAST_REF_FRAMES; ++ref_frame) {
         ref_cnt_fb(pool->frame_bufs,
diff --git a/vp10/encoder/firstpass.c b/vp10/encoder/firstpass.c
index 1d1bf520165c64d0e0baadd7c39b9e4993a5d833..2b3c85f783dda90ef6c79c6c9d430f8ef014e70b 100644
--- a/vp10/encoder/firstpass.c
+++ b/vp10/encoder/firstpass.c
@@ -2578,7 +2578,18 @@ static void configure_buffer_updates(VP10_COMP *cpi) {
       cpi->refresh_last_frame = 1;
       cpi->refresh_golden_frame = 0;
 #if CONFIG_EXT_REFS
-      cpi->refresh_bwd_ref_frame = 0;
+      // If we have extra ALT_REFs, we can use the farthest ALT (ALT0) as
+      // the BWD_REF.
+      if (cpi->num_extra_arfs) {
+        int tmp = cpi->bwd_fb_idx;
+
+        cpi->rc.is_bwd_ref_frame = 1;
+        cpi->bwd_fb_idx = cpi->alt_fb_idx;
+        cpi->alt_fb_idx = cpi->arf_map[0];;
+        cpi->arf_map[0] = tmp;
+      } else {
+        cpi->rc.is_bwd_ref_frame = 0;
+      }
 #endif  // CONFIG_EXT_REFS
       cpi->refresh_alt_ref_frame = 0;
       break;
@@ -2619,12 +2630,13 @@ static void configure_buffer_updates(VP10_COMP *cpi) {
       cpi->refresh_alt_ref_frame = 0;
       cpi->rc.is_bwd_ref_frame = 1;
       if (cpi->num_extra_arfs) {
-        // Allow BRF uses the farthest ALT_REF (ALT0) as BWD_REF by swapping
+        // Allow BRF use the farthest ALT_REF (ALT0) as BWD_REF by swapping
         // the virtual indices.
         // NOTE: The indices will be swapped back after this frame is encoded
         //       (in vp10_update_reference_frames()).
         int tmp = cpi->bwd_fb_idx;
-        cpi->bwd_fb_idx = cpi->arf_map[0];
+        cpi->bwd_fb_idx = cpi->alt_fb_idx;
+        cpi->alt_fb_idx = cpi->arf_map[0];
         cpi->arf_map[0] = tmp;
       }
       break;