diff --git a/av1/encoder/block.h b/av1/encoder/block.h
index 2807274ab420ccb11c2849fe3625fa603a91524b..5c7aeb4b2a16280d49b9b55e54efa505772b07b7 100644
--- a/av1/encoder/block.h
+++ b/av1/encoder/block.h
@@ -166,6 +166,10 @@ struct macroblock {
 
   int skip;
 
+#if CONFIG_CB4X4
+  int skip_chroma_rd;
+#endif
+
   // note that token_costs is the cost when eob node is skipped
   av1_coeff_cost token_costs[TX_SIZES];
 
diff --git a/av1/encoder/encodeframe.c b/av1/encoder/encodeframe.c
index 57357025deecec7cc0c99b4553b1f7dfface2c4a..69e428bfd41c8f9a3780e58cf29972947d3b3c74 100644
--- a/av1/encoder/encodeframe.c
+++ b/av1/encoder/encodeframe.c
@@ -1852,6 +1852,11 @@ static void rd_pick_sb_modes(const AV1_COMP *const cpi, TileDataEnc *tile_data,
   // Set to zero to make sure we do not use the previous encoded frame stats
   mbmi->skip = 0;
 
+#if CONFIG_CB4X4
+  x->skip_chroma_rd =
+      (bsize < BLOCK_8X8) && !is_chroma_reference(mi_row, mi_col);
+#endif
+
 #if CONFIG_AOM_HIGHBITDEPTH
   if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) {
     x->source_variance = av1_high_get_sby_perpixel_variance(
diff --git a/av1/encoder/rdopt.c b/av1/encoder/rdopt.c
index 16540d7e7443c976960682746848d5db9e7a8839..e0b9784903e04fb77fdb425a09e6d53221eb5110 100644
--- a/av1/encoder/rdopt.c
+++ b/av1/encoder/rdopt.c
@@ -9035,8 +9035,10 @@ void av1_rd_pick_intra_mode_sb(const AV1_COMP *cpi, MACROBLOCK *x,
                                    [pd[1].subsampling_x][pd[1].subsampling_y];
 
 #if CONFIG_CB4X4
-  rd_pick_intra_sbuv_mode(cpi, x, &rate_uv, &rate_uv_tokenonly, &dist_uv,
-                          &uv_skip, bsize, max_uv_tx_size);
+  max_uv_tx_size = AOMMAX(max_uv_tx_size, TX_4X4);
+  if (!x->skip_chroma_rd)
+    rd_pick_intra_sbuv_mode(cpi, x, &rate_uv, &rate_uv_tokenonly, &dist_uv,
+                            &uv_skip, bsize, max_uv_tx_size);
 #else
   rd_pick_intra_sbuv_mode(cpi, x, &rate_uv, &rate_uv_tokenonly, &dist_uv,
                           &uv_skip, AOMMAX(BLOCK_8X8, bsize), max_uv_tx_size);