Disable CDEF signaling when all segments are lossless.

This required moving the CDEF and lr signaling in the
uncompressed header to after segmentation is signaled.

The all_lossless logic was factored out into onyxc_int.h.

Change-Id: If8de03438d4e9f8cb49ed49b60a880962cc9d28b

av1/common/onyxc_int.h

@@ -323,7 +323,7 @@ typedef struct AV1Common {
 
   struct loopfilter lf;
   struct segmentation seg;
-
+  int all_lossless;
   int frame_parallel_decode;  // frame-based threading.
 
 #if CONFIG_EXT_TX
@@ -1085,6 +1085,22 @@ static INLINE void set_sb_size(AV1_COMMON *const cm, BLOCK_SIZE sb_size) {
 #endif
 }
 
+static INLINE int all_lossless(const AV1_COMMON *cm, const MACROBLOCKD *xd) {
+  int i;
+  int all_lossless = 1;
+  if (cm->seg.enabled) {
+    for (i = 0; i < MAX_SEGMENTS; ++i) {
+      if (!xd->lossless[i]) {
+        all_lossless = 0;
+        break;
+      }
+    }
+  } else {
+    all_lossless = xd->lossless[0];
+  }
+  return all_lossless;
+}
+
 #ifdef __cplusplus
 }  // extern "C"
 #endif

av1/decoder/decodeframe.c

@@ -144,25 +144,11 @@ static int decode_unsigned_max(struct aom_read_bit_buffer *rb, int max) {
   return data > max ? max : data;
 }
 
-static TX_MODE read_tx_mode(AV1_COMMON *cm, MACROBLOCKD *xd,
-                            struct aom_read_bit_buffer *rb) {
-  int i, all_lossless = 1;
+static TX_MODE read_tx_mode(AV1_COMMON *cm, struct aom_read_bit_buffer *rb) {
 #if CONFIG_TX64X64
   TX_MODE tx_mode;
 #endif
-
-  if (cm->seg.enabled) {
-    for (i = 0; i < MAX_SEGMENTS; ++i) {
-      if (!xd->lossless[i]) {
-        all_lossless = 0;
-        break;
-      }
-    }
-  } else {
-    all_lossless = xd->lossless[0];
-  }
-
-  if (all_lossless) return ONLY_4X4;
+  if (cm->all_lossless) return ONLY_4X4;
 #if CONFIG_TX64X64
   tx_mode = aom_rb_read_bit(rb) ? TX_MODE_SELECT : aom_rb_read_literal(rb, 2);
   if (tx_mode == ALLOW_32X32) tx_mode += aom_rb_read_bit(rb);
@@ -2686,7 +2672,7 @@ static void decode_partition(AV1Decoder *const pbi, MACROBLOCKD *const xd,
 
 #if CONFIG_CDEF
   if (bsize == cm->sb_size) {
-    if (!sb_all_skip(cm, mi_row, mi_col)) {
+    if (!sb_all_skip(cm, mi_row, mi_col) && !cm->all_lossless) {
       cm->mi_grid_visible[mi_row * cm->mi_stride + mi_col]->mbmi.cdef_strength =
           aom_read_literal(r, cm->cdef_bits, ACCT_STR);
     } else {
@@ -4738,12 +4724,6 @@ static size_t read_uncompressed_header(AV1Decoder *pbi,
 #endif  // CONFIG_EXT_PARTITION
 
   setup_loopfilter(cm, rb);
-#if CONFIG_CDEF
-  setup_cdef(cm, rb);
-#endif
-#if CONFIG_LOOP_RESTORATION
-  decode_restoration_mode(cm, rb);
-#endif  // CONFIG_LOOP_RESTORATION
   setup_quantization(cm, rb);
   xd->bd = (int)cm->bit_depth;
 
@@ -4801,9 +4781,17 @@ static size_t read_uncompressed_header(AV1Decoder *pbi,
                       cm->uv_dc_delta_q == 0 && cm->uv_ac_delta_q == 0;
     xd->qindex[i] = qindex;
   }
-
+  cm->all_lossless = all_lossless(cm, xd);
   setup_segmentation_dequant(cm);
-  cm->tx_mode = read_tx_mode(cm, xd, rb);
+#if CONFIG_CDEF
+  if (!cm->all_lossless) {
+    setup_cdef(cm, rb);
+  }
+#endif
+#if CONFIG_LOOP_RESTORATION
+  decode_restoration_mode(cm, rb);
+#endif  // CONFIG_LOOP_RESTORATION
+  cm->tx_mode = read_tx_mode(cm, rb);
   cm->reference_mode = read_frame_reference_mode(cm, rb);
 #if CONFIG_EXT_INTER
   read_compound_tools(cm, rb);
@@ -5540,7 +5528,7 @@ void av1_decode_frame(AV1Decoder *pbi, const uint8_t *data,
   }
 
 #if CONFIG_CDEF
-  if (!cm->skip_loop_filter) {
+  if (!cm->skip_loop_filter && !cm->all_lossless) {
     av1_cdef_frame(&pbi->cur_buf->buf, cm, &pbi->mb);
   }
 #endif  // CONFIG_CDEF

av1/encoder/bitstream.c

@@ -3245,7 +3245,7 @@ static void write_modes_sb(AV1_COMP *const cpi, const TileInfo *const tile,
 
 #if CONFIG_CDEF
   if (bsize == cm->sb_size && !sb_all_skip(cm, mi_row, mi_col) &&
-      cm->cdef_bits != 0) {
+      cm->cdef_bits != 0 && !cm->all_lossless) {
     aom_write_literal(w, cm->mi_grid_visible[mi_row * cm->mi_stride + mi_col]
                              ->mbmi.cdef_strength,
                       cm->cdef_bits);
@@ -3870,21 +3870,9 @@ static void update_seg_probs(AV1_COMP *cpi, aom_writer *w) {
 }
 #endif
 
-static void write_tx_mode(AV1_COMMON *cm, MACROBLOCKD *xd, TX_MODE *mode,
+static void write_tx_mode(AV1_COMMON *cm, TX_MODE *mode,
                           struct aom_write_bit_buffer *wb) {
-  int i, all_lossless = 1;
-
-  if (cm->seg.enabled) {
-    for (i = 0; i < MAX_SEGMENTS; ++i) {
-      if (!xd->lossless[i]) {
-        all_lossless = 0;
-        break;
-      }
-    }
-  } else {
-    all_lossless = xd->lossless[0];
-  }
-  if (all_lossless) {
+  if (cm->all_lossless) {
     *mode = ONLY_4X4;
     return;
   }
@@ -4796,12 +4784,6 @@ static void write_uncompressed_header(AV1_COMP *cpi,
 #endif  // CONFIG_EXT_PARTITION
 
   encode_loopfilter(cm, wb);
-#if CONFIG_CDEF
-  encode_cdef(cm, wb);
-#endif
-#if CONFIG_LOOP_RESTORATION
-  encode_restoration_mode(cm, wb);
-#endif  // CONFIG_LOOP_RESTORATION
   encode_quantization(cm, wb);
   encode_segmentation(cm, xd, wb);
 #if CONFIG_DELTA_Q
@@ -4834,8 +4816,15 @@ static void write_uncompressed_header(AV1_COMP *cpi,
     }
   }
 #endif
-
-  write_tx_mode(cm, xd, &cm->tx_mode, wb);
+#if CONFIG_CDEF
+  if (!cm->all_lossless) {
+    encode_cdef(cm, wb);
+  }
+#endif
+#if CONFIG_LOOP_RESTORATION
+  encode_restoration_mode(cm, wb);
+#endif  // CONFIG_LOOP_RESTORATION
+  write_tx_mode(cm, &cm->tx_mode, wb);
 
   if (cpi->allow_comp_inter_inter) {
     const int use_hybrid_pred = cm->reference_mode == REFERENCE_MODE_SELECT;

av1/encoder/encodeframe.c

@@ -4521,20 +4521,8 @@ static MV_REFERENCE_FRAME get_frame_type(const AV1_COMP *cpi) {
     return LAST_FRAME;
 }
 
-static TX_MODE select_tx_mode(const AV1_COMP *cpi, MACROBLOCKD *const xd) {
-  int i, all_lossless = 1;
-
-  if (cpi->common.seg.enabled) {
-    for (i = 0; i < MAX_SEGMENTS; ++i) {
-      if (!xd->lossless[i]) {
-        all_lossless = 0;
-        break;
-      }
-    }
-  } else {
-    all_lossless = xd->lossless[0];
-  }
-  if (all_lossless) return ONLY_4X4;
+static TX_MODE select_tx_mode(const AV1_COMP *cpi) {
+  if (cpi->common.all_lossless) return ONLY_4X4;
   if (cpi->sf.tx_size_search_method == USE_LARGESTALL)
     return ALLOW_32X32 + CONFIG_TX64X64;
   else if (cpi->sf.tx_size_search_method == USE_FULL_RD ||
@@ -5004,10 +4992,10 @@ static void encode_frame_internal(AV1_COMP *cpi) {
                       cm->uv_dc_delta_q == 0 && cm->uv_ac_delta_q == 0;
     xd->qindex[i] = qindex;
   }
-
+  cm->all_lossless = all_lossless(cm, xd);
   if (!cm->seg.enabled && xd->lossless[0]) x->optimize = 0;
 
-  cm->tx_mode = select_tx_mode(cpi, xd);
+  cm->tx_mode = select_tx_mode(cpi);
 
 #if CONFIG_DELTA_Q
   // Fix delta q resolution for the moment