Frame buffer marking enabling non-intra-based loss recovery

Change-Id: Iaaf9244678cde74cf3e216f3b8c9d293f206f27e

av1/av1_cx_iface.c

@@ -1049,6 +1049,12 @@ static aom_codec_err_t encoder_encode(aom_codec_alg_priv_t *ctx,
            -1 != av1_get_compressed_data(cpi, &lib_flags, &size, cx_data,
                                          &dst_time_stamp, &dst_end_time_stamp,
                                          !img)) {
+#if CONFIG_REFERENCE_BUFFER
+      if (cpi->common.invalid_delta_frame_id_minus1) {
+        ctx->base.err_detail = "Invalid delta_frame_id_minus1";
+        return AOM_CODEC_ERROR;
+      }
+#endif
       if (size) {
         aom_codec_cx_pkt_t pkt;
 

av1/av1_dx_iface.c

@@ -212,7 +212,18 @@ static aom_codec_err_t decoder_peek_si_internal(
     si->is_kf = !aom_rb_read_bit(&rb);
     show_frame = aom_rb_read_bit(&rb);
     error_resilient = aom_rb_read_bit(&rb);
-
+#if CONFIG_REFERENCE_BUFFER
+    {
+      /* TODO: Move outside frame loop or inside key-frame branch */
+      int FidLen;
+      SequenceHeader seq_params;
+      read_sequence_header(&seq_params);
+      if (seq_params.frame_id_numbers_present_flag) {
+        FidLen = seq_params.frame_id_length_minus7 + 7;
+        aom_rb_read_literal(&rb, FidLen);
+      }
+    }
+#endif
     if (si->is_kf) {
       if (!av1_read_sync_code(&rb)) return AOM_CODEC_UNSUP_BITSTREAM;
 

av1/common/onyxc_int.h

@@ -48,6 +48,13 @@ extern "C" {
 // normal reference pool.
 #define FRAME_BUFFERS (REF_FRAMES + 7)
 
+#if CONFIG_REFERENCE_BUFFER
+/* Constant values while waiting for the sequence header */
+#define FRAME_ID_NUMBERS_PRESENT_FLAG 1
+#define FRAME_ID_LENGTH_MINUS7 8         // Allows frame id up to 2^15-1
+#define DELTA_FRAME_ID_LENGTH_MINUS2 12  // Allows frame id deltas up to 2^14-1
+#endif
+
 #if CONFIG_EXT_REFS
 #define FRAME_CONTEXTS_LOG2 3
 #else
@@ -404,8 +411,24 @@ typedef struct AV1Common {
 #if CONFIG_TILE_GROUPS
   int num_tg;
 #endif
+#if CONFIG_REFERENCE_BUFFER
+  int current_frame_id;
+  int ref_frame_id[REF_FRAMES];
+  int valid_for_referencing[REF_FRAMES];
+  int refresh_mask;
+  int invalid_delta_frame_id_minus1;
+#endif
 } AV1_COMMON;
 
+#if CONFIG_REFERENCE_BUFFER
+/* Initial version of sequence header structure */
+typedef struct SequenceHeader {
+  int frame_id_numbers_present_flag;
+  int frame_id_length_minus7;
+  int delta_frame_id_length_minus2;
+} SequenceHeader;
+#endif
+
 // TODO(hkuang): Don't need to lock the whole pool after implementing atomic
 // frame reference count.
 static void lock_buffer_pool(BufferPool *const pool) {

av1/decoder/decodeframe.c

@@ -3507,6 +3507,15 @@ static void read_bitdepth_colorspace_sampling(AV1_COMMON *cm,
   }
 }
 
+#if CONFIG_REFERENCE_BUFFER
+void read_sequence_header(SequenceHeader *seq_params) {
+  /* Placeholder for actually reading from the bitstream */
+  seq_params->frame_id_numbers_present_flag = FRAME_ID_NUMBERS_PRESENT_FLAG;
+  seq_params->frame_id_length_minus7 = FRAME_ID_LENGTH_MINUS7;
+  seq_params->delta_frame_id_length_minus2 = DELTA_FRAME_ID_LENGTH_MINUS2;
+}
+#endif
+
 static size_t read_uncompressed_header(AV1Decoder *pbi,
                                        struct aom_read_bit_buffer *rb) {
   AV1_COMMON *const cm = &pbi->common;
@@ -3515,6 +3524,12 @@ static size_t read_uncompressed_header(AV1Decoder *pbi,
   RefCntBuffer *const frame_bufs = pool->frame_bufs;
   int i, mask, ref_index = 0;
   size_t sz;
+
+#if CONFIG_REFERENCE_BUFFER
+  /* TODO: Move outside frame loop or inside key-frame branch */
+  read_sequence_header(&pbi->seq_params);
+#endif
+
   cm->last_frame_type = cm->frame_type;
   cm->last_intra_only = cm->intra_only;
 
@@ -3543,7 +3558,18 @@ static size_t read_uncompressed_header(AV1Decoder *pbi,
   if (cm->show_existing_frame) {
     // Show an existing frame directly.
     const int frame_to_show = cm->ref_frame_map[aom_rb_read_literal(rb, 3)];
-
+#if CONFIG_REFERENCE_BUFFER
+    if (pbi->seq_params.frame_id_numbers_present_flag) {
+      int FidLen = pbi->seq_params.frame_id_length_minus7 + 7;
+      int display_frame_id = aom_rb_read_literal(rb, FidLen);
+      /* Compare display_frame_id with ref_frame_id and check valid for
+      * referencing */
+      if (display_frame_id != cm->ref_frame_id[frame_to_show] ||
+          cm->valid_for_referencing[frame_to_show] == 0)
+        aom_internal_error(&cm->error, AOM_CODEC_CORRUPT_FRAME,
+                           "Reference buffer frame ID mismatch");
+    }
+#endif
     lock_buffer_pool(pool);
     if (frame_to_show < 0 || frame_bufs[frame_to_show].ref_count < 1) {
       unlock_buffer_pool(pool);
@@ -3569,7 +3595,47 @@ static size_t read_uncompressed_header(AV1Decoder *pbi,
   cm->frame_type = (FRAME_TYPE)aom_rb_read_bit(rb);
   cm->show_frame = aom_rb_read_bit(rb);
   cm->error_resilient_mode = aom_rb_read_bit(rb);
+#if CONFIG_REFERENCE_BUFFER
+  if (pbi->seq_params.frame_id_numbers_present_flag) {
+    int FidLen = pbi->seq_params.frame_id_length_minus7 + 7;
+    int DiffLen = pbi->seq_params.delta_frame_id_length_minus2 + 2;
+    int PrevFrameId = 0;
+    if (cm->frame_type != KEY_FRAME) {
+      PrevFrameId = cm->current_frame_id;
+    }
+    cm->current_frame_id = aom_rb_read_literal(rb, FidLen);
 
+    if (cm->frame_type != KEY_FRAME) {
+      int DiffFrameID;
+      if (cm->current_frame_id > PrevFrameId) {
+        DiffFrameID = cm->current_frame_id - PrevFrameId;
+      } else {
+        DiffFrameID = (1 << FidLen) + cm->current_frame_id - PrevFrameId;
+      }
+      /* Check current_frame_id for conformance */
+      if (PrevFrameId == cm->current_frame_id ||
+          DiffFrameID >= (1 << (FidLen - 1))) {
+        aom_internal_error(&cm->error, AOM_CODEC_CORRUPT_FRAME,
+                           "Invalid value of current_frame_id");
+      }
+    }
+    /* Check if some frames need to be marked as not valid for referencing */
+    for (i = 0; i < REF_FRAMES; i++) {
+      if (cm->frame_type == KEY_FRAME) {
+        cm->valid_for_referencing[i] = 0;
+      } else if (cm->current_frame_id - (1 << DiffLen) > 0) {
+        if (cm->ref_frame_id[i] > cm->current_frame_id ||
+            cm->ref_frame_id[i] < cm->current_frame_id - (1 << DiffLen))
+          cm->valid_for_referencing[i] = 0;
+      } else {
+        if (cm->ref_frame_id[i] > cm->current_frame_id &&
+            cm->ref_frame_id[i] <
+                (1 << FidLen) + cm->current_frame_id - (1 << DiffLen))
+          cm->valid_for_referencing[i] = 0;
+      }
+    }
+  }
+#endif
   if (cm->frame_type == KEY_FRAME) {
     if (!av1_read_sync_code(rb))
       aom_internal_error(&cm->error, AOM_CODEC_UNSUP_BITSTREAM,
@@ -3645,6 +3711,22 @@ static size_t read_uncompressed_header(AV1Decoder *pbi,
         ref_frame->idx = idx;
         ref_frame->buf = &frame_bufs[idx].buf;
         cm->ref_frame_sign_bias[LAST_FRAME + i] = aom_rb_read_bit(rb);
+#if CONFIG_REFERENCE_BUFFER
+        if (pbi->seq_params.frame_id_numbers_present_flag) {
+          int FidLen = pbi->seq_params.frame_id_length_minus7 + 7;
+          int DiffLen = pbi->seq_params.delta_frame_id_length_minus2 + 2;
+          int delta_frame_id_minus1 = aom_rb_read_literal(rb, DiffLen);
+          int refFrameId = ((cm->current_frame_id -
+                             (delta_frame_id_minus1 + 1) + (1 << FidLen)) %
+                            (1 << FidLen));
+          /* Compare values derived from delta_frame_id_minus1 and
+          * refresh_frame_flags. Also, check valid for referencing */
+          if (refFrameId != cm->ref_frame_id[ref] ||
+              cm->valid_for_referencing[ref] == 0)
+            aom_internal_error(&cm->error, AOM_CODEC_CORRUPT_FRAME,
+                               "Reference buffer frame ID mismatch");
+        }
+#endif
       }
 
 #if CONFIG_FRAME_SIZE
@@ -3675,6 +3757,22 @@ static size_t read_uncompressed_header(AV1Decoder *pbi,
       }
     }
   }
+
+#if CONFIG_REFERENCE_BUFFER
+  if (pbi->seq_params.frame_id_numbers_present_flag) {
+    /* If bitmask is set, update reference frame id values and
+    mark frames as valid for reference */
+    int refresh_frame_flags =
+        cm->frame_type == KEY_FRAME ? 0xFF : pbi->refresh_frame_flags;
+    for (i = 0; i < REF_FRAMES; i++) {
+      if ((refresh_frame_flags >> i) & 1) {
+        cm->ref_frame_id[i] = cm->current_frame_id;
+        cm->valid_for_referencing[i] = 1;
+      }
+    }
+  }
+#endif
+
 #if CONFIG_AOM_HIGHBITDEPTH
   get_frame_new_buffer(cm)->bit_depth = cm->bit_depth;
 #endif

av1/decoder/decodeframe.h

@@ -19,6 +19,11 @@ extern "C" {
 struct AV1Decoder;
 struct aom_read_bit_buffer;
 
+#if CONFIG_REFERENCE_BUFFER
+/* Placeholder for now */
+void read_sequence_header(SequenceHeader *seq_params);
+#endif
+
 int av1_read_sync_code(struct aom_read_bit_buffer *const rb);
 void av1_read_frame_size(struct aom_read_bit_buffer *rb, int *width,
                          int *height);

av1/decoder/decoder.h

@@ -128,6 +128,9 @@ typedef struct AV1Decoder {
   int tg_start;  // First tile in the current tilegroup
   int tg_size_bit_offset;
 #endif
+#if CONFIG_REFERENCE_BUFFER
+  SequenceHeader seq_params;
+#endif
 } AV1Decoder;
 
 int av1_receive_compressed_data(struct AV1Decoder *pbi, size_t size,

av1/encoder/bitstream.c

@@ -3632,11 +3632,25 @@ static void write_bitdepth_colorspace_sampling(
   }
 }
 
+#if CONFIG_REFERENCE_BUFFER
+void write_sequence_header(SequenceHeader *seq_params) {
+  /* Placeholder for actually writing to the bitstream */
+  seq_params->frame_id_numbers_present_flag = FRAME_ID_NUMBERS_PRESENT_FLAG;
+  seq_params->frame_id_length_minus7 = FRAME_ID_LENGTH_MINUS7;
+  seq_params->delta_frame_id_length_minus2 = DELTA_FRAME_ID_LENGTH_MINUS2;
+}
+#endif
+
 static void write_uncompressed_header(AV1_COMP *cpi,
                                       struct aom_write_bit_buffer *wb) {
   AV1_COMMON *const cm = &cpi->common;
   MACROBLOCKD *const xd = &cpi->td.mb.e_mbd;
 
+#if CONFIG_REFERENCE_BUFFER
+  /* TODO: Move outside frame loop or inside key-frame branch */
+  write_sequence_header(&cpi->seq_params);
+#endif
+
   aom_wb_write_literal(wb, AOM_FRAME_MARKER, 2);
 
   write_profile(cm->profile, wb);
@@ -3671,6 +3685,14 @@ static void write_uncompressed_header(AV1_COMP *cpi,
   aom_wb_write_bit(wb, cm->show_frame);
   aom_wb_write_bit(wb, cm->error_resilient_mode);
 
+#if CONFIG_REFERENCE_BUFFER
+  cm->invalid_delta_frame_id_minus1 = 0;
+  if (cpi->seq_params.frame_id_numbers_present_flag) {
+    int FidLen = cpi->seq_params.frame_id_length_minus7 + 7;
+    aom_wb_write_literal(wb, cm->current_frame_id, FidLen);
+  }
+#endif
+
   if (cm->frame_type == KEY_FRAME) {
     write_sync_code(wb);
     write_bitdepth_colorspace_sampling(cm, wb);
@@ -3732,6 +3754,21 @@ static void write_uncompressed_header(AV1_COMP *cpi,
         aom_wb_write_literal(wb, get_ref_frame_map_idx(cpi, ref_frame),
                              REF_FRAMES_LOG2);
         aom_wb_write_bit(wb, cm->ref_frame_sign_bias[ref_frame]);
+#if CONFIG_REFERENCE_BUFFER
+        if (cpi->seq_params.frame_id_numbers_present_flag) {
+          int i = get_ref_frame_map_idx(cpi, ref_frame);
+          int FidLen = cpi->seq_params.frame_id_length_minus7 + 7;
+          int DiffLen = cpi->seq_params.delta_frame_id_length_minus2 + 2;
+          int delta_frame_id_minus1 =
+              ((cm->current_frame_id - cm->ref_frame_id[i] + (1 << FidLen)) %
+               (1 << FidLen)) -
+              1;
+          if (delta_frame_id_minus1 < 0 ||
+              delta_frame_id_minus1 >= (1 << DiffLen))
+            cm->invalid_delta_frame_id_minus1 = 1;
+          aom_wb_write_literal(wb, delta_frame_id_minus1, DiffLen);
+        }
+#endif
       }
 
 #if CONFIG_FRAME_SIZE
@@ -3751,6 +3788,10 @@ static void write_uncompressed_header(AV1_COMP *cpi,
     }
   }
 
+#if CONFIG_REFERENCE_BUFFER
+  cm->refresh_mask = cm->frame_type == KEY_FRAME ? 0xFF : get_refresh_mask(cpi);
+#endif
+
   if (!cm->error_resilient_mode) {
     aom_wb_write_bit(
         wb, cm->refresh_frame_context == REFRESH_FRAME_CONTEXT_FORWARD);

av1/encoder/bitstream.h

@@ -18,6 +18,10 @@ extern "C" {
 
 #include "av1/encoder/encoder.h"
 
+#if CONFIG_REFERENCE_BUFFER
+void write_sequence_header(SequenceHeader *seq_params);
+#endif
+
 void av1_pack_bitstream(AV1_COMP *const cpi, uint8_t *dest, size_t *size);
 
 void av1_encode_token_init(void);

av1/encoder/encoder.c

@@ -4609,6 +4609,23 @@ static void encode_frame_to_data_rate(AV1_COMP *cpi, size_t *size,
          MAX_MODES * sizeof(*cpi->mode_chosen_counts));
 #endif
 
+#if CONFIG_REFERENCE_BUFFER
+  {
+    /* Non-normative definition of current_frame_id ("frame counter" with
+    * wraparound) */
+    int FidLen = FRAME_ID_LENGTH_MINUS7 + 7;
+    if (cm->current_frame_id == -1) {
+      /* quasi-random initialization of current_frame_id for a key frame */
+      int lsb = cpi->Source->y_buffer[0] & 0xff;
+      int msb = cpi->Source->y_buffer[1] & 0xff;
+      cm->current_frame_id = ((msb << 8) + lsb) % (1 << FidLen);
+    } else {
+      cm->current_frame_id =
+          (cm->current_frame_id + 1 + (1 << FidLen)) % (1 << FidLen);
+    }
+  }
+#endif
+
   if (cpi->sf.recode_loop == DISALLOW_RECODE) {
     encode_without_recode_loop(cpi);
   } else {
@@ -4659,6 +4676,18 @@ static void encode_frame_to_data_rate(AV1_COMP *cpi, size_t *size,
   // Build the bitstream
   av1_pack_bitstream(cpi, dest, size);
 
+#if CONFIG_REFERENCE_BUFFER
+  {
+    int i;
+    /* Update reference frame id values based on the value of refresh_mask */
+    for (i = 0; i < REF_FRAMES; i++) {
+      if ((cm->refresh_mask >> i) & 1) {
+        cm->ref_frame_id[i] = cm->current_frame_id;
+      }
+    }
+  }
+#endif
+
 #if DUMP_RECON_FRAMES == 1
   // NOTE(zoeliu): For debug - Output the filtered reconstructed video.
   if (cm->show_frame) dump_filtered_recon_frames(cpi);
@@ -5371,6 +5400,12 @@ int av1_get_compressed_data(AV1_COMP *cpi, unsigned int *frame_flags,
   cm->max_qmlevel = cpi->oxcf.qm_maxlevel;
 #endif
 
+#if CONFIG_REFERENCE_BUFFER
+  if (*time_stamp == 0) {
+    cpi->common.current_frame_id = -1;
+  }
+#endif
+
   if (oxcf->pass == 1) {
     cpi->td.mb.e_mbd.lossless[0] = is_lossless_requested(oxcf);
     av1_first_pass(cpi, source);

av1/encoder/encoder.h

@@ -622,6 +622,9 @@ typedef struct AV1_COMP {
 #if CONFIG_GLOBAL_MOTION
   int global_motion_used[TOTAL_REFS_PER_FRAME];
 #endif
+#if CONFIG_REFERENCE_BUFFER
+  SequenceHeader seq_params;
+#endif
 } AV1_COMP;
 
 void av1_initialize_enc(void);

configure

@@ -295,6 +295,7 @@ EXPERIMENT_LIST="
     ec_adapt
     simp_mv_pred
     rd_debug
+    reference_buffer
 "
 CONFIG_LIST="
     dependency_tracking