Add a simple frame data inspection API.

This patch adds a decoder control that exposes frame data via a simple
 callback.

Change-Id: Icae73ae6b5da8a7783db9fadb1fff4a85d77174b

aom/aomdx.h

@@ -41,6 +41,23 @@ extern aom_codec_iface_t *aom_codec_av1_dx(void);
  */
 typedef struct Accounting Accounting;
 
+/** Callback that inspects decoder frame data.
+ */
+typedef void (*aom_inspect_cb)(void *decoder, void *ctx);
+
+/*!\brief Structure to hold inspection callback and context.
+ *
+ * Defines a structure to hold the inspection callback function and calling
+ * context.
+ */
+typedef struct aom_inspect_init {
+  /*! Inspection callback. */
+  aom_inspect_cb inspect_cb;
+
+  /*! Inspection context. */
+  void *inspect_ctx;
+} aom_inspect_init;
+
 /*!\enum aom_dec_control_id
  * \brief AOM decoder control functions
  *
@@ -129,6 +146,12 @@ enum aom_dec_control_id {
   AV1_SET_DECODE_TILE_ROW,
   AV1_SET_DECODE_TILE_COL,
 
+  /** control function to set an aom_inspect_cb callback that is invoked each
+   * time a frame is decoded.  When compiled without --enable-inspection, this
+   * returns AOM_CODEC_INCAPABLE.
+   */
+  AV1_SET_INSPECTION_CALLBACK,
+
   AOM_DECODER_CTRL_ID_MAX,
 };
 
@@ -184,6 +207,8 @@ AOM_CTRL_USE_TYPE(AV1_SET_DECODE_TILE_ROW, int)
 #define AOM_CTRL_AV1_SET_DECODE_TILE_ROW
 AOM_CTRL_USE_TYPE(AV1_SET_DECODE_TILE_COL, int)
 #define AOM_CTRL_AV1_SET_DECODE_TILE_COL
+AOM_CTRL_USE_TYPE(AV1_SET_INSPECTION_CALLBACK, aom_inspect_init *)
+#define AOM_CTRL_AV1_SET_INSPECTION_CALLBACK
 /*!\endcond */
 /*! @} - end defgroup aom_decoder */
 

av1/av1.cmake

@@ -85,6 +85,13 @@ set(AOM_AV1_DECODER_SOURCES
     "${AOM_ROOT}/av1/decoder/dthread.c"
     "${AOM_ROOT}/av1/decoder/dthread.h")
 
+if (CONFIG_INSPECTION)
+  set(AOM_AV1_DECODER_SOURCES
+      ${AOM_AV1_DECODER_SOURCES}
+      "${AOM_ROOT}/av1/decoder/inspection.c"
+      "${AOM_ROOT}/av1/decoder/inspection.h")
+endif ()
+
 set(AOM_AV1_ENCODER_SOURCES
     "${AOM_ROOT}/av1/av1_cx_iface.c"
     "${AOM_ROOT}/av1/encoder/aq_complexity.c"

av1/av1_dx.mk

@@ -33,6 +33,11 @@ AV1_DX_SRCS-yes += decoder/decoder.h
 AV1_DX_SRCS-yes += decoder/dsubexp.c
 AV1_DX_SRCS-yes += decoder/dsubexp.h
 
+ifeq ($(CONFIG_INSPECTION),yes)
+AV1_DX_SRCS-yes += decoder/inspection.c
+AV1_DX_SRCS-yes += decoder/inspection.h
+endif
+
 ifeq ($(CONFIG_PVQ),yes)
 # PVQ from daala
 AV1_DX_SRCS-yes += decoder/pvq_decoder.c

av1/av1_dx_iface.c

@@ -80,6 +80,11 @@ struct aom_codec_alg_priv {
   void *ext_priv;  // Private data associated with the external frame buffers.
   aom_get_frame_buffer_cb_fn_t get_ext_fb_cb;
   aom_release_frame_buffer_cb_fn_t release_ext_fb_cb;
+
+#if CONFIG_INSPECTION
+  aom_inspect_cb inspect_cb;
+  void *inspect_ctx;
+#endif
 };
 
 static aom_codec_err_t decoder_init(aom_codec_ctx_t *ctx,
@@ -483,6 +488,10 @@ static aom_codec_err_t decode_one(aom_codec_alg_priv_t *ctx,
     // decrypt config between frames.
     frame_worker_data->pbi->decrypt_cb = ctx->decrypt_cb;
     frame_worker_data->pbi->decrypt_state = ctx->decrypt_state;
+#if CONFIG_INSPECTION
+    frame_worker_data->pbi->inspect_cb = ctx->inspect_cb;
+    frame_worker_data->pbi->inspect_ctx = ctx->inspect_ctx;
+#endif
 
 #if CONFIG_EXT_TILE
     frame_worker_data->pbi->dec_tile_row = ctx->decode_tile_row;
@@ -1133,6 +1142,20 @@ static aom_codec_err_t ctrl_set_decode_tile_col(aom_codec_alg_priv_t *ctx,
   return AOM_CODEC_OK;
 }
 
+static aom_codec_err_t ctrl_set_inspection_callback(aom_codec_alg_priv_t *ctx,
+                                                    va_list args) {
+#if !CONFIG_INSPECTION
+  (void)ctx;
+  (void)args;
+  return AOM_CODEC_INCAPABLE;
+#else
+  aom_inspect_init *init = va_arg(args, aom_inspect_init *);
+  ctx->inspect_cb = init->inspect_cb;
+  ctx->inspect_ctx = init->inspect_ctx;
+  return AOM_CODEC_OK;
+#endif
+}
+
 static aom_codec_ctrl_fn_map_t decoder_ctrl_maps[] = {
   { AOM_COPY_REFERENCE, ctrl_copy_reference },
 
@@ -1149,6 +1172,7 @@ static aom_codec_ctrl_fn_map_t decoder_ctrl_maps[] = {
   { AV1_SET_SKIP_LOOP_FILTER, ctrl_set_skip_loop_filter },
   { AV1_SET_DECODE_TILE_ROW, ctrl_set_decode_tile_row },
   { AV1_SET_DECODE_TILE_COL, ctrl_set_decode_tile_col },
+  { AV1_SET_INSPECTION_CALLBACK, ctrl_set_inspection_callback },
 
   // Getters
   { AOMD_GET_FRAME_CORRUPTED, ctrl_get_frame_corrupted },

av1/decoder/decodeframe.c

@@ -33,6 +33,9 @@
 #include "av1/common/clpf.h"
 #include "av1/common/dering.h"
 #endif
+#if CONFIG_INSPECTION
+#include "av1/decoder/inspection.h"
+#endif
 #include "av1/common/common.h"
 #include "av1/common/entropy.h"
 #include "av1/common/entropymode.h"
@@ -5034,6 +5037,12 @@ void av1_decode_frame(AV1Decoder *pbi, const uint8_t *data,
                        "Decode failed. Frame data is corrupted.");
   }
 
+#if CONFIG_INSPECTION
+  if (pbi->inspect_cb != NULL) {
+    (*pbi->inspect_cb)(pbi, pbi->inspect_ctx);
+  }
+#endif
+
   // Non frame parallel update frame context here.
   if (!cm->error_resilient_mode && !context_updated)
     cm->frame_contexts[cm->frame_context_idx] = *cm->fc;

av1/decoder/decoder.h

@@ -25,6 +25,9 @@
 #if CONFIG_ACCOUNTING
 #include "av1/common/accounting.h"
 #endif
+#if CONFIG_INSPECTION
+#include "av1/decoder/inspection.h"
+#endif
 
 #if CONFIG_PVQ
 #include "aom_dsp/entdec.h"
@@ -137,6 +140,10 @@ typedef struct AV1Decoder {
 #if CONFIG_REFERENCE_BUFFER
   SequenceHeader seq_params;
 #endif
+#if CONFIG_INSPECTION
+  aom_inspect_cb inspect_cb;
+  void *inspect_ctx;
+#endif
 } AV1Decoder;
 
 int av1_receive_compressed_data(struct AV1Decoder *pbi, size_t size,

av1/decoder/inspection.c

+#include "av1/decoder/decoder.h"
+#include "av1/decoder/inspection.h"
+#include "av1/common/enums.h"
+
+void ifd_init(insp_frame_data *fd, int frame_width, int frame_height) {
+  fd->mi_cols = ALIGN_POWER_OF_TWO(frame_width, MI_SIZE_LOG2) >> MI_SIZE_LOG2;
+  fd->mi_rows = ALIGN_POWER_OF_TWO(frame_height, MI_SIZE_LOG2) >> MI_SIZE_LOG2;
+  fd->mi_grid = (insp_mi_data *)aom_malloc(sizeof(insp_mi_data) * fd->mi_rows *
+                                           fd->mi_cols);
+}
+
+void ifd_clear(insp_frame_data *fd) {
+  aom_free(fd->mi_grid);
+  fd->mi_grid = NULL;
+}
+
+/* TODO(negge) This function may be called by more than one thread when using
+               a multi-threaded decoder and this may cause a data race. */
+int ifd_inspect(insp_frame_data *fd, void *decoder) {
+  struct AV1Decoder *pbi = (struct AV1Decoder *)decoder;
+  AV1_COMMON *const cm = &pbi->common;
+  // TODO(negge): Should this function just call ifd_clear() and ifd_init()?
+  if (fd->mi_rows != cm->mi_rows || fd->mi_cols != cm->mi_cols) {
+    return 0;
+  }
+  fd->show_frame = cm->show_frame;
+  fd->frame_type = cm->frame_type;
+  fd->base_qindex = cm->base_qindex;
+#if CONFIG_ACCOUNTING
+  fd->accounting = &pbi->accounting;
+#endif
+#if CONFIG_CDEF
+// TODO(negge): copy per frame CDEF data
+#endif
+  int i, j;
+  for (i = 0; i < MAX_SEGMENTS; i++) {
+    for (j = 0; j < 2; j++) {
+      fd->y_dequant[i][j] = cm->y_dequant[i][j];
+      fd->uv_dequant[i][j] = cm->uv_dequant[i][j];
+    }
+  }
+  for (j = 0; j < cm->mi_rows; j++) {
+    for (i = 0; i < cm->mi_cols; i++) {
+      const MB_MODE_INFO *mbmi =
+          &cm->mi_grid_visible[j * cm->mi_stride + i]->mbmi;
+      insp_mi_data *mi = &fd->mi_grid[j * cm->mi_cols + j];
+      // Segment
+      mi->segment_id = mbmi->segment_id;
+      // Motion Vectors
+      mi->mv[0].row = mbmi->mv[0].as_mv.row;
+      mi->mv[0].col = mbmi->mv[0].as_mv.col;
+      mi->mv[1].row = mbmi->mv[1].as_mv.row;
+      mi->mv[1].col = mbmi->mv[1].as_mv.col;
+      // Reference Frames
+      mi->ref_frame[0] = mbmi->ref_frame[0];
+      mi->ref_frame[1] = mbmi->ref_frame[1];
+      // Prediction Mode
+      mi->mode = mbmi->mode;
+      // Block Size
+      mi->sb_type = mbmi->sb_type;
+      // Skip Flag
+      mi->skip = mbmi->skip;
+      // Filters
+      mi->filter = mbmi->interp_filter;
+      // Transform
+      mi->tx_type = mbmi->tx_type;
+      mi->tx_size = mbmi->tx_size;
+#if CONFIG_CDEF
+// TODO(negge): copy per block CDEF data
+#endif
+    }
+  }
+  return 1;
+}

av1/decoder/inspection.h

+#ifndef AOM_INSPECTION_H_
+#define AOM_INSPECTION_H_
+
+#if CONFIG_ACCOUNTING
+#include "av1/common/accounting.h"
+#endif
+
+typedef void (*aom_inspect_cb)(void *decoder, void *data);
+
+typedef struct insp_mv insp_mv;
+
+struct insp_mv {
+  int16_t row;
+  int16_t col;
+};
+
+typedef struct insp_mi_data insp_mi_data;
+
+struct insp_mi_data {
+  insp_mv mv[2];
+  int8_t ref_frame[2];
+  int8_t mode;
+  int8_t sb_type;
+  int8_t skip;
+  int8_t segment_id;
+  int8_t filter;
+  int8_t tx_type;
+  int8_t tx_size;
+#if CONFIG_CDEF
+// TODO(negge): add per block CDEF data
+#endif
+};
+
+typedef struct insp_frame_data insp_frame_data;
+
+struct insp_frame_data {
+#if CONFIG_ACCOUNTING
+  Accounting *accounting;
+#endif
+  insp_mi_data *mi_grid;
+  int show_frame;
+  int frame_type;
+  int base_qindex;
+  int mi_rows;
+  int mi_cols;
+  int16_t y_dequant[MAX_SEGMENTS][2];
+  int16_t uv_dequant[MAX_SEGMENTS][2];
+#if CONFIG_CDEF
+// TODO(negge): add per frame CDEF data
+#endif
+};
+
+void ifd_init(insp_frame_data *fd, int frame_width, int frame_height);
+void ifd_clear(insp_frame_data *fd);
+int ifd_inspect(insp_frame_data *fd, void *decoder);
+
+#endif

configure

@@ -60,6 +60,7 @@ Advanced options:
   ${toggle_webm_io}               enable input from and output to WebM container
   ${toggle_libyuv}                enable libyuv
   ${toggle_accounting}            enable bit accounting
+  ${toggle_inspection}            enable bitstream inspection
 
 Codecs:
   Codecs can be selectively enabled or disabled individually, or by family:
@@ -354,6 +355,7 @@ CONFIG_LIST="
     webm_io
     libyuv
     accounting
+    inspection
     decode_perf_tests
     encode_perf_tests
     multi_res_encoding
@@ -415,6 +417,7 @@ CMDLINE_SELECT="
     webm_io
     libyuv
     accounting
+    inspection
     decode_perf_tests
     encode_perf_tests
     multi_res_encoding