decode_perf_test.cc 8.8 KB
Newer Older
1
/*
2
 * Copyright (c) 2016, Alliance for Open Media. All rights reserved
3
 *
4 5 6 7 8 9 10 11
 * This source code is subject to the terms of the BSD 2 Clause License and
 * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
 * was not distributed with this source code in the LICENSE file, you can
 * obtain it at www.aomedia.org/license/software. If the Alliance for Open
 * Media Patent License 1.0 was not distributed with this source code in the
 * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
*/

12

13
#include <string>
14 15
#include "test/codec_factory.h"
#include "test/decode_test_driver.h"
16 17
#include "test/encode_test_driver.h"
#include "test/i420_video_source.h"
18 19 20 21
#include "test/ivf_video_source.h"
#include "test/md5_helper.h"
#include "test/util.h"
#include "test/webm_video_source.h"
Adrian Grange's avatar
Adrian Grange committed
22
#include "aom_ports/aom_timer.h"
23
#include "./ivfenc.h"
Adrian Grange's avatar
Adrian Grange committed
24
#include "./aom_version.h"
25 26 27 28 29 30 31 32

using std::tr1::make_tuple;

namespace {

#define VIDEO_NAME 0
#define THREADS 1

33
const int kMaxPsnr = 100;
34
const double kUsecsInSec = 1000000.0;
35
const char kNewEncodeOutputFile[] = "new_encode.ivf";
36 37 38 39

/*
 DecodePerfTest takes a tuple of filename + number of threads to decode with
 */
40
typedef std::tr1::tuple<const char *, unsigned> DecodePerfParam;
41

Adrian Grange's avatar
Adrian Grange committed
42
const DecodePerfParam kAV1DecodePerfVectors[] = {
Adrian Grange's avatar
Adrian Grange committed
43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61
  make_tuple("av10-2-bbb_426x240_tile_1x1_180kbps.webm", 1),
  make_tuple("av10-2-bbb_640x360_tile_1x2_337kbps.webm", 2),
  make_tuple("av10-2-bbb_854x480_tile_1x2_651kbps.webm", 2),
  make_tuple("av10-2-bbb_1280x720_tile_1x4_1310kbps.webm", 4),
  make_tuple("av10-2-bbb_1920x1080_tile_1x1_2581kbps.webm", 1),
  make_tuple("av10-2-bbb_1920x1080_tile_1x4_2586kbps.webm", 4),
  make_tuple("av10-2-bbb_1920x1080_tile_1x4_fpm_2304kbps.webm", 4),
  make_tuple("av10-2-sintel_426x182_tile_1x1_171kbps.webm", 1),
  make_tuple("av10-2-sintel_640x272_tile_1x2_318kbps.webm", 2),
  make_tuple("av10-2-sintel_854x364_tile_1x2_621kbps.webm", 2),
  make_tuple("av10-2-sintel_1280x546_tile_1x4_1257kbps.webm", 4),
  make_tuple("av10-2-sintel_1920x818_tile_1x4_fpm_2279kbps.webm", 4),
  make_tuple("av10-2-tos_426x178_tile_1x1_181kbps.webm", 1),
  make_tuple("av10-2-tos_640x266_tile_1x2_336kbps.webm", 2),
  make_tuple("av10-2-tos_854x356_tile_1x2_656kbps.webm", 2),
  make_tuple("av10-2-tos_854x356_tile_1x2_fpm_546kbps.webm", 2),
  make_tuple("av10-2-tos_1280x534_tile_1x4_1306kbps.webm", 4),
  make_tuple("av10-2-tos_1280x534_tile_1x4_fpm_952kbps.webm", 4),
  make_tuple("av10-2-tos_1920x800_tile_1x4_fpm_2335kbps.webm", 4),
62 63 64 65 66 67 68 69 70 71 72 73 74
};

/*
 In order to reflect real world performance as much as possible, Perf tests
 *DO NOT* do any correctness checks. Please run them alongside correctness
 tests to ensure proper codec integrity. Furthermore, in this test we
 deliberately limit the amount of system calls we make to avoid OS
 preemption.

 TODO(joshualitt) create a more detailed perf measurement test to collect
   power/temp/min max frame decode times/etc
 */

clang-format's avatar
clang-format committed
75
class DecodePerfTest : public ::testing::TestWithParam<DecodePerfParam> {};
76 77 78 79 80

TEST_P(DecodePerfTest, PerfTest) {
  const char *const video_name = GET_PARAM(VIDEO_NAME);
  const unsigned threads = GET_PARAM(THREADS);

Yaowu Xu's avatar
Yaowu Xu committed
81
  libaom_test::WebMVideoSource video(video_name);
82 83
  video.Init();

Adrian Grange's avatar
Adrian Grange committed
84
  aom_codec_dec_cfg_t cfg = aom_codec_dec_cfg_t();
85
  cfg.threads = threads;
Adrian Grange's avatar
Adrian Grange committed
86
  libaom_test::AV1Decoder decoder(cfg, 0);
87

Adrian Grange's avatar
Adrian Grange committed
88 89
  aom_usec_timer t;
  aom_usec_timer_start(&t);
90 91 92 93 94

  for (video.Begin(); video.cxdata() != NULL; video.Next()) {
    decoder.DecodeFrame(video.cxdata(), video.frame_size());
  }

Adrian Grange's avatar
Adrian Grange committed
95 96
  aom_usec_timer_mark(&t);
  const double elapsed_secs = double(aom_usec_timer_elapsed(&t)) / kUsecsInSec;
97 98 99 100
  const unsigned frames = video.frame_number();
  const double fps = double(frames) / elapsed_secs;

  printf("{\n");
Joshua Litt's avatar
Joshua Litt committed
101
  printf("\t\"type\" : \"decode_perf_test\",\n");
102 103 104 105 106 107 108 109 110
  printf("\t\"version\" : \"%s\",\n", VERSION_STRING_NOSP);
  printf("\t\"videoName\" : \"%s\",\n", video_name);
  printf("\t\"threadCount\" : %u,\n", threads);
  printf("\t\"decodeTimeSecs\" : %f,\n", elapsed_secs);
  printf("\t\"totalFrames\" : %u,\n", frames);
  printf("\t\"framesPerSecond\" : %f\n", fps);
  printf("}\n");
}

Adrian Grange's avatar
Adrian Grange committed
111 112
INSTANTIATE_TEST_CASE_P(AV1, DecodePerfTest,
                        ::testing::ValuesIn(kAV1DecodePerfVectors));
113

Adrian Grange's avatar
Adrian Grange committed
114
class AV1NewEncodeDecodePerfTest
Yaowu Xu's avatar
Yaowu Xu committed
115 116
    : public ::libaom_test::EncoderTest,
      public ::libaom_test::CodecTestWithParam<libaom_test::TestMode> {
117
 protected:
Adrian Grange's avatar
Adrian Grange committed
118
  AV1NewEncodeDecodePerfTest()
clang-format's avatar
clang-format committed
119 120
      : EncoderTest(GET_PARAM(0)), encoding_mode_(GET_PARAM(1)), speed_(0),
        outfile_(0), out_frames_(0) {}
121

Adrian Grange's avatar
Adrian Grange committed
122
  virtual ~AV1NewEncodeDecodePerfTest() {}
123 124 125 126 127 128 129 130 131 132 133 134 135 136 137

  virtual void SetUp() {
    InitializeConfig();
    SetMode(encoding_mode_);

    cfg_.g_lag_in_frames = 25;
    cfg_.rc_min_quantizer = 2;
    cfg_.rc_max_quantizer = 56;
    cfg_.rc_dropframe_thresh = 0;
    cfg_.rc_undershoot_pct = 50;
    cfg_.rc_overshoot_pct = 50;
    cfg_.rc_buf_sz = 1000;
    cfg_.rc_buf_initial_sz = 500;
    cfg_.rc_buf_optimal_sz = 600;
    cfg_.rc_resize_allowed = 0;
Adrian Grange's avatar
Adrian Grange committed
138
    cfg_.rc_end_usage = AOM_VBR;
139 140
  }

Yaowu Xu's avatar
Yaowu Xu committed
141 142
  virtual void PreEncodeFrameHook(::libaom_test::VideoSource *video,
                                  ::libaom_test::Encoder *encoder) {
143
    if (video->frame() == 1) {
Adrian Grange's avatar
Adrian Grange committed
144
      encoder->Control(AOME_SET_CPUUSED, speed_);
Adrian Grange's avatar
Adrian Grange committed
145 146
      encoder->Control(AV1E_SET_FRAME_PARALLEL_DECODING, 1);
      encoder->Control(AV1E_SET_TILE_COLUMNS, 2);
147 148 149 150
    }
  }

  virtual void BeginPassHook(unsigned int /*pass*/) {
Adrian Grange's avatar
Adrian Grange committed
151
    const std::string data_path = getenv("LIBAOM_TEST_DATA_PATH");
152 153
    const std::string path_to_source = data_path + "/" + kNewEncodeOutputFile;
    outfile_ = fopen(path_to_source.c_str(), "wb");
154
    ASSERT_TRUE(outfile_ != NULL);
155 156 157
  }

  virtual void EndPassHook() {
158
    if (outfile_ != NULL) {
159
      if (!fseek(outfile_, 0, SEEK_SET))
Adrian Grange's avatar
Adrian Grange committed
160
        ivf_write_file_header(outfile_, &cfg_, AV1_FOURCC, out_frames_);
161 162 163 164 165
      fclose(outfile_);
      outfile_ = NULL;
    }
  }

Adrian Grange's avatar
Adrian Grange committed
166
  virtual void FramePktHook(const aom_codec_cx_pkt_t *pkt) {
167 168 169 170
    ++out_frames_;

    // Write initial file header if first frame.
    if (pkt->data.frame.pts == 0)
Adrian Grange's avatar
Adrian Grange committed
171
      ivf_write_file_header(outfile_, &cfg_, AV1_FOURCC, out_frames_);
172 173 174

    // Write frame header and data.
    ivf_write_frame_header(outfile_, out_frames_, pkt->data.frame.sz);
175 176
    ASSERT_EQ(fwrite(pkt->data.frame.buf, 1, pkt->data.frame.sz, outfile_),
              pkt->data.frame.sz);
177 178
  }

179
  virtual bool DoDecode() { return false; }
180

clang-format's avatar
clang-format committed
181
  void set_speed(unsigned int speed) { speed_ = speed; }
182 183

 private:
Yaowu Xu's avatar
Yaowu Xu committed
184
  libaom_test::TestMode encoding_mode_;
185 186 187 188 189 190 191 192
  uint32_t speed_;
  FILE *outfile_;
  uint32_t out_frames_;
};

struct EncodePerfTestVideo {
  EncodePerfTestVideo(const char *name_, uint32_t width_, uint32_t height_,
                      uint32_t bitrate_, int frames_)
clang-format's avatar
clang-format committed
193
      : name(name_), width(width_), height(height_), bitrate(bitrate_),
194 195 196 197 198 199 200 201
        frames(frames_) {}
  const char *name;
  uint32_t width;
  uint32_t height;
  uint32_t bitrate;
  int frames;
};

Adrian Grange's avatar
Adrian Grange committed
202
const EncodePerfTestVideo kAV1EncodePerfTestVectors[] = {
203 204 205
  EncodePerfTestVideo("niklas_1280_720_30.yuv", 1280, 720, 600, 470),
};

Adrian Grange's avatar
Adrian Grange committed
206
TEST_P(AV1NewEncodeDecodePerfTest, PerfTest) {
207 208 209 210
  SetUp();

  // TODO(JBB): Make this work by going through the set of given files.
  const int i = 0;
Adrian Grange's avatar
Adrian Grange committed
211
  const aom_rational timebase = { 33333333, 1000000000 };
212
  cfg_.g_timebase = timebase;
Adrian Grange's avatar
Adrian Grange committed
213
  cfg_.rc_target_bitrate = kAV1EncodePerfTestVectors[i].bitrate;
214

Adrian Grange's avatar
Adrian Grange committed
215
  init_flags_ = AOM_CODEC_USE_PSNR;
216

Adrian Grange's avatar
Adrian Grange committed
217
  const char *video_name = kAV1EncodePerfTestVectors[i].name;
Yaowu Xu's avatar
Yaowu Xu committed
218
  libaom_test::I420VideoSource video(
Adrian Grange's avatar
Adrian Grange committed
219 220 221
      video_name, kAV1EncodePerfTestVectors[i].width,
      kAV1EncodePerfTestVectors[i].height, timebase.den, timebase.num, 0,
      kAV1EncodePerfTestVectors[i].frames);
222 223 224 225 226 227
  set_speed(2);

  ASSERT_NO_FATAL_FAILURE(RunLoop(&video));

  const uint32_t threads = 4;

Yaowu Xu's avatar
Yaowu Xu committed
228
  libaom_test::IVFVideoSource decode_video(kNewEncodeOutputFile);
229 230
  decode_video.Init();

Adrian Grange's avatar
Adrian Grange committed
231
  aom_codec_dec_cfg_t cfg = aom_codec_dec_cfg_t();
232
  cfg.threads = threads;
Adrian Grange's avatar
Adrian Grange committed
233
  libaom_test::AV1Decoder decoder(cfg, 0);
234

Adrian Grange's avatar
Adrian Grange committed
235 236
  aom_usec_timer t;
  aom_usec_timer_start(&t);
237 238 239 240 241 242

  for (decode_video.Begin(); decode_video.cxdata() != NULL;
       decode_video.Next()) {
    decoder.DecodeFrame(decode_video.cxdata(), decode_video.frame_size());
  }

Adrian Grange's avatar
Adrian Grange committed
243
  aom_usec_timer_mark(&t);
244
  const double elapsed_secs =
Adrian Grange's avatar
Adrian Grange committed
245
      static_cast<double>(aom_usec_timer_elapsed(&t)) / kUsecsInSec;
246
  const unsigned decode_frames = decode_video.frame_number();
247
  const double fps = static_cast<double>(decode_frames) / elapsed_secs;
248 249 250 251 252 253 254 255 256 257 258 259

  printf("{\n");
  printf("\t\"type\" : \"decode_perf_test\",\n");
  printf("\t\"version\" : \"%s\",\n", VERSION_STRING_NOSP);
  printf("\t\"videoName\" : \"%s\",\n", kNewEncodeOutputFile);
  printf("\t\"threadCount\" : %u,\n", threads);
  printf("\t\"decodeTimeSecs\" : %f,\n", elapsed_secs);
  printf("\t\"totalFrames\" : %u,\n", decode_frames);
  printf("\t\"framesPerSecond\" : %f\n", fps);
  printf("}\n");
}

Adrian Grange's avatar
Adrian Grange committed
260
AV1_INSTANTIATE_TEST_CASE(AV1NewEncodeDecodePerfTest,
Yaowu Xu's avatar
Yaowu Xu committed
261
                           ::testing::Values(::libaom_test::kTwoPassGood));
262
}  // namespace