error_resilience_test.cc 9.26 KB
Newer Older
1
/*
2
 * Copyright (c) 2016, Alliance for Open Media. All rights reserved
Frank Galligan's avatar
Frank Galligan committed
3
 *
4 5 6 7 8 9 10 11
 * This source code is subject to the terms of the BSD 2 Clause License and
 * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
 * was not distributed with this source code in the LICENSE file, you can
 * obtain it at www.aomedia.org/license/software. If the Alliance for Open
 * Media Patent License 1.0 was not distributed with this source code in the
 * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
*/

12
#include "third_party/googletest/src/include/gtest/gtest.h"
13
#include "test/codec_factory.h"
14 15
#include "test/encode_test_driver.h"
#include "test/i420_video_source.h"
16
#include "test/util.h"
17 18 19

namespace {

20 21
const int kMaxErrorFrames = 12;
const int kMaxDroppableFrames = 12;
22

clang-format's avatar
clang-format committed
23
class ErrorResilienceTestLarge
Yaowu Xu's avatar
Yaowu Xu committed
24 25
    : public ::libaom_test::EncoderTest,
      public ::libaom_test::CodecTestWith2Params<libaom_test::TestMode, bool> {
26
 protected:
27
  ErrorResilienceTestLarge()
28 29
      : EncoderTest(GET_PARAM(0)), psnr_(0.0), nframes_(0), mismatch_psnr_(0.0),
        mismatch_nframes_(0), encoding_mode_(GET_PARAM(1)) {
30 31
    Reset();
  }
32

33
  virtual ~ErrorResilienceTestLarge() {}
34

35 36 37
  void Reset() {
    error_nframes_ = 0;
    droppable_nframes_ = 0;
Marco's avatar
Marco committed
38
    pattern_switch_ = 0;
39 40
  }

41 42 43 44 45 46 47 48
  virtual void SetUp() {
    InitializeConfig();
    SetMode(encoding_mode_);
  }

  virtual void BeginPassHook(unsigned int /*pass*/) {
    psnr_ = 0.0;
    nframes_ = 0;
49 50
    mismatch_psnr_ = 0.0;
    mismatch_nframes_ = 0;
51 52
  }

Adrian Grange's avatar
Adrian Grange committed
53
  virtual void PSNRPktHook(const aom_codec_cx_pkt_t *pkt) {
54 55 56 57
    psnr_ += pkt->data.psnr.psnr[0];
    nframes_++;
  }

58
  virtual void PreEncodeFrameHook(libaom_test::VideoSource *video) {
clang-format's avatar
clang-format committed
59
    frame_flags_ &=
Adrian Grange's avatar
Adrian Grange committed
60
        ~(AOM_EFLAG_NO_UPD_LAST | AOM_EFLAG_NO_UPD_GF | AOM_EFLAG_NO_UPD_ARF);
61
    // For temporal layer case.
62 63
    if (droppable_nframes_ > 0 &&
        (cfg_.g_pass == AOM_RC_LAST_PASS || cfg_.g_pass == AOM_RC_ONE_PASS)) {
64
      for (unsigned int i = 0; i < droppable_nframes_; ++i) {
65
        if (droppable_frames_[i] == video->frame()) {
clang-format's avatar
clang-format committed
66 67
          std::cout << "Encoding droppable frame: " << droppable_frames_[i]
                    << "\n";
68 69 70
          frame_flags_ |= (AOM_EFLAG_NO_UPD_LAST | AOM_EFLAG_NO_UPD_GF |
                           AOM_EFLAG_NO_UPD_ARF);
          return;
clang-format's avatar
clang-format committed
71 72
        }
      }
73 74 75
    }
  }

76
  double GetAveragePsnr() const {
clang-format's avatar
clang-format committed
77
    if (nframes_) return psnr_ / nframes_;
78 79 80
    return 0.0;
  }

81
  double GetAverageMismatchPsnr() const {
clang-format's avatar
clang-format committed
82
    if (mismatch_nframes_) return mismatch_psnr_ / mismatch_nframes_;
83 84 85 86 87
    return 0.0;
  }

  virtual bool DoDecode() const {
    if (error_nframes_ > 0 &&
Adrian Grange's avatar
Adrian Grange committed
88
        (cfg_.g_pass == AOM_RC_LAST_PASS || cfg_.g_pass == AOM_RC_ONE_PASS)) {
89 90 91 92 93 94 95 96 97 98 99
      for (unsigned int i = 0; i < error_nframes_; ++i) {
        if (error_frames_[i] == nframes_ - 1) {
          std::cout << "             Skipping decoding frame: "
                    << error_frames_[i] << "\n";
          return 0;
        }
      }
    }
    return 1;
  }

Adrian Grange's avatar
Adrian Grange committed
100
  virtual void MismatchHook(const aom_image_t *img1, const aom_image_t *img2) {
101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126
    double mismatch_psnr = compute_psnr(img1, img2);
    mismatch_psnr_ += mismatch_psnr;
    ++mismatch_nframes_;
    // std::cout << "Mismatch frame psnr: " << mismatch_psnr << "\n";
  }

  void SetErrorFrames(int num, unsigned int *list) {
    if (num > kMaxErrorFrames)
      num = kMaxErrorFrames;
    else if (num < 0)
      num = 0;
    error_nframes_ = num;
    for (unsigned int i = 0; i < error_nframes_; ++i)
      error_frames_[i] = list[i];
  }

  void SetDroppableFrames(int num, unsigned int *list) {
    if (num > kMaxDroppableFrames)
      num = kMaxDroppableFrames;
    else if (num < 0)
      num = 0;
    droppable_nframes_ = num;
    for (unsigned int i = 0; i < droppable_nframes_; ++i)
      droppable_frames_[i] = list[i];
  }

clang-format's avatar
clang-format committed
127
  unsigned int GetMismatchFrames() { return mismatch_nframes_; }
128

clang-format's avatar
clang-format committed
129
  void SetPatternSwitch(int frame_switch) { pattern_switch_ = frame_switch; }
Marco's avatar
Marco committed
130

131 132 133
 private:
  double psnr_;
  unsigned int nframes_;
134 135
  unsigned int error_nframes_;
  unsigned int droppable_nframes_;
Marco's avatar
Marco committed
136
  unsigned int pattern_switch_;
137 138 139 140
  double mismatch_psnr_;
  unsigned int mismatch_nframes_;
  unsigned int error_frames_[kMaxErrorFrames];
  unsigned int droppable_frames_[kMaxDroppableFrames];
Yaowu Xu's avatar
Yaowu Xu committed
141
  libaom_test::TestMode encoding_mode_;
142 143
};

144
TEST_P(ErrorResilienceTestLarge, OnVersusOff) {
Adrian Grange's avatar
Adrian Grange committed
145
  const aom_rational timebase = { 33333333, 1000000000 };
146 147
  cfg_.g_timebase = timebase;
  cfg_.rc_target_bitrate = 2000;
148
  cfg_.g_lag_in_frames = 10;
149

Adrian Grange's avatar
Adrian Grange committed
150
  init_flags_ = AOM_CODEC_USE_PSNR;
151

Yaowu Xu's avatar
Yaowu Xu committed
152
  libaom_test::I420VideoSource video("hantro_collage_w352h288.yuv", 352, 288,
153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174
                                     timebase.den, timebase.num, 0, 30);

  // Error resilient mode OFF.
  cfg_.g_error_resilient = 0;
  ASSERT_NO_FATAL_FAILURE(RunLoop(&video));
  const double psnr_resilience_off = GetAveragePsnr();
  EXPECT_GT(psnr_resilience_off, 25.0);

  // Error resilient mode ON.
  cfg_.g_error_resilient = 1;
  ASSERT_NO_FATAL_FAILURE(RunLoop(&video));
  const double psnr_resilience_on = GetAveragePsnr();
  EXPECT_GT(psnr_resilience_on, 25.0);

  // Test that turning on error resilient mode hurts by 10% at most.
  if (psnr_resilience_off > 0.0) {
    const double psnr_ratio = psnr_resilience_on / psnr_resilience_off;
    EXPECT_GE(psnr_ratio, 0.9);
    EXPECT_LE(psnr_ratio, 1.1);
  }
}

175 176 177 178
// Check for successful decoding and no encoder/decoder mismatch
// if we lose (i.e., drop before decoding) a set of droppable
// frames (i.e., frames that don't update any reference buffers).
// Check both isolated and consecutive loss.
179
TEST_P(ErrorResilienceTestLarge, DropFramesWithoutRecovery) {
Adrian Grange's avatar
Adrian Grange committed
180
  const aom_rational timebase = { 33333333, 1000000000 };
181
  cfg_.g_timebase = timebase;
182
  cfg_.rc_target_bitrate = 500;
183 184 185
  // FIXME(debargha): Fix this to work for any lag.
  // Currently this test only works for lag = 0
  cfg_.g_lag_in_frames = 0;
186

Adrian Grange's avatar
Adrian Grange committed
187
  init_flags_ = AOM_CODEC_USE_PSNR;
188

Yaowu Xu's avatar
Yaowu Xu committed
189
  libaom_test::I420VideoSource video("hantro_collage_w352h288.yuv", 352, 288,
190
                                     timebase.den, timebase.num, 0, 40);
191 192 193

  // Error resilient mode ON.
  cfg_.g_error_resilient = 1;
Adrian Grange's avatar
Adrian Grange committed
194
  cfg_.kf_mode = AOM_KF_DISABLED;
195 196 197 198 199

  // Set an arbitrary set of error frames same as droppable frames.
  // In addition to isolated loss/drop, add a long consecutive series
  // (of size 9) of dropped frames.
  unsigned int num_droppable_frames = 11;
clang-format's avatar
clang-format committed
200 201
  unsigned int droppable_frame_list[] = { 5,  16, 22, 23, 24, 25,
                                          26, 27, 28, 29, 30 };
202 203 204 205
  SetDroppableFrames(num_droppable_frames, droppable_frame_list);
  SetErrorFrames(num_droppable_frames, droppable_frame_list);
  ASSERT_NO_FATAL_FAILURE(RunLoop(&video));
  // Test that no mismatches have been found
clang-format's avatar
clang-format committed
206 207
  std::cout << "             Mismatch frames: " << GetMismatchFrames() << "\n";
  EXPECT_EQ(GetMismatchFrames(), (unsigned int)0);
208

209
  // Reset previously set of error/droppable frames.
210 211
  Reset();

212 213 214 215 216
#if 0
  // TODO(jkoleszar): This test is disabled for the time being as too
  // sensitive. It's not clear how to set a reasonable threshold for
  // this behavior.

217 218 219 220 221
  // Now set an arbitrary set of error frames that are non-droppable
  unsigned int num_error_frames = 3;
  unsigned int error_frame_list[] = {3, 10, 20};
  SetErrorFrames(num_error_frames, error_frame_list);
  ASSERT_NO_FATAL_FAILURE(RunLoop(&video));
222

223 224 225 226 227 228 229 230
  // Test that dropping an arbitrary set of inter frames does not hurt too much
  // Note the Average Mismatch PSNR is the average of the PSNR between
  // decoded frame and encoder's version of the same frame for all frames
  // with mismatch.
  const double psnr_resilience_mismatch = GetAverageMismatchPsnr();
  std::cout << "             Mismatch PSNR: "
            << psnr_resilience_mismatch << "\n";
  EXPECT_GT(psnr_resilience_mismatch, 20.0);
231
#endif
232 233
}

clang-format's avatar
clang-format committed
234
class ErrorResilienceTestLargeCodecControls
Yaowu Xu's avatar
Yaowu Xu committed
235 236
    : public ::libaom_test::EncoderTest,
      public ::libaom_test::CodecTestWithParam<libaom_test::TestMode> {
Marco's avatar
Marco committed
237 238
 protected:
  ErrorResilienceTestLargeCodecControls()
clang-format's avatar
clang-format committed
239
      : EncoderTest(GET_PARAM(0)), encoding_mode_(GET_PARAM(1)) {
Marco's avatar
Marco committed
240 241 242 243 244 245 246 247
    Reset();
  }

  virtual ~ErrorResilienceTestLargeCodecControls() {}

  void Reset() {
    last_pts_ = 0;
    tot_frame_number_ = 0;
248
    bits_total_ = 0;
Marco's avatar
Marco committed
249 250 251 252 253 254 255 256
    duration_ = 0.0;
  }

  virtual void SetUp() {
    InitializeConfig();
    SetMode(encoding_mode_);
  }

Adrian Grange's avatar
Adrian Grange committed
257
  virtual void FramePktHook(const aom_codec_cx_pkt_t *pkt) {
Marco's avatar
Marco committed
258
    // Time since last timestamp = duration.
Adrian Grange's avatar
Adrian Grange committed
259
    aom_codec_pts_t duration = pkt->data.frame.pts - last_pts_;
Marco's avatar
Marco committed
260 261 262 263 264 265 266 267
    if (duration > 1) {
      // Update counter for total number of frames (#frames input to encoder).
      // Needed for setting the proper layer_id below.
      tot_frame_number_ += static_cast<int>(duration - 1);
    }
    const size_t frame_size_in_bits = pkt->data.frame.sz * 8;
    // Update the total encoded bits. For temporal layers, update the cumulative
    // encoded bits per layer.
268
    bits_total_ += frame_size_in_bits;
Marco's avatar
Marco committed
269 270 271 272 273
    // Update the most recent pts.
    last_pts_ = pkt->data.frame.pts;
    ++tot_frame_number_;
  }

274
  virtual void EndPassHook(void) { duration_ = (last_pts_ + 1) * timebase_; }
clang-format's avatar
clang-format committed
275 276

 private:
Yaowu Xu's avatar
Yaowu Xu committed
277
  libaom_test::TestMode encoding_mode_;
Adrian Grange's avatar
Adrian Grange committed
278
  aom_codec_pts_t last_pts_;
clang-format's avatar
clang-format committed
279
  double timebase_;
280
  int64_t bits_total_;
clang-format's avatar
clang-format committed
281 282 283
  double duration_;
  int tot_frame_number_;
};
Marco's avatar
Marco committed
284

285
AV1_INSTANTIATE_TEST_CASE(ErrorResilienceTestLarge, ONE_PASS_TEST_MODES,
clang-format's avatar
clang-format committed
286
                          ::testing::Values(false));
287
}  // namespace