temporal_filter.c 24.7 KB
Newer Older
Jingning Han's avatar
Jingning Han committed
1
/*
Yaowu Xu's avatar
Yaowu Xu committed
2
 * Copyright (c) 2016, Alliance for Open Media. All rights reserved
Jingning Han's avatar
Jingning Han committed
3
 *
Yaowu Xu's avatar
Yaowu Xu committed
4
5
6
7
8
9
 * This source code is subject to the terms of the BSD 2 Clause License and
 * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
 * was not distributed with this source code in the LICENSE file, you can
 * obtain it at www.aomedia.org/license/software. If the Alliance for Open
 * Media Patent License 1.0 was not distributed with this source code in the
 * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
Jingning Han's avatar
Jingning Han committed
10
11
12
13
14
 */

#include <math.h>
#include <limits.h>

15
#include "./aom_config.h"
16
17
18
19
20
#include "av1/common/alloccommon.h"
#include "av1/common/onyxc_int.h"
#include "av1/common/quant_common.h"
#include "av1/common/reconinter.h"
#include "av1/common/odintrin.h"
21
#include "av1/encoder/av1_quantize.h"
22
23
24
25
26
27
28
#include "av1/encoder/extend.h"
#include "av1/encoder/firstpass.h"
#include "av1/encoder/mcomp.h"
#include "av1/encoder/encoder.h"
#include "av1/encoder/ratectrl.h"
#include "av1/encoder/segmentation.h"
#include "av1/encoder/temporal_filter.h"
Yaowu Xu's avatar
Yaowu Xu committed
29
30
#include "aom_dsp/aom_dsp_common.h"
#include "aom_mem/aom_mem.h"
31
#include "aom_ports/mem.h"
Yaowu Xu's avatar
Yaowu Xu committed
32
33
#include "aom_ports/aom_timer.h"
#include "aom_scale/aom_scale.h"
Jingning Han's avatar
Jingning Han committed
34

35
36
37
38
static void temporal_filter_predictors_mb_c(
    MACROBLOCKD *xd, uint8_t *y_mb_ptr, uint8_t *u_mb_ptr, uint8_t *v_mb_ptr,
    int stride, int uv_block_width, int uv_block_height, int mv_row, int mv_col,
    uint8_t *pred, struct scale_factors *scale, int x, int y) {
Jingning Han's avatar
Jingning Han committed
39
40
41
42
  const int which_mv = 0;
  const MV mv = { mv_row, mv_col };
  enum mv_precision mv_precision_uv;
  int uv_stride;
43
  // TODO(angiebird): change plane setting accordingly
44
  ConvolveParams conv_params = get_conv_params(which_mv, which_mv, 0);
45

46
#if USE_TEMPORALFILTER_12TAP
47
#if CONFIG_DUAL_FILTER
James Zern's avatar
James Zern committed
48
49
50
51
  const InterpFilter interp_filter[4] = { TEMPORALFILTER_12TAP,
                                          TEMPORALFILTER_12TAP,
                                          TEMPORALFILTER_12TAP,
                                          TEMPORALFILTER_12TAP };
52
#else
James Zern's avatar
James Zern committed
53
  const InterpFilter interp_filter = TEMPORALFILTER_12TAP;
54
#endif
55
  (void)xd;
56
#else
James Zern's avatar
James Zern committed
57
  const InterpFilter interp_filter = xd->mi[0]->mbmi.interp_filter;
58
#endif  // USE_TEMPORALFILTER_12TAP
59
60
61
62
#if CONFIG_GLOBAL_MOTION || CONFIG_WARPED_MOTION
  WarpTypesAllowed warp_types;
  memset(&warp_types, 0, sizeof(WarpTypesAllowed));
#endif  // CONFIG_GLOBAL_MOTION || CONFIG_WARPED_MOTION
63

Jingning Han's avatar
Jingning Han committed
64
65
66
67
68
69
70
71
  if (uv_block_width == 8) {
    uv_stride = (stride + 1) >> 1;
    mv_precision_uv = MV_PRECISION_Q4;
  } else {
    uv_stride = stride;
    mv_precision_uv = MV_PRECISION_Q3;
  }

72
#if CONFIG_HIGHBITDEPTH
Jingning Han's avatar
Jingning Han committed
73
  if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) {
Yaowu Xu's avatar
Yaowu Xu committed
74
75
    av1_highbd_build_inter_predictor(y_mb_ptr, stride, &pred[0], 16, &mv, scale,
                                     16, 16, which_mv, interp_filter,
76
77
78
#if CONFIG_GLOBAL_MOTION || CONFIG_WARPED_MOTION
                                     &warp_types, x, y,
#endif  // CONFIG_GLOBAL_MOTION || CONFIG_WARPED_MOTION
79
                                     0, MV_PRECISION_Q3, x, y, xd);
Yaowu Xu's avatar
Yaowu Xu committed
80
81
82
83

    av1_highbd_build_inter_predictor(u_mb_ptr, uv_stride, &pred[256],
                                     uv_block_width, &mv, scale, uv_block_width,
                                     uv_block_height, which_mv, interp_filter,
84
85
86
#if CONFIG_GLOBAL_MOTION || CONFIG_WARPED_MOTION
                                     &warp_types, x, y,
#endif  // CONFIG_GLOBAL_MOTION || CONFIG_WARPED_MOTION
87
                                     1, mv_precision_uv, x, y, xd);
Yaowu Xu's avatar
Yaowu Xu committed
88
89
90
91

    av1_highbd_build_inter_predictor(v_mb_ptr, uv_stride, &pred[512],
                                     uv_block_width, &mv, scale, uv_block_width,
                                     uv_block_height, which_mv, interp_filter,
92
93
94
#if CONFIG_GLOBAL_MOTION || CONFIG_WARPED_MOTION
                                     &warp_types, x, y,
#endif  // CONFIG_GLOBAL_MOTION || CONFIG_WARPED_MOTION
95
                                     2, mv_precision_uv, x, y, xd);
Jingning Han's avatar
Jingning Han committed
96
97
    return;
  }
98
#endif  // CONFIG_HIGHBITDEPTH
Yaowu Xu's avatar
Yaowu Xu committed
99
  av1_build_inter_predictor(y_mb_ptr, stride, &pred[0], 16, &mv, scale, 16, 16,
100
                            &conv_params, interp_filter,
101
102
103
#if CONFIG_GLOBAL_MOTION || CONFIG_WARPED_MOTION
                            &warp_types, x, y, 0, 0,
#endif  // CONFIG_GLOBAL_MOTION || CONFIG_WARPED_MOTION
104
                            MV_PRECISION_Q3, x, y, xd);
105

Yaowu Xu's avatar
Yaowu Xu committed
106
107
  av1_build_inter_predictor(u_mb_ptr, uv_stride, &pred[256], uv_block_width,
                            &mv, scale, uv_block_width, uv_block_height,
108
                            &conv_params, interp_filter,
109
110
111
#if CONFIG_GLOBAL_MOTION || CONFIG_WARPED_MOTION
                            &warp_types, x, y, 1, 0,
#endif  // CONFIG_GLOBAL_MOTION || CONFIG_WARPED_MOTION
112
                            mv_precision_uv, x, y, xd);
113

Yaowu Xu's avatar
Yaowu Xu committed
114
115
  av1_build_inter_predictor(v_mb_ptr, uv_stride, &pred[512], uv_block_width,
                            &mv, scale, uv_block_width, uv_block_height,
116
                            &conv_params, interp_filter,
117
118
119
#if CONFIG_GLOBAL_MOTION || CONFIG_WARPED_MOTION
                            &warp_types, x, y, 2, 0,
#endif  // CONFIG_GLOBAL_MOTION || CONFIG_WARPED_MOTION
120
                            mv_precision_uv, x, y, xd);
Jingning Han's avatar
Jingning Han committed
121
122
}

Yaowu Xu's avatar
Yaowu Xu committed
123
124
125
126
127
void av1_temporal_filter_apply_c(uint8_t *frame1, unsigned int stride,
                                 uint8_t *frame2, unsigned int block_width,
                                 unsigned int block_height, int strength,
                                 int filter_weight, unsigned int *accumulator,
                                 uint16_t *count) {
Jingning Han's avatar
Jingning Han committed
128
129
130
131
132
133
134
  unsigned int i, j, k;
  int modifier;
  int byte = 0;
  const int rounding = strength > 0 ? 1 << (strength - 1) : 0;

  for (i = 0, k = 0; i < block_height; i++) {
    for (j = 0; j < block_width; j++, k++) {
Jingning Han's avatar
Jingning Han committed
135
136
137
138
139
140
141
142
      int pixel_value = *frame2;

      // non-local mean approach
      int diff_sse[9] = { 0 };
      int idx, idy, index = 0;

      for (idy = -1; idy <= 1; ++idy) {
        for (idx = -1; idx <= 1; ++idx) {
Debargha Mukherjee's avatar
Debargha Mukherjee committed
143
144
          int row = (int)i + idy;
          int col = (int)j + idx;
Jingning Han's avatar
Jingning Han committed
145

146
147
          if (row >= 0 && row < (int)block_height && col >= 0 &&
              col < (int)block_width) {
Jingning Han's avatar
Jingning Han committed
148
            int diff = frame1[byte + idy * (int)stride + idx] -
149
                       frame2[idy * (int)block_width + idx];
Jingning Han's avatar
Jingning Han committed
150
151
152
153
154
155
156
157
158
            diff_sse[index] = diff * diff;
            ++index;
          }
        }
      }

      assert(index > 0);

      modifier = 0;
159
      for (idx = 0; idx < 9; ++idx) modifier += diff_sse[idx];
Jingning Han's avatar
Jingning Han committed
160
161
162
163
164

      modifier *= 3;
      modifier /= index;

      ++frame2;
Jingning Han's avatar
Jingning Han committed
165

166
      modifier += rounding;
Jingning Han's avatar
Jingning Han committed
167
168
      modifier >>= strength;

169
      if (modifier > 16) modifier = 16;
Jingning Han's avatar
Jingning Han committed
170
171
172
173
174
175
176
177
178
179
180
181
182
183

      modifier = 16 - modifier;
      modifier *= filter_weight;

      count[k] += modifier;
      accumulator[k] += modifier * pixel_value;

      byte++;
    }

    byte += stride - block_width;
  }
}

184
#if CONFIG_HIGHBITDEPTH
Yaowu Xu's avatar
Yaowu Xu committed
185
void av1_highbd_temporal_filter_apply_c(
186
187
188
    uint8_t *frame1_8, unsigned int stride, uint8_t *frame2_8,
    unsigned int block_width, unsigned int block_height, int strength,
    int filter_weight, unsigned int *accumulator, uint16_t *count) {
Jingning Han's avatar
Jingning Han committed
189
190
191
192
193
194
195
196
197
  uint16_t *frame1 = CONVERT_TO_SHORTPTR(frame1_8);
  uint16_t *frame2 = CONVERT_TO_SHORTPTR(frame2_8);
  unsigned int i, j, k;
  int modifier;
  int byte = 0;
  const int rounding = strength > 0 ? 1 << (strength - 1) : 0;

  for (i = 0, k = 0; i < block_height; i++) {
    for (j = 0; j < block_width; j++, k++) {
198
199
200
201
202
203
204
205
      int pixel_value = *frame2;

      // non-local mean approach
      int diff_sse[9] = { 0 };
      int idx, idy, index = 0;

      for (idy = -1; idy <= 1; ++idy) {
        for (idx = -1; idx <= 1; ++idx) {
Yaowu Xu's avatar
Yaowu Xu committed
206
207
          int row = (int)i + idy;
          int col = (int)j + idx;
208

209
210
          if (row >= 0 && row < (int)block_height && col >= 0 &&
              col < (int)block_width) {
211
            int diff = frame1[byte + idy * (int)stride + idx] -
212
                       frame2[idy * (int)block_width + idx];
213
214
215
216
217
218
219
220
221
            diff_sse[index] = diff * diff;
            ++index;
          }
        }
      }

      assert(index > 0);

      modifier = 0;
222
      for (idx = 0; idx < 9; ++idx) modifier += diff_sse[idx];
223

Jingning Han's avatar
Jingning Han committed
224
      modifier *= 3;
225
226
227
228
      modifier /= index;

      ++frame2;

Jingning Han's avatar
Jingning Han committed
229
230
231
      modifier += rounding;
      modifier >>= strength;

232
      if (modifier > 16) modifier = 16;
Jingning Han's avatar
Jingning Han committed
233
234
235
236
237
238
239
240
241
242
243
244
245

      modifier = 16 - modifier;
      modifier *= filter_weight;

      count[k] += modifier;
      accumulator[k] += modifier * pixel_value;

      byte++;
    }

    byte += stride - block_width;
  }
}
246
#endif  // CONFIG_HIGHBITDEPTH
Jingning Han's avatar
Jingning Han committed
247

Yaowu Xu's avatar
Yaowu Xu committed
248
static int temporal_filter_find_matching_mb_c(AV1_COMP *cpi,
Jingning Han's avatar
Jingning Han committed
249
250
251
252
253
254
255
256
257
258
259
260
                                              uint8_t *arf_frame_buf,
                                              uint8_t *frame_ptr_buf,
                                              int stride) {
  MACROBLOCK *const x = &cpi->td.mb;
  MACROBLOCKD *const xd = &x->e_mbd;
  const MV_SPEED_FEATURES *const mv_sf = &cpi->sf.mv;
  int step_param;
  int sadpb = x->sadperbit16;
  int bestsme = INT_MAX;
  int distortion;
  unsigned int sse;
  int cost_list[5];
Alex Converse's avatar
Alex Converse committed
261
  MvLimits tmp_mv_limits = x->mv_limits;
Jingning Han's avatar
Jingning Han committed
262

263
  MV best_ref_mv1 = { 0, 0 };
Jingning Han's avatar
Jingning Han committed
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
  MV best_ref_mv1_full; /* full-pixel value of best_ref_mv1 */

  // Save input state
  struct buf_2d src = x->plane[0].src;
  struct buf_2d pre = xd->plane[0].pre[0];

  best_ref_mv1_full.col = best_ref_mv1.col >> 3;
  best_ref_mv1_full.row = best_ref_mv1.row >> 3;

  // Setup frame pointers
  x->plane[0].src.buf = arf_frame_buf;
  x->plane[0].src.stride = stride;
  xd->plane[0].pre[0].buf = frame_ptr_buf;
  xd->plane[0].pre[0].stride = stride;

  step_param = mv_sf->reduce_first_step_size;
Yaowu Xu's avatar
Yaowu Xu committed
280
  step_param = AOMMIN(step_param, MAX_MVSEARCH_STEPS - 2);
Jingning Han's avatar
Jingning Han committed
281

Alex Converse's avatar
Alex Converse committed
282
  av1_set_mv_search_range(&x->mv_limits, &best_ref_mv1);
283

Jingning Han's avatar
Jingning Han committed
284
285
286
  x->mvcost = x->mv_cost_stack[0];
  x->nmvjointcost = x->nmv_vec_cost[0];

287
  // Use mv costing from x->mvcost directly
Yaowu Xu's avatar
Yaowu Xu committed
288
289
290
  av1_hex_search(x, &best_ref_mv1_full, step_param, sadpb, 1,
                 cond_cost_list(cpi, cost_list), &cpi->fn_ptr[BLOCK_16X16], 0,
                 &best_ref_mv1);
Jingning Han's avatar
Jingning Han committed
291

Alex Converse's avatar
Alex Converse committed
292
  x->mv_limits = tmp_mv_limits;
293

Jingning Han's avatar
Jingning Han committed
294
  // Ignore mv costing by sending NULL pointer instead of cost array
295
296
297
  bestsme = cpi->find_fractional_mv_step(
      x, &best_ref_mv1, cpi->common.allow_high_precision_mv, x->errorperbit,
      &cpi->fn_ptr[BLOCK_16X16], 0, mv_sf->subpel_iters_per_step,
298
299
300
301
302
      cond_cost_list(cpi, cost_list), NULL, NULL, &distortion, &sse, NULL,
#if CONFIG_EXT_INTER
      NULL, 0, 0,
#endif
      0, 0, 0);
Jingning Han's avatar
Jingning Han committed
303

hui su's avatar
hui su committed
304
305
  x->e_mbd.mi[0]->bmi[0].as_mv[0] = x->best_mv;

Jingning Han's avatar
Jingning Han committed
306
307
308
309
310
311
312
  // Restore input state
  x->plane[0].src = src;
  xd->plane[0].pre[0] = pre;

  return bestsme;
}

Yaowu Xu's avatar
Yaowu Xu committed
313
static void temporal_filter_iterate_c(AV1_COMP *cpi,
Todd Nguyen's avatar
Todd Nguyen committed
314
315
316
#if CONFIG_BGSPRITE
                                      YV12_BUFFER_CONFIG *target,
#endif  // CONFIG_BGSPRITE
Jingning Han's avatar
Jingning Han committed
317
                                      YV12_BUFFER_CONFIG **frames,
318
                                      int frame_count, int alt_ref_index,
Jingning Han's avatar
Jingning Han committed
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
                                      int strength,
                                      struct scale_factors *scale) {
  int byte;
  int frame;
  int mb_col, mb_row;
  unsigned int filter_weight;
  int mb_cols = (frames[alt_ref_index]->y_crop_width + 15) >> 4;
  int mb_rows = (frames[alt_ref_index]->y_crop_height + 15) >> 4;
  int mb_y_offset = 0;
  int mb_uv_offset = 0;
  DECLARE_ALIGNED(16, unsigned int, accumulator[16 * 16 * 3]);
  DECLARE_ALIGNED(16, uint16_t, count[16 * 16 * 3]);
  MACROBLOCKD *mbd = &cpi->td.mb.e_mbd;
  YV12_BUFFER_CONFIG *f = frames[alt_ref_index];
  uint8_t *dst1, *dst2;
334
#if CONFIG_HIGHBITDEPTH
335
336
  DECLARE_ALIGNED(16, uint16_t, predictor16[16 * 16 * 3]);
  DECLARE_ALIGNED(16, uint8_t, predictor8[16 * 16 * 3]);
Jingning Han's avatar
Jingning Han committed
337
338
  uint8_t *predictor;
#else
339
  DECLARE_ALIGNED(16, uint8_t, predictor[16 * 16 * 3]);
Jingning Han's avatar
Jingning Han committed
340
341
#endif
  const int mb_uv_height = 16 >> mbd->plane[1].subsampling_y;
342
  const int mb_uv_width = 16 >> mbd->plane[1].subsampling_x;
Jingning Han's avatar
Jingning Han committed
343
344

  // Save input state
345
  uint8_t *input_buffer[MAX_MB_PLANE];
Jingning Han's avatar
Jingning Han committed
346
  int i;
347
#if CONFIG_HIGHBITDEPTH
Jingning Han's avatar
Jingning Han committed
348
349
350
351
352
353
354
  if (mbd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) {
    predictor = CONVERT_TO_BYTEPTR(predictor16);
  } else {
    predictor = predictor8;
  }
#endif

355
  for (i = 0; i < MAX_MB_PLANE; i++) input_buffer[i] = mbd->plane[i].pre[0].buf;
Jingning Han's avatar
Jingning Han committed
356
357
358

  for (mb_row = 0; mb_row < mb_rows; mb_row++) {
    // Source frames are extended to 16 pixels. This is different than
Yaowu Xu's avatar
Yaowu Xu committed
359
    //  L/A/G reference frames that have a border of 32 (AV1ENCBORDERINPIXELS)
Jingning Han's avatar
Jingning Han committed
360
361
    // A 6/8 tap filter is used for motion search.  This requires 2 pixels
    //  before and 3 pixels after.  So the largest Y mv on a border would
Yaowu Xu's avatar
Yaowu Xu committed
362
    //  then be 16 - AOM_INTERP_EXTEND. The UV blocks are half the size of the
Jingning Han's avatar
Jingning Han committed
363
    //  Y and therefore only extended by 8.  The largest mv that a UV block
Yaowu Xu's avatar
Yaowu Xu committed
364
365
366
    //  can support is 8 - AOM_INTERP_EXTEND.  A UV mv is half of a Y mv.
    //  (16 - AOM_INTERP_EXTEND) >> 1 which is greater than
    //  8 - AOM_INTERP_EXTEND.
Jingning Han's avatar
Jingning Han committed
367
    // To keep the mv in play for both Y and UV planes the max that it
Yaowu Xu's avatar
Yaowu Xu committed
368
    //  can be on a border is therefore 16 - (2*AOM_INTERP_EXTEND+1).
Alex Converse's avatar
Alex Converse committed
369
370
371
    cpi->td.mb.mv_limits.row_min =
        -((mb_row * 16) + (17 - 2 * AOM_INTERP_EXTEND));
    cpi->td.mb.mv_limits.row_max =
Yaowu Xu's avatar
Yaowu Xu committed
372
        ((mb_rows - 1 - mb_row) * 16) + (17 - 2 * AOM_INTERP_EXTEND);
Jingning Han's avatar
Jingning Han committed
373
374

    for (mb_col = 0; mb_col < mb_cols; mb_col++) {
375
      int j, k;
Jingning Han's avatar
Jingning Han committed
376
377
378
379
380
      int stride;

      memset(accumulator, 0, 16 * 16 * 3 * sizeof(accumulator[0]));
      memset(count, 0, 16 * 16 * 3 * sizeof(count[0]));

Alex Converse's avatar
Alex Converse committed
381
382
383
      cpi->td.mb.mv_limits.col_min =
          -((mb_col * 16) + (17 - 2 * AOM_INTERP_EXTEND));
      cpi->td.mb.mv_limits.col_max =
Yaowu Xu's avatar
Yaowu Xu committed
384
          ((mb_cols - 1 - mb_col) * 16) + (17 - 2 * AOM_INTERP_EXTEND);
Jingning Han's avatar
Jingning Han committed
385
386

      for (frame = 0; frame < frame_count; frame++) {
387
        const int thresh_low = 10000;
Jingning Han's avatar
Jingning Han committed
388
389
        const int thresh_high = 20000;

390
        if (frames[frame] == NULL) continue;
Jingning Han's avatar
Jingning Han committed
391
392
393
394
395
396
397
398

        mbd->mi[0]->bmi[0].as_mv[0].as_mv.row = 0;
        mbd->mi[0]->bmi[0].as_mv[0].as_mv.col = 0;

        if (frame == alt_ref_index) {
          filter_weight = 2;
        } else {
          // Find best match in this frame by MC
399
400
401
          int err = temporal_filter_find_matching_mb_c(
              cpi, frames[alt_ref_index]->y_buffer + mb_y_offset,
              frames[frame]->y_buffer + mb_y_offset, frames[frame]->y_stride);
Jingning Han's avatar
Jingning Han committed
402
403
404
405

          // Assign higher weight to matching MB if it's error
          // score is lower. If not applying MC default behavior
          // is to weight all MBs equal.
406
          filter_weight = err < thresh_low ? 2 : err < thresh_high ? 1 : 0;
Jingning Han's avatar
Jingning Han committed
407
408
409
410
        }

        if (filter_weight != 0) {
          // Construct the predictors
411
412
          temporal_filter_predictors_mb_c(
              mbd, frames[frame]->y_buffer + mb_y_offset,
Jingning Han's avatar
Jingning Han committed
413
              frames[frame]->u_buffer + mb_uv_offset,
414
415
416
              frames[frame]->v_buffer + mb_uv_offset, frames[frame]->y_stride,
              mb_uv_width, mb_uv_height, mbd->mi[0]->bmi[0].as_mv[0].as_mv.row,
              mbd->mi[0]->bmi[0].as_mv[0].as_mv.col, predictor, scale,
Jingning Han's avatar
Jingning Han committed
417
418
              mb_col * 16, mb_row * 16);

419
// Apply the filter (YUV)
420
#if CONFIG_HIGHBITDEPTH
Jingning Han's avatar
Jingning Han committed
421
422
          if (mbd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) {
            int adj_strength = strength + 2 * (mbd->bd - 8);
Yaowu Xu's avatar
Yaowu Xu committed
423
            av1_highbd_temporal_filter_apply(
424
425
                f->y_buffer + mb_y_offset, f->y_stride, predictor, 16, 16,
                adj_strength, filter_weight, accumulator, count);
Yaowu Xu's avatar
Yaowu Xu committed
426
            av1_highbd_temporal_filter_apply(
427
428
429
                f->u_buffer + mb_uv_offset, f->uv_stride, predictor + 256,
                mb_uv_width, mb_uv_height, adj_strength, filter_weight,
                accumulator + 256, count + 256);
Yaowu Xu's avatar
Yaowu Xu committed
430
            av1_highbd_temporal_filter_apply(
431
432
433
                f->v_buffer + mb_uv_offset, f->uv_stride, predictor + 512,
                mb_uv_width, mb_uv_height, adj_strength, filter_weight,
                accumulator + 512, count + 512);
Jingning Han's avatar
Jingning Han committed
434
          } else {
435
#endif  // CONFIG_HIGHBITDEPTH
Yaowu Xu's avatar
Yaowu Xu committed
436
437
438
439
            av1_temporal_filter_apply_c(f->y_buffer + mb_y_offset, f->y_stride,
                                        predictor, 16, 16, strength,
                                        filter_weight, accumulator, count);
            av1_temporal_filter_apply_c(
440
441
442
                f->u_buffer + mb_uv_offset, f->uv_stride, predictor + 256,
                mb_uv_width, mb_uv_height, strength, filter_weight,
                accumulator + 256, count + 256);
Yaowu Xu's avatar
Yaowu Xu committed
443
            av1_temporal_filter_apply_c(
444
445
446
                f->v_buffer + mb_uv_offset, f->uv_stride, predictor + 512,
                mb_uv_width, mb_uv_height, strength, filter_weight,
                accumulator + 512, count + 512);
447
#if CONFIG_HIGHBITDEPTH
Jingning Han's avatar
Jingning Han committed
448
          }
449
#endif  // CONFIG_HIGHBITDEPTH
Jingning Han's avatar
Jingning Han committed
450
451
452
        }
      }

453
// Normalize filter output to produce AltRef frame
454
#if CONFIG_HIGHBITDEPTH
Jingning Han's avatar
Jingning Han committed
455
456
457
      if (mbd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) {
        uint16_t *dst1_16;
        uint16_t *dst2_16;
Todd Nguyen's avatar
Todd Nguyen committed
458
459
460
#if CONFIG_BGSPRITE
        dst1 = target->y_buffer;
#else
Jingning Han's avatar
Jingning Han committed
461
        dst1 = cpi->alt_ref_buffer.y_buffer;
Todd Nguyen's avatar
Todd Nguyen committed
462
#endif  // CONFIG_BGSPRITE
Jingning Han's avatar
Jingning Han committed
463
        dst1_16 = CONVERT_TO_SHORTPTR(dst1);
Todd Nguyen's avatar
Todd Nguyen committed
464
465
466
#if CONFIG_BGSPRITE
        stride = target->y_stride;
#else
Jingning Han's avatar
Jingning Han committed
467
        stride = cpi->alt_ref_buffer.y_stride;
Todd Nguyen's avatar
Todd Nguyen committed
468
#endif  // CONFIG_BGSPRITE
Jingning Han's avatar
Jingning Han committed
469
470
471
        byte = mb_y_offset;
        for (i = 0, k = 0; i < 16; i++) {
          for (j = 0; j < 16; j++, k++) {
Yaowu Xu's avatar
Yaowu Xu committed
472
473
            dst1_16[byte] =
                (uint16_t)OD_DIVU(accumulator[k] + (count[k] >> 1), count[k]);
Jingning Han's avatar
Jingning Han committed
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492

            // move to next pixel
            byte++;
          }

          byte += stride - 16;
        }

        dst1 = cpi->alt_ref_buffer.u_buffer;
        dst2 = cpi->alt_ref_buffer.v_buffer;
        dst1_16 = CONVERT_TO_SHORTPTR(dst1);
        dst2_16 = CONVERT_TO_SHORTPTR(dst2);
        stride = cpi->alt_ref_buffer.uv_stride;
        byte = mb_uv_offset;
        for (i = 0, k = 256; i < mb_uv_height; i++) {
          for (j = 0; j < mb_uv_width; j++, k++) {
            int m = k + 256;

            // U
Yaowu Xu's avatar
Yaowu Xu committed
493
494
            dst1_16[byte] =
                (uint16_t)OD_DIVU(accumulator[k] + (count[k] >> 1), count[k]);
Jingning Han's avatar
Jingning Han committed
495
496

            // V
Yaowu Xu's avatar
Yaowu Xu committed
497
498
            dst2_16[byte] =
                (uint16_t)OD_DIVU(accumulator[m] + (count[m] >> 1), count[m]);
Jingning Han's avatar
Jingning Han committed
499
500
501
502
503
504
505
506

            // move to next pixel
            byte++;
          }

          byte += stride - mb_uv_width;
        }
      } else {
507
#endif  // CONFIG_HIGHBITDEPTH
Todd Nguyen's avatar
Todd Nguyen committed
508
509
510
511
512
513
514
#if CONFIG_BGSPRITE
        dst1 = target->y_buffer;
        stride = target->y_stride;
#else
      dst1 = cpi->alt_ref_buffer.y_buffer;
      stride = cpi->alt_ref_buffer.y_stride;
#endif  // CONFIG_BGSPRITE
Jingning Han's avatar
Jingning Han committed
515
516
517
        byte = mb_y_offset;
        for (i = 0, k = 0; i < 16; i++) {
          for (j = 0; j < 16; j++, k++) {
Yaowu Xu's avatar
Yaowu Xu committed
518
519
            dst1[byte] =
                (uint8_t)OD_DIVU(accumulator[k] + (count[k] >> 1), count[k]);
Jingning Han's avatar
Jingning Han committed
520
521
522
523
524
525

            // move to next pixel
            byte++;
          }
          byte += stride - 16;
        }
Todd Nguyen's avatar
Todd Nguyen committed
526
527
528
529
530
531
532
533
534
#if CONFIG_BGSPRITE
        dst1 = target->u_buffer;
        dst2 = target->v_buffer;
        stride = target->uv_stride;
#else
      dst1 = cpi->alt_ref_buffer.u_buffer;
      dst2 = cpi->alt_ref_buffer.v_buffer;
      stride = cpi->alt_ref_buffer.uv_stride;
#endif  // CONFIG_BGSPRITE
Jingning Han's avatar
Jingning Han committed
535
536
537
538
539
540
        byte = mb_uv_offset;
        for (i = 0, k = 256; i < mb_uv_height; i++) {
          for (j = 0; j < mb_uv_width; j++, k++) {
            int m = k + 256;

            // U
Yaowu Xu's avatar
Yaowu Xu committed
541
542
            dst1[byte] =
                (uint8_t)OD_DIVU(accumulator[k] + (count[k] >> 1), count[k]);
Jingning Han's avatar
Jingning Han committed
543
544

            // V
Yaowu Xu's avatar
Yaowu Xu committed
545
546
            dst2[byte] =
                (uint8_t)OD_DIVU(accumulator[m] + (count[m] >> 1), count[m]);
Jingning Han's avatar
Jingning Han committed
547
548
549
550
551
552

            // move to next pixel
            byte++;
          }
          byte += stride - mb_uv_width;
        }
553
#if CONFIG_HIGHBITDEPTH
Jingning Han's avatar
Jingning Han committed
554
      }
555
#endif  // CONFIG_HIGHBITDEPTH
Jingning Han's avatar
Jingning Han committed
556
557
558
559
560
561
562
563
      mb_y_offset += 16;
      mb_uv_offset += mb_uv_width;
    }
    mb_y_offset += 16 * (f->y_stride - mb_cols);
    mb_uv_offset += mb_uv_height * f->uv_stride - mb_uv_width * mb_cols;
  }

  // Restore input state
564
  for (i = 0; i < MAX_MB_PLANE; i++) mbd->plane[i].pre[0].buf = input_buffer[i];
Jingning Han's avatar
Jingning Han committed
565
566
567
}

// Apply buffer limits and context specific adjustments to arnr filter.
Yaowu Xu's avatar
Yaowu Xu committed
568
static void adjust_arnr_filter(AV1_COMP *cpi, int distance, int group_boost,
Jingning Han's avatar
Jingning Han committed
569
                               int *arnr_frames, int *arnr_strength) {
Yaowu Xu's avatar
Yaowu Xu committed
570
  const AV1EncoderConfig *const oxcf = &cpi->oxcf;
Jingning Han's avatar
Jingning Han committed
571
  const int frames_after_arf =
Yaowu Xu's avatar
Yaowu Xu committed
572
      av1_lookahead_depth(cpi->lookahead) - distance - 1;
Jingning Han's avatar
Jingning Han committed
573
574
575
576
577
  int frames_fwd = (cpi->oxcf.arnr_max_frames - 1) >> 1;
  int frames_bwd;
  int q, frames, strength;

  // Define the forward and backwards filter limits for this arnr group.
578
579
  if (frames_fwd > frames_after_arf) frames_fwd = frames_after_arf;
  if (frames_fwd > distance) frames_fwd = distance;
Jingning Han's avatar
Jingning Han committed
580
581
582
583
584

  frames_bwd = frames_fwd;

  // For even length filter there is one more frame backward
  // than forward: e.g. len=6 ==> bbbAff, len=7 ==> bbbAfff.
585
  if (frames_bwd < distance) frames_bwd += (oxcf->arnr_max_frames + 1) & 0x1;
Jingning Han's avatar
Jingning Han committed
586
587
588
589
590
591

  // Set the baseline active filter size.
  frames = frames_bwd + 1 + frames_fwd;

  // Adjust the strength based on active max q.
  if (cpi->common.current_video_frame > 1)
Yaowu Xu's avatar
Yaowu Xu committed
592
593
    q = ((int)av1_convert_qindex_to_q(cpi->rc.avg_frame_qindex[INTER_FRAME],
                                      cpi->common.bit_depth));
Jingning Han's avatar
Jingning Han committed
594
  else
Yaowu Xu's avatar
Yaowu Xu committed
595
596
    q = ((int)av1_convert_qindex_to_q(cpi->rc.avg_frame_qindex[KEY_FRAME],
                                      cpi->common.bit_depth));
Jingning Han's avatar
Jingning Han committed
597
598
599
600
  if (q > 16) {
    strength = oxcf->arnr_strength;
  } else {
    strength = oxcf->arnr_strength - ((16 - q) / 2);
601
    if (strength < 0) strength = 0;
Jingning Han's avatar
Jingning Han committed
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
  }

  // Adjust number of frames in filter and strength based on gf boost level.
  if (frames > group_boost / 150) {
    frames = group_boost / 150;
    frames += !(frames & 1);
  }

  if (strength > group_boost / 300) {
    strength = group_boost / 300;
  }

  // Adjustments for second level arf in multi arf case.
  if (cpi->oxcf.pass == 2 && cpi->multi_arf_allowed) {
    const GF_GROUP *const gf_group = &cpi->twopass.gf_group;
    if (gf_group->rf_level[gf_group->index] != GF_ARF_STD) {
      strength >>= 1;
    }
  }

  *arnr_frames = frames;
  *arnr_strength = strength;
}

626
627
void av1_temporal_filter(AV1_COMP *cpi,
#if CONFIG_BGSPRITE
Todd Nguyen's avatar
Todd Nguyen committed
628
                         YV12_BUFFER_CONFIG *bg, YV12_BUFFER_CONFIG *target,
629
630
#endif  // CONFIG_BGSPRITE
                         int distance) {
Jingning Han's avatar
Jingning Han committed
631
632
633
634
635
636
637
638
  RATE_CONTROL *const rc = &cpi->rc;
  int frame;
  int frames_to_blur;
  int start_frame;
  int strength;
  int frames_to_blur_backward;
  int frames_to_blur_forward;
  struct scale_factors sf;
639
  YV12_BUFFER_CONFIG *frames[MAX_LAG_BUFFERS] = { NULL };
640
641
#if CONFIG_EXT_REFS
  const GF_GROUP *const gf_group = &cpi->twopass.gf_group;
Zoe Liu's avatar
Zoe Liu committed
642
#endif  // CONFIG_EXT_REFS
Jingning Han's avatar
Jingning Han committed
643
644
645

  // Apply context specific adjustments to the arnr filter parameters.
  adjust_arnr_filter(cpi, distance, rc->gfu_boost, &frames_to_blur, &strength);
Yaowu Xu's avatar
Yaowu Xu committed
646
647
648
649
// TODO(weitinglin): Currently, we enforce the filtering strength on
//                   extra ARFs' to be zeros. We should investigate in which
//                   case it is more beneficial to use non-zero strength
//                   filtering.
650
#if CONFIG_EXT_REFS
Zoe Liu's avatar
Zoe Liu committed
651
652
653
654
655
656
#if CONFIG_ALTREF2
  if (gf_group->update_type[gf_group->index] == INTNL_ARF_UPDATE) {
    strength = 0;
    frames_to_blur = 1;
  }
#else   // !CONFIG_ALTREF2
657
658
659
660
  if (gf_group->rf_level[gf_group->index] == GF_ARF_LOW) {
    strength = 0;
    frames_to_blur = 1;
  }
Zoe Liu's avatar
Zoe Liu committed
661
662
#endif  // CONFIG_ALTREF2
#endif  // CONFIG_EXT_REFS
663
664
665
666
667
668
669

#if CONFIG_EXT_REFS
  if (strength == 0 && frames_to_blur == 1) {
    cpi->is_arf_filter_off[gf_group->arf_update_idx[gf_group->index]] = 1;
  } else {
    cpi->is_arf_filter_off[gf_group->arf_update_idx[gf_group->index]] = 0;
  }
Zoe Liu's avatar
Zoe Liu committed
670
#endif  // CONFIG_EXT_REFS
671

Jingning Han's avatar
Jingning Han committed
672
673
674
675
676
677
678
  frames_to_blur_backward = (frames_to_blur / 2);
  frames_to_blur_forward = ((frames_to_blur - 1) / 2);
  start_frame = distance + frames_to_blur_forward;

  // Setup frame pointers, NULL indicates frame not included in filter.
  for (frame = 0; frame < frames_to_blur; ++frame) {
    const int which_buffer = start_frame - frame;
679
680
681
682
683
684
685
686
687
688
689
690
#if CONFIG_BGSPRITE
    if (frame == frames_to_blur_backward && bg != NULL) {
      // Insert bg into frames at ARF index.
      frames[frames_to_blur - 1 - frame] = bg;
    } else {
#endif  // CONFIG_BGSPRITE
      struct lookahead_entry *buf =
          av1_lookahead_peek(cpi->lookahead, which_buffer);
      frames[frames_to_blur - 1 - frame] = &buf->img;
#if CONFIG_BGSPRITE
    }
#endif  // CONFIG_BGSPRITE
Jingning Han's avatar
Jingning Han committed
691
692
693
  }

  if (frames_to_blur > 0) {
694
695
696
// Setup scaling factors. Scaling on each of the arnr frames is not
// supported.
// ARF is produced at the native frame size and resized when coded.
697
#if CONFIG_HIGHBITDEPTH
Yaowu Xu's avatar
Yaowu Xu committed
698
    av1_setup_scale_factors_for_frame(
699
700
701
        &sf, frames[0]->y_crop_width, frames[0]->y_crop_height,
        frames[0]->y_crop_width, frames[0]->y_crop_height,
        cpi->common.use_highbitdepth);
Jingning Han's avatar
Jingning Han committed
702
#else
Yaowu Xu's avatar
Yaowu Xu committed
703
    av1_setup_scale_factors_for_frame(
704
705
        &sf, frames[0]->y_crop_width, frames[0]->y_crop_height,
        frames[0]->y_crop_width, frames[0]->y_crop_height);
706
#endif  // CONFIG_HIGHBITDEPTH
Jingning Han's avatar
Jingning Han committed
707
708
  }

Todd Nguyen's avatar
Todd Nguyen committed
709
710
711
712
713
714
  temporal_filter_iterate_c(cpi,
#if CONFIG_BGSPRITE
                            target,
#endif  // CONFIG_BGSPRITE
                            frames, frames_to_blur, frames_to_blur_backward,
                            strength, &sf);
Jingning Han's avatar
Jingning Han committed
715
}