highbd_sad4d_sse2.asm 9.15 KB
Newer Older
1
;
Yaowu Xu's avatar
Yaowu Xu committed
2
; Copyright (c) 2016, Alliance for Open Media. All rights reserved
3
;
Yaowu Xu's avatar
Yaowu Xu committed
4 5 6 7 8 9 10 11
; This source code is subject to the terms of the BSD 2 Clause License and
; the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
; was not distributed with this source code in the LICENSE file, you can
; obtain it at www.aomedia.org/license/software. If the Alliance for Open
; Media Patent License 1.0 was not distributed with this source code in the
; PATENTS file, you can obtain it at www.aomedia.org/license/patent.
;

12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214
;

%include "third_party/x86inc/x86inc.asm"

SECTION .text

; HIGH_PROCESS_4x2x4 first, off_{first,second}_{src,ref}, advance_at_end
%macro HIGH_PROCESS_4x2x4 5-6 0
  movh                  m0, [srcq +%2*2]
%if %1 == 1
  movu                  m4, [ref1q+%3*2]
  movu                  m5, [ref2q+%3*2]
  movu                  m6, [ref3q+%3*2]
  movu                  m7, [ref4q+%3*2]
  movhps                m0, [srcq +%4*2]
  movhps                m4, [ref1q+%5*2]
  movhps                m5, [ref2q+%5*2]
  movhps                m6, [ref3q+%5*2]
  movhps                m7, [ref4q+%5*2]
  mova                  m3, m0
  mova                  m2, m0
  psubusw               m3, m4
  psubusw               m2, m5
  psubusw               m4, m0
  psubusw               m5, m0
  por                   m4, m3
  por                   m5, m2
  pmaddwd               m4, m1
  pmaddwd               m5, m1
  mova                  m3, m0
  mova                  m2, m0
  psubusw               m3, m6
  psubusw               m2, m7
  psubusw               m6, m0
  psubusw               m7, m0
  por                   m6, m3
  por                   m7, m2
  pmaddwd               m6, m1
  pmaddwd               m7, m1
%else
  movu                  m2, [ref1q+%3*2]
  movhps                m0, [srcq +%4*2]
  movhps                m2, [ref1q+%5*2]
  mova                  m3, m0
  psubusw               m3, m2
  psubusw               m2, m0
  por                   m2, m3
  pmaddwd               m2, m1
  paddd                 m4, m2

  movu                  m2, [ref2q+%3*2]
  mova                  m3, m0
  movhps                m2, [ref2q+%5*2]
  psubusw               m3, m2
  psubusw               m2, m0
  por                   m2, m3
  pmaddwd               m2, m1
  paddd                 m5, m2

  movu                  m2, [ref3q+%3*2]
  mova                  m3, m0
  movhps                m2, [ref3q+%5*2]
  psubusw               m3, m2
  psubusw               m2, m0
  por                   m2, m3
  pmaddwd               m2, m1
  paddd                 m6, m2

  movu                  m2, [ref4q+%3*2]
  mova                  m3, m0
  movhps                m2, [ref4q+%5*2]
  psubusw               m3, m2
  psubusw               m2, m0
  por                   m2, m3
  pmaddwd               m2, m1
  paddd                 m7, m2
%endif
%if %6 == 1
  lea                 srcq, [srcq +src_strideq*4]
  lea                ref1q, [ref1q+ref_strideq*4]
  lea                ref2q, [ref2q+ref_strideq*4]
  lea                ref3q, [ref3q+ref_strideq*4]
  lea                ref4q, [ref4q+ref_strideq*4]
%endif
%endmacro

; PROCESS_8x2x4 first, off_{first,second}_{src,ref}, advance_at_end
%macro HIGH_PROCESS_8x2x4 5-6 0
  ; 1st 8 px
  mova                  m0, [srcq +%2*2]
%if %1 == 1
  movu                  m4, [ref1q+%3*2]
  movu                  m5, [ref2q+%3*2]
  movu                  m6, [ref3q+%3*2]
  movu                  m7, [ref4q+%3*2]
  mova                  m3, m0
  mova                  m2, m0
  psubusw               m3, m4
  psubusw               m2, m5
  psubusw               m4, m0
  psubusw               m5, m0
  por                   m4, m3
  por                   m5, m2
  pmaddwd               m4, m1
  pmaddwd               m5, m1
  mova                  m3, m0
  mova                  m2, m0
  psubusw               m3, m6
  psubusw               m2, m7
  psubusw               m6, m0
  psubusw               m7, m0
  por                   m6, m3
  por                   m7, m2
  pmaddwd               m6, m1
  pmaddwd               m7, m1
%else
  mova                  m3, m0
  movu                  m2, [ref1q+%3*2]
  psubusw               m3, m2
  psubusw               m2, m0
  por                   m2, m3
  mova                  m3, m0
  pmaddwd               m2, m1
  paddd                 m4, m2
  movu                  m2, [ref2q+%3*2]
  psubusw               m3, m2
  psubusw               m2, m0
  por                   m2, m3
  mova                  m3, m0
  pmaddwd               m2, m1
  paddd                 m5, m2
  movu                  m2, [ref3q+%3*2]
  psubusw               m3, m2
  psubusw               m2, m0
  por                   m2, m3
  mova                  m3, m0
  pmaddwd               m2, m1
  paddd                 m6, m2
  movu                  m2, [ref4q+%3*2]
  psubusw               m3, m2
  psubusw               m2, m0
  por                   m2, m3
  pmaddwd               m2, m1
  paddd                 m7, m2
%endif

  ; 2nd 8 px
  mova                  m0, [srcq +(%4)*2]
  mova                  m3, m0
  movu                  m2, [ref1q+(%5)*2]
  psubusw               m3, m2
  psubusw               m2, m0
  por                   m2, m3
  mova                  m3, m0
  pmaddwd               m2, m1
  paddd                 m4, m2
  movu                  m2, [ref2q+(%5)*2]
  psubusw               m3, m2
  psubusw               m2, m0
  por                   m2, m3
  mova                  m3, m0
  pmaddwd               m2, m1
  paddd                 m5, m2
  movu                  m2, [ref3q+(%5)*2]
  psubusw               m3, m2
  psubusw               m2, m0
  por                   m2, m3
  mova                  m3, m0
  pmaddwd               m2, m1
  paddd                 m6, m2
  movu                  m2, [ref4q+(%5)*2]
  psubusw               m3, m2
  psubusw               m2, m0
%if %6 == 1
  lea                 srcq, [srcq +src_strideq*4]
  lea                ref1q, [ref1q+ref_strideq*4]
  lea                ref2q, [ref2q+ref_strideq*4]
  lea                ref3q, [ref3q+ref_strideq*4]
  lea                ref4q, [ref4q+ref_strideq*4]
%endif
  por                   m2, m3
  pmaddwd               m2, m1
  paddd                 m7, m2
%endmacro

; HIGH_PROCESS_16x2x4 first, off_{first,second}_{src,ref}, advance_at_end
%macro HIGH_PROCESS_16x2x4 5-6 0
  HIGH_PROCESS_8x2x4 %1, %2, %3, (%2 + 8), (%3 + 8)
  HIGH_PROCESS_8x2x4  0, %4, %5, (%4 + 8), (%5 + 8), %6
%endmacro

; HIGH_PROCESS_32x2x4 first, off_{first,second}_{src,ref}, advance_at_end
%macro HIGH_PROCESS_32x2x4 5-6 0
  HIGH_PROCESS_16x2x4 %1, %2, %3, (%2 + 16), (%3 + 16)
  HIGH_PROCESS_16x2x4  0, %4, %5, (%4 + 16), (%5 + 16), %6
%endmacro

; HIGH_PROCESS_64x2x4 first, off_{first,second}_{src,ref}, advance_at_end
%macro HIGH_PROCESS_64x2x4 5-6 0
  HIGH_PROCESS_32x2x4 %1, %2, %3, (%2 + 32), (%3 + 32)
  HIGH_PROCESS_32x2x4  0, %4, %5, (%4 + 32), (%5 + 32), %6
%endmacro

Yaowu Xu's avatar
Yaowu Xu committed
215
; void aom_highbd_sadNxNx4d_sse2(uint8_t *src,    int src_stride,
216
;                         uint8_t *ref[4], int ref_stride,
Johann's avatar
Johann committed
217
;                         uint32_t res[4]);
218 219 220
; where NxN = 64x64, 32x32, 16x16, 16x8, 8x16 or 8x8
%macro HIGH_SADNXN4D 2
%if UNIX64
221 222
cglobal highbd_sad%1x%2x4d, 5, 8, 8, src, src_stride, ref1, ref_stride, \
                              res, ref2, ref3, ref4
223
%else
224 225
cglobal highbd_sad%1x%2x4d, 4, 7, 8, src, src_stride, ref1, ref_stride, \
                              ref2, ref3, ref4
226 227
%endif

228 229 230 231 232 233 234
; set m1
  push                srcq
  mov                 srcd, 0x00010001
  movd                  m1, srcd
  pshufd                m1, m1, 0x0
  pop                 srcq

235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290
  movsxdifnidn src_strideq, src_strided
  movsxdifnidn ref_strideq, ref_strided
  mov                ref2q, [ref1q+gprsize*1]
  mov                ref3q, [ref1q+gprsize*2]
  mov                ref4q, [ref1q+gprsize*3]
  mov                ref1q, [ref1q+gprsize*0]

; convert byte pointers to short pointers
  shl                 srcq, 1
  shl                ref2q, 1
  shl                ref3q, 1
  shl                ref4q, 1
  shl                ref1q, 1

  HIGH_PROCESS_%1x2x4 1, 0, 0, src_strideq, ref_strideq, 1
%rep (%2-4)/2
  HIGH_PROCESS_%1x2x4 0, 0, 0, src_strideq, ref_strideq, 1
%endrep
  HIGH_PROCESS_%1x2x4 0, 0, 0, src_strideq, ref_strideq, 0
  ; N.B. HIGH_PROCESS outputs dwords (32 bits)
  ; so in high bit depth even the smallest width (4) needs 128bits i.e. XMM
  movhlps               m0, m4
  movhlps               m1, m5
  movhlps               m2, m6
  movhlps               m3, m7
  paddd                 m4, m0
  paddd                 m5, m1
  paddd                 m6, m2
  paddd                 m7, m3
  punpckldq             m4, m5
  punpckldq             m6, m7
  movhlps               m0, m4
  movhlps               m1, m6
  paddd                 m4, m0
  paddd                 m6, m1
  punpcklqdq            m4, m6
  movifnidn             r4, r4mp
  movu                [r4], m4
  RET
%endmacro


INIT_XMM sse2
HIGH_SADNXN4D 64, 64
HIGH_SADNXN4D 64, 32
HIGH_SADNXN4D 32, 64
HIGH_SADNXN4D 32, 32
HIGH_SADNXN4D 32, 16
HIGH_SADNXN4D 16, 32
HIGH_SADNXN4D 16, 16
HIGH_SADNXN4D 16,  8
HIGH_SADNXN4D  8, 16
HIGH_SADNXN4D  8,  8
HIGH_SADNXN4D  8,  4
HIGH_SADNXN4D  4,  8
HIGH_SADNXN4D  4,  4
291 292 293 294 295 296
%if CONFIG_EXT_PARTITION_TYPES
HIGH_SADNXN4D  4, 16
HIGH_SADNXN4D 16,  4
HIGH_SADNXN4D  8, 32
HIGH_SADNXN4D 32,  8
%endif