; /*
; * Provide AVX2 luma optical flow functions for VVC decoding
; * Copyright (c) 2024 Nuo Mi
; *
; * This file is part of FFmpeg.
; *
; * FFmpeg is free software; you can redistribute it and/or
; * modify it under the terms of the GNU Lesser General Public
; * License as published by the Free Software Foundation; either
; * version 2.1 of the License, or (at your option) any later version.
; *
; * FFmpeg is distributed in the hope that it will be useful,
; * but WITHOUT ANY WARRANTY; without even the implied warranty of
; * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
; * Lesser General Public License for more details.
; *
; * You should have received a copy of the GNU Lesser General Public
; * License along with FFmpeg; if not, write to the Free Software
; * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
; */
%include "libavutil/x86/x86util.asm"

%define MAX_PB_SIZE             128
%define SRC_STRIDE              (MAX_PB_SIZE * 2)
%define SRC_PS                  2                                                       ; source pixel size, sizeof(int16_t)
%define BDOF_STACK_SIZE         10                                                      ; (4 + 1) * 2, 4 lines + the first line, *2 for h and v
%define bdof_stack_offset(line) ((line) * 2 % BDOF_STACK_SIZE * mmsize)
%define SHIFT                   6
%define SHIFT2                  4

SECTION_RODATA 32
pd_15                   times 8 dd 15
pd_m15                  times 8 dd -15

pb_shuffle     times 2 db   0, 1, 0xff, 0xff, 8, 9, 0xff, 0xff, 6, 7, 0xff, 0xff, 14, 15, 0xff, 0xff
pd_perm_w16            dd   0, 1, 2, 4, 3, 5, 6, 7
%if ARCH_X86_64

%if HAVE_AVX2_EXTERNAL

SECTION .text

INIT_YMM avx2

; dst = (src0 >> shift) - (src1 >> shift)
%macro DIFF 5 ; dst, src0, src1, shift, tmp
    psraw   %1, %2, %4
    psraw   %5, %3, %4
    psubw   %1, %5
%endmacro

%macro LOAD_GRAD_H 4 ; dst, src, off, tmp
    movu    %1, [%2 + %3 + 2 * SRC_PS]
    movu    %4, [%2 + %3]

    DIFF    %1, %1, %4, SHIFT, %4
%endmacro

%macro SUM_GRAD 2 ;(dst/grad0, grad1)
    paddw  %1, %2
    psraw  %1, 1    ; shift3
%endmacro

%macro APPLY_BDOF_MIN_BLOCK_LINE 5 ; dst, vx, vy, tmp, line_num
%define off bdof_stack_offset(%5)
    pmullw                        %1, %2, [rsp + off + 0 * mmsize]               ; vx * (gradient_h[0] - gradient_h[1])
    pmullw                        %4, %3, [rsp + off + 1 * mmsize]               ; vy * (gradient_v[0] - gradient_v[1])
    paddw                         %1, [src0q + (%5 + 1) * SRC_STRIDE + SRC_PS]
    paddw                         %4, [src1q + (%5 + 1) * SRC_STRIDE + SRC_PS]
    paddsw                        %1, %4                                         ; src0[x] + src1[x] + bdof_offset
    pmulhrsw                      %1, m11
%endmacro

%macro SAVE 2-3 ""; dst, src, jump target
    cmp                 pixel_maxd, (1 << 8) - 1
    jne               %%save_16bpc

    packuswb                   m%2, m%2

    cmp                         wd, 16
    je                       %%w16_8
    movq                        %1, xm%2
%ifnidn %3, ""
    jmp                         %3
%else
    jmp                      %%end
%endif

%%save_16bpc:
    CLIPW                      m%2, m9, m10
    cmp                         wd, 16
    jne                       %%w8_16
    movu                        %1, m%2
%ifnidn %3, ""
    jmp                         %3
%else
    jmp                      %%end
%endif

%%w16_8:
    vpermq                     m%2, m%2, q0020
%%w8_16:
    movu                        %1, xm%2
%%end:
%endmacro

; [rsp + even * mmsize] are gradient_h[0] - gradient_h[1]
; [rsp +  odd * mmsize] are gradient_v[0] - gradient_v[1]
%macro APPLY_BDOF_MIN_BLOCK 3-4 ""; block_num, vx, vy, jump target
    pxor                          m9, m9

    movd                        xm10, pixel_maxd
    vpbroadcastw                 m10, xm10

    lea                        tmp0d, [pixel_maxd + 1]
    movd                        xm11, tmp0d
    VPBROADCASTW                 m11, xm11                 ;shift_4 for pmulhrsw

    APPLY_BDOF_MIN_BLOCK_LINE    m6, %2, %3, m7, (%1) * 4 + 0
    SAVE                        [dstq + 0 * dsq], 6

    APPLY_BDOF_MIN_BLOCK_LINE    m6, %2, %3, m7, (%1) * 4 + 1
    SAVE                        [dstq + 1 * dsq], 6

    APPLY_BDOF_MIN_BLOCK_LINE    m6, %2, %3, m7, (%1) * 4 + 2
    SAVE                        [dstq + 2 * dsq], 6

    APPLY_BDOF_MIN_BLOCK_LINE    m6, %2, %3, m7, (%1) * 4 + 3
    SAVE                        [dstq + ds3q], 6, %4
%endmacro

%macro SUM_MIN_BLOCK_W16 4-5 ; src/dst, shuffle, perm, tmp, [dst]
    pshufb  %4, %1, %2
    vpermd  %4, %3, %4
%if %0 == 4
    paddw   %1, %4
%else
    paddw   %5, %1, %4
%endif
%endmacro

%macro SUM_MIN_BLOCK_W8 3-4 ; src/dst, shuffle, tmp, [dst]
    pshufb  xm%3, xm%1, xm%2
%if %0 == 3
    paddw   xm%1, xm%3
%else
    paddw   xm%4, xm%1, xm%3
%endif
%endmacro

%macro BDOF_PROF_GRAD 2-3 0 ; line_no, last_line, assign (instead of add) to dst regs
%assign i0 (%1 + 0) % 3
%assign j0 (%1 + 1) % 3
%assign k0 (%1 + 2) % 3
%assign i1 3 + (%1 + 0) % 3
%assign j1 3 + (%1 + 1) % 3
%assign k1 3 + (%1 + 2) % 3

; we cached src0 in m0 to m2
%define t0 m %+ i0
%define c0 m %+ j0
%define b0 m %+ k0

; we cached src1 in m3 to m5
%define t1 m %+ i1
%define c1 m %+ j1
%define b1 m %+ k1
%define ndiff t1
%define off bdof_stack_offset(%1)

    movu                        b0, [src0q + (%1 + 2) * SRC_STRIDE + SRC_PS]
    movu                        b1, [src1q + (%1 + 2) * SRC_STRIDE + SRC_PS]

    ; gradient_v[0], gradient_v[1]
    DIFF                        m6,  b0,  t0, SHIFT, t0
    DIFF                        m7,  b1,  t1, SHIFT, t1

    ; save gradient_v[0] - gradient_v[1]
    psubw                      m10, m6, m7
    mova      [rsp + off + mmsize], m10

    ; gradient_h[0], gradient_h[1]
    LOAD_GRAD_H                 m8, src0q, (%1 + 1) * SRC_STRIDE, t0
    LOAD_GRAD_H                 m9, src1q, (%1 + 1) * SRC_STRIDE, t1

    ; save gradient_h[0] - gradient_h[1]
    psubw                      m11, m8, m9
    mova               [rsp + off], m11

    SUM_GRAD                    m8, m9                  ; temph
    SUM_GRAD                    m6, m7                  ; tempv

    DIFF                     ndiff, c1, c0, SHIFT2, t0  ; -diff

    ; use t0, t1 as temporary buffers
    mova                        t0, [pb_shuffle]

    psignw                      m7, ndiff, m8           ; sgxdi
    psignw                      m9, ndiff, m6           ; sgydi
    psignw                     m10, m8, m6              ; sgxgy

    pabsw                       m6, m6                  ; sgy2
    pabsw                       m8, m8                  ; sgx2

    cmp                         wd, 16

    je                       %%w16
    SUM_MIN_BLOCK_W8             6, i0, i1
    SUM_MIN_BLOCK_W8             7, i0, i1
    SUM_MIN_BLOCK_W8             8, i0, i1
    SUM_MIN_BLOCK_W8             9, i0, i1
%if (%3)
    SUM_MIN_BLOCK_W8            10, i0, i1, 13
%else
    SUM_MIN_BLOCK_W8            10, i0, i1
%endif
    jmp                     %%wend

%%w16:
    mova                        t1, [pd_perm_w16]
    SUM_MIN_BLOCK_W16           m6, t0, t1, m11
    SUM_MIN_BLOCK_W16           m7, t0, t1, m11
    SUM_MIN_BLOCK_W16           m8, t0, t1, m11
    SUM_MIN_BLOCK_W16           m9, t0, t1, m11
%if (%3)
    SUM_MIN_BLOCK_W16          m10, t0, t1, m11, m13
%else
    SUM_MIN_BLOCK_W16          m10, t0, t1, m11
%endif

%%wend:
    vpblendd                    m11, m8, m7, 10101010b
    vpblendd                     m7, m8, m7, 01010101b
    pshufd                       m7, m7, q2301
    paddw                        m8, m7, m11                ;4 x (2sgx2, 2sgxdi)

    vpblendd                    m11, m6, m9, 10101010b
    vpblendd                     m9, m6, m9, 01010101b
    pshufd                       m9, m9, q2301
    paddw                        m6, m9, m11                ;4 x (2sgy2, 2sgydi)

    vpblendw                    m11, m8, m6, 10101010b
    vpblendw                     m6, m8, m6, 01010101b
    pshuflw                      m6, m6, q2301
    pshufhw                      m6, m6, q2301
%if (%3)
    paddw                       m12, m6, m11                ; 4 x (4sgx2, 4sgy2, 4sgxdi, 4sgydi)
%else
    paddw                        m8, m6, m11                ; 4 x (4sgx2, 4sgy2, 4sgxdi, 4sgydi)
%endif

%if (%1) == 0
    ; pad for top and directly output to m12, m13
    paddw                      m12, m8,  m8
    paddw                      m13, m10, m10
%elifn (%3)
%if (%2)
    ; pad for bottom
    paddw                       m8, m8
    paddw                      m10, m10
%endif

    paddw                      m12, m8
    paddw                      m13, m10
%endif
%endmacro


%macro LOG2 3 ; dst, src, tmp
    cvtdq2ps             %1, %2
    ; The exponent contains log2 biased by 127 unless the value is zero.
    ; dst is only used as shift count where the value to be shifted is
    ; always zero if src is zero, so avoid using saturated subtraction.
    pcmpeqd              %3, %3
    psrld                %3, 25        ; pd_127
    psrld                %1, 23        ; floating point exponent
    psubd                %1, %3
%endmacro

; %1: 4 (sgx2, sgy2, sgxdi, gydi)
; %2: 4 (4sgxgy)
%macro BDOF_VX_VY 2       ;
    pshufd                 m%1, m%1, q3120
    vextracti128           xm7, m%1, 1

    punpcklqdq             xm8, xm%1, xm7           ; 4 (sgx2, sgy2)
    punpckhqdq             xm9, xm%1, xm7           ; 4 (sgxdi, sgydi)

    ; Promote to dword since vpsrlvw is AVX-512 only
    pmovzxwd                m8, xm8
    pmovsxwd                m9, xm9
    LOG2                   m10, m8, m7              ; 4 (log2(sgx2), log2(sgy2))

    pslld                   m9, 2                   ; 4 (sgxdi, sgydi)

    vpsravd                m11, m9, m10
    CLIPD                  m11, [pd_m15], [pd_15]   ; 4 (vx, junk)

    pshuflw                m%1, m11, q0000
    pshufhw                m%1, m%1, q0000          ; 4 (4vx)

    psllq                   m6, m%2, 32
    paddw                  m%2, m6

    pmaddwd                m%2, m%1                 ; 4 (junk, vx * sgxgy)
    psrad                  m%2, 1
    psubd                   m9, m%2                 ; 4 (junk, (sgydi << 2) - (vx * sgxgy >> 1))

    vpsravd                m%2, m9, m10
    CLIPD                  m%2, [pd_m15], [pd_15]   ; 4 (junk, vy)

    pshuflw                m%2, m%2, q2222
    pshufhw                m%2, m%2, q2222          ; 4 (4vy)
%endmacro


%macro BDOF_MINI_BLOCKS 2 ; (block_num, last_block)

%if (%1) == 0
    movu                    m0, [src0q + 0 * SRC_STRIDE + SRC_PS]
    movu                    m1, [src0q + 1 * SRC_STRIDE + SRC_PS]
    movu                    m3, [src1q + 0 * SRC_STRIDE + SRC_PS]
    movu                    m4, [src1q + 1 * SRC_STRIDE + SRC_PS]

    BDOF_PROF_GRAD           0, 0
%endif

%if (%1) != 1
    BDOF_PROF_GRAD  %1 * 4 + 1, 0
    BDOF_PROF_GRAD  %1 * 4 + 2, 0
%endif

%if (%2)
    BDOF_PROF_GRAD  %1 * 4 + 3, %2
    BDOF_VX_VY              12, 13
%if UNIX64
    APPLY_BDOF_MIN_BLOCK    %1, m12, m13
%else
    APPLY_BDOF_MIN_BLOCK    %1, m12, m13, .end
%endif

%else
    mova                   m14, m12
    mova                   m15, m13

    BDOF_PROF_GRAD  %1 * 4 + 3, 0, 1
    BDOF_PROF_GRAD  %1 * 4 + 4, 0
    paddw                  m14, m12
    paddw                  m15, m13

    BDOF_VX_VY              14, 15
    APPLY_BDOF_MIN_BLOCK    %1, m14, m15
    lea                   dstq, [dstq + 4 * dsq]
%endif
%endmacro

%macro BDOF_WRAPPER 2 ; bpp, is_nonadjacent
;void ff_vvc_apply_bdof_%1(uint8_t *dst, const ptrdiff_t dst_stride, const int16_t *src0,
;                          const int16_t *src1, const int w, const int h)
cglobal vvc_apply_bdof_%1
    ; r6 is not used for parameter passing and is volatile both on UNIX64
    ; and Win64, so it can be freely used
    mov                    r6d, (1<<%1)-1
%if %2
    jmp        vvc_apply_bdof_ %+ cpuname
%endif
%endmacro

%macro VVC_OF_AVX2 0
    BDOF_WRAPPER 12, 1
    BDOF_WRAPPER  8, 1
    BDOF_WRAPPER 10, 0

vvc_apply_bdof_ %+ cpuname:
; the prologue on Win64 is big (10 xmm regs need saving), so use PROLOGUE
; to avoid duplicating it.
PROLOGUE 6, 9, 16, BDOF_STACK_SIZE*32, dst, ds, src0, src1, w, h, pixel_max, ds3, tmp0
    lea                   ds3q, [dsq * 3]
    sub                  src0q, SRC_STRIDE + SRC_PS
    sub                  src1q, SRC_STRIDE + SRC_PS

    BDOF_MINI_BLOCKS         0, 0

    BDOF_PROF_GRAD  1 * 4 + 1, 0
    BDOF_PROF_GRAD  1 * 4 + 2, 0

    cmp                     hd, 16
    je                    .h16
    BDOF_MINI_BLOCKS         1, 1
%if UNIX64
    RET
%else
    jmp                   .end
%endif

.h16:
    BDOF_MINI_BLOCKS         1, 0
    BDOF_MINI_BLOCKS         2, 0
    BDOF_MINI_BLOCKS         3, 1

.end:
    RET
%endmacro

VVC_OF_AVX2

%endif ; HAVE_AVX2_EXTERNAL

%endif ; ARCH_X86_64
