mirror of
https://git.ffmpeg.org/ffmpeg.git
synced 2024-09-19 21:06:42 +00:00
9b4655c3a1
T-Head C908 (cycles): vp7_idct_dc_add_c: 108.5 vp7_idct_dc_add_rvv_i32: 56.2 (before) vp7_idct_dc_add_rvv_i32: 47.2 (after) vp8_idct_dc_add_c: 96.2 vp8_idct_dc_add_rvv_i32: 43.0 (before) vp8_idct_dc_add_rvv_i32: 34.0 (after)
552 lines
15 KiB
ArmAsm
552 lines
15 KiB
ArmAsm
/*
|
|
* Copyright (c) 2024 Institue of Software Chinese Academy of Sciences (ISCAS).
|
|
* Copyright © 2024 Rémi Denis-Courmont.
|
|
*
|
|
* This file is part of FFmpeg.
|
|
*
|
|
* FFmpeg is free software; you can redistribute it and/or
|
|
* modify it under the terms of the GNU Lesser General Public
|
|
* License as published by the Free Software Foundation; either
|
|
* version 2.1 of the License, or (at your option) any later version.
|
|
*
|
|
* FFmpeg is distributed in the hope that it will be useful,
|
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
|
* Lesser General Public License for more details.
|
|
*
|
|
* You should have received a copy of the GNU Lesser General Public
|
|
* License along with FFmpeg; if not, write to the Free Software
|
|
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
|
|
*/
|
|
|
|
#include "libavutil/riscv/asm.S"
|
|
|
|
.macro vsetvlstatic8 len
|
|
.if \len <= 4
|
|
vsetivli zero, \len, e8, mf4, ta, ma
|
|
.elseif \len <= 8
|
|
vsetivli zero, \len, e8, mf2, ta, ma
|
|
.elseif \len <= 16
|
|
vsetivli zero, \len, e8, m1, ta, ma
|
|
.elseif \len <= 31
|
|
vsetivli zero, \len, e8, m2, ta, ma
|
|
.endif
|
|
.endm
|
|
|
|
.macro vsetvlstatic16 len
|
|
.if \len <= 4
|
|
vsetivli zero, \len, e16, mf2, ta, ma
|
|
.elseif \len <= 8
|
|
vsetivli zero, \len, e16, m1, ta, ma
|
|
.elseif \len <= 16
|
|
vsetivli zero, \len, e16, m2, ta, ma
|
|
.endif
|
|
.endm
|
|
|
|
#if __riscv_xlen >= 64
|
|
func ff_vp8_luma_dc_wht_rvv, zve64x
|
|
lpad 0
|
|
vsetivli zero, 1, e64, m1, ta, ma
|
|
vlseg4e64.v v4, (a1)
|
|
vsetivli zero, 4, e16, mf2, ta, ma
|
|
vwadd.vv v1, v5, v6
|
|
addi t1, sp, -48
|
|
vwadd.vv v0, v4, v7
|
|
addi t2, sp, -32
|
|
vwsub.vv v2, v5, v6
|
|
addi t3, sp, -16
|
|
vwsub.vv v3, v4, v7
|
|
addi sp, sp, -64
|
|
vsetvli zero, zero, e32, m1, ta, ma
|
|
vadd.vv v4, v0, v1
|
|
vadd.vv v5, v3, v2
|
|
vse32.v v4, (sp)
|
|
vsub.vv v6, v0, v1
|
|
vse32.v v5, (t1)
|
|
vsub.vv v7, v3, v2
|
|
vse32.v v6, (t2)
|
|
vse32.v v7, (t3)
|
|
vlseg4e32.v v4, (sp)
|
|
vadd.vv v0, v4, v7
|
|
sd zero, (a1)
|
|
vadd.vv v1, v5, v6
|
|
sd zero, 8(a1)
|
|
vsub.vv v2, v5, v6
|
|
sd zero, 16(a1)
|
|
vsub.vv v3, v4, v7
|
|
sd zero, 24(a1)
|
|
vadd.vi v0, v0, 3 # rounding mode not supported, do it manually
|
|
li t0, 4 * 16 * 2
|
|
vadd.vi v3, v3, 3
|
|
addi t1, a0, 16 * 2
|
|
vadd.vv v4, v0, v1
|
|
addi t2, a0, 16 * 2 * 2
|
|
vadd.vv v5, v3, v2
|
|
addi t3, a0, 16 * 2 * 3
|
|
vsub.vv v6, v0, v1
|
|
vsub.vv v7, v3, v2
|
|
vsetvli zero, zero, e16, mf2, ta, ma
|
|
vnsra.wi v0, v4, 3
|
|
addi sp, sp, 64
|
|
vnsra.wi v1, v5, 3
|
|
vsse16.v v0, (a0), t0
|
|
vnsra.wi v2, v6, 3
|
|
vsse16.v v1, (t1), t0
|
|
vnsra.wi v3, v7, 3
|
|
vsse16.v v2, (t2), t0
|
|
vsse16.v v3, (t3), t0
|
|
ret
|
|
endfunc
|
|
#endif
|
|
|
|
func ff_vp8_idct_add_rvv, zve32x
|
|
lpad 0
|
|
csrwi vxrm, 0
|
|
vsetivli zero, 4, e16, mf2, ta, ma
|
|
addi a3, a1, 1 * 4 * 2
|
|
addi a4, a1, 2 * 4 * 2
|
|
addi a5, a1, 3 * 4 * 2
|
|
li t1, 20091
|
|
li t2, 35468
|
|
jal t0, 1f
|
|
vsseg4e16.v v0, (a1)
|
|
jal t0, 1f
|
|
vlsseg4e8.v v4, (a0), a2
|
|
vssra.vi v0, v0, 3
|
|
sd zero, (a1)
|
|
vssra.vi v1, v1, 3
|
|
sd zero, 8(a1)
|
|
vssra.vi v2, v2, 3
|
|
sd zero, 16(a1)
|
|
vssra.vi v3, v3, 3
|
|
sd zero, 24(a1)
|
|
vsetvli zero, zero, e8, mf4, ta, ma
|
|
vwaddu.wv v0, v0, v4
|
|
vwaddu.wv v1, v1, v5
|
|
vwaddu.wv v2, v2, v6
|
|
vwaddu.wv v3, v3, v7
|
|
vsetvli zero, zero, e16, mf2, ta, ma
|
|
vmax.vx v0, v0, zero
|
|
vmax.vx v1, v1, zero
|
|
vmax.vx v2, v2, zero
|
|
vmax.vx v3, v3, zero
|
|
vsetvli zero, zero, e8, mf4, ta, ma
|
|
vnclipu.wi v4, v0, 0
|
|
vnclipu.wi v5, v1, 0
|
|
vnclipu.wi v6, v2, 0
|
|
vnclipu.wi v7, v3, 0
|
|
vssseg4e8.v v4, (a0), a2
|
|
ret
|
|
1:
|
|
vle16.v v0, (a1)
|
|
vle16.v v2, (a4)
|
|
vle16.v v1, (a3)
|
|
vle16.v v3, (a5)
|
|
vadd.vv v4, v0, v2 # t0
|
|
vsub.vv v5, v0, v2 # t1
|
|
vmulhsu.vx v8, v3, t1
|
|
vmulhsu.vx v6, v1, t2
|
|
vadd.vv v8, v8, v3
|
|
vmulhsu.vx v7, v1, t1
|
|
vmulhsu.vx v9, v3, t2
|
|
vadd.vv v7, v7, v1
|
|
vsub.vv v6, v6, v8 # t2
|
|
vadd.vv v7, v7, v9 # t3
|
|
vadd.vv v1, v5, v6
|
|
vsub.vv v2, v5, v6
|
|
vadd.vv v0, v4, v7
|
|
vsub.vv v3, v4, v7
|
|
jr t0
|
|
endfunc
|
|
|
|
func ff_vp8_idct_dc_add_rvv, zve32x
|
|
lpad 0
|
|
lh a3, (a1)
|
|
addi a3, a3, 4
|
|
srai a3, a3, 3
|
|
# fall through
|
|
endfunc
|
|
|
|
# a3 = DC
|
|
func ff_vp78_idct_dc_add_rvv, zve32x
|
|
vsetivli zero, 4, e8, mf4, ta, ma
|
|
sh zero, (a1)
|
|
vlse32.v v8, (a0), a2
|
|
vsetivli zero, 16, e8, m1, ta, ma
|
|
bgez a3, 1f
|
|
|
|
# block[0] < 0
|
|
neg a3, a3
|
|
vssubu.vx v8, v8, a3
|
|
vsetivli zero, 4, e8, mf4, ta, ma
|
|
vsse32.v v8, (a0), a2
|
|
ret
|
|
|
|
1: # block[0] >= 0
|
|
vsaddu.vx v8, v8, a3
|
|
vsetivli zero, 4, e8, mf4, ta, ma
|
|
vsse32.v v8, (a0), a2
|
|
ret
|
|
endfunc
|
|
|
|
func ff_vp8_idct_dc_add4y_rvv, zve32x
|
|
lpad 0
|
|
li t0, 32
|
|
vsetivli zero, 4, e16, mf2, ta, ma
|
|
li t1, 4 - (128 << 3)
|
|
vlse16.v v8, (a1), t0
|
|
vadd.vx v8, v8, t1
|
|
vsra.vi v8, v8, 3
|
|
# fall through
|
|
endfunc
|
|
|
|
.variant_cc ff_vp78_idct_dc_add4y_rvv
|
|
# v8 = [dc0 - 128, dc1 - 128, dc2 - 128, dc3 - 128]
|
|
func ff_vp78_idct_dc_add4y_rvv, zve32x
|
|
vsetivli zero, 16, e16, m2, ta, ma
|
|
vid.v v4
|
|
li a4, 4
|
|
vsrl.vi v4, v4, 2
|
|
li t1, 128
|
|
vrgather.vv v0, v8, v4 # replicate each DC four times
|
|
vsetvli zero, zero, e8, m1, ta, ma
|
|
1:
|
|
vle8.v v8, (a0)
|
|
addi a4, a4, -1
|
|
vwaddu.wv v16, v0, v8
|
|
sh zero, (a1)
|
|
vnclip.wi v8, v16, 0
|
|
addi a1, a1, 32
|
|
vxor.vx v8, v8, t1
|
|
vse8.v v8, (a0)
|
|
add a0, a0, a2
|
|
bnez a4, 1b
|
|
|
|
ret
|
|
endfunc
|
|
|
|
func ff_vp8_idct_dc_add4uv_rvv, zve32x
|
|
lpad 0
|
|
li t0, 32
|
|
vsetivli zero, 4, e16, mf2, ta, ma
|
|
li t1, 4 - (128 << 3)
|
|
vlse16.v v8, (a1), t0
|
|
vadd.vx v8, v8, t1
|
|
vsra.vi v8, v8, 3
|
|
# fall through
|
|
endfunc
|
|
|
|
.variant_cc ff_vp78_idct_dc_add4uv_rvv
|
|
func ff_vp78_idct_dc_add4uv_rvv, zve64x
|
|
vsetivli zero, 16, e16, m2, ta, ma
|
|
vid.v v4
|
|
li a4, 4
|
|
vsrl.vi v4, v4, 2
|
|
li t1, 128
|
|
vrgather.vv v0, v8, v4 # replicate each DC four times
|
|
slli t2, a2, 2
|
|
vsetivli zero, 2, e64, m1, ta, ma
|
|
1:
|
|
vlse64.v v8, (a0), t2
|
|
addi a4, a4, -1
|
|
vsetivli zero, 16, e8, m1, ta, ma
|
|
vwaddu.wv v16, v0, v8
|
|
sh zero, (a1)
|
|
vnclip.wi v8, v16, 0
|
|
addi a1, a1, 32
|
|
vxor.vx v8, v8, t1
|
|
vsetivli zero, 2, e64, m1, ta, ma
|
|
vsse64.v v8, (a0), t2
|
|
add a0, a0, a2
|
|
bnez a4, 1b
|
|
|
|
ret
|
|
endfunc
|
|
|
|
.macro bilin_load_h dst mn
|
|
addi t5, a2, 1
|
|
vle8.v \dst, (a2)
|
|
vle8.v v2, (t5)
|
|
vwmulu.vx v28, \dst, t1
|
|
vwmaccu.vx v28, \mn, v2
|
|
vwaddu.wx v24, v28, t4
|
|
vnsra.wi \dst, v24, 3
|
|
.endm
|
|
|
|
.macro put_vp8_bilin_h_v type mn
|
|
func ff_put_vp8_bilin4_\type\()_rvv, zve32x
|
|
lpad 0
|
|
vsetvlstatic8 4
|
|
.Lbilin_\type:
|
|
li t1, 8
|
|
li t4, 4
|
|
sub t1, t1, \mn
|
|
1:
|
|
add t0, a2, a3
|
|
add t2, a0, a1
|
|
addi a4, a4, -2
|
|
.ifc \type,v
|
|
add t3, t0, a3
|
|
.else
|
|
addi t5, a2, 1
|
|
addi t3, t0, 1
|
|
vle8.v v2, (t5)
|
|
.endif
|
|
vle8.v v0, (a2)
|
|
vle8.v v4, (t0)
|
|
vle8.v v6, (t3)
|
|
vwmulu.vx v28, v0, t1
|
|
vwmulu.vx v26, v4, t1
|
|
.ifc \type,v
|
|
vwmaccu.vx v28, \mn, v4
|
|
.else
|
|
vwmaccu.vx v28, \mn, v2
|
|
.endif
|
|
vwmaccu.vx v26, \mn, v6
|
|
vwaddu.wx v24, v28, t4
|
|
vwaddu.wx v22, v26, t4
|
|
vnsra.wi v30, v24, 3
|
|
vnsra.wi v0, v22, 3
|
|
vse8.v v30, (a0)
|
|
vse8.v v0, (t2)
|
|
add a2, t0, a3
|
|
add a0, t2, a1
|
|
bnez a4, 1b
|
|
|
|
ret
|
|
endfunc
|
|
.endm
|
|
|
|
put_vp8_bilin_h_v h a5
|
|
put_vp8_bilin_h_v v a6
|
|
|
|
func ff_put_vp8_bilin4_hv_rvv, zve32x
|
|
lpad 0
|
|
vsetvlstatic8 4
|
|
.Lbilin_hv:
|
|
li t3, 8
|
|
sub t1, t3, a5
|
|
sub t2, t3, a6
|
|
li t4, 4
|
|
bilin_load_h v4, a5
|
|
add a2, a2, a3
|
|
1:
|
|
addi a4, a4, -1
|
|
vwmulu.vx v20, v4, t2
|
|
bilin_load_h v4, a5
|
|
vwmaccu.vx v20, a6, v4
|
|
vwaddu.wx v24, v20, t4
|
|
vnsra.wi v0, v24, 3
|
|
vse8.v v0, (a0)
|
|
add a2, a2, a3
|
|
add a0, a0, a1
|
|
bnez a4, 1b
|
|
|
|
ret
|
|
endfunc
|
|
|
|
.irp len,16,8
|
|
func ff_put_vp8_bilin\len\()_h_rvv, zve32x
|
|
lpad 0
|
|
vsetvlstatic8 \len
|
|
j .Lbilin_h
|
|
endfunc
|
|
|
|
func ff_put_vp8_bilin\len\()_v_rvv, zve32x
|
|
lpad 0
|
|
vsetvlstatic8 \len
|
|
j .Lbilin_v
|
|
endfunc
|
|
|
|
func ff_put_vp8_bilin\len\()_hv_rvv, zve32x
|
|
lpad 0
|
|
vsetvlstatic8 \len
|
|
j .Lbilin_hv
|
|
endfunc
|
|
.endr
|
|
|
|
const subpel_filters
|
|
.byte 0, -6, 123, 12, -1, 0
|
|
.byte 2, -11, 108, 36, -8, 1
|
|
.byte 0, -9, 93, 50, -6, 0
|
|
.byte 3, -16, 77, 77, -16, 3
|
|
.byte 0, -6, 50, 93, -9, 0
|
|
.byte 1, -8, 36, 108, -11, 2
|
|
.byte 0, -1, 12, 123, -6, 0
|
|
endconst
|
|
|
|
.macro epel_filter size type regtype
|
|
.ifc \type,v
|
|
addi \regtype\()0, a6, -1
|
|
.else
|
|
addi \regtype\()0, a5, -1
|
|
.endif
|
|
lla \regtype\()2, subpel_filters
|
|
sh1add \regtype\()0, \regtype\()0, \regtype\()0
|
|
sh1add \regtype\()0, \regtype\()0, \regtype\()2
|
|
.irp n,1,2,3,4
|
|
lb \regtype\n, \n(\regtype\()0)
|
|
.endr
|
|
.ifc \size,6
|
|
lb \regtype\()5, 5(\regtype\()0)
|
|
lb \regtype\()0, (\regtype\()0)
|
|
.endif
|
|
.endm
|
|
|
|
.macro epel_load dst len size type from_mem regtype
|
|
.ifc \type,v
|
|
sub t6, a2, a3
|
|
add a7, a2, a3
|
|
.else
|
|
addi t6, a2, -1
|
|
addi a7, a2, 1
|
|
.endif
|
|
|
|
.if \from_mem
|
|
vle8.v v24, (a2)
|
|
vle8.v v22, (t6)
|
|
vle8.v v26, (a7)
|
|
.ifc \type,v
|
|
add a7, a7, a3
|
|
.else
|
|
addi a7, a7, 1
|
|
.endif
|
|
vle8.v v28, (a7)
|
|
vwmulu.vx v16, v24, \regtype\()2
|
|
vwmulu.vx v20, v26, \regtype\()3
|
|
.ifc \size,6
|
|
.ifc \type,v
|
|
sub t6, t6, a3
|
|
add a7, a7, a3
|
|
.else
|
|
addi t6, t6, -1
|
|
addi a7, a7, 1
|
|
.endif
|
|
vle8.v v24, (t6)
|
|
vle8.v v26, (a7)
|
|
vwmaccu.vx v16, \regtype\()0, v24
|
|
vwmaccu.vx v16, \regtype\()5, v26
|
|
.endif
|
|
vwmaccsu.vx v16, \regtype\()1, v22
|
|
vwmaccsu.vx v16, \regtype\()4, v28
|
|
.else
|
|
vwmulu.vx v16, v4, \regtype\()2
|
|
vwmulu.vx v20, v6, \regtype\()3
|
|
.ifc \size,6
|
|
vwmaccu.vx v16, \regtype\()0, v0
|
|
vwmaccu.vx v16, \regtype\()5, v10
|
|
.endif
|
|
vwmaccsu.vx v16, \regtype\()1, v2
|
|
vwmaccsu.vx v16, \regtype\()4, v8
|
|
.endif
|
|
li t6, 64
|
|
vwadd.wx v16, v16, t6
|
|
vsetvlstatic16 \len
|
|
vwadd.vv v24, v16, v20
|
|
vnsra.wi v24, v24, 7
|
|
vmax.vx v24, v24, zero
|
|
vsetvlstatic8 \len
|
|
vnclipu.wi \dst, v24, 0
|
|
.endm
|
|
|
|
.macro epel_load_inc dst len size type from_mem regtype
|
|
epel_load \dst \len \size \type \from_mem \regtype
|
|
add a2, a2, a3
|
|
.endm
|
|
|
|
.macro epel len size type
|
|
func ff_put_vp8_epel\len\()_\type\()\size\()_rvv, zve32x, zba
|
|
lpad 0
|
|
epel_filter \size \type t
|
|
vsetvlstatic8 \len
|
|
1:
|
|
addi a4, a4, -1
|
|
epel_load_inc v30 \len \size \type 1 t
|
|
vse8.v v30, (a0)
|
|
add a0, a0, a1
|
|
bnez a4, 1b
|
|
|
|
ret
|
|
endfunc
|
|
.endm
|
|
|
|
.macro epel_hv len hsize vsize
|
|
func ff_put_vp8_epel\len\()_h\hsize\()v\vsize\()_rvv, zve32x, zba
|
|
lpad 0
|
|
#if __riscv_xlen == 64
|
|
addi sp, sp, -48
|
|
.irp n,0,1,2,3,4,5
|
|
sd s\n, (\n * 8)(sp)
|
|
.endr
|
|
#else
|
|
addi sp, sp, -32
|
|
.irp n,0,1,2,3,4,5
|
|
sw s\n, (\n * 4)(sp)
|
|
.endr
|
|
#endif
|
|
sub a2, a2, a3
|
|
epel_filter \hsize h t
|
|
epel_filter \vsize v s
|
|
vsetvlstatic8 \len
|
|
.if \hsize == 6 || \vsize == 6
|
|
sub a2, a2, a3
|
|
epel_load_inc v0 \len \hsize h 1 t
|
|
.endif
|
|
epel_load_inc v2 \len \hsize h 1 t
|
|
epel_load_inc v4 \len \hsize h 1 t
|
|
epel_load_inc v6 \len \hsize h 1 t
|
|
epel_load_inc v8 \len \hsize h 1 t
|
|
.if \hsize == 6 || \vsize == 6
|
|
epel_load_inc v10 \len \hsize h 1 t
|
|
.endif
|
|
addi a4, a4, -1
|
|
1:
|
|
addi a4, a4, -1
|
|
epel_load v30 \len \vsize v 0 s
|
|
vse8.v v30, (a0)
|
|
.if \hsize == 6 || \vsize == 6
|
|
vmv.v.v v0, v2
|
|
.endif
|
|
vmv.v.v v2, v4
|
|
vmv.v.v v4, v6
|
|
vmv.v.v v6, v8
|
|
.if \hsize == 6 || \vsize == 6
|
|
vmv.v.v v8, v10
|
|
epel_load_inc v10 \len \hsize h 1 t
|
|
.else
|
|
epel_load_inc v8 \len 4 h 1 t
|
|
.endif
|
|
add a0, a0, a1
|
|
bnez a4, 1b
|
|
epel_load v30 \len \vsize v 0 s
|
|
vse8.v v30, (a0)
|
|
|
|
#if __riscv_xlen == 64
|
|
.irp n,0,1,2,3,4,5
|
|
ld s\n, (\n * 8)(sp)
|
|
.endr
|
|
addi sp, sp, 48
|
|
#else
|
|
.irp n,0,1,2,3,4,5
|
|
lw s\n, (\n * 4)(sp)
|
|
.endr
|
|
addi sp, sp, 32
|
|
#endif
|
|
|
|
ret
|
|
endfunc
|
|
.endm
|
|
|
|
.irp len,16,8,4
|
|
epel \len 6 h
|
|
epel \len 4 h
|
|
epel \len 6 v
|
|
epel \len 4 v
|
|
#if __riscv_xlen <= 64
|
|
epel_hv \len 6 6
|
|
epel_hv \len 4 4
|
|
epel_hv \len 6 4
|
|
epel_hv \len 4 6
|
|
#endif
|
|
.endr
|