From f5ed254528062c9d97e285e95996e21640c938b7 Mon Sep 17 00:00:00 2001 From: Andreas Rheinhardt Date: Wed, 22 Apr 2026 14:50:22 +0200 Subject: [PATCH] swscale/x86/yuv2yuvX: Port ff_yuv2yuvX_mmxext to SSE2 The mmx function performs two registers in parallel; given the larger register size of SSE2, the same amount of data can be processed in one register with some speedups. (Given that this function is used for tail-processing, not processing more data is important.) Switching to SSE2 also fixes a bug introduced in 554c2bc7086f49ef5a6a989ad6bc4bc11807eb6f: Since said commit, only half the dither values were used. This seems not to matter in practice, as the functions here use dither only in the following form: ((filtersize-1)*8+dither)>>4. The dither values used here come from ff_dither_8x8_128 which has the property that ff_dither_8x8_128[i][j] and ff_dither_8x8_128[i][j+4] always lead to the same result in the above formula. Old benchmarks: yuv2yuvX_8_2_0_512_approximate_c: 2309.9 ( 1.00x) yuv2yuvX_8_2_0_512_approximate_mmxext: 250.2 ( 9.23x) yuv2yuvX_8_2_0_512_approximate_sse3: 98.8 (23.39x) yuv2yuvX_8_2_0_512_approximate_avx2: 52.9 (43.63x) yuv2yuvX_8_2_16_512_approximate_c: 2263.0 ( 1.00x) yuv2yuvX_8_2_16_512_approximate_mmxext: 245.3 ( 9.22x) yuv2yuvX_8_2_16_512_approximate_sse3: 114.3 (19.80x) yuv2yuvX_8_2_16_512_approximate_avx2: 85.6 (26.45x) yuv2yuvX_8_2_32_512_approximate_c: 2155.8 ( 1.00x) yuv2yuvX_8_2_32_512_approximate_mmxext: 235.6 ( 9.15x) yuv2yuvX_8_2_32_512_approximate_sse3: 93.6 (23.04x) yuv2yuvX_8_2_32_512_approximate_avx2: 78.1 (27.60x) yuv2yuvX_8_2_48_512_approximate_c: 2084.8 ( 1.00x) yuv2yuvX_8_2_48_512_approximate_mmxext: 230.2 ( 9.05x) yuv2yuvX_8_2_48_512_approximate_sse3: 105.0 (19.85x) yuv2yuvX_8_2_48_512_approximate_avx2: 71.9 (29.00x) yuv2yuvX_8_4_0_512_approximate_c: 3496.3 ( 1.00x) yuv2yuvX_8_4_0_512_approximate_mmxext: 455.0 ( 7.68x) yuv2yuvX_8_4_0_512_approximate_sse3: 157.5 (22.20x) yuv2yuvX_8_4_0_512_approximate_avx2: 88.4 (39.53x) yuv2yuvX_8_4_16_512_approximate_c: 3380.9 ( 1.00x) yuv2yuvX_8_4_16_512_approximate_mmxext: 440.0 ( 7.68x) yuv2yuvX_8_4_16_512_approximate_sse3: 175.0 (19.32x) yuv2yuvX_8_4_16_512_approximate_avx2: 134.1 (25.22x) yuv2yuvX_8_4_32_512_approximate_c: 3277.6 ( 1.00x) yuv2yuvX_8_4_32_512_approximate_mmxext: 427.2 ( 7.67x) yuv2yuvX_8_4_32_512_approximate_sse3: 149.7 (21.89x) yuv2yuvX_8_4_32_512_approximate_avx2: 115.5 (28.37x) yuv2yuvX_8_4_48_512_approximate_c: 3167.8 ( 1.00x) yuv2yuvX_8_4_48_512_approximate_mmxext: 414.9 ( 7.63x) yuv2yuvX_8_4_48_512_approximate_sse3: 164.1 (19.31x) yuv2yuvX_8_4_48_512_approximate_avx2: 101.2 (31.30x) yuv2yuvX_8_8_0_512_approximate_c: 5987.5 ( 1.00x) yuv2yuvX_8_8_0_512_approximate_mmxext: 854.1 ( 7.01x) yuv2yuvX_8_8_0_512_approximate_sse3: 294.6 (20.32x) yuv2yuvX_8_8_0_512_approximate_avx2: 144.1 (41.56x) yuv2yuvX_8_8_16_512_approximate_c: 5848.9 ( 1.00x) yuv2yuvX_8_8_16_512_approximate_mmxext: 834.4 ( 7.01x) yuv2yuvX_8_8_16_512_approximate_sse3: 312.1 (18.74x) yuv2yuvX_8_8_16_512_approximate_avx2: 214.9 (27.22x) yuv2yuvX_8_8_32_512_approximate_c: 5610.1 ( 1.00x) yuv2yuvX_8_8_32_512_approximate_mmxext: 811.6 ( 6.91x) yuv2yuvX_8_8_32_512_approximate_sse3: 277.5 (20.21x) yuv2yuvX_8_8_32_512_approximate_avx2: 189.8 (29.55x) yuv2yuvX_8_8_48_512_approximate_c: 5415.8 ( 1.00x) yuv2yuvX_8_8_48_512_approximate_mmxext: 782.3 ( 6.92x) yuv2yuvX_8_8_48_512_approximate_sse3: 289.4 (18.72x) yuv2yuvX_8_8_48_512_approximate_avx2: 165.3 (32.76x) yuv2yuvX_8_16_0_512_approximate_c: 11100.7 ( 1.00x) yuv2yuvX_8_16_0_512_approximate_mmxext: 1682.1 ( 6.60x) yuv2yuvX_8_16_0_512_approximate_sse3: 558.8 (19.86x) yuv2yuvX_8_16_0_512_approximate_avx2: 280.1 (39.63x) yuv2yuvX_8_16_16_512_approximate_c: 10772.1 ( 1.00x) yuv2yuvX_8_16_16_512_approximate_mmxext: 1611.0 ( 6.69x) yuv2yuvX_8_16_16_512_approximate_sse3: 578.1 (18.63x) yuv2yuvX_8_16_16_512_approximate_avx2: 418.8 (25.72x) yuv2yuvX_8_16_32_512_approximate_c: 10381.5 ( 1.00x) yuv2yuvX_8_16_32_512_approximate_mmxext: 1560.4 ( 6.65x) yuv2yuvX_8_16_32_512_approximate_sse3: 525.8 (19.74x) yuv2yuvX_8_16_32_512_approximate_avx2: 370.7 (28.01x) yuv2yuvX_8_16_48_512_approximate_c: 10046.1 ( 1.00x) yuv2yuvX_8_16_48_512_approximate_mmxext: 1512.4 ( 6.64x) yuv2yuvX_8_16_48_512_approximate_sse3: 546.0 (18.40x) yuv2yuvX_8_16_48_512_approximate_avx2: 315.0 (31.89x) New benchmarks: yuv2yuvX_8_2_0_512_approximate_c: 2302.5 ( 1.00x) yuv2yuvX_8_2_0_512_approximate_sse2: 184.4 (12.49x) yuv2yuvX_8_2_0_512_approximate_sse3: 100.1 (23.01x) yuv2yuvX_8_2_0_512_approximate_avx2: 54.9 (41.98x) yuv2yuvX_8_2_16_512_approximate_c: 2224.6 ( 1.00x) yuv2yuvX_8_2_16_512_approximate_sse2: 180.0 (12.36x) yuv2yuvX_8_2_16_512_approximate_sse3: 109.5 (20.31x) yuv2yuvX_8_2_16_512_approximate_avx2: 81.3 (27.35x) yuv2yuvX_8_2_32_512_approximate_c: 2165.3 ( 1.00x) yuv2yuvX_8_2_32_512_approximate_sse2: 176.6 (12.26x) yuv2yuvX_8_2_32_512_approximate_sse3: 93.7 (23.11x) yuv2yuvX_8_2_32_512_approximate_avx2: 73.1 (29.61x) yuv2yuvX_8_2_48_512_approximate_c: 2088.0 ( 1.00x) yuv2yuvX_8_2_48_512_approximate_sse2: 170.7 (12.23x) yuv2yuvX_8_2_48_512_approximate_sse3: 103.4 (20.20x) yuv2yuvX_8_2_48_512_approximate_avx2: 69.4 (30.10x) yuv2yuvX_8_4_0_512_approximate_c: 3496.8 ( 1.00x) yuv2yuvX_8_4_0_512_approximate_sse2: 320.3 (10.92x) yuv2yuvX_8_4_0_512_approximate_sse3: 158.8 (22.02x) yuv2yuvX_8_4_0_512_approximate_avx2: 86.4 (40.49x) yuv2yuvX_8_4_16_512_approximate_c: 3443.5 ( 1.00x) yuv2yuvX_8_4_16_512_approximate_sse2: 325.3 (10.59x) yuv2yuvX_8_4_16_512_approximate_sse3: 171.9 (20.03x) yuv2yuvX_8_4_16_512_approximate_avx2: 123.6 (27.85x) yuv2yuvX_8_4_32_512_approximate_c: 3272.2 ( 1.00x) yuv2yuvX_8_4_32_512_approximate_sse2: 302.7 (10.81x) yuv2yuvX_8_4_32_512_approximate_sse3: 148.9 (21.98x) yuv2yuvX_8_4_32_512_approximate_avx2: 110.6 (29.58x) yuv2yuvX_8_4_48_512_approximate_c: 3166.3 ( 1.00x) yuv2yuvX_8_4_48_512_approximate_sse2: 291.0 (10.88x) yuv2yuvX_8_4_48_512_approximate_sse3: 162.9 (19.44x) yuv2yuvX_8_4_48_512_approximate_avx2: 102.3 (30.95x) yuv2yuvX_8_8_0_512_approximate_c: 5967.6 ( 1.00x) yuv2yuvX_8_8_0_512_approximate_sse2: 691.2 ( 8.63x) yuv2yuvX_8_8_0_512_approximate_sse3: 294.2 (20.28x) yuv2yuvX_8_8_0_512_approximate_avx2: 154.9 (38.52x) yuv2yuvX_8_8_16_512_approximate_c: 5780.2 ( 1.00x) yuv2yuvX_8_8_16_512_approximate_sse2: 606.2 ( 9.53x) yuv2yuvX_8_8_16_512_approximate_sse3: 309.3 (18.69x) yuv2yuvX_8_8_16_512_approximate_avx2: 208.7 (27.69x) yuv2yuvX_8_8_32_512_approximate_c: 5604.3 ( 1.00x) yuv2yuvX_8_8_32_512_approximate_sse2: 592.3 ( 9.46x) yuv2yuvX_8_8_32_512_approximate_sse3: 281.1 (19.94x) yuv2yuvX_8_8_32_512_approximate_avx2: 185.4 (30.23x) yuv2yuvX_8_8_48_512_approximate_c: 5413.7 ( 1.00x) yuv2yuvX_8_8_48_512_approximate_sse2: 570.4 ( 9.49x) yuv2yuvX_8_8_48_512_approximate_sse3: 294.9 (18.36x) yuv2yuvX_8_8_48_512_approximate_avx2: 166.5 (32.51x) yuv2yuvX_8_16_0_512_approximate_c: 11099.4 ( 1.00x) yuv2yuvX_8_16_0_512_approximate_sse2: 1213.6 ( 9.15x) yuv2yuvX_8_16_0_512_approximate_sse3: 563.0 (19.72x) yuv2yuvX_8_16_0_512_approximate_avx2: 294.8 (37.65x) yuv2yuvX_8_16_16_512_approximate_c: 10718.1 ( 1.00x) yuv2yuvX_8_16_16_512_approximate_sse2: 1121.2 ( 9.56x) yuv2yuvX_8_16_16_512_approximate_sse3: 563.7 (19.01x) yuv2yuvX_8_16_16_512_approximate_avx2: 389.5 (27.51x) yuv2yuvX_8_16_32_512_approximate_c: 10373.3 ( 1.00x) yuv2yuvX_8_16_32_512_approximate_sse2: 1096.2 ( 9.46x) yuv2yuvX_8_16_32_512_approximate_sse3: 526.7 (19.70x) yuv2yuvX_8_16_32_512_approximate_avx2: 354.7 (29.24x) yuv2yuvX_8_16_48_512_approximate_c: 10066.9 ( 1.00x) yuv2yuvX_8_16_48_512_approximate_sse2: 1055.8 ( 9.53x) yuv2yuvX_8_16_48_512_approximate_sse3: 527.9 (19.07x) yuv2yuvX_8_16_48_512_approximate_avx2: 313.7 (32.09x) Signed-off-by: Andreas Rheinhardt --- libswscale/x86/swscale.c | 51 ++++++++++++++++++------------------- libswscale/x86/yuv2yuvX.asm | 44 ++++++++++++++++++-------------- tests/checkasm/sw_scale.c | 6 ++--- 3 files changed, 53 insertions(+), 48 deletions(-) diff --git a/libswscale/x86/swscale.c b/libswscale/x86/swscale.c index 26fd8aed3c..25461d787f 100644 --- a/libswscale/x86/swscale.c +++ b/libswscale/x86/swscale.c @@ -174,19 +174,6 @@ void ff_updateMMXDitherTables(SwsInternal *c, int dstY) } } -#define YUV2YUVX_FUNC_MMX(opt, step) \ -void ff_yuv2yuvX_ ##opt(const int16_t *filter, int filterSize, int srcOffset, \ - uint8_t *dest, int dstW, \ - const uint8_t *dither, int offset); \ -static void yuv2yuvX_ ##opt(const int16_t *filter, int filterSize, \ - const int16_t **src, uint8_t *dest, int dstW, \ - const uint8_t *dither, int offset) \ -{ \ - if(dstW > 0) \ - ff_yuv2yuvX_ ##opt(filter, filterSize - 1, 0, dest - offset, dstW + offset, dither, offset); \ - return; \ -} - #define YUV2YUVX_FUNC(opt, step) \ void ff_yuv2yuvX_ ##opt(const int16_t *filter, int filterSize, int srcOffset, \ uint8_t *dest, int dstW, \ @@ -198,26 +185,36 @@ static void yuv2yuvX_ ##opt(const int16_t *filter, int filterSize, \ int remainder = (dstW % step); \ int pixelsProcessed = dstW - remainder; \ if(((uintptr_t)dest) & 15){ \ - yuv2yuvX_mmxext(filter, filterSize, src, dest, dstW, dither, offset); \ + yuv2yuvX_sse2(filter, filterSize, src, dest, dstW, dither, offset); \ return; \ } \ if(pixelsProcessed > 0) \ ff_yuv2yuvX_ ##opt(filter, filterSize - 1, 0, dest - offset, pixelsProcessed + offset, dither, offset); \ if(remainder > 0){ \ - ff_yuv2yuvX_mmxext(filter, filterSize - 1, pixelsProcessed, dest - offset, pixelsProcessed + remainder + offset, dither, offset); \ + ff_yuv2yuvX_sse2(filter, filterSize - 1, pixelsProcessed, dest - offset, pixelsProcessed + remainder + offset, dither, offset); \ } \ return; \ } -#if HAVE_MMXEXT_EXTERNAL -YUV2YUVX_FUNC_MMX(mmxext, 16) -#endif +#if HAVE_SSE2_EXTERNAL +void ff_yuv2yuvX_sse2(const int16_t *filter, int filterSize, int srcOffset, + uint8_t *dest, int dstW, + const uint8_t *dither, int offset); +static void yuv2yuvX_sse2(const int16_t *filter, int filterSize, + const int16_t **src, uint8_t *dest, int dstW, + const uint8_t *dither, int offset) +{ + if (dstW > 0) + ff_yuv2yuvX_sse2(filter, filterSize - 1, 0, dest - offset, dstW + offset, dither, offset); + return; +} #if HAVE_SSE3_EXTERNAL YUV2YUVX_FUNC(sse3, 32) #endif #if HAVE_AVX2_EXTERNAL YUV2YUVX_FUNC(avx2, 64) #endif +#endif #define SCALE_FUNC(filter_n, from_bpc, to_bpc, opt) \ void ff_hscale ## from_bpc ## to ## to_bpc ## _ ## filter_n ## _ ## opt( \ @@ -513,18 +510,20 @@ av_cold void ff_sws_init_swscale_x86(SwsInternal *c) } #endif } else { -#if HAVE_MMXEXT_EXTERNAL - c->use_mmx_vfilter = 1; - c->yuv2planeX = yuv2yuvX_mmxext; +#if HAVE_SSE2_EXTERNAL + if (EXTERNAL_SSE2(cpu_flags)) { + c->use_mmx_vfilter = 1; + c->yuv2planeX = yuv2yuvX_sse2; #if HAVE_SSE3_EXTERNAL - if (EXTERNAL_SSE3(cpu_flags)) - c->yuv2planeX = yuv2yuvX_sse3; + if (EXTERNAL_SSE3(cpu_flags)) + c->yuv2planeX = yuv2yuvX_sse3; #endif #if HAVE_AVX2_EXTERNAL - if (EXTERNAL_AVX2_FAST(cpu_flags)) - c->yuv2planeX = yuv2yuvX_avx2; + if (EXTERNAL_AVX2_FAST(cpu_flags)) + c->yuv2planeX = yuv2yuvX_avx2; #endif -#endif /* HAVE_MMXEXT_EXTERNAL */ + } +#endif /* HAVE_SSE2_EXTERNAL */ #if HAVE_MMXEXT_INLINE if (!(c->opts.flags & SWS_FULL_CHR_H_INT)) { switch (c->opts.dst_format) { diff --git a/libswscale/x86/yuv2yuvX.asm b/libswscale/x86/yuv2yuvX.asm index 7137be2e17..66eff06e03 100644 --- a/libswscale/x86/yuv2yuvX.asm +++ b/libswscale/x86/yuv2yuvX.asm @@ -34,13 +34,13 @@ SECTION .text ;----------------------------------------------------------------------------- %macro YUV2YUVX_FUNC 0 -cglobal yuv2yuvX, 7, 7, 8, filter, filterSize, src, dest, dstW, dither, offset +cglobal yuv2yuvX, 7, 7, 6+2*cpuflag(sse3), filter, filterSize, src, dest, dstW, dither, offset %if notcpuflag(sse3) -%define movr mova +%define movr movq %define unroll 1 %else %define movr movdqu -%define unroll 2 +%define unroll 4 %endif movsxdifnidn dstWq, dstWd movsxdifnidn offsetq, offsetd @@ -67,14 +67,14 @@ cglobal yuv2yuvX, 7, 7, 8, filter, filterSize, src, dest, dstW, dither, offset mov srcq, [filterSizeq] punpcklbw m3, m0 psllw m1, m1, 3 - paddw m3, m3, m1 - psraw m7, m3, 4 + paddw m1, m3 + psraw m1, 4 .outerloop: - mova m4, m7 - mova m3, m7 + mova m3, m1 %if cpuflag(sse3) - mova m6, m7 - mova m1, m7 + mova m4, m1 + mova m6, m1 + mova m7, m1 %endif .loop: %if cpuflag(avx2) @@ -82,31 +82,37 @@ cglobal yuv2yuvX, 7, 7, 8, filter, filterSize, src, dest, dstW, dither, offset %elif cpuflag(sse3) movddup m0, [filterSizeq + 8] %else - mova m0, [filterSizeq + 8] + movq m0, [filterSizeq + 8] + punpcklqdq m0, m0 %endif + +%if cpuflag(sse3) pmulhw m2, m0, [srcq + offsetq * 2] pmulhw m5, m0, [srcq + offsetq * 2 + mmsize] paddw m3, m3, m2 paddw m4, m4, m5 -%if cpuflag(sse3) pmulhw m2, m0, [srcq + offsetq * 2 + 2 * mmsize] pmulhw m5, m0, [srcq + offsetq * 2 + 3 * mmsize] paddw m6, m6, m2 - paddw m1, m1, m5 + paddw m7, m7, m5 +%else + movu m2, [srcq + offsetq * 2] + pmulhw m2, m0 + paddw m3, m2 %endif add filterSizeq, 0x10 mov srcq, [filterSizeq] test srcq, srcq jnz .loop psraw m3, m3, 3 +%if cpuflag(sse3) psraw m4, m4, 3 -%if cpuflag(sse3) psraw m6, m6, 3 - psraw m1, m1, 3 -%endif + psraw m7, m7, 3 packuswb m3, m3, m4 -%if cpuflag(sse3) - packuswb m6, m6, m1 + packuswb m6, m6, m7 +%else + packuswb m3, m3 %endif mov srcq, [filterq] %if cpuflag(avx2) @@ -117,14 +123,14 @@ cglobal yuv2yuvX, 7, 7, 8, filter, filterSize, src, dest, dstW, dither, offset %if cpuflag(sse3) movr [destq + offsetq + mmsize], m6 %endif - add offsetq, mmsize * unroll + add offsetq, mmsize / 2 * unroll mov filterSizeq, filterq cmp offsetq, dstWq jb .outerloop RET %endmacro -INIT_MMX mmxext +INIT_XMM sse2 YUV2YUVX_FUNC INIT_XMM sse3 YUV2YUVX_FUNC diff --git a/tests/checkasm/sw_scale.c b/tests/checkasm/sw_scale.c index f3dd0019e5..3c33785122 100644 --- a/tests/checkasm/sw_scale.c +++ b/tests/checkasm/sw_scale.c @@ -181,9 +181,9 @@ static void check_yuv2yuvX(int accurate, int bit_depth, int dst_pix_format) static const int input_sizes[] = {8, 24, 128, 144, 256, 512}; const char *accurate_str = (accurate) ? "accurate" : "approximate"; - declare_func_emms(AV_CPU_FLAG_MMX, void, const int16_t *filter, - int filterSize, const int16_t **src, uint8_t *dest, - int dstW, const uint8_t *dither, int offset); + declare_func(void, const int16_t *filter, + int filterSize, const int16_t **src, uint8_t *dest, + int dstW, const uint8_t *dither, int offset); const int16_t **src; LOCAL_ALIGNED_16(int16_t, src_pixels, [LARGEST_FILTER * LARGEST_INPUT_SIZE]);