x86-64: Prefer EVEX512 string and memory functions on AMD Zen5
Resolves: RHEL-175520
This commit is contained in:
parent
92cd2bbb68
commit
2c1dcfb960
153
glibc-RHEL-175520-1.patch
Normal file
153
glibc-RHEL-175520-1.patch
Normal file
@ -0,0 +1,153 @@
|
||||
commit 43388b3ac09c757af1d94c9071e52b79b4bd8bb4
|
||||
Author: Noah Goldstein <goldstein.w.n@gmail.com>
|
||||
Date: Mon Jan 10 15:35:36 2022 -0600
|
||||
|
||||
string/test-str*cmp: remove stupid_[strcmp, strncmp, wcscmp, wcsncmp].
|
||||
|
||||
These implementations just add to test duration. Since we have
|
||||
simple_* implementations we already have a safe reference
|
||||
implementation.
|
||||
|
||||
Signed-off-by: Noah Goldstein <goldstein.w.n@gmail.com>
|
||||
|
||||
diff --git a/string/test-strcmp.c b/string/test-strcmp.c
|
||||
index a0255b9625fbcedd..a6b5344f5710effc 100644
|
||||
--- a/string/test-strcmp.c
|
||||
+++ b/string/test-strcmp.c
|
||||
@@ -36,7 +36,6 @@
|
||||
# define STRLEN wcslen
|
||||
# define MEMCPY wmemcpy
|
||||
# define SIMPLE_STRCMP simple_wcscmp
|
||||
-# define STUPID_STRCMP stupid_wcscmp
|
||||
# define CHAR wchar_t
|
||||
# define UCHAR wchar_t
|
||||
# define CHARBYTES 4
|
||||
@@ -66,25 +65,6 @@ simple_wcscmp (const wchar_t *s1, const wchar_t *s2)
|
||||
return c1 < c2 ? -1 : 1;
|
||||
}
|
||||
|
||||
-int
|
||||
-stupid_wcscmp (const wchar_t *s1, const wchar_t *s2)
|
||||
-{
|
||||
- size_t ns1 = wcslen (s1) + 1;
|
||||
- size_t ns2 = wcslen (s2) + 1;
|
||||
- size_t n = ns1 < ns2 ? ns1 : ns2;
|
||||
- int ret = 0;
|
||||
-
|
||||
- wchar_t c1, c2;
|
||||
-
|
||||
- while (n--) {
|
||||
- c1 = *s1++;
|
||||
- c2 = *s2++;
|
||||
- if ((ret = c1 < c2 ? -1 : c1 == c2 ? 0 : 1) != 0)
|
||||
- break;
|
||||
- }
|
||||
- return ret;
|
||||
-}
|
||||
-
|
||||
#else
|
||||
# include <limits.h>
|
||||
|
||||
@@ -94,7 +74,6 @@ stupid_wcscmp (const wchar_t *s1, const wchar_t *s2)
|
||||
# define STRLEN strlen
|
||||
# define MEMCPY memcpy
|
||||
# define SIMPLE_STRCMP simple_strcmp
|
||||
-# define STUPID_STRCMP stupid_strcmp
|
||||
# define CHAR char
|
||||
# define UCHAR unsigned char
|
||||
# define CHARBYTES 1
|
||||
@@ -115,24 +94,10 @@ simple_strcmp (const char *s1, const char *s2)
|
||||
return ret;
|
||||
}
|
||||
|
||||
-int
|
||||
-stupid_strcmp (const char *s1, const char *s2)
|
||||
-{
|
||||
- size_t ns1 = strlen (s1) + 1;
|
||||
- size_t ns2 = strlen (s2) + 1;
|
||||
- size_t n = ns1 < ns2 ? ns1 : ns2;
|
||||
- int ret = 0;
|
||||
-
|
||||
- while (n--)
|
||||
- if ((ret = *(unsigned char *) s1++ - *(unsigned char *) s2++) != 0)
|
||||
- break;
|
||||
- return ret;
|
||||
-}
|
||||
#endif
|
||||
|
||||
typedef int (*proto_t) (const CHAR *, const CHAR *);
|
||||
|
||||
-IMPL (STUPID_STRCMP, 1)
|
||||
IMPL (SIMPLE_STRCMP, 1)
|
||||
IMPL (STRCMP, 1)
|
||||
|
||||
diff --git a/string/test-strncmp.c b/string/test-strncmp.c
|
||||
index 56e23670ae7f90e4..a761136e94d65efc 100644
|
||||
--- a/string/test-strncmp.c
|
||||
+++ b/string/test-strncmp.c
|
||||
@@ -34,7 +34,6 @@
|
||||
# define STRDUP wcsdup
|
||||
# define MEMCPY wmemcpy
|
||||
# define SIMPLE_STRNCMP simple_wcsncmp
|
||||
-# define STUPID_STRNCMP stupid_wcsncmp
|
||||
# define CHAR wchar_t
|
||||
# define UCHAR wchar_t
|
||||
# define CHARBYTES 4
|
||||
@@ -58,25 +57,6 @@ simple_wcsncmp (const CHAR *s1, const CHAR *s2, size_t n)
|
||||
return 0;
|
||||
}
|
||||
|
||||
-int
|
||||
-stupid_wcsncmp (const CHAR *s1, const CHAR *s2, size_t n)
|
||||
-{
|
||||
- wchar_t c1, c2;
|
||||
- size_t ns1 = wcsnlen (s1, n) + 1, ns2 = wcsnlen (s2, n) + 1;
|
||||
-
|
||||
- n = ns1 < n ? ns1 : n;
|
||||
- n = ns2 < n ? ns2 : n;
|
||||
-
|
||||
- while (n--)
|
||||
- {
|
||||
- c1 = *s1++;
|
||||
- c2 = *s2++;
|
||||
- if (c1 != c2)
|
||||
- return c1 > c2 ? 1 : -1;
|
||||
- }
|
||||
- return 0;
|
||||
-}
|
||||
-
|
||||
#else
|
||||
# define L(str) str
|
||||
# define STRNCMP strncmp
|
||||
@@ -84,7 +64,6 @@ stupid_wcsncmp (const CHAR *s1, const CHAR *s2, size_t n)
|
||||
# define STRDUP strdup
|
||||
# define MEMCPY memcpy
|
||||
# define SIMPLE_STRNCMP simple_strncmp
|
||||
-# define STUPID_STRNCMP stupid_strncmp
|
||||
# define CHAR char
|
||||
# define UCHAR unsigned char
|
||||
# define CHARBYTES 1
|
||||
@@ -102,23 +81,10 @@ simple_strncmp (const char *s1, const char *s2, size_t n)
|
||||
return ret;
|
||||
}
|
||||
|
||||
-int
|
||||
-stupid_strncmp (const char *s1, const char *s2, size_t n)
|
||||
-{
|
||||
- size_t ns1 = strnlen (s1, n) + 1, ns2 = strnlen (s2, n) + 1;
|
||||
- int ret = 0;
|
||||
-
|
||||
- n = ns1 < n ? ns1 : n;
|
||||
- n = ns2 < n ? ns2 : n;
|
||||
- while (n-- && (ret = *(unsigned char *) s1++ - * (unsigned char *) s2++) == 0);
|
||||
- return ret;
|
||||
-}
|
||||
-
|
||||
#endif
|
||||
|
||||
typedef int (*proto_t) (const CHAR *, const CHAR *, size_t);
|
||||
|
||||
-IMPL (STUPID_STRNCMP, 0)
|
||||
IMPL (SIMPLE_STRNCMP, 0)
|
||||
IMPL (STRNCMP, 1)
|
||||
|
||||
977
glibc-RHEL-175520-10.patch
Normal file
977
glibc-RHEL-175520-10.patch
Normal file
@ -0,0 +1,977 @@
|
||||
commit a3c50bf46a1ca6d9d2b7d879176d345abf95a9de
|
||||
Author: Noah Goldstein <goldstein.w.n@gmail.com>
|
||||
Date: Thu Sep 21 09:38:37 2023 -0500
|
||||
|
||||
x86: Prepare `strrchr-evex` and `strrchr-evex512` for AVX10
|
||||
|
||||
This commit refactors `strrchr-evex` and `strrchr-evex512` to use a
|
||||
common implementation: `strrchr-evex-base.S`.
|
||||
|
||||
The motivation is `strrchr-evex` needed to be refactored to not use
|
||||
64-bit masked registers in preperation for AVX10.
|
||||
|
||||
Once vec-width masked register combining was removed, the EVEX and
|
||||
EVEX512 implementations can easily be implemented in the same file
|
||||
without any major overhead.
|
||||
|
||||
The net result is performance improvements (measured on TGL) for both
|
||||
`strrchr-evex` and `strrchr-evex512`. Although, note there are some
|
||||
regressions in the test suite and it may be many of the cases that
|
||||
make the total-geomean of improvement/regression across bench-strrchr
|
||||
are cold. The point of the performance measurement is to show there
|
||||
are no major regressions, but the primary motivation is preperation
|
||||
for AVX10.
|
||||
|
||||
Benchmarks where taken on TGL:
|
||||
https://www.intel.com/content/www/us/en/products/sku/213799/intel-core-i711850h-processor-24m-cache-up-to-4-80-ghz/specifications.html
|
||||
|
||||
EVEX geometric_mean(N=5) of all benchmarks New / Original : 0.74
|
||||
EVEX512 geometric_mean(N=5) of all benchmarks New / Original: 0.87
|
||||
|
||||
Full check passes on x86.
|
||||
|
||||
Conflicts:
|
||||
sysdeps/x86_64/multiarch/strrchr-evex-base.S
|
||||
(fixup context, missing spell fixes for removed comments)
|
||||
sysdeps/x86_64/multiarch/strrchr-evex.S
|
||||
(adapted downstream)
|
||||
|
||||
diff --git a/sysdeps/x86_64/multiarch/strrchr-evex-base.S b/sysdeps/x86_64/multiarch/strrchr-evex-base.S
|
||||
index 81cab3e0178c8b1e..8cb38be69fef5a45 100644
|
||||
--- a/sysdeps/x86_64/multiarch/strrchr-evex-base.S
|
||||
+++ b/sysdeps/x86_64/multiarch/strrchr-evex-base.S
|
||||
@@ -1,4 +1,4 @@
|
||||
-/* Placeholder function, not used by any processor at the moment.
|
||||
+/* Implementation for strrchr using evex256 and evex512.
|
||||
Copyright (C) 2022 Free Software Foundation, Inc.
|
||||
This file is part of the GNU C Library.
|
||||
|
||||
@@ -16,8 +16,6 @@
|
||||
License along with the GNU C Library; if not, see
|
||||
<https://www.gnu.org/licenses/>. */
|
||||
|
||||
-/* UNUSED. Exists purely as reference implementation. */
|
||||
-
|
||||
#include <isa-level.h>
|
||||
|
||||
#if ISA_SHOULD_BUILD (4)
|
||||
@@ -25,240 +23,351 @@
|
||||
# include <sysdep.h>
|
||||
|
||||
# ifdef USE_AS_WCSRCHR
|
||||
+# if VEC_SIZE == 64
|
||||
+# define RCX_M cx
|
||||
+# define KORTEST_M kortestw
|
||||
+# else
|
||||
+# define RCX_M cl
|
||||
+# define KORTEST_M kortestb
|
||||
+# endif
|
||||
+
|
||||
+# define SHIFT_REG VRCX
|
||||
# define CHAR_SIZE 4
|
||||
-# define VPBROADCAST vpbroadcastd
|
||||
-# define VPCMPEQ vpcmpeqd
|
||||
-# define VPMINU vpminud
|
||||
+# define VPCMP vpcmpd
|
||||
+# define VPMIN vpminud
|
||||
+# define VPCOMPRESS vpcompressd
|
||||
# define VPTESTN vptestnmd
|
||||
+# define VPTEST vptestmd
|
||||
+# define VPBROADCAST vpbroadcastd
|
||||
+# define VPCMPEQ vpcmpeqd
|
||||
+
|
||||
# else
|
||||
+# define SHIFT_REG VRDI
|
||||
# define CHAR_SIZE 1
|
||||
-# define VPBROADCAST vpbroadcastb
|
||||
-# define VPCMPEQ vpcmpeqb
|
||||
-# define VPMINU vpminub
|
||||
+# define VPCMP vpcmpb
|
||||
+# define VPMIN vpminub
|
||||
+# define VPCOMPRESS vpcompressb
|
||||
# define VPTESTN vptestnmb
|
||||
+# define VPTEST vptestmb
|
||||
+# define VPBROADCAST vpbroadcastb
|
||||
+# define VPCMPEQ vpcmpeqb
|
||||
+
|
||||
+# define RCX_M VRCX
|
||||
+# define KORTEST_M KORTEST
|
||||
# endif
|
||||
|
||||
-# define PAGE_SIZE 4096
|
||||
+# define VMATCH VMM(0)
|
||||
# define CHAR_PER_VEC (VEC_SIZE / CHAR_SIZE)
|
||||
+# define PAGE_SIZE 4096
|
||||
|
||||
.section SECTION(.text), "ax", @progbits
|
||||
-/* Aligning entry point to 64 byte, provides better performance for
|
||||
- one vector length string. */
|
||||
-ENTRY_P2ALIGN (STRRCHR, 6)
|
||||
-
|
||||
- /* Broadcast CHAR to VMM(0). */
|
||||
- VPBROADCAST %esi, %VMM(0)
|
||||
+ /* Aligning entry point to 64 byte, provides better performance for
|
||||
+ one vector length string. */
|
||||
+ENTRY_P2ALIGN(STRRCHR, 6)
|
||||
movl %edi, %eax
|
||||
- sall $20, %eax
|
||||
- cmpl $((PAGE_SIZE - VEC_SIZE) << 20), %eax
|
||||
- ja L(page_cross)
|
||||
+ /* Broadcast CHAR to VMATCH. */
|
||||
+ VPBROADCAST %esi, %VMATCH
|
||||
|
||||
-L(page_cross_continue):
|
||||
- /* Compare [w]char for null, mask bit will be set for match. */
|
||||
- VMOVU (%rdi), %VMM(1)
|
||||
+ andl $(PAGE_SIZE - 1), %eax
|
||||
+ cmpl $(PAGE_SIZE - VEC_SIZE), %eax
|
||||
+ jg L(cross_page_boundary)
|
||||
|
||||
- VPTESTN %VMM(1), %VMM(1), %k1
|
||||
- KMOV %k1, %VRCX
|
||||
- test %VRCX, %VRCX
|
||||
- jz L(align_more)
|
||||
-
|
||||
- VPCMPEQ %VMM(1), %VMM(0), %k0
|
||||
- KMOV %k0, %VRAX
|
||||
- BLSMSK %VRCX, %VRCX
|
||||
- and %VRCX, %VRAX
|
||||
- jz L(ret)
|
||||
-
|
||||
- BSR %VRAX, %VRAX
|
||||
+ VMOVU (%rdi), %VMM(1)
|
||||
+ /* k0 has a 1 for each zero CHAR in YMM1. */
|
||||
+ VPTESTN %VMM(1), %VMM(1), %k0
|
||||
+ KMOV %k0, %VGPR(rsi)
|
||||
+ test %VGPR(rsi), %VGPR(rsi)
|
||||
+ jz L(aligned_more)
|
||||
+ /* fallthrough: zero CHAR in first VEC. */
|
||||
+L(page_cross_return):
|
||||
+ /* K1 has a 1 for each search CHAR match in VEC(1). */
|
||||
+ VPCMPEQ %VMATCH, %VMM(1), %k1
|
||||
+ KMOV %k1, %VGPR(rax)
|
||||
+ /* Build mask up until first zero CHAR (used to mask of
|
||||
+ potential search CHAR matches past the end of the string). */
|
||||
+ blsmsk %VGPR(rsi), %VGPR(rsi)
|
||||
+ /* Use `and` here to remove any out of bounds matches so we can
|
||||
+ do a reverse scan on `rax` to find the last match. */
|
||||
+ and %VGPR(rsi), %VGPR(rax)
|
||||
+ jz L(ret0)
|
||||
+ /* Get last match. */
|
||||
+ bsr %VGPR(rax), %VGPR(rax)
|
||||
# ifdef USE_AS_WCSRCHR
|
||||
leaq (%rdi, %rax, CHAR_SIZE), %rax
|
||||
# else
|
||||
- add %rdi, %rax
|
||||
+ addq %rdi, %rax
|
||||
# endif
|
||||
-L(ret):
|
||||
+L(ret0):
|
||||
ret
|
||||
|
||||
-L(vector_x2_end):
|
||||
- VPCMPEQ %VMM(2), %VMM(0), %k2
|
||||
- KMOV %k2, %VRAX
|
||||
- BLSMSK %VRCX, %VRCX
|
||||
- and %VRCX, %VRAX
|
||||
- jz L(vector_x1_ret)
|
||||
-
|
||||
- BSR %VRAX, %VRAX
|
||||
- leaq (VEC_SIZE)(%rdi, %rax, CHAR_SIZE), %rax
|
||||
- ret
|
||||
-
|
||||
- /* Check the first vector at very last to look for match. */
|
||||
-L(vector_x1_ret):
|
||||
- VPCMPEQ %VMM(1), %VMM(0), %k2
|
||||
- KMOV %k2, %VRAX
|
||||
- test %VRAX, %VRAX
|
||||
- jz L(ret)
|
||||
-
|
||||
- BSR %VRAX, %VRAX
|
||||
+ /* Returns for first vec x1/x2/x3 have hard coded backward
|
||||
+ search path for earlier matches. */
|
||||
+ .p2align 4,, 6
|
||||
+L(first_vec_x1):
|
||||
+ VPCMPEQ %VMATCH, %VMM(2), %k1
|
||||
+ KMOV %k1, %VGPR(rax)
|
||||
+ blsmsk %VGPR(rcx), %VGPR(rcx)
|
||||
+ /* eax non-zero if search CHAR in range. */
|
||||
+ and %VGPR(rcx), %VGPR(rax)
|
||||
+ jnz L(first_vec_x1_return)
|
||||
+
|
||||
+ /* fallthrough: no match in YMM2 then need to check for earlier
|
||||
+ matches (in YMM1). */
|
||||
+ .p2align 4,, 4
|
||||
+L(first_vec_x0_test):
|
||||
+ VPCMPEQ %VMATCH, %VMM(1), %k1
|
||||
+ KMOV %k1, %VGPR(rax)
|
||||
+ test %VGPR(rax), %VGPR(rax)
|
||||
+ jz L(ret1)
|
||||
+ bsr %VGPR(rax), %VGPR(rax)
|
||||
# ifdef USE_AS_WCSRCHR
|
||||
leaq (%rsi, %rax, CHAR_SIZE), %rax
|
||||
# else
|
||||
- add %rsi, %rax
|
||||
+ addq %rsi, %rax
|
||||
# endif
|
||||
+L(ret1):
|
||||
ret
|
||||
|
||||
-L(align_more):
|
||||
- /* Zero r8 to store match result. */
|
||||
- xorl %r8d, %r8d
|
||||
- /* Save pointer of first vector, in case if no match found. */
|
||||
+ .p2align 4,, 10
|
||||
+L(first_vec_x3):
|
||||
+ VPCMPEQ %VMATCH, %VMM(4), %k1
|
||||
+ KMOV %k1, %VGPR(rax)
|
||||
+ blsmsk %VGPR(rcx), %VGPR(rcx)
|
||||
+ /* If no search CHAR match in range check YMM1/YMM2/YMM3. */
|
||||
+ and %VGPR(rcx), %VGPR(rax)
|
||||
+ jz L(first_vec_x1_or_x2)
|
||||
+ bsr %VGPR(rax), %VGPR(rax)
|
||||
+ leaq (VEC_SIZE * 3)(%rdi, %rax, CHAR_SIZE), %rax
|
||||
+ ret
|
||||
+ .p2align 4,, 4
|
||||
+
|
||||
+L(first_vec_x2):
|
||||
+ VPCMPEQ %VMATCH, %VMM(3), %k1
|
||||
+ KMOV %k1, %VGPR(rax)
|
||||
+ blsmsk %VGPR(rcx), %VGPR(rcx)
|
||||
+ /* Check YMM3 for last match first. If no match try YMM2/YMM1. */
|
||||
+ and %VGPR(rcx), %VGPR(rax)
|
||||
+ jz L(first_vec_x0_x1_test)
|
||||
+ bsr %VGPR(rax), %VGPR(rax)
|
||||
+ leaq (VEC_SIZE * 2)(%r8, %rax, CHAR_SIZE), %rax
|
||||
+ ret
|
||||
+
|
||||
+ .p2align 4,, 6
|
||||
+L(first_vec_x0_x1_test):
|
||||
+ VPCMPEQ %VMATCH, %VMM(2), %k1
|
||||
+ KMOV %k1, %VGPR(rax)
|
||||
+ /* Check YMM2 for last match first. If no match try YMM1. */
|
||||
+ test %VGPR(rax), %VGPR(rax)
|
||||
+ jz L(first_vec_x0_test)
|
||||
+ .p2align 4,, 4
|
||||
+L(first_vec_x1_return):
|
||||
+ bsr %VGPR(rax), %VGPR(rax)
|
||||
+ leaq (VEC_SIZE)(%r8, %rax, CHAR_SIZE), %rax
|
||||
+ ret
|
||||
+
|
||||
+ .p2align 4,, 12
|
||||
+L(aligned_more):
|
||||
+L(page_cross_continue):
|
||||
+ /* Need to keep original pointer incase VEC(1) has last match. */
|
||||
movq %rdi, %rsi
|
||||
- /* Align pointer to vector size. */
|
||||
andq $-VEC_SIZE, %rdi
|
||||
- /* Loop unroll for 2 vector loop. */
|
||||
- VMOVA (VEC_SIZE)(%rdi), %VMM(2)
|
||||
+
|
||||
+ VMOVU VEC_SIZE(%rdi), %VMM(2)
|
||||
VPTESTN %VMM(2), %VMM(2), %k0
|
||||
KMOV %k0, %VRCX
|
||||
+ movq %rdi, %r8
|
||||
test %VRCX, %VRCX
|
||||
- jnz L(vector_x2_end)
|
||||
+ jnz L(first_vec_x1)
|
||||
+
|
||||
+ VMOVU (VEC_SIZE * 2)(%rdi), %VMM(3)
|
||||
+ VPTESTN %VMM(3), %VMM(3), %k0
|
||||
+ KMOV %k0, %VRCX
|
||||
+
|
||||
+ test %VRCX, %VRCX
|
||||
+ jnz L(first_vec_x2)
|
||||
+
|
||||
+ VMOVU (VEC_SIZE * 3)(%rdi), %VMM(4)
|
||||
+ VPTESTN %VMM(4), %VMM(4), %k0
|
||||
+ KMOV %k0, %VRCX
|
||||
+
|
||||
+ /* Intentionally use 64-bit here. EVEX256 version needs 1-byte
|
||||
+ padding for efficient nop before loop alignment. */
|
||||
+ test %rcx, %rcx
|
||||
+ jnz L(first_vec_x3)
|
||||
|
||||
- /* Save pointer of second vector, in case if no match
|
||||
- found. */
|
||||
- movq %rdi, %r9
|
||||
- /* Align address to VEC_SIZE * 2 for loop. */
|
||||
andq $-(VEC_SIZE * 2), %rdi
|
||||
+ .p2align 4
|
||||
+L(first_aligned_loop):
|
||||
+ /* Preserve VEC(1), VEC(2), VEC(3), and VEC(4) until we can
|
||||
+ gurantee they don't store a match. */
|
||||
+ VMOVA (VEC_SIZE * 4)(%rdi), %VMM(5)
|
||||
+ VMOVA (VEC_SIZE * 5)(%rdi), %VMM(6)
|
||||
|
||||
- .p2align 4,,11
|
||||
-L(loop):
|
||||
- /* 2 vector loop, as it provide better performance as compared
|
||||
- to 4 vector loop. */
|
||||
- VMOVA (VEC_SIZE * 2)(%rdi), %VMM(3)
|
||||
- VMOVA (VEC_SIZE * 3)(%rdi), %VMM(4)
|
||||
- VPCMPEQ %VMM(3), %VMM(0), %k1
|
||||
- VPCMPEQ %VMM(4), %VMM(0), %k2
|
||||
- VPMINU %VMM(3), %VMM(4), %VMM(5)
|
||||
- VPTESTN %VMM(5), %VMM(5), %k0
|
||||
- KOR %k1, %k2, %k3
|
||||
- subq $-(VEC_SIZE * 2), %rdi
|
||||
- /* If k0 and k3 zero, match and end of string not found. */
|
||||
- KORTEST %k0, %k3
|
||||
- jz L(loop)
|
||||
-
|
||||
- /* If k0 is non zero, end of string found. */
|
||||
- KORTEST %k0, %k0
|
||||
- jnz L(endloop)
|
||||
-
|
||||
- lea VEC_SIZE(%rdi), %r8
|
||||
- /* A match found, it need to be stored in r8 before loop
|
||||
- continue. */
|
||||
- /* Check second vector first. */
|
||||
- KMOV %k2, %VRDX
|
||||
- test %VRDX, %VRDX
|
||||
- jnz L(loop_vec_x2_match)
|
||||
+ VPCMP $4, %VMM(5), %VMATCH, %k2
|
||||
+ VPCMP $4, %VMM(6), %VMATCH, %k3{%k2}
|
||||
|
||||
+ VPMIN %VMM(5), %VMM(6), %VMM(7)
|
||||
+
|
||||
+ VPTEST %VMM(7), %VMM(7), %k1{%k3}
|
||||
+ subq $(VEC_SIZE * -2), %rdi
|
||||
+ KORTEST_M %k1, %k1
|
||||
+ jc L(first_aligned_loop)
|
||||
+
|
||||
+ VPTESTN %VMM(7), %VMM(7), %k1
|
||||
KMOV %k1, %VRDX
|
||||
- /* Match is in first vector, rdi offset need to be substracted
|
||||
- by VEC_SIZE. */
|
||||
- sub $VEC_SIZE, %r8
|
||||
-
|
||||
- /* If second vector doesn't have match, first vector must
|
||||
- have match. */
|
||||
-L(loop_vec_x2_match):
|
||||
- BSR %VRDX, %VRDX
|
||||
-# ifdef USE_AS_WCSRCHR
|
||||
- sal $2, %rdx
|
||||
-# endif
|
||||
- add %rdx, %r8
|
||||
- jmp L(loop)
|
||||
+ test %VRDX, %VRDX
|
||||
+ jz L(second_aligned_loop_prep)
|
||||
|
||||
-L(endloop):
|
||||
- /* Check if string end in first loop vector. */
|
||||
- VPTESTN %VMM(3), %VMM(3), %k0
|
||||
- KMOV %k0, %VRCX
|
||||
- test %VRCX, %VRCX
|
||||
- jnz L(loop_vector_x1_end)
|
||||
+ KORTEST_M %k3, %k3
|
||||
+ jnc L(return_first_aligned_loop)
|
||||
|
||||
- /* Check if it has match in first loop vector. */
|
||||
- KMOV %k1, %VRAX
|
||||
+ .p2align 4,, 6
|
||||
+L(first_vec_x1_or_x2_or_x3):
|
||||
+ VPCMPEQ %VMM(4), %VMATCH, %k4
|
||||
+ KMOV %k4, %VRAX
|
||||
test %VRAX, %VRAX
|
||||
- jz L(loop_vector_x2_end)
|
||||
-
|
||||
- BSR %VRAX, %VRAX
|
||||
- leaq (%rdi, %rax, CHAR_SIZE), %r8
|
||||
+ jz L(first_vec_x1_or_x2)
|
||||
+ bsr %VRAX, %VRAX
|
||||
+ leaq (VEC_SIZE * 3)(%r8, %rax, CHAR_SIZE), %rax
|
||||
+ ret
|
||||
|
||||
- /* String must end in second loop vector. */
|
||||
-L(loop_vector_x2_end):
|
||||
- VPTESTN %VMM(4), %VMM(4), %k0
|
||||
+ .p2align 4,, 8
|
||||
+L(return_first_aligned_loop):
|
||||
+ VPTESTN %VMM(5), %VMM(5), %k0
|
||||
KMOV %k0, %VRCX
|
||||
+ blsmsk %VRCX, %VRCX
|
||||
+ jnc L(return_first_new_match_first)
|
||||
+ blsmsk %VRDX, %VRDX
|
||||
+ VPCMPEQ %VMM(6), %VMATCH, %k0
|
||||
+ KMOV %k0, %VRAX
|
||||
+ addq $VEC_SIZE, %rdi
|
||||
+ and %VRDX, %VRAX
|
||||
+ jnz L(return_first_new_match_ret)
|
||||
+ subq $VEC_SIZE, %rdi
|
||||
+L(return_first_new_match_first):
|
||||
KMOV %k2, %VRAX
|
||||
- BLSMSK %VRCX, %VRCX
|
||||
- /* Check if it has match in second loop vector. */
|
||||
+# ifdef USE_AS_WCSRCHR
|
||||
+ xorl $((1 << CHAR_PER_VEC)- 1), %VRAX
|
||||
and %VRCX, %VRAX
|
||||
- jz L(check_last_match)
|
||||
+# else
|
||||
+ andn %VRCX, %VRAX, %VRAX
|
||||
+# endif
|
||||
+ jz L(first_vec_x1_or_x2_or_x3)
|
||||
+L(return_first_new_match_ret):
|
||||
+ bsr %VRAX, %VRAX
|
||||
+ leaq (VEC_SIZE * 2)(%rdi, %rax, CHAR_SIZE), %rax
|
||||
+ ret
|
||||
|
||||
- BSR %VRAX, %VRAX
|
||||
- leaq (VEC_SIZE)(%rdi, %rax, CHAR_SIZE), %rax
|
||||
+ .p2align 4,, 10
|
||||
+L(first_vec_x1_or_x2):
|
||||
+ VPCMPEQ %VMM(3), %VMATCH, %k3
|
||||
+ KMOV %k3, %VRAX
|
||||
+ test %VRAX, %VRAX
|
||||
+ jz L(first_vec_x0_x1_test)
|
||||
+ bsr %VRAX, %VRAX
|
||||
+ leaq (VEC_SIZE * 2)(%r8, %rax, CHAR_SIZE), %rax
|
||||
ret
|
||||
|
||||
- /* String end in first loop vector. */
|
||||
-L(loop_vector_x1_end):
|
||||
- KMOV %k1, %VRAX
|
||||
- BLSMSK %VRCX, %VRCX
|
||||
- /* Check if it has match in second loop vector. */
|
||||
- and %VRCX, %VRAX
|
||||
- jz L(check_last_match)
|
||||
+ .p2align 4
|
||||
+ /* We can throw away the work done for the first 4x checks here
|
||||
+ as we have a later match. This is the 'fast' path persay. */
|
||||
+L(second_aligned_loop_prep):
|
||||
+L(second_aligned_loop_set_furthest_match):
|
||||
+ movq %rdi, %rsi
|
||||
+ VMOVA %VMM(5), %VMM(7)
|
||||
+ VMOVA %VMM(6), %VMM(8)
|
||||
+ .p2align 4
|
||||
+L(second_aligned_loop):
|
||||
+ VMOVU (VEC_SIZE * 4)(%rdi), %VMM(5)
|
||||
+ VMOVU (VEC_SIZE * 5)(%rdi), %VMM(6)
|
||||
+ VPCMP $4, %VMM(5), %VMATCH, %k2
|
||||
+ VPCMP $4, %VMM(6), %VMATCH, %k3{%k2}
|
||||
+
|
||||
+ VPMIN %VMM(5), %VMM(6), %VMM(4)
|
||||
+
|
||||
+ VPTEST %VMM(4), %VMM(4), %k1{%k3}
|
||||
+ subq $(VEC_SIZE * -2), %rdi
|
||||
+ KMOV %k1, %VRCX
|
||||
+ inc %RCX_M
|
||||
+ jz L(second_aligned_loop)
|
||||
+ VPTESTN %VMM(4), %VMM(4), %k1
|
||||
+ KMOV %k1, %VRDX
|
||||
+ test %VRDX, %VRDX
|
||||
+ jz L(second_aligned_loop_set_furthest_match)
|
||||
|
||||
- BSR %VRAX, %VRAX
|
||||
- leaq (%rdi, %rax, CHAR_SIZE), %rax
|
||||
- ret
|
||||
+ KORTEST_M %k3, %k3
|
||||
+ jnc L(return_new_match)
|
||||
+ /* branch here because there is a significant advantage interms
|
||||
+ of output dependency chance in using edx. */
|
||||
|
||||
- /* No match in first and second loop vector. */
|
||||
-L(check_last_match):
|
||||
- /* Check if any match recorded in r8. */
|
||||
- test %r8, %r8
|
||||
- jz L(vector_x2_ret)
|
||||
- movq %r8, %rax
|
||||
+L(return_old_match):
|
||||
+ VPCMPEQ %VMM(8), %VMATCH, %k0
|
||||
+ KMOV %k0, %VRCX
|
||||
+ bsr %VRCX, %VRCX
|
||||
+ jnz L(return_old_match_ret)
|
||||
+
|
||||
+ VPCMPEQ %VMM(7), %VMATCH, %k0
|
||||
+ KMOV %k0, %VRCX
|
||||
+ bsr %VRCX, %VRCX
|
||||
+ subq $VEC_SIZE, %rsi
|
||||
+L(return_old_match_ret):
|
||||
+ leaq (VEC_SIZE * 3)(%rsi, %rcx, CHAR_SIZE), %rax
|
||||
ret
|
||||
|
||||
- /* No match recorded in r8. Check the second saved vector
|
||||
- in begining. */
|
||||
-L(vector_x2_ret):
|
||||
- VPCMPEQ %VMM(2), %VMM(0), %k2
|
||||
+L(return_new_match):
|
||||
+ VPTESTN %VMM(5), %VMM(5), %k0
|
||||
+ KMOV %k0, %VRCX
|
||||
+ blsmsk %VRCX, %VRCX
|
||||
+ jnc L(return_new_match_first)
|
||||
+ dec %VRDX
|
||||
+ VPCMPEQ %VMM(6), %VMATCH, %k0
|
||||
+ KMOV %k0, %VRAX
|
||||
+ addq $VEC_SIZE, %rdi
|
||||
+ and %VRDX, %VRAX
|
||||
+ jnz L(return_new_match_ret)
|
||||
+ subq $VEC_SIZE, %rdi
|
||||
+L(return_new_match_first):
|
||||
KMOV %k2, %VRAX
|
||||
- test %VRAX, %VRAX
|
||||
- jz L(vector_x1_ret)
|
||||
-
|
||||
- /* Match found in the second saved vector. */
|
||||
- BSR %VRAX, %VRAX
|
||||
- leaq (VEC_SIZE)(%r9, %rax, CHAR_SIZE), %rax
|
||||
+# ifdef USE_AS_WCSRCHR
|
||||
+ xorl $((1 << CHAR_PER_VEC)- 1), %VRAX
|
||||
+ and %VRCX, %VRAX
|
||||
+# else
|
||||
+ andn %VRCX, %VRAX, %VRAX
|
||||
+# endif
|
||||
+ jz L(return_old_match)
|
||||
+L(return_new_match_ret):
|
||||
+ bsr %VRAX, %VRAX
|
||||
+ leaq (VEC_SIZE * 2)(%rdi, %rax, CHAR_SIZE), %rax
|
||||
ret
|
||||
|
||||
-L(page_cross):
|
||||
- mov %rdi, %rax
|
||||
- movl %edi, %ecx
|
||||
+ .p2align 4,, 4
|
||||
+L(cross_page_boundary):
|
||||
+ xorq %rdi, %rax
|
||||
+ mov $-1, %VRDX
|
||||
+ VMOVU (PAGE_SIZE - VEC_SIZE)(%rax), %VMM(6)
|
||||
+ VPTESTN %VMM(6), %VMM(6), %k0
|
||||
+ KMOV %k0, %VRSI
|
||||
|
||||
# ifdef USE_AS_WCSRCHR
|
||||
- /* Calculate number of compare result bits to be skipped for
|
||||
- wide string alignment adjustment. */
|
||||
- andl $(VEC_SIZE - 1), %ecx
|
||||
- sarl $2, %ecx
|
||||
+ movl %edi, %ecx
|
||||
+ and $(VEC_SIZE - 1), %ecx
|
||||
+ shrl $2, %ecx
|
||||
# endif
|
||||
- /* ecx contains number of w[char] to be skipped as a result
|
||||
- of address alignment. */
|
||||
- andq $-VEC_SIZE, %rax
|
||||
- VMOVA (%rax), %VMM(1)
|
||||
- VPTESTN %VMM(1), %VMM(1), %k1
|
||||
- KMOV %k1, %VRAX
|
||||
- SHR %cl, %VRAX
|
||||
- jz L(page_cross_continue)
|
||||
- VPCMPEQ %VMM(1), %VMM(0), %k0
|
||||
- KMOV %k0, %VRDX
|
||||
- SHR %cl, %VRDX
|
||||
- BLSMSK %VRAX, %VRAX
|
||||
- and %VRDX, %VRAX
|
||||
- jz L(ret)
|
||||
- BSR %VRAX, %VRAX
|
||||
+ shlx %SHIFT_REG, %VRDX, %VRDX
|
||||
+
|
||||
# ifdef USE_AS_WCSRCHR
|
||||
- leaq (%rdi, %rax, CHAR_SIZE), %rax
|
||||
+ kmovw %edx, %k1
|
||||
# else
|
||||
- add %rdi, %rax
|
||||
+ KMOV %VRDX, %k1
|
||||
# endif
|
||||
|
||||
- ret
|
||||
-END (STRRCHR)
|
||||
+ VPCOMPRESS %VMM(6), %VMM(1){%k1}{z}
|
||||
+ /* We could technically just jmp back after the vpcompress but
|
||||
+ it doesn't save any 16-byte blocks. */
|
||||
+ shrx %SHIFT_REG, %VRSI, %VRSI
|
||||
+ test %VRSI, %VRSI
|
||||
+ jnz L(page_cross_return)
|
||||
+ jmp L(page_cross_continue)
|
||||
+ /* 1-byte from cache line. */
|
||||
+END(STRRCHR)
|
||||
#endif
|
||||
diff --git a/sysdeps/x86_64/multiarch/strrchr-evex.S b/sysdeps/x86_64/multiarch/strrchr-evex.S
|
||||
index f5b6d755ceb85ae2..3bf6a5101422e4d1 100644
|
||||
--- a/sysdeps/x86_64/multiarch/strrchr-evex.S
|
||||
+++ b/sysdeps/x86_64/multiarch/strrchr-evex.S
|
||||
@@ -1,374 +1,8 @@
|
||||
-/* strrchr/wcsrchr optimized with 256-bit EVEX instructions.
|
||||
- Copyright (C) 2021 Free Software Foundation, Inc.
|
||||
- This file is part of the GNU C Library.
|
||||
-
|
||||
- The GNU C Library is free software; you can redistribute it and/or
|
||||
- modify it under the terms of the GNU Lesser General Public
|
||||
- License as published by the Free Software Foundation; either
|
||||
- version 2.1 of the License, or (at your option) any later version.
|
||||
-
|
||||
- The GNU C Library is distributed in the hope that it will be useful,
|
||||
- but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
- Lesser General Public License for more details.
|
||||
-
|
||||
- You should have received a copy of the GNU Lesser General Public
|
||||
- License along with the GNU C Library; if not, see
|
||||
- <https://www.gnu.org/licenses/>. */
|
||||
-
|
||||
-#if IS_IN (libc)
|
||||
-
|
||||
-# include <sysdep.h>
|
||||
-
|
||||
# ifndef STRRCHR
|
||||
# define STRRCHR __strrchr_evex
|
||||
# endif
|
||||
|
||||
-# define VMOVU vmovdqu64
|
||||
-# define VMOVA vmovdqa64
|
||||
-
|
||||
-# ifdef USE_AS_WCSRCHR
|
||||
-# define SHIFT_REG esi
|
||||
-
|
||||
-# define kunpck kunpckbw
|
||||
-# define kmov_2x kmovd
|
||||
-# define maskz_2x ecx
|
||||
-# define maskm_2x eax
|
||||
-# define CHAR_SIZE 4
|
||||
-# define VPMIN vpminud
|
||||
-# define VPTESTN vptestnmd
|
||||
-# define VPBROADCAST vpbroadcastd
|
||||
-# define VPCMP vpcmpd
|
||||
-# else
|
||||
-# define SHIFT_REG edi
|
||||
-
|
||||
-# define kunpck kunpckdq
|
||||
-# define kmov_2x kmovq
|
||||
-# define maskz_2x rcx
|
||||
-# define maskm_2x rax
|
||||
-
|
||||
-# define CHAR_SIZE 1
|
||||
-# define VPMIN vpminub
|
||||
-# define VPTESTN vptestnmb
|
||||
-# define VPBROADCAST vpbroadcastb
|
||||
-# define VPCMP vpcmpb
|
||||
-# endif
|
||||
-
|
||||
-# define XMMZERO xmm16
|
||||
-# define YMMZERO ymm16
|
||||
-# define YMMMATCH ymm17
|
||||
-# define YMMSAVE ymm18
|
||||
-
|
||||
-# define YMM1 ymm19
|
||||
-# define YMM2 ymm20
|
||||
-# define YMM3 ymm21
|
||||
-# define YMM4 ymm22
|
||||
-# define YMM5 ymm23
|
||||
-# define YMM6 ymm24
|
||||
-# define YMM7 ymm25
|
||||
-# define YMM8 ymm26
|
||||
-
|
||||
-
|
||||
-# define VEC_SIZE 32
|
||||
-# define PAGE_SIZE 4096
|
||||
- .section .text.evex, "ax", @progbits
|
||||
-ENTRY(STRRCHR)
|
||||
- movl %edi, %eax
|
||||
- /* Broadcast CHAR to YMMMATCH. */
|
||||
- VPBROADCAST %esi, %YMMMATCH
|
||||
-
|
||||
- andl $(PAGE_SIZE - 1), %eax
|
||||
- cmpl $(PAGE_SIZE - VEC_SIZE), %eax
|
||||
- jg L(cross_page_boundary)
|
||||
-
|
||||
-L(page_cross_continue):
|
||||
- VMOVU (%rdi), %YMM1
|
||||
- /* k0 has a 1 for each zero CHAR in YMM1. */
|
||||
- VPTESTN %YMM1, %YMM1, %k0
|
||||
- kmovd %k0, %ecx
|
||||
- testl %ecx, %ecx
|
||||
- jz L(aligned_more)
|
||||
- /* fallthrough: zero CHAR in first VEC. */
|
||||
-
|
||||
- /* K1 has a 1 for each search CHAR match in YMM1. */
|
||||
- VPCMP $0, %YMMMATCH, %YMM1, %k1
|
||||
- kmovd %k1, %eax
|
||||
- /* Build mask up until first zero CHAR (used to mask of
|
||||
- potential search CHAR matches past the end of the string).
|
||||
- */
|
||||
- blsmskl %ecx, %ecx
|
||||
- andl %ecx, %eax
|
||||
- jz L(ret0)
|
||||
- /* Get last match (the `andl` removed any out of bounds
|
||||
- matches). */
|
||||
- bsrl %eax, %eax
|
||||
-# ifdef USE_AS_WCSRCHR
|
||||
- leaq (%rdi, %rax, CHAR_SIZE), %rax
|
||||
-# else
|
||||
- addq %rdi, %rax
|
||||
-# endif
|
||||
-L(ret0):
|
||||
- ret
|
||||
-
|
||||
- /* Returns for first vec x1/x2/x3 have hard coded backward
|
||||
- search path for earlier matches. */
|
||||
- .p2align 4,, 6
|
||||
-L(first_vec_x1):
|
||||
- VPCMP $0, %YMMMATCH, %YMM2, %k1
|
||||
- kmovd %k1, %eax
|
||||
- blsmskl %ecx, %ecx
|
||||
- /* eax non-zero if search CHAR in range. */
|
||||
- andl %ecx, %eax
|
||||
- jnz L(first_vec_x1_return)
|
||||
-
|
||||
- /* fallthrough: no match in YMM2 then need to check for earlier
|
||||
- matches (in YMM1). */
|
||||
- .p2align 4,, 4
|
||||
-L(first_vec_x0_test):
|
||||
- VPCMP $0, %YMMMATCH, %YMM1, %k1
|
||||
- kmovd %k1, %eax
|
||||
- testl %eax, %eax
|
||||
- jz L(ret1)
|
||||
- bsrl %eax, %eax
|
||||
-# ifdef USE_AS_WCSRCHR
|
||||
- leaq (%rsi, %rax, CHAR_SIZE), %rax
|
||||
-# else
|
||||
- addq %rsi, %rax
|
||||
-# endif
|
||||
-L(ret1):
|
||||
- ret
|
||||
-
|
||||
- .p2align 4,, 10
|
||||
-L(first_vec_x1_or_x2):
|
||||
- VPCMP $0, %YMM3, %YMMMATCH, %k3
|
||||
- VPCMP $0, %YMM2, %YMMMATCH, %k2
|
||||
- /* K2 and K3 have 1 for any search CHAR match. Test if any
|
||||
- matches between either of them. Otherwise check YMM1. */
|
||||
- kortestd %k2, %k3
|
||||
- jz L(first_vec_x0_test)
|
||||
-
|
||||
- /* Guranteed that YMM2 and YMM3 are within range so merge the
|
||||
- two bitmasks then get last result. */
|
||||
- kunpck %k2, %k3, %k3
|
||||
- kmovq %k3, %rax
|
||||
- bsrq %rax, %rax
|
||||
- leaq (VEC_SIZE)(%r8, %rax, CHAR_SIZE), %rax
|
||||
- ret
|
||||
-
|
||||
- .p2align 4,, 6
|
||||
-L(first_vec_x3):
|
||||
- VPCMP $0, %YMMMATCH, %YMM4, %k1
|
||||
- kmovd %k1, %eax
|
||||
- blsmskl %ecx, %ecx
|
||||
- /* If no search CHAR match in range check YMM1/YMM2/YMM3. */
|
||||
- andl %ecx, %eax
|
||||
- jz L(first_vec_x1_or_x2)
|
||||
- bsrl %eax, %eax
|
||||
- leaq (VEC_SIZE * 3)(%rdi, %rax, CHAR_SIZE), %rax
|
||||
- ret
|
||||
-
|
||||
- .p2align 4,, 6
|
||||
-L(first_vec_x0_x1_test):
|
||||
- VPCMP $0, %YMMMATCH, %YMM2, %k1
|
||||
- kmovd %k1, %eax
|
||||
- /* Check YMM2 for last match first. If no match try YMM1. */
|
||||
- testl %eax, %eax
|
||||
- jz L(first_vec_x0_test)
|
||||
- .p2align 4,, 4
|
||||
-L(first_vec_x1_return):
|
||||
- bsrl %eax, %eax
|
||||
- leaq (VEC_SIZE)(%rdi, %rax, CHAR_SIZE), %rax
|
||||
- ret
|
||||
-
|
||||
- .p2align 4,, 10
|
||||
-L(first_vec_x2):
|
||||
- VPCMP $0, %YMMMATCH, %YMM3, %k1
|
||||
- kmovd %k1, %eax
|
||||
- blsmskl %ecx, %ecx
|
||||
- /* Check YMM3 for last match first. If no match try YMM2/YMM1.
|
||||
- */
|
||||
- andl %ecx, %eax
|
||||
- jz L(first_vec_x0_x1_test)
|
||||
- bsrl %eax, %eax
|
||||
- leaq (VEC_SIZE * 2)(%rdi, %rax, CHAR_SIZE), %rax
|
||||
- ret
|
||||
-
|
||||
-
|
||||
- .p2align 4
|
||||
-L(aligned_more):
|
||||
- /* Need to keep original pointer incase YMM1 has last match. */
|
||||
- movq %rdi, %rsi
|
||||
- andq $-VEC_SIZE, %rdi
|
||||
- VMOVU VEC_SIZE(%rdi), %YMM2
|
||||
- VPTESTN %YMM2, %YMM2, %k0
|
||||
- kmovd %k0, %ecx
|
||||
- testl %ecx, %ecx
|
||||
- jnz L(first_vec_x1)
|
||||
-
|
||||
- VMOVU (VEC_SIZE * 2)(%rdi), %YMM3
|
||||
- VPTESTN %YMM3, %YMM3, %k0
|
||||
- kmovd %k0, %ecx
|
||||
- testl %ecx, %ecx
|
||||
- jnz L(first_vec_x2)
|
||||
-
|
||||
- VMOVU (VEC_SIZE * 3)(%rdi), %YMM4
|
||||
- VPTESTN %YMM4, %YMM4, %k0
|
||||
- kmovd %k0, %ecx
|
||||
- movq %rdi, %r8
|
||||
- testl %ecx, %ecx
|
||||
- jnz L(first_vec_x3)
|
||||
-
|
||||
- andq $-(VEC_SIZE * 2), %rdi
|
||||
- .p2align 4
|
||||
-L(first_aligned_loop):
|
||||
- /* Preserve YMM1, YMM2, YMM3, and YMM4 until we can gurantee
|
||||
- they don't store a match. */
|
||||
- VMOVA (VEC_SIZE * 4)(%rdi), %YMM5
|
||||
- VMOVA (VEC_SIZE * 5)(%rdi), %YMM6
|
||||
-
|
||||
- VPCMP $0, %YMM5, %YMMMATCH, %k2
|
||||
- vpxord %YMM6, %YMMMATCH, %YMM7
|
||||
-
|
||||
- VPMIN %YMM5, %YMM6, %YMM8
|
||||
- VPMIN %YMM8, %YMM7, %YMM7
|
||||
-
|
||||
- VPTESTN %YMM7, %YMM7, %k1
|
||||
- subq $(VEC_SIZE * -2), %rdi
|
||||
- kortestd %k1, %k2
|
||||
- jz L(first_aligned_loop)
|
||||
-
|
||||
- VPCMP $0, %YMM6, %YMMMATCH, %k3
|
||||
- VPTESTN %YMM8, %YMM8, %k1
|
||||
- ktestd %k1, %k1
|
||||
- jz L(second_aligned_loop_prep)
|
||||
-
|
||||
- kortestd %k2, %k3
|
||||
- jnz L(return_first_aligned_loop)
|
||||
-
|
||||
- .p2align 4,, 6
|
||||
-L(first_vec_x1_or_x2_or_x3):
|
||||
- VPCMP $0, %YMM4, %YMMMATCH, %k4
|
||||
- kmovd %k4, %eax
|
||||
- testl %eax, %eax
|
||||
- jz L(first_vec_x1_or_x2)
|
||||
- bsrl %eax, %eax
|
||||
- leaq (VEC_SIZE * 3)(%r8, %rax, CHAR_SIZE), %rax
|
||||
- ret
|
||||
-
|
||||
- .p2align 4,, 8
|
||||
-L(return_first_aligned_loop):
|
||||
- VPTESTN %YMM5, %YMM5, %k0
|
||||
- kunpck %k0, %k1, %k0
|
||||
- kmov_2x %k0, %maskz_2x
|
||||
-
|
||||
- blsmsk %maskz_2x, %maskz_2x
|
||||
- kunpck %k2, %k3, %k3
|
||||
- kmov_2x %k3, %maskm_2x
|
||||
- and %maskz_2x, %maskm_2x
|
||||
- jz L(first_vec_x1_or_x2_or_x3)
|
||||
-
|
||||
- bsr %maskm_2x, %maskm_2x
|
||||
- leaq (VEC_SIZE * 2)(%rdi, %rax, CHAR_SIZE), %rax
|
||||
- ret
|
||||
-
|
||||
- .p2align 4
|
||||
- /* We can throw away the work done for the first 4x checks here
|
||||
- as we have a later match. This is the 'fast' path persay.
|
||||
- */
|
||||
-L(second_aligned_loop_prep):
|
||||
-L(second_aligned_loop_set_furthest_match):
|
||||
- movq %rdi, %rsi
|
||||
- kunpck %k2, %k3, %k4
|
||||
-
|
||||
- .p2align 4
|
||||
-L(second_aligned_loop):
|
||||
- VMOVU (VEC_SIZE * 4)(%rdi), %YMM1
|
||||
- VMOVU (VEC_SIZE * 5)(%rdi), %YMM2
|
||||
-
|
||||
- VPCMP $0, %YMM1, %YMMMATCH, %k2
|
||||
- vpxord %YMM2, %YMMMATCH, %YMM3
|
||||
-
|
||||
- VPMIN %YMM1, %YMM2, %YMM4
|
||||
- VPMIN %YMM3, %YMM4, %YMM3
|
||||
-
|
||||
- VPTESTN %YMM3, %YMM3, %k1
|
||||
- subq $(VEC_SIZE * -2), %rdi
|
||||
- kortestd %k1, %k2
|
||||
- jz L(second_aligned_loop)
|
||||
-
|
||||
- VPCMP $0, %YMM2, %YMMMATCH, %k3
|
||||
- VPTESTN %YMM4, %YMM4, %k1
|
||||
- ktestd %k1, %k1
|
||||
- jz L(second_aligned_loop_set_furthest_match)
|
||||
-
|
||||
- kortestd %k2, %k3
|
||||
- /* branch here because there is a significant advantage interms
|
||||
- of output dependency chance in using edx. */
|
||||
- jnz L(return_new_match)
|
||||
-L(return_old_match):
|
||||
- kmovq %k4, %rax
|
||||
- bsrq %rax, %rax
|
||||
- leaq (VEC_SIZE * 2)(%rsi, %rax, CHAR_SIZE), %rax
|
||||
- ret
|
||||
-
|
||||
-L(return_new_match):
|
||||
- VPTESTN %YMM1, %YMM1, %k0
|
||||
- kunpck %k0, %k1, %k0
|
||||
- kmov_2x %k0, %maskz_2x
|
||||
-
|
||||
- blsmsk %maskz_2x, %maskz_2x
|
||||
- kunpck %k2, %k3, %k3
|
||||
- kmov_2x %k3, %maskm_2x
|
||||
- and %maskz_2x, %maskm_2x
|
||||
- jz L(return_old_match)
|
||||
-
|
||||
- bsr %maskm_2x, %maskm_2x
|
||||
- leaq (VEC_SIZE * 2)(%rdi, %rax, CHAR_SIZE), %rax
|
||||
- ret
|
||||
-
|
||||
-L(cross_page_boundary):
|
||||
- /* eax contains all the page offset bits of src (rdi). `xor rdi,
|
||||
- rax` sets pointer will all page offset bits cleared so
|
||||
- offset of (PAGE_SIZE - VEC_SIZE) will get last aligned VEC
|
||||
- before page cross (guranteed to be safe to read). Doing this
|
||||
- as opposed to `movq %rdi, %rax; andq $-VEC_SIZE, %rax` saves
|
||||
- a bit of code size. */
|
||||
- xorq %rdi, %rax
|
||||
- VMOVU (PAGE_SIZE - VEC_SIZE)(%rax), %YMM1
|
||||
- VPTESTN %YMM1, %YMM1, %k0
|
||||
- kmovd %k0, %ecx
|
||||
-
|
||||
- /* Shift out zero CHAR matches that are before the begining of
|
||||
- src (rdi). */
|
||||
-# ifdef USE_AS_WCSRCHR
|
||||
- movl %edi, %esi
|
||||
- andl $(VEC_SIZE - 1), %esi
|
||||
- shrl $2, %esi
|
||||
-# endif
|
||||
- shrxl %SHIFT_REG, %ecx, %ecx
|
||||
-
|
||||
- testl %ecx, %ecx
|
||||
- jz L(page_cross_continue)
|
||||
-
|
||||
- /* Found zero CHAR so need to test for search CHAR. */
|
||||
- VPCMP $0, %YMMMATCH, %YMM1, %k1
|
||||
- kmovd %k1, %eax
|
||||
- /* Shift out search CHAR matches that are before the begining of
|
||||
- src (rdi). */
|
||||
- shrxl %SHIFT_REG, %eax, %eax
|
||||
-
|
||||
- /* Check if any search CHAR match in range. */
|
||||
- blsmskl %ecx, %ecx
|
||||
- andl %ecx, %eax
|
||||
- jz L(ret3)
|
||||
- bsrl %eax, %eax
|
||||
-# ifdef USE_AS_WCSRCHR
|
||||
- leaq (%rdi, %rax, CHAR_SIZE), %rax
|
||||
-# else
|
||||
- addq %rdi, %rax
|
||||
-# endif
|
||||
-L(ret3):
|
||||
- ret
|
||||
+#include "x86-evex256-vecs.h"
|
||||
+#include "reg-macros.h"
|
||||
|
||||
-END(STRRCHR)
|
||||
-#endif
|
||||
+#include "strrchr-evex-base.S"
|
||||
diff --git a/sysdeps/x86_64/multiarch/wcsrchr-evex.S b/sysdeps/x86_64/multiarch/wcsrchr-evex.S
|
||||
index c64602f7dc154ad6..7bd265a24977f339 100644
|
||||
--- a/sysdeps/x86_64/multiarch/wcsrchr-evex.S
|
||||
+++ b/sysdeps/x86_64/multiarch/wcsrchr-evex.S
|
||||
@@ -1,3 +1,4 @@
|
||||
#define STRRCHR __wcsrchr_evex
|
||||
#define USE_AS_WCSRCHR 1
|
||||
+#define USE_WIDE_CHAR 1
|
||||
#include "strrchr-evex.S"
|
||||
157
glibc-RHEL-175520-11.patch
Normal file
157
glibc-RHEL-175520-11.patch
Normal file
@ -0,0 +1,157 @@
|
||||
commit b7f8b6b64b135e6dba5083d688675874d6809c91
|
||||
Author: Noah Goldstein <goldstein.w.n@gmail.com>
|
||||
Date: Wed Nov 1 16:42:07 2023 -0500
|
||||
|
||||
x86: Fix unchecked AVX512-VBMI2 usage in strrchr-evex-base.S
|
||||
|
||||
strrchr-evex-base used `vpcompress{b|d}` in the page cross logic but
|
||||
was missing the CPU_FEATURE checks for VBMI2 in the
|
||||
ifunc/ifunc-impl-list.
|
||||
|
||||
The fix is either to add those checks or change the logic to not use
|
||||
`vpcompress{b|d}`. Choosing the latter here so that the strrchr-evex
|
||||
implementation is usable on SKX.
|
||||
|
||||
New implementation is a bit slower, but this is in a cold path so its
|
||||
probably okay.
|
||||
|
||||
diff --git a/sysdeps/x86_64/multiarch/strrchr-evex-base.S b/sysdeps/x86_64/multiarch/strrchr-evex-base.S
|
||||
index 8cb38be69fef5a45..2c1db616294bdc2c 100644
|
||||
--- a/sysdeps/x86_64/multiarch/strrchr-evex-base.S
|
||||
+++ b/sysdeps/x86_64/multiarch/strrchr-evex-base.S
|
||||
@@ -35,18 +35,20 @@
|
||||
# define CHAR_SIZE 4
|
||||
# define VPCMP vpcmpd
|
||||
# define VPMIN vpminud
|
||||
-# define VPCOMPRESS vpcompressd
|
||||
# define VPTESTN vptestnmd
|
||||
# define VPTEST vptestmd
|
||||
# define VPBROADCAST vpbroadcastd
|
||||
# define VPCMPEQ vpcmpeqd
|
||||
|
||||
# else
|
||||
-# define SHIFT_REG VRDI
|
||||
+# if VEC_SIZE == 64
|
||||
+# define SHIFT_REG VRCX
|
||||
+# else
|
||||
+# define SHIFT_REG VRDI
|
||||
+# endif
|
||||
# define CHAR_SIZE 1
|
||||
# define VPCMP vpcmpb
|
||||
# define VPMIN vpminub
|
||||
-# define VPCOMPRESS vpcompressb
|
||||
# define VPTESTN vptestnmb
|
||||
# define VPTEST vptestmb
|
||||
# define VPBROADCAST vpbroadcastb
|
||||
@@ -56,6 +58,12 @@
|
||||
# define KORTEST_M KORTEST
|
||||
# endif
|
||||
|
||||
+# if VEC_SIZE == 32 || (defined USE_AS_WCSRCHR)
|
||||
+# define SHIFT_R(cnt, val) shrx cnt, val, val
|
||||
+# else
|
||||
+# define SHIFT_R(cnt, val) shr %cl, val
|
||||
+# endif
|
||||
+
|
||||
# define VMATCH VMM(0)
|
||||
# define CHAR_PER_VEC (VEC_SIZE / CHAR_SIZE)
|
||||
# define PAGE_SIZE 4096
|
||||
@@ -71,7 +79,7 @@ ENTRY_P2ALIGN(STRRCHR, 6)
|
||||
andl $(PAGE_SIZE - 1), %eax
|
||||
cmpl $(PAGE_SIZE - VEC_SIZE), %eax
|
||||
jg L(cross_page_boundary)
|
||||
-
|
||||
+L(page_cross_continue):
|
||||
VMOVU (%rdi), %VMM(1)
|
||||
/* k0 has a 1 for each zero CHAR in YMM1. */
|
||||
VPTESTN %VMM(1), %VMM(1), %k0
|
||||
@@ -79,7 +87,7 @@ ENTRY_P2ALIGN(STRRCHR, 6)
|
||||
test %VGPR(rsi), %VGPR(rsi)
|
||||
jz L(aligned_more)
|
||||
/* fallthrough: zero CHAR in first VEC. */
|
||||
-L(page_cross_return):
|
||||
+
|
||||
/* K1 has a 1 for each search CHAR match in VEC(1). */
|
||||
VPCMPEQ %VMATCH, %VMM(1), %k1
|
||||
KMOV %k1, %VGPR(rax)
|
||||
@@ -167,7 +175,6 @@ L(first_vec_x1_return):
|
||||
|
||||
.p2align 4,, 12
|
||||
L(aligned_more):
|
||||
-L(page_cross_continue):
|
||||
/* Need to keep original pointer incase VEC(1) has last match. */
|
||||
movq %rdi, %rsi
|
||||
andq $-VEC_SIZE, %rdi
|
||||
@@ -340,34 +347,54 @@ L(return_new_match_ret):
|
||||
leaq (VEC_SIZE * 2)(%rdi, %rax, CHAR_SIZE), %rax
|
||||
ret
|
||||
|
||||
- .p2align 4,, 4
|
||||
L(cross_page_boundary):
|
||||
+ /* eax contains all the page offset bits of src (rdi). `xor rdi,
|
||||
+ rax` sets pointer will all page offset bits cleared so
|
||||
+ offset of (PAGE_SIZE - VEC_SIZE) will get last aligned VEC
|
||||
+ before page cross (guaranteed to be safe to read). Doing this
|
||||
+ as opposed to `movq %rdi, %rax; andq $-VEC_SIZE, %rax` saves
|
||||
+ a bit of code size. */
|
||||
xorq %rdi, %rax
|
||||
- mov $-1, %VRDX
|
||||
- VMOVU (PAGE_SIZE - VEC_SIZE)(%rax), %VMM(6)
|
||||
- VPTESTN %VMM(6), %VMM(6), %k0
|
||||
+ VMOVU (PAGE_SIZE - VEC_SIZE)(%rax), %VMM(1)
|
||||
+ VPTESTN %VMM(1), %VMM(1), %k0
|
||||
KMOV %k0, %VRSI
|
||||
|
||||
-# ifdef USE_AS_WCSRCHR
|
||||
+ /* Shift out zero CHAR matches that are before the beginning of
|
||||
+ src (rdi). */
|
||||
+# if VEC_SIZE == 64 || (defined USE_AS_WCSRCHR)
|
||||
movl %edi, %ecx
|
||||
- and $(VEC_SIZE - 1), %ecx
|
||||
+# endif
|
||||
+# ifdef USE_AS_WCSRCHR
|
||||
+ andl $(VEC_SIZE - 1), %ecx
|
||||
shrl $2, %ecx
|
||||
# endif
|
||||
- shlx %SHIFT_REG, %VRDX, %VRDX
|
||||
+ SHIFT_R (%SHIFT_REG, %VRSI)
|
||||
+# if VEC_SIZE == 32 || (defined USE_AS_WCSRCHR)
|
||||
+ /* For strrchr-evex512 we use SHIFT_R as shr which will set zero
|
||||
+ flag. */
|
||||
+ test %VRSI, %VRSI
|
||||
+# endif
|
||||
+ jz L(page_cross_continue)
|
||||
|
||||
+ /* Found zero CHAR so need to test for search CHAR. */
|
||||
+ VPCMPEQ %VMATCH, %VMM(1), %k1
|
||||
+ KMOV %k1, %VRAX
|
||||
+ /* Shift out search CHAR matches that are before the beginning of
|
||||
+ src (rdi). */
|
||||
+ SHIFT_R (%SHIFT_REG, %VRAX)
|
||||
+ /* Check if any search CHAR match in range. */
|
||||
+ blsmsk %VRSI, %VRSI
|
||||
+ and %VRSI, %VRAX
|
||||
+ jz L(ret2)
|
||||
+ bsr %VRAX, %VRAX
|
||||
# ifdef USE_AS_WCSRCHR
|
||||
- kmovw %edx, %k1
|
||||
+ leaq (%rdi, %rax, CHAR_SIZE), %rax
|
||||
# else
|
||||
- KMOV %VRDX, %k1
|
||||
+ addq %rdi, %rax
|
||||
# endif
|
||||
-
|
||||
- VPCOMPRESS %VMM(6), %VMM(1){%k1}{z}
|
||||
- /* We could technically just jmp back after the vpcompress but
|
||||
- it doesn't save any 16-byte blocks. */
|
||||
- shrx %SHIFT_REG, %VRSI, %VRSI
|
||||
- test %VRSI, %VRSI
|
||||
- jnz L(page_cross_return)
|
||||
- jmp L(page_cross_continue)
|
||||
- /* 1-byte from cache line. */
|
||||
+L(ret2):
|
||||
+ ret
|
||||
+ /* 3 bytes from cache-line for evex. */
|
||||
+ /* 0 bytes from cache-line for evex512. */
|
||||
END(STRRCHR)
|
||||
#endif
|
||||
1276
glibc-RHEL-175520-12.patch
Normal file
1276
glibc-RHEL-175520-12.patch
Normal file
File diff suppressed because it is too large
Load Diff
154
glibc-RHEL-175520-13.patch
Normal file
154
glibc-RHEL-175520-13.patch
Normal file
@ -0,0 +1,154 @@
|
||||
commit 7da08862471dfec6fdae731c2a5f351ad485c71f
|
||||
Author: Noah Goldstein <goldstein.w.n@gmail.com>
|
||||
Date: Tue Aug 13 23:29:14 2024 +0800
|
||||
|
||||
x86: Fix bug in strchrnul-evex512 [BZ #32078]
|
||||
|
||||
Issue was we were expecting not matches with CHAR before the start of
|
||||
the string in the page cross case.
|
||||
|
||||
The check code in the page cross case:
|
||||
```
|
||||
and $0xffffffffffffffc0,%rax
|
||||
vmovdqa64 (%rax),%zmm17
|
||||
vpcmpneqb %zmm17,%zmm16,%k1
|
||||
vptestmb %zmm17,%zmm17,%k0{%k1}
|
||||
kmovq %k0,%rax
|
||||
inc %rax
|
||||
shr %cl,%rax
|
||||
je L(continue)
|
||||
```
|
||||
|
||||
expects that all characters that neither match null nor CHAR will be
|
||||
1s in `rax` prior to the `inc`. Then the `inc` will overflow all of
|
||||
the 1s where no relevant match was found.
|
||||
|
||||
This is incorrect in the page-cross case, as the
|
||||
`vmovdqa64 (%rax),%zmm17` loads from before the start of the input
|
||||
string.
|
||||
|
||||
If there are matches with CHAR before the start of the string, `rax`
|
||||
won't properly overflow.
|
||||
|
||||
The fix is quite simple. Just replace:
|
||||
|
||||
```
|
||||
inc %rax
|
||||
shr %cl,%rax
|
||||
```
|
||||
With:
|
||||
```
|
||||
sar %cl,%rax
|
||||
inc %rax
|
||||
```
|
||||
|
||||
The arithmetic shift will clear any matches prior to the start of the
|
||||
string while maintaining the signbit so the 1s can properly overflow
|
||||
to zero in the case of no matches.
|
||||
Reviewed-by: H.J. Lu <hjl.tools@gmail.com>
|
||||
|
||||
diff --git a/string/test-strchr.c b/string/test-strchr.c
|
||||
index f24ade10485853c3..e5e55cdccd156914 100644
|
||||
--- a/string/test-strchr.c
|
||||
+++ b/string/test-strchr.c
|
||||
@@ -227,6 +227,69 @@ check1 (void)
|
||||
check_result (impl, s, c, exp_result);
|
||||
}
|
||||
|
||||
+static void
|
||||
+check2 (void)
|
||||
+{
|
||||
+ CHAR *s = (CHAR *) (buf1 + getpagesize () - 4 * sizeof (CHAR));
|
||||
+ CHAR *s_begin = (CHAR *) (buf1 + getpagesize () - 64);
|
||||
+#ifndef USE_FOR_STRCHRNUL
|
||||
+ CHAR *exp_result = NULL;
|
||||
+#else
|
||||
+ CHAR *exp_result = s + 1;
|
||||
+#endif
|
||||
+ CHAR val = 0x12;
|
||||
+ for (; s_begin != s; ++s_begin)
|
||||
+ *s_begin = val;
|
||||
+
|
||||
+ s[0] = val + 1;
|
||||
+ s[1] = 0;
|
||||
+ s[2] = val + 1;
|
||||
+ s[3] = val + 1;
|
||||
+
|
||||
+ {
|
||||
+ FOR_EACH_IMPL (impl, 0)
|
||||
+ check_result (impl, s, val, exp_result);
|
||||
+ }
|
||||
+ s[3] = val;
|
||||
+ {
|
||||
+ FOR_EACH_IMPL (impl, 0)
|
||||
+ check_result (impl, s, val, exp_result);
|
||||
+ }
|
||||
+ exp_result = s;
|
||||
+ s[0] = val;
|
||||
+ {
|
||||
+ FOR_EACH_IMPL (impl, 0)
|
||||
+ check_result (impl, s, val, exp_result);
|
||||
+ }
|
||||
+
|
||||
+ s[3] = val + 1;
|
||||
+ {
|
||||
+ FOR_EACH_IMPL (impl, 0)
|
||||
+ check_result (impl, s, val, exp_result);
|
||||
+ }
|
||||
+
|
||||
+ s[0] = val + 1;
|
||||
+ s[1] = val + 1;
|
||||
+ s[2] = val + 1;
|
||||
+ s[3] = val + 1;
|
||||
+ s[4] = val;
|
||||
+ exp_result = s + 4;
|
||||
+ {
|
||||
+ FOR_EACH_IMPL (impl, 0)
|
||||
+ check_result (impl, s, val, exp_result);
|
||||
+ }
|
||||
+ s[4] = 0;
|
||||
+#ifndef USE_FOR_STRCHRNUL
|
||||
+ exp_result = NULL;
|
||||
+#else
|
||||
+ exp_result = s + 4;
|
||||
+#endif
|
||||
+ {
|
||||
+ FOR_EACH_IMPL (impl, 0)
|
||||
+ check_result (impl, s, val, exp_result);
|
||||
+ }
|
||||
+}
|
||||
+
|
||||
int
|
||||
test_main (void)
|
||||
{
|
||||
@@ -235,7 +298,7 @@ test_main (void)
|
||||
test_init ();
|
||||
|
||||
check1 ();
|
||||
-
|
||||
+ check2 ();
|
||||
printf ("%20s", "");
|
||||
FOR_EACH_IMPL (impl, 0)
|
||||
printf ("\t%s", impl->name);
|
||||
diff --git a/sysdeps/x86_64/multiarch/strchr-evex-base.S b/sysdeps/x86_64/multiarch/strchr-evex-base.S
|
||||
index 75fee8c82ade14f1..f50923ec839e0599 100644
|
||||
--- a/sysdeps/x86_64/multiarch/strchr-evex-base.S
|
||||
+++ b/sysdeps/x86_64/multiarch/strchr-evex-base.S
|
||||
@@ -124,13 +124,13 @@ L(page_cross):
|
||||
VPCMPNE %VMM(1), %VMM(0), %k1
|
||||
VPTEST %VMM(1), %VMM(1), %k0{%k1}
|
||||
KMOV %k0, %VRAX
|
||||
-# ifdef USE_AS_WCSCHR
|
||||
+ sar %cl, %VRAX
|
||||
+#ifdef USE_AS_WCSCHR
|
||||
sub $VEC_MATCH_MASK, %VRAX
|
||||
-# else
|
||||
+#else
|
||||
inc %VRAX
|
||||
-# endif
|
||||
+#endif
|
||||
/* Ignore number of character for alignment adjustment. */
|
||||
- shr %cl, %VRAX
|
||||
jz L(align_more)
|
||||
|
||||
bsf %VRAX, %VRAX
|
||||
783
glibc-RHEL-175520-14.patch
Normal file
783
glibc-RHEL-175520-14.patch
Normal file
@ -0,0 +1,783 @@
|
||||
commit 294a8927694ed866ffc40833f1b6d96cd649df0a
|
||||
Author: Matthew Sterrett <matthew.sterrett@intel.com>
|
||||
Date: Fri Aug 9 15:05:09 2024 -0700
|
||||
|
||||
x86: Unifies 'strnlen-evex' and 'strnlen-evex512' implementations.
|
||||
|
||||
This commit uses a common implementation 'strnlen-evex-base.S' for both
|
||||
'strnlen-evex' and 'strnlen-evex512'
|
||||
|
||||
This patch serves both to reduce the number of implementations, and it also does some small optimizations that benefit strnlen-evex and strnlen-evex512.
|
||||
|
||||
All tests pass on x86.
|
||||
|
||||
Benchmarks were taken on SKX.
|
||||
https://www.intel.com/content/www/us/en/products/sku/123613/intel-core-i97900x-xseries-processor-13-75m-cache-up-to-4-30-ghz/specifications.html
|
||||
|
||||
Geometric mean for strnlen-evex over all benchmarks (N=10) was (new/old) 0.881
|
||||
Geometric mean for strnlen-evex512 over all benchmarks (N=10) was (new/old) 0.953
|
||||
|
||||
Code Size Changes:
|
||||
strnlen-evex : +31 bytes
|
||||
strnlen-evex512 : +156 bytes
|
||||
Reviewed-by: Noah Goldstein <goldstein.w.n@gmail.com>
|
||||
|
||||
Conflicts:
|
||||
sysdeps/x86_64/multiarch/strnlen-evex.S
|
||||
(adapt to 2.34 branch)
|
||||
sysdeps/x86_64/multiarch/strnlen-evex512.S
|
||||
(fixup context)
|
||||
|
||||
diff --git a/sysdeps/x86_64/multiarch/strnlen-evex-base.S b/sysdeps/x86_64/multiarch/strnlen-evex-base.S
|
||||
new file mode 100644
|
||||
index 0000000000000000..1c2cfdfe067140f1
|
||||
--- /dev/null
|
||||
+++ b/sysdeps/x86_64/multiarch/strnlen-evex-base.S
|
||||
@@ -0,0 +1,462 @@
|
||||
+/* strnlen/wcsnlen optimized with 256/512-bit EVEX instructions.
|
||||
+ Copyright (C) 2022-2024 Free Software Foundation, Inc.
|
||||
+ This file is part of the GNU C Library.
|
||||
+
|
||||
+ The GNU C Library is free software; you can redistribute it and/or
|
||||
+ modify it under the terms of the GNU Lesser General Public
|
||||
+ License as published by the Free Software Foundation; either
|
||||
+ version 2.1 of the License, or (at your option) any later version.
|
||||
+
|
||||
+ The GNU C Library is distributed in the hope that it will be useful,
|
||||
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
+ Lesser General Public License for more details.
|
||||
+
|
||||
+ You should have received a copy of the GNU Lesser General Public
|
||||
+ License along with the GNU C Library; if not, see
|
||||
+ <https://www.gnu.org/licenses/>. */
|
||||
+
|
||||
+
|
||||
+#include <isa-level.h>
|
||||
+
|
||||
+#if ISA_SHOULD_BUILD (4)
|
||||
+
|
||||
+# include <sysdep.h>
|
||||
+
|
||||
+#ifdef USE_AS_WCSLEN
|
||||
+# define VPCMPEQ vpcmpeqd
|
||||
+# define VPTESTN vptestnmd
|
||||
+# define VPMINU vpminud
|
||||
+# define CHAR_SIZE 4
|
||||
+#else
|
||||
+# define VPCMPEQ vpcmpeqb
|
||||
+# define VPTESTN vptestnmb
|
||||
+# define VPMINU vpminub
|
||||
+# define CHAR_SIZE 1
|
||||
+#endif
|
||||
+
|
||||
+#define XZERO VMM_128(0)
|
||||
+#define VZERO VMM(0)
|
||||
+#define PAGE_SIZE 4096
|
||||
+#define CHAR_PER_VEC (VEC_SIZE / CHAR_SIZE)
|
||||
+
|
||||
+#if CHAR_PER_VEC == 32
|
||||
+# define SUB_SHORT(imm, reg) subb $(imm), %VGPR_SZ(reg, 8)
|
||||
+#else
|
||||
+# define SUB_SHORT(imm, reg) subl $(imm), %VGPR_SZ(reg, 32)
|
||||
+#endif
|
||||
+
|
||||
+#ifdef USE_AS_WCSLEN
|
||||
+/* For wide-character, we care more about limitting code size
|
||||
+ than optimally aligning targets, so just cap nop padding
|
||||
+ reasonably low. */
|
||||
+# define P2ALIGN(...) .p2align 4,, 6
|
||||
+# define P2ALIGN_CLAMPED(...) P2ALIGN(__VA_ARGS__)
|
||||
+#else
|
||||
+# define P2ALIGN(x) .p2align x
|
||||
+# define P2ALIGN_CLAMPED(x, y) .p2align x,, y
|
||||
+#endif
|
||||
+
|
||||
+ .section SECTION(.text), "ax", @progbits
|
||||
+ /* Aligning entry point to 64 byte, provides better performance for
|
||||
+ one vector length string. */
|
||||
+ENTRY_P2ALIGN(STRNLEN, 6)
|
||||
+ /* rdi is pointer to array, rsi is the upper limit. */
|
||||
+
|
||||
+ /* Check zero length. */
|
||||
+ test %RSI_LP, %RSI_LP
|
||||
+ jz L(zero)
|
||||
+
|
||||
+#ifdef __ILP32__
|
||||
+ /* Clear the upper 32 bits. */
|
||||
+ movl %esi, %esi
|
||||
+#endif
|
||||
+
|
||||
+ vpxorq %XZERO, %XZERO, %XZERO
|
||||
+
|
||||
+ /* Check that we won't cross a page boundary with our first load. */
|
||||
+ movl %edi, %eax
|
||||
+ shll $20, %eax
|
||||
+ cmpl $((PAGE_SIZE - VEC_SIZE) << 20), %eax
|
||||
+ ja L(crosses_page_boundary)
|
||||
+
|
||||
+ /* Check the first VEC_SIZE bytes. Each bit in K0 represents a
|
||||
+ null byte. */
|
||||
+ VPCMPEQ (%rdi), %VZERO, %k0
|
||||
+ KMOV %k0, %VRCX
|
||||
+
|
||||
+ /* If src (rcx) is zero, bsf does not change the result. NB:
|
||||
+ Must use 64-bit bsf here so that upper bits of len are not
|
||||
+ cleared. */
|
||||
+ movq %rsi, %rax
|
||||
+ bsfq %rcx, %rax
|
||||
+
|
||||
+ /* If rax > CHAR_PER_VEC then rcx must have been zero (no null
|
||||
+ CHAR) and rsi must be > CHAR_PER_VEC. */
|
||||
+ cmpq $CHAR_PER_VEC, %rax
|
||||
+ ja L(more_1x_vec)
|
||||
+
|
||||
+ /* Check if first match in bounds. */
|
||||
+ cmpq %rax, %rsi
|
||||
+ cmovb %esi, %eax
|
||||
+ ret
|
||||
+
|
||||
+#if VEC_SIZE == 32
|
||||
+ P2ALIGN_CLAMPED(4, 2)
|
||||
+L(zero):
|
||||
+L(max_0):
|
||||
+ movl %esi, %eax
|
||||
+ ret
|
||||
+#endif
|
||||
+
|
||||
+ P2ALIGN_CLAMPED(4, 10)
|
||||
+L(more_1x_vec):
|
||||
+L(cross_page_continue):
|
||||
+ /* After this calculation, rax stores the number of elements
|
||||
+ left to be processed The complexity comes from the fact some
|
||||
+ elements get read twice due to alignment and we need to be
|
||||
+ sure we don't count them twice (else, it would just be rsi -
|
||||
+ CHAR_PER_VEC). */
|
||||
+
|
||||
+#ifdef USE_AS_WCSLEN
|
||||
+ /* Need to compute directly for wcslen as CHAR_SIZE * rsi can
|
||||
+ overflow. */
|
||||
+ movq %rdi, %rax
|
||||
+ andq $(VEC_SIZE * -1), %rdi
|
||||
+ subq %rdi, %rax
|
||||
+ sarq $2, %rax
|
||||
+ leaq -(CHAR_PER_VEC * 1)(%rax, %rsi), %rax
|
||||
+#else
|
||||
+ /* Calculate ptr + N - VEC_SIZE, then mask off the low bits,
|
||||
+ then subtract ptr to get the new aligned limit value. */
|
||||
+ leaq (VEC_SIZE * -1)(%rsi, %rdi), %rax
|
||||
+ andq $(VEC_SIZE * -1), %rdi
|
||||
+ subq %rdi, %rax
|
||||
+#endif
|
||||
+
|
||||
+ VPCMPEQ VEC_SIZE(%rdi), %VZERO, %k0
|
||||
+
|
||||
+ /* Checking here is faster for 256-bit but not 512-bit */
|
||||
+#if VEC_SIZE == 0
|
||||
+ KMOV %k0, %VRDX
|
||||
+ test %VRDX, %VRDX
|
||||
+ jnz L(last_vec_check)
|
||||
+#endif
|
||||
+
|
||||
+ cmpq $(CHAR_PER_VEC * 2), %rax
|
||||
+ ja L(more_2x_vec)
|
||||
+
|
||||
+L(last_2x_vec_or_less):
|
||||
+
|
||||
+ /* Checking here is faster for 512-bit but not 256-bit */
|
||||
+#if VEC_SIZE != 0
|
||||
+ KMOV %k0, %VRDX
|
||||
+ test %VRDX, %VRDX
|
||||
+ jnz L(last_vec_check)
|
||||
+#endif
|
||||
+
|
||||
+ /* Check for the end of data. */
|
||||
+ SUB_SHORT (CHAR_PER_VEC, rax)
|
||||
+ jbe L(max_0)
|
||||
+
|
||||
+ /* Check the final remaining vector. */
|
||||
+ VPCMPEQ (VEC_SIZE * 2)(%rdi), %VZERO, %k0
|
||||
+ KMOV %k0, %VRDX
|
||||
+ test %VRDX, %VRDX
|
||||
+#if VEC_SIZE == 32
|
||||
+ jz L(max_0)
|
||||
+#else
|
||||
+ jnz L(last_vec_check)
|
||||
+ P2ALIGN_CLAMPED(4, 2)
|
||||
+L(zero):
|
||||
+L(max_0):
|
||||
+ movl %esi, %eax
|
||||
+ ret
|
||||
+
|
||||
+#endif
|
||||
+ P2ALIGN_CLAMPED(4, 4)
|
||||
+L(last_vec_check):
|
||||
+ bsf %VRDX, %VRDX
|
||||
+ sub %eax, %edx
|
||||
+ lea (%rsi, %rdx), %eax
|
||||
+ cmovae %esi, %eax
|
||||
+ ret
|
||||
+
|
||||
+
|
||||
+#if VEC_SIZE == 32
|
||||
+ P2ALIGN_CLAMPED(4, 8)
|
||||
+#endif
|
||||
+L(last_4x_vec_or_less):
|
||||
+ addl $(CHAR_PER_VEC * -4), %eax
|
||||
+ VPCMPEQ (VEC_SIZE * 5)(%rdi), %VZERO, %k0
|
||||
+
|
||||
+#if VEC_SIZE == 64
|
||||
+ KMOV %k0, %VRDX
|
||||
+ test %VRDX, %VRDX
|
||||
+ jnz L(last_vec_check)
|
||||
+#endif
|
||||
+
|
||||
+ subq $(VEC_SIZE * -4), %rdi
|
||||
+ cmpl $(CHAR_PER_VEC * 2), %eax
|
||||
+ jbe L(last_2x_vec_or_less)
|
||||
+
|
||||
+ P2ALIGN_CLAMPED(4, 6)
|
||||
+L(more_2x_vec):
|
||||
+ /* Remaining length >= 2 * CHAR_PER_VEC so do VEC0/VEC1 without
|
||||
+ rechecking bounds. */
|
||||
+
|
||||
+ /* Already checked in 256-bit case */
|
||||
+#if VEC_SIZE != 0
|
||||
+ KMOV %k0, %VRDX
|
||||
+
|
||||
+ test %VRDX, %VRDX
|
||||
+ jnz L(first_vec_x1)
|
||||
+#endif
|
||||
+
|
||||
+ VPCMPEQ (VEC_SIZE * 2)(%rdi), %VZERO, %k0
|
||||
+ KMOV %k0, %VRDX
|
||||
+
|
||||
+ test %VRDX, %VRDX
|
||||
+ jnz L(first_vec_x2)
|
||||
+
|
||||
+ cmpq $(CHAR_PER_VEC * 4), %rax
|
||||
+ ja L(more_4x_vec)
|
||||
+
|
||||
+
|
||||
+ VPCMPEQ (VEC_SIZE * 3)(%rdi), %VZERO, %k0
|
||||
+ KMOV %k0, %VRDX
|
||||
+ addl $(CHAR_PER_VEC * -2), %eax
|
||||
+ test %VRDX, %VRDX
|
||||
+ jnz L(last_vec_check)
|
||||
+
|
||||
+ subb $(CHAR_PER_VEC), %al
|
||||
+ jbe L(max_1)
|
||||
+
|
||||
+ VPCMPEQ (VEC_SIZE * 4)(%rdi), %VZERO, %k0
|
||||
+ KMOV %k0, %VRDX
|
||||
+
|
||||
+ test %VRDX, %VRDX
|
||||
+ jnz L(last_vec_check)
|
||||
+L(max_1):
|
||||
+ movl %esi, %eax
|
||||
+ ret
|
||||
+
|
||||
+
|
||||
+ P2ALIGN_CLAMPED(4, 14)
|
||||
+L(first_vec_x2):
|
||||
+#if VEC_SIZE == 64
|
||||
+ /* If VEC_SIZE == 64 we can fit logic for full return label in
|
||||
+ spare bytes before next cache line. */
|
||||
+ bsf %VRDX, %VRDX
|
||||
+ sub %eax, %esi
|
||||
+ leal (CHAR_PER_VEC * 1)(%rsi, %rdx), %eax
|
||||
+ ret
|
||||
+ P2ALIGN_CLAMPED(4, 6)
|
||||
+#else
|
||||
+ addl $CHAR_PER_VEC, %esi
|
||||
+#endif
|
||||
+L(first_vec_x1):
|
||||
+ bsf %VRDX, %VRDX
|
||||
+ sub %eax, %esi
|
||||
+ leal (CHAR_PER_VEC * 0)(%rsi, %rdx), %eax
|
||||
+ ret
|
||||
+
|
||||
+#if VEC_SIZE == 64
|
||||
+ P2ALIGN_CLAMPED(4, 6)
|
||||
+L(first_vec_x4):
|
||||
+# if VEC_SIZE == 64
|
||||
+ /* If VEC_SIZE == 64 we can fit logic for full return label in
|
||||
+ spare bytes before next cache line. */
|
||||
+ bsf %VRDX, %VRDX
|
||||
+ sub %eax, %esi
|
||||
+ leal (CHAR_PER_VEC * 3)(%rsi, %rdx), %eax
|
||||
+ ret
|
||||
+ P2ALIGN_CLAMPED(4, 6)
|
||||
+# else
|
||||
+ addl $CHAR_PER_VEC, %esi
|
||||
+# endif
|
||||
+L(first_vec_x3):
|
||||
+ bsf %VRDX, %VRDX
|
||||
+ sub %eax, %esi
|
||||
+ leal (CHAR_PER_VEC * 2)(%rsi, %rdx), %eax
|
||||
+ ret
|
||||
+#endif
|
||||
+
|
||||
+ P2ALIGN_CLAMPED(6, 20)
|
||||
+L(more_4x_vec):
|
||||
+ VPCMPEQ (VEC_SIZE * 3)(%rdi), %VZERO, %k0
|
||||
+ KMOV %k0, %VRDX
|
||||
+ test %VRDX, %VRDX
|
||||
+ jnz L(first_vec_x3)
|
||||
+
|
||||
+ VPCMPEQ (VEC_SIZE * 4)(%rdi), %VZERO, %k0
|
||||
+ KMOV %k0, %VRDX
|
||||
+ test %VRDX, %VRDX
|
||||
+ jnz L(first_vec_x4)
|
||||
+
|
||||
+ /* Check if at last VEC_SIZE * 4 length before aligning for the
|
||||
+ loop. */
|
||||
+ cmpq $(CHAR_PER_VEC * 8), %rax
|
||||
+ jbe L(last_4x_vec_or_less)
|
||||
+
|
||||
+
|
||||
+ /* Compute number of words checked after aligning. */
|
||||
+#ifdef USE_AS_WCSLEN
|
||||
+ /* Need to compute directly for wcslen as CHAR_SIZE * rsi can
|
||||
+ overflow. */
|
||||
+ leaq (VEC_SIZE * -3)(%rdi), %rdx
|
||||
+#else
|
||||
+ leaq (VEC_SIZE * -3)(%rdi, %rax), %rax
|
||||
+#endif
|
||||
+
|
||||
+ subq $(VEC_SIZE * -1), %rdi
|
||||
+
|
||||
+ /* Align data to VEC_SIZE * 4. */
|
||||
+#if VEC_SIZE == 64
|
||||
+ /* Saves code size. No evex512 processor has partial register
|
||||
+ stalls. If that change this can be replaced with `andq
|
||||
+ $-(VEC_SIZE * 4), %rdi`. */
|
||||
+ xorb %dil, %dil
|
||||
+#else
|
||||
+ andq $-(VEC_SIZE * 4), %rdi
|
||||
+#endif
|
||||
+
|
||||
+#ifdef USE_AS_WCSLEN
|
||||
+ subq %rdi, %rdx
|
||||
+ sarq $2, %rdx
|
||||
+ addq %rdx, %rax
|
||||
+#else
|
||||
+ subq %rdi, %rax
|
||||
+#endif
|
||||
+
|
||||
+ // mov %rdi, %rdx
|
||||
+
|
||||
+ P2ALIGN(6)
|
||||
+L(loop):
|
||||
+ /* VPMINU and VPCMP combination provide better performance as
|
||||
+ compared to alternative combinations. */
|
||||
+ VMOVA (VEC_SIZE * 4)(%rdi), %VMM(1)
|
||||
+ VPMINU (VEC_SIZE * 5)(%rdi), %VMM(1), %VMM(2)
|
||||
+ VMOVA (VEC_SIZE * 6)(%rdi), %VMM(3)
|
||||
+ VPMINU (VEC_SIZE * 7)(%rdi), %VMM(3), %VMM(4)
|
||||
+
|
||||
+ VPTESTN %VMM(2), %VMM(2), %k0
|
||||
+ VPTESTN %VMM(4), %VMM(4), %k1
|
||||
+
|
||||
+ subq $-(VEC_SIZE * 4), %rdi
|
||||
+ KORTEST %k0, %k1
|
||||
+
|
||||
+ jnz L(loopend)
|
||||
+ subq $(CHAR_PER_VEC * 4), %rax
|
||||
+ ja L(loop)
|
||||
+ mov %rsi, %rax
|
||||
+ ret
|
||||
+
|
||||
+
|
||||
+#if VEC_SIZE == 32
|
||||
+ P2ALIGN_CLAMPED(4, 6)
|
||||
+L(first_vec_x4):
|
||||
+# if VEC_SIZE == 64
|
||||
+ /* If VEC_SIZE == 64 we can fit logic for full return label in
|
||||
+ spare bytes before next cache line. */
|
||||
+ bsf %VRDX, %VRDX
|
||||
+ sub %eax, %esi
|
||||
+ leal (CHAR_PER_VEC * 3)(%rsi, %rdx), %eax
|
||||
+ ret
|
||||
+ P2ALIGN_CLAMPED(4, 6)
|
||||
+# else
|
||||
+ addl $CHAR_PER_VEC, %esi
|
||||
+# endif
|
||||
+L(first_vec_x3):
|
||||
+ bsf %VRDX, %VRDX
|
||||
+ sub %eax, %esi
|
||||
+ leal (CHAR_PER_VEC * 2)(%rsi, %rdx), %eax
|
||||
+ ret
|
||||
+#endif
|
||||
+
|
||||
+
|
||||
+ P2ALIGN_CLAMPED(4, 11)
|
||||
+L(loopend):
|
||||
+ /* We found a null terminator in one of the 4 vectors. */
|
||||
+
|
||||
+ /* Check the first vector. */
|
||||
+ movq %rax, %r8
|
||||
+ VPTESTN %VMM(1), %VMM(1), %k2
|
||||
+ KMOV %k2, %VRCX
|
||||
+ bsf %rcx, %r8
|
||||
+
|
||||
+ cmpq $(CHAR_PER_VEC), %r8
|
||||
+ jbe L(end_vec)
|
||||
+
|
||||
+ /* Check the second vector. */
|
||||
+ subq $(CHAR_PER_VEC), %rax
|
||||
+ movq %rax, %r8
|
||||
+ KMOV %k0, %VRCX
|
||||
+ bsf %rcx, %r8
|
||||
+
|
||||
+ cmpq $(CHAR_PER_VEC), %r8
|
||||
+ jbe L(end_vec)
|
||||
+
|
||||
+ /* Check the third vector. */
|
||||
+ subq $(CHAR_PER_VEC), %rax
|
||||
+ movq %rax, %r8
|
||||
+ VPTESTN %VMM(3), %VMM(3), %k2
|
||||
+ KMOV %k2, %VRCX
|
||||
+ bsf %rcx, %r8
|
||||
+
|
||||
+ cmpq $(CHAR_PER_VEC), %r8
|
||||
+ jbe L(end_vec)
|
||||
+
|
||||
+ /* It is in the fourth vector. */
|
||||
+ subq $(CHAR_PER_VEC), %rax
|
||||
+ movq %rax, %r8
|
||||
+ KMOV %k1, %VRCX
|
||||
+ bsf %rcx, %r8
|
||||
+
|
||||
+ P2ALIGN_CLAMPED(4, 3)
|
||||
+L(end_vec):
|
||||
+ /* Get the number that has been processed. */
|
||||
+ movq %rsi, %rcx
|
||||
+ subq %rax, %rcx
|
||||
+
|
||||
+ /* Add that to the offset we found the null terminator at. */
|
||||
+ leaq (%r8, %rcx), %rax
|
||||
+
|
||||
+ /* Take the min of that and the limit. */
|
||||
+ cmpq %rsi, %rax
|
||||
+ cmovnb %rsi, %rax
|
||||
+ ret
|
||||
+
|
||||
+ P2ALIGN_CLAMPED(4, 11)
|
||||
+L(crosses_page_boundary):
|
||||
+ /* Align data backwards to VEC_SIZE. */
|
||||
+ shrl $20, %eax
|
||||
+ movq %rdi, %rcx
|
||||
+ andq $-VEC_SIZE, %rcx
|
||||
+ VPCMPEQ (%rcx), %VZERO, %k0
|
||||
+
|
||||
+ KMOV %k0, %VRCX
|
||||
+#ifdef USE_AS_WCSLEN
|
||||
+ shrl $2, %eax
|
||||
+ andl $(CHAR_PER_VEC - 1), %eax
|
||||
+#endif
|
||||
+ /* By this point rax contains number of bytes we need to skip. */
|
||||
+ shrx %VRAX, %VRCX, %VRCX
|
||||
+
|
||||
+ /* Calculates CHAR_PER_VEC - eax and stores in eax. */
|
||||
+ negl %eax
|
||||
+ andl $(CHAR_PER_VEC - 1), %eax
|
||||
+
|
||||
+ movq %rsi, %rdx
|
||||
+ bsf %VRCX, %VRDX
|
||||
+ cmpq %rax, %rdx
|
||||
+ ja L(cross_page_continue)
|
||||
+
|
||||
+ /* The vector had a null terminator or we are at the limit. */
|
||||
+ movl %edx, %eax
|
||||
+ cmpq %rdx, %rsi
|
||||
+ cmovb %esi, %eax
|
||||
+ ret
|
||||
+
|
||||
+END(STRNLEN)
|
||||
+#endif
|
||||
diff --git a/sysdeps/x86_64/multiarch/strnlen-evex.S b/sysdeps/x86_64/multiarch/strnlen-evex.S
|
||||
index 722022f303cc0ab5..c41288906cdd2bc4 100644
|
||||
--- a/sysdeps/x86_64/multiarch/strnlen-evex.S
|
||||
+++ b/sysdeps/x86_64/multiarch/strnlen-evex.S
|
||||
@@ -1,4 +1,7 @@
|
||||
-#define STRLEN __strnlen_evex
|
||||
-#define USE_AS_STRNLEN 1
|
||||
+#ifndef STRNLEN
|
||||
+#define STRNLEN __strnlen_evex
|
||||
+#endif
|
||||
|
||||
-#include "strlen-evex.S"
|
||||
+#include "x86-evex256-vecs.h"
|
||||
+#include "reg-macros.h"
|
||||
+#include "strnlen-evex-base.S"
|
||||
diff --git a/sysdeps/x86_64/multiarch/strnlen-evex512.S b/sysdeps/x86_64/multiarch/strnlen-evex512.S
|
||||
index ebf22c259f9b6362..07f0bb375cb671d7 100644
|
||||
--- a/sysdeps/x86_64/multiarch/strnlen-evex512.S
|
||||
+++ b/sysdeps/x86_64/multiarch/strnlen-evex512.S
|
||||
@@ -1,264 +1,7 @@
|
||||
-/* Placeholder function, not used by any processor at the moment.
|
||||
- Copyright (C) 2022-2023 Free Software Foundation, Inc.
|
||||
- This file is part of the GNU C Library.
|
||||
-
|
||||
- The GNU C Library is free software; you can redistribute it and/or
|
||||
- modify it under the terms of the GNU Lesser General Public
|
||||
- License as published by the Free Software Foundation; either
|
||||
- version 2.1 of the License, or (at your option) any later version.
|
||||
-
|
||||
- The GNU C Library is distributed in the hope that it will be useful,
|
||||
- but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
- Lesser General Public License for more details.
|
||||
-
|
||||
- You should have received a copy of the GNU Lesser General Public
|
||||
- License along with the GNU C Library; if not, see
|
||||
- <https://www.gnu.org/licenses/>. */
|
||||
-
|
||||
#ifndef STRNLEN
|
||||
#define STRNLEN __strnlen_evex512
|
||||
#endif
|
||||
|
||||
#include "x86-evex512-vecs.h"
|
||||
#include "reg-macros.h"
|
||||
-
|
||||
-#include <isa-level.h>
|
||||
-
|
||||
-#if ISA_SHOULD_BUILD (4)
|
||||
-
|
||||
-# include <sysdep.h>
|
||||
-
|
||||
-# ifdef USE_AS_WCSLEN
|
||||
-# define VPCMPEQ vpcmpeqd
|
||||
-# define VPTESTN vptestnmd
|
||||
-# define VPMINU vpminud
|
||||
-# define CHAR_SIZE 4
|
||||
-# else
|
||||
-# define VPCMPEQ vpcmpeqb
|
||||
-# define VPTESTN vptestnmb
|
||||
-# define VPMINU vpminub
|
||||
-# define CHAR_SIZE 1
|
||||
-# endif
|
||||
-
|
||||
-# define PAGE_SIZE 4096
|
||||
-# define CHAR_PER_VEC (VEC_SIZE / CHAR_SIZE)
|
||||
-
|
||||
- .section SECTION(.text),"ax",@progbits
|
||||
-/* Aligning entry point to 64 byte, provides better performance for
|
||||
- one vector length string. */
|
||||
-ENTRY_P2ALIGN (STRNLEN, 6)
|
||||
- /* Check zero length. */
|
||||
- test %RSI_LP, %RSI_LP
|
||||
- jz L(ret_max)
|
||||
-# ifdef __ILP32__
|
||||
- /* Clear the upper 32 bits. */
|
||||
- movl %esi, %esi
|
||||
-# endif
|
||||
-
|
||||
- movl %edi, %eax
|
||||
- vpxorq %VMM_128(0), %VMM_128(0), %VMM_128(0)
|
||||
- sall $20, %eax
|
||||
- cmpl $((PAGE_SIZE - VEC_SIZE) << 20), %eax
|
||||
- ja L(page_cross)
|
||||
-
|
||||
- /* Compare [w]char for null, mask bit will be set for match. */
|
||||
- VPCMPEQ (%rdi), %VMM(0), %k0
|
||||
- KMOV %k0, %VRCX
|
||||
- /* Store max length in rax. */
|
||||
- mov %rsi, %rax
|
||||
- /* If rcx is 0, rax will have max length. We can not use VRCX
|
||||
- and VRAX here for evex256 because, upper 32 bits may be
|
||||
- undefined for ecx and eax. */
|
||||
- bsfq %rcx, %rax
|
||||
- cmp $CHAR_PER_VEC, %rax
|
||||
- ja L(align_more)
|
||||
- cmpq %rax, %rsi
|
||||
- cmovb %esi, %eax
|
||||
- ret
|
||||
-
|
||||
- /* At this point vector max length reached. */
|
||||
- .p2align 4,,3
|
||||
-L(ret_max):
|
||||
- movq %rsi, %rax
|
||||
- ret
|
||||
-
|
||||
-L(align_more):
|
||||
- mov %rdi, %rax
|
||||
- /* Align rax to VEC_SIZE. */
|
||||
- andq $-VEC_SIZE, %rax
|
||||
- movq %rdi, %rdx
|
||||
- subq %rax, %rdx
|
||||
-# ifdef USE_AS_WCSLEN
|
||||
- shr $2, %VRDX
|
||||
-# endif
|
||||
- /* At this point rdx contains [w]chars already compared. */
|
||||
- leaq -CHAR_PER_VEC(%rsi, %rdx), %rdx
|
||||
- /* At this point rdx contains number of w[char] needs to go.
|
||||
- Now onwards rdx will keep decrementing with each compare. */
|
||||
-
|
||||
- /* Loop unroll 4 times for 4 vector loop. */
|
||||
- VPCMPEQ VEC_SIZE(%rax), %VMM(0), %k0
|
||||
- subq $-VEC_SIZE, %rax
|
||||
- KMOV %k0, %VRCX
|
||||
- test %VRCX, %VRCX
|
||||
- jnz L(ret_vec_x1)
|
||||
-
|
||||
- subq $CHAR_PER_VEC, %rdx
|
||||
- jbe L(ret_max)
|
||||
-
|
||||
- VPCMPEQ VEC_SIZE(%rax), %VMM(0), %k0
|
||||
- KMOV %k0, %VRCX
|
||||
- test %VRCX, %VRCX
|
||||
- jnz L(ret_vec_x2)
|
||||
-
|
||||
- subq $CHAR_PER_VEC, %rdx
|
||||
- jbe L(ret_max)
|
||||
-
|
||||
- VPCMPEQ (VEC_SIZE * 2)(%rax), %VMM(0), %k0
|
||||
- KMOV %k0, %VRCX
|
||||
- test %VRCX, %VRCX
|
||||
- jnz L(ret_vec_x3)
|
||||
-
|
||||
- subq $CHAR_PER_VEC, %rdx
|
||||
- jbe L(ret_max)
|
||||
-
|
||||
- VPCMPEQ (VEC_SIZE * 3)(%rax), %VMM(0), %k0
|
||||
- KMOV %k0, %VRCX
|
||||
- test %VRCX, %VRCX
|
||||
- jnz L(ret_vec_x4)
|
||||
-
|
||||
- subq $CHAR_PER_VEC, %rdx
|
||||
- jbe L(ret_max)
|
||||
- /* Save pointer before 4 x VEC_SIZE alignment. */
|
||||
- movq %rax, %rcx
|
||||
-
|
||||
- /* Align address to VEC_SIZE * 4 for loop. */
|
||||
- andq $-(VEC_SIZE * 4), %rax
|
||||
-
|
||||
- subq %rax, %rcx
|
||||
-# ifdef USE_AS_WCSLEN
|
||||
- shr $2, %VRCX
|
||||
-# endif
|
||||
- /* rcx contains number of [w]char will be recompared due to
|
||||
- alignment fixes. rdx must be incremented by rcx to offset
|
||||
- alignment adjustment. */
|
||||
- addq %rcx, %rdx
|
||||
- /* Need jump as we don't want to add/subtract rdx for first
|
||||
- iteration of 4 x VEC_SIZE aligned loop. */
|
||||
-
|
||||
- .p2align 4,,11
|
||||
-L(loop):
|
||||
- /* VPMINU and VPCMP combination provide better performance as
|
||||
- compared to alternative combinations. */
|
||||
- VMOVA (VEC_SIZE * 4)(%rax), %VMM(1)
|
||||
- VPMINU (VEC_SIZE * 5)(%rax), %VMM(1), %VMM(2)
|
||||
- VMOVA (VEC_SIZE * 6)(%rax), %VMM(3)
|
||||
- VPMINU (VEC_SIZE * 7)(%rax), %VMM(3), %VMM(4)
|
||||
-
|
||||
- VPTESTN %VMM(2), %VMM(2), %k0
|
||||
- VPTESTN %VMM(4), %VMM(4), %k1
|
||||
-
|
||||
- subq $-(VEC_SIZE * 4), %rax
|
||||
- KORTEST %k0, %k1
|
||||
-
|
||||
- jnz L(loopend)
|
||||
- subq $(CHAR_PER_VEC * 4), %rdx
|
||||
- ja L(loop)
|
||||
- mov %rsi, %rax
|
||||
- ret
|
||||
-
|
||||
-L(loopend):
|
||||
-
|
||||
- VPTESTN %VMM(1), %VMM(1), %k2
|
||||
- KMOV %k2, %VRCX
|
||||
- test %VRCX, %VRCX
|
||||
- jnz L(ret_vec_x1)
|
||||
-
|
||||
- KMOV %k0, %VRCX
|
||||
- /* At this point, if k0 is non zero, null char must be in the
|
||||
- second vector. */
|
||||
- test %VRCX, %VRCX
|
||||
- jnz L(ret_vec_x2)
|
||||
-
|
||||
- VPTESTN %VMM(3), %VMM(3), %k3
|
||||
- KMOV %k3, %VRCX
|
||||
- test %VRCX, %VRCX
|
||||
- jnz L(ret_vec_x3)
|
||||
- /* At this point null [w]char must be in the fourth vector so no
|
||||
- need to check. */
|
||||
- KMOV %k1, %VRCX
|
||||
-
|
||||
- /* Fourth, third, second vector terminating are pretty much
|
||||
- same, implemented this way to avoid branching and reuse code
|
||||
- from pre loop exit condition. */
|
||||
-L(ret_vec_x4):
|
||||
- bsf %VRCX, %VRCX
|
||||
- subq %rdi, %rax
|
||||
-# ifdef USE_AS_WCSLEN
|
||||
- subq $-(VEC_SIZE * 3), %rax
|
||||
- shrq $2, %rax
|
||||
- addq %rcx, %rax
|
||||
-# else
|
||||
- leaq (VEC_SIZE * 3)(%rcx, %rax), %rax
|
||||
-# endif
|
||||
-
|
||||
- cmpq %rsi, %rax
|
||||
- cmovnb %rsi, %rax
|
||||
- ret
|
||||
-
|
||||
-L(ret_vec_x3):
|
||||
- bsf %VRCX, %VRCX
|
||||
- subq %rdi, %rax
|
||||
-# ifdef USE_AS_WCSLEN
|
||||
- subq $-(VEC_SIZE * 2), %rax
|
||||
- shrq $2, %rax
|
||||
- addq %rcx, %rax
|
||||
-# else
|
||||
- leaq (VEC_SIZE * 2)(%rcx, %rax), %rax
|
||||
-# endif
|
||||
- cmpq %rsi, %rax
|
||||
- cmovnb %rsi, %rax
|
||||
- ret
|
||||
-
|
||||
-L(ret_vec_x2):
|
||||
- subq $-VEC_SIZE, %rax
|
||||
-L(ret_vec_x1):
|
||||
- bsf %VRCX, %VRCX
|
||||
- subq %rdi, %rax
|
||||
-# ifdef USE_AS_WCSLEN
|
||||
- shrq $2, %rax
|
||||
-# endif
|
||||
- addq %rcx, %rax
|
||||
- cmpq %rsi, %rax
|
||||
- cmovnb %rsi, %rax
|
||||
- ret
|
||||
-
|
||||
-L(page_cross):
|
||||
- mov %rdi, %rax
|
||||
- movl %edi, %ecx
|
||||
- andl $(VEC_SIZE - 1), %ecx
|
||||
-# ifdef USE_AS_WCSLEN
|
||||
- sarl $2, %ecx
|
||||
-# endif
|
||||
- /* ecx contains number of w[char] to be skipped as a result
|
||||
- of address alignment. */
|
||||
- andq $-VEC_SIZE, %rax
|
||||
- VPCMPEQ (%rax), %VMM(0), %k0
|
||||
- KMOV %k0, %VRDX
|
||||
- /* Ignore number of character for alignment adjustment. */
|
||||
- shr %cl, %VRDX
|
||||
- jnz L(page_cross_end)
|
||||
- movl $CHAR_PER_VEC, %eax
|
||||
- sub %ecx, %eax
|
||||
- cmp %rax, %rsi
|
||||
- ja L(align_more)
|
||||
-
|
||||
-L(page_cross_end):
|
||||
- bsf %VRDX, %VRAX
|
||||
- cmpq %rsi, %rax
|
||||
- cmovnb %esi, %eax
|
||||
- ret
|
||||
-
|
||||
-END (STRNLEN)
|
||||
-#endif
|
||||
+#include "strnlen-evex-base.S"
|
||||
59
glibc-RHEL-175520-15.patch
Normal file
59
glibc-RHEL-175520-15.patch
Normal file
@ -0,0 +1,59 @@
|
||||
Partial backport of b79f8ff26aa6.
|
||||
Upstream commit was skipped during backporting because most of the files
|
||||
impacted were re-written later. This missing backport left
|
||||
wcsnlen-evex.S pointing to strlen-evex.S. After strlen-evex.S dropped
|
||||
bounds checking (Patch 12) and strnlen-evex-base.S was introduced (Patch
|
||||
14), wcsnlen-evex.S must be updated to point to the new strnlen
|
||||
implementation.
|
||||
|
||||
commit b79f8ff26aa6151d2d2167afcddcd1ec46cfbc81
|
||||
Author: Noah Goldstein <goldstein.w.n@gmail.com>
|
||||
Date: Tue Oct 18 17:44:05 2022 -0700
|
||||
|
||||
x86: Optimize strnlen-evex.S and implement with VMM headers
|
||||
|
||||
Optimizations are:
|
||||
1. Use the fact that bsf(0) leaves the destination unchanged to save a
|
||||
branch in short string case.
|
||||
2. Restructure code so that small strings are given the hot path.
|
||||
- This is a net-zero on the benchmark suite but in general makes
|
||||
sense as smaller sizes are far more common.
|
||||
3. Use more code-size efficient instructions.
|
||||
- tzcnt ... -> bsf ...
|
||||
- vpcmpb $0 ... -> vpcmpeq ...
|
||||
4. Align labels less aggressively, especially if it doesn't save fetch
|
||||
blocks / causes the basic-block to span extra cache-lines.
|
||||
|
||||
The optimizations (especially for point 2) make the strnlen and
|
||||
strlen code essentially incompatible so split strnlen-evex
|
||||
to a new file.
|
||||
|
||||
Code Size Changes:
|
||||
strlen-evex.S : -23 bytes
|
||||
strnlen-evex.S : -167 bytes
|
||||
|
||||
Net perf changes:
|
||||
|
||||
Reported as geometric mean of all improvements / regressions from N=10
|
||||
runs of the benchtests. Value as New Time / Old Time so < 1.0 is
|
||||
improvement and 1.0 is regression.
|
||||
|
||||
strlen-evex.S : 0.992 (No real change)
|
||||
strnlen-evex.S : 0.947
|
||||
|
||||
Full results attached in email.
|
||||
|
||||
Full check passes on x86-64.
|
||||
|
||||
diff --git a/sysdeps/x86_64/multiarch/wcsnlen-evex.S b/sysdeps/x86_64/multiarch/wcsnlen-evex.S
|
||||
index 24773bb4e2327ca4..920d860047e5a96c 100644
|
||||
--- a/sysdeps/x86_64/multiarch/wcsnlen-evex.S
|
||||
+++ b/sysdeps/x86_64/multiarch/wcsnlen-evex.S
|
||||
@@ -1,5 +1,4 @@
|
||||
-#define STRLEN __wcsnlen_evex
|
||||
+#define STRNLEN __wcsnlen_evex
|
||||
#define USE_AS_WCSLEN 1
|
||||
-#define USE_AS_STRNLEN 1
|
||||
|
||||
-#include "strlen-evex.S"
|
||||
+#include "strnlen-evex.S"
|
||||
257
glibc-RHEL-175520-16.patch
Normal file
257
glibc-RHEL-175520-16.patch
Normal file
@ -0,0 +1,257 @@
|
||||
commit cd5fda114ece002945ace3d54a8f80a4f67d1fbb
|
||||
Author: Sajan Karumanchi <sajan.karumanchi@gmail.com>
|
||||
Date: Thu Mar 26 09:21:30 2026 +0000
|
||||
|
||||
x86_64: Prefer EVEX512 code-path on AMD Zen5 CPUs
|
||||
|
||||
Introduced a synthetic architecture preference flag (Prefer_EVEX512)
|
||||
and enabled it for AMD Zen5 (CPUID Family 0x1A) when AVX-512 is supported.
|
||||
|
||||
This flag modifies IFUNC dispatch to prefer 512-bit EVEX variants over
|
||||
256-bit EVEX variants for string and memory functions on Zen5 processors,
|
||||
leveraging their native 512-bit execution units for improved throughput.
|
||||
When Prefer_EVEX512 is set, the dispatcher selects evex512 implementations;
|
||||
otherwise, it falls back to evex (256-bit) variants.
|
||||
|
||||
The implementation updates the IFUNC selection logic in ifunc-avx2.h and
|
||||
ifunc-evex.h to check for the Prefer_EVEX512 flag before dispatching to
|
||||
EVEX512 implementations. This change affects six string/memory functions:
|
||||
|
||||
- strchr
|
||||
- strlen
|
||||
- strnlen
|
||||
- strrchr
|
||||
- strchrnul
|
||||
- memchr
|
||||
|
||||
Benchmarks conducted on AMD Zen5 hardware demonstrate significant
|
||||
performance improvements across all affected functions:
|
||||
|
||||
Function Baseline Patched Avg Avg Avg Max
|
||||
Variant Variant Baseline Patched Change Improve
|
||||
(ns) (ns) % %
|
||||
------------+----------+----------+-----------+----------+--------+--------
|
||||
STRCHR evex evex512 16.408 12.293 25.08% 37.69%
|
||||
STRLEN evex evex512 16.862 11.436 32.18% 56.74%
|
||||
STRNLEN evex evex512 18.493 11.762 36.40% 64.40%
|
||||
STRRCHR evex evex512 15.154 10.874 28.24% 44.38%
|
||||
STRCHRNUL evex evex512 16.464 12.605 23.44% 45.56%
|
||||
MEMCHR evex evex512 9.984 8.268 17.19% 39.99%
|
||||
|
||||
Additionally, a tunable option (glibc.cpu.x86_cpu_features.preferred)
|
||||
is provided to allow runtime control of the Prefer_EVEX512 flag for testing
|
||||
and compatibility.
|
||||
|
||||
Reviewed-by: Ganesh Gopalasubramanian <Ganesh.Gopalasubramanian@amd.com>
|
||||
Reviewed-by: H.J. Lu <hjl.tools@gmail.com>
|
||||
|
||||
Conflicts:
|
||||
sysdeps/x86/cpu-features.c
|
||||
(adapt to missing b93dddfaf440aa12f45d7c356f6ffe9f27d35577)
|
||||
sysdeps/x86/cpu-tunables.c
|
||||
(fixup: extra "disable" parameter downstream)
|
||||
sysdeps/x86/include/cpu-features-preferred_feature_index_1.def
|
||||
(fixup context: missing 5bcf6265f215326d14dfacdce8532792c2c7f8f8)
|
||||
sysdeps/x86_64/multiarch/ifunc-avx2.h
|
||||
(fixup context)
|
||||
sysdeps/x86_64/multiarch/ifunc-evex.h
|
||||
(fixup context)
|
||||
sysdeps/x86_64/multiarch/strchr.c
|
||||
(fixup context)
|
||||
|
||||
diff --git a/sysdeps/x86/cpu-features.c b/sysdeps/x86/cpu-features.c
|
||||
index 846c17366a38abc2..fd0c8be971b3119c 100644
|
||||
--- a/sysdeps/x86/cpu-features.c
|
||||
+++ b/sysdeps/x86/cpu-features.c
|
||||
@@ -855,6 +855,12 @@ https://www.intel.com/content/www/us/en/support/articles/000059422/processors.ht
|
||||
|
||||
ecx = cpu_features->features[CPUID_INDEX_1].cpuid.ecx;
|
||||
|
||||
+ /* Prefer EVEX512 string/memory variants on AMD Zen5 (Family 0x1A)
|
||||
+ when AVX-512 is usable. */
|
||||
+ if (family == 0x1A && CPU_FEATURE_USABLE_P (cpu_features, AVX512F))
|
||||
+ cpu_features->preferred[index_arch_Prefer_EVEX512]
|
||||
+ |= bit_arch_Prefer_EVEX512;
|
||||
+
|
||||
if (CPU_FEATURE_USABLE_P (cpu_features, AVX))
|
||||
{
|
||||
/* Since the FMA4 bit is in CPUID_INDEX_80000001 and
|
||||
diff --git a/sysdeps/x86/cpu-tunables.c b/sysdeps/x86/cpu-tunables.c
|
||||
index 957db3ad229ba39f..1d1da66ee4e9f639 100644
|
||||
--- a/sysdeps/x86/cpu-tunables.c
|
||||
+++ b/sysdeps/x86/cpu-tunables.c
|
||||
@@ -220,6 +220,12 @@ TUNABLE_CALLBACK (set_hwcaps) (tunable_val_t *valp)
|
||||
disable, 11);
|
||||
}
|
||||
break;
|
||||
+ case 14:
|
||||
+ {
|
||||
+ CHECK_GLIBC_IFUNC_PREFERRED_NEED_BOTH
|
||||
+ (n, cpu_features, Prefer_EVEX512, AVX512F, disable, 14);
|
||||
+ }
|
||||
+ break;
|
||||
case 15:
|
||||
{
|
||||
CHECK_GLIBC_IFUNC_PREFERRED_BOTH (n, cpu_features,
|
||||
diff --git a/sysdeps/x86/include/cpu-features-preferred_feature_index_1.def b/sysdeps/x86/include/cpu-features-preferred_feature_index_1.def
|
||||
index 1530d594b3a0c88e..21955eabc7a4fe91 100644
|
||||
--- a/sysdeps/x86/include/cpu-features-preferred_feature_index_1.def
|
||||
+++ b/sysdeps/x86/include/cpu-features-preferred_feature_index_1.def
|
||||
@@ -33,3 +33,4 @@ BIT (Prefer_No_AVX512)
|
||||
BIT (MathVec_Prefer_No_AVX512)
|
||||
BIT (Prefer_FSRM)
|
||||
BIT (Avoid_Short_Distance_REP_MOVSB)
|
||||
+BIT (Prefer_EVEX512)
|
||||
diff --git a/sysdeps/x86_64/multiarch/ifunc-avx2.h b/sysdeps/x86_64/multiarch/ifunc-avx2.h
|
||||
index 877f007dd6e38fe8..dad8377750c39ec3 100644
|
||||
--- a/sysdeps/x86_64/multiarch/ifunc-avx2.h
|
||||
+++ b/sysdeps/x86_64/multiarch/ifunc-avx2.h
|
||||
@@ -1,4 +1,4 @@
|
||||
-/* Common definition for ifunc selections optimized with SSE2 and AVX2.
|
||||
+/* Common definition for ifunc selections optimized with SSE2, AVX2 and EVEX512.
|
||||
All versions must be listed in ifunc-impl-list.c.
|
||||
Copyright (C) 2017-2021 Free Software Foundation, Inc.
|
||||
This file is part of the GNU C Library.
|
||||
@@ -23,6 +23,9 @@ extern __typeof (REDIRECT_NAME) OPTIMIZE (sse2) attribute_hidden;
|
||||
extern __typeof (REDIRECT_NAME) OPTIMIZE (avx2) attribute_hidden;
|
||||
extern __typeof (REDIRECT_NAME) OPTIMIZE (avx2_rtm) attribute_hidden;
|
||||
extern __typeof (REDIRECT_NAME) OPTIMIZE (evex) attribute_hidden;
|
||||
+#ifdef USE_EVEX512
|
||||
+extern __typeof (REDIRECT_NAME) OPTIMIZE (evex512) attribute_hidden;
|
||||
+#endif
|
||||
|
||||
static inline void *
|
||||
IFUNC_SELECTOR (void)
|
||||
@@ -37,7 +40,13 @@ IFUNC_SELECTOR (void)
|
||||
{
|
||||
if (CPU_FEATURE_USABLE_P (cpu_features, AVX512VL)
|
||||
&& CPU_FEATURE_USABLE_P (cpu_features, AVX512BW))
|
||||
- return OPTIMIZE (evex);
|
||||
+ {
|
||||
+#ifdef USE_EVEX512
|
||||
+ if (CPU_FEATURES_ARCH_P (cpu_features, Prefer_EVEX512))
|
||||
+ return OPTIMIZE (evex512);
|
||||
+#endif
|
||||
+ return OPTIMIZE (evex);
|
||||
+ }
|
||||
|
||||
if (CPU_FEATURE_USABLE_P (cpu_features, RTM))
|
||||
return OPTIMIZE (avx2_rtm);
|
||||
diff --git a/sysdeps/x86_64/multiarch/ifunc-evex.h b/sysdeps/x86_64/multiarch/ifunc-evex.h
|
||||
index fc391edb8abc7d7f..440c1cc2e2cc0e73 100644
|
||||
--- a/sysdeps/x86_64/multiarch/ifunc-evex.h
|
||||
+++ b/sysdeps/x86_64/multiarch/ifunc-evex.h
|
||||
@@ -1,4 +1,4 @@
|
||||
-/* Common definition for ifunc selection optimized with EVEX.
|
||||
+/* Common definition for ifunc selection optimized with EVEX and EVEX512.
|
||||
All versions must be listed in ifunc-impl-list.c.
|
||||
Copyright (C) 2017-2021 Free Software Foundation, Inc.
|
||||
This file is part of the GNU C Library.
|
||||
@@ -25,6 +25,9 @@ extern __typeof (REDIRECT_NAME) OPTIMIZE (avx2_rtm) attribute_hidden;
|
||||
extern __typeof (REDIRECT_NAME) OPTIMIZE (evex) attribute_hidden;
|
||||
extern __typeof (REDIRECT_NAME) OPTIMIZE (evex_rtm) attribute_hidden;
|
||||
|
||||
+#ifdef USE_EVEX512
|
||||
+extern __typeof (REDIRECT_NAME) OPTIMIZE (evex512) attribute_hidden;
|
||||
+#endif
|
||||
|
||||
static inline void *
|
||||
IFUNC_SELECTOR (void)
|
||||
@@ -38,6 +41,11 @@ IFUNC_SELECTOR (void)
|
||||
if (CPU_FEATURE_USABLE_P (cpu_features, AVX512VL)
|
||||
&& CPU_FEATURE_USABLE_P (cpu_features, AVX512BW))
|
||||
{
|
||||
+#ifdef USE_EVEX512
|
||||
+ if (CPU_FEATURES_ARCH_P (cpu_features, Prefer_EVEX512))
|
||||
+ return OPTIMIZE (evex512);
|
||||
+#endif
|
||||
+
|
||||
if (CPU_FEATURE_USABLE_P (cpu_features, RTM))
|
||||
return OPTIMIZE (evex_rtm);
|
||||
|
||||
diff --git a/sysdeps/x86_64/multiarch/memchr.c b/sysdeps/x86_64/multiarch/memchr.c
|
||||
index 5a4131cb8fd6d860..ae1c2b121d62b9a2 100644
|
||||
--- a/sysdeps/x86_64/multiarch/memchr.c
|
||||
+++ b/sysdeps/x86_64/multiarch/memchr.c
|
||||
@@ -24,6 +24,7 @@
|
||||
# undef memchr
|
||||
|
||||
# define SYMBOL_NAME memchr
|
||||
+# define USE_EVEX512 1
|
||||
# include "ifunc-evex.h"
|
||||
|
||||
libc_ifunc_redirected (__redirect_memchr, memchr, IFUNC_SELECTOR ());
|
||||
diff --git a/sysdeps/x86_64/multiarch/strchr.c b/sysdeps/x86_64/multiarch/strchr.c
|
||||
index 691770f335b70b5e..de7a338d53047d78 100644
|
||||
--- a/sysdeps/x86_64/multiarch/strchr.c
|
||||
+++ b/sysdeps/x86_64/multiarch/strchr.c
|
||||
@@ -31,6 +31,7 @@ extern __typeof (REDIRECT_NAME) OPTIMIZE (sse2_no_bsf) attribute_hidden;
|
||||
extern __typeof (REDIRECT_NAME) OPTIMIZE (avx2) attribute_hidden;
|
||||
extern __typeof (REDIRECT_NAME) OPTIMIZE (avx2_rtm) attribute_hidden;
|
||||
extern __typeof (REDIRECT_NAME) OPTIMIZE (evex) attribute_hidden;
|
||||
+extern __typeof (REDIRECT_NAME) OPTIMIZE (evex512) attribute_hidden;
|
||||
|
||||
static inline void *
|
||||
IFUNC_SELECTOR (void)
|
||||
@@ -43,7 +44,12 @@ IFUNC_SELECTOR (void)
|
||||
{
|
||||
if (CPU_FEATURE_USABLE_P (cpu_features, AVX512VL)
|
||||
&& CPU_FEATURE_USABLE_P (cpu_features, AVX512BW))
|
||||
- return OPTIMIZE (evex);
|
||||
+ {
|
||||
+ if (CPU_FEATURES_ARCH_P (cpu_features, Prefer_EVEX512))
|
||||
+ return OPTIMIZE (evex512);
|
||||
+
|
||||
+ return OPTIMIZE (evex);
|
||||
+ }
|
||||
|
||||
if (CPU_FEATURE_USABLE_P (cpu_features, RTM))
|
||||
return OPTIMIZE (avx2_rtm);
|
||||
diff --git a/sysdeps/x86_64/multiarch/strchrnul.c b/sysdeps/x86_64/multiarch/strchrnul.c
|
||||
index 7631927dd44d8bf1..eadb2d41ed86636a 100644
|
||||
--- a/sysdeps/x86_64/multiarch/strchrnul.c
|
||||
+++ b/sysdeps/x86_64/multiarch/strchrnul.c
|
||||
@@ -26,6 +26,7 @@
|
||||
# undef strchrnul
|
||||
|
||||
# define SYMBOL_NAME strchrnul
|
||||
+# define USE_EVEX512 1
|
||||
# include "ifunc-avx2.h"
|
||||
|
||||
libc_ifunc_redirected (__redirect_strchrnul, __strchrnul,
|
||||
diff --git a/sysdeps/x86_64/multiarch/strlen.c b/sysdeps/x86_64/multiarch/strlen.c
|
||||
index f438b00375c8e791..13a6e63a898784e9 100644
|
||||
--- a/sysdeps/x86_64/multiarch/strlen.c
|
||||
+++ b/sysdeps/x86_64/multiarch/strlen.c
|
||||
@@ -24,6 +24,7 @@
|
||||
# undef strlen
|
||||
|
||||
# define SYMBOL_NAME strlen
|
||||
+# define USE_EVEX512 1
|
||||
# include "ifunc-avx2.h"
|
||||
|
||||
libc_ifunc_redirected (__redirect_strlen, strlen, IFUNC_SELECTOR ());
|
||||
diff --git a/sysdeps/x86_64/multiarch/strnlen.c b/sysdeps/x86_64/multiarch/strnlen.c
|
||||
index 413c2f38ab01d205..02d97b437918f4ed 100644
|
||||
--- a/sysdeps/x86_64/multiarch/strnlen.c
|
||||
+++ b/sysdeps/x86_64/multiarch/strnlen.c
|
||||
@@ -26,6 +26,7 @@
|
||||
# undef strnlen
|
||||
|
||||
# define SYMBOL_NAME strnlen
|
||||
+# define USE_EVEX512 1
|
||||
# include "ifunc-avx2.h"
|
||||
|
||||
libc_ifunc_redirected (__redirect_strnlen, __strnlen, IFUNC_SELECTOR ());
|
||||
diff --git a/sysdeps/x86_64/multiarch/strrchr.c b/sysdeps/x86_64/multiarch/strrchr.c
|
||||
index d46b23543f08333f..2b5cc738d2d2a74d 100644
|
||||
--- a/sysdeps/x86_64/multiarch/strrchr.c
|
||||
+++ b/sysdeps/x86_64/multiarch/strrchr.c
|
||||
@@ -23,6 +23,7 @@
|
||||
# undef strrchr
|
||||
|
||||
# define SYMBOL_NAME strrchr
|
||||
+# define USE_EVEX512 1
|
||||
# include "ifunc-avx2.h"
|
||||
|
||||
libc_ifunc_redirected (__redirect_strrchr, strrchr, IFUNC_SELECTOR ());
|
||||
78
glibc-RHEL-175520-2.patch
Normal file
78
glibc-RHEL-175520-2.patch
Normal file
@ -0,0 +1,78 @@
|
||||
commit 0281c7a7ec8f3f46d8e6f5f3d7fca548946dbfce
|
||||
Author: Noah Goldstein <goldstein.w.n@gmail.com>
|
||||
Date: Sun Feb 13 19:40:37 2022 -0600
|
||||
|
||||
String: Strength memset tests in test-memset.c
|
||||
|
||||
The prior sentinel logic was broken and was checking the SIMPLE_MEMSET
|
||||
as opposed to the tested implementation. As well `s` (the test buffer)
|
||||
was not reset between implementation tests so it was possible for a
|
||||
buggy implementation to be hidden by a previously executed correct
|
||||
one.
|
||||
Reviewed-by: Siddhesh Poyarekar <siddhesh@sourceware.org>
|
||||
|
||||
diff --git a/string/test-memset.c b/string/test-memset.c
|
||||
index 82bfcd6ad4a7a0fb..fe89bb57c1c85892 100644
|
||||
--- a/string/test-memset.c
|
||||
+++ b/string/test-memset.c
|
||||
@@ -107,26 +107,28 @@ SIMPLE_MEMSET (CHAR *s, int c, size_t n)
|
||||
}
|
||||
|
||||
static void
|
||||
-do_one_test (impl_t *impl, CHAR *s, int c __attribute ((unused)), size_t n)
|
||||
+do_one_test (impl_t *impl, CHAR *s, int c __attribute ((unused)), size_t n, int space_below, int space_above)
|
||||
{
|
||||
- CHAR buf[n + 2];
|
||||
- CHAR *tstbuf = buf + 1;
|
||||
- CHAR sentinel = c - 1;
|
||||
- buf[0] = sentinel;
|
||||
- buf[n + 1] = sentinel;
|
||||
+ CHAR buf[n];
|
||||
+ CHAR sentinel = ~c;
|
||||
+ if (space_below)
|
||||
+ s[-1] = sentinel;
|
||||
+ if (space_above)
|
||||
+ s[n] = sentinel;
|
||||
+ SIMPLE_MEMSET(s, ~c, n);
|
||||
#ifdef TEST_BZERO
|
||||
- simple_bzero (tstbuf, n);
|
||||
+ simple_bzero (buf, n);
|
||||
CALL (impl, s, n);
|
||||
- if (memcmp (s, tstbuf, n) != 0
|
||||
- || buf[0] != sentinel
|
||||
- || buf[n + 1] != sentinel)
|
||||
+ if (memcmp (s, buf, n) != 0
|
||||
+ || (space_below && s[-1] != sentinel)
|
||||
+ || (space_above && s[n] != sentinel))
|
||||
#else
|
||||
CHAR *res = CALL (impl, s, c, n);
|
||||
if (res != s
|
||||
- || SIMPLE_MEMSET (tstbuf, c, n) != tstbuf
|
||||
- || MEMCMP (s, tstbuf, n) != 0
|
||||
- || buf[0] != sentinel
|
||||
- || buf[n + 1] != sentinel)
|
||||
+ || SIMPLE_MEMSET (buf, c, n) != buf
|
||||
+ || MEMCMP (s, buf, n) != 0
|
||||
+ || (space_below && s[-1] != sentinel)
|
||||
+ || (space_above && s[n] != sentinel))
|
||||
#endif /* !TEST_BZERO */
|
||||
{
|
||||
error (0, 0, "Wrong result in function %s", impl->name);
|
||||
@@ -138,12 +140,16 @@ do_one_test (impl_t *impl, CHAR *s, int c __attribute ((unused)), size_t n)
|
||||
static void
|
||||
do_test (size_t align, int c, size_t len)
|
||||
{
|
||||
+ int space_below, space_above;
|
||||
align &= 4095;
|
||||
if ((align + len) * sizeof (CHAR) > page_size)
|
||||
return;
|
||||
|
||||
+ space_below = !!align;
|
||||
+ space_above = !((align + len + 1) * sizeof (CHAR) > page_size);
|
||||
+
|
||||
FOR_EACH_IMPL (impl, 0)
|
||||
- do_one_test (impl, (CHAR *) (buf1) + align, c, len);
|
||||
+ do_one_test (impl, (CHAR *) (buf1) + align, c, len, space_below, space_above);
|
||||
}
|
||||
|
||||
#ifndef TEST_BZERO
|
||||
830
glibc-RHEL-175520-3.patch
Normal file
830
glibc-RHEL-175520-3.patch
Normal file
@ -0,0 +1,830 @@
|
||||
commit 67e3b0c63c35769c1ba28fa2a32446332bb4fcef
|
||||
Author: Siddhesh Poyarekar <siddhesh@sourceware.org>
|
||||
Date: Wed Apr 6 20:53:24 2022 +0530
|
||||
|
||||
tests/string: Drop simple/stupid/builtin tests
|
||||
|
||||
In most cases the simple/stupid/builtin functions were in there to
|
||||
benchmark optimized implementations against. Only in some cases the
|
||||
functions are used to check expected results.
|
||||
|
||||
Remove these tests from IMPL() and only keep them in wherever they're
|
||||
used for a specific purpose, e.g. to generate expected results.
|
||||
|
||||
This improves timing of `make subdirs=string` by over a minute and a
|
||||
half (over 15%) on a Whiskey Lake laptop.
|
||||
|
||||
Signed-off-by: Siddhesh Poyarekar <siddhesh@sourceware.org>
|
||||
Reviewed-by: Noah Goldstein <libc-alpha@sourceware.org>
|
||||
|
||||
Conflicts:
|
||||
string/test-memcpy-support.h
|
||||
(modified string/test-memcpy.c instead)
|
||||
string/test-memset.c
|
||||
(fixup context)
|
||||
|
||||
diff --git a/string/test-memccpy.c b/string/test-memccpy.c
|
||||
index e0b1d5ae5283ccb1..5f7eeda5a224d7f5 100644
|
||||
--- a/string/test-memccpy.c
|
||||
+++ b/string/test-memccpy.c
|
||||
@@ -21,13 +21,9 @@
|
||||
#define TEST_NAME "memccpy"
|
||||
#include "test-string.h"
|
||||
|
||||
-void *simple_memccpy (void *, const void *, int, size_t);
|
||||
-void *stupid_memccpy (void *, const void *, int, size_t);
|
||||
-
|
||||
-IMPL (stupid_memccpy, 0)
|
||||
-IMPL (simple_memccpy, 0)
|
||||
IMPL (memccpy, 1)
|
||||
|
||||
+/* Naive implementation to verify results. */
|
||||
void *
|
||||
simple_memccpy (void *dst, const void *src, int c, size_t n)
|
||||
{
|
||||
@@ -41,18 +37,6 @@ simple_memccpy (void *dst, const void *src, int c, size_t n)
|
||||
return NULL;
|
||||
}
|
||||
|
||||
-void *
|
||||
-stupid_memccpy (void *dst, const void *src, int c, size_t n)
|
||||
-{
|
||||
- void *p = memchr (src, c, n);
|
||||
-
|
||||
- if (p != NULL)
|
||||
- return mempcpy (dst, src, p - src + 1);
|
||||
-
|
||||
- memcpy (dst, src, n);
|
||||
- return NULL;
|
||||
-}
|
||||
-
|
||||
typedef void *(*proto_t) (void *, const void *, int c, size_t);
|
||||
|
||||
static void
|
||||
diff --git a/string/test-memchr.c b/string/test-memchr.c
|
||||
index ce964284aa16508f..6e5f7795a09febcb 100644
|
||||
--- a/string/test-memchr.c
|
||||
+++ b/string/test-memchr.c
|
||||
@@ -45,11 +45,10 @@
|
||||
#endif /* WIDE */
|
||||
|
||||
typedef CHAR *(*proto_t) (const CHAR *, int, size_t);
|
||||
-CHAR *SIMPLE_MEMCHR (const CHAR *, int, size_t);
|
||||
|
||||
-IMPL (SIMPLE_MEMCHR, 0)
|
||||
IMPL (MEMCHR, 1)
|
||||
|
||||
+/* Naive implementation to verify results. */
|
||||
CHAR *
|
||||
SIMPLE_MEMCHR (const CHAR *s, int c, size_t n)
|
||||
{
|
||||
diff --git a/string/test-memcmp.c b/string/test-memcmp.c
|
||||
index fbda26a41e9731a5..480b3a0adcf3a644 100644
|
||||
--- a/string/test-memcmp.c
|
||||
+++ b/string/test-memcmp.c
|
||||
@@ -73,7 +73,6 @@ simple_memcmp (const char *s1, const char *s2, size_t n)
|
||||
|
||||
typedef int (*proto_t) (const CHAR *, const CHAR *, size_t);
|
||||
|
||||
-IMPL (SIMPLE_MEMCMP, 0)
|
||||
IMPL (MEMCMP, 1)
|
||||
|
||||
static int
|
||||
diff --git a/string/test-memcpy.c b/string/test-memcpy.c
|
||||
index c9dfc88fedddc775..63c930627821ed93 100644
|
||||
--- a/string/test-memcpy.c
|
||||
+++ b/string/test-memcpy.c
|
||||
@@ -24,13 +24,9 @@
|
||||
# define TEST_NAME "memcpy"
|
||||
# include "test-string.h"
|
||||
|
||||
-char *simple_memcpy (char *, const char *, size_t);
|
||||
-char *builtin_memcpy (char *, const char *, size_t);
|
||||
-
|
||||
-IMPL (simple_memcpy, 0)
|
||||
-IMPL (builtin_memcpy, 0)
|
||||
IMPL (memcpy, 1)
|
||||
|
||||
+/* Naive implementation to verify results. */
|
||||
char *
|
||||
simple_memcpy (char *dst, const char *src, size_t n)
|
||||
{
|
||||
@@ -40,11 +36,6 @@ simple_memcpy (char *dst, const char *src, size_t n)
|
||||
return ret;
|
||||
}
|
||||
|
||||
-char *
|
||||
-builtin_memcpy (char *dst, const char *src, size_t n)
|
||||
-{
|
||||
- return __builtin_memcpy (dst, src, n);
|
||||
-}
|
||||
#endif
|
||||
|
||||
typedef char *(*proto_t) (char *, const char *, size_t);
|
||||
diff --git a/string/test-memmem.c b/string/test-memmem.c
|
||||
index 624d3106bfebf628..260116fcd80072cc 100644
|
||||
--- a/string/test-memmem.c
|
||||
+++ b/string/test-memmem.c
|
||||
@@ -24,11 +24,10 @@
|
||||
#include "test-string.h"
|
||||
|
||||
typedef char *(*proto_t) (const void *, size_t, const void *, size_t);
|
||||
-void *simple_memmem (const void *, size_t, const void *, size_t);
|
||||
|
||||
-IMPL (simple_memmem, 0)
|
||||
IMPL (memmem, 1)
|
||||
|
||||
+/* Naive implementation to verify results. */
|
||||
void *
|
||||
simple_memmem (const void *haystack, size_t haystack_len, const void *needle,
|
||||
size_t needle_len)
|
||||
diff --git a/string/test-memmove.c b/string/test-memmove.c
|
||||
index b271248b1d903634..c4f9f6d11330016d 100644
|
||||
--- a/string/test-memmove.c
|
||||
+++ b/string/test-memmove.c
|
||||
@@ -30,23 +30,23 @@ char *simple_memmove (char *, const char *, size_t);
|
||||
|
||||
#ifdef TEST_BCOPY
|
||||
typedef void (*proto_t) (const char *, char *, size_t);
|
||||
-void simple_bcopy (const char *, char *, size_t);
|
||||
|
||||
-IMPL (simple_bcopy, 0)
|
||||
IMPL (bcopy, 1)
|
||||
|
||||
+/* Naive implementation to verify results. */
|
||||
void
|
||||
simple_bcopy (const char *src, char *dst, size_t n)
|
||||
{
|
||||
simple_memmove (dst, src, n);
|
||||
}
|
||||
+
|
||||
#else
|
||||
typedef char *(*proto_t) (char *, const char *, size_t);
|
||||
|
||||
-IMPL (simple_memmove, 0)
|
||||
IMPL (memmove, 1)
|
||||
#endif
|
||||
|
||||
+/* Naive implementation to verify results. */
|
||||
char *
|
||||
inhibit_loop_to_libcall
|
||||
simple_memmove (char *dst, const char *src, size_t n)
|
||||
diff --git a/string/test-mempcpy.c b/string/test-mempcpy.c
|
||||
index f80b014d41921462..1554af9b1587d5ca 100644
|
||||
--- a/string/test-mempcpy.c
|
||||
+++ b/string/test-mempcpy.c
|
||||
@@ -23,11 +23,9 @@
|
||||
#define TEST_NAME "mempcpy"
|
||||
#include "test-string.h"
|
||||
|
||||
-char *simple_mempcpy (char *, const char *, size_t);
|
||||
-
|
||||
-IMPL (simple_mempcpy, 0)
|
||||
IMPL (mempcpy, 1)
|
||||
|
||||
+/* Naive implementation to verify results. */
|
||||
char *
|
||||
simple_mempcpy (char *dst, const char *src, size_t n)
|
||||
{
|
||||
diff --git a/string/test-memrchr.c b/string/test-memrchr.c
|
||||
index 1a3b9ce69d2729c2..5166f22a37893c98 100644
|
||||
--- a/string/test-memrchr.c
|
||||
+++ b/string/test-memrchr.c
|
||||
@@ -22,11 +22,10 @@
|
||||
#include "test-string.h"
|
||||
|
||||
typedef char *(*proto_t) (const char *, int, size_t);
|
||||
-char *simple_memrchr (const char *, int, size_t);
|
||||
|
||||
-IMPL (simple_memrchr, 0)
|
||||
IMPL (memrchr, 1)
|
||||
|
||||
+/* Naive implementation to verify results. */
|
||||
char *
|
||||
simple_memrchr (const char *s, int c, size_t n)
|
||||
{
|
||||
diff --git a/string/test-memset.c b/string/test-memset.c
|
||||
index fe89bb57c1c85892..bca571860ff82a19 100644
|
||||
--- a/string/test-memset.c
|
||||
+++ b/string/test-memset.c
|
||||
@@ -51,51 +51,19 @@
|
||||
# define BIG_CHAR WCHAR_MAX
|
||||
#endif /* WIDE */
|
||||
|
||||
-CHAR *SIMPLE_MEMSET (CHAR *, int, size_t);
|
||||
-
|
||||
#ifdef TEST_BZERO
|
||||
typedef void (*proto_t) (char *, size_t);
|
||||
-void simple_bzero (char *, size_t);
|
||||
-void builtin_bzero (char *, size_t);
|
||||
-
|
||||
-IMPL (simple_bzero, 0)
|
||||
-IMPL (builtin_bzero, 0)
|
||||
-#ifdef TEST_EXPLICIT_BZERO
|
||||
+# ifdef TEST_EXPLICIT_BZERO
|
||||
IMPL (explicit_bzero, 1)
|
||||
-#else
|
||||
+# else
|
||||
IMPL (bzero, 1)
|
||||
-#endif
|
||||
-
|
||||
-void
|
||||
-simple_bzero (char *s, size_t n)
|
||||
-{
|
||||
- SIMPLE_MEMSET (s, 0, n);
|
||||
-}
|
||||
-
|
||||
-void
|
||||
-builtin_bzero (char *s, size_t n)
|
||||
-{
|
||||
- __builtin_bzero (s, n);
|
||||
-}
|
||||
+# endif
|
||||
#else
|
||||
typedef CHAR *(*proto_t) (CHAR *, int, size_t);
|
||||
-
|
||||
-IMPL (SIMPLE_MEMSET, 0)
|
||||
-# ifndef WIDE
|
||||
-char *builtin_memset (char *, int, size_t);
|
||||
-IMPL (builtin_memset, 0)
|
||||
-# endif /* !WIDE */
|
||||
IMPL (MEMSET, 1)
|
||||
-
|
||||
-# ifndef WIDE
|
||||
-char *
|
||||
-builtin_memset (char *s, int c, size_t n)
|
||||
-{
|
||||
- return __builtin_memset (s, c, n);
|
||||
-}
|
||||
-# endif /* !WIDE */
|
||||
#endif /* !TEST_BZERO */
|
||||
|
||||
+/* Naive implementation to verify results. */
|
||||
CHAR *
|
||||
inhibit_loop_to_libcall
|
||||
SIMPLE_MEMSET (CHAR *s, int c, size_t n)
|
||||
@@ -117,7 +85,7 @@ do_one_test (impl_t *impl, CHAR *s, int c __attribute ((unused)), size_t n, int
|
||||
s[n] = sentinel;
|
||||
SIMPLE_MEMSET(s, ~c, n);
|
||||
#ifdef TEST_BZERO
|
||||
- simple_bzero (buf, n);
|
||||
+ SIMPLE_MEMSET (buf, 0, n);
|
||||
CALL (impl, s, n);
|
||||
if (memcmp (s, buf, n) != 0
|
||||
|| (space_below && s[-1] != sentinel)
|
||||
diff --git a/string/test-strcasecmp.c b/string/test-strcasecmp.c
|
||||
index b380150cb74b5a8e..f4e002510cb6797b 100644
|
||||
--- a/string/test-strcasecmp.c
|
||||
+++ b/string/test-strcasecmp.c
|
||||
@@ -24,14 +24,11 @@
|
||||
#include "test-string.h"
|
||||
|
||||
typedef int (*proto_t) (const char *, const char *);
|
||||
-static int simple_strcasecmp (const char *, const char *);
|
||||
-static int stupid_strcasecmp (const char *, const char *);
|
||||
|
||||
-IMPL (stupid_strcasecmp, 0)
|
||||
-IMPL (simple_strcasecmp, 0)
|
||||
IMPL (strcasecmp, 1)
|
||||
|
||||
-static int
|
||||
+/* Naive implementation to verify results. */
|
||||
+int
|
||||
simple_strcasecmp (const char *s1, const char *s2)
|
||||
{
|
||||
int ret;
|
||||
@@ -43,24 +40,6 @@ simple_strcasecmp (const char *s1, const char *s2)
|
||||
return ret;
|
||||
}
|
||||
|
||||
-static int
|
||||
-stupid_strcasecmp (const char *s1, const char *s2)
|
||||
-{
|
||||
- size_t ns1 = strlen (s1) + 1, ns2 = strlen (s2) + 1;
|
||||
- size_t n = ns1 < ns2 ? ns1 : ns2;
|
||||
- int ret = 0;
|
||||
-
|
||||
- while (n--)
|
||||
- {
|
||||
- if ((ret = ((unsigned char) tolower (*s1)
|
||||
- - (unsigned char) tolower (*s2))) != 0)
|
||||
- break;
|
||||
- ++s1;
|
||||
- ++s2;
|
||||
- }
|
||||
- return ret;
|
||||
-}
|
||||
-
|
||||
static void
|
||||
do_one_test (impl_t *impl, const char *s1, const char *s2, int exp_result)
|
||||
{
|
||||
diff --git a/string/test-strcasestr.c b/string/test-strcasestr.c
|
||||
index 045156e6b8b87f51..0cdc5dcd97974694 100644
|
||||
--- a/string/test-strcasestr.c
|
||||
+++ b/string/test-strcasestr.c
|
||||
@@ -22,15 +22,15 @@
|
||||
#include "test-string.h"
|
||||
|
||||
|
||||
-#define STRCASESTR simple_strcasestr
|
||||
+#define STRCASESTR c_strcasestr
|
||||
#define NO_ALIAS
|
||||
#define __strncasecmp strncasecmp
|
||||
#define __strnlen strnlen
|
||||
#include "strcasestr.c"
|
||||
|
||||
-
|
||||
+/* Naive implementation to verify results. */
|
||||
static char *
|
||||
-stupid_strcasestr (const char *s1, const char *s2)
|
||||
+simple_strcasestr (const char *s1, const char *s2)
|
||||
{
|
||||
ssize_t s1len = strlen (s1);
|
||||
ssize_t s2len = strlen (s2);
|
||||
@@ -54,8 +54,7 @@ stupid_strcasestr (const char *s1, const char *s2)
|
||||
|
||||
typedef char *(*proto_t) (const char *, const char *);
|
||||
|
||||
-IMPL (stupid_strcasestr, 0)
|
||||
-IMPL (simple_strcasestr, 0)
|
||||
+IMPL (c_strcasestr, 0)
|
||||
IMPL (strcasestr, 1)
|
||||
|
||||
|
||||
@@ -130,7 +129,7 @@ check1 (void)
|
||||
const char s2[] = "OK";
|
||||
char *exp_result;
|
||||
|
||||
- exp_result = stupid_strcasestr (s1, s2);
|
||||
+ exp_result = simple_strcasestr (s1, s2);
|
||||
FOR_EACH_IMPL (impl, 0)
|
||||
check_result (impl, s1, s2, exp_result);
|
||||
}
|
||||
diff --git a/string/test-strcat.c b/string/test-strcat.c
|
||||
index dea1558b0c64c311..97cc1a2136800ee9 100644
|
||||
--- a/string/test-strcat.c
|
||||
+++ b/string/test-strcat.c
|
||||
@@ -55,11 +55,10 @@
|
||||
#endif /* WIDE */
|
||||
|
||||
typedef CHAR *(*proto_t) (CHAR *, const CHAR *);
|
||||
-CHAR *SIMPLE_STRCAT (CHAR *, const CHAR *);
|
||||
|
||||
-IMPL (SIMPLE_STRCAT, 0)
|
||||
IMPL (STRCAT, 1)
|
||||
|
||||
+/* Naive implementation to verify results. */
|
||||
CHAR *
|
||||
SIMPLE_STRCAT (CHAR *dst, const CHAR *src)
|
||||
{
|
||||
diff --git a/string/test-strchr.c b/string/test-strchr.c
|
||||
index 6c8ca54a7df8fa2b..f24ade10485853c3 100644
|
||||
--- a/string/test-strchr.c
|
||||
+++ b/string/test-strchr.c
|
||||
@@ -37,7 +37,6 @@
|
||||
#ifndef WIDE
|
||||
# ifdef USE_FOR_STRCHRNUL
|
||||
# define STRCHR strchrnul
|
||||
-# define stupid_STRCHR stupid_STRCHRNUL
|
||||
# define simple_STRCHR simple_STRCHRNUL
|
||||
# else
|
||||
# define STRCHR strchr
|
||||
@@ -53,7 +52,6 @@
|
||||
# include <wchar.h>
|
||||
# ifdef USE_FOR_STRCHRNUL
|
||||
# define STRCHR wcschrnul
|
||||
-# define stupid_STRCHR stupid_WCSCHRNUL
|
||||
# define simple_STRCHR simple_WCSCHRNUL
|
||||
# else
|
||||
# define STRCHR wcschr
|
||||
@@ -76,17 +74,9 @@
|
||||
|
||||
typedef CHAR *(*proto_t) (const CHAR *, int);
|
||||
|
||||
+/* Naive implementation to verify results. */
|
||||
CHAR *
|
||||
simple_STRCHR (const CHAR *s, int c)
|
||||
-{
|
||||
- for (; *s != (CHAR) c; ++s)
|
||||
- if (*s == '\0')
|
||||
- return NULLRET ((CHAR *) s);
|
||||
- return (CHAR *) s;
|
||||
-}
|
||||
-
|
||||
-CHAR *
|
||||
-stupid_STRCHR (const CHAR *s, int c)
|
||||
{
|
||||
size_t n = STRLEN (s) + 1;
|
||||
|
||||
@@ -96,8 +86,6 @@ stupid_STRCHR (const CHAR *s, int c)
|
||||
return NULLRET ((CHAR *) s - 1);
|
||||
}
|
||||
|
||||
-IMPL (stupid_STRCHR, 0)
|
||||
-IMPL (simple_STRCHR, 0)
|
||||
IMPL (STRCHR, 1)
|
||||
|
||||
static int
|
||||
@@ -233,7 +221,7 @@ check1 (void)
|
||||
{
|
||||
CHAR s[] __attribute__((aligned(16))) = L ("\xff");
|
||||
CHAR c = L ('\xfe');
|
||||
- CHAR *exp_result = stupid_STRCHR (s, c);
|
||||
+ CHAR *exp_result = simple_STRCHR (s, c);
|
||||
|
||||
FOR_EACH_IMPL (impl, 0)
|
||||
check_result (impl, s, c, exp_result);
|
||||
diff --git a/string/test-strcmp.c b/string/test-strcmp.c
|
||||
index a6b5344f5710effc..7eeaf472de5ce178 100644
|
||||
--- a/string/test-strcmp.c
|
||||
+++ b/string/test-strcmp.c
|
||||
@@ -98,7 +98,6 @@ simple_strcmp (const char *s1, const char *s2)
|
||||
|
||||
typedef int (*proto_t) (const CHAR *, const CHAR *);
|
||||
|
||||
-IMPL (SIMPLE_STRCMP, 1)
|
||||
IMPL (STRCMP, 1)
|
||||
|
||||
static int
|
||||
diff --git a/string/test-strcpy.c b/string/test-strcpy.c
|
||||
index 4c0a04f85d70104f..271b9251154fb0f6 100644
|
||||
--- a/string/test-strcpy.c
|
||||
+++ b/string/test-strcpy.c
|
||||
@@ -56,11 +56,9 @@
|
||||
# define STRCPY wcscpy
|
||||
# endif
|
||||
|
||||
-CHAR *SIMPLE_STRCPY (CHAR *, const CHAR *);
|
||||
-
|
||||
-IMPL (SIMPLE_STRCPY, 0)
|
||||
IMPL (STRCPY, 1)
|
||||
|
||||
+/* Naive implementation to verify results. */
|
||||
CHAR *
|
||||
SIMPLE_STRCPY (CHAR *dst, const CHAR *src)
|
||||
{
|
||||
diff --git a/string/test-strlen.c b/string/test-strlen.c
|
||||
index c9a7afb339a28114..06919fa9ce289b39 100644
|
||||
--- a/string/test-strlen.c
|
||||
+++ b/string/test-strlen.c
|
||||
@@ -39,6 +39,7 @@
|
||||
|
||||
typedef size_t (*proto_t) (const CHAR *);
|
||||
|
||||
+/* Naive implementation to verify results. */
|
||||
size_t
|
||||
simple_STRLEN (const CHAR *s)
|
||||
{
|
||||
@@ -57,7 +58,6 @@ builtin_strlen (const CHAR *p)
|
||||
IMPL (builtin_strlen, 0)
|
||||
#endif
|
||||
|
||||
-IMPL (simple_STRLEN, 0)
|
||||
IMPL (STRLEN, 1)
|
||||
|
||||
|
||||
diff --git a/string/test-strncasecmp.c b/string/test-strncasecmp.c
|
||||
index ace94e83eca9bd72..6662dab672378ae1 100644
|
||||
--- a/string/test-strncasecmp.c
|
||||
+++ b/string/test-strncasecmp.c
|
||||
@@ -25,12 +25,10 @@
|
||||
|
||||
typedef int (*proto_t) (const char *, const char *, size_t);
|
||||
static int simple_strncasecmp (const char *, const char *, size_t);
|
||||
-static int stupid_strncasecmp (const char *, const char *, size_t);
|
||||
|
||||
-IMPL (stupid_strncasecmp, 0)
|
||||
-IMPL (simple_strncasecmp, 0)
|
||||
IMPL (strncasecmp, 1)
|
||||
|
||||
+/* Naive implementation to verify results. */
|
||||
static int
|
||||
simple_strncasecmp (const char *s1, const char *s2, size_t n)
|
||||
{
|
||||
@@ -50,27 +48,6 @@ simple_strncasecmp (const char *s1, const char *s2, size_t n)
|
||||
return ret;
|
||||
}
|
||||
|
||||
-static int
|
||||
-stupid_strncasecmp (const char *s1, const char *s2, size_t max)
|
||||
-{
|
||||
- size_t ns1 = strlen (s1) + 1;
|
||||
- size_t ns2 = strlen (s2) + 1;
|
||||
- size_t n = ns1 < ns2 ? ns1 : ns2;
|
||||
- if (n > max)
|
||||
- n = max;
|
||||
- int ret = 0;
|
||||
-
|
||||
- while (n--)
|
||||
- {
|
||||
- if ((ret = ((unsigned char) tolower (*s1)
|
||||
- - (unsigned char) tolower (*s2))) != 0)
|
||||
- break;
|
||||
- ++s1;
|
||||
- ++s2;
|
||||
- }
|
||||
- return ret;
|
||||
-}
|
||||
-
|
||||
static int
|
||||
check_result (impl_t *impl, const char *s1, const char *s2, size_t n,
|
||||
int exp_result)
|
||||
diff --git a/string/test-strncat.c b/string/test-strncat.c
|
||||
index 37ea26ea057938a4..982c1dff0421ce45 100644
|
||||
--- a/string/test-strncat.c
|
||||
+++ b/string/test-strncat.c
|
||||
@@ -29,7 +29,6 @@
|
||||
# define CHAR char
|
||||
# define UCHAR unsigned char
|
||||
# define SIMPLE_STRNCAT simple_strncat
|
||||
-# define STUPID_STRNCAT stupid_strncat
|
||||
# define STRLEN strlen
|
||||
# define MEMSET memset
|
||||
# define MEMCPY memcpy
|
||||
@@ -42,7 +41,6 @@
|
||||
# define CHAR wchar_t
|
||||
# define UCHAR wchar_t
|
||||
# define SIMPLE_STRNCAT simple_wcsncat
|
||||
-# define STUPID_STRNCAT stupid_wcsncat
|
||||
# define STRLEN wcslen
|
||||
# define MEMSET wmemset
|
||||
# define MEMCPY wmemcpy
|
||||
@@ -52,14 +50,12 @@
|
||||
#endif /* WIDE */
|
||||
|
||||
typedef CHAR *(*proto_t) (CHAR *, const CHAR *, size_t);
|
||||
-CHAR *STUPID_STRNCAT (CHAR *, const CHAR *, size_t);
|
||||
-CHAR *SIMPLE_STRNCAT (CHAR *, const CHAR *, size_t);
|
||||
|
||||
-IMPL (STUPID_STRNCAT, 0)
|
||||
IMPL (STRNCAT, 2)
|
||||
|
||||
+/* Naive implementation to verify results. */
|
||||
CHAR *
|
||||
-STUPID_STRNCAT (CHAR *dst, const CHAR *src, size_t n)
|
||||
+SIMPLE_STRNCAT (CHAR *dst, const CHAR *src, size_t n)
|
||||
{
|
||||
CHAR *ret = dst;
|
||||
while (*dst++ != '\0');
|
||||
diff --git a/string/test-strncmp.c b/string/test-strncmp.c
|
||||
index a761136e94d65efc..5e481d260b52ff9d 100644
|
||||
--- a/string/test-strncmp.c
|
||||
+++ b/string/test-strncmp.c
|
||||
@@ -85,7 +85,6 @@ simple_strncmp (const char *s1, const char *s2, size_t n)
|
||||
|
||||
typedef int (*proto_t) (const CHAR *, const CHAR *, size_t);
|
||||
|
||||
-IMPL (SIMPLE_STRNCMP, 0)
|
||||
IMPL (STRNCMP, 1)
|
||||
|
||||
|
||||
diff --git a/string/test-strncpy.c b/string/test-strncpy.c
|
||||
index 84a3b83bbc185802..7ee1b53ef5a219fb 100644
|
||||
--- a/string/test-strncpy.c
|
||||
+++ b/string/test-strncpy.c
|
||||
@@ -48,21 +48,16 @@
|
||||
# include "test-string.h"
|
||||
# ifndef WIDE
|
||||
# define SIMPLE_STRNCPY simple_strncpy
|
||||
-# define STUPID_STRNCPY stupid_strncpy
|
||||
# define STRNCPY strncpy
|
||||
# else
|
||||
# define SIMPLE_STRNCPY simple_wcsncpy
|
||||
-# define STUPID_STRNCPY stupid_wcsncpy
|
||||
# define STRNCPY wcsncpy
|
||||
# endif /* WIDE */
|
||||
|
||||
-CHAR *SIMPLE_STRNCPY (CHAR *, const CHAR *, size_t);
|
||||
-CHAR *STUPID_STRNCPY (CHAR *, const CHAR *, size_t);
|
||||
|
||||
-IMPL (STUPID_STRNCPY, 0)
|
||||
-IMPL (SIMPLE_STRNCPY, 0)
|
||||
IMPL (STRNCPY, 1)
|
||||
|
||||
+/* Naive implementation to verify results. */
|
||||
CHAR *
|
||||
SIMPLE_STRNCPY (CHAR *dst, const CHAR *src, size_t n)
|
||||
{
|
||||
@@ -77,18 +72,6 @@ SIMPLE_STRNCPY (CHAR *dst, const CHAR *src, size_t n)
|
||||
return ret;
|
||||
}
|
||||
|
||||
-CHAR *
|
||||
-STUPID_STRNCPY (CHAR *dst, const CHAR *src, size_t n)
|
||||
-{
|
||||
- size_t nc = STRNLEN (src, n);
|
||||
- size_t i;
|
||||
-
|
||||
- for (i = 0; i < nc; ++i)
|
||||
- dst[i] = src[i];
|
||||
- for (; i < n; ++i)
|
||||
- dst[i] = '\0';
|
||||
- return dst;
|
||||
-}
|
||||
#endif /* !STRNCPY_RESULT */
|
||||
|
||||
typedef CHAR *(*proto_t) (CHAR *, const CHAR *, size_t);
|
||||
diff --git a/string/test-strnlen.c b/string/test-strnlen.c
|
||||
index eac84cd17526d5d9..18d1c93f3e0cd315 100644
|
||||
--- a/string/test-strnlen.c
|
||||
+++ b/string/test-strnlen.c
|
||||
@@ -43,11 +43,10 @@
|
||||
#endif /* !WIDE */
|
||||
|
||||
typedef size_t (*proto_t) (const CHAR *, size_t);
|
||||
-size_t SIMPLE_STRNLEN (const CHAR *, size_t);
|
||||
|
||||
-IMPL (SIMPLE_STRNLEN, 0)
|
||||
IMPL (STRNLEN, 1)
|
||||
|
||||
+/* Naive implementation to verify results. */
|
||||
size_t
|
||||
SIMPLE_STRNLEN (const CHAR *s, size_t maxlen)
|
||||
{
|
||||
diff --git a/string/test-strpbrk.c b/string/test-strpbrk.c
|
||||
index 3f81c2edc35af161..89586a9ab685eec6 100644
|
||||
--- a/string/test-strpbrk.c
|
||||
+++ b/string/test-strpbrk.c
|
||||
@@ -48,22 +48,17 @@
|
||||
# ifndef WIDE
|
||||
# define STRPBRK strpbrk
|
||||
# define SIMPLE_STRPBRK simple_strpbrk
|
||||
-# define STUPID_STRPBRK stupid_strpbrk
|
||||
# else
|
||||
# include <wchar.h>
|
||||
# define STRPBRK wcspbrk
|
||||
# define SIMPLE_STRPBRK simple_wcspbrk
|
||||
-# define STUPID_STRPBRK stupid_wcspbrk
|
||||
# endif /* WIDE */
|
||||
|
||||
typedef CHAR *(*proto_t) (const CHAR *, const CHAR *);
|
||||
-CHAR *SIMPLE_STRPBRK (const CHAR *, const CHAR *);
|
||||
-CHAR *STUPID_STRPBRK (const CHAR *, const CHAR *);
|
||||
|
||||
-IMPL (STUPID_STRPBRK, 0)
|
||||
-IMPL (SIMPLE_STRPBRK, 0)
|
||||
IMPL (STRPBRK, 1)
|
||||
|
||||
+/* Naive implementation to verify results. */
|
||||
CHAR *
|
||||
SIMPLE_STRPBRK (const CHAR *s, const CHAR *rej)
|
||||
{
|
||||
@@ -73,22 +68,10 @@ SIMPLE_STRPBRK (const CHAR *s, const CHAR *rej)
|
||||
while ((c = *s++) != '\0')
|
||||
for (r = rej; *r != '\0'; ++r)
|
||||
if (*r == c)
|
||||
- return (CHAR *) s - 1;
|
||||
+ return (CHAR *) s - 1;
|
||||
return NULL;
|
||||
}
|
||||
|
||||
-CHAR *
|
||||
-STUPID_STRPBRK (const CHAR *s, const CHAR *rej)
|
||||
-{
|
||||
- size_t ns = STRLEN (s), nrej = STRLEN (rej);
|
||||
- size_t i, j;
|
||||
-
|
||||
- for (i = 0; i < ns; ++i)
|
||||
- for (j = 0; j < nrej; ++j)
|
||||
- if (s[i] == rej[j])
|
||||
- return (CHAR *) s + i;
|
||||
- return NULL;
|
||||
-}
|
||||
#endif /* !STRPBRK_RESULT */
|
||||
|
||||
static void
|
||||
diff --git a/string/test-strrchr.c b/string/test-strrchr.c
|
||||
index 9f85dc681ef075b8..d6018f2324eb8ad0 100644
|
||||
--- a/string/test-strrchr.c
|
||||
+++ b/string/test-strrchr.c
|
||||
@@ -45,11 +45,10 @@
|
||||
#endif
|
||||
|
||||
typedef CHAR *(*proto_t) (const CHAR *, int);
|
||||
-CHAR *SIMPLE_STRRCHR (const CHAR *, int);
|
||||
|
||||
-IMPL (SIMPLE_STRRCHR, 0)
|
||||
IMPL (STRRCHR, 1)
|
||||
|
||||
+/* Naive implementation to verify results. */
|
||||
CHAR *
|
||||
SIMPLE_STRRCHR (const CHAR *s, int c)
|
||||
{
|
||||
diff --git a/string/test-strspn.c b/string/test-strspn.c
|
||||
index ccbed1ca2f6cf8f0..24d79770cde1f28d 100644
|
||||
--- a/string/test-strspn.c
|
||||
+++ b/string/test-strspn.c
|
||||
@@ -30,7 +30,6 @@
|
||||
# define CHAR char
|
||||
# define UCHAR unsigned char
|
||||
# define SIMPLE_STRSPN simple_strspn
|
||||
-# define STUPID_STRSPN stupid_strspn
|
||||
# define STRLEN strlen
|
||||
# define STRCHR strchr
|
||||
# define BIG_CHAR CHAR_MAX
|
||||
@@ -41,7 +40,6 @@
|
||||
# define CHAR wchar_t
|
||||
# define UCHAR wchar_t
|
||||
# define SIMPLE_STRSPN simple_wcsspn
|
||||
-# define STUPID_STRSPN stupid_wcsspn
|
||||
# define STRLEN wcslen
|
||||
# define STRCHR wcschr
|
||||
# define BIG_CHAR WCHAR_MAX
|
||||
@@ -49,13 +47,10 @@
|
||||
#endif /* WIDE */
|
||||
|
||||
typedef size_t (*proto_t) (const CHAR *, const CHAR *);
|
||||
-size_t SIMPLE_STRSPN (const CHAR *, const CHAR *);
|
||||
-size_t STUPID_STRSPN (const CHAR *, const CHAR *);
|
||||
|
||||
-IMPL (STUPID_STRSPN, 0)
|
||||
-IMPL (SIMPLE_STRSPN, 0)
|
||||
IMPL (STRSPN, 1)
|
||||
|
||||
+/* Naive implementation to verify results. */
|
||||
size_t
|
||||
SIMPLE_STRSPN (const CHAR *s, const CHAR *acc)
|
||||
{
|
||||
@@ -73,23 +68,6 @@ SIMPLE_STRSPN (const CHAR *s, const CHAR *acc)
|
||||
return s - str - 1;
|
||||
}
|
||||
|
||||
-size_t
|
||||
-STUPID_STRSPN (const CHAR *s, const CHAR *acc)
|
||||
-{
|
||||
- size_t ns = STRLEN (s), nacc = STRLEN (acc);
|
||||
- size_t i, j;
|
||||
-
|
||||
- for (i = 0; i < ns; ++i)
|
||||
- {
|
||||
- for (j = 0; j < nacc; ++j)
|
||||
- if (s[i] == acc[j])
|
||||
- break;
|
||||
- if (j == nacc)
|
||||
- return i;
|
||||
- }
|
||||
- return i;
|
||||
-}
|
||||
-
|
||||
static void
|
||||
do_one_test (impl_t *impl, const CHAR *s, const CHAR *acc, size_t exp_res)
|
||||
{
|
||||
diff --git a/string/test-strstr.c b/string/test-strstr.c
|
||||
index 2c40d929f27843fa..958c4842af92c1f1 100644
|
||||
--- a/string/test-strstr.c
|
||||
+++ b/string/test-strstr.c
|
||||
@@ -22,14 +22,14 @@
|
||||
#include "test-string.h"
|
||||
|
||||
|
||||
-#define STRSTR simple_strstr
|
||||
+#define STRSTR c_strstr
|
||||
#define libc_hidden_builtin_def(arg) /* nothing */
|
||||
#define __strnlen strnlen
|
||||
#include "strstr.c"
|
||||
|
||||
-
|
||||
+/* Naive implementation to verify results. */
|
||||
static char *
|
||||
-stupid_strstr (const char *s1, const char *s2)
|
||||
+simple_strstr (const char *s1, const char *s2)
|
||||
{
|
||||
ssize_t s1len = strlen (s1);
|
||||
ssize_t s2len = strlen (s2);
|
||||
@@ -53,8 +53,7 @@ stupid_strstr (const char *s1, const char *s2)
|
||||
|
||||
typedef char *(*proto_t) (const char *, const char *);
|
||||
|
||||
-IMPL (stupid_strstr, 0)
|
||||
-IMPL (simple_strstr, 0)
|
||||
+IMPL (c_strstr, 0)
|
||||
IMPL (strstr, 1)
|
||||
|
||||
|
||||
@@ -131,7 +130,7 @@ check1 (void)
|
||||
const char s2[] = "_EF_BF_BD_EF_BF_BD_EF_BF_BD_EF_BF_BD_EF_BF_BD";
|
||||
char *exp_result;
|
||||
|
||||
- exp_result = stupid_strstr (s1, s2);
|
||||
+ exp_result = simple_strstr (s1, s2);
|
||||
FOR_EACH_IMPL (impl, 0)
|
||||
check_result (impl, s1, s2, exp_result);
|
||||
}
|
||||
@@ -164,7 +163,7 @@ check2 (void)
|
||||
char *s2_page_cross = (void *) buf2 + page_size_real - 8;
|
||||
strcpy (s2_page_cross, s2_stack);
|
||||
|
||||
- exp_result = stupid_strstr (s1_stack, s2_stack);
|
||||
+ exp_result = simple_strstr (s1_stack, s2_stack);
|
||||
FOR_EACH_IMPL (impl, 0)
|
||||
{
|
||||
check_result (impl, s1_stack, s2_stack, exp_result);
|
||||
@@ -202,7 +201,7 @@ pr23637 (void)
|
||||
/* Ensure we don't match at the first 'x'. */
|
||||
h[0] = 'x';
|
||||
|
||||
- char *exp_result = stupid_strstr (h, n);
|
||||
+ char *exp_result = simple_strstr (h, n);
|
||||
FOR_EACH_IMPL (impl, 0)
|
||||
check_result (impl, h, n, exp_result);
|
||||
}
|
||||
716
glibc-RHEL-175520-4.patch
Normal file
716
glibc-RHEL-175520-4.patch
Normal file
@ -0,0 +1,716 @@
|
||||
commit 52ab7604db35e0421bc3d2468a3af52b2c513a7b
|
||||
Author: Noah Goldstein <goldstein.w.n@gmail.com>
|
||||
Date: Fri Oct 14 22:00:25 2022 -0500
|
||||
|
||||
x86: Update VEC macros to complete API for evex/evex512 impls
|
||||
|
||||
1) Copy so that backport will be easier.
|
||||
2) Make section only define if there is not a previous definition
|
||||
3) Add `VEC_lo` definition for proper reg-width but in the
|
||||
ymm/zmm0-15 range.
|
||||
4) Add macros for accessing GPRs based on VEC_SIZE
|
||||
This is to make it easier to do think like:
|
||||
```
|
||||
vpcmpb %VEC(0), %VEC(1), %k0
|
||||
kmov{d|q} %k0, %{eax|rax}
|
||||
test %{eax|rax}
|
||||
```
|
||||
It adds macro s.t any GPR can get the proper width with:
|
||||
`V{upcase_GPR_name}`
|
||||
|
||||
and any mask insn can get the proper width with:
|
||||
`{upcase_mask_insn_without_postfix}`
|
||||
|
||||
This commit does not change libc.so
|
||||
|
||||
Tested build on x86-64
|
||||
|
||||
diff --git a/sysdeps/x86_64/multiarch/reg-macros.h b/sysdeps/x86_64/multiarch/reg-macros.h
|
||||
new file mode 100644
|
||||
index 0000000000000000..c8ea330256d00e5c
|
||||
--- /dev/null
|
||||
+++ b/sysdeps/x86_64/multiarch/reg-macros.h
|
||||
@@ -0,0 +1,168 @@
|
||||
+/* This file was generated by: gen-reg-macros.py.
|
||||
+
|
||||
+ Copyright (C) 2022 Free Software Foundation, Inc.
|
||||
+ This file is part of the GNU C Library.
|
||||
+
|
||||
+ The GNU C Library is free software; you can redistribute it and/or
|
||||
+ modify it under the terms of the GNU Lesser General Public
|
||||
+ License as published by the Free Software Foundation; either
|
||||
+ version 2.1 of the License, or (at your option) any later version.
|
||||
+
|
||||
+ The GNU C Library is distributed in the hope that it will be useful,
|
||||
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
+ Lesser General Public License for more details.
|
||||
+
|
||||
+ You should have received a copy of the GNU Lesser General Public
|
||||
+ License along with the GNU C Library; if not, see
|
||||
+ <https://www.gnu.org/licenses/>. */
|
||||
+
|
||||
+#ifndef _REG_MACROS_H
|
||||
+#define _REG_MACROS_H 1
|
||||
+
|
||||
+#define rax_8 al
|
||||
+#define rax_16 ax
|
||||
+#define rax_32 eax
|
||||
+#define rax_64 rax
|
||||
+#define rbx_8 bl
|
||||
+#define rbx_16 bx
|
||||
+#define rbx_32 ebx
|
||||
+#define rbx_64 rbx
|
||||
+#define rcx_8 cl
|
||||
+#define rcx_16 cx
|
||||
+#define rcx_32 ecx
|
||||
+#define rcx_64 rcx
|
||||
+#define rdx_8 dl
|
||||
+#define rdx_16 dx
|
||||
+#define rdx_32 edx
|
||||
+#define rdx_64 rdx
|
||||
+#define rbp_8 bpl
|
||||
+#define rbp_16 bp
|
||||
+#define rbp_32 ebp
|
||||
+#define rbp_64 rbp
|
||||
+#define rsp_8 spl
|
||||
+#define rsp_16 sp
|
||||
+#define rsp_32 esp
|
||||
+#define rsp_64 rsp
|
||||
+#define rsi_8 sil
|
||||
+#define rsi_16 si
|
||||
+#define rsi_32 esi
|
||||
+#define rsi_64 rsi
|
||||
+#define rdi_8 dil
|
||||
+#define rdi_16 di
|
||||
+#define rdi_32 edi
|
||||
+#define rdi_64 rdi
|
||||
+#define r8_8 r8b
|
||||
+#define r8_16 r8w
|
||||
+#define r8_32 r8d
|
||||
+#define r8_64 r8
|
||||
+#define r9_8 r9b
|
||||
+#define r9_16 r9w
|
||||
+#define r9_32 r9d
|
||||
+#define r9_64 r9
|
||||
+#define r10_8 r10b
|
||||
+#define r10_16 r10w
|
||||
+#define r10_32 r10d
|
||||
+#define r10_64 r10
|
||||
+#define r11_8 r11b
|
||||
+#define r11_16 r11w
|
||||
+#define r11_32 r11d
|
||||
+#define r11_64 r11
|
||||
+#define r12_8 r12b
|
||||
+#define r12_16 r12w
|
||||
+#define r12_32 r12d
|
||||
+#define r12_64 r12
|
||||
+#define r13_8 r13b
|
||||
+#define r13_16 r13w
|
||||
+#define r13_32 r13d
|
||||
+#define r13_64 r13
|
||||
+#define r14_8 r14b
|
||||
+#define r14_16 r14w
|
||||
+#define r14_32 r14d
|
||||
+#define r14_64 r14
|
||||
+#define r15_8 r15b
|
||||
+#define r15_16 r15w
|
||||
+#define r15_32 r15d
|
||||
+#define r15_64 r15
|
||||
+
|
||||
+#define kmov_8 kmovb
|
||||
+#define kmov_16 kmovw
|
||||
+#define kmov_32 kmovd
|
||||
+#define kmov_64 kmovq
|
||||
+#define kortest_8 kortestb
|
||||
+#define kortest_16 kortestw
|
||||
+#define kortest_32 kortestd
|
||||
+#define kortest_64 kortestq
|
||||
+#define kor_8 korb
|
||||
+#define kor_16 korw
|
||||
+#define kor_32 kord
|
||||
+#define kor_64 korq
|
||||
+#define ktest_8 ktestb
|
||||
+#define ktest_16 ktestw
|
||||
+#define ktest_32 ktestd
|
||||
+#define ktest_64 ktestq
|
||||
+#define kand_8 kandb
|
||||
+#define kand_16 kandw
|
||||
+#define kand_32 kandd
|
||||
+#define kand_64 kandq
|
||||
+#define kxor_8 kxorb
|
||||
+#define kxor_16 kxorw
|
||||
+#define kxor_32 kxord
|
||||
+#define kxor_64 kxorq
|
||||
+#define knot_8 knotb
|
||||
+#define knot_16 knotw
|
||||
+#define knot_32 knotd
|
||||
+#define knot_64 knotq
|
||||
+#define kxnor_8 kxnorb
|
||||
+#define kxnor_16 kxnorw
|
||||
+#define kxnor_32 kxnord
|
||||
+#define kxnor_64 kxnorq
|
||||
+#define kunpack_8 kunpackbw
|
||||
+#define kunpack_16 kunpackwd
|
||||
+#define kunpack_32 kunpackdq
|
||||
+
|
||||
+/* Common API for accessing proper width GPR is V{upcase_GPR_name}. */
|
||||
+#define VRAX VGPR(rax)
|
||||
+#define VRBX VGPR(rbx)
|
||||
+#define VRCX VGPR(rcx)
|
||||
+#define VRDX VGPR(rdx)
|
||||
+#define VRBP VGPR(rbp)
|
||||
+#define VRSP VGPR(rsp)
|
||||
+#define VRSI VGPR(rsi)
|
||||
+#define VRDI VGPR(rdi)
|
||||
+#define VR8 VGPR(r8)
|
||||
+#define VR9 VGPR(r9)
|
||||
+#define VR10 VGPR(r10)
|
||||
+#define VR11 VGPR(r11)
|
||||
+#define VR12 VGPR(r12)
|
||||
+#define VR13 VGPR(r13)
|
||||
+#define VR14 VGPR(r14)
|
||||
+#define VR15 VGPR(r15)
|
||||
+
|
||||
+/* Common API for accessing proper width mask insn is {upcase_mask_insn}. */
|
||||
+#define KMOV VKINSN(kmov)
|
||||
+#define KORTEST VKINSN(kortest)
|
||||
+#define KOR VKINSN(kor)
|
||||
+#define KTEST VKINSN(ktest)
|
||||
+#define KAND VKINSN(kand)
|
||||
+#define KXOR VKINSN(kxor)
|
||||
+#define KNOT VKINSN(knot)
|
||||
+#define KXNOR VKINSN(kxnor)
|
||||
+#define KUNPACK VKINSN(kunpack)
|
||||
+
|
||||
+#ifdef USE_WIDE_CHAR
|
||||
+# define REG_WIDTH 32
|
||||
+#else
|
||||
+# define REG_WIDTH VEC_SIZE
|
||||
+#endif
|
||||
+
|
||||
+#define VPASTER(x, y) x##_##y
|
||||
+#define VEVALUATOR(x, y) VPASTER(x, y)
|
||||
+
|
||||
+#define VGPR_SZ(reg_name, reg_size) VEVALUATOR(reg_name, reg_size)
|
||||
+#define VKINSN_SZ(insn, reg_size) VEVALUATOR(insn, reg_size)
|
||||
+
|
||||
+#define VGPR(reg_name) VGPR_SZ(reg_name, REG_WIDTH)
|
||||
+#define VKINSN(mask_insn) VKINSN_SZ(mask_insn, REG_WIDTH)
|
||||
+
|
||||
+#endif
|
||||
diff --git a/sysdeps/x86_64/multiarch/scripts/gen-reg-macros.py b/sysdeps/x86_64/multiarch/scripts/gen-reg-macros.py
|
||||
new file mode 100644
|
||||
index 0000000000000000..9fb6903212492034
|
||||
--- /dev/null
|
||||
+++ b/sysdeps/x86_64/multiarch/scripts/gen-reg-macros.py
|
||||
@@ -0,0 +1,133 @@
|
||||
+#!/usr/bin/python3
|
||||
+# Copyright (C) 2022 Free Software Foundation, Inc.
|
||||
+# This file is part of the GNU C Library.
|
||||
+#
|
||||
+# The GNU C Library is free software; you can redistribute it and/or
|
||||
+# modify it under the terms of the GNU Lesser General Public
|
||||
+# License as published by the Free Software Foundation; either
|
||||
+# version 2.1 of the License, or (at your option) any later version.
|
||||
+#
|
||||
+# The GNU C Library is distributed in the hope that it will be useful,
|
||||
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
+# Lesser General Public License for more details.
|
||||
+#
|
||||
+# You should have received a copy of the GNU Lesser General Public
|
||||
+# License along with the GNU C Library; if not, see
|
||||
+# <https://www.gnu.org/licenses/>.
|
||||
+"""Generate macros for getting GPR name of a certain size
|
||||
+
|
||||
+Inputs: None
|
||||
+Output: Prints header fill to stdout
|
||||
+
|
||||
+API:
|
||||
+ V{upcase_GPR_name}
|
||||
+ - Get register name REG_WIDTH component of `upcase_GPR_name`
|
||||
+ {upcase_mask_insn_without_postfix}
|
||||
+ - Get proper REG_WIDTH mask insn for `upcase_mask_insn_without_postfix`
|
||||
+ VGPR(reg_name)
|
||||
+ - Get register name REG_WIDTH component of `reg_name`
|
||||
+ VKINSN(mask_insn)
|
||||
+ - Get proper REG_WIDTH mask insn for `mask_insn`
|
||||
+ VGPR_SZ(reg_name, reg_size)
|
||||
+ - Get register name `reg_size` component of `reg_name`
|
||||
+ VKINSN_SZ(mask_insn, insn_size)
|
||||
+ - Get proper `insn_size` mask insn for `mask_insn`
|
||||
+"""
|
||||
+
|
||||
+import sys
|
||||
+import os
|
||||
+from datetime import datetime
|
||||
+
|
||||
+registers = [["rax", "eax", "ax", "al"], ["rbx", "ebx", "bx", "bl"],
|
||||
+ ["rcx", "ecx", "cx", "cl"], ["rdx", "edx", "dx", "dl"],
|
||||
+ ["rbp", "ebp", "bp", "bpl"], ["rsp", "esp", "sp", "spl"],
|
||||
+ ["rsi", "esi", "si", "sil"], ["rdi", "edi", "di", "dil"],
|
||||
+ ["r8", "r8d", "r8w", "r8b"], ["r9", "r9d", "r9w", "r9b"],
|
||||
+ ["r10", "r10d", "r10w", "r10b"], ["r11", "r11d", "r11w", "r11b"],
|
||||
+ ["r12", "r12d", "r12w", "r12b"], ["r13", "r13d", "r13w", "r13b"],
|
||||
+ ["r14", "r14d", "r14w", "r14b"], ["r15", "r15d", "r15w", "r15b"]]
|
||||
+
|
||||
+mask_insns = [
|
||||
+ "kmov",
|
||||
+ "kortest",
|
||||
+ "kor",
|
||||
+ "ktest",
|
||||
+ "kand",
|
||||
+ "kxor",
|
||||
+ "knot",
|
||||
+ "kxnor",
|
||||
+]
|
||||
+mask_insns_ext = ["b", "w", "d", "q"]
|
||||
+
|
||||
+cr = """
|
||||
+ Copyright (C) {} Free Software Foundation, Inc.
|
||||
+ This file is part of the GNU C Library.
|
||||
+
|
||||
+ The GNU C Library is free software; you can redistribute it and/or
|
||||
+ modify it under the terms of the GNU Lesser General Public
|
||||
+ License as published by the Free Software Foundation; either
|
||||
+ version 2.1 of the License, or (at your option) any later version.
|
||||
+
|
||||
+ The GNU C Library is distributed in the hope that it will be useful,
|
||||
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
+ Lesser General Public License for more details.
|
||||
+
|
||||
+ You should have received a copy of the GNU Lesser General Public
|
||||
+ License along with the GNU C Library; if not, see
|
||||
+ <https://www.gnu.org/licenses/>. */
|
||||
+"""
|
||||
+
|
||||
+print("/* This file was generated by: {}.".format(os.path.basename(
|
||||
+ sys.argv[0])))
|
||||
+print(cr.format(datetime.today().year))
|
||||
+
|
||||
+print("#ifndef _REG_MACROS_H")
|
||||
+print("#define _REG_MACROS_H\t1")
|
||||
+print("")
|
||||
+for reg in registers:
|
||||
+ for i in range(0, 4):
|
||||
+ print("#define {}_{}\t{}".format(reg[0], 8 << i, reg[3 - i]))
|
||||
+
|
||||
+print("")
|
||||
+for mask_insn in mask_insns:
|
||||
+ for i in range(0, 4):
|
||||
+ print("#define {}_{}\t{}{}".format(mask_insn, 8 << i, mask_insn,
|
||||
+ mask_insns_ext[i]))
|
||||
+for i in range(0, 3):
|
||||
+ print("#define kunpack_{}\tkunpack{}{}".format(8 << i, mask_insns_ext[i],
|
||||
+ mask_insns_ext[i + 1]))
|
||||
+mask_insns.append("kunpack")
|
||||
+
|
||||
+print("")
|
||||
+print(
|
||||
+ "/* Common API for accessing proper width GPR is V{upcase_GPR_name}. */")
|
||||
+for reg in registers:
|
||||
+ print("#define V{}\tVGPR({})".format(reg[0].upper(), reg[0]))
|
||||
+
|
||||
+print("")
|
||||
+
|
||||
+print(
|
||||
+ "/* Common API for accessing proper width mask insn is {upcase_mask_insn}. */"
|
||||
+)
|
||||
+for mask_insn in mask_insns:
|
||||
+ print("#define {} \tVKINSN({})".format(mask_insn.upper(), mask_insn))
|
||||
+print("")
|
||||
+
|
||||
+print("#ifdef USE_WIDE_CHAR")
|
||||
+print("# define REG_WIDTH 32")
|
||||
+print("#else")
|
||||
+print("# define REG_WIDTH VEC_SIZE")
|
||||
+print("#endif")
|
||||
+print("")
|
||||
+print("#define VPASTER(x, y)\tx##_##y")
|
||||
+print("#define VEVALUATOR(x, y)\tVPASTER(x, y)")
|
||||
+print("")
|
||||
+print("#define VGPR_SZ(reg_name, reg_size)\tVEVALUATOR(reg_name, reg_size)")
|
||||
+print("#define VKINSN_SZ(insn, reg_size)\tVEVALUATOR(insn, reg_size)")
|
||||
+print("")
|
||||
+print("#define VGPR(reg_name)\tVGPR_SZ(reg_name, REG_WIDTH)")
|
||||
+print("#define VKINSN(mask_insn)\tVKINSN_SZ(mask_insn, REG_WIDTH)")
|
||||
+
|
||||
+print("\n#endif")
|
||||
diff --git a/sysdeps/x86_64/multiarch/x86-avx-rtm-vecs.h b/sysdeps/x86_64/multiarch/x86-avx-rtm-vecs.h
|
||||
new file mode 100644
|
||||
index 0000000000000000..0b326c8a703eabb3
|
||||
--- /dev/null
|
||||
+++ b/sysdeps/x86_64/multiarch/x86-avx-rtm-vecs.h
|
||||
@@ -0,0 +1,35 @@
|
||||
+/* Common config for AVX-RTM VECs
|
||||
+ All versions must be listed in ifunc-impl-list.c.
|
||||
+ Copyright (C) 2022 Free Software Foundation, Inc.
|
||||
+ This file is part of the GNU C Library.
|
||||
+
|
||||
+ The GNU C Library is free software; you can redistribute it and/or
|
||||
+ modify it under the terms of the GNU Lesser General Public
|
||||
+ License as published by the Free Software Foundation; either
|
||||
+ version 2.1 of the License, or (at your option) any later version.
|
||||
+
|
||||
+ The GNU C Library is distributed in the hope that it will be useful,
|
||||
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
+ Lesser General Public License for more details.
|
||||
+
|
||||
+ You should have received a copy of the GNU Lesser General Public
|
||||
+ License along with the GNU C Library; if not, see
|
||||
+ <https://www.gnu.org/licenses/>. */
|
||||
+
|
||||
+#ifndef _X86_AVX_RTM_VECS_H
|
||||
+#define _X86_AVX_RTM_VECS_H 1
|
||||
+
|
||||
+#define COND_VZEROUPPER COND_VZEROUPPER_XTEST
|
||||
+#define ZERO_UPPER_VEC_REGISTERS_RETURN \
|
||||
+ ZERO_UPPER_VEC_REGISTERS_RETURN_XTEST
|
||||
+
|
||||
+#define VZEROUPPER_RETURN jmp L(return_vzeroupper)
|
||||
+
|
||||
+#define USE_WITH_RTM 1
|
||||
+#include "x86-avx-vecs.h"
|
||||
+
|
||||
+#undef SECTION
|
||||
+#define SECTION(p) p##.avx.rtm
|
||||
+
|
||||
+#endif
|
||||
diff --git a/sysdeps/x86_64/multiarch/x86-avx-vecs.h b/sysdeps/x86_64/multiarch/x86-avx-vecs.h
|
||||
new file mode 100644
|
||||
index 0000000000000000..dca1089060eeba93
|
||||
--- /dev/null
|
||||
+++ b/sysdeps/x86_64/multiarch/x86-avx-vecs.h
|
||||
@@ -0,0 +1,47 @@
|
||||
+/* Common config for AVX VECs
|
||||
+ All versions must be listed in ifunc-impl-list.c.
|
||||
+ Copyright (C) 2022 Free Software Foundation, Inc.
|
||||
+ This file is part of the GNU C Library.
|
||||
+
|
||||
+ The GNU C Library is free software; you can redistribute it and/or
|
||||
+ modify it under the terms of the GNU Lesser General Public
|
||||
+ License as published by the Free Software Foundation; either
|
||||
+ version 2.1 of the License, or (at your option) any later version.
|
||||
+
|
||||
+ The GNU C Library is distributed in the hope that it will be useful,
|
||||
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
+ Lesser General Public License for more details.
|
||||
+
|
||||
+ You should have received a copy of the GNU Lesser General Public
|
||||
+ License along with the GNU C Library; if not, see
|
||||
+ <https://www.gnu.org/licenses/>. */
|
||||
+
|
||||
+#ifndef _X86_AVX_VECS_H
|
||||
+#define _X86_AVX_VECS_H 1
|
||||
+
|
||||
+#ifdef VEC_SIZE
|
||||
+# error "Multiple VEC configs included!"
|
||||
+#endif
|
||||
+
|
||||
+#define VEC_SIZE 32
|
||||
+#include "x86-vec-macros.h"
|
||||
+
|
||||
+#define USE_WITH_AVX 1
|
||||
+#define SECTION(p) p##.avx
|
||||
+
|
||||
+/* 4-byte mov instructions with AVX2. */
|
||||
+#define MOV_SIZE 4
|
||||
+/* 1 (ret) + 3 (vzeroupper). */
|
||||
+#define RET_SIZE 4
|
||||
+#define VZEROUPPER vzeroupper
|
||||
+
|
||||
+#define VMOVU vmovdqu
|
||||
+#define VMOVA vmovdqa
|
||||
+#define VMOVNT vmovntdq
|
||||
+
|
||||
+/* Often need to access xmm portion. */
|
||||
+#define VMM_128 VMM_any_xmm
|
||||
+#define VMM VMM_any_ymm
|
||||
+
|
||||
+#endif
|
||||
diff --git a/sysdeps/x86_64/multiarch/x86-evex-vecs-common.h b/sysdeps/x86_64/multiarch/x86-evex-vecs-common.h
|
||||
new file mode 100644
|
||||
index 0000000000000000..f331e9d8ecfc371e
|
||||
--- /dev/null
|
||||
+++ b/sysdeps/x86_64/multiarch/x86-evex-vecs-common.h
|
||||
@@ -0,0 +1,39 @@
|
||||
+/* Common config for EVEX256 and EVEX512 VECs
|
||||
+ All versions must be listed in ifunc-impl-list.c.
|
||||
+ Copyright (C) 2022 Free Software Foundation, Inc.
|
||||
+ This file is part of the GNU C Library.
|
||||
+
|
||||
+ The GNU C Library is free software; you can redistribute it and/or
|
||||
+ modify it under the terms of the GNU Lesser General Public
|
||||
+ License as published by the Free Software Foundation; either
|
||||
+ version 2.1 of the License, or (at your option) any later version.
|
||||
+
|
||||
+ The GNU C Library is distributed in the hope that it will be useful,
|
||||
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
+ Lesser General Public License for more details.
|
||||
+
|
||||
+ You should have received a copy of the GNU Lesser General Public
|
||||
+ License along with the GNU C Library; if not, see
|
||||
+ <https://www.gnu.org/licenses/>. */
|
||||
+
|
||||
+#ifndef _X86_EVEX_VECS_COMMON_H
|
||||
+#define _X86_EVEX_VECS_COMMON_H 1
|
||||
+
|
||||
+#include "x86-vec-macros.h"
|
||||
+
|
||||
+/* 6-byte mov instructions with EVEX. */
|
||||
+#define MOV_SIZE 6
|
||||
+/* No vzeroupper needed. */
|
||||
+#define RET_SIZE 1
|
||||
+#define VZEROUPPER
|
||||
+
|
||||
+#define VMOVU vmovdqu64
|
||||
+#define VMOVA vmovdqa64
|
||||
+#define VMOVNT vmovntdq
|
||||
+
|
||||
+#define VMM_128 VMM_hi_xmm
|
||||
+#define VMM_256 VMM_hi_ymm
|
||||
+#define VMM_512 VMM_hi_zmm
|
||||
+
|
||||
+#endif
|
||||
diff --git a/sysdeps/x86_64/multiarch/x86-evex256-vecs.h b/sysdeps/x86_64/multiarch/x86-evex256-vecs.h
|
||||
new file mode 100644
|
||||
index 0000000000000000..8337b955045deb64
|
||||
--- /dev/null
|
||||
+++ b/sysdeps/x86_64/multiarch/x86-evex256-vecs.h
|
||||
@@ -0,0 +1,38 @@
|
||||
+/* Common config for EVEX256 VECs
|
||||
+ All versions must be listed in ifunc-impl-list.c.
|
||||
+ Copyright (C) 2022 Free Software Foundation, Inc.
|
||||
+ This file is part of the GNU C Library.
|
||||
+
|
||||
+ The GNU C Library is free software; you can redistribute it and/or
|
||||
+ modify it under the terms of the GNU Lesser General Public
|
||||
+ License as published by the Free Software Foundation; either
|
||||
+ version 2.1 of the License, or (at your option) any later version.
|
||||
+
|
||||
+ The GNU C Library is distributed in the hope that it will be useful,
|
||||
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
+ Lesser General Public License for more details.
|
||||
+
|
||||
+ You should have received a copy of the GNU Lesser General Public
|
||||
+ License along with the GNU C Library; if not, see
|
||||
+ <https://www.gnu.org/licenses/>. */
|
||||
+
|
||||
+#ifndef _EVEX256_VECS_H
|
||||
+#define _EVEX256_VECS_H 1
|
||||
+
|
||||
+#ifdef VEC_SIZE
|
||||
+# error "Multiple VEC configs included!"
|
||||
+#endif
|
||||
+
|
||||
+#define VEC_SIZE 32
|
||||
+#include "x86-evex-vecs-common.h"
|
||||
+
|
||||
+#define USE_WITH_EVEX256 1
|
||||
+
|
||||
+#ifndef SECTION
|
||||
+# define SECTION(p) p##.evex
|
||||
+#endif
|
||||
+
|
||||
+#define VMM VMM_256
|
||||
+#define VMM_lo VMM_any_ymm
|
||||
+#endif
|
||||
diff --git a/sysdeps/x86_64/multiarch/x86-evex512-vecs.h b/sysdeps/x86_64/multiarch/x86-evex512-vecs.h
|
||||
new file mode 100644
|
||||
index 0000000000000000..7dc5c23ad04e6128
|
||||
--- /dev/null
|
||||
+++ b/sysdeps/x86_64/multiarch/x86-evex512-vecs.h
|
||||
@@ -0,0 +1,38 @@
|
||||
+/* Common config for EVEX512 VECs
|
||||
+ All versions must be listed in ifunc-impl-list.c.
|
||||
+ Copyright (C) 2022 Free Software Foundation, Inc.
|
||||
+ This file is part of the GNU C Library.
|
||||
+
|
||||
+ The GNU C Library is free software; you can redistribute it and/or
|
||||
+ modify it under the terms of the GNU Lesser General Public
|
||||
+ License as published by the Free Software Foundation; either
|
||||
+ version 2.1 of the License, or (at your option) any later version.
|
||||
+
|
||||
+ The GNU C Library is distributed in the hope that it will be useful,
|
||||
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
+ Lesser General Public License for more details.
|
||||
+
|
||||
+ You should have received a copy of the GNU Lesser General Public
|
||||
+ License along with the GNU C Library; if not, see
|
||||
+ <https://www.gnu.org/licenses/>. */
|
||||
+
|
||||
+#ifndef _EVEX512_VECS_H
|
||||
+#define _EVEX512_VECS_H 1
|
||||
+
|
||||
+#ifdef VEC_SIZE
|
||||
+# error "Multiple VEC configs included!"
|
||||
+#endif
|
||||
+
|
||||
+#define VEC_SIZE 64
|
||||
+#include "x86-evex-vecs-common.h"
|
||||
+
|
||||
+#define USE_WITH_EVEX512 1
|
||||
+
|
||||
+#ifndef SECTION
|
||||
+# define SECTION(p) p##.evex512
|
||||
+#endif
|
||||
+
|
||||
+#define VMM VMM_512
|
||||
+#define VMM_lo VMM_any_zmm
|
||||
+#endif
|
||||
diff --git a/sysdeps/x86_64/multiarch/x86-sse2-vecs.h b/sysdeps/x86_64/multiarch/x86-sse2-vecs.h
|
||||
new file mode 100644
|
||||
index 0000000000000000..b8bbd5dc29cf2975
|
||||
--- /dev/null
|
||||
+++ b/sysdeps/x86_64/multiarch/x86-sse2-vecs.h
|
||||
@@ -0,0 +1,47 @@
|
||||
+/* Common config for SSE2 VECs
|
||||
+ All versions must be listed in ifunc-impl-list.c.
|
||||
+ Copyright (C) 2022 Free Software Foundation, Inc.
|
||||
+ This file is part of the GNU C Library.
|
||||
+
|
||||
+ The GNU C Library is free software; you can redistribute it and/or
|
||||
+ modify it under the terms of the GNU Lesser General Public
|
||||
+ License as published by the Free Software Foundation; either
|
||||
+ version 2.1 of the License, or (at your option) any later version.
|
||||
+
|
||||
+ The GNU C Library is distributed in the hope that it will be useful,
|
||||
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
+ Lesser General Public License for more details.
|
||||
+
|
||||
+ You should have received a copy of the GNU Lesser General Public
|
||||
+ License along with the GNU C Library; if not, see
|
||||
+ <https://www.gnu.org/licenses/>. */
|
||||
+
|
||||
+#ifndef _X86_SSE2_VECS_H
|
||||
+#define _X86_SSE2_VECS_H 1
|
||||
+
|
||||
+#ifdef VEC_SIZE
|
||||
+# error "Multiple VEC configs included!"
|
||||
+#endif
|
||||
+
|
||||
+#define VEC_SIZE 16
|
||||
+#include "x86-vec-macros.h"
|
||||
+
|
||||
+#define USE_WITH_SSE2 1
|
||||
+#define SECTION(p) p
|
||||
+
|
||||
+/* 3-byte mov instructions with SSE2. */
|
||||
+#define MOV_SIZE 3
|
||||
+/* No vzeroupper needed. */
|
||||
+#define RET_SIZE 1
|
||||
+#define VZEROUPPER
|
||||
+
|
||||
+#define VMOVU movups
|
||||
+#define VMOVA movaps
|
||||
+#define VMOVNT movntdq
|
||||
+
|
||||
+#define VMM_128 VMM_any_xmm
|
||||
+#define VMM VMM_any_xmm
|
||||
+
|
||||
+
|
||||
+#endif
|
||||
diff --git a/sysdeps/x86_64/multiarch/x86-vec-macros.h b/sysdeps/x86_64/multiarch/x86-vec-macros.h
|
||||
new file mode 100644
|
||||
index 0000000000000000..7d6bb31d55f1a0d1
|
||||
--- /dev/null
|
||||
+++ b/sysdeps/x86_64/multiarch/x86-vec-macros.h
|
||||
@@ -0,0 +1,90 @@
|
||||
+/* Macro helpers for VEC_{type}({vec_num})
|
||||
+ All versions must be listed in ifunc-impl-list.c.
|
||||
+ Copyright (C) 2022 Free Software Foundation, Inc.
|
||||
+ This file is part of the GNU C Library.
|
||||
+
|
||||
+ The GNU C Library is free software; you can redistribute it and/or
|
||||
+ modify it under the terms of the GNU Lesser General Public
|
||||
+ License as published by the Free Software Foundation; either
|
||||
+ version 2.1 of the License, or (at your option) any later version.
|
||||
+
|
||||
+ The GNU C Library is distributed in the hope that it will be useful,
|
||||
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
+ Lesser General Public License for more details.
|
||||
+
|
||||
+ You should have received a copy of the GNU Lesser General Public
|
||||
+ License along with the GNU C Library; if not, see
|
||||
+ <https://www.gnu.org/licenses/>. */
|
||||
+
|
||||
+#ifndef _X86_VEC_MACROS_H
|
||||
+#define _X86_VEC_MACROS_H 1
|
||||
+
|
||||
+#ifndef VEC_SIZE
|
||||
+# error "Never include this file directly. Always include a vector config."
|
||||
+#endif
|
||||
+
|
||||
+/* Defines so we can use SSE2 / AVX2 / EVEX / EVEX512 encoding with same
|
||||
+ VMM(N) values. */
|
||||
+#define VMM_hi_xmm0 xmm16
|
||||
+#define VMM_hi_xmm1 xmm17
|
||||
+#define VMM_hi_xmm2 xmm18
|
||||
+#define VMM_hi_xmm3 xmm19
|
||||
+#define VMM_hi_xmm4 xmm20
|
||||
+#define VMM_hi_xmm5 xmm21
|
||||
+#define VMM_hi_xmm6 xmm22
|
||||
+#define VMM_hi_xmm7 xmm23
|
||||
+#define VMM_hi_xmm8 xmm24
|
||||
+#define VMM_hi_xmm9 xmm25
|
||||
+#define VMM_hi_xmm10 xmm26
|
||||
+#define VMM_hi_xmm11 xmm27
|
||||
+#define VMM_hi_xmm12 xmm28
|
||||
+#define VMM_hi_xmm13 xmm29
|
||||
+#define VMM_hi_xmm14 xmm30
|
||||
+#define VMM_hi_xmm15 xmm31
|
||||
+
|
||||
+#define VMM_hi_ymm0 ymm16
|
||||
+#define VMM_hi_ymm1 ymm17
|
||||
+#define VMM_hi_ymm2 ymm18
|
||||
+#define VMM_hi_ymm3 ymm19
|
||||
+#define VMM_hi_ymm4 ymm20
|
||||
+#define VMM_hi_ymm5 ymm21
|
||||
+#define VMM_hi_ymm6 ymm22
|
||||
+#define VMM_hi_ymm7 ymm23
|
||||
+#define VMM_hi_ymm8 ymm24
|
||||
+#define VMM_hi_ymm9 ymm25
|
||||
+#define VMM_hi_ymm10 ymm26
|
||||
+#define VMM_hi_ymm11 ymm27
|
||||
+#define VMM_hi_ymm12 ymm28
|
||||
+#define VMM_hi_ymm13 ymm29
|
||||
+#define VMM_hi_ymm14 ymm30
|
||||
+#define VMM_hi_ymm15 ymm31
|
||||
+
|
||||
+#define VMM_hi_zmm0 zmm16
|
||||
+#define VMM_hi_zmm1 zmm17
|
||||
+#define VMM_hi_zmm2 zmm18
|
||||
+#define VMM_hi_zmm3 zmm19
|
||||
+#define VMM_hi_zmm4 zmm20
|
||||
+#define VMM_hi_zmm5 zmm21
|
||||
+#define VMM_hi_zmm6 zmm22
|
||||
+#define VMM_hi_zmm7 zmm23
|
||||
+#define VMM_hi_zmm8 zmm24
|
||||
+#define VMM_hi_zmm9 zmm25
|
||||
+#define VMM_hi_zmm10 zmm26
|
||||
+#define VMM_hi_zmm11 zmm27
|
||||
+#define VMM_hi_zmm12 zmm28
|
||||
+#define VMM_hi_zmm13 zmm29
|
||||
+#define VMM_hi_zmm14 zmm30
|
||||
+#define VMM_hi_zmm15 zmm31
|
||||
+
|
||||
+#define PRIMITIVE_VMM(vec, num) vec##num
|
||||
+
|
||||
+#define VMM_any_xmm(i) PRIMITIVE_VMM(xmm, i)
|
||||
+#define VMM_any_ymm(i) PRIMITIVE_VMM(ymm, i)
|
||||
+#define VMM_any_zmm(i) PRIMITIVE_VMM(zmm, i)
|
||||
+
|
||||
+#define VMM_hi_xmm(i) PRIMITIVE_VMM(VMM_hi_xmm, i)
|
||||
+#define VMM_hi_ymm(i) PRIMITIVE_VMM(VMM_hi_ymm, i)
|
||||
+#define VMM_hi_zmm(i) PRIMITIVE_VMM(VMM_hi_zmm, i)
|
||||
+
|
||||
+#endif
|
||||
264
glibc-RHEL-175520-5.patch
Normal file
264
glibc-RHEL-175520-5.patch
Normal file
@ -0,0 +1,264 @@
|
||||
commit be066536bd313f1eec6e36fb92a96d39bf76f483
|
||||
Author: Noah Goldstein <goldstein.w.n@gmail.com>
|
||||
Date: Fri Oct 14 22:00:30 2022 -0500
|
||||
|
||||
x86: Update strlen-evex-base to use new reg/vec macros.
|
||||
|
||||
To avoid duplicate the VMM / GPR / mask insn macros in all incoming
|
||||
evex512 files use the macros defined in 'reg-macros.h' and
|
||||
'{vec}-macros.h'
|
||||
|
||||
This commit does not change libc.so
|
||||
|
||||
Tested build on x86-64
|
||||
|
||||
diff --git a/sysdeps/x86_64/multiarch/strlen-evex-base.S b/sysdeps/x86_64/multiarch/strlen-evex-base.S
|
||||
index 278c899691d89ba7..176babee1e0a9e89 100644
|
||||
--- a/sysdeps/x86_64/multiarch/strlen-evex-base.S
|
||||
+++ b/sysdeps/x86_64/multiarch/strlen-evex-base.S
|
||||
@@ -32,42 +32,10 @@
|
||||
# define CHAR_SIZE 1
|
||||
# endif
|
||||
|
||||
-# define XMM0 xmm16
|
||||
# define PAGE_SIZE 4096
|
||||
# define CHAR_PER_VEC (VEC_SIZE / CHAR_SIZE)
|
||||
|
||||
-# if VEC_SIZE == 64
|
||||
-# define KMOV kmovq
|
||||
-# define KORTEST kortestq
|
||||
-# define RAX rax
|
||||
-# define RCX rcx
|
||||
-# define RDX rdx
|
||||
-# define SHR shrq
|
||||
-# define TEXTSUFFIX evex512
|
||||
-# define VMM0 zmm16
|
||||
-# define VMM1 zmm17
|
||||
-# define VMM2 zmm18
|
||||
-# define VMM3 zmm19
|
||||
-# define VMM4 zmm20
|
||||
-# define VMOVA vmovdqa64
|
||||
-# elif VEC_SIZE == 32
|
||||
-/* Currently Unused. */
|
||||
-# define KMOV kmovd
|
||||
-# define KORTEST kortestd
|
||||
-# define RAX eax
|
||||
-# define RCX ecx
|
||||
-# define RDX edx
|
||||
-# define SHR shrl
|
||||
-# define TEXTSUFFIX evex256
|
||||
-# define VMM0 ymm16
|
||||
-# define VMM1 ymm17
|
||||
-# define VMM2 ymm18
|
||||
-# define VMM3 ymm19
|
||||
-# define VMM4 ymm20
|
||||
-# define VMOVA vmovdqa32
|
||||
-# endif
|
||||
-
|
||||
- .section .text.TEXTSUFFIX, "ax", @progbits
|
||||
+ .section SECTION(.text),"ax",@progbits
|
||||
/* Aligning entry point to 64 byte, provides better performance for
|
||||
one vector length string. */
|
||||
ENTRY_P2ALIGN (STRLEN, 6)
|
||||
@@ -82,18 +50,18 @@ ENTRY_P2ALIGN (STRLEN, 6)
|
||||
# endif
|
||||
|
||||
movl %edi, %eax
|
||||
- vpxorq %XMM0, %XMM0, %XMM0
|
||||
+ vpxorq %VMM_128(0), %VMM_128(0), %VMM_128(0)
|
||||
andl $(PAGE_SIZE - 1), %eax
|
||||
cmpl $(PAGE_SIZE - VEC_SIZE), %eax
|
||||
ja L(page_cross)
|
||||
|
||||
/* Compare [w]char for null, mask bit will be set for match. */
|
||||
- VPCMP $0, (%rdi), %VMM0, %k0
|
||||
- KMOV %k0, %RAX
|
||||
- test %RAX, %RAX
|
||||
+ VPCMP $0, (%rdi), %VMM(0), %k0
|
||||
+ KMOV %k0, %VRAX
|
||||
+ test %VRAX, %VRAX
|
||||
jz L(align_more)
|
||||
|
||||
- bsf %RAX, %RAX
|
||||
+ bsf %VRAX, %VRAX
|
||||
# ifdef USE_AS_STRNLEN
|
||||
cmpq %rsi, %rax
|
||||
cmovnb %rsi, %rax
|
||||
@@ -116,7 +84,7 @@ L(align_more):
|
||||
movq %rax, %rdx
|
||||
subq %rdi, %rdx
|
||||
# ifdef USE_AS_WCSLEN
|
||||
- SHR $2, %RDX
|
||||
+ shr $2, %VRDX
|
||||
# endif
|
||||
/* At this point rdx contains [w]chars already compared. */
|
||||
subq %rsi, %rdx
|
||||
@@ -127,9 +95,9 @@ L(align_more):
|
||||
# endif
|
||||
|
||||
/* Loop unroll 4 times for 4 vector loop. */
|
||||
- VPCMP $0, (%rax), %VMM0, %k0
|
||||
- KMOV %k0, %RCX
|
||||
- test %RCX, %RCX
|
||||
+ VPCMP $0, (%rax), %VMM(0), %k0
|
||||
+ KMOV %k0, %VRCX
|
||||
+ test %VRCX, %VRCX
|
||||
jnz L(ret_vec_x1)
|
||||
|
||||
# ifdef USE_AS_STRNLEN
|
||||
@@ -137,9 +105,9 @@ L(align_more):
|
||||
jbe L(ret_max)
|
||||
# endif
|
||||
|
||||
- VPCMP $0, VEC_SIZE(%rax), %VMM0, %k0
|
||||
- KMOV %k0, %RCX
|
||||
- test %RCX, %RCX
|
||||
+ VPCMP $0, VEC_SIZE(%rax), %VMM(0), %k0
|
||||
+ KMOV %k0, %VRCX
|
||||
+ test %VRCX, %VRCX
|
||||
jnz L(ret_vec_x2)
|
||||
|
||||
# ifdef USE_AS_STRNLEN
|
||||
@@ -147,9 +115,9 @@ L(align_more):
|
||||
jbe L(ret_max)
|
||||
# endif
|
||||
|
||||
- VPCMP $0, (VEC_SIZE * 2)(%rax), %VMM0, %k0
|
||||
- KMOV %k0, %RCX
|
||||
- test %RCX, %RCX
|
||||
+ VPCMP $0, (VEC_SIZE * 2)(%rax), %VMM(0), %k0
|
||||
+ KMOV %k0, %VRCX
|
||||
+ test %VRCX, %VRCX
|
||||
jnz L(ret_vec_x3)
|
||||
|
||||
# ifdef USE_AS_STRNLEN
|
||||
@@ -157,9 +125,9 @@ L(align_more):
|
||||
jbe L(ret_max)
|
||||
# endif
|
||||
|
||||
- VPCMP $0, (VEC_SIZE * 3)(%rax), %VMM0, %k0
|
||||
- KMOV %k0, %RCX
|
||||
- test %RCX, %RCX
|
||||
+ VPCMP $0, (VEC_SIZE * 3)(%rax), %VMM(0), %k0
|
||||
+ KMOV %k0, %VRCX
|
||||
+ test %VRCX, %VRCX
|
||||
jnz L(ret_vec_x4)
|
||||
|
||||
# ifdef USE_AS_STRNLEN
|
||||
@@ -175,7 +143,7 @@ L(align_more):
|
||||
# ifdef USE_AS_STRNLEN
|
||||
subq %rax, %rcx
|
||||
# ifdef USE_AS_WCSLEN
|
||||
- SHR $2, %RCX
|
||||
+ shr $2, %VRCX
|
||||
# endif
|
||||
/* rcx contains number of [w]char will be recompared due to
|
||||
alignment fixes. rdx must be incremented by rcx to offset
|
||||
@@ -195,42 +163,42 @@ L(loop_entry):
|
||||
# endif
|
||||
/* VPMINU and VPCMP combination provide better performance as
|
||||
compared to alternative combinations. */
|
||||
- VMOVA (VEC_SIZE * 4)(%rax), %VMM1
|
||||
- VPMINU (VEC_SIZE * 5)(%rax), %VMM1, %VMM2
|
||||
- VMOVA (VEC_SIZE * 6)(%rax), %VMM3
|
||||
- VPMINU (VEC_SIZE * 7)(%rax), %VMM3, %VMM4
|
||||
+ VMOVA (VEC_SIZE * 4)(%rax), %VMM(1)
|
||||
+ VPMINU (VEC_SIZE * 5)(%rax), %VMM(1), %VMM(2)
|
||||
+ VMOVA (VEC_SIZE * 6)(%rax), %VMM(3)
|
||||
+ VPMINU (VEC_SIZE * 7)(%rax), %VMM(3), %VMM(4)
|
||||
|
||||
- VPTESTN %VMM2, %VMM2, %k0
|
||||
- VPTESTN %VMM4, %VMM4, %k1
|
||||
+ VPTESTN %VMM(2), %VMM(2), %k0
|
||||
+ VPTESTN %VMM(4), %VMM(4), %k1
|
||||
|
||||
subq $-(VEC_SIZE * 4), %rax
|
||||
KORTEST %k0, %k1
|
||||
jz L(loop)
|
||||
|
||||
- VPTESTN %VMM1, %VMM1, %k2
|
||||
- KMOV %k2, %RCX
|
||||
- test %RCX, %RCX
|
||||
+ VPTESTN %VMM(1), %VMM(1), %k2
|
||||
+ KMOV %k2, %VRCX
|
||||
+ test %VRCX, %VRCX
|
||||
jnz L(ret_vec_x1)
|
||||
|
||||
- KMOV %k0, %RCX
|
||||
+ KMOV %k0, %VRCX
|
||||
/* At this point, if k0 is non zero, null char must be in the
|
||||
second vector. */
|
||||
- test %RCX, %RCX
|
||||
+ test %VRCX, %VRCX
|
||||
jnz L(ret_vec_x2)
|
||||
|
||||
- VPTESTN %VMM3, %VMM3, %k3
|
||||
- KMOV %k3, %RCX
|
||||
- test %RCX, %RCX
|
||||
+ VPTESTN %VMM(3), %VMM(3), %k3
|
||||
+ KMOV %k3, %VRCX
|
||||
+ test %VRCX, %VRCX
|
||||
jnz L(ret_vec_x3)
|
||||
/* At this point null [w]char must be in the fourth vector so no
|
||||
need to check. */
|
||||
- KMOV %k1, %RCX
|
||||
+ KMOV %k1, %VRCX
|
||||
|
||||
/* Fourth, third, second vector terminating are pretty much
|
||||
same, implemented this way to avoid branching and reuse code
|
||||
from pre loop exit condition. */
|
||||
L(ret_vec_x4):
|
||||
- bsf %RCX, %RCX
|
||||
+ bsf %VRCX, %VRCX
|
||||
subq %rdi, %rax
|
||||
# ifdef USE_AS_WCSLEN
|
||||
subq $-(VEC_SIZE * 3), %rax
|
||||
@@ -246,7 +214,7 @@ L(ret_vec_x4):
|
||||
ret
|
||||
|
||||
L(ret_vec_x3):
|
||||
- bsf %RCX, %RCX
|
||||
+ bsf %VRCX, %VRCX
|
||||
subq %rdi, %rax
|
||||
# ifdef USE_AS_WCSLEN
|
||||
subq $-(VEC_SIZE * 2), %rax
|
||||
@@ -264,7 +232,7 @@ L(ret_vec_x3):
|
||||
L(ret_vec_x2):
|
||||
subq $-VEC_SIZE, %rax
|
||||
L(ret_vec_x1):
|
||||
- bsf %RCX, %RCX
|
||||
+ bsf %VRCX, %VRCX
|
||||
subq %rdi, %rax
|
||||
# ifdef USE_AS_WCSLEN
|
||||
shrq $2, %rax
|
||||
@@ -285,13 +253,13 @@ L(page_cross):
|
||||
/* ecx contains number of w[char] to be skipped as a result
|
||||
of address alignment. */
|
||||
xorq %rdi, %rax
|
||||
- VPCMP $0, (PAGE_SIZE - VEC_SIZE)(%rax), %VMM0, %k0
|
||||
- KMOV %k0, %RAX
|
||||
+ VPCMP $0, (PAGE_SIZE - VEC_SIZE)(%rax), %VMM(0), %k0
|
||||
+ KMOV %k0, %VRAX
|
||||
/* Ignore number of character for alignment adjustment. */
|
||||
- SHR %cl, %RAX
|
||||
+ shr %cl, %VRAX
|
||||
jz L(align_more)
|
||||
|
||||
- bsf %RAX, %RAX
|
||||
+ bsf %VRAX, %VRAX
|
||||
# ifdef USE_AS_STRNLEN
|
||||
cmpq %rsi, %rax
|
||||
cmovnb %rsi, %rax
|
||||
diff --git a/sysdeps/x86_64/multiarch/strlen-evex512.S b/sysdeps/x86_64/multiarch/strlen-evex512.S
|
||||
index 116f8981c8954e2e..10c3415c8a1e5380 100644
|
||||
--- a/sysdeps/x86_64/multiarch/strlen-evex512.S
|
||||
+++ b/sysdeps/x86_64/multiarch/strlen-evex512.S
|
||||
@@ -2,6 +2,6 @@
|
||||
# define STRLEN __strlen_evex512
|
||||
#endif
|
||||
|
||||
-#define VEC_SIZE 64
|
||||
-
|
||||
+#include "x86-evex512-vecs.h"
|
||||
+#include "reg-macros.h"
|
||||
#include "strlen-evex-base.S"
|
||||
453
glibc-RHEL-175520-6.patch
Normal file
453
glibc-RHEL-175520-6.patch
Normal file
@ -0,0 +1,453 @@
|
||||
commit 451c6e58540e8571e31581c04c4829e5d2cfe8ac
|
||||
Author: Sunil K Pandey <skpgkp2@gmail.com>
|
||||
Date: Thu Aug 18 06:48:07 2022 -0700
|
||||
|
||||
x86_64: Implement evex512 version of memchr, rawmemchr and wmemchr
|
||||
|
||||
This patch implements following evex512 version of string functions.
|
||||
evex512 version takes up to 30% less cycle as compared to evex,
|
||||
depending on length and alignment.
|
||||
|
||||
- memchr function using 512 bit vectors.
|
||||
- rawmemchr function using 512 bit vectors.
|
||||
- wmemchr function using 512 bit vectors.
|
||||
|
||||
Code size data:
|
||||
|
||||
memchr-evex.o 762 byte
|
||||
memchr-evex512.o 576 byte (-24%)
|
||||
|
||||
rawmemchr-evex.o 461 byte
|
||||
rawmemchr-evex512.o 412 byte (-11%)
|
||||
|
||||
wmemchr-evex.o 794 byte
|
||||
wmemchr-evex512.o 552 byte (-30%)
|
||||
|
||||
Placeholder function, not used by any processor at the moment.
|
||||
|
||||
Reviewed-by: Noah Goldstein <goldstein.w.n@gmail.com>
|
||||
|
||||
Conflicts:
|
||||
sysdeps/x86_64/multiarch/ifunc-impl-list.c
|
||||
(Adapting to missing commit 703f4341083afa7d71987aa96a35eab81309e634)
|
||||
|
||||
diff --git a/sysdeps/x86_64/multiarch/Makefile b/sysdeps/x86_64/multiarch/Makefile
|
||||
index b9ea5b60c2be1b0a..67d4b3df693b7110 100644
|
||||
--- a/sysdeps/x86_64/multiarch/Makefile
|
||||
+++ b/sysdeps/x86_64/multiarch/Makefile
|
||||
@@ -4,6 +4,7 @@ sysdep_routines += \
|
||||
memchr-avx2 \
|
||||
memchr-avx2-rtm \
|
||||
memchr-evex \
|
||||
+ memchr-evex512 \
|
||||
memchr-evex-rtm \
|
||||
memchr-sse2 \
|
||||
memcmp-avx2-movbe \
|
||||
@@ -36,6 +37,7 @@ sysdep_routines += \
|
||||
rawmemchr-avx2 \
|
||||
rawmemchr-avx2-rtm \
|
||||
rawmemchr-evex \
|
||||
+ rawmemchr-evex512 \
|
||||
rawmemchr-evex-rtm \
|
||||
rawmemchr-sse2 \
|
||||
stpcpy-avx2 \
|
||||
@@ -174,6 +176,7 @@ sysdep_routines += \
|
||||
wmemchr-avx2 \
|
||||
wmemchr-avx2-rtm \
|
||||
wmemchr-evex \
|
||||
+ wmemchr-evex512 \
|
||||
wmemchr-evex-rtm \
|
||||
wmemchr-sse2 \
|
||||
wmemcmp-avx2-movbe \
|
||||
diff --git a/sysdeps/x86_64/multiarch/ifunc-impl-list.c b/sysdeps/x86_64/multiarch/ifunc-impl-list.c
|
||||
index 84f9e73e2b7df816..6037de5c422a6f7a 100644
|
||||
--- a/sysdeps/x86_64/multiarch/ifunc-impl-list.c
|
||||
+++ b/sysdeps/x86_64/multiarch/ifunc-impl-list.c
|
||||
@@ -54,6 +54,11 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,
|
||||
&& CPU_FEATURE_USABLE (AVX512BW)
|
||||
&& CPU_FEATURE_USABLE (BMI2)),
|
||||
__memchr_evex)
|
||||
+ IFUNC_IMPL_ADD (array, i, memchr,
|
||||
+ (CPU_FEATURE_USABLE (AVX512VL)
|
||||
+ && CPU_FEATURE_USABLE (AVX512BW)
|
||||
+ && CPU_FEATURE_USABLE (BMI2)),
|
||||
+ __memchr_evex512)
|
||||
IFUNC_IMPL_ADD (array, i, memchr,
|
||||
(CPU_FEATURE_USABLE (AVX512VL)
|
||||
&& CPU_FEATURE_USABLE (AVX512BW)
|
||||
@@ -304,6 +309,11 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,
|
||||
&& CPU_FEATURE_USABLE (AVX512BW)
|
||||
&& CPU_FEATURE_USABLE (BMI2)),
|
||||
__rawmemchr_evex)
|
||||
+ IFUNC_IMPL_ADD (array, i, rawmemchr,
|
||||
+ (CPU_FEATURE_USABLE (AVX512VL)
|
||||
+ && CPU_FEATURE_USABLE (AVX512BW)
|
||||
+ && CPU_FEATURE_USABLE (BMI2)),
|
||||
+ __rawmemchr_evex512)
|
||||
IFUNC_IMPL_ADD (array, i, rawmemchr,
|
||||
(CPU_FEATURE_USABLE (AVX512VL)
|
||||
&& CPU_FEATURE_USABLE (AVX512BW)
|
||||
@@ -821,6 +831,11 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,
|
||||
&& CPU_FEATURE_USABLE (AVX512BW)
|
||||
&& CPU_FEATURE_USABLE (BMI2)),
|
||||
__wmemchr_evex)
|
||||
+ IFUNC_IMPL_ADD (array, i, wmemchr,
|
||||
+ (CPU_FEATURE_USABLE (AVX512VL)
|
||||
+ && CPU_FEATURE_USABLE (AVX512BW)
|
||||
+ && CPU_FEATURE_USABLE (BMI2)),
|
||||
+ __wmemchr_evex512)
|
||||
IFUNC_IMPL_ADD (array, i, wmemchr,
|
||||
(CPU_FEATURE_USABLE (AVX512VL)
|
||||
&& CPU_FEATURE_USABLE (AVX512BW)
|
||||
diff --git a/sysdeps/x86_64/multiarch/memchr-evex-base.S b/sysdeps/x86_64/multiarch/memchr-evex-base.S
|
||||
new file mode 100644
|
||||
index 0000000000000000..6ebc9a66e812a644
|
||||
--- /dev/null
|
||||
+++ b/sysdeps/x86_64/multiarch/memchr-evex-base.S
|
||||
@@ -0,0 +1,304 @@
|
||||
+/* Placeholder function, not used by any processor at the moment.
|
||||
+ Copyright (C) 2022 Free Software Foundation, Inc.
|
||||
+ This file is part of the GNU C Library.
|
||||
+
|
||||
+ The GNU C Library is free software; you can redistribute it and/or
|
||||
+ modify it under the terms of the GNU Lesser General Public
|
||||
+ License as published by the Free Software Foundation; either
|
||||
+ version 2.1 of the License, or (at your option) any later version.
|
||||
+
|
||||
+ The GNU C Library is distributed in the hope that it will be useful,
|
||||
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
+ Lesser General Public License for more details.
|
||||
+
|
||||
+ You should have received a copy of the GNU Lesser General Public
|
||||
+ License along with the GNU C Library; if not, see
|
||||
+ <https://www.gnu.org/licenses/>. */
|
||||
+
|
||||
+/* UNUSED. Exists purely as reference implementation. */
|
||||
+
|
||||
+#include <isa-level.h>
|
||||
+
|
||||
+#if ISA_SHOULD_BUILD (4)
|
||||
+
|
||||
+# include <sysdep.h>
|
||||
+
|
||||
+# ifdef USE_AS_WMEMCHR
|
||||
+# define CHAR_SIZE 4
|
||||
+# define VPBROADCAST vpbroadcastd
|
||||
+# define VPCMPEQ vpcmpeqd
|
||||
+# define VPCMPNE vpcmpneqd
|
||||
+# define VPMINU vpminud
|
||||
+# define VPTESTNM vptestnmd
|
||||
+# else
|
||||
+# define CHAR_SIZE 1
|
||||
+# define VPBROADCAST vpbroadcastb
|
||||
+# define VPCMPEQ vpcmpeqb
|
||||
+# define VPCMPNE vpcmpneqb
|
||||
+# define VPMINU vpminub
|
||||
+# define VPTESTNM vptestnmb
|
||||
+# endif
|
||||
+
|
||||
+# define PAGE_SIZE 4096
|
||||
+# define CHAR_PER_VEC (VEC_SIZE / CHAR_SIZE)
|
||||
+
|
||||
+ .section SECTION(.text), "ax", @progbits
|
||||
+/* Aligning entry point to 64 byte, provides better performance for
|
||||
+ one vector length string. */
|
||||
+ENTRY_P2ALIGN (MEMCHR, 6)
|
||||
+# ifndef USE_AS_RAWMEMCHR
|
||||
+ /* Check for zero length. */
|
||||
+ test %RDX_LP, %RDX_LP
|
||||
+ jz L(zero)
|
||||
+
|
||||
+# ifdef __ILP32__
|
||||
+ /* Clear the upper 32 bits. */
|
||||
+ movl %edx, %edx
|
||||
+# endif
|
||||
+# endif
|
||||
+
|
||||
+ /* Broadcast CHAR to VMM(1). */
|
||||
+ VPBROADCAST %esi, %VMM(1)
|
||||
+ movl %edi, %eax
|
||||
+ andl $(PAGE_SIZE - 1), %eax
|
||||
+ cmpl $(PAGE_SIZE - VEC_SIZE), %eax
|
||||
+ ja L(page_cross)
|
||||
+
|
||||
+ /* Compare [w]char for null, mask bit will be set for match. */
|
||||
+ VPCMPEQ (%rdi), %VMM(1), %k0
|
||||
+
|
||||
+ KMOV %k0, %VRCX
|
||||
+# ifndef USE_AS_RAWMEMCHR
|
||||
+ mov %rdx, %rsi
|
||||
+ /* Need to use bsfq here as upper 32 bit of rsi may zero out
|
||||
+ for 'bsf %ecx, %esi', if %ecx is 0. */
|
||||
+ bsfq %rcx, %rsi
|
||||
+ cmp $CHAR_PER_VEC, %rsi
|
||||
+ ja L(align_more)
|
||||
+# ifdef USE_AS_WMEMCHR
|
||||
+ leaq (%rdi, %rsi, CHAR_SIZE), %rdi
|
||||
+# else
|
||||
+ addq %rsi, %rdi
|
||||
+# endif
|
||||
+ xor %eax, %eax
|
||||
+ cmp %rsi, %rdx
|
||||
+ cmova %rdi, %rax
|
||||
+# else
|
||||
+ bsf %VRCX, %VRAX
|
||||
+ jz L(align_more)
|
||||
+ add %rdi, %rax
|
||||
+# endif
|
||||
+ ret
|
||||
+
|
||||
+ .p2align 5,,5
|
||||
+L(page_cross):
|
||||
+ movl %eax, %ecx
|
||||
+ andl $(VEC_SIZE - 1), %ecx
|
||||
+# ifdef USE_AS_WMEMCHR
|
||||
+ shrl $2, %ecx
|
||||
+# endif
|
||||
+ xorq %rdi, %rax
|
||||
+ VPCMPEQ (PAGE_SIZE - VEC_SIZE)(%rax), %VMM(1), %k0
|
||||
+ KMOV %k0, %VRSI
|
||||
+ shr %cl, %VRSI
|
||||
+# ifndef USE_AS_RAWMEMCHR
|
||||
+ jnz L(page_cross_end)
|
||||
+ movl $CHAR_PER_VEC, %eax
|
||||
+ sub %ecx, %eax
|
||||
+ cmp %rax, %rdx
|
||||
+ ja L(align_more)
|
||||
+# else
|
||||
+ jz L(align_more)
|
||||
+# endif
|
||||
+
|
||||
+L(page_cross_end):
|
||||
+# ifndef USE_AS_RAWMEMCHR
|
||||
+ bsf %VRSI, %VRCX
|
||||
+ jz L(zero)
|
||||
+ leaq (%rdi, %rcx, CHAR_SIZE), %rdi
|
||||
+ xor %eax, %eax
|
||||
+ cmp %rcx, %rdx
|
||||
+ cmova %rdi, %rax
|
||||
+# else
|
||||
+ bsf %VRSI, %VRAX
|
||||
+ add %rdi, %rax
|
||||
+# endif
|
||||
+ ret
|
||||
+
|
||||
+# ifndef USE_AS_RAWMEMCHR
|
||||
+L(zero):
|
||||
+ xorl %eax, %eax
|
||||
+ ret
|
||||
+# endif
|
||||
+
|
||||
+L(ret_vec_x2):
|
||||
+ subq $-VEC_SIZE, %rdi
|
||||
+L(ret_vec_x1):
|
||||
+ bsf %VRAX, %VRAX
|
||||
+# ifndef USE_AS_RAWMEMCHR
|
||||
+ cmp %rax, %rdx
|
||||
+ jbe L(zero)
|
||||
+# endif
|
||||
+# ifdef USE_AS_WMEMCHR
|
||||
+ leaq (%rdi, %rax, CHAR_SIZE), %rax
|
||||
+# else
|
||||
+ add %rdi, %rax
|
||||
+# endif
|
||||
+ ret
|
||||
+
|
||||
+ .p2align 5,,5
|
||||
+L(align_more):
|
||||
+# ifndef USE_AS_RAWMEMCHR
|
||||
+ mov %rdi, %rax
|
||||
+# endif
|
||||
+ subq $-VEC_SIZE, %rdi
|
||||
+ /* Align rdi to VEC_SIZE. */
|
||||
+ andq $-VEC_SIZE, %rdi
|
||||
+
|
||||
+# ifndef USE_AS_RAWMEMCHR
|
||||
+ subq %rdi, %rax
|
||||
+# ifdef USE_AS_WMEMCHR
|
||||
+ sar $2, %rax
|
||||
+# endif
|
||||
+ addq %rax, %rdx
|
||||
+# endif
|
||||
+
|
||||
+ /* Loop unroll 4 times for 4 vector loop. */
|
||||
+ VPCMPEQ (%rdi), %VMM(1), %k0
|
||||
+
|
||||
+ KMOV %k0, %VRAX
|
||||
+ test %VRAX, %VRAX
|
||||
+ jnz L(ret_vec_x1)
|
||||
+
|
||||
+# ifndef USE_AS_RAWMEMCHR
|
||||
+ subq $CHAR_PER_VEC, %rdx
|
||||
+ jbe L(zero)
|
||||
+# endif
|
||||
+
|
||||
+ VPCMPEQ VEC_SIZE(%rdi), %VMM(1), %k0
|
||||
+
|
||||
+ KMOV %k0, %VRAX
|
||||
+ test %VRAX, %VRAX
|
||||
+ jnz L(ret_vec_x2)
|
||||
+
|
||||
+# ifndef USE_AS_RAWMEMCHR
|
||||
+ subq $CHAR_PER_VEC, %rdx
|
||||
+ jbe L(zero)
|
||||
+# endif
|
||||
+
|
||||
+ VPCMPEQ (VEC_SIZE * 2)(%rdi), %VMM(1), %k0
|
||||
+
|
||||
+ KMOV %k0, %VRAX
|
||||
+ test %VRAX, %VRAX
|
||||
+ jnz L(ret_vec_x3)
|
||||
+
|
||||
+# ifndef USE_AS_RAWMEMCHR
|
||||
+ subq $CHAR_PER_VEC, %rdx
|
||||
+ jbe L(zero)
|
||||
+# endif
|
||||
+
|
||||
+ VPCMPEQ (VEC_SIZE * 3)(%rdi), %VMM(1), %k0
|
||||
+
|
||||
+ KMOV %k0, %VRAX
|
||||
+ test %VRAX, %VRAX
|
||||
+ jnz L(ret_vec_x4)
|
||||
+
|
||||
+# ifndef USE_AS_RAWMEMCHR
|
||||
+ subq $CHAR_PER_VEC, %rdx
|
||||
+ jbe L(zero)
|
||||
+ /* Save pointer to find alignment adjustment. */
|
||||
+ movq %rdi, %rax
|
||||
+# endif
|
||||
+ /* Align address to VEC_SIZE * 4 for loop. */
|
||||
+ andq $-(VEC_SIZE * 4), %rdi
|
||||
+
|
||||
+ /* Add alignment difference to rdx. */
|
||||
+# ifndef USE_AS_RAWMEMCHR
|
||||
+ subq %rdi, %rax
|
||||
+# ifdef USE_AS_WMEMCHR
|
||||
+ shr $2, %VRAX
|
||||
+# endif
|
||||
+ addq %rax, %rdx
|
||||
+# endif
|
||||
+
|
||||
+ /* 4 vector loop. */
|
||||
+ .p2align 5,,11
|
||||
+L(loop):
|
||||
+
|
||||
+ VPCMPNE (VEC_SIZE * 4)(%rdi), %VMM(1), %k1
|
||||
+ vpxorq (VEC_SIZE * 5)(%rdi), %VMM(1), %VMM(2)
|
||||
+ vpxorq (VEC_SIZE * 6)(%rdi), %VMM(1), %VMM(3)
|
||||
+ VPCMPEQ (VEC_SIZE * 7)(%rdi), %VMM(1), %k3
|
||||
+ VPMINU %VMM(2), %VMM(3), %VMM(3){%k1}{z}
|
||||
+ VPTESTNM %VMM(3), %VMM(3), %k2
|
||||
+
|
||||
+ subq $-(VEC_SIZE * 4), %rdi
|
||||
+ KORTEST %k2, %k3
|
||||
+# ifdef USE_AS_RAWMEMCHR
|
||||
+ jz L(loop)
|
||||
+# else
|
||||
+ jnz L(loopend)
|
||||
+ subq $(CHAR_PER_VEC * 4), %rdx
|
||||
+ ja L(loop)
|
||||
+L(zero_2):
|
||||
+ xor %eax, %eax
|
||||
+ ret
|
||||
+# endif
|
||||
+
|
||||
+L(loopend):
|
||||
+ VPCMPEQ (%rdi), %VMM(1), %k1
|
||||
+ KMOV %k1, %VRAX
|
||||
+ test %VRAX, %VRAX
|
||||
+ jnz L(ret_vec_x1)
|
||||
+
|
||||
+# ifndef USE_AS_RAWMEMCHR
|
||||
+ subq $CHAR_PER_VEC, %rdx
|
||||
+ jbe L(zero_2)
|
||||
+# endif
|
||||
+
|
||||
+ VPCMPEQ VEC_SIZE(%rdi), %VMM(1), %k1
|
||||
+ KMOV %k1, %VRAX
|
||||
+ test %VRAX, %VRAX
|
||||
+ jnz L(ret_vec_x2)
|
||||
+
|
||||
+# ifndef USE_AS_RAWMEMCHR
|
||||
+ subq $CHAR_PER_VEC, %rdx
|
||||
+ jbe L(zero_2)
|
||||
+# endif
|
||||
+
|
||||
+ VPCMPEQ (VEC_SIZE * 2)(%rdi), %VMM(1), %k1
|
||||
+ KMOV %k1, %VRAX
|
||||
+ test %VRAX, %VRAX
|
||||
+ jnz L(ret_vec_x3)
|
||||
+
|
||||
+# ifndef USE_AS_RAWMEMCHR
|
||||
+ subq $CHAR_PER_VEC, %rdx
|
||||
+ jbe L(zero_2)
|
||||
+# endif
|
||||
+
|
||||
+ /* At this point null [w]char must be in the fourth vector so no
|
||||
+ need to check. */
|
||||
+ KMOV %k3, %VRAX
|
||||
+
|
||||
+L(ret_vec_x4):
|
||||
+ bsf %VRAX, %VRAX
|
||||
+# ifndef USE_AS_RAWMEMCHR
|
||||
+ cmp %rax, %rdx
|
||||
+ jbe L(zero)
|
||||
+# endif
|
||||
+ leaq (VEC_SIZE * 3)(%rdi, %rax, CHAR_SIZE), %rax
|
||||
+ ret
|
||||
+
|
||||
+ .p2align 5,,5
|
||||
+L(ret_vec_x3):
|
||||
+ bsf %VRAX, %VRAX
|
||||
+# ifndef USE_AS_RAWMEMCHR
|
||||
+ cmp %rax, %rdx
|
||||
+ jbe L(zero)
|
||||
+# endif
|
||||
+ leaq (VEC_SIZE * 2)(%rdi, %rax, CHAR_SIZE), %rax
|
||||
+ ret
|
||||
+
|
||||
+END (MEMCHR)
|
||||
+#endif
|
||||
diff --git a/sysdeps/x86_64/multiarch/memchr-evex512.S b/sysdeps/x86_64/multiarch/memchr-evex512.S
|
||||
new file mode 100644
|
||||
index 0000000000000000..002f8c84893181e0
|
||||
--- /dev/null
|
||||
+++ b/sysdeps/x86_64/multiarch/memchr-evex512.S
|
||||
@@ -0,0 +1,8 @@
|
||||
+# ifndef MEMCHR
|
||||
+# define MEMCHR __memchr_evex512
|
||||
+# endif
|
||||
+
|
||||
+#include "x86-evex512-vecs.h"
|
||||
+#include "reg-macros.h"
|
||||
+
|
||||
+#include "memchr-evex-base.S"
|
||||
diff --git a/sysdeps/x86_64/multiarch/rawmemchr-evex512.S b/sysdeps/x86_64/multiarch/rawmemchr-evex512.S
|
||||
new file mode 100644
|
||||
index 0000000000000000..302d3cb0554e736e
|
||||
--- /dev/null
|
||||
+++ b/sysdeps/x86_64/multiarch/rawmemchr-evex512.S
|
||||
@@ -0,0 +1,7 @@
|
||||
+#ifndef RAWMEMCHR
|
||||
+# define RAWMEMCHR __rawmemchr_evex512
|
||||
+#endif
|
||||
+#define USE_AS_RAWMEMCHR 1
|
||||
+#define MEMCHR RAWMEMCHR
|
||||
+
|
||||
+#include "memchr-evex512.S"
|
||||
diff --git a/sysdeps/x86_64/multiarch/wmemchr-evex512.S b/sysdeps/x86_64/multiarch/wmemchr-evex512.S
|
||||
new file mode 100644
|
||||
index 0000000000000000..78ec4ee5ad7fb7b5
|
||||
--- /dev/null
|
||||
+++ b/sysdeps/x86_64/multiarch/wmemchr-evex512.S
|
||||
@@ -0,0 +1,9 @@
|
||||
+#ifndef WMEMCHR
|
||||
+# define WMEMCHR __wmemchr_evex512
|
||||
+#endif
|
||||
+
|
||||
+#define MEMCHR WMEMCHR
|
||||
+#define USE_AS_WMEMCHR 1
|
||||
+
|
||||
+#define USE_WIDE_CHAR 1
|
||||
+#include "memchr-evex512.S"
|
||||
427
glibc-RHEL-175520-7.patch
Normal file
427
glibc-RHEL-175520-7.patch
Normal file
@ -0,0 +1,427 @@
|
||||
commit 59e501f204fa196d6571b523459ba528bbef7783
|
||||
Author: Sunil K Pandey <skpgkp2@gmail.com>
|
||||
Date: Tue Jul 26 13:54:56 2022 -0700
|
||||
|
||||
x86_64: Implement evex512 version of strchrnul, strchr and wcschr
|
||||
|
||||
This patch implements following evex512 version of string functions.
|
||||
evex512 version takes up to 30% less cycle as compared to evex,
|
||||
depending on length and alignment.
|
||||
|
||||
- strchrnul function using 512 bit vectors.
|
||||
- strchr function using 512 bit vectors.
|
||||
- wcschr function using 512 bit vectors.
|
||||
|
||||
Code size data:
|
||||
|
||||
strchrnul-evex.o 599 byte
|
||||
strchrnul-evex512.o 569 byte (-5%)
|
||||
|
||||
strchr-evex.o 639 byte
|
||||
strchr-evex512.o 595 byte (-7%)
|
||||
|
||||
wcschr-evex.o 644 byte
|
||||
wcschr-evex512.o 607 byte (-6%)
|
||||
|
||||
Placeholder function, not used by any processor at the moment.
|
||||
|
||||
Reviewed-by: Noah Goldstein <goldstein.w.n@gmail.com>
|
||||
|
||||
Conflicts:
|
||||
sysdeps/x86_64/multiarch/ifunc-impl-list.c
|
||||
(fixup macro)
|
||||
|
||||
diff --git a/sysdeps/x86_64/multiarch/Makefile b/sysdeps/x86_64/multiarch/Makefile
|
||||
index 67d4b3df693b7110..1c8bc2af0ef9c367 100644
|
||||
--- a/sysdeps/x86_64/multiarch/Makefile
|
||||
+++ b/sysdeps/x86_64/multiarch/Makefile
|
||||
@@ -67,11 +67,13 @@ sysdep_routines += \
|
||||
strchr-avx2 \
|
||||
strchr-avx2-rtm \
|
||||
strchr-evex \
|
||||
+ strchr-evex512 \
|
||||
strchr-sse2 \
|
||||
strchr-sse2-no-bsf \
|
||||
strchrnul-avx2 \
|
||||
strchrnul-avx2-rtm \
|
||||
strchrnul-evex \
|
||||
+ strchrnul-evex512 \
|
||||
strchrnul-sse2 \
|
||||
strcmp-avx2 \
|
||||
strcmp-avx2-rtm \
|
||||
@@ -146,6 +148,7 @@ sysdep_routines += \
|
||||
wcschr-avx2 \
|
||||
wcschr-avx2-rtm \
|
||||
wcschr-evex \
|
||||
+ wcschr-evex512 \
|
||||
wcschr-sse2 \
|
||||
wcscmp-avx2 \
|
||||
wcscmp-avx2-rtm \
|
||||
diff --git a/sysdeps/x86_64/multiarch/ifunc-impl-list.c b/sysdeps/x86_64/multiarch/ifunc-impl-list.c
|
||||
index 6037de5c422a6f7a..cf482eff56fb663c 100644
|
||||
--- a/sysdeps/x86_64/multiarch/ifunc-impl-list.c
|
||||
+++ b/sysdeps/x86_64/multiarch/ifunc-impl-list.c
|
||||
@@ -484,6 +484,10 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,
|
||||
&& CPU_FEATURE_USABLE (AVX512BW)
|
||||
&& CPU_FEATURE_USABLE (BMI2)),
|
||||
__strchr_evex)
|
||||
+ IFUNC_IMPL_ADD (array, i, strchr,
|
||||
+ (CPU_FEATURE_USABLE (AVX512VL)
|
||||
+ && CPU_FEATURE_USABLE (AVX512BW)),
|
||||
+ __strchr_evex512)
|
||||
IFUNC_IMPL_ADD (array, i, strchr, 1, __strchr_sse2_no_bsf)
|
||||
IFUNC_IMPL_ADD (array, i, strchr, 1, __strchr_sse2))
|
||||
|
||||
@@ -503,6 +507,10 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,
|
||||
&& CPU_FEATURE_USABLE (AVX512BW)
|
||||
&& CPU_FEATURE_USABLE (BMI2)),
|
||||
__strchrnul_evex)
|
||||
+ IFUNC_IMPL_ADD (array, i, strchrnul,
|
||||
+ (CPU_FEATURE_USABLE (AVX512VL)
|
||||
+ && CPU_FEATURE_USABLE (AVX512BW)),
|
||||
+ __strchrnul_evex512)
|
||||
IFUNC_IMPL_ADD (array, i, strchrnul, 1, __strchrnul_sse2))
|
||||
|
||||
/* Support sysdeps/x86_64/multiarch/strrchr.c. */
|
||||
@@ -698,6 +706,10 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,
|
||||
&& CPU_FEATURE_USABLE (AVX512BW)
|
||||
&& CPU_FEATURE_USABLE (BMI2)),
|
||||
__wcschr_evex)
|
||||
+ IFUNC_IMPL_ADD (array, i, wcschr,
|
||||
+ (CPU_FEATURE_USABLE (AVX512VL)
|
||||
+ && CPU_FEATURE_USABLE (AVX512BW)),
|
||||
+ __wcschr_evex512)
|
||||
IFUNC_IMPL_ADD (array, i, wcschr, 1, __wcschr_sse2))
|
||||
|
||||
/* Support sysdeps/x86_64/multiarch/wcsrchr.c. */
|
||||
diff --git a/sysdeps/x86_64/multiarch/strchr-evex-base.S b/sysdeps/x86_64/multiarch/strchr-evex-base.S
|
||||
new file mode 100644
|
||||
index 0000000000000000..75fee8c82ade14f1
|
||||
--- /dev/null
|
||||
+++ b/sysdeps/x86_64/multiarch/strchr-evex-base.S
|
||||
@@ -0,0 +1,282 @@
|
||||
+/* Placeholder function, not used by any processor at the moment.
|
||||
+ Copyright (C) 2022 Free Software Foundation, Inc.
|
||||
+ This file is part of the GNU C Library.
|
||||
+
|
||||
+ The GNU C Library is free software; you can redistribute it and/or
|
||||
+ modify it under the terms of the GNU Lesser General Public
|
||||
+ License as published by the Free Software Foundation; either
|
||||
+ version 2.1 of the License, or (at your option) any later version.
|
||||
+
|
||||
+ The GNU C Library is distributed in the hope that it will be useful,
|
||||
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
+ Lesser General Public License for more details.
|
||||
+
|
||||
+ You should have received a copy of the GNU Lesser General Public
|
||||
+ License along with the GNU C Library; if not, see
|
||||
+ <https://www.gnu.org/licenses/>. */
|
||||
+
|
||||
+/* UNUSED. Exists purely as reference implementation. */
|
||||
+
|
||||
+#include <isa-level.h>
|
||||
+
|
||||
+#if ISA_SHOULD_BUILD (4)
|
||||
+
|
||||
+# include <sysdep.h>
|
||||
+
|
||||
+# ifdef USE_AS_WCSCHR
|
||||
+# define CHAR_REG esi
|
||||
+# define CHAR_SIZE 4
|
||||
+# define VPBROADCAST vpbroadcastd
|
||||
+# define VPCMP vpcmpd
|
||||
+# define VPCMPNE vpcmpneqd
|
||||
+# define VPMINU vpminud
|
||||
+# define VPTEST vptestmd
|
||||
+# define VPTESTN vptestnmd
|
||||
+# else
|
||||
+# define CHAR_REG sil
|
||||
+# define CHAR_SIZE 1
|
||||
+# define VPBROADCAST vpbroadcastb
|
||||
+# define VPCMP vpcmpb
|
||||
+# define VPCMPNE vpcmpneqb
|
||||
+# define VPMINU vpminub
|
||||
+# define VPTEST vptestmb
|
||||
+# define VPTESTN vptestnmb
|
||||
+# endif
|
||||
+
|
||||
+# define PAGE_SIZE 4096
|
||||
+# define CHAR_PER_VEC (VEC_SIZE / CHAR_SIZE)
|
||||
+# define VEC_MATCH_MASK ((1 << CHAR_PER_VEC) - 1)
|
||||
+
|
||||
+ .section SECTION(.text), "ax", @progbits
|
||||
+/* Aligning entry point to 64 byte, provides better performance for
|
||||
+ one vector length string. */
|
||||
+ENTRY_P2ALIGN (STRCHR, 6)
|
||||
+
|
||||
+ /* Broadcast CHAR to VMM(0). */
|
||||
+ VPBROADCAST %esi, %VMM(0)
|
||||
+ movl %edi, %eax
|
||||
+ sall $20,%eax
|
||||
+ cmpl $((PAGE_SIZE - VEC_SIZE) << 20), %eax
|
||||
+ ja L(page_cross)
|
||||
+
|
||||
+ VMOVU (%rdi), %VMM(1)
|
||||
+ VPCMPNE %VMM(1), %VMM(0), %k1
|
||||
+ VPTEST %VMM(1), %VMM(1), %k0{%k1}
|
||||
+ KMOV %k0, %VRAX
|
||||
+ /* Compare [w]char for null, mask bit will be set for match. */
|
||||
+
|
||||
+# ifdef USE_AS_WCSCHR
|
||||
+ sub $VEC_MATCH_MASK, %VRAX
|
||||
+# else
|
||||
+ inc %VRAX
|
||||
+# endif
|
||||
+ jz L(align_more)
|
||||
+
|
||||
+ bsf %VRAX, %VRAX
|
||||
+
|
||||
+# ifdef USE_AS_WCSCHR
|
||||
+ leaq (%rdi, %rax, CHAR_SIZE), %rax
|
||||
+# else
|
||||
+ add %rdi, %rax
|
||||
+# endif
|
||||
+# ifndef USE_AS_STRCHRNUL
|
||||
+ cmp (%rax), %CHAR_REG
|
||||
+ jne L(zero)
|
||||
+ ret
|
||||
+L(zero):
|
||||
+ xorl %eax, %eax
|
||||
+# endif
|
||||
+ ret
|
||||
+
|
||||
+L(ret_vec_x3):
|
||||
+ subq $-VEC_SIZE, %rdi
|
||||
+L(ret_vec_x2):
|
||||
+ subq $-VEC_SIZE, %rdi
|
||||
+L(ret_vec_x1):
|
||||
+ bsf %VRAX, %VRAX
|
||||
+# ifdef USE_AS_WCSCHR
|
||||
+ leaq (%rdi, %rax, CHAR_SIZE), %rax
|
||||
+# else
|
||||
+ add %rdi, %rax
|
||||
+# endif
|
||||
+
|
||||
+# ifndef USE_AS_STRCHRNUL
|
||||
+ cmp (%rax), %CHAR_REG
|
||||
+ jne L(zero)
|
||||
+# endif
|
||||
+ ret
|
||||
+
|
||||
+L(page_cross):
|
||||
+ mov %rdi, %rax
|
||||
+ movl %edi, %ecx
|
||||
+# ifdef USE_AS_WCSCHR
|
||||
+ /* Calculate number of compare result bits to be skipped for
|
||||
+ wide string alignment adjustment. */
|
||||
+ andl $(VEC_SIZE - 1), %ecx
|
||||
+ sarl $2, %ecx
|
||||
+# endif
|
||||
+ /* ecx contains number of w[char] to be skipped as a result
|
||||
+ of address alignment. */
|
||||
+ andq $-VEC_SIZE, %rax
|
||||
+
|
||||
+ VMOVA (%rax), %VMM(1)
|
||||
+ VPCMPNE %VMM(1), %VMM(0), %k1
|
||||
+ VPTEST %VMM(1), %VMM(1), %k0{%k1}
|
||||
+ KMOV %k0, %VRAX
|
||||
+# ifdef USE_AS_WCSCHR
|
||||
+ sub $VEC_MATCH_MASK, %VRAX
|
||||
+# else
|
||||
+ inc %VRAX
|
||||
+# endif
|
||||
+ /* Ignore number of character for alignment adjustment. */
|
||||
+ shr %cl, %VRAX
|
||||
+ jz L(align_more)
|
||||
+
|
||||
+ bsf %VRAX, %VRAX
|
||||
+# ifdef USE_AS_WCSCHR
|
||||
+ leaq (%rdi, %rax, CHAR_SIZE), %rax
|
||||
+# else
|
||||
+ addq %rdi, %rax
|
||||
+# endif
|
||||
+
|
||||
+# ifndef USE_AS_STRCHRNUL
|
||||
+ cmp (%rax), %CHAR_REG
|
||||
+ jne L(zero)
|
||||
+# endif
|
||||
+ ret
|
||||
+
|
||||
+L(align_more):
|
||||
+ /* Align rax to VEC_SIZE. */
|
||||
+ andq $-VEC_SIZE, %rdi
|
||||
+
|
||||
+ /* Loop unroll 4 times for 4 vector loop. */
|
||||
+ VMOVA VEC_SIZE(%rdi), %VMM(1)
|
||||
+ VPCMPNE %VMM(1), %VMM(0), %k1
|
||||
+ VPTEST %VMM(1), %VMM(1), %k0{%k1}
|
||||
+
|
||||
+ /* Increment rdi by vector size for further comparison and
|
||||
+ return. */
|
||||
+ subq $-VEC_SIZE, %rdi
|
||||
+ KMOV %k0, %VRAX
|
||||
+
|
||||
+# ifdef USE_AS_WCSCHR
|
||||
+ sub $VEC_MATCH_MASK, %VRAX
|
||||
+# else
|
||||
+ inc %VRAX
|
||||
+# endif
|
||||
+ jnz L(ret_vec_x1)
|
||||
+
|
||||
+ VMOVA VEC_SIZE(%rdi), %VMM(1)
|
||||
+ VPCMPNE %VMM(1), %VMM(0), %k1
|
||||
+ VPTEST %VMM(1), %VMM(1), %k0{%k1}
|
||||
+ KMOV %k0, %VRAX
|
||||
+# ifdef USE_AS_WCSCHR
|
||||
+ sub $VEC_MATCH_MASK, %VRAX
|
||||
+# else
|
||||
+ inc %VRAX
|
||||
+# endif
|
||||
+ jnz L(ret_vec_x2)
|
||||
+
|
||||
+ VMOVA (VEC_SIZE * 2)(%rdi), %VMM(1)
|
||||
+ VPCMPNE %VMM(1), %VMM(0), %k1
|
||||
+ VPTEST %VMM(1), %VMM(1), %k0{%k1}
|
||||
+ KMOV %k0, %VRAX
|
||||
+# ifdef USE_AS_WCSCHR
|
||||
+ sub $VEC_MATCH_MASK, %VRAX
|
||||
+# else
|
||||
+ inc %VRAX
|
||||
+# endif
|
||||
+ jnz L(ret_vec_x3)
|
||||
+
|
||||
+ VMOVA (VEC_SIZE * 3)(%rdi), %VMM(1)
|
||||
+ VPCMPNE %VMM(1), %VMM(0), %k1
|
||||
+ VPTEST %VMM(1), %VMM(1), %k0{%k1}
|
||||
+ KMOV %k0, %VRDX
|
||||
+# ifdef USE_AS_WCSCHR
|
||||
+ sub $VEC_MATCH_MASK, %VRDX
|
||||
+# else
|
||||
+ inc %VRDX
|
||||
+# endif
|
||||
+ jnz L(ret_vec_x4)
|
||||
+
|
||||
+
|
||||
+ /* Align address to VEC_SIZE * 4 for loop. */
|
||||
+ andq $-(VEC_SIZE * 4), %rdi
|
||||
+L(loop):
|
||||
+ /* VPMINU and VPCMP combination provide better performance as
|
||||
+ compared to alternative combinations. */
|
||||
+ VMOVA (VEC_SIZE * 4)(%rdi), %VMM(1)
|
||||
+ VMOVA (VEC_SIZE * 5)(%rdi), %VMM(2)
|
||||
+ VMOVA (VEC_SIZE * 6)(%rdi), %VMM(3)
|
||||
+ VMOVA (VEC_SIZE * 7)(%rdi), %VMM(4)
|
||||
+
|
||||
+ VPCMPNE %VMM(1), %VMM(0), %k1
|
||||
+ VPCMPNE %VMM(2), %VMM(0), %k2
|
||||
+
|
||||
+ VPMINU %VMM(2), %VMM(1), %VMM(2)
|
||||
+
|
||||
+ VPCMPNE %VMM(3), %VMM(0), %k3{%k1}
|
||||
+ VPCMPNE %VMM(4), %VMM(0), %k4{%k2}
|
||||
+
|
||||
+ VPMINU %VMM(4), %VMM(3), %VMM(4)
|
||||
+ VPMINU %VMM(2), %VMM(4), %VMM(4){%k3}{z}
|
||||
+
|
||||
+ VPTEST %VMM(4), %VMM(4), %k5{%k4}
|
||||
+
|
||||
+ KMOV %k5, %VRDX
|
||||
+ subq $-(VEC_SIZE * 4), %rdi
|
||||
+# ifdef USE_AS_WCSCHR
|
||||
+ sub $VEC_MATCH_MASK, %VRDX
|
||||
+# else
|
||||
+ inc %VRDX
|
||||
+# endif
|
||||
+ jz L(loop)
|
||||
+
|
||||
+ VPTEST %VMM(1), %VMM(1), %k0{%k1}
|
||||
+ KMOV %k0, %VRAX
|
||||
+# ifdef USE_AS_WCSCHR
|
||||
+ sub $VEC_MATCH_MASK, %VRAX
|
||||
+# else
|
||||
+ inc %VRAX
|
||||
+# endif
|
||||
+ jnz L(ret_vec_x1)
|
||||
+
|
||||
+ VPTEST %VMM(2), %VMM(2), %k0{%k2}
|
||||
+ KMOV %k0, %VRAX
|
||||
+ /* At this point, if k1 is non zero, null char must be in the
|
||||
+ second vector. */
|
||||
+# ifdef USE_AS_WCSCHR
|
||||
+ sub $VEC_MATCH_MASK, %VRAX
|
||||
+# else
|
||||
+ inc %VRAX
|
||||
+# endif
|
||||
+ jnz L(ret_vec_x2)
|
||||
+
|
||||
+ VPTEST %VMM(3), %VMM(3), %k0{%k3}
|
||||
+ KMOV %k0, %VRAX
|
||||
+# ifdef USE_AS_WCSCHR
|
||||
+ sub $VEC_MATCH_MASK, %VRAX
|
||||
+# else
|
||||
+ inc %VRAX
|
||||
+# endif
|
||||
+ jnz L(ret_vec_x3)
|
||||
+ /* At this point null [w]char must be in the fourth vector so no
|
||||
+ need to check. */
|
||||
+
|
||||
+L(ret_vec_x4):
|
||||
+ bsf %VRDX, %VRDX
|
||||
+ leaq (VEC_SIZE * 3)(%rdi, %rdx, CHAR_SIZE), %rax
|
||||
+# ifndef USE_AS_STRCHRNUL
|
||||
+ cmp (%rax), %CHAR_REG
|
||||
+ jne L(zero_2)
|
||||
+# endif
|
||||
+ ret
|
||||
+
|
||||
+# ifndef USE_AS_STRCHRNUL
|
||||
+L(zero_2):
|
||||
+ xor %eax, %eax
|
||||
+ ret
|
||||
+# endif
|
||||
+END (STRCHR)
|
||||
+#endif
|
||||
diff --git a/sysdeps/x86_64/multiarch/strchr-evex512.S b/sysdeps/x86_64/multiarch/strchr-evex512.S
|
||||
new file mode 100644
|
||||
index 0000000000000000..a4ac0229523d1b01
|
||||
--- /dev/null
|
||||
+++ b/sysdeps/x86_64/multiarch/strchr-evex512.S
|
||||
@@ -0,0 +1,8 @@
|
||||
+# ifndef STRCHR
|
||||
+# define STRCHR __strchr_evex512
|
||||
+# endif
|
||||
+
|
||||
+#include "x86-evex512-vecs.h"
|
||||
+#include "reg-macros.h"
|
||||
+
|
||||
+#include "strchr-evex-base.S"
|
||||
diff --git a/sysdeps/x86_64/multiarch/strchrnul-evex512.S b/sysdeps/x86_64/multiarch/strchrnul-evex512.S
|
||||
new file mode 100644
|
||||
index 0000000000000000..1be0b12f385da936
|
||||
--- /dev/null
|
||||
+++ b/sysdeps/x86_64/multiarch/strchrnul-evex512.S
|
||||
@@ -0,0 +1,8 @@
|
||||
+#ifndef STRCHRNUL
|
||||
+# define STRCHRNUL __strchrnul_evex512
|
||||
+#endif
|
||||
+
|
||||
+#define STRCHR STRCHRNUL
|
||||
+#define USE_AS_STRCHRNUL 1
|
||||
+
|
||||
+#include "strchr-evex512.S"
|
||||
diff --git a/sysdeps/x86_64/multiarch/wcschr-evex512.S b/sysdeps/x86_64/multiarch/wcschr-evex512.S
|
||||
new file mode 100644
|
||||
index 0000000000000000..3fe4e77a706b2bce
|
||||
--- /dev/null
|
||||
+++ b/sysdeps/x86_64/multiarch/wcschr-evex512.S
|
||||
@@ -0,0 +1,9 @@
|
||||
+#ifndef WCSCHR
|
||||
+# define WCSCHR __wcschr_evex512
|
||||
+#endif
|
||||
+
|
||||
+#define STRCHR WCSCHR
|
||||
+#define USE_AS_WCSCHR 1
|
||||
+
|
||||
+#define USE_WIDE_CHAR 1
|
||||
+#include "strchr-evex512.S"
|
||||
210
glibc-RHEL-175520-8.patch
Normal file
210
glibc-RHEL-175520-8.patch
Normal file
@ -0,0 +1,210 @@
|
||||
commit e96971482de05eff92c1408b694c320cedd2d167
|
||||
Author: Sunil K Pandey <skpgkp2@gmail.com>
|
||||
Date: Mon Oct 3 12:00:53 2022 -0700
|
||||
|
||||
x86-64: Improve evex512 version of strlen functions
|
||||
|
||||
This patch improves following functionality
|
||||
- Replace VPCMP with VPCMPEQ.
|
||||
- Replace page cross check logic with sall.
|
||||
- Remove extra lea from align_more.
|
||||
- Remove uncondition loop jump.
|
||||
- Use bsf to check max length in first vector.
|
||||
|
||||
Reviewed-by: Noah Goldstein <goldstein.w.n@gmail.com>
|
||||
|
||||
diff --git a/sysdeps/x86_64/multiarch/strlen-evex-base.S b/sysdeps/x86_64/multiarch/strlen-evex-base.S
|
||||
index 176babee1e0a9e89..742ac9c9253ecb60 100644
|
||||
--- a/sysdeps/x86_64/multiarch/strlen-evex-base.S
|
||||
+++ b/sysdeps/x86_64/multiarch/strlen-evex-base.S
|
||||
@@ -21,12 +21,12 @@
|
||||
# include <sysdep.h>
|
||||
|
||||
# ifdef USE_AS_WCSLEN
|
||||
-# define VPCMP vpcmpd
|
||||
+# define VPCMPEQ vpcmpeqd
|
||||
# define VPTESTN vptestnmd
|
||||
# define VPMINU vpminud
|
||||
# define CHAR_SIZE 4
|
||||
# else
|
||||
-# define VPCMP vpcmpb
|
||||
+# define VPCMPEQ vpcmpeqb
|
||||
# define VPTESTN vptestnmb
|
||||
# define VPMINU vpminub
|
||||
# define CHAR_SIZE 1
|
||||
@@ -51,20 +51,29 @@ ENTRY_P2ALIGN (STRLEN, 6)
|
||||
|
||||
movl %edi, %eax
|
||||
vpxorq %VMM_128(0), %VMM_128(0), %VMM_128(0)
|
||||
- andl $(PAGE_SIZE - 1), %eax
|
||||
- cmpl $(PAGE_SIZE - VEC_SIZE), %eax
|
||||
+ sall $20, %eax
|
||||
+ cmpl $((PAGE_SIZE - VEC_SIZE) << 20), %eax
|
||||
ja L(page_cross)
|
||||
|
||||
/* Compare [w]char for null, mask bit will be set for match. */
|
||||
- VPCMP $0, (%rdi), %VMM(0), %k0
|
||||
+ VPCMPEQ (%rdi), %VMM(0), %k0
|
||||
+# ifdef USE_AS_STRNLEN
|
||||
+ KMOV %k0, %VRCX
|
||||
+ /* Store max length in rax. */
|
||||
+ mov %rsi, %rax
|
||||
+ /* If rcx is 0, rax will have max length. We can not use VRCX
|
||||
+ and VRAX here for evex256 because, upper 32 bits may be
|
||||
+ undefined for ecx and eax. */
|
||||
+ bsfq %rcx, %rax
|
||||
+ cmp $CHAR_PER_VEC, %rax
|
||||
+ ja L(align_more)
|
||||
+ cmpq %rax, %rsi
|
||||
+ cmovb %esi, %eax
|
||||
+# else
|
||||
KMOV %k0, %VRAX
|
||||
test %VRAX, %VRAX
|
||||
jz L(align_more)
|
||||
-
|
||||
bsf %VRAX, %VRAX
|
||||
-# ifdef USE_AS_STRNLEN
|
||||
- cmpq %rsi, %rax
|
||||
- cmovnb %rsi, %rax
|
||||
# endif
|
||||
ret
|
||||
|
||||
@@ -77,25 +86,24 @@ L(ret_max):
|
||||
# endif
|
||||
|
||||
L(align_more):
|
||||
- leaq VEC_SIZE(%rdi), %rax
|
||||
+ mov %rdi, %rax
|
||||
/* Align rax to VEC_SIZE. */
|
||||
andq $-VEC_SIZE, %rax
|
||||
# ifdef USE_AS_STRNLEN
|
||||
- movq %rax, %rdx
|
||||
- subq %rdi, %rdx
|
||||
+ movq %rdi, %rdx
|
||||
+ subq %rax, %rdx
|
||||
# ifdef USE_AS_WCSLEN
|
||||
shr $2, %VRDX
|
||||
# endif
|
||||
/* At this point rdx contains [w]chars already compared. */
|
||||
- subq %rsi, %rdx
|
||||
- jae L(ret_max)
|
||||
- negq %rdx
|
||||
+ leaq -CHAR_PER_VEC(%rsi, %rdx), %rdx
|
||||
/* At this point rdx contains number of w[char] needs to go.
|
||||
Now onwards rdx will keep decrementing with each compare. */
|
||||
# endif
|
||||
|
||||
/* Loop unroll 4 times for 4 vector loop. */
|
||||
- VPCMP $0, (%rax), %VMM(0), %k0
|
||||
+ VPCMPEQ VEC_SIZE(%rax), %VMM(0), %k0
|
||||
+ subq $-VEC_SIZE, %rax
|
||||
KMOV %k0, %VRCX
|
||||
test %VRCX, %VRCX
|
||||
jnz L(ret_vec_x1)
|
||||
@@ -105,7 +113,7 @@ L(align_more):
|
||||
jbe L(ret_max)
|
||||
# endif
|
||||
|
||||
- VPCMP $0, VEC_SIZE(%rax), %VMM(0), %k0
|
||||
+ VPCMPEQ VEC_SIZE(%rax), %VMM(0), %k0
|
||||
KMOV %k0, %VRCX
|
||||
test %VRCX, %VRCX
|
||||
jnz L(ret_vec_x2)
|
||||
@@ -115,7 +123,7 @@ L(align_more):
|
||||
jbe L(ret_max)
|
||||
# endif
|
||||
|
||||
- VPCMP $0, (VEC_SIZE * 2)(%rax), %VMM(0), %k0
|
||||
+ VPCMPEQ (VEC_SIZE * 2)(%rax), %VMM(0), %k0
|
||||
KMOV %k0, %VRCX
|
||||
test %VRCX, %VRCX
|
||||
jnz L(ret_vec_x3)
|
||||
@@ -125,7 +133,7 @@ L(align_more):
|
||||
jbe L(ret_max)
|
||||
# endif
|
||||
|
||||
- VPCMP $0, (VEC_SIZE * 3)(%rax), %VMM(0), %k0
|
||||
+ VPCMPEQ (VEC_SIZE * 3)(%rax), %VMM(0), %k0
|
||||
KMOV %k0, %VRCX
|
||||
test %VRCX, %VRCX
|
||||
jnz L(ret_vec_x4)
|
||||
@@ -151,16 +159,10 @@ L(align_more):
|
||||
addq %rcx, %rdx
|
||||
/* Need jump as we don't want to add/subtract rdx for first
|
||||
iteration of 4 x VEC_SIZE aligned loop. */
|
||||
- jmp L(loop_entry)
|
||||
# endif
|
||||
|
||||
.p2align 4,,11
|
||||
L(loop):
|
||||
-# ifdef USE_AS_STRNLEN
|
||||
- subq $(CHAR_PER_VEC * 4), %rdx
|
||||
- jbe L(ret_max)
|
||||
-L(loop_entry):
|
||||
-# endif
|
||||
/* VPMINU and VPCMP combination provide better performance as
|
||||
compared to alternative combinations. */
|
||||
VMOVA (VEC_SIZE * 4)(%rax), %VMM(1)
|
||||
@@ -173,7 +175,18 @@ L(loop_entry):
|
||||
|
||||
subq $-(VEC_SIZE * 4), %rax
|
||||
KORTEST %k0, %k1
|
||||
- jz L(loop)
|
||||
+
|
||||
+# ifndef USE_AS_STRNLEN
|
||||
+ jz L(loop)
|
||||
+# else
|
||||
+ jnz L(loopend)
|
||||
+ subq $(CHAR_PER_VEC * 4), %rdx
|
||||
+ ja L(loop)
|
||||
+ mov %rsi, %rax
|
||||
+ ret
|
||||
+# endif
|
||||
+
|
||||
+L(loopend):
|
||||
|
||||
VPTESTN %VMM(1), %VMM(1), %k2
|
||||
KMOV %k2, %VRCX
|
||||
@@ -245,24 +258,34 @@ L(ret_vec_x1):
|
||||
ret
|
||||
|
||||
L(page_cross):
|
||||
- movl %eax, %ecx
|
||||
-# ifdef USE_AS_WCSLEN
|
||||
+ mov %rdi, %rax
|
||||
+ movl %edi, %ecx
|
||||
andl $(VEC_SIZE - 1), %ecx
|
||||
+# ifdef USE_AS_WCSLEN
|
||||
sarl $2, %ecx
|
||||
# endif
|
||||
/* ecx contains number of w[char] to be skipped as a result
|
||||
of address alignment. */
|
||||
- xorq %rdi, %rax
|
||||
- VPCMP $0, (PAGE_SIZE - VEC_SIZE)(%rax), %VMM(0), %k0
|
||||
- KMOV %k0, %VRAX
|
||||
+ andq $-VEC_SIZE, %rax
|
||||
+ VPCMPEQ (%rax), %VMM(0), %k0
|
||||
+ KMOV %k0, %VRDX
|
||||
/* Ignore number of character for alignment adjustment. */
|
||||
- shr %cl, %VRAX
|
||||
+ shr %cl, %VRDX
|
||||
+# ifdef USE_AS_STRNLEN
|
||||
+ jnz L(page_cross_end)
|
||||
+ movl $CHAR_PER_VEC, %eax
|
||||
+ sub %ecx, %eax
|
||||
+ cmp %rax, %rsi
|
||||
+ ja L(align_more)
|
||||
+# else
|
||||
jz L(align_more)
|
||||
+# endif
|
||||
|
||||
- bsf %VRAX, %VRAX
|
||||
+L(page_cross_end):
|
||||
+ bsf %VRDX, %VRAX
|
||||
# ifdef USE_AS_STRNLEN
|
||||
cmpq %rsi, %rax
|
||||
- cmovnb %rsi, %rax
|
||||
+ cmovnb %esi, %eax
|
||||
# endif
|
||||
ret
|
||||
|
||||
388
glibc-RHEL-175520-9.patch
Normal file
388
glibc-RHEL-175520-9.patch
Normal file
@ -0,0 +1,388 @@
|
||||
commit faaf733f49211439475e50f06716b303ee2644bf
|
||||
Author: Sunil K Pandey <skpgkp2@gmail.com>
|
||||
Date: Tue Aug 9 07:57:29 2022 -0700
|
||||
|
||||
x86_64: Implement evex512 version of strrchr and wcsrchr
|
||||
|
||||
Changes from v1:
|
||||
Use vec api for register.
|
||||
Replace VPCMP with VPCMPEQ
|
||||
Restructure and remove 1 unconditional jump.
|
||||
Change page cross logic to use sall.
|
||||
|
||||
This patch implements following evex512 version of string functions.
|
||||
evex512 version takes up to 30% less cycle as compared to evex,
|
||||
depending on length and alignment.
|
||||
|
||||
- strrchr function using 512 bit vectors.
|
||||
- wcsrchr function using 512 bit vectors.
|
||||
|
||||
Code size data:
|
||||
|
||||
strrchr-evex.o 879 byte
|
||||
strrchr-evex512.o 601 byte (-32%)
|
||||
|
||||
wcsrchr-evex.o 882 byte
|
||||
wcsrchr-evex512.o 572 byte (-35%)
|
||||
|
||||
Placeholder function, not used by any processor at the moment.
|
||||
|
||||
Reviewed-by: Noah Goldstein <goldstein.w.n@gmail.com>
|
||||
|
||||
Conflicts:
|
||||
sysdeps/x86_64/multiarch/Makefile
|
||||
(fixup context)
|
||||
sysdeps/x86_64/multiarch/ifunc-impl-list.c
|
||||
(fixup macro name)
|
||||
|
||||
diff --git a/sysdeps/x86_64/multiarch/Makefile b/sysdeps/x86_64/multiarch/Makefile
|
||||
index 1c8bc2af0ef9c367..80009869b2a76c19 100644
|
||||
--- a/sysdeps/x86_64/multiarch/Makefile
|
||||
+++ b/sysdeps/x86_64/multiarch/Makefile
|
||||
@@ -129,6 +129,7 @@ sysdep_routines += \
|
||||
strrchr-avx2 \
|
||||
strrchr-avx2-rtm \
|
||||
strrchr-evex \
|
||||
+ strrchr-evex512 \
|
||||
strrchr-sse2 \
|
||||
strspn-c \
|
||||
strspn-sse2 \
|
||||
@@ -175,6 +176,7 @@ sysdep_routines += \
|
||||
wcsrchr-avx2 \
|
||||
wcsrchr-avx2-rtm \
|
||||
wcsrchr-evex \
|
||||
+ wcsrchr-evex512 \
|
||||
wcsrchr-sse2 \
|
||||
wmemchr-avx2 \
|
||||
wmemchr-avx2-rtm \
|
||||
diff --git a/sysdeps/x86_64/multiarch/ifunc-impl-list.c b/sysdeps/x86_64/multiarch/ifunc-impl-list.c
|
||||
index cf482eff56fb663c..00bb9a56c039fcae 100644
|
||||
--- a/sysdeps/x86_64/multiarch/ifunc-impl-list.c
|
||||
+++ b/sysdeps/x86_64/multiarch/ifunc-impl-list.c
|
||||
@@ -532,6 +532,11 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,
|
||||
&& CPU_FEATURE_USABLE (BMI1)
|
||||
&& CPU_FEATURE_USABLE (BMI2)),
|
||||
__strrchr_evex)
|
||||
+ IFUNC_IMPL_ADD (array, i, strrchr,
|
||||
+ (CPU_FEATURE_USABLE (AVX512VL)
|
||||
+ && CPU_FEATURE_USABLE (AVX512BW)
|
||||
+ && CPU_FEATURE_USABLE (BMI2)),
|
||||
+ __strrchr_evex512)
|
||||
IFUNC_IMPL_ADD (array, i, strrchr, 1, __strrchr_sse2))
|
||||
|
||||
/* Support sysdeps/x86_64/multiarch/strcmp.c. */
|
||||
@@ -731,6 +736,11 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,
|
||||
&& CPU_FEATURE_USABLE (BMI1)
|
||||
&& CPU_FEATURE_USABLE (BMI2)),
|
||||
__wcsrchr_evex)
|
||||
+ IFUNC_IMPL_ADD (array, i, wcsrchr,
|
||||
+ (CPU_FEATURE_USABLE (AVX512VL)
|
||||
+ && CPU_FEATURE_USABLE (AVX512BW)
|
||||
+ && CPU_FEATURE_USABLE (BMI2)),
|
||||
+ __wcsrchr_evex512)
|
||||
IFUNC_IMPL_ADD (array, i, wcsrchr, 1, __wcsrchr_sse2))
|
||||
|
||||
/* Support sysdeps/x86_64/multiarch/wcscmp.c. */
|
||||
diff --git a/sysdeps/x86_64/multiarch/strrchr-evex-base.S b/sysdeps/x86_64/multiarch/strrchr-evex-base.S
|
||||
new file mode 100644
|
||||
index 0000000000000000..81cab3e0178c8b1e
|
||||
--- /dev/null
|
||||
+++ b/sysdeps/x86_64/multiarch/strrchr-evex-base.S
|
||||
@@ -0,0 +1,264 @@
|
||||
+/* Placeholder function, not used by any processor at the moment.
|
||||
+ Copyright (C) 2022 Free Software Foundation, Inc.
|
||||
+ This file is part of the GNU C Library.
|
||||
+
|
||||
+ The GNU C Library is free software; you can redistribute it and/or
|
||||
+ modify it under the terms of the GNU Lesser General Public
|
||||
+ License as published by the Free Software Foundation; either
|
||||
+ version 2.1 of the License, or (at your option) any later version.
|
||||
+
|
||||
+ The GNU C Library is distributed in the hope that it will be useful,
|
||||
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
+ Lesser General Public License for more details.
|
||||
+
|
||||
+ You should have received a copy of the GNU Lesser General Public
|
||||
+ License along with the GNU C Library; if not, see
|
||||
+ <https://www.gnu.org/licenses/>. */
|
||||
+
|
||||
+/* UNUSED. Exists purely as reference implementation. */
|
||||
+
|
||||
+#include <isa-level.h>
|
||||
+
|
||||
+#if ISA_SHOULD_BUILD (4)
|
||||
+
|
||||
+# include <sysdep.h>
|
||||
+
|
||||
+# ifdef USE_AS_WCSRCHR
|
||||
+# define CHAR_SIZE 4
|
||||
+# define VPBROADCAST vpbroadcastd
|
||||
+# define VPCMPEQ vpcmpeqd
|
||||
+# define VPMINU vpminud
|
||||
+# define VPTESTN vptestnmd
|
||||
+# else
|
||||
+# define CHAR_SIZE 1
|
||||
+# define VPBROADCAST vpbroadcastb
|
||||
+# define VPCMPEQ vpcmpeqb
|
||||
+# define VPMINU vpminub
|
||||
+# define VPTESTN vptestnmb
|
||||
+# endif
|
||||
+
|
||||
+# define PAGE_SIZE 4096
|
||||
+# define CHAR_PER_VEC (VEC_SIZE / CHAR_SIZE)
|
||||
+
|
||||
+ .section SECTION(.text), "ax", @progbits
|
||||
+/* Aligning entry point to 64 byte, provides better performance for
|
||||
+ one vector length string. */
|
||||
+ENTRY_P2ALIGN (STRRCHR, 6)
|
||||
+
|
||||
+ /* Broadcast CHAR to VMM(0). */
|
||||
+ VPBROADCAST %esi, %VMM(0)
|
||||
+ movl %edi, %eax
|
||||
+ sall $20, %eax
|
||||
+ cmpl $((PAGE_SIZE - VEC_SIZE) << 20), %eax
|
||||
+ ja L(page_cross)
|
||||
+
|
||||
+L(page_cross_continue):
|
||||
+ /* Compare [w]char for null, mask bit will be set for match. */
|
||||
+ VMOVU (%rdi), %VMM(1)
|
||||
+
|
||||
+ VPTESTN %VMM(1), %VMM(1), %k1
|
||||
+ KMOV %k1, %VRCX
|
||||
+ test %VRCX, %VRCX
|
||||
+ jz L(align_more)
|
||||
+
|
||||
+ VPCMPEQ %VMM(1), %VMM(0), %k0
|
||||
+ KMOV %k0, %VRAX
|
||||
+ BLSMSK %VRCX, %VRCX
|
||||
+ and %VRCX, %VRAX
|
||||
+ jz L(ret)
|
||||
+
|
||||
+ BSR %VRAX, %VRAX
|
||||
+# ifdef USE_AS_WCSRCHR
|
||||
+ leaq (%rdi, %rax, CHAR_SIZE), %rax
|
||||
+# else
|
||||
+ add %rdi, %rax
|
||||
+# endif
|
||||
+L(ret):
|
||||
+ ret
|
||||
+
|
||||
+L(vector_x2_end):
|
||||
+ VPCMPEQ %VMM(2), %VMM(0), %k2
|
||||
+ KMOV %k2, %VRAX
|
||||
+ BLSMSK %VRCX, %VRCX
|
||||
+ and %VRCX, %VRAX
|
||||
+ jz L(vector_x1_ret)
|
||||
+
|
||||
+ BSR %VRAX, %VRAX
|
||||
+ leaq (VEC_SIZE)(%rdi, %rax, CHAR_SIZE), %rax
|
||||
+ ret
|
||||
+
|
||||
+ /* Check the first vector at very last to look for match. */
|
||||
+L(vector_x1_ret):
|
||||
+ VPCMPEQ %VMM(1), %VMM(0), %k2
|
||||
+ KMOV %k2, %VRAX
|
||||
+ test %VRAX, %VRAX
|
||||
+ jz L(ret)
|
||||
+
|
||||
+ BSR %VRAX, %VRAX
|
||||
+# ifdef USE_AS_WCSRCHR
|
||||
+ leaq (%rsi, %rax, CHAR_SIZE), %rax
|
||||
+# else
|
||||
+ add %rsi, %rax
|
||||
+# endif
|
||||
+ ret
|
||||
+
|
||||
+L(align_more):
|
||||
+ /* Zero r8 to store match result. */
|
||||
+ xorl %r8d, %r8d
|
||||
+ /* Save pointer of first vector, in case if no match found. */
|
||||
+ movq %rdi, %rsi
|
||||
+ /* Align pointer to vector size. */
|
||||
+ andq $-VEC_SIZE, %rdi
|
||||
+ /* Loop unroll for 2 vector loop. */
|
||||
+ VMOVA (VEC_SIZE)(%rdi), %VMM(2)
|
||||
+ VPTESTN %VMM(2), %VMM(2), %k0
|
||||
+ KMOV %k0, %VRCX
|
||||
+ test %VRCX, %VRCX
|
||||
+ jnz L(vector_x2_end)
|
||||
+
|
||||
+ /* Save pointer of second vector, in case if no match
|
||||
+ found. */
|
||||
+ movq %rdi, %r9
|
||||
+ /* Align address to VEC_SIZE * 2 for loop. */
|
||||
+ andq $-(VEC_SIZE * 2), %rdi
|
||||
+
|
||||
+ .p2align 4,,11
|
||||
+L(loop):
|
||||
+ /* 2 vector loop, as it provide better performance as compared
|
||||
+ to 4 vector loop. */
|
||||
+ VMOVA (VEC_SIZE * 2)(%rdi), %VMM(3)
|
||||
+ VMOVA (VEC_SIZE * 3)(%rdi), %VMM(4)
|
||||
+ VPCMPEQ %VMM(3), %VMM(0), %k1
|
||||
+ VPCMPEQ %VMM(4), %VMM(0), %k2
|
||||
+ VPMINU %VMM(3), %VMM(4), %VMM(5)
|
||||
+ VPTESTN %VMM(5), %VMM(5), %k0
|
||||
+ KOR %k1, %k2, %k3
|
||||
+ subq $-(VEC_SIZE * 2), %rdi
|
||||
+ /* If k0 and k3 zero, match and end of string not found. */
|
||||
+ KORTEST %k0, %k3
|
||||
+ jz L(loop)
|
||||
+
|
||||
+ /* If k0 is non zero, end of string found. */
|
||||
+ KORTEST %k0, %k0
|
||||
+ jnz L(endloop)
|
||||
+
|
||||
+ lea VEC_SIZE(%rdi), %r8
|
||||
+ /* A match found, it need to be stored in r8 before loop
|
||||
+ continue. */
|
||||
+ /* Check second vector first. */
|
||||
+ KMOV %k2, %VRDX
|
||||
+ test %VRDX, %VRDX
|
||||
+ jnz L(loop_vec_x2_match)
|
||||
+
|
||||
+ KMOV %k1, %VRDX
|
||||
+ /* Match is in first vector, rdi offset need to be substracted
|
||||
+ by VEC_SIZE. */
|
||||
+ sub $VEC_SIZE, %r8
|
||||
+
|
||||
+ /* If second vector doesn't have match, first vector must
|
||||
+ have match. */
|
||||
+L(loop_vec_x2_match):
|
||||
+ BSR %VRDX, %VRDX
|
||||
+# ifdef USE_AS_WCSRCHR
|
||||
+ sal $2, %rdx
|
||||
+# endif
|
||||
+ add %rdx, %r8
|
||||
+ jmp L(loop)
|
||||
+
|
||||
+L(endloop):
|
||||
+ /* Check if string end in first loop vector. */
|
||||
+ VPTESTN %VMM(3), %VMM(3), %k0
|
||||
+ KMOV %k0, %VRCX
|
||||
+ test %VRCX, %VRCX
|
||||
+ jnz L(loop_vector_x1_end)
|
||||
+
|
||||
+ /* Check if it has match in first loop vector. */
|
||||
+ KMOV %k1, %VRAX
|
||||
+ test %VRAX, %VRAX
|
||||
+ jz L(loop_vector_x2_end)
|
||||
+
|
||||
+ BSR %VRAX, %VRAX
|
||||
+ leaq (%rdi, %rax, CHAR_SIZE), %r8
|
||||
+
|
||||
+ /* String must end in second loop vector. */
|
||||
+L(loop_vector_x2_end):
|
||||
+ VPTESTN %VMM(4), %VMM(4), %k0
|
||||
+ KMOV %k0, %VRCX
|
||||
+ KMOV %k2, %VRAX
|
||||
+ BLSMSK %VRCX, %VRCX
|
||||
+ /* Check if it has match in second loop vector. */
|
||||
+ and %VRCX, %VRAX
|
||||
+ jz L(check_last_match)
|
||||
+
|
||||
+ BSR %VRAX, %VRAX
|
||||
+ leaq (VEC_SIZE)(%rdi, %rax, CHAR_SIZE), %rax
|
||||
+ ret
|
||||
+
|
||||
+ /* String end in first loop vector. */
|
||||
+L(loop_vector_x1_end):
|
||||
+ KMOV %k1, %VRAX
|
||||
+ BLSMSK %VRCX, %VRCX
|
||||
+ /* Check if it has match in second loop vector. */
|
||||
+ and %VRCX, %VRAX
|
||||
+ jz L(check_last_match)
|
||||
+
|
||||
+ BSR %VRAX, %VRAX
|
||||
+ leaq (%rdi, %rax, CHAR_SIZE), %rax
|
||||
+ ret
|
||||
+
|
||||
+ /* No match in first and second loop vector. */
|
||||
+L(check_last_match):
|
||||
+ /* Check if any match recorded in r8. */
|
||||
+ test %r8, %r8
|
||||
+ jz L(vector_x2_ret)
|
||||
+ movq %r8, %rax
|
||||
+ ret
|
||||
+
|
||||
+ /* No match recorded in r8. Check the second saved vector
|
||||
+ in begining. */
|
||||
+L(vector_x2_ret):
|
||||
+ VPCMPEQ %VMM(2), %VMM(0), %k2
|
||||
+ KMOV %k2, %VRAX
|
||||
+ test %VRAX, %VRAX
|
||||
+ jz L(vector_x1_ret)
|
||||
+
|
||||
+ /* Match found in the second saved vector. */
|
||||
+ BSR %VRAX, %VRAX
|
||||
+ leaq (VEC_SIZE)(%r9, %rax, CHAR_SIZE), %rax
|
||||
+ ret
|
||||
+
|
||||
+L(page_cross):
|
||||
+ mov %rdi, %rax
|
||||
+ movl %edi, %ecx
|
||||
+
|
||||
+# ifdef USE_AS_WCSRCHR
|
||||
+ /* Calculate number of compare result bits to be skipped for
|
||||
+ wide string alignment adjustment. */
|
||||
+ andl $(VEC_SIZE - 1), %ecx
|
||||
+ sarl $2, %ecx
|
||||
+# endif
|
||||
+ /* ecx contains number of w[char] to be skipped as a result
|
||||
+ of address alignment. */
|
||||
+ andq $-VEC_SIZE, %rax
|
||||
+ VMOVA (%rax), %VMM(1)
|
||||
+ VPTESTN %VMM(1), %VMM(1), %k1
|
||||
+ KMOV %k1, %VRAX
|
||||
+ SHR %cl, %VRAX
|
||||
+ jz L(page_cross_continue)
|
||||
+ VPCMPEQ %VMM(1), %VMM(0), %k0
|
||||
+ KMOV %k0, %VRDX
|
||||
+ SHR %cl, %VRDX
|
||||
+ BLSMSK %VRAX, %VRAX
|
||||
+ and %VRDX, %VRAX
|
||||
+ jz L(ret)
|
||||
+ BSR %VRAX, %VRAX
|
||||
+# ifdef USE_AS_WCSRCHR
|
||||
+ leaq (%rdi, %rax, CHAR_SIZE), %rax
|
||||
+# else
|
||||
+ add %rdi, %rax
|
||||
+# endif
|
||||
+
|
||||
+ ret
|
||||
+END (STRRCHR)
|
||||
+#endif
|
||||
diff --git a/sysdeps/x86_64/multiarch/strrchr-evex512.S b/sysdeps/x86_64/multiarch/strrchr-evex512.S
|
||||
new file mode 100644
|
||||
index 0000000000000000..7d81e663759a8b28
|
||||
--- /dev/null
|
||||
+++ b/sysdeps/x86_64/multiarch/strrchr-evex512.S
|
||||
@@ -0,0 +1,8 @@
|
||||
+# ifndef STRRCHR
|
||||
+# define STRRCHR __strrchr_evex512
|
||||
+# endif
|
||||
+
|
||||
+#include "x86-evex512-vecs.h"
|
||||
+#include "reg-macros.h"
|
||||
+
|
||||
+#include "strrchr-evex-base.S"
|
||||
diff --git a/sysdeps/x86_64/multiarch/wcsrchr-evex512.S b/sysdeps/x86_64/multiarch/wcsrchr-evex512.S
|
||||
new file mode 100644
|
||||
index 0000000000000000..f241f63716401a97
|
||||
--- /dev/null
|
||||
+++ b/sysdeps/x86_64/multiarch/wcsrchr-evex512.S
|
||||
@@ -0,0 +1,13 @@
|
||||
+#ifndef WCSRCHR
|
||||
+# define WCSRCHR __wcsrchr_evex512
|
||||
+#endif
|
||||
+
|
||||
+#define STRRCHR WCSRCHR
|
||||
+#define USE_AS_WCSRCHR 1
|
||||
+
|
||||
+#define USE_WIDE_CHAR 1
|
||||
+
|
||||
+#include "x86-evex512-vecs.h"
|
||||
+#include "reg-macros.h"
|
||||
+
|
||||
+#include "strrchr-evex512.S"
|
||||
Loading…
Reference in New Issue
Block a user