glibc/glibc-upstream-2.34-175.patch

654 lines
18 KiB
Diff
Raw Normal View History

Import glibc-2.34-32.fc35 from f35 * Thu Apr 28 2022 Carlos O'Donell <carlos@redhat.com> - 2.34-32 - Sync with upstream branch release/2.34/master, commit c66c92181ddbd82306537a608e8c0282587131de: - posix/glob.c: update from gnulib (BZ#25659) - linux: Fix fchmodat with AT_SYMLINK_NOFOLLOW for 64 bit time_t (BZ#29097) * Wed Apr 27 2022 Carlos O'Donell <carlos@redhat.com> - 2.34-31 - Sync with upstream branch release/2.34/master, commit 55640ed3fde48360a8e8083be4843bd2dc7cecfe: - i386: Regenerate ulps - linux: Fix missing internal 64 bit time_t stat usage - x86: Optimize L(less_vec) case in memcmp-evex-movbe.S - x86: Don't set Prefer_No_AVX512 for processors with AVX512 and AVX-VNNI - x86-64: Use notl in EVEX strcmp [BZ #28646] - x86: Shrink memcmp-sse4.S code size - x86: Double size of ERMS rep_movsb_threshold in dl-cacheinfo.h - x86: Optimize memmove-vec-unaligned-erms.S - x86-64: Replace movzx with movzbl - x86-64: Remove Prefer_AVX2_STRCMP - x86-64: Improve EVEX strcmp with masked load - x86: Replace sse2 instructions with avx in memcmp-evex-movbe.S - x86: Optimize memset-vec-unaligned-erms.S - x86: Optimize memcmp-evex-movbe.S for frontend behavior and size - x86: Modify ENTRY in sysdep.h so that p2align can be specified - x86-64: Optimize load of all bits set into ZMM register [BZ #28252] - scripts/glibcelf.py: Mark as UNSUPPORTED on Python 3.5 and earlier - dlfcn: Do not use rtld_active () to determine ld.so state (bug 29078) - INSTALL: Rephrase -with-default-link documentation - misc: Fix rare fortify crash on wchar funcs. [BZ 29030] - Default to --with-default-link=no (bug 25812) - scripts: Add glibcelf.py module * Thu Apr 21 2022 Carlos O'Donell <carlos@redhat.com> - 2.34-30 - Sync with upstream branch release/2.34/master, commit 71326f1f2fd09dafb9c34404765fb88129e94237: - nptl: Fix pthread_cancel cancelhandling atomic operations - mips: Fix mips64n32 64 bit time_t stat support (BZ#29069) - hurd: Fix arbitrary error code - nptl: Handle spurious EINTR when thread cancellation is disabled (BZ#29029) - S390: Add new s390 platform z16. - NEWS: Update fixed bug list for LD_AUDIT backports. - hppa: Fix bind-now audit (BZ #28857) - elf: Replace tst-audit24bmod2.so with tst-audit24bmod2 - Fix elf/tst-audit25a with default bind now toolchains - elf: Fix runtime linker auditing on aarch64 (BZ #26643) - elf: Issue la_symbind for bind-now (BZ #23734) - elf: Fix initial-exec TLS access on audit modules (BZ #28096) - elf: Add la_activity during application exit - elf: Do not fail for failed dlmopen on audit modules (BZ #28061) - elf: Issue audit la_objopen for vDSO - elf: Add audit tests for modules with TLSDESC - elf: Avoid unnecessary slowdown from profiling with audit (BZ#15533) - elf: Add _dl_audit_pltexit - elf: Add _dl_audit_pltenter - elf: Add _dl_audit_preinit - elf: Add _dl_audit_symbind_alt and _dl_audit_symbind - elf: Add _dl_audit_objclose - elf: Add _dl_audit_objsearch - elf: Add _dl_audit_activity_map and _dl_audit_activity_nsid - elf: Add _dl_audit_objopen - elf: Move la_activity (LA_ACT_ADD) after _dl_add_to_namespace_list() (BZ #28062) - elf: Move LAV_CURRENT to link_lavcurrent.h - elf: Fix elf_get_dynamic_info() for bootstrap - elf: Fix dynamic-link.h usage on rtld.c - elf: Fix elf_get_dynamic_info definition - elf: Avoid nested functions in the loader [BZ #27220] - powerpc: Delete unneeded ELF_MACHINE_BEFORE_RTLD_RELOC - hppa: Use END instead of PSEUDO_END in swapcontext.S - hppa: Implement swapcontext in assembler (bug 28960) Resolves: #2003291 Resolves: #2064181 Resolves: #2072328 Resolves: #2075713 Resolves: #2077838
2022-04-21 22:26:44 +00:00
commit 5ec3416853c4150c4d13312e05f93a053586d528
Author: Noah Goldstein <goldstein.w.n@gmail.com>
Date: Tue Sep 21 18:45:03 2021 -0500
x86: Optimize memcmp-evex-movbe.S for frontend behavior and size
No bug.
The frontend optimizations are to:
1. Reorganize logically connected basic blocks so they are either in
the same cache line or adjacent cache lines.
2. Avoid cases when basic blocks unnecissarily cross cache lines.
3. Try and 32 byte align any basic blocks possible without sacrificing
code size. Smaller / Less hot basic blocks are used for this.
Overall code size shrunk by 168 bytes. This should make up for any
extra costs due to aligning to 64 bytes.
In general performance before deviated a great deal dependending on
whether entry alignment % 64 was 0, 16, 32, or 48. These changes
essentially make it so that the current implementation is at least
equal to the best alignment of the original for any arguments.
The only additional optimization is in the page cross case. Branch on
equals case was removed from the size == [4, 7] case. As well the [4,
7] and [2, 3] case where swapped as [4, 7] is likely a more hot
argument size.
test-memcmp and test-wmemcmp are both passing.
(cherry picked from commit 1bd8b8d58fc9967cc073d2c13bfb6befefca2faa)
diff --git a/sysdeps/x86_64/multiarch/memcmp-evex-movbe.S b/sysdeps/x86_64/multiarch/memcmp-evex-movbe.S
index 654dc7ac8ccb9445..2761b54f2e7dea9f 100644
--- a/sysdeps/x86_64/multiarch/memcmp-evex-movbe.S
+++ b/sysdeps/x86_64/multiarch/memcmp-evex-movbe.S
@@ -34,7 +34,24 @@
area.
7. Use 2 vector compares when size is 2 * CHAR_PER_VEC or less.
8. Use 4 vector compares when size is 4 * CHAR_PER_VEC or less.
- 9. Use 8 vector compares when size is 8 * CHAR_PER_VEC or less. */
+ 9. Use 8 vector compares when size is 8 * CHAR_PER_VEC or less.
+
+When possible the implementation tries to optimize for frontend in the
+following ways:
+Throughput:
+ 1. All code sections that fit are able to run optimally out of the
+ LSD.
+ 2. All code sections that fit are able to run optimally out of the
+ DSB
+ 3. Basic blocks are contained in minimum number of fetch blocks
+ necessary.
+
+Latency:
+ 1. Logically connected basic blocks are put in the same
+ cache-line.
+ 2. Logically connected basic blocks that do not fit in the same
+ cache-line are put in adjacent lines. This can get beneficial
+ L2 spatial prefetching and L1 next-line prefetching. */
# include <sysdep.h>
@@ -47,9 +64,11 @@
# ifdef USE_AS_WMEMCMP
# define CHAR_SIZE 4
# define VPCMP vpcmpd
+# define VPTEST vptestmd
# else
# define CHAR_SIZE 1
# define VPCMP vpcmpub
+# define VPTEST vptestmb
# endif
# define VEC_SIZE 32
@@ -75,7 +94,9 @@
*/
.section .text.evex,"ax",@progbits
-ENTRY (MEMCMP)
+/* Cache align memcmp entry. This allows for much more thorough
+ frontend optimization. */
+ENTRY_P2ALIGN (MEMCMP, 6)
# ifdef __ILP32__
/* Clear the upper 32 bits. */
movl %edx, %edx
@@ -89,7 +110,7 @@ ENTRY (MEMCMP)
VPCMP $4, (%rdi), %YMM1, %k1
kmovd %k1, %eax
/* NB: eax must be destination register if going to
- L(return_vec_[0,2]). For L(return_vec_3 destination register
+ L(return_vec_[0,2]). For L(return_vec_3) destination register
must be ecx. */
testl %eax, %eax
jnz L(return_vec_0)
@@ -121,10 +142,6 @@ ENTRY (MEMCMP)
testl %ecx, %ecx
jnz L(return_vec_3)
- /* Zero YMM0. 4x VEC reduction is done with vpxor + vtern so
- compare with zero to get a mask is needed. */
- vpxorq %XMM0, %XMM0, %XMM0
-
/* Go to 4x VEC loop. */
cmpq $(CHAR_PER_VEC * 8), %rdx
ja L(more_8x_vec)
@@ -148,47 +165,61 @@ ENTRY (MEMCMP)
VMOVU (VEC_SIZE * 2)(%rsi), %YMM3
vpxorq (VEC_SIZE * 2)(%rdi), %YMM3, %YMM3
- /* Or together YMM1, YMM2, and YMM3 into YMM3. */
- vpternlogd $0xfe, %YMM1, %YMM2, %YMM3
VMOVU (VEC_SIZE * 3)(%rsi), %YMM4
/* Ternary logic to xor (VEC_SIZE * 3)(%rdi) with YMM4 while
- oring with YMM3. Result is stored in YMM4. */
- vpternlogd $0xde, (VEC_SIZE * 3)(%rdi), %YMM3, %YMM4
- /* Compare YMM4 with 0. If any 1s s1 and s2 don't match. */
- VPCMP $4, %YMM4, %YMM0, %k1
+ oring with YMM1. Result is stored in YMM4. */
+ vpternlogd $0xde, (VEC_SIZE * 3)(%rdi), %YMM1, %YMM4
+
+ /* Or together YMM2, YMM3, and YMM4 into YMM4. */
+ vpternlogd $0xfe, %YMM2, %YMM3, %YMM4
+
+ /* Test YMM4 against itself. Store any CHAR mismatches in k1.
+ */
+ VPTEST %YMM4, %YMM4, %k1
+ /* k1 must go to ecx for L(return_vec_0_1_2_3). */
kmovd %k1, %ecx
testl %ecx, %ecx
jnz L(return_vec_0_1_2_3)
/* NB: eax must be zero to reach here. */
ret
- /* NB: aligning 32 here allows for the rest of the jump targets
- to be tuned for 32 byte alignment. Most important this ensures
- the L(more_8x_vec) loop is 32 byte aligned. */
- .p2align 5
-L(less_vec):
- /* Check if one or less CHAR. This is necessary for size = 0 but
- is also faster for size = CHAR_SIZE. */
- cmpl $1, %edx
- jbe L(one_or_less)
+ .p2align 4
+L(8x_end_return_vec_0_1_2_3):
+ movq %rdx, %rdi
+L(8x_return_vec_0_1_2_3):
+ addq %rdi, %rsi
+L(return_vec_0_1_2_3):
+ VPTEST %YMM1, %YMM1, %k0
+ kmovd %k0, %eax
+ testl %eax, %eax
+ jnz L(return_vec_0)
- /* Check if loading one VEC from either s1 or s2 could cause a
- page cross. This can have false positives but is by far the
- fastest method. */
- movl %edi, %eax
- orl %esi, %eax
- andl $(PAGE_SIZE - 1), %eax
- cmpl $(PAGE_SIZE - VEC_SIZE), %eax
- jg L(page_cross_less_vec)
+ VPTEST %YMM2, %YMM2, %k0
+ kmovd %k0, %eax
+ testl %eax, %eax
+ jnz L(return_vec_1)
- /* No page cross possible. */
- VMOVU (%rsi), %YMM2
- VPCMP $4, (%rdi), %YMM2, %k1
- kmovd %k1, %eax
- /* Create mask in ecx for potentially in bound matches. */
- bzhil %edx, %eax, %eax
- jnz L(return_vec_0)
+ VPTEST %YMM3, %YMM3, %k0
+ kmovd %k0, %eax
+ testl %eax, %eax
+ jnz L(return_vec_2)
+L(return_vec_3):
+ /* bsf saves 1 byte from tzcnt. This keep L(return_vec_3) in one
+ fetch block and the entire L(*return_vec_0_1_2_3) in 1 cache
+ line. */
+ bsfl %ecx, %ecx
+# ifdef USE_AS_WMEMCMP
+ movl (VEC_SIZE * 3)(%rdi, %rcx, CHAR_SIZE), %eax
+ xorl %edx, %edx
+ cmpl (VEC_SIZE * 3)(%rsi, %rcx, CHAR_SIZE), %eax
+ setg %dl
+ leal -1(%rdx, %rdx), %eax
+# else
+ movzbl (VEC_SIZE * 3)(%rdi, %rcx), %eax
+ movzbl (VEC_SIZE * 3)(%rsi, %rcx), %ecx
+ subl %ecx, %eax
+# endif
ret
.p2align 4
@@ -209,10 +240,11 @@ L(return_vec_0):
# endif
ret
- /* NB: No p2align necessary. Alignment % 16 is naturally 1
- which is good enough for a target not in a loop. */
+ .p2align 4
L(return_vec_1):
- tzcntl %eax, %eax
+ /* bsf saves 1 byte over tzcnt and keeps L(return_vec_1) in one
+ fetch block. */
+ bsfl %eax, %eax
# ifdef USE_AS_WMEMCMP
movl VEC_SIZE(%rdi, %rax, CHAR_SIZE), %ecx
xorl %edx, %edx
@@ -226,10 +258,11 @@ L(return_vec_1):
# endif
ret
- /* NB: No p2align necessary. Alignment % 16 is naturally 2
- which is good enough for a target not in a loop. */
+ .p2align 4,, 10
L(return_vec_2):
- tzcntl %eax, %eax
+ /* bsf saves 1 byte over tzcnt and keeps L(return_vec_2) in one
+ fetch block. */
+ bsfl %eax, %eax
# ifdef USE_AS_WMEMCMP
movl (VEC_SIZE * 2)(%rdi, %rax, CHAR_SIZE), %ecx
xorl %edx, %edx
@@ -243,40 +276,6 @@ L(return_vec_2):
# endif
ret
- .p2align 4
-L(8x_return_vec_0_1_2_3):
- /* Returning from L(more_8x_vec) requires restoring rsi. */
- addq %rdi, %rsi
-L(return_vec_0_1_2_3):
- VPCMP $4, %YMM1, %YMM0, %k0
- kmovd %k0, %eax
- testl %eax, %eax
- jnz L(return_vec_0)
-
- VPCMP $4, %YMM2, %YMM0, %k0
- kmovd %k0, %eax
- testl %eax, %eax
- jnz L(return_vec_1)
-
- VPCMP $4, %YMM3, %YMM0, %k0
- kmovd %k0, %eax
- testl %eax, %eax
- jnz L(return_vec_2)
-L(return_vec_3):
- tzcntl %ecx, %ecx
-# ifdef USE_AS_WMEMCMP
- movl (VEC_SIZE * 3)(%rdi, %rcx, CHAR_SIZE), %eax
- xorl %edx, %edx
- cmpl (VEC_SIZE * 3)(%rsi, %rcx, CHAR_SIZE), %eax
- setg %dl
- leal -1(%rdx, %rdx), %eax
-# else
- movzbl (VEC_SIZE * 3)(%rdi, %rcx), %eax
- movzbl (VEC_SIZE * 3)(%rsi, %rcx), %ecx
- subl %ecx, %eax
-# endif
- ret
-
.p2align 4
L(more_8x_vec):
/* Set end of s1 in rdx. */
@@ -288,21 +287,19 @@ L(more_8x_vec):
andq $-VEC_SIZE, %rdi
/* Adjust because first 4x vec where check already. */
subq $-(VEC_SIZE * 4), %rdi
+
.p2align 4
L(loop_4x_vec):
VMOVU (%rsi, %rdi), %YMM1
vpxorq (%rdi), %YMM1, %YMM1
-
VMOVU VEC_SIZE(%rsi, %rdi), %YMM2
vpxorq VEC_SIZE(%rdi), %YMM2, %YMM2
-
VMOVU (VEC_SIZE * 2)(%rsi, %rdi), %YMM3
vpxorq (VEC_SIZE * 2)(%rdi), %YMM3, %YMM3
- vpternlogd $0xfe, %YMM1, %YMM2, %YMM3
-
VMOVU (VEC_SIZE * 3)(%rsi, %rdi), %YMM4
- vpternlogd $0xde, (VEC_SIZE * 3)(%rdi), %YMM3, %YMM4
- VPCMP $4, %YMM4, %YMM0, %k1
+ vpternlogd $0xde, (VEC_SIZE * 3)(%rdi), %YMM1, %YMM4
+ vpternlogd $0xfe, %YMM2, %YMM3, %YMM4
+ VPTEST %YMM4, %YMM4, %k1
kmovd %k1, %ecx
testl %ecx, %ecx
jnz L(8x_return_vec_0_1_2_3)
@@ -319,28 +316,25 @@ L(loop_4x_vec):
cmpl $(VEC_SIZE * 2), %edi
jae L(8x_last_2x_vec)
+ vpxorq (VEC_SIZE * 2)(%rdx), %YMM3, %YMM3
+
VMOVU (%rsi, %rdx), %YMM1
vpxorq (%rdx), %YMM1, %YMM1
VMOVU VEC_SIZE(%rsi, %rdx), %YMM2
vpxorq VEC_SIZE(%rdx), %YMM2, %YMM2
-
- vpxorq (VEC_SIZE * 2)(%rdx), %YMM3, %YMM3
- vpternlogd $0xfe, %YMM1, %YMM2, %YMM3
-
VMOVU (VEC_SIZE * 3)(%rsi, %rdx), %YMM4
- vpternlogd $0xde, (VEC_SIZE * 3)(%rdx), %YMM3, %YMM4
- VPCMP $4, %YMM4, %YMM0, %k1
+ vpternlogd $0xde, (VEC_SIZE * 3)(%rdx), %YMM1, %YMM4
+ vpternlogd $0xfe, %YMM2, %YMM3, %YMM4
+ VPTEST %YMM4, %YMM4, %k1
kmovd %k1, %ecx
- /* Restore s1 pointer to rdi. */
- movq %rdx, %rdi
testl %ecx, %ecx
- jnz L(8x_return_vec_0_1_2_3)
+ jnz L(8x_end_return_vec_0_1_2_3)
/* NB: eax must be zero to reach here. */
ret
/* Only entry is from L(more_8x_vec). */
- .p2align 4
+ .p2align 4,, 10
L(8x_last_2x_vec):
VPCMP $4, (VEC_SIZE * 2)(%rdx), %YMM3, %k1
kmovd %k1, %eax
@@ -355,7 +349,31 @@ L(8x_last_1x_vec):
jnz L(8x_return_vec_3)
ret
- .p2align 4
+ /* Not ideally aligned (at offset +9 bytes in fetch block) but
+ not aligning keeps it in the same cache line as
+ L(8x_last_1x/2x_vec) so likely worth it. As well, saves code
+ size. */
+ .p2align 4,, 4
+L(8x_return_vec_2):
+ subq $VEC_SIZE, %rdx
+L(8x_return_vec_3):
+ bsfl %eax, %eax
+# ifdef USE_AS_WMEMCMP
+ leaq (%rdx, %rax, CHAR_SIZE), %rax
+ movl (VEC_SIZE * 3)(%rax), %ecx
+ xorl %edx, %edx
+ cmpl (VEC_SIZE * 3)(%rsi, %rax), %ecx
+ setg %dl
+ leal -1(%rdx, %rdx), %eax
+# else
+ addq %rdx, %rax
+ movzbl (VEC_SIZE * 3)(%rsi, %rax), %ecx
+ movzbl (VEC_SIZE * 3)(%rax), %eax
+ subl %ecx, %eax
+# endif
+ ret
+
+ .p2align 4,, 10
L(last_2x_vec):
/* Check second to last VEC. */
VMOVU -(VEC_SIZE * 2)(%rsi, %rdx, CHAR_SIZE), %YMM1
@@ -374,26 +392,49 @@ L(last_1x_vec):
jnz L(return_vec_0_end)
ret
- .p2align 4
-L(8x_return_vec_2):
- subq $VEC_SIZE, %rdx
-L(8x_return_vec_3):
- tzcntl %eax, %eax
+ .p2align 4,, 10
+L(return_vec_1_end):
+ /* Use bsf to save code size. This is necessary to have
+ L(one_or_less) fit in aligning bytes between. */
+ bsfl %eax, %eax
+ addl %edx, %eax
# ifdef USE_AS_WMEMCMP
- leaq (%rdx, %rax, CHAR_SIZE), %rax
- movl (VEC_SIZE * 3)(%rax), %ecx
+ movl -(VEC_SIZE * 2)(%rdi, %rax, CHAR_SIZE), %ecx
xorl %edx, %edx
- cmpl (VEC_SIZE * 3)(%rsi, %rax), %ecx
+ cmpl -(VEC_SIZE * 2)(%rsi, %rax, CHAR_SIZE), %ecx
setg %dl
leal -1(%rdx, %rdx), %eax
# else
- addq %rdx, %rax
- movzbl (VEC_SIZE * 3)(%rsi, %rax), %ecx
- movzbl (VEC_SIZE * 3)(%rax), %eax
+ movzbl -(VEC_SIZE * 2)(%rsi, %rax), %ecx
+ movzbl -(VEC_SIZE * 2)(%rdi, %rax), %eax
subl %ecx, %eax
# endif
ret
+ /* NB: L(one_or_less) fits in alignment padding between
+ L(return_vec_1_end) and L(return_vec_0_end). */
+# ifdef USE_AS_WMEMCMP
+L(one_or_less):
+ jb L(zero)
+ movl (%rdi), %ecx
+ xorl %edx, %edx
+ cmpl (%rsi), %ecx
+ je L(zero)
+ setg %dl
+ leal -1(%rdx, %rdx), %eax
+ ret
+# else
+L(one_or_less):
+ jb L(zero)
+ movzbl (%rsi), %ecx
+ movzbl (%rdi), %eax
+ subl %ecx, %eax
+ ret
+# endif
+L(zero):
+ xorl %eax, %eax
+ ret
+
.p2align 4
L(return_vec_0_end):
tzcntl %eax, %eax
@@ -412,23 +453,56 @@ L(return_vec_0_end):
ret
.p2align 4
-L(return_vec_1_end):
+L(less_vec):
+ /* Check if one or less CHAR. This is necessary for size == 0
+ but is also faster for size == CHAR_SIZE. */
+ cmpl $1, %edx
+ jbe L(one_or_less)
+
+ /* Check if loading one VEC from either s1 or s2 could cause a
+ page cross. This can have false positives but is by far the
+ fastest method. */
+ movl %edi, %eax
+ orl %esi, %eax
+ andl $(PAGE_SIZE - 1), %eax
+ cmpl $(PAGE_SIZE - VEC_SIZE), %eax
+ jg L(page_cross_less_vec)
+
+ /* No page cross possible. */
+ VMOVU (%rsi), %YMM2
+ VPCMP $4, (%rdi), %YMM2, %k1
+ kmovd %k1, %eax
+ /* Check if any matches where in bounds. Intentionally not
+ storing result in eax to limit dependency chain if it goes to
+ L(return_vec_0_lv). */
+ bzhil %edx, %eax, %edx
+ jnz L(return_vec_0_lv)
+ xorl %eax, %eax
+ ret
+
+ /* Essentially duplicate of L(return_vec_0). Ends up not costing
+ any code as shrinks L(less_vec) by allowing 2-byte encoding of
+ the jump and ends up fitting in aligning bytes. As well fits on
+ same cache line as L(less_vec) so also saves a line from having
+ to be fetched on cold calls to memcmp. */
+ .p2align 4,, 4
+L(return_vec_0_lv):
tzcntl %eax, %eax
- addl %edx, %eax
# ifdef USE_AS_WMEMCMP
- movl -(VEC_SIZE * 2)(%rdi, %rax, CHAR_SIZE), %ecx
+ movl (%rdi, %rax, CHAR_SIZE), %ecx
xorl %edx, %edx
- cmpl -(VEC_SIZE * 2)(%rsi, %rax, CHAR_SIZE), %ecx
+ cmpl (%rsi, %rax, CHAR_SIZE), %ecx
+ /* NB: no partial register stall here because xorl zero idiom
+ above. */
setg %dl
leal -1(%rdx, %rdx), %eax
# else
- movzbl -(VEC_SIZE * 2)(%rsi, %rax), %ecx
- movzbl -(VEC_SIZE * 2)(%rdi, %rax), %eax
+ movzbl (%rsi, %rax), %ecx
+ movzbl (%rdi, %rax), %eax
subl %ecx, %eax
# endif
ret
-
.p2align 4
L(page_cross_less_vec):
/* if USE_AS_WMEMCMP it can only be 0, 4, 8, 12, 16, 20, 24, 28
@@ -439,108 +513,84 @@ L(page_cross_less_vec):
cmpl $8, %edx
jae L(between_8_15)
cmpl $4, %edx
- jae L(between_4_7)
-L(between_2_3):
- /* Load as big endian to avoid branches. */
- movzwl (%rdi), %eax
- movzwl (%rsi), %ecx
- shll $8, %eax
- shll $8, %ecx
- bswap %eax
- bswap %ecx
- movzbl -1(%rdi, %rdx), %edi
- movzbl -1(%rsi, %rdx), %esi
- orl %edi, %eax
- orl %esi, %ecx
- /* Subtraction is okay because the upper 8 bits are zero. */
- subl %ecx, %eax
- ret
- .p2align 4
-L(one_or_less):
- jb L(zero)
- movzbl (%rsi), %ecx
- movzbl (%rdi), %eax
- subl %ecx, %eax
+ jb L(between_2_3)
+
+ /* Load as big endian with overlapping movbe to avoid branches.
+ */
+ movbe (%rdi), %eax
+ movbe (%rsi), %ecx
+ shlq $32, %rax
+ shlq $32, %rcx
+ movbe -4(%rdi, %rdx), %edi
+ movbe -4(%rsi, %rdx), %esi
+ orq %rdi, %rax
+ orq %rsi, %rcx
+ subq %rcx, %rax
+ /* edx is guranteed to be positive int32 in range [4, 7]. */
+ cmovne %edx, %eax
+ /* ecx is -1 if rcx > rax. Otherwise 0. */
+ sbbl %ecx, %ecx
+ /* If rcx > rax, then ecx is 0 and eax is positive. If rcx ==
+ rax then eax and ecx are zero. If rax < rax then ecx is -1 so
+ eax doesn't matter. */
+ orl %ecx, %eax
ret
- .p2align 4
+ .p2align 4,, 8
L(between_8_15):
# endif
/* If USE_AS_WMEMCMP fall through into 8-15 byte case. */
- vmovq (%rdi), %XMM1
- vmovq (%rsi), %XMM2
- VPCMP $4, %XMM1, %XMM2, %k1
+ vmovq (%rdi), %xmm1
+ vmovq (%rsi), %xmm2
+ VPCMP $4, %xmm1, %xmm2, %k1
kmovd %k1, %eax
testl %eax, %eax
- jnz L(return_vec_0)
+ jnz L(return_vec_0_lv)
/* Use overlapping loads to avoid branches. */
- leaq -8(%rdi, %rdx, CHAR_SIZE), %rdi
- leaq -8(%rsi, %rdx, CHAR_SIZE), %rsi
- vmovq (%rdi), %XMM1
- vmovq (%rsi), %XMM2
- VPCMP $4, %XMM1, %XMM2, %k1
+ vmovq -8(%rdi, %rdx, CHAR_SIZE), %xmm1
+ vmovq -8(%rsi, %rdx, CHAR_SIZE), %xmm2
+ VPCMP $4, %xmm1, %xmm2, %k1
+ addl $(CHAR_PER_VEC - (8 / CHAR_SIZE)), %edx
kmovd %k1, %eax
testl %eax, %eax
- jnz L(return_vec_0)
- ret
-
- .p2align 4
-L(zero):
- xorl %eax, %eax
+ jnz L(return_vec_0_end)
ret
- .p2align 4
+ .p2align 4,, 8
L(between_16_31):
/* From 16 to 31 bytes. No branch when size == 16. */
- VMOVU (%rsi), %XMM2
- VPCMP $4, (%rdi), %XMM2, %k1
+
+ /* Use movups to save code size. */
+ movups (%rsi), %xmm2
+ VPCMP $4, (%rdi), %xmm2, %k1
kmovd %k1, %eax
testl %eax, %eax
- jnz L(return_vec_0)
-
+ jnz L(return_vec_0_lv)
/* Use overlapping loads to avoid branches. */
-
- VMOVU -16(%rsi, %rdx, CHAR_SIZE), %XMM2
- leaq -16(%rdi, %rdx, CHAR_SIZE), %rdi
- leaq -16(%rsi, %rdx, CHAR_SIZE), %rsi
- VPCMP $4, (%rdi), %XMM2, %k1
+ movups -16(%rsi, %rdx, CHAR_SIZE), %xmm2
+ VPCMP $4, -16(%rdi, %rdx, CHAR_SIZE), %xmm2, %k1
+ addl $(CHAR_PER_VEC - (16 / CHAR_SIZE)), %edx
kmovd %k1, %eax
testl %eax, %eax
- jnz L(return_vec_0)
- ret
-
-# ifdef USE_AS_WMEMCMP
- .p2align 4
-L(one_or_less):
- jb L(zero)
- movl (%rdi), %ecx
- xorl %edx, %edx
- cmpl (%rsi), %ecx
- je L(zero)
- setg %dl
- leal -1(%rdx, %rdx), %eax
+ jnz L(return_vec_0_end)
ret
-# else
- .p2align 4
-L(between_4_7):
- /* Load as big endian with overlapping movbe to avoid branches.
- */
- movbe (%rdi), %eax
- movbe (%rsi), %ecx
- shlq $32, %rax
- shlq $32, %rcx
- movbe -4(%rdi, %rdx), %edi
- movbe -4(%rsi, %rdx), %esi
- orq %rdi, %rax
- orq %rsi, %rcx
- subq %rcx, %rax
- jz L(zero_4_7)
- sbbl %eax, %eax
- orl $1, %eax
-L(zero_4_7):
+# ifndef USE_AS_WMEMCMP
+L(between_2_3):
+ /* Load as big endian to avoid branches. */
+ movzwl (%rdi), %eax
+ movzwl (%rsi), %ecx
+ shll $8, %eax
+ shll $8, %ecx
+ bswap %eax
+ bswap %ecx
+ movzbl -1(%rdi, %rdx), %edi
+ movzbl -1(%rsi, %rdx), %esi
+ orl %edi, %eax
+ orl %esi, %ecx
+ /* Subtraction is okay because the upper 8 bits are zero. */
+ subl %ecx, %eax
ret
# endif
-
END (MEMCMP)
#endif