73667d0be6
* Thu Apr 28 2022 Carlos O'Donell <carlos@redhat.com> - 2.34-32 - Sync with upstream branch release/2.34/master, commit c66c92181ddbd82306537a608e8c0282587131de: - posix/glob.c: update from gnulib (BZ#25659) - linux: Fix fchmodat with AT_SYMLINK_NOFOLLOW for 64 bit time_t (BZ#29097) * Wed Apr 27 2022 Carlos O'Donell <carlos@redhat.com> - 2.34-31 - Sync with upstream branch release/2.34/master, commit 55640ed3fde48360a8e8083be4843bd2dc7cecfe: - i386: Regenerate ulps - linux: Fix missing internal 64 bit time_t stat usage - x86: Optimize L(less_vec) case in memcmp-evex-movbe.S - x86: Don't set Prefer_No_AVX512 for processors with AVX512 and AVX-VNNI - x86-64: Use notl in EVEX strcmp [BZ #28646] - x86: Shrink memcmp-sse4.S code size - x86: Double size of ERMS rep_movsb_threshold in dl-cacheinfo.h - x86: Optimize memmove-vec-unaligned-erms.S - x86-64: Replace movzx with movzbl - x86-64: Remove Prefer_AVX2_STRCMP - x86-64: Improve EVEX strcmp with masked load - x86: Replace sse2 instructions with avx in memcmp-evex-movbe.S - x86: Optimize memset-vec-unaligned-erms.S - x86: Optimize memcmp-evex-movbe.S for frontend behavior and size - x86: Modify ENTRY in sysdep.h so that p2align can be specified - x86-64: Optimize load of all bits set into ZMM register [BZ #28252] - scripts/glibcelf.py: Mark as UNSUPPORTED on Python 3.5 and earlier - dlfcn: Do not use rtld_active () to determine ld.so state (bug 29078) - INSTALL: Rephrase -with-default-link documentation - misc: Fix rare fortify crash on wchar funcs. [BZ 29030] - Default to --with-default-link=no (bug 25812) - scripts: Add glibcelf.py module * Thu Apr 21 2022 Carlos O'Donell <carlos@redhat.com> - 2.34-30 - Sync with upstream branch release/2.34/master, commit 71326f1f2fd09dafb9c34404765fb88129e94237: - nptl: Fix pthread_cancel cancelhandling atomic operations - mips: Fix mips64n32 64 bit time_t stat support (BZ#29069) - hurd: Fix arbitrary error code - nptl: Handle spurious EINTR when thread cancellation is disabled (BZ#29029) - S390: Add new s390 platform z16. - NEWS: Update fixed bug list for LD_AUDIT backports. - hppa: Fix bind-now audit (BZ #28857) - elf: Replace tst-audit24bmod2.so with tst-audit24bmod2 - Fix elf/tst-audit25a with default bind now toolchains - elf: Fix runtime linker auditing on aarch64 (BZ #26643) - elf: Issue la_symbind for bind-now (BZ #23734) - elf: Fix initial-exec TLS access on audit modules (BZ #28096) - elf: Add la_activity during application exit - elf: Do not fail for failed dlmopen on audit modules (BZ #28061) - elf: Issue audit la_objopen for vDSO - elf: Add audit tests for modules with TLSDESC - elf: Avoid unnecessary slowdown from profiling with audit (BZ#15533) - elf: Add _dl_audit_pltexit - elf: Add _dl_audit_pltenter - elf: Add _dl_audit_preinit - elf: Add _dl_audit_symbind_alt and _dl_audit_symbind - elf: Add _dl_audit_objclose - elf: Add _dl_audit_objsearch - elf: Add _dl_audit_activity_map and _dl_audit_activity_nsid - elf: Add _dl_audit_objopen - elf: Move la_activity (LA_ACT_ADD) after _dl_add_to_namespace_list() (BZ #28062) - elf: Move LAV_CURRENT to link_lavcurrent.h - elf: Fix elf_get_dynamic_info() for bootstrap - elf: Fix dynamic-link.h usage on rtld.c - elf: Fix elf_get_dynamic_info definition - elf: Avoid nested functions in the loader [BZ #27220] - powerpc: Delete unneeded ELF_MACHINE_BEFORE_RTLD_RELOC - hppa: Use END instead of PSEUDO_END in swapcontext.S - hppa: Implement swapcontext in assembler (bug 28960) Resolves: #2003291 Resolves: #2064181 Resolves: #2072328 Resolves: #2075713 Resolves: #2077838
255 lines
11 KiB
Diff
255 lines
11 KiB
Diff
commit 16245986fb9bfe396113fc7dfd1929f69a9e748e
|
|
Author: H.J. Lu <hjl.tools@gmail.com>
|
|
Date: Fri Aug 20 06:42:24 2021 -0700
|
|
|
|
x86-64: Optimize load of all bits set into ZMM register [BZ #28252]
|
|
|
|
Optimize loads of all bits set into ZMM register in AVX512 SVML codes
|
|
by replacing
|
|
|
|
vpbroadcastq .L_2il0floatpacket.16(%rip), %zmmX
|
|
|
|
and
|
|
|
|
vmovups .L_2il0floatpacket.13(%rip), %zmmX
|
|
|
|
with
|
|
vpternlogd $0xff, %zmmX, %zmmX, %zmmX
|
|
|
|
This fixes BZ #28252.
|
|
|
|
(cherry picked from commit 78c9ec9000f873abe7a15a91b87080a2e4308260)
|
|
|
|
diff --git a/sysdeps/x86_64/fpu/multiarch/svml_d_cos8_core_avx512.S b/sysdeps/x86_64/fpu/multiarch/svml_d_cos8_core_avx512.S
|
|
index e68fcdbb16a79f36..58e588a3d42a8bc9 100644
|
|
--- a/sysdeps/x86_64/fpu/multiarch/svml_d_cos8_core_avx512.S
|
|
+++ b/sysdeps/x86_64/fpu/multiarch/svml_d_cos8_core_avx512.S
|
|
@@ -265,7 +265,7 @@ WRAPPER_IMPL_AVX512 _ZGVdN4v_cos
|
|
vmovaps %zmm0, %zmm8
|
|
|
|
/* Check for large arguments path */
|
|
- vpbroadcastq .L_2il0floatpacket.16(%rip), %zmm2
|
|
+ vpternlogd $0xff, %zmm2, %zmm2, %zmm2
|
|
|
|
/*
|
|
ARGUMENT RANGE REDUCTION:
|
|
@@ -456,8 +456,3 @@ WRAPPER_IMPL_AVX512 _ZGVdN4v_cos
|
|
jmp .LBL_2_7
|
|
#endif
|
|
END (_ZGVeN8v_cos_skx)
|
|
-
|
|
- .section .rodata, "a"
|
|
-.L_2il0floatpacket.16:
|
|
- .long 0xffffffff,0xffffffff
|
|
- .type .L_2il0floatpacket.16,@object
|
|
diff --git a/sysdeps/x86_64/fpu/multiarch/svml_d_log8_core_avx512.S b/sysdeps/x86_64/fpu/multiarch/svml_d_log8_core_avx512.S
|
|
index dfa2acafc486b56b..f5f117d474f66176 100644
|
|
--- a/sysdeps/x86_64/fpu/multiarch/svml_d_log8_core_avx512.S
|
|
+++ b/sysdeps/x86_64/fpu/multiarch/svml_d_log8_core_avx512.S
|
|
@@ -274,7 +274,7 @@ WRAPPER_IMPL_AVX512 _ZGVdN4v_log
|
|
|
|
/* preserve mantissa, set input exponent to 2^(-10) */
|
|
vpternlogq $248, _ExpMask(%rax), %zmm3, %zmm2
|
|
- vpbroadcastq .L_2il0floatpacket.12(%rip), %zmm1
|
|
+ vpternlogd $0xff, %zmm1, %zmm1, %zmm1
|
|
vpsrlq $32, %zmm4, %zmm6
|
|
|
|
/* reciprocal approximation good to at least 11 bits */
|
|
@@ -461,8 +461,3 @@ WRAPPER_IMPL_AVX512 _ZGVdN4v_log
|
|
jmp .LBL_2_7
|
|
#endif
|
|
END (_ZGVeN8v_log_skx)
|
|
-
|
|
- .section .rodata, "a"
|
|
-.L_2il0floatpacket.12:
|
|
- .long 0xffffffff,0xffffffff
|
|
- .type .L_2il0floatpacket.12,@object
|
|
diff --git a/sysdeps/x86_64/fpu/multiarch/svml_d_sin8_core_avx512.S b/sysdeps/x86_64/fpu/multiarch/svml_d_sin8_core_avx512.S
|
|
index be8ab7c6e0e33819..48d251db16ccab9d 100644
|
|
--- a/sysdeps/x86_64/fpu/multiarch/svml_d_sin8_core_avx512.S
|
|
+++ b/sysdeps/x86_64/fpu/multiarch/svml_d_sin8_core_avx512.S
|
|
@@ -261,7 +261,7 @@ WRAPPER_IMPL_AVX512 _ZGVdN4v_sin
|
|
andq $-64, %rsp
|
|
subq $1280, %rsp
|
|
movq __svml_d_trig_data@GOTPCREL(%rip), %rax
|
|
- vpbroadcastq .L_2il0floatpacket.14(%rip), %zmm14
|
|
+ vpternlogd $0xff, %zmm1, %zmm1, %zmm14
|
|
vmovups __dAbsMask(%rax), %zmm7
|
|
vmovups __dInvPI(%rax), %zmm2
|
|
vmovups __dRShifter(%rax), %zmm1
|
|
@@ -458,8 +458,3 @@ WRAPPER_IMPL_AVX512 _ZGVdN4v_sin
|
|
jmp .LBL_2_7
|
|
#endif
|
|
END (_ZGVeN8v_sin_skx)
|
|
-
|
|
- .section .rodata, "a"
|
|
-.L_2il0floatpacket.14:
|
|
- .long 0xffffffff,0xffffffff
|
|
- .type .L_2il0floatpacket.14,@object
|
|
diff --git a/sysdeps/x86_64/fpu/multiarch/svml_d_sincos8_core_avx512.S b/sysdeps/x86_64/fpu/multiarch/svml_d_sincos8_core_avx512.S
|
|
index 611887082a545854..a4944a4feef6aa98 100644
|
|
--- a/sysdeps/x86_64/fpu/multiarch/svml_d_sincos8_core_avx512.S
|
|
+++ b/sysdeps/x86_64/fpu/multiarch/svml_d_sincos8_core_avx512.S
|
|
@@ -430,7 +430,7 @@ WRAPPER_IMPL_AVX512_fFF _ZGVdN4vl8l8_sincos
|
|
|
|
/* SinPoly = SinR*SinPoly */
|
|
vfmadd213pd %zmm5, %zmm5, %zmm4
|
|
- vpbroadcastq .L_2il0floatpacket.15(%rip), %zmm3
|
|
+ vpternlogd $0xff, %zmm3, %zmm3, %zmm3
|
|
|
|
/* Update Cos result's sign */
|
|
vxorpd %zmm2, %zmm1, %zmm1
|
|
@@ -741,8 +741,3 @@ END (_ZGVeN8vvv_sincos_knl)
|
|
ENTRY (_ZGVeN8vvv_sincos_skx)
|
|
WRAPPER_AVX512_vvv_vl8l8 _ZGVeN8vl8l8_sincos_skx
|
|
END (_ZGVeN8vvv_sincos_skx)
|
|
-
|
|
- .section .rodata, "a"
|
|
-.L_2il0floatpacket.15:
|
|
- .long 0xffffffff,0xffffffff
|
|
- .type .L_2il0floatpacket.15,@object
|
|
diff --git a/sysdeps/x86_64/fpu/multiarch/svml_s_cosf16_core_avx512.S b/sysdeps/x86_64/fpu/multiarch/svml_s_cosf16_core_avx512.S
|
|
index f671d60d5dab5a0e..fe8474fed943e8ad 100644
|
|
--- a/sysdeps/x86_64/fpu/multiarch/svml_s_cosf16_core_avx512.S
|
|
+++ b/sysdeps/x86_64/fpu/multiarch/svml_s_cosf16_core_avx512.S
|
|
@@ -278,7 +278,7 @@ WRAPPER_IMPL_AVX512 _ZGVdN8v_cosf
|
|
X = X - Y*PI1 - Y*PI2 - Y*PI3
|
|
*/
|
|
vmovaps %zmm0, %zmm6
|
|
- vmovups .L_2il0floatpacket.13(%rip), %zmm12
|
|
+ vpternlogd $0xff, %zmm12, %zmm12, %zmm12
|
|
vmovups __sRShifter(%rax), %zmm3
|
|
vmovups __sPI1_FMA(%rax), %zmm5
|
|
vmovups __sA9_FMA(%rax), %zmm9
|
|
@@ -453,8 +453,3 @@ WRAPPER_IMPL_AVX512 _ZGVdN8v_cosf
|
|
jmp .LBL_2_7
|
|
#endif
|
|
END (_ZGVeN16v_cosf_skx)
|
|
-
|
|
- .section .rodata, "a"
|
|
-.L_2il0floatpacket.13:
|
|
- .long 0xffffffff,0xffffffff,0xffffffff,0xffffffff,0xffffffff,0xffffffff,0xffffffff,0xffffffff,0xffffffff,0xffffffff,0xffffffff,0xffffffff,0xffffffff,0xffffffff,0xffffffff,0xffffffff
|
|
- .type .L_2il0floatpacket.13,@object
|
|
diff --git a/sysdeps/x86_64/fpu/multiarch/svml_s_expf16_core_avx512.S b/sysdeps/x86_64/fpu/multiarch/svml_s_expf16_core_avx512.S
|
|
index 637bfe3c06ab9ad4..229b7828cde04db2 100644
|
|
--- a/sysdeps/x86_64/fpu/multiarch/svml_s_expf16_core_avx512.S
|
|
+++ b/sysdeps/x86_64/fpu/multiarch/svml_s_expf16_core_avx512.S
|
|
@@ -264,7 +264,7 @@ WRAPPER_IMPL_AVX512 _ZGVdN8v_expf
|
|
vmovaps %zmm0, %zmm7
|
|
|
|
/* compare against threshold */
|
|
- vmovups .L_2il0floatpacket.13(%rip), %zmm3
|
|
+ vpternlogd $0xff, %zmm3, %zmm3, %zmm3
|
|
vmovups __sInvLn2(%rax), %zmm4
|
|
vmovups __sShifter(%rax), %zmm1
|
|
vmovups __sLn2hi(%rax), %zmm6
|
|
@@ -440,8 +440,3 @@ WRAPPER_IMPL_AVX512 _ZGVdN8v_expf
|
|
|
|
#endif
|
|
END (_ZGVeN16v_expf_skx)
|
|
-
|
|
- .section .rodata, "a"
|
|
-.L_2il0floatpacket.13:
|
|
- .long 0xffffffff,0xffffffff,0xffffffff,0xffffffff,0xffffffff,0xffffffff,0xffffffff,0xffffffff,0xffffffff,0xffffffff,0xffffffff,0xffffffff,0xffffffff,0xffffffff,0xffffffff,0xffffffff
|
|
- .type .L_2il0floatpacket.13,@object
|
|
diff --git a/sysdeps/x86_64/fpu/multiarch/svml_s_logf16_core_avx512.S b/sysdeps/x86_64/fpu/multiarch/svml_s_logf16_core_avx512.S
|
|
index 9d790fbf0ad6c8ec..fa2aae986f543582 100644
|
|
--- a/sysdeps/x86_64/fpu/multiarch/svml_s_logf16_core_avx512.S
|
|
+++ b/sysdeps/x86_64/fpu/multiarch/svml_s_logf16_core_avx512.S
|
|
@@ -235,7 +235,7 @@ WRAPPER_IMPL_AVX512 _ZGVdN8v_logf
|
|
andq $-64, %rsp
|
|
subq $1280, %rsp
|
|
movq __svml_slog_data@GOTPCREL(%rip), %rax
|
|
- vmovups .L_2il0floatpacket.7(%rip), %zmm6
|
|
+ vpternlogd $0xff, %zmm6, %zmm6, %zmm6
|
|
vmovups _iBrkValue(%rax), %zmm4
|
|
vmovups _sPoly_7(%rax), %zmm8
|
|
|
|
@@ -409,8 +409,3 @@ WRAPPER_IMPL_AVX512 _ZGVdN8v_logf
|
|
|
|
#endif
|
|
END (_ZGVeN16v_logf_skx)
|
|
-
|
|
- .section .rodata, "a"
|
|
-.L_2il0floatpacket.7:
|
|
- .long 0xffffffff,0xffffffff,0xffffffff,0xffffffff,0xffffffff,0xffffffff,0xffffffff,0xffffffff,0xffffffff,0xffffffff,0xffffffff,0xffffffff,0xffffffff,0xffffffff,0xffffffff,0xffffffff
|
|
- .type .L_2il0floatpacket.7,@object
|
|
diff --git a/sysdeps/x86_64/fpu/multiarch/svml_s_powf16_core_avx512.S b/sysdeps/x86_64/fpu/multiarch/svml_s_powf16_core_avx512.S
|
|
index c5c43c46ff7af5a3..6aea2a4f11d1f85f 100644
|
|
--- a/sysdeps/x86_64/fpu/multiarch/svml_s_powf16_core_avx512.S
|
|
+++ b/sysdeps/x86_64/fpu/multiarch/svml_s_powf16_core_avx512.S
|
|
@@ -385,7 +385,7 @@ WRAPPER_IMPL_AVX512_ff _ZGVdN8vv_powf
|
|
vpsrlq $32, %zmm3, %zmm2
|
|
vpmovqd %zmm2, %ymm11
|
|
vcvtps2pd %ymm14, %zmm13
|
|
- vmovups .L_2il0floatpacket.23(%rip), %zmm14
|
|
+ vpternlogd $0xff, %zmm14, %zmm14, %zmm14
|
|
vmovaps %zmm14, %zmm26
|
|
vpandd _ABSMASK(%rax), %zmm1, %zmm8
|
|
vpcmpd $1, _INF(%rax), %zmm8, %k2
|
|
@@ -427,7 +427,7 @@ WRAPPER_IMPL_AVX512_ff _ZGVdN8vv_powf
|
|
vpmovqd %zmm11, %ymm5
|
|
vpxord %zmm10, %zmm10, %zmm10
|
|
vgatherdpd _Log2Rcp_lookup(%rax,%ymm4), %zmm10{%k3}
|
|
- vpbroadcastq .L_2il0floatpacket.24(%rip), %zmm4
|
|
+ vpternlogd $0xff, %zmm4, %zmm4, %zmm4
|
|
vpxord %zmm11, %zmm11, %zmm11
|
|
vcvtdq2pd %ymm7, %zmm7
|
|
vgatherdpd _Log2Rcp_lookup(%rax,%ymm5), %zmm11{%k1}
|
|
@@ -643,11 +643,3 @@ WRAPPER_IMPL_AVX512_ff _ZGVdN8vv_powf
|
|
jmp .LBL_2_7
|
|
#endif
|
|
END (_ZGVeN16vv_powf_skx)
|
|
-
|
|
- .section .rodata, "a"
|
|
-.L_2il0floatpacket.23:
|
|
- .long 0xffffffff,0xffffffff,0xffffffff,0xffffffff,0xffffffff,0xffffffff,0xffffffff,0xffffffff,0xffffffff,0xffffffff,0xffffffff,0xffffffff,0xffffffff,0xffffffff,0xffffffff,0xffffffff
|
|
- .type .L_2il0floatpacket.23,@object
|
|
-.L_2il0floatpacket.24:
|
|
- .long 0xffffffff,0xffffffff
|
|
- .type .L_2il0floatpacket.24,@object
|
|
diff --git a/sysdeps/x86_64/fpu/multiarch/svml_s_sincosf16_core_avx512.S b/sysdeps/x86_64/fpu/multiarch/svml_s_sincosf16_core_avx512.S
|
|
index 9cf359c86ff9bd70..a446c504f63c9399 100644
|
|
--- a/sysdeps/x86_64/fpu/multiarch/svml_s_sincosf16_core_avx512.S
|
|
+++ b/sysdeps/x86_64/fpu/multiarch/svml_s_sincosf16_core_avx512.S
|
|
@@ -317,7 +317,7 @@ WRAPPER_IMPL_AVX512_fFF _ZGVdN8vvv_sincosf
|
|
|
|
/* Result sign calculations */
|
|
vpternlogd $150, %zmm0, %zmm14, %zmm1
|
|
- vmovups .L_2il0floatpacket.13(%rip), %zmm14
|
|
+ vpternlogd $0xff, %zmm14, %zmm14, %zmm14
|
|
|
|
/* Add correction term 0.5 for cos() part */
|
|
vaddps %zmm8, %zmm5, %zmm15
|
|
@@ -748,8 +748,3 @@ END (_ZGVeN16vvv_sincosf_knl)
|
|
ENTRY (_ZGVeN16vvv_sincosf_skx)
|
|
WRAPPER_AVX512_vvv_vl4l4 _ZGVeN16vl4l4_sincosf_skx
|
|
END (_ZGVeN16vvv_sincosf_skx)
|
|
-
|
|
- .section .rodata, "a"
|
|
-.L_2il0floatpacket.13:
|
|
- .long 0xffffffff,0xffffffff,0xffffffff,0xffffffff,0xffffffff,0xffffffff,0xffffffff,0xffffffff,0xffffffff,0xffffffff,0xffffffff,0xffffffff,0xffffffff,0xffffffff,0xffffffff,0xffffffff
|
|
- .type .L_2il0floatpacket.13,@object
|
|
diff --git a/sysdeps/x86_64/fpu/multiarch/svml_s_sinf16_core_avx512.S b/sysdeps/x86_64/fpu/multiarch/svml_s_sinf16_core_avx512.S
|
|
index bd05109a62181f22..c1b352d0ad1992cd 100644
|
|
--- a/sysdeps/x86_64/fpu/multiarch/svml_s_sinf16_core_avx512.S
|
|
+++ b/sysdeps/x86_64/fpu/multiarch/svml_s_sinf16_core_avx512.S
|
|
@@ -280,7 +280,7 @@ WRAPPER_IMPL_AVX512 _ZGVdN8v_sinf
|
|
movq __svml_s_trig_data@GOTPCREL(%rip), %rax
|
|
|
|
/* Check for large and special values */
|
|
- vmovups .L_2il0floatpacket.11(%rip), %zmm14
|
|
+ vpternlogd $0xff, %zmm14, %zmm14, %zmm14
|
|
vmovups __sAbsMask(%rax), %zmm5
|
|
vmovups __sInvPI(%rax), %zmm1
|
|
vmovups __sRShifter(%rax), %zmm2
|
|
@@ -472,8 +472,3 @@ WRAPPER_IMPL_AVX512 _ZGVdN8v_sinf
|
|
jmp .LBL_2_7
|
|
#endif
|
|
END (_ZGVeN16v_sinf_skx)
|
|
-
|
|
- .section .rodata, "a"
|
|
-.L_2il0floatpacket.11:
|
|
- .long 0xffffffff,0xffffffff,0xffffffff,0xffffffff,0xffffffff,0xffffffff,0xffffffff,0xffffffff,0xffffffff,0xffffffff,0xffffffff,0xffffffff,0xffffffff,0xffffffff,0xffffffff,0xffffffff
|
|
- .type .L_2il0floatpacket.11,@object
|