91 lines
3.8 KiB
Diff
91 lines
3.8 KiB
Diff
commit 65ae73be01604699493d387d8ea6bba41df004ab
|
|
Author: Noah Goldstein <goldstein.w.n@gmail.com>
|
|
Date: Wed Aug 14 14:37:30 2024 +0800
|
|
|
|
x86: Use `Avoid_Non_Temporal_Memset` to control non-temporal path
|
|
|
|
This is just a refactor and there should be no behavioral change from
|
|
this commit.
|
|
|
|
The goal is to make `Avoid_Non_Temporal_Memset` a more universal knob
|
|
for controlling whether we use non-temporal memset rather than having
|
|
extra logic based on vendor.
|
|
Reviewed-by: H.J. Lu <hjl.tools@gmail.com>
|
|
|
|
(cherry picked from commit b93dddfaf440aa12f45d7c356f6ffe9f27d35577)
|
|
|
|
diff --git a/sysdeps/x86/cpu-features.c b/sysdeps/x86/cpu-features.c
|
|
index 77b5638daafe9a1e..4490c0a782e25d8d 100644
|
|
--- a/sysdeps/x86/cpu-features.c
|
|
+++ b/sysdeps/x86/cpu-features.c
|
|
@@ -758,6 +758,12 @@ init_cpu_features (struct cpu_features *cpu_features)
|
|
unsigned int stepping = 0;
|
|
enum cpu_features_kind kind;
|
|
|
|
+ /* Default is avoid non-temporal memset for non Intel/AMD hardware. This is,
|
|
+ as of writing this, we only have benchmarks indicatings it profitability
|
|
+ on Intel/AMD. */
|
|
+ cpu_features->preferred[index_arch_Avoid_Non_Temporal_Memset]
|
|
+ |= bit_arch_Avoid_Non_Temporal_Memset;
|
|
+
|
|
cpu_features->cachesize_non_temporal_divisor = 4;
|
|
#if !HAS_CPUID
|
|
if (__get_cpuid_max (0, 0) == 0)
|
|
@@ -783,6 +789,11 @@ init_cpu_features (struct cpu_features *cpu_features)
|
|
|
|
update_active (cpu_features);
|
|
|
|
+ /* Benchmarks indicate non-temporal memset can be profitable on Intel
|
|
+ hardware. */
|
|
+ cpu_features->preferred[index_arch_Avoid_Non_Temporal_Memset]
|
|
+ &= ~bit_arch_Avoid_Non_Temporal_Memset;
|
|
+
|
|
if (family == 0x06)
|
|
{
|
|
model += extended_model;
|
|
@@ -993,6 +1004,11 @@ https://www.intel.com/content/www/us/en/support/articles/000059422/processors.ht
|
|
|
|
ecx = cpu_features->features[CPUID_INDEX_1].cpuid.ecx;
|
|
|
|
+ /* Benchmarks indicate non-temporal memset can be profitable on AMD
|
|
+ hardware. */
|
|
+ cpu_features->preferred[index_arch_Avoid_Non_Temporal_Memset]
|
|
+ &= ~bit_arch_Avoid_Non_Temporal_Memset;
|
|
+
|
|
if (CPU_FEATURE_USABLE_P (cpu_features, AVX))
|
|
{
|
|
/* Since the FMA4 bit is in CPUID_INDEX_80000001 and
|
|
diff --git a/sysdeps/x86/dl-cacheinfo.h b/sysdeps/x86/dl-cacheinfo.h
|
|
index 66e2b83fea0dc744..10ad18061a1b47af 100644
|
|
--- a/sysdeps/x86/dl-cacheinfo.h
|
|
+++ b/sysdeps/x86/dl-cacheinfo.h
|
|
@@ -986,14 +986,6 @@ dl_init_cacheinfo (struct cpu_features *cpu_features)
|
|
if (CPU_FEATURE_USABLE_P (cpu_features, FSRM))
|
|
rep_movsb_threshold = 2112;
|
|
|
|
- /* Non-temporal stores are more performant on Intel and AMD hardware above
|
|
- non_temporal_threshold. Enable this for both Intel and AMD hardware. */
|
|
- unsigned long int memset_non_temporal_threshold = SIZE_MAX;
|
|
- if (!CPU_FEATURES_ARCH_P (cpu_features, Avoid_Non_Temporal_Memset)
|
|
- && (cpu_features->basic.kind == arch_kind_intel
|
|
- || cpu_features->basic.kind == arch_kind_amd))
|
|
- memset_non_temporal_threshold = non_temporal_threshold;
|
|
-
|
|
/* For AMD CPUs that support ERMS (Zen3+), REP MOVSB is in a lot of
|
|
cases slower than the vectorized path (and for some alignments,
|
|
it is really slow, check BZ #30994). */
|
|
@@ -1015,6 +1007,13 @@ dl_init_cacheinfo (struct cpu_features *cpu_features)
|
|
if (tunable_size != 0)
|
|
shared = tunable_size;
|
|
|
|
+ /* Non-temporal stores are more performant on some hardware above
|
|
+ non_temporal_threshold. Currently Prefer_Non_Temporal is set for for both
|
|
+ Intel and AMD hardware. */
|
|
+ unsigned long int memset_non_temporal_threshold = SIZE_MAX;
|
|
+ if (!CPU_FEATURES_ARCH_P (cpu_features, Avoid_Non_Temporal_Memset))
|
|
+ memset_non_temporal_threshold = non_temporal_threshold;
|
|
+
|
|
tunable_size = TUNABLE_GET (x86_non_temporal_threshold, long int, NULL);
|
|
if (tunable_size > minimum_non_temporal_threshold
|
|
&& tunable_size <= maximum_non_temporal_threshold)
|