Compare commits

..

No commits in common. "c8" and "c9s" have entirely different histories.
c8 ... c9s

1598 changed files with 123072 additions and 192885 deletions

7
.gitignore vendored
View File

@ -1 +1,6 @@
SOURCES/glibc-2.28.tar.xz
# Release tarballs.
/glibc-*.tar.[gx]z
# Generated (source) RPMs.
/*.rpm
# Expanded source trees.
/glibc-*/

View File

@ -1 +0,0 @@
ccb5dc9e51a9884df8488f86982439d47b283b2a SOURCES/glibc-2.28.tar.xz

File diff suppressed because it is too large Load Diff

2
README.scripts Normal file
View File

@ -0,0 +1,2 @@
All of the useful glibc maintainer scripts are located at:
https://pagure.io/glibc-maintainer-scripts

View File

@ -1,496 +0,0 @@
# This file names the currently supported and somewhat tested locales.
# If you have any additions please file a glibc bug report.
SUPPORTED-LOCALES=\
C.UTF-8/UTF-8 \
aa_DJ.UTF-8/UTF-8 \
aa_DJ/ISO-8859-1 \
aa_ER/UTF-8 \
aa_ER@saaho/UTF-8 \
aa_ET/UTF-8 \
af_ZA.UTF-8/UTF-8 \
af_ZA/ISO-8859-1 \
agr_PE/UTF-8 \
ak_GH/UTF-8 \
am_ET/UTF-8 \
an_ES.UTF-8/UTF-8 \
an_ES/ISO-8859-15 \
anp_IN/UTF-8 \
ar_AE.UTF-8/UTF-8 \
ar_AE/ISO-8859-6 \
ar_BH.UTF-8/UTF-8 \
ar_BH/ISO-8859-6 \
ar_DZ.UTF-8/UTF-8 \
ar_DZ/ISO-8859-6 \
ar_EG.UTF-8/UTF-8 \
ar_EG/ISO-8859-6 \
ar_IN/UTF-8 \
ar_IQ.UTF-8/UTF-8 \
ar_IQ/ISO-8859-6 \
ar_JO.UTF-8/UTF-8 \
ar_JO/ISO-8859-6 \
ar_KW.UTF-8/UTF-8 \
ar_KW/ISO-8859-6 \
ar_LB.UTF-8/UTF-8 \
ar_LB/ISO-8859-6 \
ar_LY.UTF-8/UTF-8 \
ar_LY/ISO-8859-6 \
ar_MA.UTF-8/UTF-8 \
ar_MA/ISO-8859-6 \
ar_OM.UTF-8/UTF-8 \
ar_OM/ISO-8859-6 \
ar_QA.UTF-8/UTF-8 \
ar_QA/ISO-8859-6 \
ar_SA.UTF-8/UTF-8 \
ar_SA/ISO-8859-6 \
ar_SD.UTF-8/UTF-8 \
ar_SD/ISO-8859-6 \
ar_SS/UTF-8 \
ar_SY.UTF-8/UTF-8 \
ar_SY/ISO-8859-6 \
ar_TN.UTF-8/UTF-8 \
ar_TN/ISO-8859-6 \
ar_YE.UTF-8/UTF-8 \
ar_YE/ISO-8859-6 \
ayc_PE/UTF-8 \
az_AZ/UTF-8 \
az_IR/UTF-8 \
as_IN/UTF-8 \
ast_ES.UTF-8/UTF-8 \
ast_ES/ISO-8859-15 \
be_BY.UTF-8/UTF-8 \
be_BY/CP1251 \
be_BY@latin/UTF-8 \
bem_ZM/UTF-8 \
ber_DZ/UTF-8 \
ber_MA/UTF-8 \
bg_BG.UTF-8/UTF-8 \
bg_BG/CP1251 \
bhb_IN.UTF-8/UTF-8 \
bho_IN/UTF-8 \
bho_NP/UTF-8 \
bi_VU/UTF-8 \
bn_BD/UTF-8 \
bn_IN/UTF-8 \
bo_CN/UTF-8 \
bo_IN/UTF-8 \
br_FR.UTF-8/UTF-8 \
br_FR/ISO-8859-1 \
br_FR@euro/ISO-8859-15 \
brx_IN/UTF-8 \
bs_BA.UTF-8/UTF-8 \
bs_BA/ISO-8859-2 \
byn_ER/UTF-8 \
ca_AD.UTF-8/UTF-8 \
ca_AD/ISO-8859-15 \
ca_ES.UTF-8/UTF-8 \
ca_ES/ISO-8859-1 \
ca_ES@euro/ISO-8859-15 \
ca_ES@valencia/UTF-8 \
ca_FR.UTF-8/UTF-8 \
ca_FR/ISO-8859-15 \
ca_IT.UTF-8/UTF-8 \
ca_IT/ISO-8859-15 \
ce_RU/UTF-8 \
chr_US/UTF-8 \
cmn_TW/UTF-8 \
crh_UA/UTF-8 \
cs_CZ.UTF-8/UTF-8 \
cs_CZ/ISO-8859-2 \
csb_PL/UTF-8 \
cv_RU/UTF-8 \
cy_GB.UTF-8/UTF-8 \
cy_GB/ISO-8859-14 \
da_DK.UTF-8/UTF-8 \
da_DK/ISO-8859-1 \
da_DK.ISO-8859-15/ISO-8859-15 \
de_AT.UTF-8/UTF-8 \
de_AT/ISO-8859-1 \
de_AT@euro/ISO-8859-15 \
de_BE.UTF-8/UTF-8 \
de_BE/ISO-8859-1 \
de_BE@euro/ISO-8859-15 \
de_CH.UTF-8/UTF-8 \
de_CH/ISO-8859-1 \
de_DE.UTF-8/UTF-8 \
de_DE/ISO-8859-1 \
de_DE@euro/ISO-8859-15 \
de_IT.UTF-8/UTF-8 \
de_IT/ISO-8859-1 \
de_LI.UTF-8/UTF-8 \
de_LU.UTF-8/UTF-8 \
de_LU/ISO-8859-1 \
de_LU@euro/ISO-8859-15 \
doi_IN/UTF-8 \
dsb_DE/UTF-8 \
dv_MV/UTF-8 \
dz_BT/UTF-8 \
el_GR.UTF-8/UTF-8 \
el_GR/ISO-8859-7 \
el_GR@euro/ISO-8859-7 \
el_CY.UTF-8/UTF-8 \
el_CY/ISO-8859-7 \
en_AG/UTF-8 \
en_AU.UTF-8/UTF-8 \
en_AU/ISO-8859-1 \
en_BW.UTF-8/UTF-8 \
en_BW/ISO-8859-1 \
en_CA.UTF-8/UTF-8 \
en_CA/ISO-8859-1 \
en_DK.UTF-8/UTF-8 \
en_DK/ISO-8859-1 \
en_GB.UTF-8/UTF-8 \
en_GB/ISO-8859-1 \
en_GB.ISO-8859-15/ISO-8859-15 \
en_HK.UTF-8/UTF-8 \
en_HK/ISO-8859-1 \
en_IE.UTF-8/UTF-8 \
en_IE/ISO-8859-1 \
en_IE@euro/ISO-8859-15 \
en_IL/UTF-8 \
en_IN/UTF-8 \
en_NG/UTF-8 \
en_NZ.UTF-8/UTF-8 \
en_NZ/ISO-8859-1 \
en_PH.UTF-8/UTF-8 \
en_PH/ISO-8859-1 \
en_SC.UTF-8/UTF-8 \
en_SG.UTF-8/UTF-8 \
en_SG/ISO-8859-1 \
en_US.UTF-8/UTF-8 \
en_US/ISO-8859-1 \
en_US.ISO-8859-15/ISO-8859-15 \
en_US@ampm/UTF-8 \
en_US.UTF-8@ampm/UTF-8 \
en_ZA.UTF-8/UTF-8 \
en_ZA/ISO-8859-1 \
en_ZM/UTF-8 \
en_ZW.UTF-8/UTF-8 \
en_ZW/ISO-8859-1 \
eo/UTF-8 \
es_AR.UTF-8/UTF-8 \
es_AR/ISO-8859-1 \
es_BO.UTF-8/UTF-8 \
es_BO/ISO-8859-1 \
es_CL.UTF-8/UTF-8 \
es_CL/ISO-8859-1 \
es_CO.UTF-8/UTF-8 \
es_CO/ISO-8859-1 \
es_CR.UTF-8/UTF-8 \
es_CR/ISO-8859-1 \
es_CU/UTF-8 \
es_DO.UTF-8/UTF-8 \
es_DO/ISO-8859-1 \
es_EC.UTF-8/UTF-8 \
es_EC/ISO-8859-1 \
es_ES.UTF-8/UTF-8 \
es_ES/ISO-8859-1 \
es_ES@euro/ISO-8859-15 \
es_GT.UTF-8/UTF-8 \
es_GT/ISO-8859-1 \
es_HN.UTF-8/UTF-8 \
es_HN/ISO-8859-1 \
es_MX.UTF-8/UTF-8 \
es_MX/ISO-8859-1 \
es_NI.UTF-8/UTF-8 \
es_NI/ISO-8859-1 \
es_PA.UTF-8/UTF-8 \
es_PA/ISO-8859-1 \
es_PE.UTF-8/UTF-8 \
es_PE/ISO-8859-1 \
es_PR.UTF-8/UTF-8 \
es_PR/ISO-8859-1 \
es_PY.UTF-8/UTF-8 \
es_PY/ISO-8859-1 \
es_SV.UTF-8/UTF-8 \
es_SV/ISO-8859-1 \
es_US.UTF-8/UTF-8 \
es_US/ISO-8859-1 \
es_UY.UTF-8/UTF-8 \
es_UY/ISO-8859-1 \
es_VE.UTF-8/UTF-8 \
es_VE/ISO-8859-1 \
et_EE.UTF-8/UTF-8 \
et_EE/ISO-8859-1 \
et_EE.ISO-8859-15/ISO-8859-15 \
eu_ES.UTF-8/UTF-8 \
eu_ES/ISO-8859-1 \
eu_ES@euro/ISO-8859-15 \
fa_IR/UTF-8 \
ff_SN/UTF-8 \
fi_FI.UTF-8/UTF-8 \
fi_FI/ISO-8859-1 \
fi_FI@euro/ISO-8859-15 \
fil_PH/UTF-8 \
fo_FO.UTF-8/UTF-8 \
fo_FO/ISO-8859-1 \
fr_BE.UTF-8/UTF-8 \
fr_BE/ISO-8859-1 \
fr_BE@euro/ISO-8859-15 \
fr_CA.UTF-8/UTF-8 \
fr_CA/ISO-8859-1 \
fr_CH.UTF-8/UTF-8 \
fr_CH/ISO-8859-1 \
fr_FR.UTF-8/UTF-8 \
fr_FR/ISO-8859-1 \
fr_FR@euro/ISO-8859-15 \
fr_LU.UTF-8/UTF-8 \
fr_LU/ISO-8859-1 \
fr_LU@euro/ISO-8859-15 \
fur_IT/UTF-8 \
fy_NL/UTF-8 \
fy_DE/UTF-8 \
ga_IE.UTF-8/UTF-8 \
ga_IE/ISO-8859-1 \
ga_IE@euro/ISO-8859-15 \
gd_GB.UTF-8/UTF-8 \
gd_GB/ISO-8859-15 \
gez_ER/UTF-8 \
gez_ER@abegede/UTF-8 \
gez_ET/UTF-8 \
gez_ET@abegede/UTF-8 \
gl_ES.UTF-8/UTF-8 \
gl_ES/ISO-8859-1 \
gl_ES@euro/ISO-8859-15 \
gu_IN/UTF-8 \
gv_GB.UTF-8/UTF-8 \
gv_GB/ISO-8859-1 \
ha_NG/UTF-8 \
hak_TW/UTF-8 \
he_IL.UTF-8/UTF-8 \
he_IL/ISO-8859-8 \
hi_IN/UTF-8 \
hif_FJ/UTF-8 \
hne_IN/UTF-8 \
hr_HR.UTF-8/UTF-8 \
hr_HR/ISO-8859-2 \
hsb_DE/ISO-8859-2 \
hsb_DE.UTF-8/UTF-8 \
ht_HT/UTF-8 \
hu_HU.UTF-8/UTF-8 \
hu_HU/ISO-8859-2 \
hy_AM/UTF-8 \
hy_AM.ARMSCII-8/ARMSCII-8 \
ia_FR/UTF-8 \
id_ID.UTF-8/UTF-8 \
id_ID/ISO-8859-1 \
ig_NG/UTF-8 \
ik_CA/UTF-8 \
is_IS.UTF-8/UTF-8 \
is_IS/ISO-8859-1 \
it_CH.UTF-8/UTF-8 \
it_CH/ISO-8859-1 \
it_IT.UTF-8/UTF-8 \
it_IT/ISO-8859-1 \
it_IT@euro/ISO-8859-15 \
iu_CA/UTF-8 \
ja_JP.EUC-JP/EUC-JP \
ja_JP.UTF-8/UTF-8 \
ka_GE.UTF-8/UTF-8 \
ka_GE/GEORGIAN-PS \
kab_DZ/UTF-8 \
kk_KZ.UTF-8/UTF-8 \
kk_KZ/PT154 \
kl_GL.UTF-8/UTF-8 \
kl_GL/ISO-8859-1 \
km_KH/UTF-8 \
kn_IN/UTF-8 \
ko_KR.EUC-KR/EUC-KR \
ko_KR.UTF-8/UTF-8 \
kok_IN/UTF-8 \
ks_IN/UTF-8 \
ks_IN@devanagari/UTF-8 \
ku_TR.UTF-8/UTF-8 \
ku_TR/ISO-8859-9 \
kw_GB.UTF-8/UTF-8 \
kw_GB/ISO-8859-1 \
ky_KG/UTF-8 \
lb_LU/UTF-8 \
lg_UG.UTF-8/UTF-8 \
lg_UG/ISO-8859-10 \
li_BE/UTF-8 \
li_NL/UTF-8 \
lij_IT/UTF-8 \
ln_CD/UTF-8 \
lo_LA/UTF-8 \
lt_LT.UTF-8/UTF-8 \
lt_LT/ISO-8859-13 \
lv_LV.UTF-8/UTF-8 \
lv_LV/ISO-8859-13 \
lzh_TW/UTF-8 \
mag_IN/UTF-8 \
mai_IN/UTF-8 \
mai_NP/UTF-8 \
mfe_MU/UTF-8 \
mg_MG.UTF-8/UTF-8 \
mg_MG/ISO-8859-15 \
mhr_RU/UTF-8 \
mi_NZ.UTF-8/UTF-8 \
mi_NZ/ISO-8859-13 \
miq_NI/UTF-8 \
mjw_IN/UTF-8 \
mk_MK.UTF-8/UTF-8 \
mk_MK/ISO-8859-5 \
ml_IN/UTF-8 \
mn_MN/UTF-8 \
mni_IN/UTF-8 \
mr_IN/UTF-8 \
ms_MY.UTF-8/UTF-8 \
ms_MY/ISO-8859-1 \
mt_MT.UTF-8/UTF-8 \
mt_MT/ISO-8859-3 \
my_MM/UTF-8 \
nan_TW/UTF-8 \
nan_TW@latin/UTF-8 \
nb_NO.UTF-8/UTF-8 \
nb_NO/ISO-8859-1 \
nds_DE/UTF-8 \
nds_NL/UTF-8 \
ne_NP/UTF-8 \
nhn_MX/UTF-8 \
niu_NU/UTF-8 \
niu_NZ/UTF-8 \
nl_AW/UTF-8 \
nl_BE.UTF-8/UTF-8 \
nl_BE/ISO-8859-1 \
nl_BE@euro/ISO-8859-15 \
nl_NL.UTF-8/UTF-8 \
nl_NL/ISO-8859-1 \
nl_NL@euro/ISO-8859-15 \
nn_NO.UTF-8/UTF-8 \
nn_NO/ISO-8859-1 \
nr_ZA/UTF-8 \
nso_ZA/UTF-8 \
oc_FR.UTF-8/UTF-8 \
oc_FR/ISO-8859-1 \
om_ET/UTF-8 \
om_KE.UTF-8/UTF-8 \
om_KE/ISO-8859-1 \
or_IN/UTF-8 \
os_RU/UTF-8 \
pa_IN/UTF-8 \
pa_PK/UTF-8 \
pap_AW/UTF-8 \
pap_CW/UTF-8 \
pl_PL.UTF-8/UTF-8 \
pl_PL/ISO-8859-2 \
ps_AF/UTF-8 \
pt_BR.UTF-8/UTF-8 \
pt_BR/ISO-8859-1 \
pt_PT.UTF-8/UTF-8 \
pt_PT/ISO-8859-1 \
pt_PT@euro/ISO-8859-15 \
quz_PE/UTF-8 \
raj_IN/UTF-8 \
ro_RO.UTF-8/UTF-8 \
ro_RO/ISO-8859-2 \
ru_RU.KOI8-R/KOI8-R \
ru_RU.UTF-8/UTF-8 \
ru_RU/ISO-8859-5 \
ru_UA.UTF-8/UTF-8 \
ru_UA/KOI8-U \
rw_RW/UTF-8 \
sa_IN/UTF-8 \
sah_RU/UTF-8 \
sat_IN/UTF-8 \
sc_IT/UTF-8 \
sd_IN/UTF-8 \
sd_IN@devanagari/UTF-8 \
se_NO/UTF-8 \
sgs_LT/UTF-8 \
shn_MM/UTF-8 \
shs_CA/UTF-8 \
si_LK/UTF-8 \
sid_ET/UTF-8 \
sk_SK.UTF-8/UTF-8 \
sk_SK/ISO-8859-2 \
sl_SI.UTF-8/UTF-8 \
sl_SI/ISO-8859-2 \
sm_WS/UTF-8 \
so_DJ.UTF-8/UTF-8 \
so_DJ/ISO-8859-1 \
so_ET/UTF-8 \
so_KE.UTF-8/UTF-8 \
so_KE/ISO-8859-1 \
so_SO.UTF-8/UTF-8 \
so_SO/ISO-8859-1 \
sq_AL.UTF-8/UTF-8 \
sq_AL/ISO-8859-1 \
sq_MK/UTF-8 \
sr_ME/UTF-8 \
sr_RS/UTF-8 \
sr_RS@latin/UTF-8 \
ss_ZA/UTF-8 \
st_ZA.UTF-8/UTF-8 \
st_ZA/ISO-8859-1 \
sv_FI.UTF-8/UTF-8 \
sv_FI/ISO-8859-1 \
sv_FI@euro/ISO-8859-15 \
sv_SE.UTF-8/UTF-8 \
sv_SE/ISO-8859-1 \
sv_SE.ISO-8859-15/ISO-8859-15 \
sw_KE/UTF-8 \
sw_TZ/UTF-8 \
szl_PL/UTF-8 \
ta_IN/UTF-8 \
ta_LK/UTF-8 \
tcy_IN.UTF-8/UTF-8 \
te_IN/UTF-8 \
tg_TJ.UTF-8/UTF-8 \
tg_TJ/KOI8-T \
th_TH.UTF-8/UTF-8 \
th_TH/TIS-620 \
the_NP/UTF-8 \
ti_ER/UTF-8 \
ti_ET/UTF-8 \
tig_ER/UTF-8 \
tk_TM/UTF-8 \
tl_PH.UTF-8/UTF-8 \
tl_PH/ISO-8859-1 \
tn_ZA/UTF-8 \
to_TO/UTF-8 \
tpi_PG/UTF-8 \
tr_CY.UTF-8/UTF-8 \
tr_CY/ISO-8859-9 \
tr_TR.UTF-8/UTF-8 \
tr_TR/ISO-8859-9 \
ts_ZA/UTF-8 \
tt_RU/UTF-8 \
tt_RU@iqtelif/UTF-8 \
ug_CN/UTF-8 \
uk_UA.UTF-8/UTF-8 \
uk_UA/KOI8-U \
unm_US/UTF-8 \
ur_IN/UTF-8 \
ur_PK/UTF-8 \
uz_UZ.UTF-8/UTF-8 \
uz_UZ/ISO-8859-1 \
uz_UZ@cyrillic/UTF-8 \
ve_ZA/UTF-8 \
vi_VN/UTF-8 \
wa_BE/ISO-8859-1 \
wa_BE@euro/ISO-8859-15 \
wa_BE.UTF-8/UTF-8 \
wae_CH/UTF-8 \
wal_ET/UTF-8 \
wo_SN/UTF-8 \
xh_ZA.UTF-8/UTF-8 \
xh_ZA/ISO-8859-1 \
yi_US.UTF-8/UTF-8 \
yi_US/CP1255 \
yo_NG/UTF-8 \
yue_HK/UTF-8 \
yuw_PG/UTF-8 \
zh_CN.GB18030/GB18030 \
zh_CN.GBK/GBK \
zh_CN.UTF-8/UTF-8 \
zh_CN/GB2312 \
zh_HK.UTF-8/UTF-8 \
zh_HK/BIG5-HKSCS \
zh_SG.UTF-8/UTF-8 \
zh_SG.GBK/GBK \
zh_SG/GB2312 \
zh_TW.EUC-TW/EUC-TW \
zh_TW.UTF-8/UTF-8 \
zh_TW/BIG5 \
zu_ZA.UTF-8/UTF-8 \
zu_ZA/ISO-8859-1 \

View File

@ -1,862 +0,0 @@
#define _GNU_SOURCE
#include <assert.h>
#include <dirent.h>
#include <errno.h>
#include <fcntl.h>
#include <locale.h>
#include <stdarg.h>
#include <stdbool.h>
#include <stdio.h>
#include <stdlib.h>
#include <getopt.h>
#include <string.h>
#include <sys/mman.h>
#include <sys/stat.h>
#include <unistd.h>
#include "../locale/hashval.h"
#define __LC_LAST 13
#include "../locale/locarchive.h"
#include "../crypt/md5.h"
const char *alias_file = DATADIR "/locale/locale.alias";
const char *locar_file = PREFIX "/lib/locale/locale-archive";
const char *tmpl_file = PREFIX "/lib/locale/locale-archive.tmpl";
const char *loc_path = PREFIX "/lib/locale/";
/* Flags set by `--verbose` option. */
int be_quiet = 1;
int verbose = 0;
int max_locarchive_open_retry = 10;
const char *output_prefix;
/* Endianness should have been taken care of by localedef. We don't need to do
additional swapping. We need this variable exported however, since
locarchive.c uses it to determine if it needs to swap endianness of a value
before writing to or reading from the archive. */
bool swap_endianness_p = false;
static const char *locnames[] =
{
#define DEFINE_CATEGORY(category, category_name, items, a) \
[category] = category_name,
#include "../locale/categories.def"
#undef DEFINE_CATEGORY
};
static int
is_prime (unsigned long candidate)
{
/* No even number and none less than 10 will be passed here. */
unsigned long int divn = 3;
unsigned long int sq = divn * divn;
while (sq < candidate && candidate % divn != 0)
{
++divn;
sq += 4 * divn;
++divn;
}
return candidate % divn != 0;
}
unsigned long
next_prime (unsigned long seed)
{
/* Make it definitely odd. */
seed |= 1;
while (!is_prime (seed))
seed += 2;
return seed;
}
void
error (int status, int errnum, const char *message, ...)
{
va_list args;
va_start (args, message);
fflush (stdout);
fprintf (stderr, "%s: ", program_invocation_name);
vfprintf (stderr, message, args);
va_end (args);
if (errnum)
fprintf (stderr, ": %s", strerror (errnum));
putc ('\n', stderr);
fflush (stderr);
if (status)
exit (errnum == EROFS ? 0 : status);
}
void *
xmalloc (size_t size)
{
void *p = malloc (size);
if (p == NULL)
error (EXIT_FAILURE, errno, "could not allocate %zd bytes of memory", size);
return p;
}
static void
open_tmpl_archive (struct locarhandle *ah)
{
struct stat64 st;
int fd;
struct locarhead head;
const char *archivefname = ah->fname == NULL ? tmpl_file : ah->fname;
/* Open the archive. We must have exclusive write access. */
fd = open64 (archivefname, O_RDONLY);
if (fd == -1)
error (EXIT_FAILURE, errno, "cannot open locale archive template file \"%s\"",
archivefname);
if (fstat64 (fd, &st) < 0)
error (EXIT_FAILURE, errno, "cannot stat locale archive template file \"%s\"",
archivefname);
/* Read the header. */
if (TEMP_FAILURE_RETRY (read (fd, &head, sizeof (head))) != sizeof (head))
error (EXIT_FAILURE, errno, "cannot read archive header");
ah->fd = fd;
ah->mmaped = (head.sumhash_offset
+ head.sumhash_size * sizeof (struct sumhashent));
if (ah->mmaped > (unsigned long) st.st_size)
error (EXIT_FAILURE, 0, "locale archive template file truncated");
ah->mmaped = st.st_size;
ah->reserved = st.st_size;
/* Now we know how large the administrative information part is.
Map all of it. */
ah->addr = mmap64 (NULL, ah->mmaped, PROT_READ, MAP_SHARED, fd, 0);
if (ah->addr == MAP_FAILED)
error (EXIT_FAILURE, errno, "cannot map archive header");
}
/* Open the locale archive. */
extern void open_archive (struct locarhandle *ah, bool readonly);
/* Close the locale archive. */
extern void close_archive (struct locarhandle *ah);
/* Add given locale data to the archive. */
extern int add_locale_to_archive (struct locarhandle *ah, const char *name,
locale_data_t data, bool replace);
extern void add_alias (struct locarhandle *ah, const char *alias,
bool replace, const char *oldname,
uint32_t *locrec_offset_p);
extern struct namehashent *
insert_name (struct locarhandle *ah,
const char *name, size_t name_len, bool replace);
struct nameent
{
char *name;
struct locrecent *locrec;
};
struct dataent
{
const unsigned char *sum;
uint32_t file_offset;
};
static int
nameentcmp (const void *a, const void *b)
{
struct locrecent *la = ((const struct nameent *) a)->locrec;
struct locrecent *lb = ((const struct nameent *) b)->locrec;
uint32_t start_a = -1, end_a = 0;
uint32_t start_b = -1, end_b = 0;
int cnt;
for (cnt = 0; cnt < __LC_LAST; ++cnt)
if (cnt != LC_ALL)
{
if (la->record[cnt].offset < start_a)
start_a = la->record[cnt].offset;
if (la->record[cnt].offset + la->record[cnt].len > end_a)
end_a = la->record[cnt].offset + la->record[cnt].len;
}
assert (start_a != (uint32_t)-1);
assert (end_a != 0);
for (cnt = 0; cnt < __LC_LAST; ++cnt)
if (cnt != LC_ALL)
{
if (lb->record[cnt].offset < start_b)
start_b = lb->record[cnt].offset;
if (lb->record[cnt].offset + lb->record[cnt].len > end_b)
end_b = lb->record[cnt].offset + lb->record[cnt].len;
}
assert (start_b != (uint32_t)-1);
assert (end_b != 0);
if (start_a != start_b)
return (int)start_a - (int)start_b;
return (int)end_a - (int)end_b;
}
static int
dataentcmp (const void *a, const void *b)
{
if (((const struct dataent *) a)->file_offset
< ((const struct dataent *) b)->file_offset)
return -1;
if (((const struct dataent *) a)->file_offset
> ((const struct dataent *) b)->file_offset)
return 1;
return 0;
}
static int
sumsearchfn (const void *key, const void *ent)
{
uint32_t keyn = *(uint32_t *)key;
uint32_t entn = ((struct dataent *)ent)->file_offset;
if (keyn < entn)
return -1;
if (keyn > entn)
return 1;
return 0;
}
static void
compute_data (struct locarhandle *ah, struct nameent *name, size_t sumused,
struct dataent *files, locale_data_t data)
{
int cnt;
struct locrecent *locrec = name->locrec;
struct dataent *file;
data[LC_ALL].addr = ((char *) ah->addr) + locrec->record[LC_ALL].offset;
data[LC_ALL].size = locrec->record[LC_ALL].len;
for (cnt = 0; cnt < __LC_LAST; ++cnt)
if (cnt != LC_ALL)
{
data[cnt].addr = ((char *) ah->addr) + locrec->record[cnt].offset;
data[cnt].size = locrec->record[cnt].len;
if (data[cnt].addr >= data[LC_ALL].addr
&& data[cnt].addr + data[cnt].size
<= data[LC_ALL].addr + data[LC_ALL].size)
__md5_buffer (data[cnt].addr, data[cnt].size, data[cnt].sum);
else
{
file = bsearch (&locrec->record[cnt].offset, files, sumused,
sizeof (*files), sumsearchfn);
if (file == NULL)
error (EXIT_FAILURE, 0, "inconsistent template file");
memcpy (data[cnt].sum, file->sum, sizeof (data[cnt].sum));
}
}
}
static int
fill_archive (struct locarhandle *tmpl_ah,
const char *fname,
size_t install_langs_count, char *install_langs_list[],
size_t nlist, char *list[],
const char *primary)
{
struct locarhandle ah;
struct locarhead *head;
int result = 0;
struct nameent *names;
struct namehashent *namehashtab;
size_t cnt, used;
struct dataent *files;
struct sumhashent *sumhashtab;
size_t sumused;
struct locrecent *primary_locrec = NULL;
struct nameent *primary_nameent = NULL;
head = tmpl_ah->addr;
names = (struct nameent *) malloc (head->namehash_used
* sizeof (struct nameent));
files = (struct dataent *) malloc (head->sumhash_used
* sizeof (struct dataent));
if (names == NULL || files == NULL)
error (EXIT_FAILURE, errno, "could not allocate tables");
namehashtab = (struct namehashent *) ((char *) tmpl_ah->addr
+ head->namehash_offset);
sumhashtab = (struct sumhashent *) ((char *) tmpl_ah->addr
+ head->sumhash_offset);
for (cnt = used = 0; cnt < head->namehash_size; ++cnt)
if (namehashtab[cnt].locrec_offset != 0)
{
char * name;
int i;
assert (used < head->namehash_used);
name = tmpl_ah->addr + namehashtab[cnt].name_offset;
if (install_langs_count == 0)
{
/* Always intstall the entry. */
names[used].name = name;
names[used++].locrec
= (struct locrecent *) ((char *) tmpl_ah->addr +
namehashtab[cnt].locrec_offset);
}
else
{
/* Only install the entry if the user asked for it via
--install-langs. */
for (i = 0; i < install_langs_count; i++)
{
/* Add one for "_" and one for the null terminator. */
size_t len = strlen (install_langs_list[i]) + 2;
char *install_lang = (char *)xmalloc (len);
strcpy (install_lang, install_langs_list[i]);
if (strchr (install_lang, '_') == NULL)
strcat (install_lang, "_");
if (strncmp (name, install_lang, strlen (install_lang)) == 0)
{
names[used].name = name;
names[used++].locrec
= (struct locrecent *) ((char *)tmpl_ah->addr
+ namehashtab[cnt].locrec_offset);
}
free (install_lang);
}
}
}
/* Sort the names. */
qsort (names, used, sizeof (struct nameent), nameentcmp);
for (cnt = sumused = 0; cnt < head->sumhash_size; ++cnt)
if (sumhashtab[cnt].file_offset != 0)
{
assert (sumused < head->sumhash_used);
files[sumused].sum = (const unsigned char *) sumhashtab[cnt].sum;
files[sumused++].file_offset = sumhashtab[cnt].file_offset;
}
/* Sort by file locations. */
qsort (files, sumused, sizeof (struct dataent), dataentcmp);
/* Open the archive. This call never returns if we cannot
successfully open the archive. */
ah.fname = NULL;
if (fname != NULL)
ah.fname = fname;
open_archive (&ah, false);
if (primary != NULL)
{
for (cnt = 0; cnt < used; ++cnt)
if (strcmp (names[cnt].name, primary) == 0)
break;
if (cnt < used)
{
locale_data_t data;
compute_data (tmpl_ah, &names[cnt], sumused, files, data);
result |= add_locale_to_archive (&ah, primary, data, 0);
primary_locrec = names[cnt].locrec;
primary_nameent = &names[cnt];
}
}
for (cnt = 0; cnt < used; ++cnt)
if (&names[cnt] == primary_nameent)
continue;
else if ((cnt > 0 && names[cnt - 1].locrec == names[cnt].locrec)
|| names[cnt].locrec == primary_locrec)
{
const char *oldname;
struct namehashent *namehashent;
uint32_t locrec_offset;
if (names[cnt].locrec == primary_locrec)
oldname = primary;
else
oldname = names[cnt - 1].name;
namehashent = insert_name (&ah, oldname, strlen (oldname), true);
assert (namehashent->name_offset != 0);
assert (namehashent->locrec_offset != 0);
locrec_offset = namehashent->locrec_offset;
add_alias (&ah, names[cnt].name, 0, oldname, &locrec_offset);
}
else
{
locale_data_t data;
compute_data (tmpl_ah, &names[cnt], sumused, files, data);
result |= add_locale_to_archive (&ah, names[cnt].name, data, 0);
}
while (nlist-- > 0)
{
const char *fname = *list++;
size_t fnamelen = strlen (fname);
struct stat64 st;
DIR *dirp;
struct dirent64 *d;
int seen;
locale_data_t data;
int cnt;
/* First see whether this really is a directory and whether it
contains all the require locale category files. */
if (stat64 (fname, &st) < 0)
{
error (0, 0, "stat of \"%s\" failed: %s: ignored", fname,
strerror (errno));
continue;
}
if (!S_ISDIR (st.st_mode))
{
error (0, 0, "\"%s\" is no directory; ignored", fname);
continue;
}
dirp = opendir (fname);
if (dirp == NULL)
{
error (0, 0, "cannot open directory \"%s\": %s: ignored",
fname, strerror (errno));
continue;
}
seen = 0;
while ((d = readdir64 (dirp)) != NULL)
{
for (cnt = 0; cnt < __LC_LAST; ++cnt)
if (cnt != LC_ALL)
if (strcmp (d->d_name, locnames[cnt]) == 0)
{
unsigned char d_type;
/* We have an object of the required name. If it's
a directory we have to look at a file with the
prefix "SYS_". Otherwise we have found what we
are looking for. */
#ifdef _DIRENT_HAVE_D_TYPE
d_type = d->d_type;
if (d_type != DT_REG)
#endif
{
char fullname[fnamelen + 2 * strlen (d->d_name) + 7];
#ifdef _DIRENT_HAVE_D_TYPE
if (d_type == DT_UNKNOWN || d_type == DT_LNK)
#endif
{
strcpy (stpcpy (stpcpy (fullname, fname), "/"),
d->d_name);
if (stat64 (fullname, &st) == -1)
/* We cannot stat the file, ignore it. */
break;
d_type = IFTODT (st.st_mode);
}
if (d_type == DT_DIR)
{
/* We have to do more tests. The file is a
directory and it therefore must contain a
regular file with the same name except a
"SYS_" prefix. */
char *t = stpcpy (stpcpy (fullname, fname), "/");
strcpy (stpcpy (stpcpy (t, d->d_name), "/SYS_"),
d->d_name);
if (stat64 (fullname, &st) == -1)
/* There is no SYS_* file or we cannot
access it. */
break;
d_type = IFTODT (st.st_mode);
}
}
/* If we found a regular file (eventually after
following a symlink) we are successful. */
if (d_type == DT_REG)
++seen;
break;
}
}
closedir (dirp);
if (seen != __LC_LAST - 1)
{
/* We don't have all locale category files. Ignore the name. */
error (0, 0, "incomplete set of locale files in \"%s\"",
fname);
continue;
}
/* Add the files to the archive. To do this we first compute
sizes and the MD5 sums of all the files. */
for (cnt = 0; cnt < __LC_LAST; ++cnt)
if (cnt != LC_ALL)
{
char fullname[fnamelen + 2 * strlen (locnames[cnt]) + 7];
int fd;
strcpy (stpcpy (stpcpy (fullname, fname), "/"), locnames[cnt]);
fd = open64 (fullname, O_RDONLY);
if (fd == -1 || fstat64 (fd, &st) == -1)
{
/* Cannot read the file. */
if (fd != -1)
close (fd);
break;
}
if (S_ISDIR (st.st_mode))
{
char *t;
close (fd);
t = stpcpy (stpcpy (fullname, fname), "/");
strcpy (stpcpy (stpcpy (t, locnames[cnt]), "/SYS_"),
locnames[cnt]);
fd = open64 (fullname, O_RDONLY);
if (fd == -1 || fstat64 (fd, &st) == -1
|| !S_ISREG (st.st_mode))
{
if (fd != -1)
close (fd);
break;
}
}
/* Map the file. */
data[cnt].addr = mmap64 (NULL, st.st_size, PROT_READ, MAP_SHARED,
fd, 0);
if (data[cnt].addr == MAP_FAILED)
{
/* Cannot map it. */
close (fd);
break;
}
data[cnt].size = st.st_size;
__md5_buffer (data[cnt].addr, st.st_size, data[cnt].sum);
/* We don't need the file descriptor anymore. */
close (fd);
}
if (cnt != __LC_LAST)
{
while (cnt-- > 0)
if (cnt != LC_ALL)
munmap (data[cnt].addr, data[cnt].size);
error (0, 0, "cannot read all files in \"%s\": ignored", fname);
continue;
}
result |= add_locale_to_archive (&ah, basename (fname), data, 0);
for (cnt = 0; cnt < __LC_LAST; ++cnt)
if (cnt != LC_ALL)
munmap (data[cnt].addr, data[cnt].size);
}
/* We are done. */
close_archive (&ah);
return result;
}
void usage()
{
printf ("\
Usage: build-locale-archive [OPTION]... [TEMPLATE-FILE] [ARCHIVE-FILE]\n\
Builds a locale archive from a template file.\n\
Options:\n\
-h, --help Print this usage message.\n\
-v, --verbose Verbose execution.\n\
-l, --install-langs=LIST Only include locales given in LIST into the \n\
locale archive. LIST is a colon separated list\n\
of locale prefixes, for example \"de:en:ja\".\n\
The special argument \"all\" means to install\n\
all languages and it must be present by itself.\n\
If \"all\" is present with any other language it\n\
will be treated as the name of a locale.\n\
If the --install-langs option is missing, all\n\
locales are installed. The colon separated list\n\
can contain any strings matching the beginning of\n\
locale names.\n\
If a string does not contain a \"_\", it is added.\n\
Examples:\n\
--install-langs=\"en\"\n\
installs en_US, en_US.iso88591,\n\
en_US.iso885915, en_US.utf8,\n\
en_GB ...\n\
--install-langs=\"en_US.utf8\"\n\
installs only en_US.utf8.\n\
--install-langs=\"ko\"\n\
installs ko_KR, ko_KR.euckr,\n\
ko_KR.utf8 but *not* kok_IN\n\
because \"ko\" does not contain\n\
\"_\" and it is silently added\n\
--install-langs\"ko:kok\"\n\
installs ko_KR, ko_KR.euckr,\n\
ko_KR.utf8, kok_IN, and\n\
kok_IN.utf8.\n\
--install-langs=\"POSIX\" will\n\
installs *no* locales at all\n\
because POSIX matches none of\n\
the locales. Actually, any string\n\
matching nothing will do that.\n\
POSIX and C will always be\n\
available because they are\n\
builtin.\n\
Aliases are installed as well,\n\
i.e. --install-langs=\"de\"\n\
will install not only every locale starting with\n\
\"de\" but also the aliases \"deutsch\"\n\
and and \"german\" although the latter does not\n\
start with \"de\".\n\
\n\
If the arguments TEMPLATE-FILE and ARCHIVE-FILE are not given the locations\n\
where the glibc used expects these files are used by default.\n\
");
}
int main (int argc, char *argv[])
{
char path[4096];
DIR *dirp;
struct dirent64 *d;
struct stat64 st;
char *list[16384], *primary;
char *lang;
int install_langs_count = 0;
int i;
char *install_langs_arg, *ila_start;
char **install_langs_list = NULL;
unsigned int cnt = 0;
struct locarhandle tmpl_ah;
char *new_locar_fname = NULL;
size_t loc_path_len = strlen (loc_path);
while (1)
{
int c;
static struct option long_options[] =
{
{"help", no_argument, 0, 'h'},
{"verbose", no_argument, 0, 'v'},
{"install-langs", required_argument, 0, 'l'},
{0, 0, 0, 0}
};
/* getopt_long stores the option index here. */
int option_index = 0;
c = getopt_long (argc, argv, "vhl:",
long_options, &option_index);
/* Detect the end of the options. */
if (c == -1)
break;
switch (c)
{
case 0:
printf ("unknown option %s", long_options[option_index].name);
if (optarg)
printf (" with arg %s", optarg);
printf ("\n");
usage ();
exit (1);
case 'v':
verbose = 1;
be_quiet = 0;
break;
case 'h':
usage ();
exit (0);
case 'l':
install_langs_arg = ila_start = strdup (optarg);
/* If the argument to --install-lang is "all", do
not limit the list of languages to install and install
them all. We do not support installing a single locale
called "all". */
#define MAGIC_INSTALL_ALL "all"
if (install_langs_arg != NULL
&& install_langs_arg[0] != '\0'
&& !(strncmp(install_langs_arg, MAGIC_INSTALL_ALL,
strlen(MAGIC_INSTALL_ALL)) == 0
&& strlen (install_langs_arg) == 3))
{
/* Count the number of languages we will install. */
while (true)
{
lang = strtok(install_langs_arg, ":;,");
if (lang == NULL)
break;
install_langs_count++;
install_langs_arg = NULL;
}
free (ila_start);
/* Reject an entire string made up of delimiters. */
if (install_langs_count == 0)
break;
/* Copy the list. */
install_langs_list = (char **)xmalloc (sizeof(char *) * install_langs_count);
install_langs_arg = ila_start = strdup (optarg);
install_langs_count = 0;
while (true)
{
lang = strtok(install_langs_arg, ":;,");
if (lang == NULL)
break;
install_langs_list[install_langs_count] = lang;
install_langs_count++;
install_langs_arg = NULL;
}
}
break;
case '?':
/* getopt_long already printed an error message. */
usage ();
exit (0);
default:
abort ();
}
}
tmpl_ah.fname = NULL;
if (optind < argc)
tmpl_ah.fname = argv[optind];
if (optind + 1 < argc)
new_locar_fname = argv[optind + 1];
if (verbose)
{
if (tmpl_ah.fname)
printf("input archive file specified on command line: %s\n",
tmpl_ah.fname);
else
printf("using default input archive file.\n");
if (new_locar_fname)
printf("output archive file specified on command line: %s\n",
new_locar_fname);
else
printf("using default output archive file.\n");
}
dirp = opendir (loc_path);
if (dirp == NULL)
error (EXIT_FAILURE, errno, "cannot open directory \"%s\"", loc_path);
open_tmpl_archive (&tmpl_ah);
if (new_locar_fname)
unlink (new_locar_fname);
else
unlink (locar_file);
primary = getenv ("LC_ALL");
if (primary == NULL)
primary = getenv ("LANG");
if (primary != NULL)
{
if (strncmp (primary, "ja", 2) != 0
&& strncmp (primary, "ko", 2) != 0
&& strncmp (primary, "zh", 2) != 0)
{
char *ptr = malloc (strlen (primary) + strlen (".utf8") + 1), *p, *q;
/* This leads to invalid locales sometimes:
de_DE.iso885915@euro -> de_DE.utf8@euro */
if (ptr != NULL)
{
p = ptr;
q = primary;
while (*q && *q != '.' && *q != '@')
*p++ = *q++;
if (*q == '.')
while (*q && *q != '@')
q++;
p = stpcpy (p, ".utf8");
strcpy (p, q);
primary = ptr;
}
else
primary = NULL;
}
}
memcpy (path, loc_path, loc_path_len);
while ((d = readdir64 (dirp)) != NULL)
{
if (strcmp (d->d_name, ".") == 0 || strcmp (d->d_name, "..") == 0)
continue;
if (strchr (d->d_name, '_') == NULL)
continue;
size_t d_name_len = strlen (d->d_name);
if (loc_path_len + d_name_len + 1 > sizeof (path))
{
error (0, 0, "too long filename \"%s\"", d->d_name);
continue;
}
memcpy (path + loc_path_len, d->d_name, d_name_len + 1);
if (stat64 (path, &st) < 0)
{
error (0, errno, "cannot stat \"%s\"", path);
continue;
}
if (! S_ISDIR (st.st_mode))
continue;
if (cnt == 16384)
{
error (0, 0, "too many directories in \"%s\"", loc_path);
break;
}
list[cnt] = strdup (path);
if (list[cnt] == NULL)
{
error (0, errno, "cannot add file to list \"%s\"", path);
continue;
}
if (primary != NULL && cnt > 0 && strcmp (primary, d->d_name) == 0)
{
char *p = list[0];
list[0] = list[cnt];
list[cnt] = p;
}
cnt++;
}
closedir (dirp);
/* Store the archive to the file specified as the second argument on the
command line or the default locale archive. */
fill_archive (&tmpl_ah, new_locar_fname,
install_langs_count, install_langs_list,
cnt, list, primary);
close_archive (&tmpl_ah);
truncate (tmpl_file, 0);
if (install_langs_count > 0)
{
free (ila_start);
free (install_langs_list);
}
char *tz_argv[] = { "/usr/sbin/tzdata-update", NULL };
execve (tz_argv[0], (char *const *)tz_argv, (char *const *)&tz_argv[1]);
exit (0);
}

View File

@ -1,236 +0,0 @@
commit 7ea06e994093fa0bcca0d0ee2c1db271d8d7885d
Author: Florian Weimer <fweimer@redhat.com>
Date: Mon Jul 21 21:43:49 2025 +0200
posix: Fix double-free after allocation failure in regcomp (bug 33185)
If a memory allocation failure occurs during bracket expression
parsing in regcomp, a double-free error may result.
Reported-by: Anastasia Belova <abelova@astralinux.ru>
Co-authored-by: Paul Eggert <eggert@cs.ucla.edu>
Reviewed-by: Andreas K. Huettel <dilfridge@gentoo.org>
Conflicts:
posix/Makefile
(tests list not reformatted/sorted downstream)
posix/tst-regcomp-bracket-free.c
(missing strerrorname_np downstream)
diff --git a/posix/Makefile b/posix/Makefile
index 83162123f9c927a0..42a0290370b40fd9 100644
--- a/posix/Makefile
+++ b/posix/Makefile
@@ -96,7 +96,7 @@ tests := test-errno tstgetopt testfnm runtests runptests \
tst-posix_fadvise tst-posix_fadvise64 \
tst-sysconf-empty-chroot tst-glob_symlinks tst-fexecve \
tst-glob-tilde test-ssize-max tst-spawn4 bug-regex37 \
- bug-regex38 tst-regcomp-truncated
+ bug-regex38 tst-regcomp-truncated tst-regcomp-bracket-free
tests-internal := bug-regex5 bug-regex20 bug-regex33 \
tst-rfc3484 tst-rfc3484-2 tst-rfc3484-3 \
tst-glob_lstat_compat tst-spawn4-compat
diff --git a/posix/regcomp.c b/posix/regcomp.c
index 545d188468c376e7..b737b22da8703d6c 100644
--- a/posix/regcomp.c
+++ b/posix/regcomp.c
@@ -3375,6 +3375,7 @@ parse_bracket_exp (re_string_t *regexp, re_dfa_t *dfa, re_token_t *token,
{
#ifdef RE_ENABLE_I18N
free_charset (mbcset);
+ mbcset = NULL;
#endif
/* Build a tree for simple bracket. */
br_token.type = SIMPLE_BRACKET;
@@ -3390,7 +3391,8 @@ parse_bracket_exp (re_string_t *regexp, re_dfa_t *dfa, re_token_t *token,
parse_bracket_exp_free_return:
re_free (sbcset);
#ifdef RE_ENABLE_I18N
- free_charset (mbcset);
+ if (__glibc_likely (mbcset != NULL))
+ free_charset (mbcset);
#endif /* RE_ENABLE_I18N */
return NULL;
}
diff --git a/posix/tst-regcomp-bracket-free.c b/posix/tst-regcomp-bracket-free.c
new file mode 100644
index 0000000000000000..e6041ddaeba3045c
--- /dev/null
+++ b/posix/tst-regcomp-bracket-free.c
@@ -0,0 +1,176 @@
+/* Test regcomp bracket parsing with injected allocation failures (bug 33185).
+ Copyright (C) 2025 Free Software Foundation, Inc.
+ This file is part of the GNU C Library.
+
+ The GNU C Library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ The GNU C Library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with the GNU C Library; if not, see
+ <https://www.gnu.org/licenses/>. */
+
+/* This test invokes regcomp multiple times, failing one memory
+ allocation in each call. The function call should fail with
+ REG_ESPACE (or succeed if it can recover from the allocation
+ failure). Previously, there was double-free bug. */
+
+#include <errno.h>
+#include <regex.h>
+#include <stdio.h>
+#include <string.h>
+#include <support/check.h>
+#include <support/namespace.h>
+#include <support/support.h>
+
+/* Data structure allocated via MAP_SHARED, so that writes from the
+ subprocess are visible. */
+struct shared_data
+{
+ /* Number of tracked allocations performed so far. */
+ volatile unsigned int allocation_count;
+
+ /* If this number is reached, one allocation fails. */
+ volatile unsigned int failing_allocation;
+
+ /* The subprocess stores the expected name here. */
+ char name[100];
+};
+
+/* Allocation count in shared mapping. */
+static struct shared_data *shared;
+
+/* Returns true if a failure should be injected for this allocation. */
+static bool
+fail_this_allocation (void)
+{
+ if (shared != NULL)
+ {
+ unsigned int count = shared->allocation_count;
+ shared->allocation_count = count + 1;
+ return count == shared->failing_allocation;
+ }
+ else
+ return false;
+}
+
+/* Failure-injecting wrappers for allocation functions used by glibc. */
+
+void *
+malloc (size_t size)
+{
+ if (fail_this_allocation ())
+ {
+ errno = ENOMEM;
+ return NULL;
+ }
+ extern __typeof (malloc) __libc_malloc;
+ return __libc_malloc (size);
+}
+
+void *
+calloc (size_t a, size_t b)
+{
+ if (fail_this_allocation ())
+ {
+ errno = ENOMEM;
+ return NULL;
+ }
+ extern __typeof (calloc) __libc_calloc;
+ return __libc_calloc (a, b);
+}
+
+void *
+realloc (void *ptr, size_t size)
+{
+ if (fail_this_allocation ())
+ {
+ errno = ENOMEM;
+ return NULL;
+ }
+ extern __typeof (realloc) __libc_realloc;
+ return __libc_realloc (ptr, size);
+}
+
+/* No-op subprocess to verify that support_isolate_in_subprocess does
+ not perform any heap allocations. */
+static void
+no_op (void *ignored)
+{
+}
+
+/* Perform a regcomp call in a subprocess. Used to count its
+ allocations. */
+static void
+initialize (void *regexp1)
+{
+ const char *regexp = regexp1;
+
+ shared->allocation_count = 0;
+
+ regex_t reg;
+ TEST_COMPARE (regcomp (&reg, regexp, 0), 0);
+}
+
+/* Perform regcomp in a subprocess with fault injection. */
+static void
+test_in_subprocess (void *regexp1)
+{
+ const char *regexp = regexp1;
+ unsigned int inject_at = shared->failing_allocation;
+
+ regex_t reg;
+ int ret = regcomp (&reg, regexp, 0);
+
+ if (ret != 0)
+ {
+ TEST_COMPARE (ret, REG_ESPACE);
+ printf ("info: allocation %u failure results in return value %d,"
+ " error %s (%d)\n",
+ inject_at, ret, strerror (errno), errno);
+ }
+}
+
+static int
+do_test (void)
+{
+ char regexp[] = "[:alpha:]";
+
+ shared = support_shared_allocate (sizeof (*shared));
+
+ /* Disable fault injection. */
+ shared->failing_allocation = ~0U;
+
+ support_isolate_in_subprocess (no_op, NULL);
+ TEST_COMPARE (shared->allocation_count, 0);
+
+ support_isolate_in_subprocess (initialize, regexp);
+
+ /* The number of allocations in the successful case, plus some
+ slack. Once the number of expected allocations is exceeded,
+ injecting further failures does not make a difference. */
+ unsigned int maximum_allocation_count = shared->allocation_count;
+ printf ("info: successful call performs %u allocations\n",
+ maximum_allocation_count);
+ maximum_allocation_count += 10;
+
+ for (unsigned int inject_at = 0; inject_at <= maximum_allocation_count;
+ ++inject_at)
+ {
+ shared->allocation_count = 0;
+ shared->failing_allocation = inject_at;
+ support_isolate_in_subprocess (test_in_subprocess, regexp);
+ }
+
+ support_shared_free (shared);
+
+ return 0;
+}
+
+#include <support/test-driver.c>

View File

@ -1,45 +0,0 @@
commit 0fceed254559836b57ee05188deac649bc505d05
Author: Florian Weimer <fweimer@redhat.com>
Date: Fri Sep 12 21:33:34 2025 +0200
nss: Group merge does not react to ERANGE during merge (bug 33361)
The break statement in CHECK_MERGE is expected to exit the surrounding
while loop, not the do-while loop with in the macro. Remove the
do-while loop from the macro. It is not needed to turn the macro
expansion into a single statement due to the way CHECK_MERGE is used
(and the statement expression would cover this anyway).
Reviewed-by: Collin Funk <collin.funk1@gmail.com>
diff --git a/nss/getXXbyYY_r.c b/nss/getXXbyYY_r.c
index eae6c3480e..2b0735fb6a 100644
--- a/nss/getXXbyYY_r.c
+++ b/nss/getXXbyYY_r.c
@@ -157,19 +157,15 @@ __merge_einval (LOOKUP_TYPE *a,
#define CHECK_MERGE(err, status) \
({ \
- do \
+ if (err) \
{ \
- if (err) \
- { \
- __set_errno (err); \
- if (err == ERANGE) \
- status = NSS_STATUS_TRYAGAIN; \
- else \
- status = NSS_STATUS_UNAVAIL; \
- break; \
- } \
+ __set_errno (err); \
+ if (err == ERANGE) \
+ status = NSS_STATUS_TRYAGAIN; \
+ else \
+ status = NSS_STATUS_UNAVAIL; \
+ break; \
} \
- while (0); \
})
/* Type of the lookup function we need here. */

View File

@ -1,30 +0,0 @@
commit 78fdb2d6b1c34ea8e779fd48f9436dfbd50b6387
Author: Florian Weimer <fweimer@redhat.com>
Date: Thu Jan 8 12:35:08 2026 +0100
Switch currency symbol for the bg_BG locale to euro
Bulgaria joined the eurozone on 2026-01-01.
Suggested-by: Йордан Гигов <jgigov@abv.bg>
Reviewed-by: Collin Funk <collin.funk1@gmail.com>
Conflicts:
localedata/locales/bg_BG
(Adjust for missing UTF-8 conversion and separator/grouping
fixes downstream)
diff -Nrup a/localedata/locales/bg_BG b/localedata/locales/bg_BG
--- a/localedata/locales/bg_BG 2018-08-01 01:10:47.000000000 -0400
+++ b/localedata/locales/bg_BG 2026-04-09 15:01:50.118100059 -0400
@@ -248,8 +248,8 @@ reorder-end
END LC_COLLATE
LC_MONETARY
-int_curr_symbol "BGN "
-currency_symbol "<U043B><U0432>."
+int_curr_symbol "EUR "
+currency_symbol "<U20AC>"
mon_decimal_point ","
mon_thousands_sep "<U202F>"
mon_grouping 3;3

View File

@ -1,59 +0,0 @@
commit 559010e471acb3cb292615b71b248aba73e5c2fe
Author: Dragan Stanojević (Nevidljivi) <neville.ravenwood@gmail.com>
Date: Wed Feb 7 16:31:04 2024 +0100
localedata: hr_HR: change currency to EUR/€
Resolves: BZ # 29845
Conflicts:
localedata/locales/hr_HR
(locale definitions do not use UTF-8 downstream)
diff --git a/localedata/locales/hr_HR b/localedata/locales/hr_HR
index 029a3794e2a17dcb..a13370461eede9c1 100644
--- a/localedata/locales/hr_HR
+++ b/localedata/locales/hr_HR
@@ -24,8 +24,8 @@ tel ""
fax ""
language "Croatian"
territory "Croatia"
-revision "2.3"
-date "2016-04-16"
+revision "2.4"
+date "2022-12-03"
category "i18n:2012";LC_IDENTIFICATION
category "i18n:2012";LC_CTYPE
@@ -171,8 +171,8 @@ nostr "ne"
END LC_MESSAGES
LC_MONETARY
-int_curr_symbol "HRK "
-currency_symbol "kn"
+int_curr_symbol "EUR "
+currency_symbol "<U20AC>"
mon_decimal_point ","
mon_thousands_sep "."
mon_grouping 3;3
diff --git a/stdlib/tst-strfmon_l.c b/stdlib/tst-strfmon_l.c
index 603d4b0c3d91818b..65313cd14b61dfe4 100644
--- a/stdlib/tst-strfmon_l.c
+++ b/stdlib/tst-strfmon_l.c
@@ -181,12 +181,12 @@ static const struct locale_pair tests[] =
"hr_HR.UTF-8",
{
{
- "HRK 1.234.567,89", "1.234.567,89 kn",
- "HRK 1234567,89", "1234567,89 kn"
+ "EUR 1.234.567,89", "1.234.567,89 €",
+ "EUR 1234567,89", "1234567,89 €"
},
{
- "-HRK 1.234.567,89", "-1.234.567,89 kn",
- "-HRK 1234567,89", "-1234567,89 kn"
+ "-EUR 1.234.567,89", "-1.234.567,89 €",
+ "-EUR 1234567,89", "-1234567,89 €"
}
}
},

View File

@ -1,70 +0,0 @@
commit e56ff82d5034ec66c6a78f517af6faa427f65b0b
Author: Carlos O'Donell <carlos@redhat.com>
Date: Thu Jan 15 15:09:38 2026 -0500
resolv: Fix NSS DNS backend for getnetbyaddr (CVE-2026-0915)
The default network value of zero for net was never tested for and
results in a DNS query constructed from uninitialized stack bytes.
The solution is to provide a default query for the case where net
is zero.
Adding a test case for this was straight forward given the existence of
tst-resolv-network and if the test is added without the fix you observe
this failure:
FAIL: resolv/tst-resolv-network
original exit status 1
error: tst-resolv-network.c:174: invalid QNAME: \146\218\129\128
error: 1 test failures
With a random QNAME resulting from the use of uninitialized stack bytes.
After the fix the test passes.
Additionally verified using wireshark before and after to ensure
on-the-wire bytes for the DNS query were as expected.
No regressions on x86_64.
Reviewed-by: Florian Weimer <fweimer@redhat.com>
diff --git a/resolv/nss_dns/dns-network.c b/resolv/nss_dns/dns-network.c
index 61bddd754f2d73c0..cbab554f1e761016 100644
--- a/resolv/nss_dns/dns-network.c
+++ b/resolv/nss_dns/dns-network.c
@@ -207,6 +207,10 @@ _nss_dns_getnetbyaddr_r (uint32_t net, int type, struct netent *result,
sprintf (qbuf, "%u.%u.%u.%u.in-addr.arpa", net_bytes[3], net_bytes[2],
net_bytes[1], net_bytes[0]);
break;
+ default:
+ /* Default network (net is originally zero). */
+ strcpy (qbuf, "0.0.0.0.in-addr.arpa");
+ break;
}
net_buffer.buf = orig_net_buffer = (querybuf *) alloca (1024);
diff --git a/resolv/tst-resolv-network.c b/resolv/tst-resolv-network.c
index 4b862d57e65276e5..afc1874160179fcc 100644
--- a/resolv/tst-resolv-network.c
+++ b/resolv/tst-resolv-network.c
@@ -46,6 +46,9 @@ handle_code (const struct resolv_response_context *ctx,
{
switch (code)
{
+ case 0:
+ send_ptr (b, qname, qclass, qtype, "0.in-addr.arpa");
+ break;
case 1:
send_ptr (b, qname, qclass, qtype, "1.in-addr.arpa");
break;
@@ -259,6 +262,9 @@ do_test (void)
"error: NO_RECOVERY\n");
/* Lookup by address, success cases. */
+ check_reverse (0,
+ "name: 0.in-addr.arpa\n"
+ "net: 0x00000000\n");
check_reverse (1,
"name: 1.in-addr.arpa\n"
"net: 0x00000001\n");

View File

@ -1,84 +0,0 @@
commit 7b543dcdf97d07fd4346feb17916e08fe83ad0ae
Author: Florian Weimer <fweimer@redhat.com>
Date: Thu Jan 15 22:29:46 2026 +0100
elf: Ignore LD_PROFILE if LD_PROFILE_OUTPUT is not set (bug 33797)
The previous default for LD_PROFILE_OUTPUT, /var/tmp, is insecure
because it's typically a 1777 directory, and other systems could
place malicious files there which interfere with execution.
Requiring the user to specify a profiling directory mitigates
the impact of bug 33797. Clear LD_PROFILE_OUTPUT alongside
with LD_PROFILE.
Rework the test not to use predictable file names.
Reviewed-by: Carlos O'Donell <carlos@redhat.com>
Conflicts:
elf/rtld.c
(different implementation of environment variable filtering
downstream, incorporate changes from upstream commit
4a133885a7c8ae7ebe34e36fcdb353f8e94c810f, adjust for
GLRO(_dl_profile_output) use in glibc-rh2047981-44.patch)
elf/tst-env-setuid.c
(no LD_PROFILE test downstream)
diff --git a/elf/rtld.c b/elf/rtld.c
index 48698f93a4873a6d..848f6f51d093f313 100644
--- a/elf/rtld.c
+++ b/elf/rtld.c
@@ -2684,11 +2684,9 @@ process_envvars (struct dl_main_state *state)
char *envline;
char *debug_output = NULL;
- /* This is the default place for profiling data file. As a side
- effect, this marks ld.so as initialized, so that the rtld_active
- function returns true from now on. */
- GLRO(dl_profile_output)
- = &"/var/tmp\0/var/profile"[__libc_enable_secure ? 9 : 0];
+ /* This marks ld.so as initialized, so that the rtld_active function
+ returns true from now on. "" means no default. */
+ GLRO(dl_profile_output) = "";
while ((envline = _dl_next_ld_env_entry (&runp)) != NULL)
{
@@ -2738,7 +2736,8 @@ process_envvars (struct dl_main_state *state)
}
/* Which shared object shall be profiled. */
- if (memcmp (envline, "PROFILE", 7) == 0 && envline[8] != '\0')
+ if (!__libc_enable_secure
+ && memcmp (envline, "PROFILE", 7) == 0 && envline[8] != '\0')
GLRO(dl_profile) = &envline[8];
break;
@@ -2899,6 +2898,15 @@ process_envvars (struct dl_main_state *state)
/* We use standard output if opening the file failed. */
GLRO(dl_debug_fd) = STDOUT_FILENO;
}
+
+ /* There is no fixed, safe directory to store profiling data, so
+ activate LD_PROFILE only if LD_PROFILE_OUTPUT is set as well. */
+ if (GLRO(dl_profile) != NULL && *GLRO(dl_profile_output) == '\0')
+ {
+ _dl_error_printf ("\
+warning: LD_PROFILE ignored because LD_PROFILE_OUTPUT not specified\n");
+ GLRO(dl_profile) = NULL;
+ }
}
#if HP_TIMING_INLINE
diff --git a/sysdeps/generic/unsecvars.h b/sysdeps/generic/unsecvars.h
index 5ea8a4a259ef753c..0b84642f71ae9351 100644
--- a/sysdeps/generic/unsecvars.h
+++ b/sysdeps/generic/unsecvars.h
@@ -21,6 +21,7 @@
"LD_ORIGIN_PATH\0" \
"LD_PRELOAD\0" \
"LD_PROFILE\0" \
+ "LD_PROFILE_OUTPUT\0" \
"LD_SHOW_AUXV\0" \
"LD_USE_LOAD_BIAS\0" \
"LOCALDOMAIN\0" \

View File

@ -1,174 +0,0 @@
commit 80cc58ea2de214f85b0a1d902a3b668ad2ecb302
Author: Adhemerval Zanella <adhemerval.zanella@linaro.org>
Date: Thu Jan 15 10:32:19 2026 -0300
posix: Reset wordexp_t fields with WRDE_REUSE (CVE-2025-15281 / BZ 33814)
The wordexp fails to properly initialize the input wordexp_t when
WRDE_REUSE is used. The wordexp_t struct is properly freed, but
reuses the old wc_wordc value and updates the we_wordv in the
wrong position. A later wordfree will then call free with an
invalid pointer.
Checked on x86_64-linux-gnu and i686-linux-gnu.
Reviewed-by: Carlos O'Donell <carlos@redhat.com>
Conflicts:
posix/Makefile
(Makefile not sorted downstream)
diff --git a/posix/Makefile b/posix/Makefile
index 42a0290370b40fd9..e546b8d667b9c6c4 100644
--- a/posix/Makefile
+++ b/posix/Makefile
@@ -96,7 +96,8 @@ tests := test-errno tstgetopt testfnm runtests runptests \
tst-posix_fadvise tst-posix_fadvise64 \
tst-sysconf-empty-chroot tst-glob_symlinks tst-fexecve \
tst-glob-tilde test-ssize-max tst-spawn4 bug-regex37 \
- bug-regex38 tst-regcomp-truncated tst-regcomp-bracket-free
+ bug-regex38 tst-regcomp-truncated tst-regcomp-bracket-free \
+ tst-wordexp-reuse
tests-internal := bug-regex5 bug-regex20 bug-regex33 \
tst-rfc3484 tst-rfc3484-2 tst-rfc3484-3 \
tst-glob_lstat_compat tst-spawn4-compat
@@ -128,7 +129,8 @@ generated += $(addprefix wordexp-test-result, 1 2 3 4 5 6 7 8 9 10) \
tst-boost.mtrace bug-ga2.mtrace bug-ga2-mem.out \
bug-glob2.mtrace bug-glob2-mem.out tst-vfork3-mem.out \
tst-vfork3.mtrace getconf.speclist tst-fnmatch-mem.out \
- tst-fnmatch.mtrace bug-regex36.mtrace
+ tst-fnmatch.mtrace bug-regex36.mtrace \
+ tst-wordexp-reuse-mem.out tst-wordexp-reuse.mtrace
ifeq ($(run-built-tests),yes)
ifeq (yes,$(build-shared))
@@ -146,7 +148,8 @@ tests-special += $(objpfx)bug-regex2-mem.out $(objpfx)bug-regex14-mem.out \
$(objpfx)tst-boost-mem.out $(objpfx)tst-getconf.out \
$(objpfx)bug-glob2-mem.out $(objpfx)tst-vfork3-mem.out \
$(objpfx)tst-fnmatch-mem.out $(objpfx)bug-regex36-mem.out \
- $(objpfx)tst-glob-tilde-mem.out
+ $(objpfx)tst-glob-tilde-mem.out \
+ $(objpfx)tst-wordexp-reuse.out
xtests-special += $(objpfx)bug-ga2-mem.out
endif
@@ -387,3 +390,10 @@ $(objpfx)posix-conf-vars-def.h: $(..)scripts/gen-posix-conf-vars.awk \
$(make-target-directory)
$(AWK) -f $(filter-out Makefile, $^) > $@.tmp
mv -f $@.tmp $@
+
+tst-wordexp-reuse-ENV += MALLOC_TRACE=$(objpfx)tst-wordexp-reuse.mtrace \
+ LD_PRELOAD=$(common-objpfx)/malloc/libc_malloc_debug.so
+
+$(objpfx)tst-wordexp-reuse-mem.out: $(objpfx)tst-wordexp-reuse.out
+ $(common-objpfx)malloc/mtrace $(objpfx)tst-wordexp-reuse.mtrace > $@; \
+ $(evaluate-test)
diff --git a/posix/tst-wordexp-reuse.c b/posix/tst-wordexp-reuse.c
new file mode 100644
index 0000000000000000..3926b9f5576750ac
--- /dev/null
+++ b/posix/tst-wordexp-reuse.c
@@ -0,0 +1,89 @@
+/* Test for wordexp with WRDE_REUSE flag.
+ Copyright (C) 2026 Free Software Foundation, Inc.
+ This file is part of the GNU C Library.
+
+ The GNU C Library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ The GNU C Library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with the GNU C Library; if not, see
+ <https://www.gnu.org/licenses/>. */
+
+#include <wordexp.h>
+#include <mcheck.h>
+
+#include <support/check.h>
+
+static int
+do_test (void)
+{
+ mtrace ();
+
+ {
+ wordexp_t p = { 0 };
+ TEST_COMPARE (wordexp ("one", &p, 0), 0);
+ TEST_COMPARE (p.we_wordc, 1);
+ TEST_COMPARE_STRING (p.we_wordv[0], "one");
+ TEST_COMPARE (wordexp ("two", &p, WRDE_REUSE), 0);
+ TEST_COMPARE (p.we_wordc, 1);
+ TEST_COMPARE_STRING (p.we_wordv[0], "two");
+ wordfree (&p);
+ }
+
+ {
+ wordexp_t p = { .we_offs = 2 };
+ TEST_COMPARE (wordexp ("one", &p, 0), 0);
+ TEST_COMPARE (p.we_wordc, 1);
+ TEST_COMPARE_STRING (p.we_wordv[0], "one");
+ TEST_COMPARE (wordexp ("two", &p, WRDE_REUSE | WRDE_DOOFFS), 0);
+ TEST_COMPARE (p.we_wordc, 1);
+ TEST_COMPARE_STRING (p.we_wordv[p.we_offs + 0], "two");
+ wordfree (&p);
+ }
+
+ {
+ wordexp_t p = { 0 };
+ TEST_COMPARE (wordexp ("one", &p, 0), 0);
+ TEST_COMPARE (p.we_wordc, 1);
+ TEST_COMPARE_STRING (p.we_wordv[0], "one");
+ TEST_COMPARE (wordexp ("two", &p, WRDE_REUSE | WRDE_APPEND), 0);
+ TEST_COMPARE (p.we_wordc, 1);
+ TEST_COMPARE_STRING (p.we_wordv[0], "two");
+ wordfree (&p);
+ }
+
+ {
+ wordexp_t p = { .we_offs = 2 };
+ TEST_COMPARE (wordexp ("one", &p, WRDE_DOOFFS), 0);
+ TEST_COMPARE (p.we_wordc, 1);
+ TEST_COMPARE_STRING (p.we_wordv[p.we_offs + 0], "one");
+ TEST_COMPARE (wordexp ("two", &p, WRDE_REUSE
+ | WRDE_DOOFFS), 0);
+ TEST_COMPARE (p.we_wordc, 1);
+ TEST_COMPARE_STRING (p.we_wordv[p.we_offs + 0], "two");
+ wordfree (&p);
+ }
+
+ {
+ wordexp_t p = { .we_offs = 2 };
+ TEST_COMPARE (wordexp ("one", &p, WRDE_DOOFFS), 0);
+ TEST_COMPARE (p.we_wordc, 1);
+ TEST_COMPARE_STRING (p.we_wordv[p.we_offs + 0], "one");
+ TEST_COMPARE (wordexp ("two", &p, WRDE_REUSE
+ | WRDE_DOOFFS | WRDE_APPEND), 0);
+ TEST_COMPARE (p.we_wordc, 1);
+ TEST_COMPARE_STRING (p.we_wordv[p.we_offs + 0], "two");
+ wordfree (&p);
+ }
+
+ return 0;
+}
+
+#include <support/test-driver.c>
diff --git a/posix/wordexp.c b/posix/wordexp.c
index 4061969c720f1f34..0f503b1877d2ce5b 100644
--- a/posix/wordexp.c
+++ b/posix/wordexp.c
@@ -2241,7 +2241,9 @@ wordexp (const char *words, wordexp_t *pwordexp, int flags)
{
/* Minimal implementation of WRDE_REUSE for now */
wordfree (pwordexp);
+ old_word.we_wordc = 0;
old_word.we_wordv = NULL;
+ pwordexp->we_wordc = 0;
}
if ((flags & WRDE_APPEND) == 0)

View File

@ -1,29 +0,0 @@
commit bed2db02f3183e93f21d506786c5f884a1dec9e7
Author: Florian Weimer <fweimer@redhat.com>
Date: Mon Jan 26 17:12:37 2026 +0100
posix: Run tst-wordexp-reuse-mem test
The test was not properly scheduled for execution with a Makefile
dependency.
Fixes commit 80cc58ea2de214f85b0a1d902a3b668ad2ecb302 ("posix: Reset
wordexp_t fields with WRDE_REUSE (CVE-2025-15281 / BZ 33814").
Conflicts:
posix/Makefile
(Makefile not sorted downstream)
diff --git a/posix/Makefile b/posix/Makefile
index e546b8d667b9c6c4..b399b1dab0a8cb9c 100644
--- a/posix/Makefile
+++ b/posix/Makefile
@@ -149,7 +149,7 @@ tests-special += $(objpfx)bug-regex2-mem.out $(objpfx)bug-regex14-mem.out \
$(objpfx)bug-glob2-mem.out $(objpfx)tst-vfork3-mem.out \
$(objpfx)tst-fnmatch-mem.out $(objpfx)bug-regex36-mem.out \
$(objpfx)tst-glob-tilde-mem.out \
- $(objpfx)tst-wordexp-reuse.out
+ $(objpfx)tst-wordexp-reuse-mem.out
xtests-special += $(objpfx)bug-ga2-mem.out
endif

View File

@ -1,259 +0,0 @@
From 97700a34f36721b11a754cf37a1cc40695ece1fd Mon Sep 17 00:00:00 2001
From: "H.J. Lu" <hjl.tools@gmail.com>
Date: Mon, 21 Jan 2019 11:23:59 -0800
Subject: [PATCH] x86-64 memchr/wmemchr: Properly handle the length parameter
[BZ# 24097]
Content-type: text/plain; charset=UTF-8
On x32, the size_t parameter may be passed in the lower 32 bits of a
64-bit register with the non-zero upper 32 bits. The string/memory
functions written in assembly can only use the lower 32 bits of a
64-bit register as length or must clear the upper 32 bits before using
the full 64-bit register for length.
This pach fixes memchr/wmemchr for x32. Tested on x86-64 and x32. On
x86-64, libc.so is the same with and withou the fix.
[BZ# 24097]
CVE-2019-6488
* sysdeps/x86_64/memchr.S: Use RDX_LP for length. Clear the
upper 32 bits of RDX register.
* sysdeps/x86_64/multiarch/memchr-avx2.S: Likewise.
* sysdeps/x86_64/x32/Makefile (tests): Add tst-size_t-memchr and
tst-size_t-wmemchr.
* sysdeps/x86_64/x32/test-size_t.h: New file.
* sysdeps/x86_64/x32/tst-size_t-memchr.c: Likewise.
* sysdeps/x86_64/x32/tst-size_t-wmemchr.c: Likewise.
---
sysdeps/x86_64/memchr.S | 10 ++--
sysdeps/x86_64/multiarch/memchr-avx2.S | 8 ++-
sysdeps/x86_64/x32/Makefile | 8 +++
sysdeps/x86_64/x32/test-size_t.h | 35 ++++++++++++
sysdeps/x86_64/x32/tst-size_t-memchr.c | 72 +++++++++++++++++++++++++
sysdeps/x86_64/x32/tst-size_t-wmemchr.c | 20 +++++++
6 files changed, 148 insertions(+), 5 deletions(-)
create mode 100644 sysdeps/x86_64/x32/test-size_t.h
create mode 100644 sysdeps/x86_64/x32/tst-size_t-memchr.c
create mode 100644 sysdeps/x86_64/x32/tst-size_t-wmemchr.c
Conflicts:
ChangeLog
(removed)
NEWS
(removed)
diff --git a/sysdeps/x86_64/memchr.S b/sysdeps/x86_64/memchr.S
index feef5d4f..cb320257 100644
--- a/sysdeps/x86_64/memchr.S
+++ b/sysdeps/x86_64/memchr.S
@@ -34,12 +34,16 @@ ENTRY(MEMCHR)
mov %edi, %ecx
#ifdef USE_AS_WMEMCHR
- test %rdx, %rdx
+ test %RDX_LP, %RDX_LP
jz L(return_null)
- shl $2, %rdx
+ shl $2, %RDX_LP
#else
+# ifdef __ILP32__
+ /* Clear the upper 32 bits. */
+ movl %edx, %edx
+# endif
punpcklbw %xmm1, %xmm1
- test %rdx, %rdx
+ test %RDX_LP, %RDX_LP
jz L(return_null)
punpcklbw %xmm1, %xmm1
#endif
diff --git a/sysdeps/x86_64/multiarch/memchr-avx2.S b/sysdeps/x86_64/multiarch/memchr-avx2.S
index 5f5e7725..c81da19b 100644
--- a/sysdeps/x86_64/multiarch/memchr-avx2.S
+++ b/sysdeps/x86_64/multiarch/memchr-avx2.S
@@ -40,16 +40,20 @@
ENTRY (MEMCHR)
# ifndef USE_AS_RAWMEMCHR
/* Check for zero length. */
- testq %rdx, %rdx
+ test %RDX_LP, %RDX_LP
jz L(null)
# endif
movl %edi, %ecx
/* Broadcast CHAR to YMM0. */
vmovd %esi, %xmm0
# ifdef USE_AS_WMEMCHR
- shl $2, %rdx
+ shl $2, %RDX_LP
vpbroadcastd %xmm0, %ymm0
# else
+# ifdef __ILP32__
+ /* Clear the upper 32 bits. */
+ movl %edx, %edx
+# endif
vpbroadcastb %xmm0, %ymm0
# endif
/* Check if we may cross page boundary with one vector load. */
diff --git a/sysdeps/x86_64/x32/Makefile b/sysdeps/x86_64/x32/Makefile
index f2ebc24f..7d528889 100644
--- a/sysdeps/x86_64/x32/Makefile
+++ b/sysdeps/x86_64/x32/Makefile
@@ -4,3 +4,11 @@ ifeq ($(subdir),math)
# 64-bit llround. Add -fno-builtin-lround to silence the compiler.
CFLAGS-s_llround.c += -fno-builtin-lround
endif
+
+ifeq ($(subdir),string)
+tests += tst-size_t-memchr
+endif
+
+ifeq ($(subdir),wcsmbs)
+tests += tst-size_t-wmemchr
+endif
diff --git a/sysdeps/x86_64/x32/test-size_t.h b/sysdeps/x86_64/x32/test-size_t.h
new file mode 100644
index 00000000..78a94086
--- /dev/null
+++ b/sysdeps/x86_64/x32/test-size_t.h
@@ -0,0 +1,35 @@
+/* Test string/memory functions with size_t in the lower 32 bits of
+ 64-bit register.
+ Copyright (C) 2019 Free Software Foundation, Inc.
+ This file is part of the GNU C Library.
+
+ The GNU C Library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ The GNU C Library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with the GNU C Library; if not, see
+ <http://www.gnu.org/licenses/>. */
+
+#define TEST_MAIN
+#include <string/test-string.h>
+
+/* On x32, parameter_t may be passed in a 64-bit register with the LEN
+ field in the lower 32 bits. When the LEN field of 64-bit register
+ is passed to string/memory function as the size_t parameter, only
+ the lower 32 bits can be used. */
+typedef struct
+{
+ union
+ {
+ size_t len;
+ void (*fn) (void);
+ };
+ void *p;
+} parameter_t;
diff --git a/sysdeps/x86_64/x32/tst-size_t-memchr.c b/sysdeps/x86_64/x32/tst-size_t-memchr.c
new file mode 100644
index 00000000..29a3daf1
--- /dev/null
+++ b/sysdeps/x86_64/x32/tst-size_t-memchr.c
@@ -0,0 +1,72 @@
+/* Test memchr with size_t in the lower 32 bits of 64-bit register.
+ Copyright (C) 2019 Free Software Foundation, Inc.
+ This file is part of the GNU C Library.
+
+ The GNU C Library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ The GNU C Library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with the GNU C Library; if not, see
+ <http://www.gnu.org/licenses/>. */
+
+#ifndef WIDE
+# define TEST_NAME "memchr"
+#else
+# define TEST_NAME "wmemchr"
+#endif /* WIDE */
+#include "test-size_t.h"
+
+#ifndef WIDE
+# define MEMCHR memchr
+# define CHAR char
+# define UCHAR unsigned char
+#else
+# include <wchar.h>
+# define MEMCHR wmemchr
+# define CHAR wchar_t
+# define UCHAR wchar_t
+#endif /* WIDE */
+
+IMPL (MEMCHR, 1)
+
+typedef CHAR * (*proto_t) (const CHAR*, int, size_t);
+
+static CHAR *
+__attribute__ ((noinline, noclone))
+do_memchr (parameter_t a, parameter_t b)
+{
+ return CALL (&b, a.p, (uintptr_t) b.p, a.len);
+}
+
+static int
+test_main (void)
+{
+ test_init ();
+
+ parameter_t src = { { page_size / sizeof (CHAR) }, buf2 };
+ parameter_t c = { { 0 }, (void *) (uintptr_t) 0x12 };
+
+ int ret = 0;
+ FOR_EACH_IMPL (impl, 0)
+ {
+ c.fn = impl->fn;
+ CHAR *res = do_memchr (src, c);
+ if (res)
+ {
+ error (0, 0, "Wrong result in function %s: %p != NULL",
+ impl->name, res);
+ ret = 1;
+ }
+ }
+
+ return ret ? EXIT_FAILURE : EXIT_SUCCESS;
+}
+
+#include <support/test-driver.c>
diff --git a/sysdeps/x86_64/x32/tst-size_t-wmemchr.c b/sysdeps/x86_64/x32/tst-size_t-wmemchr.c
new file mode 100644
index 00000000..877801d6
--- /dev/null
+++ b/sysdeps/x86_64/x32/tst-size_t-wmemchr.c
@@ -0,0 +1,20 @@
+/* Test wmemchr with size_t in the lower 32 bits of 64-bit register.
+ Copyright (C) 2019 Free Software Foundation, Inc.
+ This file is part of the GNU C Library.
+
+ The GNU C Library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ The GNU C Library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with the GNU C Library; if not, see
+ <http://www.gnu.org/licenses/>. */
+
+#define WIDE 1
+#include "tst-size_t-memchr.c"
--
GitLab

View File

@ -1,41 +0,0 @@
From ddf0992cf57a93200e0c782e2a94d0733a5a0b87 Mon Sep 17 00:00:00 2001
From: Noah Goldstein <goldstein.w.n@gmail.com>
Date: Sun, 9 Jan 2022 16:02:21 -0600
Subject: [PATCH] x86: Fix __wcsncmp_avx2 in strcmp-avx2.S [BZ# 28755]
Content-type: text/plain; charset=UTF-8
Fixes [BZ# 28755] for wcsncmp by redirecting length >= 2^56 to
__wcscmp_avx2. For x86_64 this covers the entire address range so any
length larger could not possibly be used to bound `s1` or `s2`.
test-strcmp, test-strncmp, test-wcscmp, and test-wcsncmp all pass.
Signed-off-by: Noah Goldstein <goldstein.w.n@gmail.com>
---
sysdeps/x86_64/multiarch/strcmp-avx2.S | 10 ++++++++++
1 file changed, 10 insertions(+)
diff --git a/sysdeps/x86_64/multiarch/strcmp-avx2.S b/sysdeps/x86_64/multiarch/strcmp-avx2.S
index 156c1949..8fb8eedc 100644
--- a/sysdeps/x86_64/multiarch/strcmp-avx2.S
+++ b/sysdeps/x86_64/multiarch/strcmp-avx2.S
@@ -83,6 +83,16 @@ ENTRY (STRCMP)
je L(char0)
jb L(zero)
# ifdef USE_AS_WCSCMP
+# ifndef __ILP32__
+ movq %rdx, %rcx
+ /* Check if length could overflow when multiplied by
+ sizeof(wchar_t). Checking top 8 bits will cover all potential
+ overflow cases as well as redirect cases where its impossible to
+ length to bound a valid memory region. In these cases just use
+ 'wcscmp'. */
+ shrq $56, %rcx
+ jnz __wcscmp_avx2
+# endif
/* Convert units: from wide to byte char. */
shl $2, %RDX_LP
# endif
--
GitLab

View File

@ -1,263 +0,0 @@
From 23102686ec67b856a2d4fd25ddaa1c0b8d175c4f Mon Sep 17 00:00:00 2001
From: Noah Goldstein <goldstein.w.n@gmail.com>
Date: Fri, 15 Apr 2022 12:28:01 -0500
Subject: [PATCH] x86: Cleanup page cross code in memcmp-avx2-movbe.S
Content-type: text/plain; charset=UTF-8
Old code was both inefficient and wasted code size. New code (-62
bytes) and comparable or better performance in the page cross case.
geometric_mean(N=20) of page cross cases New / Original: 0.960
size, align0, align1, ret, New Time/Old Time
1, 4095, 0, 0, 1.001
1, 4095, 0, 1, 0.999
1, 4095, 0, -1, 1.0
2, 4094, 0, 0, 1.0
2, 4094, 0, 1, 1.0
2, 4094, 0, -1, 1.0
3, 4093, 0, 0, 1.0
3, 4093, 0, 1, 1.0
3, 4093, 0, -1, 1.0
4, 4092, 0, 0, 0.987
4, 4092, 0, 1, 1.0
4, 4092, 0, -1, 1.0
5, 4091, 0, 0, 0.984
5, 4091, 0, 1, 1.002
5, 4091, 0, -1, 1.005
6, 4090, 0, 0, 0.993
6, 4090, 0, 1, 1.001
6, 4090, 0, -1, 1.003
7, 4089, 0, 0, 0.991
7, 4089, 0, 1, 1.0
7, 4089, 0, -1, 1.001
8, 4088, 0, 0, 0.875
8, 4088, 0, 1, 0.881
8, 4088, 0, -1, 0.888
9, 4087, 0, 0, 0.872
9, 4087, 0, 1, 0.879
9, 4087, 0, -1, 0.883
10, 4086, 0, 0, 0.878
10, 4086, 0, 1, 0.886
10, 4086, 0, -1, 0.873
11, 4085, 0, 0, 0.878
11, 4085, 0, 1, 0.881
11, 4085, 0, -1, 0.879
12, 4084, 0, 0, 0.873
12, 4084, 0, 1, 0.889
12, 4084, 0, -1, 0.875
13, 4083, 0, 0, 0.873
13, 4083, 0, 1, 0.863
13, 4083, 0, -1, 0.863
14, 4082, 0, 0, 0.838
14, 4082, 0, 1, 0.869
14, 4082, 0, -1, 0.877
15, 4081, 0, 0, 0.841
15, 4081, 0, 1, 0.869
15, 4081, 0, -1, 0.876
16, 4080, 0, 0, 0.988
16, 4080, 0, 1, 0.99
16, 4080, 0, -1, 0.989
17, 4079, 0, 0, 0.978
17, 4079, 0, 1, 0.981
17, 4079, 0, -1, 0.98
18, 4078, 0, 0, 0.981
18, 4078, 0, 1, 0.98
18, 4078, 0, -1, 0.985
19, 4077, 0, 0, 0.977
19, 4077, 0, 1, 0.979
19, 4077, 0, -1, 0.986
20, 4076, 0, 0, 0.977
20, 4076, 0, 1, 0.986
20, 4076, 0, -1, 0.984
21, 4075, 0, 0, 0.977
21, 4075, 0, 1, 0.983
21, 4075, 0, -1, 0.988
22, 4074, 0, 0, 0.983
22, 4074, 0, 1, 0.994
22, 4074, 0, -1, 0.993
23, 4073, 0, 0, 0.98
23, 4073, 0, 1, 0.992
23, 4073, 0, -1, 0.995
24, 4072, 0, 0, 0.989
24, 4072, 0, 1, 0.989
24, 4072, 0, -1, 0.991
25, 4071, 0, 0, 0.99
25, 4071, 0, 1, 0.999
25, 4071, 0, -1, 0.996
26, 4070, 0, 0, 0.993
26, 4070, 0, 1, 0.995
26, 4070, 0, -1, 0.998
27, 4069, 0, 0, 0.993
27, 4069, 0, 1, 0.999
27, 4069, 0, -1, 1.0
28, 4068, 0, 0, 0.997
28, 4068, 0, 1, 1.0
28, 4068, 0, -1, 0.999
29, 4067, 0, 0, 0.996
29, 4067, 0, 1, 0.999
29, 4067, 0, -1, 0.999
30, 4066, 0, 0, 0.991
30, 4066, 0, 1, 1.001
30, 4066, 0, -1, 0.999
31, 4065, 0, 0, 0.988
31, 4065, 0, 1, 0.998
31, 4065, 0, -1, 0.998
Reviewed-by: H.J. Lu <hjl.tools@gmail.com>
---
sysdeps/x86_64/multiarch/memcmp-avx2-movbe.S | 98 ++++++++++++--------
1 file changed, 61 insertions(+), 37 deletions(-)
diff --git a/sysdeps/x86_64/multiarch/memcmp-avx2-movbe.S b/sysdeps/x86_64/multiarch/memcmp-avx2-movbe.S
index 16fc673e..99258cf5 100644
--- a/sysdeps/x86_64/multiarch/memcmp-avx2-movbe.S
+++ b/sysdeps/x86_64/multiarch/memcmp-avx2-movbe.S
@@ -429,22 +429,21 @@ L(page_cross_less_vec):
# ifndef USE_AS_WMEMCMP
cmpl $8, %edx
jae L(between_8_15)
+ /* Fall through for [4, 7]. */
cmpl $4, %edx
- jae L(between_4_7)
+ jb L(between_2_3)
- /* Load as big endian to avoid branches. */
- movzwl (%rdi), %eax
- movzwl (%rsi), %ecx
- shll $8, %eax
- shll $8, %ecx
- bswap %eax
- bswap %ecx
- movzbl -1(%rdi, %rdx), %edi
- movzbl -1(%rsi, %rdx), %esi
- orl %edi, %eax
- orl %esi, %ecx
- /* Subtraction is okay because the upper 8 bits are zero. */
- subl %ecx, %eax
+ movbe (%rdi), %eax
+ movbe (%rsi), %ecx
+ shlq $32, %rax
+ shlq $32, %rcx
+ movbe -4(%rdi, %rdx), %edi
+ movbe -4(%rsi, %rdx), %esi
+ orq %rdi, %rax
+ orq %rsi, %rcx
+ subq %rcx, %rax
+ /* Fast path for return zero. */
+ jnz L(ret_nonzero)
/* No ymm register was touched. */
ret
@@ -457,9 +456,33 @@ L(one_or_less):
/* No ymm register was touched. */
ret
+ .p2align 4,, 5
+L(ret_nonzero):
+ sbbl %eax, %eax
+ orl $1, %eax
+ /* No ymm register was touched. */
+ ret
+
+ .p2align 4,, 2
+L(zero):
+ xorl %eax, %eax
+ /* No ymm register was touched. */
+ ret
+
.p2align 4
L(between_8_15):
-# endif
+ movbe (%rdi), %rax
+ movbe (%rsi), %rcx
+ subq %rcx, %rax
+ jnz L(ret_nonzero)
+ movbe -8(%rdi, %rdx), %rax
+ movbe -8(%rsi, %rdx), %rcx
+ subq %rcx, %rax
+ /* Fast path for return zero. */
+ jnz L(ret_nonzero)
+ /* No ymm register was touched. */
+ ret
+# else
/* If USE_AS_WMEMCMP fall through into 8-15 byte case. */
vmovq (%rdi), %xmm1
vmovq (%rsi), %xmm2
@@ -475,16 +498,13 @@ L(between_8_15):
VPCMPEQ %xmm1, %xmm2, %xmm2
vpmovmskb %xmm2, %eax
subl $0xffff, %eax
+ /* Fast path for return zero. */
jnz L(return_vec_0)
/* No ymm register was touched. */
ret
+# endif
- .p2align 4
-L(zero):
- xorl %eax, %eax
- ret
-
- .p2align 4
+ .p2align 4,, 10
L(between_16_31):
/* From 16 to 31 bytes. No branch when size == 16. */
vmovdqu (%rsi), %xmm2
@@ -501,11 +521,17 @@ L(between_16_31):
VPCMPEQ (%rdi), %xmm2, %xmm2
vpmovmskb %xmm2, %eax
subl $0xffff, %eax
+ /* Fast path for return zero. */
jnz L(return_vec_0)
/* No ymm register was touched. */
ret
# ifdef USE_AS_WMEMCMP
+ .p2align 4,, 2
+L(zero):
+ xorl %eax, %eax
+ ret
+
.p2align 4
L(one_or_less):
jb L(zero)
@@ -520,22 +546,20 @@ L(one_or_less):
# else
.p2align 4
-L(between_4_7):
- /* Load as big endian with overlapping movbe to avoid branches.
- */
- movbe (%rdi), %eax
- movbe (%rsi), %ecx
- shlq $32, %rax
- shlq $32, %rcx
- movbe -4(%rdi, %rdx), %edi
- movbe -4(%rsi, %rdx), %esi
- orq %rdi, %rax
- orq %rsi, %rcx
- subq %rcx, %rax
- jz L(zero_4_7)
- sbbl %eax, %eax
- orl $1, %eax
-L(zero_4_7):
+L(between_2_3):
+ /* Load as big endian to avoid branches. */
+ movzwl (%rdi), %eax
+ movzwl (%rsi), %ecx
+ bswap %eax
+ bswap %ecx
+ shrl %eax
+ shrl %ecx
+ movzbl -1(%rdi, %rdx), %edi
+ movzbl -1(%rsi, %rdx), %esi
+ orl %edi, %eax
+ orl %esi, %ecx
+ /* Subtraction is okay because the upper bit is zero. */
+ subl %ecx, %eax
/* No ymm register was touched. */
ret
# endif
--
GitLab

View File

@ -1,226 +0,0 @@
From 8162147872491bb5b48e91543b19c49a29ae6b6d Mon Sep 17 00:00:00 2001
From: Wangyang Guo <wangyang.guo@intel.com>
Date: Fri, 6 May 2022 01:50:10 +0000
Subject: [PATCH] nptl: Add backoff mechanism to spinlock loop
Content-type: text/plain; charset=UTF-8
When mutiple threads waiting for lock at the same time, once lock owner
releases the lock, waiters will see lock available and all try to lock,
which may cause an expensive CAS storm.
Binary exponential backoff with random jitter is introduced. As try-lock
attempt increases, there is more likely that a larger number threads
compete for adaptive mutex lock, so increase wait time in exponential.
A random jitter is also added to avoid synchronous try-lock from other
threads.
v2: Remove read-check before try-lock for performance.
v3:
1. Restore read-check since it works well in some platform.
2. Make backoff arch dependent, and enable it for x86_64.
3. Limit max backoff to reduce latency in large critical section.
v4: Fix strict-prototypes error in sysdeps/nptl/pthread_mutex_backoff.h
v5: Commit log updated for regression in large critical section.
Result of pthread-mutex-locks bench
Test Platform: Xeon 8280L (2 socket, 112 CPUs in total)
First Row: thread number
First Col: critical section length
Values: backoff vs upstream, time based, low is better
non-critical-length: 1
1 2 4 8 16 32 64 112 140
0 0.99 0.58 0.52 0.49 0.43 0.44 0.46 0.52 0.54
1 0.98 0.43 0.56 0.50 0.44 0.45 0.50 0.56 0.57
2 0.99 0.41 0.57 0.51 0.45 0.47 0.48 0.60 0.61
4 0.99 0.45 0.59 0.53 0.48 0.49 0.52 0.64 0.65
8 1.00 0.66 0.71 0.63 0.56 0.59 0.66 0.72 0.71
16 0.97 0.78 0.91 0.73 0.67 0.70 0.79 0.80 0.80
32 0.95 1.17 0.98 0.87 0.82 0.86 0.89 0.90 0.90
64 0.96 0.95 1.01 1.01 0.98 1.00 1.03 0.99 0.99
128 0.99 1.01 1.01 1.17 1.08 1.12 1.02 0.97 1.02
non-critical-length: 32
1 2 4 8 16 32 64 112 140
0 1.03 0.97 0.75 0.65 0.58 0.58 0.56 0.70 0.70
1 0.94 0.95 0.76 0.65 0.58 0.58 0.61 0.71 0.72
2 0.97 0.96 0.77 0.66 0.58 0.59 0.62 0.74 0.74
4 0.99 0.96 0.78 0.66 0.60 0.61 0.66 0.76 0.77
8 0.99 0.99 0.84 0.70 0.64 0.66 0.71 0.80 0.80
16 0.98 0.97 0.95 0.76 0.70 0.73 0.81 0.85 0.84
32 1.04 1.12 1.04 0.89 0.82 0.86 0.93 0.91 0.91
64 0.99 1.15 1.07 1.00 0.99 1.01 1.05 0.99 0.99
128 1.00 1.21 1.20 1.22 1.25 1.31 1.12 1.10 0.99
non-critical-length: 128
1 2 4 8 16 32 64 112 140
0 1.02 1.00 0.99 0.67 0.61 0.61 0.61 0.74 0.73
1 0.95 0.99 1.00 0.68 0.61 0.60 0.60 0.74 0.74
2 1.00 1.04 1.00 0.68 0.59 0.61 0.65 0.76 0.76
4 1.00 0.96 0.98 0.70 0.63 0.63 0.67 0.78 0.77
8 1.01 1.02 0.89 0.73 0.65 0.67 0.71 0.81 0.80
16 0.99 0.96 0.96 0.79 0.71 0.73 0.80 0.84 0.84
32 0.99 0.95 1.05 0.89 0.84 0.85 0.94 0.92 0.91
64 1.00 0.99 1.16 1.04 1.00 1.02 1.06 0.99 0.99
128 1.00 1.06 0.98 1.14 1.39 1.26 1.08 1.02 0.98
There is regression in large critical section. But adaptive mutex is
aimed for "quick" locks. Small critical section is more common when
users choose to use adaptive pthread_mutex.
Signed-off-by: Wangyang Guo <wangyang.guo@intel.com>
Reviewed-by: H.J. Lu <hjl.tools@gmail.com>
Conflicts:
pthreadP.h
(had been moved)
nptl/pthread_mutex_lock.c
(max_adaptive_count renamed)
---
nptl/pthreadP.h | 1 +
nptl/pthread_mutex_lock.c | 16 +++++++--
sysdeps/nptl/pthread_mutex_backoff.h | 35 ++++++++++++++++++
sysdeps/x86_64/nptl/pthread_mutex_backoff.h | 39 +++++++++++++++++++++
4 files changed, 89 insertions(+), 2 deletions(-)
create mode 100644 sysdeps/nptl/pthread_mutex_backoff.h
create mode 100644 sysdeps/x86_64/nptl/pthread_mutex_backoff.h
diff --git a/nptl/pthreadP.h b/nptl/pthreadP.h
index 7ddc166c..1550e3b6 100644
--- a/nptl/pthreadP.h
+++ b/nptl/pthreadP.h
@@ -33,6 +33,7 @@
#include <kernel-features.h>
#include <errno.h>
#include <internal-signals.h>
+#include <pthread_mutex_backoff.h>
/* Atomic operations on TLS memory. */
diff --git a/nptl/pthread_mutex_lock.c b/nptl/pthread_mutex_lock.c
index d96a9933..c7770fc9 100644
--- a/nptl/pthread_mutex_lock.c
+++ b/nptl/pthread_mutex_lock.c
@@ -133,14 +133,26 @@ __pthread_mutex_lock (pthread_mutex_t *mutex)
int cnt = 0;
int max_cnt = MIN (MAX_ADAPTIVE_COUNT,
mutex->__data.__spins * 2 + 10);
+ int spin_count, exp_backoff = 1;
+ unsigned int jitter = get_jitter ();
do
{
- if (cnt++ >= max_cnt)
+ /* In each loop, spin count is exponential backoff plus
+ random jitter, random range is [0, exp_backoff-1]. */
+ spin_count = exp_backoff + (jitter & (exp_backoff - 1));
+ cnt += spin_count;
+ if (cnt >= max_cnt)
{
+ /* If cnt exceeds max spin count, just go to wait
+ queue. */
LLL_MUTEX_LOCK (mutex);
break;
}
- atomic_spin_nop ();
+ do
+ atomic_spin_nop ();
+ while (--spin_count > 0);
+ /* Prepare for next loop. */
+ exp_backoff = get_next_backoff (exp_backoff);
}
while (LLL_MUTEX_READ_LOCK (mutex) != 0
|| LLL_MUTEX_TRYLOCK (mutex) != 0);
diff --git a/sysdeps/nptl/pthread_mutex_backoff.h b/sysdeps/nptl/pthread_mutex_backoff.h
new file mode 100644
index 00000000..5b26c22a
--- /dev/null
+++ b/sysdeps/nptl/pthread_mutex_backoff.h
@@ -0,0 +1,35 @@
+/* Pthread mutex backoff configuration.
+ Copyright (C) 2022 Free Software Foundation, Inc.
+ This file is part of the GNU C Library.
+
+ The GNU C Library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ The GNU C Library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with the GNU C Library; if not, see
+ <https://www.gnu.org/licenses/>. */
+#ifndef _PTHREAD_MUTEX_BACKOFF_H
+#define _PTHREAD_MUTEX_BACKOFF_H 1
+
+static inline unsigned int
+get_jitter (void)
+{
+ /* Arch dependent random jitter, return 0 disables random. */
+ return 0;
+}
+
+static inline int
+get_next_backoff (int backoff)
+{
+ /* Next backoff, return 1 disables mutex backoff. */
+ return 1;
+}
+
+#endif
diff --git a/sysdeps/x86_64/nptl/pthread_mutex_backoff.h b/sysdeps/x86_64/nptl/pthread_mutex_backoff.h
new file mode 100644
index 00000000..ec74c3d9
--- /dev/null
+++ b/sysdeps/x86_64/nptl/pthread_mutex_backoff.h
@@ -0,0 +1,39 @@
+/* Pthread mutex backoff configuration.
+ Copyright (C) 2022 Free Software Foundation, Inc.
+ This file is part of the GNU C Library.
+
+ The GNU C Library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ The GNU C Library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with the GNU C Library; if not, see
+ <https://www.gnu.org/licenses/>. */
+#ifndef _PTHREAD_MUTEX_BACKOFF_H
+#define _PTHREAD_MUTEX_BACKOFF_H 1
+
+#include <fast-jitter.h>
+
+static inline unsigned int
+get_jitter (void)
+{
+ return get_fast_jitter ();
+}
+
+#define MAX_BACKOFF 16
+
+static inline int
+get_next_backoff (int backoff)
+{
+ /* Binary expontial backoff. Limiting max backoff
+ can reduce latency in large critical section. */
+ return (backoff < MAX_BACKOFF) ? backoff << 1 : backoff;
+}
+
+#endif
--
GitLab

View File

@ -1,55 +0,0 @@
From c6272098323153db373f2986c67786ea8c85f1cf Mon Sep 17 00:00:00 2001
From: Noah Goldstein <goldstein.w.n@gmail.com>
Date: Tue, 15 Feb 2022 08:18:15 -0600
Subject: [PATCH] x86: Fallback {str|wcs}cmp RTM in the ncmp overflow case [BZ
#28896]
Content-type: text/plain; charset=UTF-8
In the overflow fallback strncmp-avx2-rtm and wcsncmp-avx2-rtm would
call strcmp-avx2 and wcscmp-avx2 respectively. This would have
not checks around vzeroupper and would trigger spurious
aborts. This commit fixes that.
test-strcmp, test-strncmp, test-wcscmp, and test-wcsncmp all pass on
AVX2 machines with and without RTM.
Co-authored-by: H.J. Lu <hjl.tools@gmail.com>
---
sysdeps/x86_64/multiarch/strcmp-avx2.S | 8 ++------
1 file changed, 2 insertions(+), 6 deletions(-)
Conflicts:
sysdeps/x86_64/multiarch/strcmp-avx2.S
(split into two patches due to upstream bug differences)
diff --git a/sysdeps/x86_64/multiarch/strcmp-avx2.S b/sysdeps/x86_64/multiarch/strcmp-avx2.S
index 28cc98b6..e267c6cb 100644
--- a/sysdeps/x86_64/multiarch/strcmp-avx2.S
+++ b/sysdeps/x86_64/multiarch/strcmp-avx2.S
@@ -345,10 +345,10 @@ L(one_or_less):
movq %LOCALE_REG, %rdx
# endif
jb L(ret_zero)
-# ifdef USE_AS_WCSCMP
/* 'nbe' covers the case where length is negative (large
unsigned). */
- jnbe __wcscmp_avx2
+ jnbe OVERFLOW_STRCMP
+# ifdef USE_AS_WCSCMP
movl (%rdi), %edx
xorl %eax, %eax
cmpl (%rsi), %edx
@@ -357,10 +357,6 @@ L(one_or_less):
negl %eax
orl $1, %eax
# else
- /* 'nbe' covers the case where length is negative (large
- unsigned). */
-
- jnbe __strcmp_avx2
movzbl (%rdi), %eax
movzbl (%rsi), %ecx
TOLOWER_gpr (%rax, %eax)
--
GitLab

View File

@ -1,60 +0,0 @@
From 259a17cc98058d2576511201f85d28cb5d9de2a2 Mon Sep 17 00:00:00 2001
From: Stefan Liebler <stli@linux.ibm.com>
Date: Mon, 28 Jun 2021 13:01:07 +0200
Subject: s390x: Update math: redirect roundeven function
After recent commit
447954a206837b5f153869cfeeeab44631c3fac9
"math: redirect roundeven function", building on
s390x fails with:
Error: symbol `__roundevenl' is already defined
Similar to aarch64/riscv fix, this patch redirects target
specific functions for s390x:
commit 3213ed770cbc5821920d16caa93c85e92dd7b9f6
"Update math: redirect roundeven function"
diff --git a/sysdeps/s390/fpu/s_roundeven.c b/sysdeps/s390/fpu/s_roundeven.c
index 40b07e054b..0773adfed0 100644
--- a/sysdeps/s390/fpu/s_roundeven.c
+++ b/sysdeps/s390/fpu/s_roundeven.c
@@ -18,6 +18,7 @@
<https://www.gnu.org/licenses/>. */
#ifdef HAVE_S390_MIN_Z196_ZARCH_ASM_SUPPORT
+# define NO_MATH_REDIRECT
# include <math.h>
# include <libm-alias-double.h>
@@ -31,7 +32,6 @@ __roundeven (double x)
__asm__ ("fidbra %0,4,%1,4" : "=f" (y) : "f" (x));
return y;
}
-hidden_def (__roundeven)
libm_alias_double (__roundeven, roundeven)
#else
diff --git a/sysdeps/s390/fpu/s_roundevenf.c b/sysdeps/s390/fpu/s_roundevenf.c
index d2fbf3d2b6..289785bc4a 100644
--- a/sysdeps/s390/fpu/s_roundevenf.c
+++ b/sysdeps/s390/fpu/s_roundevenf.c
@@ -18,6 +18,7 @@
<https://www.gnu.org/licenses/>. */
#ifdef HAVE_S390_MIN_Z196_ZARCH_ASM_SUPPORT
+# define NO_MATH_REDIRECT
# include <math.h>
# include <libm-alias-float.h>
diff --git a/sysdeps/s390/fpu/s_roundevenl.c b/sysdeps/s390/fpu/s_roundevenl.c
index 29ab7a8616..94b6459ab4 100644
--- a/sysdeps/s390/fpu/s_roundevenl.c
+++ b/sysdeps/s390/fpu/s_roundevenl.c
@@ -18,6 +18,7 @@
<https://www.gnu.org/licenses/>. */
#ifdef HAVE_S390_MIN_Z196_ZARCH_ASM_SUPPORT
+# define NO_MATH_REDIRECT
# include <math.h>
# include <math_private.h>
# include <libm-alias-ldouble.h>

View File

@ -1,74 +0,0 @@
From 1da50d4bda07f04135dca39f40e79fc9eabed1f8 Mon Sep 17 00:00:00 2001
From: "H.J. Lu" <hjl.tools@gmail.com>
Date: Fri, 26 Feb 2021 05:36:59 -0800
Subject: [PATCH] x86: Set Prefer_No_VZEROUPPER and add Prefer_AVX2_STRCMP
Content-type: text/plain; charset=UTF-8
1. Set Prefer_No_VZEROUPPER if RTM is usable to avoid RTM abort triggered
by VZEROUPPER inside a transactionally executing RTM region.
2. Since to compare 2 32-byte strings, 256-bit EVEX strcmp requires 2
loads, 3 VPCMPs and 2 KORDs while AVX2 strcmp requires 1 load, 2 VPCMPEQs,
1 VPMINU and 1 VPMOVMSKB, AVX2 strcmp is faster than EVEX strcmp. Add
Prefer_AVX2_STRCMP to prefer AVX2 strcmp family functions.
---
sysdeps/x86/cpu-features.c | 20 +++++++++++++++++--
sysdeps/x86/cpu-tunables.c | 2 ++
...cpu-features-preferred_feature_index_1.def | 1 +
3 files changed, 21 insertions(+), 2 deletions(-)
diff --git a/sysdeps/x86/cpu-features.c b/sysdeps/x86/cpu-features.c
index 91042505..3610ee5c 100644
--- a/sysdeps/x86/cpu-features.c
+++ b/sysdeps/x86/cpu-features.c
@@ -524,8 +524,24 @@ init_cpu_features (struct cpu_features *cpu_features)
cpu_features->preferred[index_arch_Prefer_No_VZEROUPPER]
|= bit_arch_Prefer_No_VZEROUPPER;
else
- cpu_features->preferred[index_arch_Prefer_No_AVX512]
- |= bit_arch_Prefer_No_AVX512;
+ {
+ cpu_features->preferred[index_arch_Prefer_No_AVX512]
+ |= bit_arch_Prefer_No_AVX512;
+
+ /* Avoid RTM abort triggered by VZEROUPPER inside a
+ transactionally executing RTM region. */
+ if (CPU_FEATURE_USABLE_P (cpu_features, RTM))
+ cpu_features->preferred[index_arch_Prefer_No_VZEROUPPER]
+ |= bit_arch_Prefer_No_VZEROUPPER;
+
+ /* Since to compare 2 32-byte strings, 256-bit EVEX strcmp
+ requires 2 loads, 3 VPCMPs and 2 KORDs while AVX2 strcmp
+ requires 1 load, 2 VPCMPEQs, 1 VPMINU and 1 VPMOVMSKB,
+ AVX2 strcmp is faster than EVEX strcmp. */
+ if (CPU_FEATURE_USABLE_P (cpu_features, AVX2))
+ cpu_features->preferred[index_arch_Prefer_AVX2_STRCMP]
+ |= bit_arch_Prefer_AVX2_STRCMP;
+ }
}
/* This spells out "AuthenticAMD". */
else if (ebx == 0x68747541 && ecx == 0x444d4163 && edx == 0x69746e65)
diff --git a/sysdeps/x86/cpu-tunables.c b/sysdeps/x86/cpu-tunables.c
index 3173b2b9..73adbaba 100644
--- a/sysdeps/x86/cpu-tunables.c
+++ b/sysdeps/x86/cpu-tunables.c
@@ -239,6 +239,8 @@ TUNABLE_CALLBACK (set_hwcaps) (tunable_val_t *valp)
CHECK_GLIBC_IFUNC_PREFERRED_BOTH (n, cpu_features,
Fast_Copy_Backward,
disable, 18);
+ CHECK_GLIBC_IFUNC_PREFERRED_NEED_BOTH
+ (n, cpu_features, Prefer_AVX2_STRCMP, AVX2, disable, 18);
}
break;
case 19:
diff --git a/sysdeps/x86/include/cpu-features-preferred_feature_index_1.def b/sysdeps/x86/include/cpu-features-preferred_feature_index_1.def
index 17a5cc42..4ca70b40 100644
--- a/sysdeps/x86/include/cpu-features-preferred_feature_index_1.def
+++ b/sysdeps/x86/include/cpu-features-preferred_feature_index_1.def
@@ -32,3 +32,4 @@ BIT (Prefer_ERMS)
BIT (Prefer_FSRM)
BIT (Prefer_No_AVX512)
BIT (MathVec_Prefer_No_AVX512)
+BIT (Prefer_AVX2_STRCMP)
--
GitLab

View File

@ -1,26 +0,0 @@
From 3213ed770cbc5821920d16caa93c85e92dd7b9f6 Mon Sep 17 00:00:00 2001
From: "H.J. Lu" <hjl.tools@gmail.com>
Date: Wed, 23 Jun 2021 13:29:41 -0700
Subject: Update math: redirect roundeven function
Redirect target specific roundeven functions for aarch64, ldbl-128ibm
and riscv.
Conflicts:
sysdeps/aarch64/*
(not needed)
sysdeps/riscv/*
(not supported)
diff --git a/sysdeps/ieee754/ldbl-128ibm/s_roundevenl.c b/sysdeps/ieee754/ldbl-128ibm/s_roundevenl.c
index 6701970f4a..90eecf496b 100644
--- a/sysdeps/ieee754/ldbl-128ibm/s_roundevenl.c
+++ b/sysdeps/ieee754/ldbl-128ibm/s_roundevenl.c
@@ -17,6 +17,7 @@
License along with the GNU C Library; if not, see
<http://www.gnu.org/licenses/>. */
+#define NO_MATH_REDIRECT
#include <math.h>
#include <math_private.h>

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

View File

@ -1,242 +0,0 @@
From 63ad43566f7a25d140dc723598aeb441ad657eed Mon Sep 17 00:00:00 2001
From: "H.J. Lu" <hjl.tools@gmail.com>
Date: Fri, 5 Mar 2021 06:46:08 -0800
Subject: [PATCH] x86-64: Add memmove family functions with 256-bit EVEX
Content-type: text/plain; charset=UTF-8
Update ifunc-memmove.h to select the function optimized with 256-bit EVEX
instructions using YMM16-YMM31 registers to avoid RTM abort with usable
AVX512VL since VZEROUPPER isn't needed at function exit.
---
sysdeps/x86_64/multiarch/Makefile | 1 +
sysdeps/x86_64/multiarch/ifunc-impl-list.c | 36 +++++++++++++++++++
sysdeps/x86_64/multiarch/ifunc-memmove.h | 21 +++++++++--
.../multiarch/memmove-evex-unaligned-erms.S | 33 +++++++++++++++++
.../multiarch/memmove-vec-unaligned-erms.S | 24 ++++++++-----
5 files changed, 104 insertions(+), 11 deletions(-)
create mode 100644 sysdeps/x86_64/multiarch/memmove-evex-unaligned-erms.S
diff --git a/sysdeps/x86_64/multiarch/Makefile b/sysdeps/x86_64/multiarch/Makefile
index 46783cd1..4563fc56 100644
--- a/sysdeps/x86_64/multiarch/Makefile
+++ b/sysdeps/x86_64/multiarch/Makefile
@@ -41,6 +41,7 @@ sysdep_routines += strncat-c stpncpy-c strncpy-c \
memset-avx2-unaligned-erms \
memset-avx512-unaligned-erms \
memchr-evex \
+ memmove-evex-unaligned-erms \
memrchr-evex \
rawmemchr-evex \
stpcpy-evex \
diff --git a/sysdeps/x86_64/multiarch/ifunc-impl-list.c b/sysdeps/x86_64/multiarch/ifunc-impl-list.c
index 082e4da3..6bd3abfc 100644
--- a/sysdeps/x86_64/multiarch/ifunc-impl-list.c
+++ b/sysdeps/x86_64/multiarch/ifunc-impl-list.c
@@ -80,6 +80,12 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,
IFUNC_IMPL_ADD (array, i, __memmove_chk,
CPU_FEATURE_USABLE (AVX),
__memmove_chk_avx_unaligned_erms)
+ IFUNC_IMPL_ADD (array, i, __memmove_chk,
+ CPU_FEATURE_USABLE (AVX512VL),
+ __memmove_chk_evex_unaligned)
+ IFUNC_IMPL_ADD (array, i, __memmove_chk,
+ CPU_FEATURE_USABLE (AVX512VL),
+ __memmove_chk_evex_unaligned_erms)
IFUNC_IMPL_ADD (array, i, __memmove_chk,
CPU_FEATURE_USABLE (SSSE3),
__memmove_chk_ssse3_back)
@@ -102,6 +108,12 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,
IFUNC_IMPL_ADD (array, i, memmove,
CPU_FEATURE_USABLE (AVX),
__memmove_avx_unaligned_erms)
+ IFUNC_IMPL_ADD (array, i, memmove,
+ CPU_FEATURE_USABLE (AVX512VL),
+ __memmove_evex_unaligned)
+ IFUNC_IMPL_ADD (array, i, memmove,
+ CPU_FEATURE_USABLE (AVX512VL),
+ __memmove_evex_unaligned_erms)
IFUNC_IMPL_ADD (array, i, memmove,
CPU_FEATURE_USABLE (AVX512F),
__memmove_avx512_no_vzeroupper)
@@ -565,6 +577,12 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,
IFUNC_IMPL_ADD (array, i, __memcpy_chk,
CPU_FEATURE_USABLE (AVX),
__memcpy_chk_avx_unaligned_erms)
+ IFUNC_IMPL_ADD (array, i, __memcpy_chk,
+ CPU_FEATURE_USABLE (AVX512VL),
+ __memcpy_chk_evex_unaligned)
+ IFUNC_IMPL_ADD (array, i, __memcpy_chk,
+ CPU_FEATURE_USABLE (AVX512VL),
+ __memcpy_chk_evex_unaligned_erms)
IFUNC_IMPL_ADD (array, i, __memcpy_chk,
CPU_FEATURE_USABLE (SSSE3),
__memcpy_chk_ssse3_back)
@@ -587,6 +605,12 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,
IFUNC_IMPL_ADD (array, i, memcpy,
CPU_FEATURE_USABLE (AVX),
__memcpy_avx_unaligned_erms)
+ IFUNC_IMPL_ADD (array, i, memcpy,
+ CPU_FEATURE_USABLE (AVX512VL),
+ __memcpy_evex_unaligned)
+ IFUNC_IMPL_ADD (array, i, memcpy,
+ CPU_FEATURE_USABLE (AVX512VL),
+ __memcpy_evex_unaligned_erms)
IFUNC_IMPL_ADD (array, i, memcpy, CPU_FEATURE_USABLE (SSSE3),
__memcpy_ssse3_back)
IFUNC_IMPL_ADD (array, i, memcpy, CPU_FEATURE_USABLE (SSSE3),
@@ -623,6 +647,12 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,
IFUNC_IMPL_ADD (array, i, __mempcpy_chk,
CPU_FEATURE_USABLE (AVX),
__mempcpy_chk_avx_unaligned_erms)
+ IFUNC_IMPL_ADD (array, i, __mempcpy_chk,
+ CPU_FEATURE_USABLE (AVX512VL),
+ __mempcpy_chk_evex_unaligned)
+ IFUNC_IMPL_ADD (array, i, __mempcpy_chk,
+ CPU_FEATURE_USABLE (AVX512VL),
+ __mempcpy_chk_evex_unaligned_erms)
IFUNC_IMPL_ADD (array, i, __mempcpy_chk,
CPU_FEATURE_USABLE (SSSE3),
__mempcpy_chk_ssse3_back)
@@ -654,6 +684,12 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,
IFUNC_IMPL_ADD (array, i, mempcpy,
CPU_FEATURE_USABLE (AVX),
__mempcpy_avx_unaligned_erms)
+ IFUNC_IMPL_ADD (array, i, mempcpy,
+ CPU_FEATURE_USABLE (AVX512VL),
+ __mempcpy_evex_unaligned)
+ IFUNC_IMPL_ADD (array, i, mempcpy,
+ CPU_FEATURE_USABLE (AVX512VL),
+ __mempcpy_evex_unaligned_erms)
IFUNC_IMPL_ADD (array, i, mempcpy, CPU_FEATURE_USABLE (SSSE3),
__mempcpy_ssse3_back)
IFUNC_IMPL_ADD (array, i, mempcpy, CPU_FEATURE_USABLE (SSSE3),
diff --git a/sysdeps/x86_64/multiarch/ifunc-memmove.h b/sysdeps/x86_64/multiarch/ifunc-memmove.h
index 5e5f0299..6f8bce5f 100644
--- a/sysdeps/x86_64/multiarch/ifunc-memmove.h
+++ b/sysdeps/x86_64/multiarch/ifunc-memmove.h
@@ -29,6 +29,10 @@ extern __typeof (REDIRECT_NAME) OPTIMIZE (ssse3_back) attribute_hidden;
extern __typeof (REDIRECT_NAME) OPTIMIZE (avx_unaligned) attribute_hidden;
extern __typeof (REDIRECT_NAME) OPTIMIZE (avx_unaligned_erms)
attribute_hidden;
+extern __typeof (REDIRECT_NAME) OPTIMIZE (evex_unaligned)
+ attribute_hidden;
+extern __typeof (REDIRECT_NAME) OPTIMIZE (evex_unaligned_erms)
+ attribute_hidden;
extern __typeof (REDIRECT_NAME) OPTIMIZE (avx512_unaligned)
attribute_hidden;
extern __typeof (REDIRECT_NAME) OPTIMIZE (avx512_unaligned_erms)
@@ -59,10 +63,21 @@ IFUNC_SELECTOR (void)
if (CPU_FEATURES_ARCH_P (cpu_features, AVX_Fast_Unaligned_Load))
{
- if (CPU_FEATURE_USABLE_P (cpu_features, ERMS))
- return OPTIMIZE (avx_unaligned_erms);
+ if (CPU_FEATURE_USABLE_P (cpu_features, AVX512VL))
+ {
+ if (CPU_FEATURE_USABLE_P (cpu_features, ERMS))
+ return OPTIMIZE (evex_unaligned_erms);
+
+ return OPTIMIZE (evex_unaligned);
+ }
+
+ if (!CPU_FEATURES_ARCH_P (cpu_features, Prefer_No_VZEROUPPER))
+ {
+ if (CPU_FEATURE_USABLE_P (cpu_features, ERMS))
+ return OPTIMIZE (avx_unaligned_erms);
- return OPTIMIZE (avx_unaligned);
+ return OPTIMIZE (avx_unaligned);
+ }
}
if (!CPU_FEATURE_USABLE_P (cpu_features, SSSE3)
diff --git a/sysdeps/x86_64/multiarch/memmove-evex-unaligned-erms.S b/sysdeps/x86_64/multiarch/memmove-evex-unaligned-erms.S
new file mode 100644
index 00000000..0cbce8f9
--- /dev/null
+++ b/sysdeps/x86_64/multiarch/memmove-evex-unaligned-erms.S
@@ -0,0 +1,33 @@
+#if IS_IN (libc)
+# define VEC_SIZE 32
+# define XMM0 xmm16
+# define XMM1 xmm17
+# define YMM0 ymm16
+# define YMM1 ymm17
+# define VEC0 ymm16
+# define VEC1 ymm17
+# define VEC2 ymm18
+# define VEC3 ymm19
+# define VEC4 ymm20
+# define VEC5 ymm21
+# define VEC6 ymm22
+# define VEC7 ymm23
+# define VEC8 ymm24
+# define VEC9 ymm25
+# define VEC10 ymm26
+# define VEC11 ymm27
+# define VEC12 ymm28
+# define VEC13 ymm29
+# define VEC14 ymm30
+# define VEC15 ymm31
+# define VEC(i) VEC##i
+# define VMOVNT vmovntdq
+# define VMOVU vmovdqu64
+# define VMOVA vmovdqa64
+# define VZEROUPPER
+
+# define SECTION(p) p##.evex
+# define MEMMOVE_SYMBOL(p,s) p##_evex_##s
+
+# include "memmove-vec-unaligned-erms.S"
+#endif
diff --git a/sysdeps/x86_64/multiarch/memmove-vec-unaligned-erms.S b/sysdeps/x86_64/multiarch/memmove-vec-unaligned-erms.S
index 274aa1c7..08e21692 100644
--- a/sysdeps/x86_64/multiarch/memmove-vec-unaligned-erms.S
+++ b/sysdeps/x86_64/multiarch/memmove-vec-unaligned-erms.S
@@ -48,6 +48,14 @@
# define MEMMOVE_CHK_SYMBOL(p,s) MEMMOVE_SYMBOL(p, s)
#endif
+#ifndef XMM0
+# define XMM0 xmm0
+#endif
+
+#ifndef YMM0
+# define YMM0 ymm0
+#endif
+
#ifndef VZEROUPPER
# if VEC_SIZE > 16
# define VZEROUPPER vzeroupper
@@ -277,20 +285,20 @@ L(less_vec):
#if VEC_SIZE > 32
L(between_32_63):
/* From 32 to 63. No branch when size == 32. */
- vmovdqu (%rsi), %ymm0
- vmovdqu -32(%rsi,%rdx), %ymm1
- vmovdqu %ymm0, (%rdi)
- vmovdqu %ymm1, -32(%rdi,%rdx)
+ VMOVU (%rsi), %YMM0
+ VMOVU -32(%rsi,%rdx), %YMM1
+ VMOVU %YMM0, (%rdi)
+ VMOVU %YMM1, -32(%rdi,%rdx)
VZEROUPPER
ret
#endif
#if VEC_SIZE > 16
/* From 16 to 31. No branch when size == 16. */
L(between_16_31):
- vmovdqu (%rsi), %xmm0
- vmovdqu -16(%rsi,%rdx), %xmm1
- vmovdqu %xmm0, (%rdi)
- vmovdqu %xmm1, -16(%rdi,%rdx)
+ VMOVU (%rsi), %XMM0
+ VMOVU -16(%rsi,%rdx), %XMM1
+ VMOVU %XMM0, (%rdi)
+ VMOVU %XMM1, -16(%rdi,%rdx)
ret
#endif
L(between_8_15):
--
GitLab

View File

@ -1,254 +0,0 @@
From 1b968b6b9b3aac702ac2f133e0dd16cfdbb415ee Mon Sep 17 00:00:00 2001
From: "H.J. Lu" <hjl.tools@gmail.com>
Date: Fri, 5 Mar 2021 07:15:03 -0800
Subject: [PATCH] x86-64: Add memset family functions with 256-bit EVEX
Content-type: text/plain; charset=UTF-8
Update ifunc-memset.h/ifunc-wmemset.h to select the function optimized
with 256-bit EVEX instructions using YMM16-YMM31 registers to avoid RTM
abort with usable AVX512VL and AVX512BW since VZEROUPPER isn't needed at
function exit.
---
sysdeps/x86_64/multiarch/Makefile | 1 +
sysdeps/x86_64/multiarch/ifunc-impl-list.c | 22 +++++++++++++++++
sysdeps/x86_64/multiarch/ifunc-memset.h | 24 +++++++++++++++----
sysdeps/x86_64/multiarch/ifunc-wmemset.h | 13 ++++++----
.../multiarch/memset-evex-unaligned-erms.S | 24 +++++++++++++++++++
.../multiarch/memset-vec-unaligned-erms.S | 20 +++++++++++-----
6 files changed, 90 insertions(+), 14 deletions(-)
create mode 100644 sysdeps/x86_64/multiarch/memset-evex-unaligned-erms.S
diff --git a/sysdeps/x86_64/multiarch/Makefile b/sysdeps/x86_64/multiarch/Makefile
index 4563fc56..1cc0a10e 100644
--- a/sysdeps/x86_64/multiarch/Makefile
+++ b/sysdeps/x86_64/multiarch/Makefile
@@ -43,6 +43,7 @@ sysdep_routines += strncat-c stpncpy-c strncpy-c \
memchr-evex \
memmove-evex-unaligned-erms \
memrchr-evex \
+ memset-evex-unaligned-erms \
rawmemchr-evex \
stpcpy-evex \
stpncpy-evex \
diff --git a/sysdeps/x86_64/multiarch/ifunc-impl-list.c b/sysdeps/x86_64/multiarch/ifunc-impl-list.c
index 6bd3abfc..7cf83485 100644
--- a/sysdeps/x86_64/multiarch/ifunc-impl-list.c
+++ b/sysdeps/x86_64/multiarch/ifunc-impl-list.c
@@ -160,6 +160,14 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,
IFUNC_IMPL_ADD (array, i, __memset_chk,
CPU_FEATURE_USABLE (AVX2),
__memset_chk_avx2_unaligned_erms)
+ IFUNC_IMPL_ADD (array, i, __memset_chk,
+ (CPU_FEATURE_USABLE (AVX512VL)
+ && CPU_FEATURE_USABLE (AVX512BW)),
+ __memset_chk_evex_unaligned)
+ IFUNC_IMPL_ADD (array, i, __memset_chk,
+ (CPU_FEATURE_USABLE (AVX512VL)
+ && CPU_FEATURE_USABLE (AVX512BW)),
+ __memset_chk_evex_unaligned_erms)
IFUNC_IMPL_ADD (array, i, __memset_chk,
CPU_FEATURE_USABLE (AVX512F),
__memset_chk_avx512_unaligned_erms)
@@ -185,6 +193,14 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,
IFUNC_IMPL_ADD (array, i, memset,
CPU_FEATURE_USABLE (AVX2),
__memset_avx2_unaligned_erms)
+ IFUNC_IMPL_ADD (array, i, memset,
+ (CPU_FEATURE_USABLE (AVX512VL)
+ && CPU_FEATURE_USABLE (AVX512BW)),
+ __memset_evex_unaligned)
+ IFUNC_IMPL_ADD (array, i, memset,
+ (CPU_FEATURE_USABLE (AVX512VL)
+ && CPU_FEATURE_USABLE (AVX512BW)),
+ __memset_evex_unaligned_erms)
IFUNC_IMPL_ADD (array, i, memset,
CPU_FEATURE_USABLE (AVX512F),
__memset_avx512_unaligned_erms)
@@ -555,6 +571,9 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,
IFUNC_IMPL_ADD (array, i, wmemset,
CPU_FEATURE_USABLE (AVX2),
__wmemset_avx2_unaligned)
+ IFUNC_IMPL_ADD (array, i, wmemset,
+ CPU_FEATURE_USABLE (AVX512VL),
+ __wmemset_evex_unaligned)
IFUNC_IMPL_ADD (array, i, wmemset,
CPU_FEATURE_USABLE (AVX512F),
__wmemset_avx512_unaligned))
@@ -723,6 +742,9 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,
IFUNC_IMPL_ADD (array, i, __wmemset_chk,
CPU_FEATURE_USABLE (AVX2),
__wmemset_chk_avx2_unaligned)
+ IFUNC_IMPL_ADD (array, i, __wmemset_chk,
+ CPU_FEATURE_USABLE (AVX512VL),
+ __wmemset_chk_evex_unaligned)
IFUNC_IMPL_ADD (array, i, __wmemset_chk,
CPU_FEATURE_USABLE (AVX512F),
__wmemset_chk_avx512_unaligned))
diff --git a/sysdeps/x86_64/multiarch/ifunc-memset.h b/sysdeps/x86_64/multiarch/ifunc-memset.h
index 708bd72e..6f31f4dc 100644
--- a/sysdeps/x86_64/multiarch/ifunc-memset.h
+++ b/sysdeps/x86_64/multiarch/ifunc-memset.h
@@ -27,6 +27,10 @@ extern __typeof (REDIRECT_NAME) OPTIMIZE (sse2_unaligned_erms)
extern __typeof (REDIRECT_NAME) OPTIMIZE (avx2_unaligned) attribute_hidden;
extern __typeof (REDIRECT_NAME) OPTIMIZE (avx2_unaligned_erms)
attribute_hidden;
+extern __typeof (REDIRECT_NAME) OPTIMIZE (evex_unaligned)
+ attribute_hidden;
+extern __typeof (REDIRECT_NAME) OPTIMIZE (evex_unaligned_erms)
+ attribute_hidden;
extern __typeof (REDIRECT_NAME) OPTIMIZE (avx512_unaligned)
attribute_hidden;
extern __typeof (REDIRECT_NAME) OPTIMIZE (avx512_unaligned_erms)
@@ -56,10 +60,22 @@ IFUNC_SELECTOR (void)
if (CPU_FEATURE_USABLE_P (cpu_features, AVX2))
{
- if (CPU_FEATURE_USABLE_P (cpu_features, ERMS))
- return OPTIMIZE (avx2_unaligned_erms);
- else
- return OPTIMIZE (avx2_unaligned);
+ if (CPU_FEATURE_USABLE_P (cpu_features, AVX512VL)
+ && CPU_FEATURE_USABLE_P (cpu_features, AVX512BW))
+ {
+ if (CPU_FEATURE_USABLE_P (cpu_features, ERMS))
+ return OPTIMIZE (evex_unaligned_erms);
+
+ return OPTIMIZE (evex_unaligned);
+ }
+
+ if (!CPU_FEATURES_ARCH_P (cpu_features, Prefer_No_VZEROUPPER))
+ {
+ if (CPU_FEATURE_USABLE_P (cpu_features, ERMS))
+ return OPTIMIZE (avx2_unaligned_erms);
+
+ return OPTIMIZE (avx2_unaligned);
+ }
}
if (CPU_FEATURE_USABLE_P (cpu_features, ERMS))
diff --git a/sysdeps/x86_64/multiarch/ifunc-wmemset.h b/sysdeps/x86_64/multiarch/ifunc-wmemset.h
index eb242210..9290c4bf 100644
--- a/sysdeps/x86_64/multiarch/ifunc-wmemset.h
+++ b/sysdeps/x86_64/multiarch/ifunc-wmemset.h
@@ -20,6 +20,7 @@
extern __typeof (REDIRECT_NAME) OPTIMIZE (sse2_unaligned) attribute_hidden;
extern __typeof (REDIRECT_NAME) OPTIMIZE (avx2_unaligned) attribute_hidden;
+extern __typeof (REDIRECT_NAME) OPTIMIZE (evex_unaligned) attribute_hidden;
extern __typeof (REDIRECT_NAME) OPTIMIZE (avx512_unaligned) attribute_hidden;
static inline void *
@@ -27,14 +28,18 @@ IFUNC_SELECTOR (void)
{
const struct cpu_features* cpu_features = __get_cpu_features ();
- if (!CPU_FEATURES_ARCH_P (cpu_features, Prefer_No_VZEROUPPER)
- && CPU_FEATURE_USABLE_P (cpu_features, AVX2)
+ if (CPU_FEATURE_USABLE_P (cpu_features, AVX2)
&& CPU_FEATURES_ARCH_P (cpu_features, AVX_Fast_Unaligned_Load))
{
if (CPU_FEATURE_USABLE_P (cpu_features, AVX512F)
- && !CPU_FEATURES_ARCH_P (cpu_features, Prefer_No_AVX512))
+ && !CPU_FEATURES_ARCH_P (cpu_features, Prefer_No_AVX512)
+ && !CPU_FEATURES_ARCH_P (cpu_features, Prefer_No_VZEROUPPER))
return OPTIMIZE (avx512_unaligned);
- else
+
+ if (CPU_FEATURE_USABLE_P (cpu_features, AVX512VL))
+ return OPTIMIZE (evex_unaligned);
+
+ if (!CPU_FEATURES_ARCH_P (cpu_features, Prefer_No_VZEROUPPER))
return OPTIMIZE (avx2_unaligned);
}
diff --git a/sysdeps/x86_64/multiarch/memset-evex-unaligned-erms.S b/sysdeps/x86_64/multiarch/memset-evex-unaligned-erms.S
new file mode 100644
index 00000000..ae0a4d6e
--- /dev/null
+++ b/sysdeps/x86_64/multiarch/memset-evex-unaligned-erms.S
@@ -0,0 +1,24 @@
+#if IS_IN (libc)
+# define VEC_SIZE 32
+# define XMM0 xmm16
+# define YMM0 ymm16
+# define VEC0 ymm16
+# define VEC(i) VEC##i
+# define VMOVU vmovdqu64
+# define VMOVA vmovdqa64
+# define VZEROUPPER
+
+# define MEMSET_VDUP_TO_VEC0_AND_SET_RETURN(d, r) \
+ movq r, %rax; \
+ vpbroadcastb d, %VEC0
+
+# define WMEMSET_VDUP_TO_VEC0_AND_SET_RETURN(d, r) \
+ movq r, %rax; \
+ vpbroadcastd d, %VEC0
+
+# define SECTION(p) p##.evex
+# define MEMSET_SYMBOL(p,s) p##_evex_##s
+# define WMEMSET_SYMBOL(p,s) p##_evex_##s
+
+# include "memset-vec-unaligned-erms.S"
+#endif
diff --git a/sysdeps/x86_64/multiarch/memset-vec-unaligned-erms.S b/sysdeps/x86_64/multiarch/memset-vec-unaligned-erms.S
index 9a0fd818..71e91a8f 100644
--- a/sysdeps/x86_64/multiarch/memset-vec-unaligned-erms.S
+++ b/sysdeps/x86_64/multiarch/memset-vec-unaligned-erms.S
@@ -34,6 +34,14 @@
# define WMEMSET_CHK_SYMBOL(p,s) WMEMSET_SYMBOL(p, s)
#endif
+#ifndef XMM0
+# define XMM0 xmm0
+#endif
+
+#ifndef YMM0
+# define YMM0 ymm0
+#endif
+
#ifndef VZEROUPPER
# if VEC_SIZE > 16
# define VZEROUPPER vzeroupper
@@ -67,7 +75,7 @@
ENTRY (__bzero)
mov %RDI_LP, %RAX_LP /* Set return value. */
mov %RSI_LP, %RDX_LP /* Set n. */
- pxor %xmm0, %xmm0
+ pxor %XMM0, %XMM0
jmp L(entry_from_bzero)
END (__bzero)
weak_alias (__bzero, bzero)
@@ -223,7 +231,7 @@ L(less_vec):
cmpb $16, %dl
jae L(between_16_31)
# endif
- MOVQ %xmm0, %rcx
+ MOVQ %XMM0, %rcx
cmpb $8, %dl
jae L(between_8_15)
cmpb $4, %dl
@@ -238,16 +246,16 @@ L(less_vec):
# if VEC_SIZE > 32
/* From 32 to 63. No branch when size == 32. */
L(between_32_63):
- vmovdqu %ymm0, -32(%rdi,%rdx)
- vmovdqu %ymm0, (%rdi)
+ VMOVU %YMM0, -32(%rdi,%rdx)
+ VMOVU %YMM0, (%rdi)
VZEROUPPER
ret
# endif
# if VEC_SIZE > 16
/* From 16 to 31. No branch when size == 16. */
L(between_16_31):
- vmovdqu %xmm0, -16(%rdi,%rdx)
- vmovdqu %xmm0, (%rdi)
+ VMOVU %XMM0, -16(%rdi,%rdx)
+ VMOVU %XMM0, (%rdi)
VZEROUPPER
ret
# endif
--
GitLab

View File

@ -1,561 +0,0 @@
From 91264fe3577fe887b4860923fa6142b5274c8965 Mon Sep 17 00:00:00 2001
From: "H.J. Lu" <hjl.tools@gmail.com>
Date: Fri, 5 Mar 2021 07:20:28 -0800
Subject: [PATCH] x86-64: Add memcmp family functions with 256-bit EVEX
Content-type: text/plain; charset=UTF-8
Update ifunc-memcmp.h to select the function optimized with 256-bit EVEX
instructions using YMM16-YMM31 registers to avoid RTM abort with usable
AVX512VL, AVX512BW and MOVBE since VZEROUPPER isn't needed at function
exit.
---
sysdeps/x86_64/multiarch/Makefile | 4 +-
sysdeps/x86_64/multiarch/ifunc-impl-list.c | 10 +
sysdeps/x86_64/multiarch/ifunc-memcmp.h | 13 +-
sysdeps/x86_64/multiarch/memcmp-evex-movbe.S | 440 ++++++++++++++++++
sysdeps/x86_64/multiarch/wmemcmp-evex-movbe.S | 4 +
5 files changed, 467 insertions(+), 4 deletions(-)
create mode 100644 sysdeps/x86_64/multiarch/memcmp-evex-movbe.S
create mode 100644 sysdeps/x86_64/multiarch/wmemcmp-evex-movbe.S
diff --git a/sysdeps/x86_64/multiarch/Makefile b/sysdeps/x86_64/multiarch/Makefile
index 1cc0a10e..9d79b138 100644
--- a/sysdeps/x86_64/multiarch/Makefile
+++ b/sysdeps/x86_64/multiarch/Makefile
@@ -41,6 +41,7 @@ sysdep_routines += strncat-c stpncpy-c strncpy-c \
memset-avx2-unaligned-erms \
memset-avx512-unaligned-erms \
memchr-evex \
+ memcmp-evex-movbe \
memmove-evex-unaligned-erms \
memrchr-evex \
memset-evex-unaligned-erms \
@@ -81,7 +82,8 @@ sysdep_routines += wmemcmp-sse4 wmemcmp-ssse3 wmemcmp-c \
wcsncmp-evex \
wcsnlen-evex \
wcsrchr-evex \
- wmemchr-evex
+ wmemchr-evex \
+ wmemcmp-evex-movbe
endif
ifeq ($(subdir),debug)
diff --git a/sysdeps/x86_64/multiarch/ifunc-impl-list.c b/sysdeps/x86_64/multiarch/ifunc-impl-list.c
index 7cf83485..c8da910e 100644
--- a/sysdeps/x86_64/multiarch/ifunc-impl-list.c
+++ b/sysdeps/x86_64/multiarch/ifunc-impl-list.c
@@ -56,6 +56,11 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,
(CPU_FEATURE_USABLE (AVX2)
&& CPU_FEATURE_USABLE (MOVBE)),
__memcmp_avx2_movbe)
+ IFUNC_IMPL_ADD (array, i, memcmp,
+ (CPU_FEATURE_USABLE (AVX512VL)
+ && CPU_FEATURE_USABLE (AVX512BW)
+ && CPU_FEATURE_USABLE (MOVBE)),
+ __memcmp_evex_movbe)
IFUNC_IMPL_ADD (array, i, memcmp, CPU_FEATURE_USABLE (SSE4_1),
__memcmp_sse4_1)
IFUNC_IMPL_ADD (array, i, memcmp, CPU_FEATURE_USABLE (SSSE3),
@@ -558,6 +563,11 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,
(CPU_FEATURE_USABLE (AVX2)
&& CPU_FEATURE_USABLE (MOVBE)),
__wmemcmp_avx2_movbe)
+ IFUNC_IMPL_ADD (array, i, wmemcmp,
+ (CPU_FEATURE_USABLE (AVX512VL)
+ && CPU_FEATURE_USABLE (AVX512BW)
+ && CPU_FEATURE_USABLE (MOVBE)),
+ __wmemcmp_evex_movbe)
IFUNC_IMPL_ADD (array, i, wmemcmp, CPU_FEATURE_USABLE (SSE4_1),
__wmemcmp_sse4_1)
IFUNC_IMPL_ADD (array, i, wmemcmp, CPU_FEATURE_USABLE (SSSE3),
diff --git a/sysdeps/x86_64/multiarch/ifunc-memcmp.h b/sysdeps/x86_64/multiarch/ifunc-memcmp.h
index 6c1f3153..3ca1f0a6 100644
--- a/sysdeps/x86_64/multiarch/ifunc-memcmp.h
+++ b/sysdeps/x86_64/multiarch/ifunc-memcmp.h
@@ -23,17 +23,24 @@ extern __typeof (REDIRECT_NAME) OPTIMIZE (sse2) attribute_hidden;
extern __typeof (REDIRECT_NAME) OPTIMIZE (ssse3) attribute_hidden;
extern __typeof (REDIRECT_NAME) OPTIMIZE (sse4_1) attribute_hidden;
extern __typeof (REDIRECT_NAME) OPTIMIZE (avx2_movbe) attribute_hidden;
+extern __typeof (REDIRECT_NAME) OPTIMIZE (evex_movbe) attribute_hidden;
static inline void *
IFUNC_SELECTOR (void)
{
const struct cpu_features* cpu_features = __get_cpu_features ();
- if (!CPU_FEATURES_ARCH_P (cpu_features, Prefer_No_VZEROUPPER)
- && CPU_FEATURE_USABLE_P (cpu_features, AVX2)
+ if (CPU_FEATURE_USABLE_P (cpu_features, AVX2)
&& CPU_FEATURE_USABLE_P (cpu_features, MOVBE)
&& CPU_FEATURES_ARCH_P (cpu_features, AVX_Fast_Unaligned_Load))
- return OPTIMIZE (avx2_movbe);
+ {
+ if (CPU_FEATURE_USABLE_P (cpu_features, AVX512VL)
+ && CPU_FEATURE_USABLE_P (cpu_features, AVX512BW))
+ return OPTIMIZE (evex_movbe);
+
+ if (!CPU_FEATURES_ARCH_P (cpu_features, Prefer_No_VZEROUPPER))
+ return OPTIMIZE (avx2_movbe);
+ }
if (CPU_FEATURE_USABLE_P (cpu_features, SSE4_1))
return OPTIMIZE (sse4_1);
diff --git a/sysdeps/x86_64/multiarch/memcmp-evex-movbe.S b/sysdeps/x86_64/multiarch/memcmp-evex-movbe.S
new file mode 100644
index 00000000..9c093972
--- /dev/null
+++ b/sysdeps/x86_64/multiarch/memcmp-evex-movbe.S
@@ -0,0 +1,440 @@
+/* memcmp/wmemcmp optimized with 256-bit EVEX instructions.
+ Copyright (C) 2021 Free Software Foundation, Inc.
+ This file is part of the GNU C Library.
+
+ The GNU C Library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ The GNU C Library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with the GNU C Library; if not, see
+ <https://www.gnu.org/licenses/>. */
+
+#if IS_IN (libc)
+
+/* memcmp/wmemcmp is implemented as:
+ 1. For size from 2 to 7 bytes, load as big endian with movbe and bswap
+ to avoid branches.
+ 2. Use overlapping compare to avoid branch.
+ 3. Use vector compare when size >= 4 bytes for memcmp or size >= 8
+ bytes for wmemcmp.
+ 4. If size is 8 * VEC_SIZE or less, unroll the loop.
+ 5. Compare 4 * VEC_SIZE at a time with the aligned first memory
+ area.
+ 6. Use 2 vector compares when size is 2 * VEC_SIZE or less.
+ 7. Use 4 vector compares when size is 4 * VEC_SIZE or less.
+ 8. Use 8 vector compares when size is 8 * VEC_SIZE or less. */
+
+# include <sysdep.h>
+
+# ifndef MEMCMP
+# define MEMCMP __memcmp_evex_movbe
+# endif
+
+# define VMOVU vmovdqu64
+
+# ifdef USE_AS_WMEMCMP
+# define VPCMPEQ vpcmpeqd
+# else
+# define VPCMPEQ vpcmpeqb
+# endif
+
+# define XMM1 xmm17
+# define XMM2 xmm18
+# define YMM1 ymm17
+# define YMM2 ymm18
+# define YMM3 ymm19
+# define YMM4 ymm20
+# define YMM5 ymm21
+# define YMM6 ymm22
+
+# define VEC_SIZE 32
+# ifdef USE_AS_WMEMCMP
+# define VEC_MASK 0xff
+# define XMM_MASK 0xf
+# else
+# define VEC_MASK 0xffffffff
+# define XMM_MASK 0xffff
+# endif
+
+/* Warning!
+ wmemcmp has to use SIGNED comparison for elements.
+ memcmp has to use UNSIGNED comparison for elemnts.
+*/
+
+ .section .text.evex,"ax",@progbits
+ENTRY (MEMCMP)
+# ifdef USE_AS_WMEMCMP
+ shl $2, %RDX_LP
+# elif defined __ILP32__
+ /* Clear the upper 32 bits. */
+ movl %edx, %edx
+# endif
+ cmp $VEC_SIZE, %RDX_LP
+ jb L(less_vec)
+
+ /* From VEC to 2 * VEC. No branch when size == VEC_SIZE. */
+ VMOVU (%rsi), %YMM2
+ VPCMPEQ (%rdi), %YMM2, %k1
+ kmovd %k1, %eax
+ subl $VEC_MASK, %eax
+ jnz L(first_vec)
+
+ cmpq $(VEC_SIZE * 2), %rdx
+ jbe L(last_vec)
+
+ /* More than 2 * VEC. */
+ cmpq $(VEC_SIZE * 8), %rdx
+ ja L(more_8x_vec)
+ cmpq $(VEC_SIZE * 4), %rdx
+ jb L(last_4x_vec)
+
+ /* From 4 * VEC to 8 * VEC, inclusively. */
+ VMOVU (%rsi), %YMM1
+ VPCMPEQ (%rdi), %YMM1, %k1
+
+ VMOVU VEC_SIZE(%rsi), %YMM2
+ VPCMPEQ VEC_SIZE(%rdi), %YMM2, %k2
+
+ VMOVU (VEC_SIZE * 2)(%rsi), %YMM3
+ VPCMPEQ (VEC_SIZE * 2)(%rdi), %YMM3, %k3
+
+ VMOVU (VEC_SIZE * 3)(%rsi), %YMM4
+ VPCMPEQ (VEC_SIZE * 3)(%rdi), %YMM4, %k4
+
+ kandd %k1, %k2, %k5
+ kandd %k3, %k4, %k6
+ kandd %k5, %k6, %k6
+
+ kmovd %k6, %eax
+ cmpl $VEC_MASK, %eax
+ jne L(4x_vec_end)
+
+ leaq -(4 * VEC_SIZE)(%rdi, %rdx), %rdi
+ leaq -(4 * VEC_SIZE)(%rsi, %rdx), %rsi
+ VMOVU (%rsi), %YMM1
+ VPCMPEQ (%rdi), %YMM1, %k1
+
+ VMOVU VEC_SIZE(%rsi), %YMM2
+ VPCMPEQ VEC_SIZE(%rdi), %YMM2, %k2
+ kandd %k1, %k2, %k5
+
+ VMOVU (VEC_SIZE * 2)(%rsi), %YMM3
+ VPCMPEQ (VEC_SIZE * 2)(%rdi), %YMM3, %k3
+ kandd %k3, %k5, %k5
+
+ VMOVU (VEC_SIZE * 3)(%rsi), %YMM4
+ VPCMPEQ (VEC_SIZE * 3)(%rdi), %YMM4, %k4
+ kandd %k4, %k5, %k5
+
+ kmovd %k5, %eax
+ cmpl $VEC_MASK, %eax
+ jne L(4x_vec_end)
+ xorl %eax, %eax
+ ret
+
+ .p2align 4
+L(last_2x_vec):
+ /* From VEC to 2 * VEC. No branch when size == VEC_SIZE. */
+ VMOVU (%rsi), %YMM2
+ VPCMPEQ (%rdi), %YMM2, %k2
+ kmovd %k2, %eax
+ subl $VEC_MASK, %eax
+ jnz L(first_vec)
+
+L(last_vec):
+ /* Use overlapping loads to avoid branches. */
+ leaq -VEC_SIZE(%rdi, %rdx), %rdi
+ leaq -VEC_SIZE(%rsi, %rdx), %rsi
+ VMOVU (%rsi), %YMM2
+ VPCMPEQ (%rdi), %YMM2, %k2
+ kmovd %k2, %eax
+ subl $VEC_MASK, %eax
+ jnz L(first_vec)
+ ret
+
+ .p2align 4
+L(first_vec):
+ /* A byte or int32 is different within 16 or 32 bytes. */
+ tzcntl %eax, %ecx
+# ifdef USE_AS_WMEMCMP
+ xorl %eax, %eax
+ movl (%rdi, %rcx, 4), %edx
+ cmpl (%rsi, %rcx, 4), %edx
+L(wmemcmp_return):
+ setl %al
+ negl %eax
+ orl $1, %eax
+# else
+ movzbl (%rdi, %rcx), %eax
+ movzbl (%rsi, %rcx), %edx
+ sub %edx, %eax
+# endif
+ ret
+
+# ifdef USE_AS_WMEMCMP
+ .p2align 4
+L(4):
+ xorl %eax, %eax
+ movl (%rdi), %edx
+ cmpl (%rsi), %edx
+ jne L(wmemcmp_return)
+ ret
+# else
+ .p2align 4
+L(between_4_7):
+ /* Load as big endian with overlapping movbe to avoid branches. */
+ movbe (%rdi), %eax
+ movbe (%rsi), %ecx
+ shlq $32, %rax
+ shlq $32, %rcx
+ movbe -4(%rdi, %rdx), %edi
+ movbe -4(%rsi, %rdx), %esi
+ orq %rdi, %rax
+ orq %rsi, %rcx
+ subq %rcx, %rax
+ je L(exit)
+ sbbl %eax, %eax
+ orl $1, %eax
+ ret
+
+ .p2align 4
+L(exit):
+ ret
+
+ .p2align 4
+L(between_2_3):
+ /* Load as big endian to avoid branches. */
+ movzwl (%rdi), %eax
+ movzwl (%rsi), %ecx
+ shll $8, %eax
+ shll $8, %ecx
+ bswap %eax
+ bswap %ecx
+ movb -1(%rdi, %rdx), %al
+ movb -1(%rsi, %rdx), %cl
+ /* Subtraction is okay because the upper 8 bits are zero. */
+ subl %ecx, %eax
+ ret
+
+ .p2align 4
+L(1):
+ movzbl (%rdi), %eax
+ movzbl (%rsi), %ecx
+ subl %ecx, %eax
+ ret
+# endif
+
+ .p2align 4
+L(zero):
+ xorl %eax, %eax
+ ret
+
+ .p2align 4
+L(less_vec):
+# ifdef USE_AS_WMEMCMP
+ /* It can only be 0, 4, 8, 12, 16, 20, 24, 28 bytes. */
+ cmpb $4, %dl
+ je L(4)
+ jb L(zero)
+# else
+ cmpb $1, %dl
+ je L(1)
+ jb L(zero)
+ cmpb $4, %dl
+ jb L(between_2_3)
+ cmpb $8, %dl
+ jb L(between_4_7)
+# endif
+ cmpb $16, %dl
+ jae L(between_16_31)
+ /* It is between 8 and 15 bytes. */
+ vmovq (%rdi), %XMM1
+ vmovq (%rsi), %XMM2
+ VPCMPEQ %XMM1, %XMM2, %k2
+ kmovw %k2, %eax
+ subl $XMM_MASK, %eax
+ jnz L(first_vec)
+ /* Use overlapping loads to avoid branches. */
+ leaq -8(%rdi, %rdx), %rdi
+ leaq -8(%rsi, %rdx), %rsi
+ vmovq (%rdi), %XMM1
+ vmovq (%rsi), %XMM2
+ VPCMPEQ %XMM1, %XMM2, %k2
+ kmovw %k2, %eax
+ subl $XMM_MASK, %eax
+ jnz L(first_vec)
+ ret
+
+ .p2align 4
+L(between_16_31):
+ /* From 16 to 31 bytes. No branch when size == 16. */
+ VMOVU (%rsi), %XMM2
+ VPCMPEQ (%rdi), %XMM2, %k2
+ kmovw %k2, %eax
+ subl $XMM_MASK, %eax
+ jnz L(first_vec)
+
+ /* Use overlapping loads to avoid branches. */
+ leaq -16(%rdi, %rdx), %rdi
+ leaq -16(%rsi, %rdx), %rsi
+ VMOVU (%rsi), %XMM2
+ VPCMPEQ (%rdi), %XMM2, %k2
+ kmovw %k2, %eax
+ subl $XMM_MASK, %eax
+ jnz L(first_vec)
+ ret
+
+ .p2align 4
+L(more_8x_vec):
+ /* More than 8 * VEC. Check the first VEC. */
+ VMOVU (%rsi), %YMM2
+ VPCMPEQ (%rdi), %YMM2, %k2
+ kmovd %k2, %eax
+ subl $VEC_MASK, %eax
+ jnz L(first_vec)
+
+ /* Align the first memory area for aligned loads in the loop.
+ Compute how much the first memory area is misaligned. */
+ movq %rdi, %rcx
+ andl $(VEC_SIZE - 1), %ecx
+ /* Get the negative of offset for alignment. */
+ subq $VEC_SIZE, %rcx
+ /* Adjust the second memory area. */
+ subq %rcx, %rsi
+ /* Adjust the first memory area which should be aligned now. */
+ subq %rcx, %rdi
+ /* Adjust length. */
+ addq %rcx, %rdx
+
+L(loop_4x_vec):
+ /* Compare 4 * VEC at a time forward. */
+ VMOVU (%rsi), %YMM1
+ VPCMPEQ (%rdi), %YMM1, %k1
+
+ VMOVU VEC_SIZE(%rsi), %YMM2
+ VPCMPEQ VEC_SIZE(%rdi), %YMM2, %k2
+ kandd %k2, %k1, %k5
+
+ VMOVU (VEC_SIZE * 2)(%rsi), %YMM3
+ VPCMPEQ (VEC_SIZE * 2)(%rdi), %YMM3, %k3
+ kandd %k3, %k5, %k5
+
+ VMOVU (VEC_SIZE * 3)(%rsi), %YMM4
+ VPCMPEQ (VEC_SIZE * 3)(%rdi), %YMM4, %k4
+ kandd %k4, %k5, %k5
+
+ kmovd %k5, %eax
+ cmpl $VEC_MASK, %eax
+ jne L(4x_vec_end)
+
+ addq $(VEC_SIZE * 4), %rdi
+ addq $(VEC_SIZE * 4), %rsi
+
+ subq $(VEC_SIZE * 4), %rdx
+ cmpq $(VEC_SIZE * 4), %rdx
+ jae L(loop_4x_vec)
+
+ /* Less than 4 * VEC. */
+ cmpq $VEC_SIZE, %rdx
+ jbe L(last_vec)
+ cmpq $(VEC_SIZE * 2), %rdx
+ jbe L(last_2x_vec)
+
+L(last_4x_vec):
+ /* From 2 * VEC to 4 * VEC. */
+ VMOVU (%rsi), %YMM2
+ VPCMPEQ (%rdi), %YMM2, %k2
+ kmovd %k2, %eax
+ subl $VEC_MASK, %eax
+ jnz L(first_vec)
+
+ addq $VEC_SIZE, %rdi
+ addq $VEC_SIZE, %rsi
+ VMOVU (%rsi), %YMM2
+ VPCMPEQ (%rdi), %YMM2, %k2
+ kmovd %k2, %eax
+ subl $VEC_MASK, %eax
+ jnz L(first_vec)
+
+ /* Use overlapping loads to avoid branches. */
+ leaq -(3 * VEC_SIZE)(%rdi, %rdx), %rdi
+ leaq -(3 * VEC_SIZE)(%rsi, %rdx), %rsi
+ VMOVU (%rsi), %YMM2
+ VPCMPEQ (%rdi), %YMM2, %k2
+ kmovd %k2, %eax
+ subl $VEC_MASK, %eax
+ jnz L(first_vec)
+
+ addq $VEC_SIZE, %rdi
+ addq $VEC_SIZE, %rsi
+ VMOVU (%rsi), %YMM2
+ VPCMPEQ (%rdi), %YMM2, %k2
+ kmovd %k2, %eax
+ subl $VEC_MASK, %eax
+ jnz L(first_vec)
+ ret
+
+ .p2align 4
+L(4x_vec_end):
+ kmovd %k1, %eax
+ subl $VEC_MASK, %eax
+ jnz L(first_vec)
+ kmovd %k2, %eax
+ subl $VEC_MASK, %eax
+ jnz L(first_vec_x1)
+ kmovd %k3, %eax
+ subl $VEC_MASK, %eax
+ jnz L(first_vec_x2)
+ kmovd %k4, %eax
+ subl $VEC_MASK, %eax
+ tzcntl %eax, %ecx
+# ifdef USE_AS_WMEMCMP
+ xorl %eax, %eax
+ movl (VEC_SIZE * 3)(%rdi, %rcx, 4), %edx
+ cmpl (VEC_SIZE * 3)(%rsi, %rcx, 4), %edx
+ jmp L(wmemcmp_return)
+# else
+ movzbl (VEC_SIZE * 3)(%rdi, %rcx), %eax
+ movzbl (VEC_SIZE * 3)(%rsi, %rcx), %edx
+ sub %edx, %eax
+# endif
+ ret
+
+ .p2align 4
+L(first_vec_x1):
+ tzcntl %eax, %ecx
+# ifdef USE_AS_WMEMCMP
+ xorl %eax, %eax
+ movl VEC_SIZE(%rdi, %rcx, 4), %edx
+ cmpl VEC_SIZE(%rsi, %rcx, 4), %edx
+ jmp L(wmemcmp_return)
+# else
+ movzbl VEC_SIZE(%rdi, %rcx), %eax
+ movzbl VEC_SIZE(%rsi, %rcx), %edx
+ sub %edx, %eax
+# endif
+ ret
+
+ .p2align 4
+L(first_vec_x2):
+ tzcntl %eax, %ecx
+# ifdef USE_AS_WMEMCMP
+ xorl %eax, %eax
+ movl (VEC_SIZE * 2)(%rdi, %rcx, 4), %edx
+ cmpl (VEC_SIZE * 2)(%rsi, %rcx, 4), %edx
+ jmp L(wmemcmp_return)
+# else
+ movzbl (VEC_SIZE * 2)(%rdi, %rcx), %eax
+ movzbl (VEC_SIZE * 2)(%rsi, %rcx), %edx
+ sub %edx, %eax
+# endif
+ ret
+END (MEMCMP)
+#endif
diff --git a/sysdeps/x86_64/multiarch/wmemcmp-evex-movbe.S b/sysdeps/x86_64/multiarch/wmemcmp-evex-movbe.S
new file mode 100644
index 00000000..4726d74a
--- /dev/null
+++ b/sysdeps/x86_64/multiarch/wmemcmp-evex-movbe.S
@@ -0,0 +1,4 @@
+#define MEMCMP __wmemcmp_evex_movbe
+#define USE_AS_WMEMCMP 1
+
+#include "memcmp-evex-movbe.S"
--
GitLab

File diff suppressed because it is too large Load Diff

View File

@ -1,735 +0,0 @@
From 4bd660be40967cd69072f69ebc2ad32bfcc1f206 Mon Sep 17 00:00:00 2001
From: "H.J. Lu" <hjl.tools@gmail.com>
Date: Tue, 23 Feb 2021 06:33:10 -0800
Subject: [PATCH] x86: Add string/memory function tests in RTM region
Content-type: text/plain; charset=UTF-8
At function exit, AVX optimized string/memory functions have VZEROUPPER
which triggers RTM abort. When such functions are called inside a
transactionally executing RTM region, RTM abort causes severe performance
degradation. Add tests to verify that string/memory functions won't
cause RTM abort in RTM region.
---
sysdeps/x86/Makefile | 23 +++++++++++
sysdeps/x86/tst-memchr-rtm.c | 54 ++++++++++++++++++++++++++
sysdeps/x86/tst-memcmp-rtm.c | 52 +++++++++++++++++++++++++
sysdeps/x86/tst-memmove-rtm.c | 53 ++++++++++++++++++++++++++
sysdeps/x86/tst-memrchr-rtm.c | 54 ++++++++++++++++++++++++++
sysdeps/x86/tst-memset-rtm.c | 45 ++++++++++++++++++++++
sysdeps/x86/tst-strchr-rtm.c | 54 ++++++++++++++++++++++++++
sysdeps/x86/tst-strcpy-rtm.c | 53 ++++++++++++++++++++++++++
sysdeps/x86/tst-string-rtm.h | 72 +++++++++++++++++++++++++++++++++++
sysdeps/x86/tst-strlen-rtm.c | 53 ++++++++++++++++++++++++++
sysdeps/x86/tst-strncmp-rtm.c | 52 +++++++++++++++++++++++++
sysdeps/x86/tst-strrchr-rtm.c | 53 ++++++++++++++++++++++++++
12 files changed, 618 insertions(+)
create mode 100644 sysdeps/x86/tst-memchr-rtm.c
create mode 100644 sysdeps/x86/tst-memcmp-rtm.c
create mode 100644 sysdeps/x86/tst-memmove-rtm.c
create mode 100644 sysdeps/x86/tst-memrchr-rtm.c
create mode 100644 sysdeps/x86/tst-memset-rtm.c
create mode 100644 sysdeps/x86/tst-strchr-rtm.c
create mode 100644 sysdeps/x86/tst-strcpy-rtm.c
create mode 100644 sysdeps/x86/tst-string-rtm.h
create mode 100644 sysdeps/x86/tst-strlen-rtm.c
create mode 100644 sysdeps/x86/tst-strncmp-rtm.c
create mode 100644 sysdeps/x86/tst-strrchr-rtm.c
diff --git a/sysdeps/x86/Makefile b/sysdeps/x86/Makefile
index 59e928e9..5be71ada 100644
--- a/sysdeps/x86/Makefile
+++ b/sysdeps/x86/Makefile
@@ -17,6 +17,29 @@ endif
ifeq ($(subdir),string)
sysdep_routines += cacheinfo
+
+tests += \
+ tst-memchr-rtm \
+ tst-memcmp-rtm \
+ tst-memmove-rtm \
+ tst-memrchr-rtm \
+ tst-memset-rtm \
+ tst-strchr-rtm \
+ tst-strcpy-rtm \
+ tst-strlen-rtm \
+ tst-strncmp-rtm \
+ tst-strrchr-rtm
+
+CFLAGS-tst-memchr-rtm.c += -mrtm
+CFLAGS-tst-memcmp-rtm.c += -mrtm
+CFLAGS-tst-memmove-rtm.c += -mrtm
+CFLAGS-tst-memrchr-rtm.c += -mrtm
+CFLAGS-tst-memset-rtm.c += -mrtm
+CFLAGS-tst-strchr-rtm.c += -mrtm
+CFLAGS-tst-strcpy-rtm.c += -mrtm
+CFLAGS-tst-strlen-rtm.c += -mrtm
+CFLAGS-tst-strncmp-rtm.c += -mrtm
+CFLAGS-tst-strrchr-rtm.c += -mrtm
endif
ifneq ($(enable-cet),no)
diff --git a/sysdeps/x86/tst-memchr-rtm.c b/sysdeps/x86/tst-memchr-rtm.c
new file mode 100644
index 00000000..e4749401
--- /dev/null
+++ b/sysdeps/x86/tst-memchr-rtm.c
@@ -0,0 +1,54 @@
+/* Test case for memchr inside a transactionally executing RTM region.
+ Copyright (C) 2021 Free Software Foundation, Inc.
+ This file is part of the GNU C Library.
+
+ The GNU C Library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ The GNU C Library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with the GNU C Library; if not, see
+ <https://www.gnu.org/licenses/>. */
+
+#include <tst-string-rtm.h>
+
+#define LOOP 3000
+#define STRING_SIZE 1024
+char string1[STRING_SIZE];
+
+__attribute__ ((noinline, noclone))
+static int
+prepare (void)
+{
+ memset (string1, 'a', STRING_SIZE);
+ string1[100] = 'c';
+ string1[STRING_SIZE - 100] = 'c';
+ char *p = memchr (string1, 'c', STRING_SIZE);
+ if (p == &string1[100])
+ return EXIT_SUCCESS;
+ else
+ return EXIT_FAILURE;
+}
+
+__attribute__ ((noinline, noclone))
+static int
+function (void)
+{
+ char *p = memchr (string1, 'c', STRING_SIZE);
+ if (p == &string1[100])
+ return 0;
+ else
+ return 1;
+}
+
+static int
+do_test (void)
+{
+ return do_test_1 ("memchr", LOOP, prepare, function);
+}
diff --git a/sysdeps/x86/tst-memcmp-rtm.c b/sysdeps/x86/tst-memcmp-rtm.c
new file mode 100644
index 00000000..e4c8a623
--- /dev/null
+++ b/sysdeps/x86/tst-memcmp-rtm.c
@@ -0,0 +1,52 @@
+/* Test case for memcmp inside a transactionally executing RTM region.
+ Copyright (C) 2021 Free Software Foundation, Inc.
+ This file is part of the GNU C Library.
+
+ The GNU C Library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ The GNU C Library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with the GNU C Library; if not, see
+ <https://www.gnu.org/licenses/>. */
+
+#include <tst-string-rtm.h>
+
+#define LOOP 3000
+#define STRING_SIZE 1024
+char string1[STRING_SIZE];
+char string2[STRING_SIZE];
+
+__attribute__ ((noinline, noclone))
+static int
+prepare (void)
+{
+ memset (string1, 'a', STRING_SIZE);
+ memset (string2, 'a', STRING_SIZE);
+ if (memcmp (string1, string2, STRING_SIZE) == 0)
+ return EXIT_SUCCESS;
+ else
+ return EXIT_FAILURE;
+}
+
+__attribute__ ((noinline, noclone))
+static int
+function (void)
+{
+ if (memcmp (string1, string2, STRING_SIZE) == 0)
+ return 0;
+ else
+ return 1;
+}
+
+static int
+do_test (void)
+{
+ return do_test_1 ("memcmp", LOOP, prepare, function);
+}
diff --git a/sysdeps/x86/tst-memmove-rtm.c b/sysdeps/x86/tst-memmove-rtm.c
new file mode 100644
index 00000000..4bf97ef1
--- /dev/null
+++ b/sysdeps/x86/tst-memmove-rtm.c
@@ -0,0 +1,53 @@
+/* Test case for memmove inside a transactionally executing RTM region.
+ Copyright (C) 2021 Free Software Foundation, Inc.
+ This file is part of the GNU C Library.
+
+ The GNU C Library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ The GNU C Library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with the GNU C Library; if not, see
+ <https://www.gnu.org/licenses/>. */
+
+#include <tst-string-rtm.h>
+
+#define LOOP 3000
+#define STRING_SIZE 1024
+char string1[STRING_SIZE];
+char string2[STRING_SIZE];
+
+__attribute__ ((noinline, noclone))
+static int
+prepare (void)
+{
+ memset (string1, 'a', STRING_SIZE);
+ if (memmove (string2, string1, STRING_SIZE) == string2
+ && memcmp (string2, string1, STRING_SIZE) == 0)
+ return EXIT_SUCCESS;
+ else
+ return EXIT_FAILURE;
+}
+
+__attribute__ ((noinline, noclone))
+static int
+function (void)
+{
+ if (memmove (string2, string1, STRING_SIZE) == string2
+ && memcmp (string2, string1, STRING_SIZE) == 0)
+ return 0;
+ else
+ return 1;
+}
+
+static int
+do_test (void)
+{
+ return do_test_1 ("memmove", LOOP, prepare, function);
+}
diff --git a/sysdeps/x86/tst-memrchr-rtm.c b/sysdeps/x86/tst-memrchr-rtm.c
new file mode 100644
index 00000000..a57a5a8e
--- /dev/null
+++ b/sysdeps/x86/tst-memrchr-rtm.c
@@ -0,0 +1,54 @@
+/* Test case for memrchr inside a transactionally executing RTM region.
+ Copyright (C) 2021 Free Software Foundation, Inc.
+ This file is part of the GNU C Library.
+
+ The GNU C Library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ The GNU C Library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with the GNU C Library; if not, see
+ <https://www.gnu.org/licenses/>. */
+
+#include <tst-string-rtm.h>
+
+#define LOOP 3000
+#define STRING_SIZE 1024
+char string1[STRING_SIZE];
+
+__attribute__ ((noinline, noclone))
+static int
+prepare (void)
+{
+ memset (string1, 'a', STRING_SIZE);
+ string1[100] = 'c';
+ string1[STRING_SIZE - 100] = 'c';
+ char *p = memrchr (string1, 'c', STRING_SIZE);
+ if (p == &string1[STRING_SIZE - 100])
+ return EXIT_SUCCESS;
+ else
+ return EXIT_FAILURE;
+}
+
+__attribute__ ((noinline, noclone))
+static int
+function (void)
+{
+ char *p = memrchr (string1, 'c', STRING_SIZE);
+ if (p == &string1[STRING_SIZE - 100])
+ return 0;
+ else
+ return 1;
+}
+
+static int
+do_test (void)
+{
+ return do_test_1 ("memrchr", LOOP, prepare, function);
+}
diff --git a/sysdeps/x86/tst-memset-rtm.c b/sysdeps/x86/tst-memset-rtm.c
new file mode 100644
index 00000000..bf343a4d
--- /dev/null
+++ b/sysdeps/x86/tst-memset-rtm.c
@@ -0,0 +1,45 @@
+/* Test case for memset inside a transactionally executing RTM region.
+ Copyright (C) 2021 Free Software Foundation, Inc.
+ This file is part of the GNU C Library.
+
+ The GNU C Library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ The GNU C Library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with the GNU C Library; if not, see
+ <https://www.gnu.org/licenses/>. */
+
+#include <tst-string-rtm.h>
+
+#define LOOP 3000
+#define STRING_SIZE 1024
+char string1[STRING_SIZE];
+
+__attribute__ ((noinline, noclone))
+static int
+prepare (void)
+{
+ memset (string1, 'a', STRING_SIZE);
+ return EXIT_SUCCESS;
+}
+
+__attribute__ ((noinline, noclone))
+static int
+function (void)
+{
+ memset (string1, 'a', STRING_SIZE);
+ return 0;
+}
+
+static int
+do_test (void)
+{
+ return do_test_1 ("memset", LOOP, prepare, function);
+}
diff --git a/sysdeps/x86/tst-strchr-rtm.c b/sysdeps/x86/tst-strchr-rtm.c
new file mode 100644
index 00000000..a82e29c0
--- /dev/null
+++ b/sysdeps/x86/tst-strchr-rtm.c
@@ -0,0 +1,54 @@
+/* Test case for strchr inside a transactionally executing RTM region.
+ Copyright (C) 2021 Free Software Foundation, Inc.
+ This file is part of the GNU C Library.
+
+ The GNU C Library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ The GNU C Library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with the GNU C Library; if not, see
+ <https://www.gnu.org/licenses/>. */
+
+#include <tst-string-rtm.h>
+
+#define LOOP 3000
+#define STRING_SIZE 1024
+char string1[STRING_SIZE];
+
+__attribute__ ((noinline, noclone))
+static int
+prepare (void)
+{
+ memset (string1, 'a', STRING_SIZE - 1);
+ string1[100] = 'c';
+ string1[STRING_SIZE - 100] = 'c';
+ char *p = strchr (string1, 'c');
+ if (p == &string1[100])
+ return EXIT_SUCCESS;
+ else
+ return EXIT_FAILURE;
+}
+
+__attribute__ ((noinline, noclone))
+static int
+function (void)
+{
+ char *p = strchr (string1, 'c');
+ if (p == &string1[100])
+ return 0;
+ else
+ return 1;
+}
+
+static int
+do_test (void)
+{
+ return do_test_1 ("strchr", LOOP, prepare, function);
+}
diff --git a/sysdeps/x86/tst-strcpy-rtm.c b/sysdeps/x86/tst-strcpy-rtm.c
new file mode 100644
index 00000000..2b2a583f
--- /dev/null
+++ b/sysdeps/x86/tst-strcpy-rtm.c
@@ -0,0 +1,53 @@
+/* Test case for strcpy inside a transactionally executing RTM region.
+ Copyright (C) 2021 Free Software Foundation, Inc.
+ This file is part of the GNU C Library.
+
+ The GNU C Library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ The GNU C Library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with the GNU C Library; if not, see
+ <https://www.gnu.org/licenses/>. */
+
+#include <tst-string-rtm.h>
+
+#define LOOP 3000
+#define STRING_SIZE 1024
+char string1[STRING_SIZE];
+char string2[STRING_SIZE];
+
+__attribute__ ((noinline, noclone))
+static int
+prepare (void)
+{
+ memset (string1, 'a', STRING_SIZE - 1);
+ if (strcpy (string2, string1) == string2
+ && strcmp (string2, string1) == 0)
+ return EXIT_SUCCESS;
+ else
+ return EXIT_FAILURE;
+}
+
+__attribute__ ((noinline, noclone))
+static int
+function (void)
+{
+ if (strcpy (string2, string1) == string2
+ && strcmp (string2, string1) == 0)
+ return 0;
+ else
+ return 1;
+}
+
+static int
+do_test (void)
+{
+ return do_test_1 ("strcpy", LOOP, prepare, function);
+}
diff --git a/sysdeps/x86/tst-string-rtm.h b/sysdeps/x86/tst-string-rtm.h
new file mode 100644
index 00000000..d2470afa
--- /dev/null
+++ b/sysdeps/x86/tst-string-rtm.h
@@ -0,0 +1,72 @@
+/* Test string function in a transactionally executing RTM region.
+ Copyright (C) 2021 Free Software Foundation, Inc.
+ This file is part of the GNU C Library.
+
+ The GNU C Library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ The GNU C Library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with the GNU C Library; if not, see
+ <https://www.gnu.org/licenses/>. */
+
+#include <string.h>
+#include <x86intrin.h>
+#include <sys/platform/x86.h>
+#include <support/check.h>
+#include <support/test-driver.h>
+
+static int
+do_test_1 (const char *name, unsigned int loop, int (*prepare) (void),
+ int (*function) (void))
+{
+ if (!CPU_FEATURE_USABLE (RTM))
+ return EXIT_UNSUPPORTED;
+
+ int status = prepare ();
+ if (status != EXIT_SUCCESS)
+ return status;
+
+ unsigned int i;
+ unsigned int naborts = 0;
+ unsigned int failed = 0;
+ for (i = 0; i < loop; i++)
+ {
+ failed |= function ();
+ if (_xbegin() == _XBEGIN_STARTED)
+ {
+ failed |= function ();
+ _xend();
+ }
+ else
+ {
+ failed |= function ();
+ ++naborts;
+ }
+ }
+
+ if (failed)
+ FAIL_EXIT1 ("%s() failed", name);
+
+ if (naborts)
+ {
+ /* NB: Low single digit (<= 5%) noise-level aborts are normal for
+ TSX. */
+ double rate = 100 * ((double) naborts) / ((double) loop);
+ if (rate > 5)
+ FAIL_EXIT1 ("TSX abort rate: %.2f%% (%d out of %d)",
+ rate, naborts, loop);
+ }
+
+ return EXIT_SUCCESS;
+}
+
+static int do_test (void);
+
+#include <support/test-driver.c>
diff --git a/sysdeps/x86/tst-strlen-rtm.c b/sysdeps/x86/tst-strlen-rtm.c
new file mode 100644
index 00000000..0dcf14db
--- /dev/null
+++ b/sysdeps/x86/tst-strlen-rtm.c
@@ -0,0 +1,53 @@
+/* Test case for strlen inside a transactionally executing RTM region.
+ Copyright (C) 2021 Free Software Foundation, Inc.
+ This file is part of the GNU C Library.
+
+ The GNU C Library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ The GNU C Library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with the GNU C Library; if not, see
+ <https://www.gnu.org/licenses/>. */
+
+#include <tst-string-rtm.h>
+
+#define LOOP 3000
+#define STRING_SIZE 1024
+char string1[STRING_SIZE];
+
+__attribute__ ((noinline, noclone))
+static int
+prepare (void)
+{
+ memset (string1, 'a', STRING_SIZE - 1);
+ string1[STRING_SIZE - 100] = '\0';
+ size_t len = strlen (string1);
+ if (len == STRING_SIZE - 100)
+ return EXIT_SUCCESS;
+ else
+ return EXIT_FAILURE;
+}
+
+__attribute__ ((noinline, noclone))
+static int
+function (void)
+{
+ size_t len = strlen (string1);
+ if (len == STRING_SIZE - 100)
+ return 0;
+ else
+ return 1;
+}
+
+static int
+do_test (void)
+{
+ return do_test_1 ("strlen", LOOP, prepare, function);
+}
diff --git a/sysdeps/x86/tst-strncmp-rtm.c b/sysdeps/x86/tst-strncmp-rtm.c
new file mode 100644
index 00000000..236ad951
--- /dev/null
+++ b/sysdeps/x86/tst-strncmp-rtm.c
@@ -0,0 +1,52 @@
+/* Test case for strncmp inside a transactionally executing RTM region.
+ Copyright (C) 2021 Free Software Foundation, Inc.
+ This file is part of the GNU C Library.
+
+ The GNU C Library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ The GNU C Library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with the GNU C Library; if not, see
+ <https://www.gnu.org/licenses/>. */
+
+#include <tst-string-rtm.h>
+
+#define LOOP 3000
+#define STRING_SIZE 1024
+char string1[STRING_SIZE];
+char string2[STRING_SIZE];
+
+__attribute__ ((noinline, noclone))
+static int
+prepare (void)
+{
+ memset (string1, 'a', STRING_SIZE - 1);
+ memset (string2, 'a', STRING_SIZE - 1);
+ if (strncmp (string1, string2, STRING_SIZE) == 0)
+ return EXIT_SUCCESS;
+ else
+ return EXIT_FAILURE;
+}
+
+__attribute__ ((noinline, noclone))
+static int
+function (void)
+{
+ if (strncmp (string1, string2, STRING_SIZE) == 0)
+ return 0;
+ else
+ return 1;
+}
+
+static int
+do_test (void)
+{
+ return do_test_1 ("strncmp", LOOP, prepare, function);
+}
diff --git a/sysdeps/x86/tst-strrchr-rtm.c b/sysdeps/x86/tst-strrchr-rtm.c
new file mode 100644
index 00000000..e32bfaf5
--- /dev/null
+++ b/sysdeps/x86/tst-strrchr-rtm.c
@@ -0,0 +1,53 @@
+/* Test case for strrchr inside a transactionally executing RTM region.
+ Copyright (C) 2021 Free Software Foundation, Inc.
+ This file is part of the GNU C Library.
+
+ The GNU C Library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ The GNU C Library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with the GNU C Library; if not, see
+ <https://www.gnu.org/licenses/>. */
+
+#include <tst-string-rtm.h>
+
+#define LOOP 3000
+#define STRING_SIZE 1024
+char string1[STRING_SIZE];
+
+__attribute__ ((noinline, noclone))
+static int
+prepare (void)
+{
+ memset (string1, 'a', STRING_SIZE - 1);
+ string1[STRING_SIZE - 100] = 'c';
+ char *p = strrchr (string1, 'c');
+ if (p == &string1[STRING_SIZE - 100])
+ return EXIT_SUCCESS;
+ else
+ return EXIT_FAILURE;
+}
+
+__attribute__ ((noinline, noclone))
+static int
+function (void)
+{
+ char *p = strrchr (string1, 'c');
+ if (p == &string1[STRING_SIZE - 100])
+ return 0;
+ else
+ return 1;
+}
+
+static int
+do_test (void)
+{
+ return do_test_1 ("strrchr", LOOP, prepare, function);
+}
--
GitLab

View File

@ -1,148 +0,0 @@
From 4e2d8f352774b56078c34648b14a2412c38384f4 Mon Sep 17 00:00:00 2001
From: "H.J. Lu" <hjl.tools@gmail.com>
Date: Sun, 7 Mar 2021 09:44:18 -0800
Subject: [PATCH] x86-64: Use ZMM16-ZMM31 in AVX512 memset family functions
Content-type: text/plain; charset=UTF-8
Update ifunc-memset.h/ifunc-wmemset.h to select the function optimized
with AVX512 instructions using ZMM16-ZMM31 registers to avoid RTM abort
with usable AVX512VL and AVX512BW since VZEROUPPER isn't needed at
function exit.
---
sysdeps/x86_64/multiarch/ifunc-impl-list.c | 14 +++++++++-----
sysdeps/x86_64/multiarch/ifunc-memset.h | 13 ++++++++-----
sysdeps/x86_64/multiarch/ifunc-wmemset.h | 12 ++++++------
.../multiarch/memset-avx512-unaligned-erms.S | 16 ++++++++--------
4 files changed, 31 insertions(+), 24 deletions(-)
diff --git a/sysdeps/x86_64/multiarch/ifunc-impl-list.c b/sysdeps/x86_64/multiarch/ifunc-impl-list.c
index c1efeec0..d969a156 100644
--- a/sysdeps/x86_64/multiarch/ifunc-impl-list.c
+++ b/sysdeps/x86_64/multiarch/ifunc-impl-list.c
@@ -211,10 +211,12 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,
&& CPU_FEATURE_USABLE (AVX512BW)),
__memset_chk_evex_unaligned_erms)
IFUNC_IMPL_ADD (array, i, __memset_chk,
- CPU_FEATURE_USABLE (AVX512F),
+ (CPU_FEATURE_USABLE (AVX512VL)
+ && CPU_FEATURE_USABLE (AVX512BW)),
__memset_chk_avx512_unaligned_erms)
IFUNC_IMPL_ADD (array, i, __memset_chk,
- CPU_FEATURE_USABLE (AVX512F),
+ (CPU_FEATURE_USABLE (AVX512VL)
+ && CPU_FEATURE_USABLE (AVX512BW)),
__memset_chk_avx512_unaligned)
IFUNC_IMPL_ADD (array, i, __memset_chk,
CPU_FEATURE_USABLE (AVX512F),
@@ -252,10 +254,12 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,
&& CPU_FEATURE_USABLE (AVX512BW)),
__memset_evex_unaligned_erms)
IFUNC_IMPL_ADD (array, i, memset,
- CPU_FEATURE_USABLE (AVX512F),
+ (CPU_FEATURE_USABLE (AVX512VL)
+ && CPU_FEATURE_USABLE (AVX512BW)),
__memset_avx512_unaligned_erms)
IFUNC_IMPL_ADD (array, i, memset,
- CPU_FEATURE_USABLE (AVX512F),
+ (CPU_FEATURE_USABLE (AVX512VL)
+ && CPU_FEATURE_USABLE (AVX512BW)),
__memset_avx512_unaligned)
IFUNC_IMPL_ADD (array, i, memset,
CPU_FEATURE_USABLE (AVX512F),
@@ -719,7 +723,7 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,
CPU_FEATURE_USABLE (AVX512VL),
__wmemset_evex_unaligned)
IFUNC_IMPL_ADD (array, i, wmemset,
- CPU_FEATURE_USABLE (AVX512F),
+ CPU_FEATURE_USABLE (AVX512VL),
__wmemset_avx512_unaligned))
#ifdef SHARED
diff --git a/sysdeps/x86_64/multiarch/ifunc-memset.h b/sysdeps/x86_64/multiarch/ifunc-memset.h
index 6f3375cc..19795938 100644
--- a/sysdeps/x86_64/multiarch/ifunc-memset.h
+++ b/sysdeps/x86_64/multiarch/ifunc-memset.h
@@ -53,13 +53,16 @@ IFUNC_SELECTOR (void)
if (CPU_FEATURE_USABLE_P (cpu_features, AVX512F)
&& !CPU_FEATURES_ARCH_P (cpu_features, Prefer_No_AVX512))
{
- if (CPU_FEATURES_ARCH_P (cpu_features, Prefer_No_VZEROUPPER))
- return OPTIMIZE (avx512_no_vzeroupper);
+ if (CPU_FEATURE_USABLE_P (cpu_features, AVX512VL)
+ && CPU_FEATURE_USABLE_P (cpu_features, AVX512BW))
+ {
+ if (CPU_FEATURE_USABLE_P (cpu_features, ERMS))
+ return OPTIMIZE (avx512_unaligned_erms);
- if (CPU_FEATURE_USABLE_P (cpu_features, ERMS))
- return OPTIMIZE (avx512_unaligned_erms);
+ return OPTIMIZE (avx512_unaligned);
+ }
- return OPTIMIZE (avx512_unaligned);
+ return OPTIMIZE (avx512_no_vzeroupper);
}
if (CPU_FEATURE_USABLE_P (cpu_features, AVX2))
diff --git a/sysdeps/x86_64/multiarch/ifunc-wmemset.h b/sysdeps/x86_64/multiarch/ifunc-wmemset.h
index bdc94c6c..98c5d406 100644
--- a/sysdeps/x86_64/multiarch/ifunc-wmemset.h
+++ b/sysdeps/x86_64/multiarch/ifunc-wmemset.h
@@ -33,13 +33,13 @@ IFUNC_SELECTOR (void)
if (CPU_FEATURE_USABLE_P (cpu_features, AVX2)
&& CPU_FEATURES_ARCH_P (cpu_features, AVX_Fast_Unaligned_Load))
{
- if (CPU_FEATURE_USABLE_P (cpu_features, AVX512F)
- && !CPU_FEATURES_ARCH_P (cpu_features, Prefer_No_AVX512)
- && !CPU_FEATURES_ARCH_P (cpu_features, Prefer_No_VZEROUPPER))
- return OPTIMIZE (avx512_unaligned);
-
if (CPU_FEATURE_USABLE_P (cpu_features, AVX512VL))
- return OPTIMIZE (evex_unaligned);
+ {
+ if (!CPU_FEATURES_ARCH_P (cpu_features, Prefer_No_AVX512))
+ return OPTIMIZE (avx512_unaligned);
+
+ return OPTIMIZE (evex_unaligned);
+ }
if (CPU_FEATURE_USABLE_P (cpu_features, RTM))
return OPTIMIZE (avx2_unaligned_rtm);
diff --git a/sysdeps/x86_64/multiarch/memset-avx512-unaligned-erms.S b/sysdeps/x86_64/multiarch/memset-avx512-unaligned-erms.S
index 0783979c..22e7b187 100644
--- a/sysdeps/x86_64/multiarch/memset-avx512-unaligned-erms.S
+++ b/sysdeps/x86_64/multiarch/memset-avx512-unaligned-erms.S
@@ -1,22 +1,22 @@
#if IS_IN (libc)
# define VEC_SIZE 64
-# define VEC(i) zmm##i
+# define XMM0 xmm16
+# define YMM0 ymm16
+# define VEC0 zmm16
+# define VEC(i) VEC##i
# define VMOVU vmovdqu64
# define VMOVA vmovdqa64
+# define VZEROUPPER
# define MEMSET_VDUP_TO_VEC0_AND_SET_RETURN(d, r) \
- vmovd d, %xmm0; \
movq r, %rax; \
- vpbroadcastb %xmm0, %xmm0; \
- vpbroadcastq %xmm0, %zmm0
+ vpbroadcastb d, %VEC0
# define WMEMSET_VDUP_TO_VEC0_AND_SET_RETURN(d, r) \
- vmovd d, %xmm0; \
movq r, %rax; \
- vpbroadcastd %xmm0, %xmm0; \
- vpbroadcastq %xmm0, %zmm0
+ vpbroadcastd d, %VEC0
-# define SECTION(p) p##.avx512
+# define SECTION(p) p##.evex512
# define MEMSET_SYMBOL(p,s) p##_avx512_##s
# define WMEMSET_SYMBOL(p,s) p##_avx512_##s
--
GitLab

View File

@ -1,230 +0,0 @@
From b304fc201d2f6baf52ea790df8643e99772243cd Mon Sep 17 00:00:00 2001
From: "H.J. Lu" <hjl.tools@gmail.com>
Date: Mon, 21 Jan 2019 11:25:56 -0800
Subject: [PATCH] x86-64 memcmp/wmemcmp: Properly handle the length parameter
[BZ# 24097]
Content-type: text/plain; charset=UTF-8
On x32, the size_t parameter may be passed in the lower 32 bits of a
64-bit register with the non-zero upper 32 bits. The string/memory
functions written in assembly can only use the lower 32 bits of a
64-bit register as length or must clear the upper 32 bits before using
the full 64-bit register for length.
This pach fixes memcmp/wmemcmp for x32. Tested on x86-64 and x32. On
x86-64, libc.so is the same with and withou the fix.
[BZ# 24097]
CVE-2019-6488
* sysdeps/x86_64/multiarch/memcmp-avx2-movbe.S: Use RDX_LP for
length. Clear the upper 32 bits of RDX register.
* sysdeps/x86_64/multiarch/memcmp-sse4.S: Likewise.
* sysdeps/x86_64/multiarch/memcmp-ssse3.S: Likewise.
* sysdeps/x86_64/x32/Makefile (tests): Add tst-size_t-memcmp and
tst-size_t-wmemcmp.
* sysdeps/x86_64/x32/tst-size_t-memcmp.c: New file.
* sysdeps/x86_64/x32/tst-size_t-wmemcmp.c: Likewise.
---
sysdeps/x86_64/multiarch/memcmp-avx2-movbe.S | 7 +-
sysdeps/x86_64/multiarch/memcmp-sse4.S | 9 ++-
sysdeps/x86_64/multiarch/memcmp-ssse3.S | 7 +-
sysdeps/x86_64/x32/Makefile | 4 +-
sysdeps/x86_64/x32/tst-size_t-memcmp.c | 76 ++++++++++++++++++++
sysdeps/x86_64/x32/tst-size_t-wmemcmp.c | 20 ++++++
6 files changed, 114 insertions(+), 9 deletions(-)
create mode 100644 sysdeps/x86_64/x32/tst-size_t-memcmp.c
create mode 100644 sysdeps/x86_64/x32/tst-size_t-wmemcmp.c
Conflicts:
ChangeLog
(removed)
diff --git a/sysdeps/x86_64/multiarch/memcmp-avx2-movbe.S b/sysdeps/x86_64/multiarch/memcmp-avx2-movbe.S
index 30f764c3..e3a35b89 100644
--- a/sysdeps/x86_64/multiarch/memcmp-avx2-movbe.S
+++ b/sysdeps/x86_64/multiarch/memcmp-avx2-movbe.S
@@ -58,9 +58,12 @@
.section .text.avx,"ax",@progbits
ENTRY (MEMCMP)
# ifdef USE_AS_WMEMCMP
- shl $2, %rdx
+ shl $2, %RDX_LP
+# elif defined __ILP32__
+ /* Clear the upper 32 bits. */
+ movl %edx, %edx
# endif
- cmpq $VEC_SIZE, %rdx
+ cmp $VEC_SIZE, %RDX_LP
jb L(less_vec)
/* From VEC to 2 * VEC. No branch when size == VEC_SIZE. */
diff --git a/sysdeps/x86_64/multiarch/memcmp-sse4.S b/sysdeps/x86_64/multiarch/memcmp-sse4.S
index 8e164f2c..302900f5 100644
--- a/sysdeps/x86_64/multiarch/memcmp-sse4.S
+++ b/sysdeps/x86_64/multiarch/memcmp-sse4.S
@@ -42,13 +42,16 @@
.section .text.sse4.1,"ax",@progbits
ENTRY (MEMCMP)
# ifdef USE_AS_WMEMCMP
- shl $2, %rdx
+ shl $2, %RDX_LP
+# elif defined __ILP32__
+ /* Clear the upper 32 bits. */
+ mov %edx, %edx
# endif
pxor %xmm0, %xmm0
- cmp $79, %rdx
+ cmp $79, %RDX_LP
ja L(79bytesormore)
# ifndef USE_AS_WMEMCMP
- cmp $1, %rdx
+ cmp $1, %RDX_LP
je L(firstbyte)
# endif
add %rdx, %rsi
diff --git a/sysdeps/x86_64/multiarch/memcmp-ssse3.S b/sysdeps/x86_64/multiarch/memcmp-ssse3.S
index 6f76c641..69d030fc 100644
--- a/sysdeps/x86_64/multiarch/memcmp-ssse3.S
+++ b/sysdeps/x86_64/multiarch/memcmp-ssse3.S
@@ -33,9 +33,12 @@
atom_text_section
ENTRY (MEMCMP)
# ifdef USE_AS_WMEMCMP
- shl $2, %rdx
- test %rdx, %rdx
+ shl $2, %RDX_LP
+ test %RDX_LP, %RDX_LP
jz L(equal)
+# elif defined __ILP32__
+ /* Clear the upper 32 bits. */
+ mov %edx, %edx
# endif
mov %rdx, %rcx
mov %rdi, %rdx
diff --git a/sysdeps/x86_64/x32/Makefile b/sysdeps/x86_64/x32/Makefile
index 7d528889..ddec7f04 100644
--- a/sysdeps/x86_64/x32/Makefile
+++ b/sysdeps/x86_64/x32/Makefile
@@ -6,9 +6,9 @@ CFLAGS-s_llround.c += -fno-builtin-lround
endif
ifeq ($(subdir),string)
-tests += tst-size_t-memchr
+tests += tst-size_t-memchr tst-size_t-memcmp
endif
ifeq ($(subdir),wcsmbs)
-tests += tst-size_t-wmemchr
+tests += tst-size_t-wmemchr tst-size_t-wmemcmp
endif
diff --git a/sysdeps/x86_64/x32/tst-size_t-memcmp.c b/sysdeps/x86_64/x32/tst-size_t-memcmp.c
new file mode 100644
index 00000000..9bd6fdb4
--- /dev/null
+++ b/sysdeps/x86_64/x32/tst-size_t-memcmp.c
@@ -0,0 +1,76 @@
+/* Test memcmp with size_t in the lower 32 bits of 64-bit register.
+ Copyright (C) 2019 Free Software Foundation, Inc.
+ This file is part of the GNU C Library.
+
+ The GNU C Library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ The GNU C Library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with the GNU C Library; if not, see
+ <http://www.gnu.org/licenses/>. */
+
+#define TEST_MAIN
+#ifdef WIDE
+# define TEST_NAME "wmemcmp"
+#else
+# define TEST_NAME "memcmp"
+#endif
+
+#include "test-size_t.h"
+
+#ifdef WIDE
+# include <inttypes.h>
+# include <wchar.h>
+
+# define MEMCMP wmemcmp
+# define CHAR wchar_t
+#else
+# define MEMCMP memcmp
+# define CHAR char
+#endif
+
+IMPL (MEMCMP, 1)
+
+typedef int (*proto_t) (const CHAR *, const CHAR *, size_t);
+
+static int
+__attribute__ ((noinline, noclone))
+do_memcmp (parameter_t a, parameter_t b)
+{
+ return CALL (&b, a.p, b.p, a.len);
+}
+
+static int
+test_main (void)
+{
+ test_init ();
+
+ parameter_t dest = { { page_size / sizeof (CHAR) }, buf1 };
+ parameter_t src = { { 0 }, buf2 };
+
+ memcpy (buf1, buf2, page_size);
+
+ int ret = 0;
+ FOR_EACH_IMPL (impl, 0)
+ {
+ src.fn = impl->fn;
+ int res = do_memcmp (dest, src);
+ if (res)
+ {
+ error (0, 0, "Wrong result in function %s: %i != 0",
+ impl->name, res);
+ ret = 1;
+ }
+ }
+
+ return ret ? EXIT_FAILURE : EXIT_SUCCESS;
+}
+
+#include <support/test-driver.c>
diff --git a/sysdeps/x86_64/x32/tst-size_t-wmemcmp.c b/sysdeps/x86_64/x32/tst-size_t-wmemcmp.c
new file mode 100644
index 00000000..e8b5ffd0
--- /dev/null
+++ b/sysdeps/x86_64/x32/tst-size_t-wmemcmp.c
@@ -0,0 +1,20 @@
+/* Test wmemcmp with size_t in the lower 32 bits of 64-bit register.
+ Copyright (C) 2019 Free Software Foundation, Inc.
+ This file is part of the GNU C Library.
+
+ The GNU C Library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ The GNU C Library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with the GNU C Library; if not, see
+ <http://www.gnu.org/licenses/>. */
+
+#define WIDE 1
+#include "tst-size_t-memcmp.c"
--
GitLab

View File

@ -1,164 +0,0 @@
From e4fda4631017e49d4ee5a2755db34289b6860fa4 Mon Sep 17 00:00:00 2001
From: "H.J. Lu" <hjl.tools@gmail.com>
Date: Sun, 7 Mar 2021 09:45:23 -0800
Subject: [PATCH] x86-64: Use ZMM16-ZMM31 in AVX512 memmove family functions
Content-type: text/plain; charset=UTF-8
Update ifunc-memmove.h to select the function optimized with AVX512
instructions using ZMM16-ZMM31 registers to avoid RTM abort with usable
AVX512VL since VZEROUPPER isn't needed at function exit.
---
sysdeps/x86_64/multiarch/ifunc-impl-list.c | 24 +++++++++---------
sysdeps/x86_64/multiarch/ifunc-memmove.h | 12 +++++----
.../multiarch/memmove-avx512-unaligned-erms.S | 25 +++++++++++++++++--
3 files changed, 42 insertions(+), 19 deletions(-)
diff --git a/sysdeps/x86_64/multiarch/ifunc-impl-list.c b/sysdeps/x86_64/multiarch/ifunc-impl-list.c
index d969a156..fec384f6 100644
--- a/sysdeps/x86_64/multiarch/ifunc-impl-list.c
+++ b/sysdeps/x86_64/multiarch/ifunc-impl-list.c
@@ -83,10 +83,10 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,
CPU_FEATURE_USABLE (AVX512F),
__memmove_chk_avx512_no_vzeroupper)
IFUNC_IMPL_ADD (array, i, __memmove_chk,
- CPU_FEATURE_USABLE (AVX512F),
+ CPU_FEATURE_USABLE (AVX512VL),
__memmove_chk_avx512_unaligned)
IFUNC_IMPL_ADD (array, i, __memmove_chk,
- CPU_FEATURE_USABLE (AVX512F),
+ CPU_FEATURE_USABLE (AVX512VL),
__memmove_chk_avx512_unaligned_erms)
IFUNC_IMPL_ADD (array, i, __memmove_chk,
CPU_FEATURE_USABLE (AVX),
@@ -148,10 +148,10 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,
CPU_FEATURE_USABLE (AVX512F),
__memmove_avx512_no_vzeroupper)
IFUNC_IMPL_ADD (array, i, memmove,
- CPU_FEATURE_USABLE (AVX512F),
+ CPU_FEATURE_USABLE (AVX512VL),
__memmove_avx512_unaligned)
IFUNC_IMPL_ADD (array, i, memmove,
- CPU_FEATURE_USABLE (AVX512F),
+ CPU_FEATURE_USABLE (AVX512VL),
__memmove_avx512_unaligned_erms)
IFUNC_IMPL_ADD (array, i, memmove, CPU_FEATURE_USABLE (SSSE3),
__memmove_ssse3_back)
@@ -733,10 +733,10 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,
CPU_FEATURE_USABLE (AVX512F),
__memcpy_chk_avx512_no_vzeroupper)
IFUNC_IMPL_ADD (array, i, __memcpy_chk,
- CPU_FEATURE_USABLE (AVX512F),
+ CPU_FEATURE_USABLE (AVX512VL),
__memcpy_chk_avx512_unaligned)
IFUNC_IMPL_ADD (array, i, __memcpy_chk,
- CPU_FEATURE_USABLE (AVX512F),
+ CPU_FEATURE_USABLE (AVX512VL),
__memcpy_chk_avx512_unaligned_erms)
IFUNC_IMPL_ADD (array, i, __memcpy_chk,
CPU_FEATURE_USABLE (AVX),
@@ -802,10 +802,10 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,
CPU_FEATURE_USABLE (AVX512F),
__memcpy_avx512_no_vzeroupper)
IFUNC_IMPL_ADD (array, i, memcpy,
- CPU_FEATURE_USABLE (AVX512F),
+ CPU_FEATURE_USABLE (AVX512VL),
__memcpy_avx512_unaligned)
IFUNC_IMPL_ADD (array, i, memcpy,
- CPU_FEATURE_USABLE (AVX512F),
+ CPU_FEATURE_USABLE (AVX512VL),
__memcpy_avx512_unaligned_erms)
IFUNC_IMPL_ADD (array, i, memcpy, 1, __memcpy_sse2_unaligned)
IFUNC_IMPL_ADD (array, i, memcpy, 1,
@@ -819,10 +819,10 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,
CPU_FEATURE_USABLE (AVX512F),
__mempcpy_chk_avx512_no_vzeroupper)
IFUNC_IMPL_ADD (array, i, __mempcpy_chk,
- CPU_FEATURE_USABLE (AVX512F),
+ CPU_FEATURE_USABLE (AVX512VL),
__mempcpy_chk_avx512_unaligned)
IFUNC_IMPL_ADD (array, i, __mempcpy_chk,
- CPU_FEATURE_USABLE (AVX512F),
+ CPU_FEATURE_USABLE (AVX512VL),
__mempcpy_chk_avx512_unaligned_erms)
IFUNC_IMPL_ADD (array, i, __mempcpy_chk,
CPU_FEATURE_USABLE (AVX),
@@ -864,10 +864,10 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,
CPU_FEATURE_USABLE (AVX512F),
__mempcpy_avx512_no_vzeroupper)
IFUNC_IMPL_ADD (array, i, mempcpy,
- CPU_FEATURE_USABLE (AVX512F),
+ CPU_FEATURE_USABLE (AVX512VL),
__mempcpy_avx512_unaligned)
IFUNC_IMPL_ADD (array, i, mempcpy,
- CPU_FEATURE_USABLE (AVX512F),
+ CPU_FEATURE_USABLE (AVX512VL),
__mempcpy_avx512_unaligned_erms)
IFUNC_IMPL_ADD (array, i, mempcpy,
CPU_FEATURE_USABLE (AVX),
diff --git a/sysdeps/x86_64/multiarch/ifunc-memmove.h b/sysdeps/x86_64/multiarch/ifunc-memmove.h
index fa09b9fb..014e95c7 100644
--- a/sysdeps/x86_64/multiarch/ifunc-memmove.h
+++ b/sysdeps/x86_64/multiarch/ifunc-memmove.h
@@ -56,13 +56,15 @@ IFUNC_SELECTOR (void)
if (CPU_FEATURE_USABLE_P (cpu_features, AVX512F)
&& !CPU_FEATURES_ARCH_P (cpu_features, Prefer_No_AVX512))
{
- if (CPU_FEATURES_ARCH_P (cpu_features, Prefer_No_VZEROUPPER))
- return OPTIMIZE (avx512_no_vzeroupper);
+ if (CPU_FEATURE_USABLE_P (cpu_features, AVX512VL))
+ {
+ if (CPU_FEATURE_USABLE_P (cpu_features, ERMS))
+ return OPTIMIZE (avx512_unaligned_erms);
- if (CPU_FEATURE_USABLE_P (cpu_features, ERMS))
- return OPTIMIZE (avx512_unaligned_erms);
+ return OPTIMIZE (avx512_unaligned);
+ }
- return OPTIMIZE (avx512_unaligned);
+ return OPTIMIZE (avx512_no_vzeroupper);
}
if (CPU_FEATURES_ARCH_P (cpu_features, AVX_Fast_Unaligned_Load))
diff --git a/sysdeps/x86_64/multiarch/memmove-avx512-unaligned-erms.S b/sysdeps/x86_64/multiarch/memmove-avx512-unaligned-erms.S
index aac1515c..848848ab 100644
--- a/sysdeps/x86_64/multiarch/memmove-avx512-unaligned-erms.S
+++ b/sysdeps/x86_64/multiarch/memmove-avx512-unaligned-erms.S
@@ -1,11 +1,32 @@
#if IS_IN (libc)
# define VEC_SIZE 64
-# define VEC(i) zmm##i
+# define XMM0 xmm16
+# define XMM1 xmm17
+# define YMM0 ymm16
+# define YMM1 ymm17
+# define VEC0 zmm16
+# define VEC1 zmm17
+# define VEC2 zmm18
+# define VEC3 zmm19
+# define VEC4 zmm20
+# define VEC5 zmm21
+# define VEC6 zmm22
+# define VEC7 zmm23
+# define VEC8 zmm24
+# define VEC9 zmm25
+# define VEC10 zmm26
+# define VEC11 zmm27
+# define VEC12 zmm28
+# define VEC13 zmm29
+# define VEC14 zmm30
+# define VEC15 zmm31
+# define VEC(i) VEC##i
# define VMOVNT vmovntdq
# define VMOVU vmovdqu64
# define VMOVA vmovdqa64
+# define VZEROUPPER
-# define SECTION(p) p##.avx512
+# define SECTION(p) p##.evex512
# define MEMMOVE_SYMBOL(p,s) p##_avx512_##s
# include "memmove-vec-unaligned-erms.S"
--
GitLab

View File

@ -1,71 +0,0 @@
From 595c22ecd8e87a27fd19270ed30fdbae9ad25426 Mon Sep 17 00:00:00 2001
From: Sunil K Pandey <skpgkp2@gmail.com>
Date: Thu, 1 Apr 2021 15:47:04 -0700
Subject: [PATCH] x86-64: Fix ifdef indentation in strlen-evex.S
Content-type: text/plain; charset=UTF-8
Fix some indentations of ifdef in file strlen-evex.S which are off by 1
and confusing to read.
---
sysdeps/x86_64/multiarch/strlen-evex.S | 16 ++++++++--------
1 file changed, 8 insertions(+), 8 deletions(-)
diff --git a/sysdeps/x86_64/multiarch/strlen-evex.S b/sysdeps/x86_64/multiarch/strlen-evex.S
index cd022509..05838190 100644
--- a/sysdeps/x86_64/multiarch/strlen-evex.S
+++ b/sysdeps/x86_64/multiarch/strlen-evex.S
@@ -276,10 +276,10 @@ L(last_2x_vec):
.p2align 4
L(first_vec_x0_check):
tzcntl %eax, %eax
-# ifdef USE_AS_WCSLEN
+# ifdef USE_AS_WCSLEN
/* NB: Multiply wchar_t count by 4 to get the number of bytes. */
sall $2, %eax
-# endif
+# endif
/* Check the end of data. */
cmpq %rax, %rsi
jbe L(max)
@@ -293,10 +293,10 @@ L(first_vec_x0_check):
.p2align 4
L(first_vec_x1_check):
tzcntl %eax, %eax
-# ifdef USE_AS_WCSLEN
+# ifdef USE_AS_WCSLEN
/* NB: Multiply wchar_t count by 4 to get the number of bytes. */
sall $2, %eax
-# endif
+# endif
/* Check the end of data. */
cmpq %rax, %rsi
jbe L(max)
@@ -311,10 +311,10 @@ L(first_vec_x1_check):
.p2align 4
L(first_vec_x2_check):
tzcntl %eax, %eax
-# ifdef USE_AS_WCSLEN
+# ifdef USE_AS_WCSLEN
/* NB: Multiply wchar_t count by 4 to get the number of bytes. */
sall $2, %eax
-# endif
+# endif
/* Check the end of data. */
cmpq %rax, %rsi
jbe L(max)
@@ -329,10 +329,10 @@ L(first_vec_x2_check):
.p2align 4
L(first_vec_x3_check):
tzcntl %eax, %eax
-# ifdef USE_AS_WCSLEN
+# ifdef USE_AS_WCSLEN
/* NB: Multiply wchar_t count by 4 to get the number of bytes. */
sall $2, %eax
-# endif
+# endif
/* Check the end of data. */
cmpq %rax, %rsi
jbe L(max)
--
GitLab

View File

@ -1,51 +0,0 @@
From 55bf411b451c13f0fb7ff3d3bf9a820020b45df1 Mon Sep 17 00:00:00 2001
From: "H.J. Lu" <hjl.tools@gmail.com>
Date: Mon, 19 Apr 2021 07:07:21 -0700
Subject: [PATCH] x86-64: Require BMI2 for __strlen_evex and __strnlen_evex
Content-type: text/plain; charset=UTF-8
Since __strlen_evex and __strnlen_evex added by
commit 1fd8c163a83d96ace1ff78fa6bac7aee084f6f77
Author: H.J. Lu <hjl.tools@gmail.com>
Date: Fri Mar 5 06:24:52 2021 -0800
x86-64: Add ifunc-avx2.h functions with 256-bit EVEX
use sarx:
c4 e2 6a f7 c0 sarx %edx,%eax,%eax
require BMI2 for __strlen_evex and __strnlen_evex in ifunc-impl-list.c.
ifunc-avx2.h already requires BMI2 for EVEX implementation.
---
sysdeps/x86_64/multiarch/ifunc-impl-list.c | 6 ++++--
1 file changed, 4 insertions(+), 2 deletions(-)
diff --git a/sysdeps/x86_64/multiarch/ifunc-impl-list.c b/sysdeps/x86_64/multiarch/ifunc-impl-list.c
index fec384f6..cbfc1a5d 100644
--- a/sysdeps/x86_64/multiarch/ifunc-impl-list.c
+++ b/sysdeps/x86_64/multiarch/ifunc-impl-list.c
@@ -293,7 +293,8 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,
__strlen_avx2_rtm)
IFUNC_IMPL_ADD (array, i, strlen,
(CPU_FEATURE_USABLE (AVX512VL)
- && CPU_FEATURE_USABLE (AVX512BW)),
+ && CPU_FEATURE_USABLE (AVX512BW)
+ && CPU_FEATURE_USABLE (BMI2)),
__strlen_evex)
IFUNC_IMPL_ADD (array, i, strlen, 1, __strlen_sse2))
@@ -308,7 +309,8 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,
__strnlen_avx2_rtm)
IFUNC_IMPL_ADD (array, i, strnlen,
(CPU_FEATURE_USABLE (AVX512VL)
- && CPU_FEATURE_USABLE (AVX512BW)),
+ && CPU_FEATURE_USABLE (AVX512BW)
+ && CPU_FEATURE_USABLE (BMI2)),
__strnlen_evex)
IFUNC_IMPL_ADD (array, i, strnlen, 1, __strnlen_sse2))
--
GitLab

View File

@ -1,584 +0,0 @@
From acfd088a1963ba51cd83c78f95c0ab25ead79e04 Mon Sep 17 00:00:00 2001
From: Noah Goldstein <goldstein.w.n@gmail.com>
Date: Mon, 3 May 2021 03:01:58 -0400
Subject: [PATCH] x86: Optimize memchr-avx2.S
Content-type: text/plain; charset=UTF-8
No bug. This commit optimizes memchr-avx2.S. The optimizations include
replacing some branches with cmovcc, avoiding some branches entirely
in the less_4x_vec case, making the page cross logic less strict,
asaving a few instructions the in loop return loop. test-memchr,
test-rawmemchr, and test-wmemchr are all passing.
Signed-off-by: Noah Goldstein <goldstein.w.n@gmail.com>
Reviewed-by: H.J. Lu <hjl.tools@gmail.com>
---
sysdeps/x86_64/multiarch/memchr-avx2.S | 425 ++++++++++++++-----------
1 file changed, 247 insertions(+), 178 deletions(-)
diff --git a/sysdeps/x86_64/multiarch/memchr-avx2.S b/sysdeps/x86_64/multiarch/memchr-avx2.S
index cf893e77..b377f22e 100644
--- a/sysdeps/x86_64/multiarch/memchr-avx2.S
+++ b/sysdeps/x86_64/multiarch/memchr-avx2.S
@@ -26,8 +26,22 @@
# ifdef USE_AS_WMEMCHR
# define VPCMPEQ vpcmpeqd
+# define VPBROADCAST vpbroadcastd
+# define CHAR_SIZE 4
# else
# define VPCMPEQ vpcmpeqb
+# define VPBROADCAST vpbroadcastb
+# define CHAR_SIZE 1
+# endif
+
+# ifdef USE_AS_RAWMEMCHR
+# define ERAW_PTR_REG ecx
+# define RRAW_PTR_REG rcx
+# define ALGN_PTR_REG rdi
+# else
+# define ERAW_PTR_REG edi
+# define RRAW_PTR_REG rdi
+# define ALGN_PTR_REG rcx
# endif
# ifndef VZEROUPPER
@@ -39,6 +53,7 @@
# endif
# define VEC_SIZE 32
+# define PAGE_SIZE 4096
.section SECTION(.text),"ax",@progbits
ENTRY (MEMCHR)
@@ -47,295 +62,349 @@ ENTRY (MEMCHR)
test %RDX_LP, %RDX_LP
jz L(null)
# endif
- movl %edi, %ecx
- /* Broadcast CHAR to YMM0. */
- vmovd %esi, %xmm0
# ifdef USE_AS_WMEMCHR
shl $2, %RDX_LP
- vpbroadcastd %xmm0, %ymm0
# else
# ifdef __ILP32__
/* Clear the upper 32 bits. */
movl %edx, %edx
# endif
- vpbroadcastb %xmm0, %ymm0
# endif
+ /* Broadcast CHAR to YMMMATCH. */
+ vmovd %esi, %xmm0
+ VPBROADCAST %xmm0, %ymm0
/* Check if we may cross page boundary with one vector load. */
- andl $(2 * VEC_SIZE - 1), %ecx
- cmpl $VEC_SIZE, %ecx
- ja L(cros_page_boundary)
+ movl %edi, %eax
+ andl $(PAGE_SIZE - 1), %eax
+ cmpl $(PAGE_SIZE - VEC_SIZE), %eax
+ ja L(cross_page_boundary)
/* Check the first VEC_SIZE bytes. */
- VPCMPEQ (%rdi), %ymm0, %ymm1
+ VPCMPEQ (%rdi), %ymm0, %ymm1
vpmovmskb %ymm1, %eax
- testl %eax, %eax
-
# ifndef USE_AS_RAWMEMCHR
- jnz L(first_vec_x0_check)
- /* Adjust length and check the end of data. */
- subq $VEC_SIZE, %rdx
- jbe L(zero)
-# else
- jnz L(first_vec_x0)
+ /* If length < CHAR_PER_VEC handle special. */
+ cmpq $VEC_SIZE, %rdx
+ jbe L(first_vec_x0)
# endif
-
- /* Align data for aligned loads in the loop. */
- addq $VEC_SIZE, %rdi
- andl $(VEC_SIZE - 1), %ecx
- andq $-VEC_SIZE, %rdi
+ testl %eax, %eax
+ jz L(aligned_more)
+ tzcntl %eax, %eax
+ addq %rdi, %rax
+ VZEROUPPER_RETURN
# ifndef USE_AS_RAWMEMCHR
- /* Adjust length. */
- addq %rcx, %rdx
+ .p2align 5
+L(first_vec_x0):
+ /* Check if first match was before length. */
+ tzcntl %eax, %eax
+ xorl %ecx, %ecx
+ cmpl %eax, %edx
+ leaq (%rdi, %rax), %rax
+ cmovle %rcx, %rax
+ VZEROUPPER_RETURN
- subq $(VEC_SIZE * 4), %rdx
- jbe L(last_4x_vec_or_less)
+L(null):
+ xorl %eax, %eax
+ ret
# endif
- jmp L(more_4x_vec)
-
.p2align 4
-L(cros_page_boundary):
- andl $(VEC_SIZE - 1), %ecx
- andq $-VEC_SIZE, %rdi
- VPCMPEQ (%rdi), %ymm0, %ymm1
+L(cross_page_boundary):
+ /* Save pointer before aligning as its original value is necessary
+ for computer return address if byte is found or adjusting length
+ if it is not and this is memchr. */
+ movq %rdi, %rcx
+ /* Align data to VEC_SIZE - 1. ALGN_PTR_REG is rcx for memchr and
+ rdi for rawmemchr. */
+ orq $(VEC_SIZE - 1), %ALGN_PTR_REG
+ VPCMPEQ -(VEC_SIZE - 1)(%ALGN_PTR_REG), %ymm0, %ymm1
vpmovmskb %ymm1, %eax
+# ifndef USE_AS_RAWMEMCHR
+ /* Calculate length until end of page (length checked for a
+ match). */
+ leaq 1(%ALGN_PTR_REG), %rsi
+ subq %RRAW_PTR_REG, %rsi
+# endif
/* Remove the leading bytes. */
- sarl %cl, %eax
- testl %eax, %eax
- jz L(aligned_more)
- tzcntl %eax, %eax
+ sarxl %ERAW_PTR_REG, %eax, %eax
# ifndef USE_AS_RAWMEMCHR
/* Check the end of data. */
- cmpq %rax, %rdx
- jbe L(zero)
+ cmpq %rsi, %rdx
+ jbe L(first_vec_x0)
# endif
- addq %rdi, %rax
- addq %rcx, %rax
+ testl %eax, %eax
+ jz L(cross_page_continue)
+ tzcntl %eax, %eax
+ addq %RRAW_PTR_REG, %rax
L(return_vzeroupper):
ZERO_UPPER_VEC_REGISTERS_RETURN
.p2align 4
-L(aligned_more):
-# ifndef USE_AS_RAWMEMCHR
- /* Calculate "rdx + rcx - VEC_SIZE" with "rdx - (VEC_SIZE - rcx)"
- instead of "(rdx + rcx) - VEC_SIZE" to void possible addition
- overflow. */
- negq %rcx
- addq $VEC_SIZE, %rcx
+L(first_vec_x1):
+ tzcntl %eax, %eax
+ incq %rdi
+ addq %rdi, %rax
+ VZEROUPPER_RETURN
- /* Check the end of data. */
- subq %rcx, %rdx
- jbe L(zero)
-# endif
+ .p2align 4
+L(first_vec_x2):
+ tzcntl %eax, %eax
+ addq $(VEC_SIZE + 1), %rdi
+ addq %rdi, %rax
+ VZEROUPPER_RETURN
+
+ .p2align 4
+L(first_vec_x3):
+ tzcntl %eax, %eax
+ addq $(VEC_SIZE * 2 + 1), %rdi
+ addq %rdi, %rax
+ VZEROUPPER_RETURN
- addq $VEC_SIZE, %rdi
-# ifndef USE_AS_RAWMEMCHR
- subq $(VEC_SIZE * 4), %rdx
- jbe L(last_4x_vec_or_less)
-# endif
+ .p2align 4
+L(first_vec_x4):
+ tzcntl %eax, %eax
+ addq $(VEC_SIZE * 3 + 1), %rdi
+ addq %rdi, %rax
+ VZEROUPPER_RETURN
-L(more_4x_vec):
+ .p2align 4
+L(aligned_more):
/* Check the first 4 * VEC_SIZE. Only one VEC_SIZE at a time
since data is only aligned to VEC_SIZE. */
- VPCMPEQ (%rdi), %ymm0, %ymm1
- vpmovmskb %ymm1, %eax
- testl %eax, %eax
- jnz L(first_vec_x0)
- VPCMPEQ VEC_SIZE(%rdi), %ymm0, %ymm1
+# ifndef USE_AS_RAWMEMCHR
+L(cross_page_continue):
+ /* Align data to VEC_SIZE - 1. */
+ xorl %ecx, %ecx
+ subl %edi, %ecx
+ orq $(VEC_SIZE - 1), %rdi
+ /* esi is for adjusting length to see if near the end. */
+ leal (VEC_SIZE * 4 + 1)(%rdi, %rcx), %esi
+# else
+ orq $(VEC_SIZE - 1), %rdi
+L(cross_page_continue):
+# endif
+ /* Load first VEC regardless. */
+ VPCMPEQ 1(%rdi), %ymm0, %ymm1
vpmovmskb %ymm1, %eax
+# ifndef USE_AS_RAWMEMCHR
+ /* Adjust length. If near end handle specially. */
+ subq %rsi, %rdx
+ jbe L(last_4x_vec_or_less)
+# endif
testl %eax, %eax
jnz L(first_vec_x1)
- VPCMPEQ (VEC_SIZE * 2)(%rdi), %ymm0, %ymm1
+ VPCMPEQ (VEC_SIZE + 1)(%rdi), %ymm0, %ymm1
vpmovmskb %ymm1, %eax
testl %eax, %eax
jnz L(first_vec_x2)
- VPCMPEQ (VEC_SIZE * 3)(%rdi), %ymm0, %ymm1
+ VPCMPEQ (VEC_SIZE * 2 + 1)(%rdi), %ymm0, %ymm1
vpmovmskb %ymm1, %eax
testl %eax, %eax
jnz L(first_vec_x3)
- addq $(VEC_SIZE * 4), %rdi
+ VPCMPEQ (VEC_SIZE * 3 + 1)(%rdi), %ymm0, %ymm1
+ vpmovmskb %ymm1, %eax
+ testl %eax, %eax
+ jnz L(first_vec_x4)
# ifndef USE_AS_RAWMEMCHR
+ /* Check if at last VEC_SIZE * 4 length. */
subq $(VEC_SIZE * 4), %rdx
- jbe L(last_4x_vec_or_less)
-# endif
-
- /* Align data to 4 * VEC_SIZE. */
- movq %rdi, %rcx
- andl $(4 * VEC_SIZE - 1), %ecx
- andq $-(4 * VEC_SIZE), %rdi
-
-# ifndef USE_AS_RAWMEMCHR
- /* Adjust length. */
+ jbe L(last_4x_vec_or_less_cmpeq)
+ /* Align data to VEC_SIZE * 4 - 1 for the loop and readjust
+ length. */
+ incq %rdi
+ movl %edi, %ecx
+ orq $(VEC_SIZE * 4 - 1), %rdi
+ andl $(VEC_SIZE * 4 - 1), %ecx
addq %rcx, %rdx
+# else
+ /* Align data to VEC_SIZE * 4 - 1 for loop. */
+ incq %rdi
+ orq $(VEC_SIZE * 4 - 1), %rdi
# endif
+ /* Compare 4 * VEC at a time forward. */
.p2align 4
L(loop_4x_vec):
- /* Compare 4 * VEC at a time forward. */
- VPCMPEQ (%rdi), %ymm0, %ymm1
- VPCMPEQ VEC_SIZE(%rdi), %ymm0, %ymm2
- VPCMPEQ (VEC_SIZE * 2)(%rdi), %ymm0, %ymm3
- VPCMPEQ (VEC_SIZE * 3)(%rdi), %ymm0, %ymm4
-
+ VPCMPEQ 1(%rdi), %ymm0, %ymm1
+ VPCMPEQ (VEC_SIZE + 1)(%rdi), %ymm0, %ymm2
+ VPCMPEQ (VEC_SIZE * 2 + 1)(%rdi), %ymm0, %ymm3
+ VPCMPEQ (VEC_SIZE * 3 + 1)(%rdi), %ymm0, %ymm4
vpor %ymm1, %ymm2, %ymm5
vpor %ymm3, %ymm4, %ymm6
vpor %ymm5, %ymm6, %ymm5
- vpmovmskb %ymm5, %eax
- testl %eax, %eax
- jnz L(4x_vec_end)
-
- addq $(VEC_SIZE * 4), %rdi
-
+ vpmovmskb %ymm5, %ecx
# ifdef USE_AS_RAWMEMCHR
- jmp L(loop_4x_vec)
+ subq $-(VEC_SIZE * 4), %rdi
+ testl %ecx, %ecx
+ jz L(loop_4x_vec)
# else
- subq $(VEC_SIZE * 4), %rdx
- ja L(loop_4x_vec)
+ testl %ecx, %ecx
+ jnz L(loop_4x_vec_end)
-L(last_4x_vec_or_less):
- /* Less than 4 * VEC and aligned to VEC_SIZE. */
- addl $(VEC_SIZE * 2), %edx
- jle L(last_2x_vec)
+ subq $-(VEC_SIZE * 4), %rdi
- VPCMPEQ (%rdi), %ymm0, %ymm1
- vpmovmskb %ymm1, %eax
- testl %eax, %eax
- jnz L(first_vec_x0)
+ subq $(VEC_SIZE * 4), %rdx
+ ja L(loop_4x_vec)
- VPCMPEQ VEC_SIZE(%rdi), %ymm0, %ymm1
+ /* Fall through into less than 4 remaining vectors of length case.
+ */
+ VPCMPEQ (VEC_SIZE * 0 + 1)(%rdi), %ymm0, %ymm1
vpmovmskb %ymm1, %eax
+ .p2align 4
+L(last_4x_vec_or_less):
+ /* Check if first VEC contained match. */
testl %eax, %eax
- jnz L(first_vec_x1)
+ jnz L(first_vec_x1_check)
- VPCMPEQ (VEC_SIZE * 2)(%rdi), %ymm0, %ymm1
- vpmovmskb %ymm1, %eax
- testl %eax, %eax
+ /* If remaining length > VEC_SIZE * 2. */
+ addl $(VEC_SIZE * 2), %edx
+ jg L(last_4x_vec)
- jnz L(first_vec_x2_check)
- subl $VEC_SIZE, %edx
- jle L(zero)
+L(last_2x_vec):
+ /* If remaining length < VEC_SIZE. */
+ addl $VEC_SIZE, %edx
+ jle L(zero_end)
- VPCMPEQ (VEC_SIZE * 3)(%rdi), %ymm0, %ymm1
+ /* Check VEC2 and compare any match with remaining length. */
+ VPCMPEQ (VEC_SIZE + 1)(%rdi), %ymm0, %ymm1
vpmovmskb %ymm1, %eax
- testl %eax, %eax
-
- jnz L(first_vec_x3_check)
- xorl %eax, %eax
+ tzcntl %eax, %eax
+ cmpl %eax, %edx
+ jbe L(set_zero_end)
+ addq $(VEC_SIZE + 1), %rdi
+ addq %rdi, %rax
+L(zero_end):
VZEROUPPER_RETURN
.p2align 4
-L(last_2x_vec):
- addl $(VEC_SIZE * 2), %edx
- VPCMPEQ (%rdi), %ymm0, %ymm1
+L(loop_4x_vec_end):
+# endif
+ /* rawmemchr will fall through into this if match was found in
+ loop. */
+
vpmovmskb %ymm1, %eax
testl %eax, %eax
+ jnz L(last_vec_x1_return)
- jnz L(first_vec_x0_check)
- subl $VEC_SIZE, %edx
- jle L(zero)
-
- VPCMPEQ VEC_SIZE(%rdi), %ymm0, %ymm1
- vpmovmskb %ymm1, %eax
+ vpmovmskb %ymm2, %eax
testl %eax, %eax
- jnz L(first_vec_x1_check)
- xorl %eax, %eax
- VZEROUPPER_RETURN
+ jnz L(last_vec_x2_return)
- .p2align 4
-L(first_vec_x0_check):
- tzcntl %eax, %eax
- /* Check the end of data. */
- cmpq %rax, %rdx
- jbe L(zero)
+ vpmovmskb %ymm3, %eax
+ /* Combine VEC3 matches (eax) with VEC4 matches (ecx). */
+ salq $32, %rcx
+ orq %rcx, %rax
+ tzcntq %rax, %rax
+# ifdef USE_AS_RAWMEMCHR
+ subq $(VEC_SIZE * 2 - 1), %rdi
+# else
+ subq $-(VEC_SIZE * 2 + 1), %rdi
+# endif
addq %rdi, %rax
VZEROUPPER_RETURN
+# ifndef USE_AS_RAWMEMCHR
.p2align 4
L(first_vec_x1_check):
tzcntl %eax, %eax
- /* Check the end of data. */
- cmpq %rax, %rdx
- jbe L(zero)
- addq $VEC_SIZE, %rax
+ /* Adjust length. */
+ subl $-(VEC_SIZE * 4), %edx
+ /* Check if match within remaining length. */
+ cmpl %eax, %edx
+ jbe L(set_zero_end)
+ incq %rdi
addq %rdi, %rax
VZEROUPPER_RETURN
+ .p2align 4
+L(set_zero_end):
+ xorl %eax, %eax
+ VZEROUPPER_RETURN
+# endif
.p2align 4
-L(first_vec_x2_check):
+L(last_vec_x1_return):
tzcntl %eax, %eax
- /* Check the end of data. */
- cmpq %rax, %rdx
- jbe L(zero)
- addq $(VEC_SIZE * 2), %rax
+# ifdef USE_AS_RAWMEMCHR
+ subq $(VEC_SIZE * 4 - 1), %rdi
+# else
+ incq %rdi
+# endif
addq %rdi, %rax
VZEROUPPER_RETURN
.p2align 4
-L(first_vec_x3_check):
+L(last_vec_x2_return):
tzcntl %eax, %eax
- /* Check the end of data. */
- cmpq %rax, %rdx
- jbe L(zero)
- addq $(VEC_SIZE * 3), %rax
+# ifdef USE_AS_RAWMEMCHR
+ subq $(VEC_SIZE * 3 - 1), %rdi
+# else
+ subq $-(VEC_SIZE + 1), %rdi
+# endif
addq %rdi, %rax
VZEROUPPER_RETURN
+# ifndef USE_AS_RAWMEMCHR
.p2align 4
-L(zero):
- xorl %eax, %eax
- jmp L(return_vzeroupper)
+L(last_4x_vec_or_less_cmpeq):
+ VPCMPEQ (VEC_SIZE * 4 + 1)(%rdi), %ymm0, %ymm1
+ vpmovmskb %ymm1, %eax
+ subq $-(VEC_SIZE * 4), %rdi
+ /* Check first VEC regardless. */
+ testl %eax, %eax
+ jnz L(first_vec_x1_check)
+ /* If remaining length <= CHAR_PER_VEC * 2. */
+ addl $(VEC_SIZE * 2), %edx
+ jle L(last_2x_vec)
.p2align 4
-L(null):
- xorl %eax, %eax
- ret
-# endif
+L(last_4x_vec):
+ VPCMPEQ (VEC_SIZE + 1)(%rdi), %ymm0, %ymm1
+ vpmovmskb %ymm1, %eax
+ testl %eax, %eax
+ jnz L(last_vec_x2_return)
- .p2align 4
-L(first_vec_x0):
- tzcntl %eax, %eax
- addq %rdi, %rax
- VZEROUPPER_RETURN
+ VPCMPEQ (VEC_SIZE * 2 + 1)(%rdi), %ymm0, %ymm1
+ vpmovmskb %ymm1, %eax
- .p2align 4
-L(first_vec_x1):
- tzcntl %eax, %eax
- addq $VEC_SIZE, %rax
- addq %rdi, %rax
- VZEROUPPER_RETURN
+ /* Create mask for possible matches within remaining length. */
+ movq $-1, %rcx
+ bzhiq %rdx, %rcx, %rcx
- .p2align 4
-L(first_vec_x2):
+ /* Test matches in data against length match. */
+ andl %ecx, %eax
+ jnz L(last_vec_x3)
+
+ /* if remaining length <= VEC_SIZE * 3 (Note this is after
+ remaining length was found to be > VEC_SIZE * 2. */
+ subl $VEC_SIZE, %edx
+ jbe L(zero_end2)
+
+ VPCMPEQ (VEC_SIZE * 3 + 1)(%rdi), %ymm0, %ymm1
+ vpmovmskb %ymm1, %eax
+ /* Shift remaining length mask for last VEC. */
+ shrq $32, %rcx
+ andl %ecx, %eax
+ jz L(zero_end2)
tzcntl %eax, %eax
- addq $(VEC_SIZE * 2), %rax
+ addq $(VEC_SIZE * 3 + 1), %rdi
addq %rdi, %rax
+L(zero_end2):
VZEROUPPER_RETURN
.p2align 4
-L(4x_vec_end):
- vpmovmskb %ymm1, %eax
- testl %eax, %eax
- jnz L(first_vec_x0)
- vpmovmskb %ymm2, %eax
- testl %eax, %eax
- jnz L(first_vec_x1)
- vpmovmskb %ymm3, %eax
- testl %eax, %eax
- jnz L(first_vec_x2)
- vpmovmskb %ymm4, %eax
- testl %eax, %eax
-L(first_vec_x3):
+L(last_vec_x3):
tzcntl %eax, %eax
- addq $(VEC_SIZE * 3), %rax
+ subq $-(VEC_SIZE * 2 + 1), %rdi
addq %rdi, %rax
VZEROUPPER_RETURN
+# endif
END (MEMCHR)
#endif
--
GitLab

View File

@ -1,388 +0,0 @@
From 645a158978f9520e74074e8c14047503be4db0f0 Mon Sep 17 00:00:00 2001
From: Noah Goldstein <goldstein.w.n@gmail.com>
Date: Wed, 9 Jun 2021 16:25:32 -0400
Subject: [PATCH] x86: Fix overflow bug with wmemchr-sse2 and wmemchr-avx2 [BZ
#27974]
Content-type: text/plain; charset=UTF-8
This commit fixes the bug mentioned in the previous commit.
The previous implementations of wmemchr in these files relied
on n * sizeof(wchar_t) which was not guranteed by the standard.
The new overflow tests added in the previous commit now
pass (As well as all the other tests).
Signed-off-by: Noah Goldstein <goldstein.w.n@gmail.com>
Reviewed-by: H.J. Lu <hjl.tools@gmail.com>
---
sysdeps/x86_64/memchr.S | 77 +++++++++++++++++++-------
sysdeps/x86_64/multiarch/memchr-avx2.S | 58 +++++++++++++------
2 files changed, 98 insertions(+), 37 deletions(-)
diff --git a/sysdeps/x86_64/memchr.S b/sysdeps/x86_64/memchr.S
index cb320257..24f9a0c5 100644
--- a/sysdeps/x86_64/memchr.S
+++ b/sysdeps/x86_64/memchr.S
@@ -21,9 +21,11 @@
#ifdef USE_AS_WMEMCHR
# define MEMCHR wmemchr
# define PCMPEQ pcmpeqd
+# define CHAR_PER_VEC 4
#else
# define MEMCHR memchr
# define PCMPEQ pcmpeqb
+# define CHAR_PER_VEC 16
#endif
/* fast SSE2 version with using pmaxub and 64 byte loop */
@@ -33,15 +35,14 @@ ENTRY(MEMCHR)
movd %esi, %xmm1
mov %edi, %ecx
+#ifdef __ILP32__
+ /* Clear the upper 32 bits. */
+ movl %edx, %edx
+#endif
#ifdef USE_AS_WMEMCHR
test %RDX_LP, %RDX_LP
jz L(return_null)
- shl $2, %RDX_LP
#else
-# ifdef __ILP32__
- /* Clear the upper 32 bits. */
- movl %edx, %edx
-# endif
punpcklbw %xmm1, %xmm1
test %RDX_LP, %RDX_LP
jz L(return_null)
@@ -60,13 +61,16 @@ ENTRY(MEMCHR)
test %eax, %eax
jnz L(matches_1)
- sub $16, %rdx
+ sub $CHAR_PER_VEC, %rdx
jbe L(return_null)
add $16, %rdi
and $15, %ecx
and $-16, %rdi
+#ifdef USE_AS_WMEMCHR
+ shr $2, %ecx
+#endif
add %rcx, %rdx
- sub $64, %rdx
+ sub $(CHAR_PER_VEC * 4), %rdx
jbe L(exit_loop)
jmp L(loop_prolog)
@@ -77,16 +81,21 @@ L(crosscache):
movdqa (%rdi), %xmm0
PCMPEQ %xmm1, %xmm0
-/* Check if there is a match. */
+ /* Check if there is a match. */
pmovmskb %xmm0, %eax
-/* Remove the leading bytes. */
+ /* Remove the leading bytes. */
sar %cl, %eax
test %eax, %eax
je L(unaligned_no_match)
-/* Check which byte is a match. */
+ /* Check which byte is a match. */
bsf %eax, %eax
-
+#ifdef USE_AS_WMEMCHR
+ mov %eax, %esi
+ shr $2, %esi
+ sub %rsi, %rdx
+#else
sub %rax, %rdx
+#endif
jbe L(return_null)
add %rdi, %rax
add %rcx, %rax
@@ -94,15 +103,18 @@ L(crosscache):
.p2align 4
L(unaligned_no_match):
- /* "rcx" is less than 16. Calculate "rdx + rcx - 16" by using
+ /* "rcx" is less than 16. Calculate "rdx + rcx - 16" by using
"rdx - (16 - rcx)" instead of "(rdx + rcx) - 16" to void
possible addition overflow. */
neg %rcx
add $16, %rcx
+#ifdef USE_AS_WMEMCHR
+ shr $2, %ecx
+#endif
sub %rcx, %rdx
jbe L(return_null)
add $16, %rdi
- sub $64, %rdx
+ sub $(CHAR_PER_VEC * 4), %rdx
jbe L(exit_loop)
.p2align 4
@@ -135,7 +147,7 @@ L(loop_prolog):
test $0x3f, %rdi
jz L(align64_loop)
- sub $64, %rdx
+ sub $(CHAR_PER_VEC * 4), %rdx
jbe L(exit_loop)
movdqa (%rdi), %xmm0
@@ -167,11 +179,14 @@ L(loop_prolog):
mov %rdi, %rcx
and $-64, %rdi
and $63, %ecx
+#ifdef USE_AS_WMEMCHR
+ shr $2, %ecx
+#endif
add %rcx, %rdx
.p2align 4
L(align64_loop):
- sub $64, %rdx
+ sub $(CHAR_PER_VEC * 4), %rdx
jbe L(exit_loop)
movdqa (%rdi), %xmm0
movdqa 16(%rdi), %xmm2
@@ -218,7 +233,7 @@ L(align64_loop):
.p2align 4
L(exit_loop):
- add $32, %edx
+ add $(CHAR_PER_VEC * 2), %edx
jle L(exit_loop_32)
movdqa (%rdi), %xmm0
@@ -238,7 +253,7 @@ L(exit_loop):
pmovmskb %xmm3, %eax
test %eax, %eax
jnz L(matches32_1)
- sub $16, %edx
+ sub $CHAR_PER_VEC, %edx
jle L(return_null)
PCMPEQ 48(%rdi), %xmm1
@@ -250,13 +265,13 @@ L(exit_loop):
.p2align 4
L(exit_loop_32):
- add $32, %edx
+ add $(CHAR_PER_VEC * 2), %edx
movdqa (%rdi), %xmm0
PCMPEQ %xmm1, %xmm0
pmovmskb %xmm0, %eax
test %eax, %eax
jnz L(matches_1)
- sub $16, %edx
+ sub $CHAR_PER_VEC, %edx
jbe L(return_null)
PCMPEQ 16(%rdi), %xmm1
@@ -293,7 +308,13 @@ L(matches32):
.p2align 4
L(matches_1):
bsf %eax, %eax
+#ifdef USE_AS_WMEMCHR
+ mov %eax, %esi
+ shr $2, %esi
+ sub %rsi, %rdx
+#else
sub %rax, %rdx
+#endif
jbe L(return_null)
add %rdi, %rax
ret
@@ -301,7 +322,13 @@ L(matches_1):
.p2align 4
L(matches16_1):
bsf %eax, %eax
+#ifdef USE_AS_WMEMCHR
+ mov %eax, %esi
+ shr $2, %esi
+ sub %rsi, %rdx
+#else
sub %rax, %rdx
+#endif
jbe L(return_null)
lea 16(%rdi, %rax), %rax
ret
@@ -309,7 +336,13 @@ L(matches16_1):
.p2align 4
L(matches32_1):
bsf %eax, %eax
+#ifdef USE_AS_WMEMCHR
+ mov %eax, %esi
+ shr $2, %esi
+ sub %rsi, %rdx
+#else
sub %rax, %rdx
+#endif
jbe L(return_null)
lea 32(%rdi, %rax), %rax
ret
@@ -317,7 +350,13 @@ L(matches32_1):
.p2align 4
L(matches48_1):
bsf %eax, %eax
+#ifdef USE_AS_WMEMCHR
+ mov %eax, %esi
+ shr $2, %esi
+ sub %rsi, %rdx
+#else
sub %rax, %rdx
+#endif
jbe L(return_null)
lea 48(%rdi, %rax), %rax
ret
diff --git a/sysdeps/x86_64/multiarch/memchr-avx2.S b/sysdeps/x86_64/multiarch/memchr-avx2.S
index b377f22e..16027abb 100644
--- a/sysdeps/x86_64/multiarch/memchr-avx2.S
+++ b/sysdeps/x86_64/multiarch/memchr-avx2.S
@@ -54,21 +54,19 @@
# define VEC_SIZE 32
# define PAGE_SIZE 4096
+# define CHAR_PER_VEC (VEC_SIZE / CHAR_SIZE)
.section SECTION(.text),"ax",@progbits
ENTRY (MEMCHR)
# ifndef USE_AS_RAWMEMCHR
/* Check for zero length. */
- test %RDX_LP, %RDX_LP
- jz L(null)
-# endif
-# ifdef USE_AS_WMEMCHR
- shl $2, %RDX_LP
-# else
# ifdef __ILP32__
- /* Clear the upper 32 bits. */
- movl %edx, %edx
+ /* Clear upper bits. */
+ and %RDX_LP, %RDX_LP
+# else
+ test %RDX_LP, %RDX_LP
# endif
+ jz L(null)
# endif
/* Broadcast CHAR to YMMMATCH. */
vmovd %esi, %xmm0
@@ -84,7 +82,7 @@ ENTRY (MEMCHR)
vpmovmskb %ymm1, %eax
# ifndef USE_AS_RAWMEMCHR
/* If length < CHAR_PER_VEC handle special. */
- cmpq $VEC_SIZE, %rdx
+ cmpq $CHAR_PER_VEC, %rdx
jbe L(first_vec_x0)
# endif
testl %eax, %eax
@@ -98,6 +96,10 @@ ENTRY (MEMCHR)
L(first_vec_x0):
/* Check if first match was before length. */
tzcntl %eax, %eax
+# ifdef USE_AS_WMEMCHR
+ /* NB: Multiply length by 4 to get byte count. */
+ sall $2, %edx
+# endif
xorl %ecx, %ecx
cmpl %eax, %edx
leaq (%rdi, %rax), %rax
@@ -110,12 +112,12 @@ L(null):
# endif
.p2align 4
L(cross_page_boundary):
- /* Save pointer before aligning as its original value is necessary
- for computer return address if byte is found or adjusting length
- if it is not and this is memchr. */
+ /* Save pointer before aligning as its original value is
+ necessary for computer return address if byte is found or
+ adjusting length if it is not and this is memchr. */
movq %rdi, %rcx
- /* Align data to VEC_SIZE - 1. ALGN_PTR_REG is rcx for memchr and
- rdi for rawmemchr. */
+ /* Align data to VEC_SIZE - 1. ALGN_PTR_REG is rcx for memchr
+ and rdi for rawmemchr. */
orq $(VEC_SIZE - 1), %ALGN_PTR_REG
VPCMPEQ -(VEC_SIZE - 1)(%ALGN_PTR_REG), %ymm0, %ymm1
vpmovmskb %ymm1, %eax
@@ -124,6 +126,10 @@ L(cross_page_boundary):
match). */
leaq 1(%ALGN_PTR_REG), %rsi
subq %RRAW_PTR_REG, %rsi
+# ifdef USE_AS_WMEMCHR
+ /* NB: Divide bytes by 4 to get wchar_t count. */
+ shrl $2, %esi
+# endif
# endif
/* Remove the leading bytes. */
sarxl %ERAW_PTR_REG, %eax, %eax
@@ -181,6 +187,10 @@ L(cross_page_continue):
orq $(VEC_SIZE - 1), %rdi
/* esi is for adjusting length to see if near the end. */
leal (VEC_SIZE * 4 + 1)(%rdi, %rcx), %esi
+# ifdef USE_AS_WMEMCHR
+ /* NB: Divide bytes by 4 to get the wchar_t count. */
+ sarl $2, %esi
+# endif
# else
orq $(VEC_SIZE - 1), %rdi
L(cross_page_continue):
@@ -213,7 +223,7 @@ L(cross_page_continue):
# ifndef USE_AS_RAWMEMCHR
/* Check if at last VEC_SIZE * 4 length. */
- subq $(VEC_SIZE * 4), %rdx
+ subq $(CHAR_PER_VEC * 4), %rdx
jbe L(last_4x_vec_or_less_cmpeq)
/* Align data to VEC_SIZE * 4 - 1 for the loop and readjust
length. */
@@ -221,6 +231,10 @@ L(cross_page_continue):
movl %edi, %ecx
orq $(VEC_SIZE * 4 - 1), %rdi
andl $(VEC_SIZE * 4 - 1), %ecx
+# ifdef USE_AS_WMEMCHR
+ /* NB: Divide bytes by 4 to get the wchar_t count. */
+ sarl $2, %ecx
+# endif
addq %rcx, %rdx
# else
/* Align data to VEC_SIZE * 4 - 1 for loop. */
@@ -250,15 +264,19 @@ L(loop_4x_vec):
subq $-(VEC_SIZE * 4), %rdi
- subq $(VEC_SIZE * 4), %rdx
+ subq $(CHAR_PER_VEC * 4), %rdx
ja L(loop_4x_vec)
- /* Fall through into less than 4 remaining vectors of length case.
- */
+ /* Fall through into less than 4 remaining vectors of length
+ case. */
VPCMPEQ (VEC_SIZE * 0 + 1)(%rdi), %ymm0, %ymm1
vpmovmskb %ymm1, %eax
.p2align 4
L(last_4x_vec_or_less):
+# ifdef USE_AS_WMEMCHR
+ /* NB: Multiply length by 4 to get byte count. */
+ sall $2, %edx
+# endif
/* Check if first VEC contained match. */
testl %eax, %eax
jnz L(first_vec_x1_check)
@@ -355,6 +373,10 @@ L(last_vec_x2_return):
L(last_4x_vec_or_less_cmpeq):
VPCMPEQ (VEC_SIZE * 4 + 1)(%rdi), %ymm0, %ymm1
vpmovmskb %ymm1, %eax
+# ifdef USE_AS_WMEMCHR
+ /* NB: Multiply length by 4 to get byte count. */
+ sall $2, %edx
+# endif
subq $-(VEC_SIZE * 4), %rdi
/* Check first VEC regardless. */
testl %eax, %eax
--
GitLab

View File

@ -1,767 +0,0 @@
From aaa23c35071537e2dcf5807e956802ed215210aa Mon Sep 17 00:00:00 2001
From: Noah Goldstein <goldstein.w.n@gmail.com>
Date: Mon, 19 Apr 2021 19:36:07 -0400
Subject: [PATCH] x86: Optimize strlen-avx2.S
Content-type: text/plain; charset=UTF-8
No bug. This commit optimizes strlen-avx2.S. The optimizations are
mostly small things but they add up to roughly 10-30% performance
improvement for strlen. The results for strnlen are bit more
ambiguous. test-strlen, test-strnlen, test-wcslen, and test-wcsnlen
are all passing.
Signed-off-by: Noah Goldstein <goldstein.w.n@gmail.com>
---
sysdeps/x86_64/multiarch/ifunc-impl-list.c | 16 +-
sysdeps/x86_64/multiarch/strlen-avx2.S | 532 +++++++++++++--------
2 files changed, 334 insertions(+), 214 deletions(-)
diff --git a/sysdeps/x86_64/multiarch/ifunc-impl-list.c b/sysdeps/x86_64/multiarch/ifunc-impl-list.c
index cbfc1a5d..f1a6460a 100644
--- a/sysdeps/x86_64/multiarch/ifunc-impl-list.c
+++ b/sysdeps/x86_64/multiarch/ifunc-impl-list.c
@@ -285,10 +285,12 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,
/* Support sysdeps/x86_64/multiarch/strlen.c. */
IFUNC_IMPL (i, name, strlen,
IFUNC_IMPL_ADD (array, i, strlen,
- CPU_FEATURE_USABLE (AVX2),
+ (CPU_FEATURE_USABLE (AVX2)
+ && CPU_FEATURE_USABLE (BMI2)),
__strlen_avx2)
IFUNC_IMPL_ADD (array, i, strlen,
(CPU_FEATURE_USABLE (AVX2)
+ && CPU_FEATURE_USABLE (BMI2)
&& CPU_FEATURE_USABLE (RTM)),
__strlen_avx2_rtm)
IFUNC_IMPL_ADD (array, i, strlen,
@@ -301,10 +303,12 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,
/* Support sysdeps/x86_64/multiarch/strnlen.c. */
IFUNC_IMPL (i, name, strnlen,
IFUNC_IMPL_ADD (array, i, strnlen,
- CPU_FEATURE_USABLE (AVX2),
+ (CPU_FEATURE_USABLE (AVX2)
+ && CPU_FEATURE_USABLE (BMI2)),
__strnlen_avx2)
IFUNC_IMPL_ADD (array, i, strnlen,
(CPU_FEATURE_USABLE (AVX2)
+ && CPU_FEATURE_USABLE (BMI2)
&& CPU_FEATURE_USABLE (RTM)),
__strnlen_avx2_rtm)
IFUNC_IMPL_ADD (array, i, strnlen,
@@ -640,10 +644,12 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,
/* Support sysdeps/x86_64/multiarch/wcslen.c. */
IFUNC_IMPL (i, name, wcslen,
IFUNC_IMPL_ADD (array, i, wcslen,
- CPU_FEATURE_USABLE (AVX2),
+ (CPU_FEATURE_USABLE (AVX2)
+ && CPU_FEATURE_USABLE (BMI2)),
__wcslen_avx2)
IFUNC_IMPL_ADD (array, i, wcslen,
(CPU_FEATURE_USABLE (AVX2)
+ && CPU_FEATURE_USABLE (BMI2)
&& CPU_FEATURE_USABLE (RTM)),
__wcslen_avx2_rtm)
IFUNC_IMPL_ADD (array, i, wcslen,
@@ -656,10 +662,12 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,
/* Support sysdeps/x86_64/multiarch/wcsnlen.c. */
IFUNC_IMPL (i, name, wcsnlen,
IFUNC_IMPL_ADD (array, i, wcsnlen,
- CPU_FEATURE_USABLE (AVX2),
+ (CPU_FEATURE_USABLE (AVX2)
+ && CPU_FEATURE_USABLE (BMI2)),
__wcsnlen_avx2)
IFUNC_IMPL_ADD (array, i, wcsnlen,
(CPU_FEATURE_USABLE (AVX2)
+ && CPU_FEATURE_USABLE (BMI2)
&& CPU_FEATURE_USABLE (RTM)),
__wcsnlen_avx2_rtm)
IFUNC_IMPL_ADD (array, i, wcsnlen,
diff --git a/sysdeps/x86_64/multiarch/strlen-avx2.S b/sysdeps/x86_64/multiarch/strlen-avx2.S
index 82826e10..be8a5db5 100644
--- a/sysdeps/x86_64/multiarch/strlen-avx2.S
+++ b/sysdeps/x86_64/multiarch/strlen-avx2.S
@@ -27,9 +27,11 @@
# ifdef USE_AS_WCSLEN
# define VPCMPEQ vpcmpeqd
# define VPMINU vpminud
+# define CHAR_SIZE 4
# else
# define VPCMPEQ vpcmpeqb
# define VPMINU vpminub
+# define CHAR_SIZE 1
# endif
# ifndef VZEROUPPER
@@ -41,349 +43,459 @@
# endif
# define VEC_SIZE 32
+# define PAGE_SIZE 4096
.section SECTION(.text),"ax",@progbits
ENTRY (STRLEN)
# ifdef USE_AS_STRNLEN
- /* Check for zero length. */
+ /* Check zero length. */
test %RSI_LP, %RSI_LP
jz L(zero)
+ /* Store max len in R8_LP before adjusting if using WCSLEN. */
+ mov %RSI_LP, %R8_LP
# ifdef USE_AS_WCSLEN
shl $2, %RSI_LP
# elif defined __ILP32__
/* Clear the upper 32 bits. */
movl %esi, %esi
# endif
- mov %RSI_LP, %R8_LP
# endif
- movl %edi, %ecx
+ movl %edi, %eax
movq %rdi, %rdx
vpxor %xmm0, %xmm0, %xmm0
-
+ /* Clear high bits from edi. Only keeping bits relevant to page
+ cross check. */
+ andl $(PAGE_SIZE - 1), %eax
/* Check if we may cross page boundary with one vector load. */
- andl $(2 * VEC_SIZE - 1), %ecx
- cmpl $VEC_SIZE, %ecx
- ja L(cros_page_boundary)
+ cmpl $(PAGE_SIZE - VEC_SIZE), %eax
+ ja L(cross_page_boundary)
/* Check the first VEC_SIZE bytes. */
- VPCMPEQ (%rdi), %ymm0, %ymm1
- vpmovmskb %ymm1, %eax
- testl %eax, %eax
-
+ VPCMPEQ (%rdi), %ymm0, %ymm1
+ vpmovmskb %ymm1, %eax
# ifdef USE_AS_STRNLEN
- jnz L(first_vec_x0_check)
- /* Adjust length and check the end of data. */
- subq $VEC_SIZE, %rsi
- jbe L(max)
-# else
- jnz L(first_vec_x0)
+ /* If length < VEC_SIZE handle special. */
+ cmpq $VEC_SIZE, %rsi
+ jbe L(first_vec_x0)
# endif
-
- /* Align data for aligned loads in the loop. */
- addq $VEC_SIZE, %rdi
- andl $(VEC_SIZE - 1), %ecx
- andq $-VEC_SIZE, %rdi
+ /* If empty continue to aligned_more. Otherwise return bit
+ position of first match. */
+ testl %eax, %eax
+ jz L(aligned_more)
+ tzcntl %eax, %eax
+# ifdef USE_AS_WCSLEN
+ shrl $2, %eax
+# endif
+ VZEROUPPER_RETURN
# ifdef USE_AS_STRNLEN
- /* Adjust length. */
- addq %rcx, %rsi
+L(zero):
+ xorl %eax, %eax
+ ret
- subq $(VEC_SIZE * 4), %rsi
- jbe L(last_4x_vec_or_less)
+ .p2align 4
+L(first_vec_x0):
+ /* Set bit for max len so that tzcnt will return min of max len
+ and position of first match. */
+ btsq %rsi, %rax
+ tzcntl %eax, %eax
+# ifdef USE_AS_WCSLEN
+ shrl $2, %eax
+# endif
+ VZEROUPPER_RETURN
# endif
- jmp L(more_4x_vec)
.p2align 4
-L(cros_page_boundary):
- andl $(VEC_SIZE - 1), %ecx
- andq $-VEC_SIZE, %rdi
- VPCMPEQ (%rdi), %ymm0, %ymm1
- vpmovmskb %ymm1, %eax
- /* Remove the leading bytes. */
- sarl %cl, %eax
- testl %eax, %eax
- jz L(aligned_more)
+L(first_vec_x1):
tzcntl %eax, %eax
+ /* Safe to use 32 bit instructions as these are only called for
+ size = [1, 159]. */
# ifdef USE_AS_STRNLEN
- /* Check the end of data. */
- cmpq %rax, %rsi
- jbe L(max)
+ /* Use ecx which was computed earlier to compute correct value.
+ */
+ subl $(VEC_SIZE * 4 + 1), %ecx
+ addl %ecx, %eax
+# else
+ subl %edx, %edi
+ incl %edi
+ addl %edi, %eax
# endif
- addq %rdi, %rax
- addq %rcx, %rax
- subq %rdx, %rax
# ifdef USE_AS_WCSLEN
- shrq $2, %rax
+ shrl $2, %eax
# endif
-L(return_vzeroupper):
- ZERO_UPPER_VEC_REGISTERS_RETURN
+ VZEROUPPER_RETURN
.p2align 4
-L(aligned_more):
+L(first_vec_x2):
+ tzcntl %eax, %eax
+ /* Safe to use 32 bit instructions as these are only called for
+ size = [1, 159]. */
# ifdef USE_AS_STRNLEN
- /* "rcx" is less than VEC_SIZE. Calculate "rdx + rcx - VEC_SIZE"
- with "rdx - (VEC_SIZE - rcx)" instead of "(rdx + rcx) - VEC_SIZE"
- to void possible addition overflow. */
- negq %rcx
- addq $VEC_SIZE, %rcx
-
- /* Check the end of data. */
- subq %rcx, %rsi
- jbe L(max)
+ /* Use ecx which was computed earlier to compute correct value.
+ */
+ subl $(VEC_SIZE * 3 + 1), %ecx
+ addl %ecx, %eax
+# else
+ subl %edx, %edi
+ addl $(VEC_SIZE + 1), %edi
+ addl %edi, %eax
# endif
+# ifdef USE_AS_WCSLEN
+ shrl $2, %eax
+# endif
+ VZEROUPPER_RETURN
- addq $VEC_SIZE, %rdi
+ .p2align 4
+L(first_vec_x3):
+ tzcntl %eax, %eax
+ /* Safe to use 32 bit instructions as these are only called for
+ size = [1, 159]. */
+# ifdef USE_AS_STRNLEN
+ /* Use ecx which was computed earlier to compute correct value.
+ */
+ subl $(VEC_SIZE * 2 + 1), %ecx
+ addl %ecx, %eax
+# else
+ subl %edx, %edi
+ addl $(VEC_SIZE * 2 + 1), %edi
+ addl %edi, %eax
+# endif
+# ifdef USE_AS_WCSLEN
+ shrl $2, %eax
+# endif
+ VZEROUPPER_RETURN
+ .p2align 4
+L(first_vec_x4):
+ tzcntl %eax, %eax
+ /* Safe to use 32 bit instructions as these are only called for
+ size = [1, 159]. */
# ifdef USE_AS_STRNLEN
- subq $(VEC_SIZE * 4), %rsi
- jbe L(last_4x_vec_or_less)
+ /* Use ecx which was computed earlier to compute correct value.
+ */
+ subl $(VEC_SIZE + 1), %ecx
+ addl %ecx, %eax
+# else
+ subl %edx, %edi
+ addl $(VEC_SIZE * 3 + 1), %edi
+ addl %edi, %eax
# endif
+# ifdef USE_AS_WCSLEN
+ shrl $2, %eax
+# endif
+ VZEROUPPER_RETURN
-L(more_4x_vec):
+ .p2align 5
+L(aligned_more):
+ /* Align data to VEC_SIZE - 1. This is the same number of
+ instructions as using andq with -VEC_SIZE but saves 4 bytes of
+ code on the x4 check. */
+ orq $(VEC_SIZE - 1), %rdi
+L(cross_page_continue):
/* Check the first 4 * VEC_SIZE. Only one VEC_SIZE at a time
since data is only aligned to VEC_SIZE. */
- VPCMPEQ (%rdi), %ymm0, %ymm1
- vpmovmskb %ymm1, %eax
- testl %eax, %eax
- jnz L(first_vec_x0)
-
- VPCMPEQ VEC_SIZE(%rdi), %ymm0, %ymm1
- vpmovmskb %ymm1, %eax
+# ifdef USE_AS_STRNLEN
+ /* + 1 because rdi is aligned to VEC_SIZE - 1. + CHAR_SIZE because
+ it simplies the logic in last_4x_vec_or_less. */
+ leaq (VEC_SIZE * 4 + CHAR_SIZE + 1)(%rdi), %rcx
+ subq %rdx, %rcx
+# endif
+ /* Load first VEC regardless. */
+ VPCMPEQ 1(%rdi), %ymm0, %ymm1
+# ifdef USE_AS_STRNLEN
+ /* Adjust length. If near end handle specially. */
+ subq %rcx, %rsi
+ jb L(last_4x_vec_or_less)
+# endif
+ vpmovmskb %ymm1, %eax
testl %eax, %eax
jnz L(first_vec_x1)
- VPCMPEQ (VEC_SIZE * 2)(%rdi), %ymm0, %ymm1
- vpmovmskb %ymm1, %eax
+ VPCMPEQ (VEC_SIZE + 1)(%rdi), %ymm0, %ymm1
+ vpmovmskb %ymm1, %eax
testl %eax, %eax
jnz L(first_vec_x2)
- VPCMPEQ (VEC_SIZE * 3)(%rdi), %ymm0, %ymm1
- vpmovmskb %ymm1, %eax
+ VPCMPEQ (VEC_SIZE * 2 + 1)(%rdi), %ymm0, %ymm1
+ vpmovmskb %ymm1, %eax
testl %eax, %eax
jnz L(first_vec_x3)
- addq $(VEC_SIZE * 4), %rdi
-
-# ifdef USE_AS_STRNLEN
- subq $(VEC_SIZE * 4), %rsi
- jbe L(last_4x_vec_or_less)
-# endif
-
- /* Align data to 4 * VEC_SIZE. */
- movq %rdi, %rcx
- andl $(4 * VEC_SIZE - 1), %ecx
- andq $-(4 * VEC_SIZE), %rdi
+ VPCMPEQ (VEC_SIZE * 3 + 1)(%rdi), %ymm0, %ymm1
+ vpmovmskb %ymm1, %eax
+ testl %eax, %eax
+ jnz L(first_vec_x4)
+ /* Align data to VEC_SIZE * 4 - 1. */
# ifdef USE_AS_STRNLEN
- /* Adjust length. */
+ /* Before adjusting length check if at last VEC_SIZE * 4. */
+ cmpq $(VEC_SIZE * 4 - 1), %rsi
+ jbe L(last_4x_vec_or_less_load)
+ incq %rdi
+ movl %edi, %ecx
+ orq $(VEC_SIZE * 4 - 1), %rdi
+ andl $(VEC_SIZE * 4 - 1), %ecx
+ /* Readjust length. */
addq %rcx, %rsi
+# else
+ incq %rdi
+ orq $(VEC_SIZE * 4 - 1), %rdi
# endif
-
+ /* Compare 4 * VEC at a time forward. */
.p2align 4
L(loop_4x_vec):
- /* Compare 4 * VEC at a time forward. */
- vmovdqa (%rdi), %ymm1
- vmovdqa VEC_SIZE(%rdi), %ymm2
- vmovdqa (VEC_SIZE * 2)(%rdi), %ymm3
- vmovdqa (VEC_SIZE * 3)(%rdi), %ymm4
- VPMINU %ymm1, %ymm2, %ymm5
- VPMINU %ymm3, %ymm4, %ymm6
- VPMINU %ymm5, %ymm6, %ymm5
-
- VPCMPEQ %ymm5, %ymm0, %ymm5
- vpmovmskb %ymm5, %eax
- testl %eax, %eax
- jnz L(4x_vec_end)
-
- addq $(VEC_SIZE * 4), %rdi
-
-# ifndef USE_AS_STRNLEN
- jmp L(loop_4x_vec)
-# else
+# ifdef USE_AS_STRNLEN
+ /* Break if at end of length. */
subq $(VEC_SIZE * 4), %rsi
- ja L(loop_4x_vec)
-
-L(last_4x_vec_or_less):
- /* Less than 4 * VEC and aligned to VEC_SIZE. */
- addl $(VEC_SIZE * 2), %esi
- jle L(last_2x_vec)
+ jb L(last_4x_vec_or_less_cmpeq)
+# endif
+ /* Save some code size by microfusing VPMINU with the load. Since
+ the matches in ymm2/ymm4 can only be returned if there where no
+ matches in ymm1/ymm3 respectively there is no issue with overlap.
+ */
+ vmovdqa 1(%rdi), %ymm1
+ VPMINU (VEC_SIZE + 1)(%rdi), %ymm1, %ymm2
+ vmovdqa (VEC_SIZE * 2 + 1)(%rdi), %ymm3
+ VPMINU (VEC_SIZE * 3 + 1)(%rdi), %ymm3, %ymm4
+
+ VPMINU %ymm2, %ymm4, %ymm5
+ VPCMPEQ %ymm5, %ymm0, %ymm5
+ vpmovmskb %ymm5, %ecx
- VPCMPEQ (%rdi), %ymm0, %ymm1
- vpmovmskb %ymm1, %eax
- testl %eax, %eax
- jnz L(first_vec_x0)
+ subq $-(VEC_SIZE * 4), %rdi
+ testl %ecx, %ecx
+ jz L(loop_4x_vec)
- VPCMPEQ VEC_SIZE(%rdi), %ymm0, %ymm1
- vpmovmskb %ymm1, %eax
- testl %eax, %eax
- jnz L(first_vec_x1)
- VPCMPEQ (VEC_SIZE * 2)(%rdi), %ymm0, %ymm1
- vpmovmskb %ymm1, %eax
+ VPCMPEQ %ymm1, %ymm0, %ymm1
+ vpmovmskb %ymm1, %eax
+ subq %rdx, %rdi
testl %eax, %eax
+ jnz L(last_vec_return_x0)
- jnz L(first_vec_x2_check)
- subl $VEC_SIZE, %esi
- jle L(max)
-
- VPCMPEQ (VEC_SIZE * 3)(%rdi), %ymm0, %ymm1
- vpmovmskb %ymm1, %eax
+ VPCMPEQ %ymm2, %ymm0, %ymm2
+ vpmovmskb %ymm2, %eax
testl %eax, %eax
-
- jnz L(first_vec_x3_check)
- movq %r8, %rax
-# ifdef USE_AS_WCSLEN
+ jnz L(last_vec_return_x1)
+
+ /* Combine last 2 VEC. */
+ VPCMPEQ %ymm3, %ymm0, %ymm3
+ vpmovmskb %ymm3, %eax
+ /* rcx has combined result from all 4 VEC. It will only be used if
+ the first 3 other VEC all did not contain a match. */
+ salq $32, %rcx
+ orq %rcx, %rax
+ tzcntq %rax, %rax
+ subq $(VEC_SIZE * 2 - 1), %rdi
+ addq %rdi, %rax
+# ifdef USE_AS_WCSLEN
shrq $2, %rax
-# endif
+# endif
VZEROUPPER_RETURN
+
+# ifdef USE_AS_STRNLEN
.p2align 4
-L(last_2x_vec):
- addl $(VEC_SIZE * 2), %esi
- VPCMPEQ (%rdi), %ymm0, %ymm1
- vpmovmskb %ymm1, %eax
- testl %eax, %eax
+L(last_4x_vec_or_less_load):
+ /* Depending on entry adjust rdi / prepare first VEC in ymm1. */
+ subq $-(VEC_SIZE * 4), %rdi
+L(last_4x_vec_or_less_cmpeq):
+ VPCMPEQ 1(%rdi), %ymm0, %ymm1
+L(last_4x_vec_or_less):
- jnz L(first_vec_x0_check)
- subl $VEC_SIZE, %esi
- jle L(max)
+ vpmovmskb %ymm1, %eax
+ /* If remaining length > VEC_SIZE * 2. This works if esi is off by
+ VEC_SIZE * 4. */
+ testl $(VEC_SIZE * 2), %esi
+ jnz L(last_4x_vec)
- VPCMPEQ VEC_SIZE(%rdi), %ymm0, %ymm1
- vpmovmskb %ymm1, %eax
+ /* length may have been negative or positive by an offset of
+ VEC_SIZE * 4 depending on where this was called from. This fixes
+ that. */
+ andl $(VEC_SIZE * 4 - 1), %esi
testl %eax, %eax
- jnz L(first_vec_x1_check)
- movq %r8, %rax
-# ifdef USE_AS_WCSLEN
- shrq $2, %rax
-# endif
- VZEROUPPER_RETURN
+ jnz L(last_vec_x1_check)
- .p2align 4
-L(first_vec_x0_check):
+ subl $VEC_SIZE, %esi
+ jb L(max)
+
+ VPCMPEQ (VEC_SIZE + 1)(%rdi), %ymm0, %ymm1
+ vpmovmskb %ymm1, %eax
tzcntl %eax, %eax
/* Check the end of data. */
- cmpq %rax, %rsi
- jbe L(max)
+ cmpl %eax, %esi
+ jb L(max)
+ subq %rdx, %rdi
+ addl $(VEC_SIZE + 1), %eax
addq %rdi, %rax
- subq %rdx, %rax
# ifdef USE_AS_WCSLEN
shrq $2, %rax
# endif
VZEROUPPER_RETURN
+# endif
.p2align 4
-L(first_vec_x1_check):
+L(last_vec_return_x0):
tzcntl %eax, %eax
- /* Check the end of data. */
- cmpq %rax, %rsi
- jbe L(max)
- addq $VEC_SIZE, %rax
+ subq $(VEC_SIZE * 4 - 1), %rdi
addq %rdi, %rax
- subq %rdx, %rax
-# ifdef USE_AS_WCSLEN
+# ifdef USE_AS_WCSLEN
shrq $2, %rax
-# endif
+# endif
VZEROUPPER_RETURN
.p2align 4
-L(first_vec_x2_check):
+L(last_vec_return_x1):
tzcntl %eax, %eax
- /* Check the end of data. */
- cmpq %rax, %rsi
- jbe L(max)
- addq $(VEC_SIZE * 2), %rax
+ subq $(VEC_SIZE * 3 - 1), %rdi
addq %rdi, %rax
- subq %rdx, %rax
-# ifdef USE_AS_WCSLEN
+# ifdef USE_AS_WCSLEN
shrq $2, %rax
-# endif
+# endif
VZEROUPPER_RETURN
+# ifdef USE_AS_STRNLEN
.p2align 4
-L(first_vec_x3_check):
+L(last_vec_x1_check):
+
tzcntl %eax, %eax
/* Check the end of data. */
- cmpq %rax, %rsi
- jbe L(max)
- addq $(VEC_SIZE * 3), %rax
+ cmpl %eax, %esi
+ jb L(max)
+ subq %rdx, %rdi
+ incl %eax
addq %rdi, %rax
- subq %rdx, %rax
# ifdef USE_AS_WCSLEN
shrq $2, %rax
# endif
VZEROUPPER_RETURN
- .p2align 4
L(max):
movq %r8, %rax
+ VZEROUPPER_RETURN
+
+ .p2align 4
+L(last_4x_vec):
+ /* Test first 2x VEC normally. */
+ testl %eax, %eax
+ jnz L(last_vec_x1)
+
+ VPCMPEQ (VEC_SIZE + 1)(%rdi), %ymm0, %ymm1
+ vpmovmskb %ymm1, %eax
+ testl %eax, %eax
+ jnz L(last_vec_x2)
+
+ /* Normalize length. */
+ andl $(VEC_SIZE * 4 - 1), %esi
+ VPCMPEQ (VEC_SIZE * 2 + 1)(%rdi), %ymm0, %ymm1
+ vpmovmskb %ymm1, %eax
+ testl %eax, %eax
+ jnz L(last_vec_x3)
+
+ subl $(VEC_SIZE * 3), %esi
+ jb L(max)
+
+ VPCMPEQ (VEC_SIZE * 3 + 1)(%rdi), %ymm0, %ymm1
+ vpmovmskb %ymm1, %eax
+ tzcntl %eax, %eax
+ /* Check the end of data. */
+ cmpl %eax, %esi
+ jb L(max)
+ subq %rdx, %rdi
+ addl $(VEC_SIZE * 3 + 1), %eax
+ addq %rdi, %rax
# ifdef USE_AS_WCSLEN
shrq $2, %rax
# endif
VZEROUPPER_RETURN
- .p2align 4
-L(zero):
- xorl %eax, %eax
- ret
-# endif
.p2align 4
-L(first_vec_x0):
+L(last_vec_x1):
+ /* essentially duplicates of first_vec_x1 but use 64 bit
+ instructions. */
tzcntl %eax, %eax
+ subq %rdx, %rdi
+ incl %eax
addq %rdi, %rax
- subq %rdx, %rax
-# ifdef USE_AS_WCSLEN
+# ifdef USE_AS_WCSLEN
shrq $2, %rax
-# endif
+# endif
VZEROUPPER_RETURN
.p2align 4
-L(first_vec_x1):
+L(last_vec_x2):
+ /* essentially duplicates of first_vec_x1 but use 64 bit
+ instructions. */
tzcntl %eax, %eax
- addq $VEC_SIZE, %rax
+ subq %rdx, %rdi
+ addl $(VEC_SIZE + 1), %eax
addq %rdi, %rax
- subq %rdx, %rax
-# ifdef USE_AS_WCSLEN
+# ifdef USE_AS_WCSLEN
shrq $2, %rax
-# endif
+# endif
VZEROUPPER_RETURN
.p2align 4
-L(first_vec_x2):
+L(last_vec_x3):
tzcntl %eax, %eax
- addq $(VEC_SIZE * 2), %rax
+ subl $(VEC_SIZE * 2), %esi
+ /* Check the end of data. */
+ cmpl %eax, %esi
+ jb L(max_end)
+ subq %rdx, %rdi
+ addl $(VEC_SIZE * 2 + 1), %eax
addq %rdi, %rax
- subq %rdx, %rax
-# ifdef USE_AS_WCSLEN
+# ifdef USE_AS_WCSLEN
shrq $2, %rax
-# endif
+# endif
+ VZEROUPPER_RETURN
+L(max_end):
+ movq %r8, %rax
VZEROUPPER_RETURN
+# endif
+ /* Cold case for crossing page with first load. */
.p2align 4
-L(4x_vec_end):
- VPCMPEQ %ymm1, %ymm0, %ymm1
- vpmovmskb %ymm1, %eax
- testl %eax, %eax
- jnz L(first_vec_x0)
- VPCMPEQ %ymm2, %ymm0, %ymm2
- vpmovmskb %ymm2, %eax
+L(cross_page_boundary):
+ /* Align data to VEC_SIZE - 1. */
+ orq $(VEC_SIZE - 1), %rdi
+ VPCMPEQ -(VEC_SIZE - 1)(%rdi), %ymm0, %ymm1
+ vpmovmskb %ymm1, %eax
+ /* Remove the leading bytes. sarxl only uses bits [5:0] of COUNT
+ so no need to manually mod rdx. */
+ sarxl %edx, %eax, %eax
+# ifdef USE_AS_STRNLEN
testl %eax, %eax
- jnz L(first_vec_x1)
- VPCMPEQ %ymm3, %ymm0, %ymm3
- vpmovmskb %ymm3, %eax
+ jnz L(cross_page_less_vec)
+ leaq 1(%rdi), %rcx
+ subq %rdx, %rcx
+ /* Check length. */
+ cmpq %rsi, %rcx
+ jb L(cross_page_continue)
+ movq %r8, %rax
+# else
testl %eax, %eax
- jnz L(first_vec_x2)
- VPCMPEQ %ymm4, %ymm0, %ymm4
- vpmovmskb %ymm4, %eax
-L(first_vec_x3):
+ jz L(cross_page_continue)
tzcntl %eax, %eax
- addq $(VEC_SIZE * 3), %rax
- addq %rdi, %rax
- subq %rdx, %rax
-# ifdef USE_AS_WCSLEN
- shrq $2, %rax
+# ifdef USE_AS_WCSLEN
+ shrl $2, %eax
+# endif
# endif
+L(return_vzeroupper):
+ ZERO_UPPER_VEC_REGISTERS_RETURN
+
+# ifdef USE_AS_STRNLEN
+ .p2align 4
+L(cross_page_less_vec):
+ tzcntl %eax, %eax
+ cmpq %rax, %rsi
+ cmovb %esi, %eax
+# ifdef USE_AS_WCSLEN
+ shrl $2, %eax
+# endif
VZEROUPPER_RETURN
+# endif
END (STRLEN)
#endif
--
GitLab

View File

@ -1,701 +0,0 @@
From 2a76821c3081d2c0231ecd2618f52662cb48fccd Mon Sep 17 00:00:00 2001
From: Noah Goldstein <goldstein.w.n@gmail.com>
Date: Mon, 3 May 2021 03:03:19 -0400
Subject: [PATCH] x86: Optimize memchr-evex.S
Content-type: text/plain; charset=UTF-8
No bug. This commit optimizes memchr-evex.S. The optimizations include
replacing some branches with cmovcc, avoiding some branches entirely
in the less_4x_vec case, making the page cross logic less strict,
saving some ALU in the alignment process, and most importantly
increasing ILP in the 4x loop. test-memchr, test-rawmemchr, and
test-wmemchr are all passing.
Signed-off-by: Noah Goldstein <goldstein.w.n@gmail.com>
Reviewed-by: H.J. Lu <hjl.tools@gmail.com>
---
sysdeps/x86_64/multiarch/memchr-evex.S | 547 +++++++++++++++----------
1 file changed, 322 insertions(+), 225 deletions(-)
diff --git a/sysdeps/x86_64/multiarch/memchr-evex.S b/sysdeps/x86_64/multiarch/memchr-evex.S
index 6dd5d67b..81d5cd64 100644
--- a/sysdeps/x86_64/multiarch/memchr-evex.S
+++ b/sysdeps/x86_64/multiarch/memchr-evex.S
@@ -26,14 +26,28 @@
# ifdef USE_AS_WMEMCHR
# define VPBROADCAST vpbroadcastd
-# define VPCMP vpcmpd
-# define SHIFT_REG r8d
+# define VPMINU vpminud
+# define VPCMP vpcmpd
+# define VPCMPEQ vpcmpeqd
+# define CHAR_SIZE 4
# else
# define VPBROADCAST vpbroadcastb
-# define VPCMP vpcmpb
-# define SHIFT_REG ecx
+# define VPMINU vpminub
+# define VPCMP vpcmpb
+# define VPCMPEQ vpcmpeqb
+# define CHAR_SIZE 1
# endif
+# ifdef USE_AS_RAWMEMCHR
+# define RAW_PTR_REG rcx
+# define ALGN_PTR_REG rdi
+# else
+# define RAW_PTR_REG rdi
+# define ALGN_PTR_REG rcx
+# endif
+
+# define XMMZERO xmm23
+# define YMMZERO ymm23
# define XMMMATCH xmm16
# define YMMMATCH ymm16
# define YMM1 ymm17
@@ -44,6 +58,8 @@
# define YMM6 ymm22
# define VEC_SIZE 32
+# define CHAR_PER_VEC (VEC_SIZE / CHAR_SIZE)
+# define PAGE_SIZE 4096
.section .text.evex,"ax",@progbits
ENTRY (MEMCHR)
@@ -51,11 +67,7 @@ ENTRY (MEMCHR)
/* Check for zero length. */
test %RDX_LP, %RDX_LP
jz L(zero)
-# endif
- movl %edi, %ecx
-# ifdef USE_AS_WMEMCHR
- shl $2, %RDX_LP
-# else
+
# ifdef __ILP32__
/* Clear the upper 32 bits. */
movl %edx, %edx
@@ -64,318 +76,403 @@ ENTRY (MEMCHR)
/* Broadcast CHAR to YMMMATCH. */
VPBROADCAST %esi, %YMMMATCH
/* Check if we may cross page boundary with one vector load. */
- andl $(2 * VEC_SIZE - 1), %ecx
- cmpl $VEC_SIZE, %ecx
- ja L(cros_page_boundary)
+ movl %edi, %eax
+ andl $(PAGE_SIZE - 1), %eax
+ cmpl $(PAGE_SIZE - VEC_SIZE), %eax
+ ja L(cross_page_boundary)
/* Check the first VEC_SIZE bytes. */
- VPCMP $0, (%rdi), %YMMMATCH, %k1
- kmovd %k1, %eax
- testl %eax, %eax
-
+ VPCMP $0, (%rdi), %YMMMATCH, %k0
+ kmovd %k0, %eax
# ifndef USE_AS_RAWMEMCHR
- jnz L(first_vec_x0_check)
- /* Adjust length and check the end of data. */
- subq $VEC_SIZE, %rdx
- jbe L(zero)
+ /* If length < CHAR_PER_VEC handle special. */
+ cmpq $CHAR_PER_VEC, %rdx
+ jbe L(first_vec_x0)
+# endif
+ testl %eax, %eax
+ jz L(aligned_more)
+ tzcntl %eax, %eax
+# ifdef USE_AS_WMEMCHR
+ /* NB: Multiply bytes by CHAR_SIZE to get the wchar_t count. */
+ leaq (%rdi, %rax, CHAR_SIZE), %rax
# else
- jnz L(first_vec_x0)
+ addq %rdi, %rax
# endif
-
- /* Align data for aligned loads in the loop. */
- addq $VEC_SIZE, %rdi
- andl $(VEC_SIZE - 1), %ecx
- andq $-VEC_SIZE, %rdi
+ ret
# ifndef USE_AS_RAWMEMCHR
- /* Adjust length. */
- addq %rcx, %rdx
-
- subq $(VEC_SIZE * 4), %rdx
- jbe L(last_4x_vec_or_less)
-# endif
- jmp L(more_4x_vec)
+L(zero):
+ xorl %eax, %eax
+ ret
+ .p2align 5
+L(first_vec_x0):
+ /* Check if first match was before length. */
+ tzcntl %eax, %eax
+ xorl %ecx, %ecx
+ cmpl %eax, %edx
+ leaq (%rdi, %rax, CHAR_SIZE), %rax
+ cmovle %rcx, %rax
+ ret
+# else
+ /* NB: first_vec_x0 is 17 bytes which will leave
+ cross_page_boundary (which is relatively cold) close enough
+ to ideal alignment. So only realign L(cross_page_boundary) if
+ rawmemchr. */
.p2align 4
-L(cros_page_boundary):
- andl $(VEC_SIZE - 1), %ecx
+# endif
+L(cross_page_boundary):
+ /* Save pointer before aligning as its original value is
+ necessary for computer return address if byte is found or
+ adjusting length if it is not and this is memchr. */
+ movq %rdi, %rcx
+ /* Align data to VEC_SIZE. ALGN_PTR_REG is rcx for memchr and rdi
+ for rawmemchr. */
+ andq $-VEC_SIZE, %ALGN_PTR_REG
+ VPCMP $0, (%ALGN_PTR_REG), %YMMMATCH, %k0
+ kmovd %k0, %r8d
# ifdef USE_AS_WMEMCHR
- /* NB: Divide shift count by 4 since each bit in K1 represent 4
+ /* NB: Divide shift count by 4 since each bit in K0 represent 4
bytes. */
- movl %ecx, %SHIFT_REG
- sarl $2, %SHIFT_REG
+ sarl $2, %eax
+# endif
+# ifndef USE_AS_RAWMEMCHR
+ movl $(PAGE_SIZE / CHAR_SIZE), %esi
+ subl %eax, %esi
# endif
- andq $-VEC_SIZE, %rdi
- VPCMP $0, (%rdi), %YMMMATCH, %k1
- kmovd %k1, %eax
- /* Remove the leading bytes. */
- sarxl %SHIFT_REG, %eax, %eax
- testl %eax, %eax
- jz L(aligned_more)
- tzcntl %eax, %eax
# ifdef USE_AS_WMEMCHR
- /* NB: Multiply wchar_t count by 4 to get the number of bytes. */
- sall $2, %eax
+ andl $(CHAR_PER_VEC - 1), %eax
# endif
+ /* Remove the leading bytes. */
+ sarxl %eax, %r8d, %eax
# ifndef USE_AS_RAWMEMCHR
/* Check the end of data. */
- cmpq %rax, %rdx
- jbe L(zero)
+ cmpq %rsi, %rdx
+ jbe L(first_vec_x0)
+# endif
+ testl %eax, %eax
+ jz L(cross_page_continue)
+ tzcntl %eax, %eax
+# ifdef USE_AS_WMEMCHR
+ /* NB: Multiply bytes by CHAR_SIZE to get the wchar_t count. */
+ leaq (%RAW_PTR_REG, %rax, CHAR_SIZE), %rax
+# else
+ addq %RAW_PTR_REG, %rax
# endif
- addq %rdi, %rax
- addq %rcx, %rax
ret
.p2align 4
-L(aligned_more):
-# ifndef USE_AS_RAWMEMCHR
- /* Calculate "rdx + rcx - VEC_SIZE" with "rdx - (VEC_SIZE - rcx)"
- instead of "(rdx + rcx) - VEC_SIZE" to void possible addition
- overflow. */
- negq %rcx
- addq $VEC_SIZE, %rcx
+L(first_vec_x1):
+ tzcntl %eax, %eax
+ leaq VEC_SIZE(%rdi, %rax, CHAR_SIZE), %rax
+ ret
- /* Check the end of data. */
- subq %rcx, %rdx
- jbe L(zero)
-# endif
+ .p2align 4
+L(first_vec_x2):
+ tzcntl %eax, %eax
+ leaq (VEC_SIZE * 2)(%rdi, %rax, CHAR_SIZE), %rax
+ ret
- addq $VEC_SIZE, %rdi
+ .p2align 4
+L(first_vec_x3):
+ tzcntl %eax, %eax
+ leaq (VEC_SIZE * 3)(%rdi, %rax, CHAR_SIZE), %rax
+ ret
-# ifndef USE_AS_RAWMEMCHR
- subq $(VEC_SIZE * 4), %rdx
- jbe L(last_4x_vec_or_less)
-# endif
+ .p2align 4
+L(first_vec_x4):
+ tzcntl %eax, %eax
+ leaq (VEC_SIZE * 4)(%rdi, %rax, CHAR_SIZE), %rax
+ ret
-L(more_4x_vec):
+ .p2align 5
+L(aligned_more):
/* Check the first 4 * VEC_SIZE. Only one VEC_SIZE at a time
since data is only aligned to VEC_SIZE. */
- VPCMP $0, (%rdi), %YMMMATCH, %k1
- kmovd %k1, %eax
- testl %eax, %eax
- jnz L(first_vec_x0)
- VPCMP $0, VEC_SIZE(%rdi), %YMMMATCH, %k1
- kmovd %k1, %eax
+# ifndef USE_AS_RAWMEMCHR
+ /* Align data to VEC_SIZE. */
+L(cross_page_continue):
+ xorl %ecx, %ecx
+ subl %edi, %ecx
+ andq $-VEC_SIZE, %rdi
+ /* esi is for adjusting length to see if near the end. */
+ leal (VEC_SIZE * 5)(%rdi, %rcx), %esi
+# ifdef USE_AS_WMEMCHR
+ /* NB: Divide bytes by 4 to get the wchar_t count. */
+ sarl $2, %esi
+# endif
+# else
+ andq $-VEC_SIZE, %rdi
+L(cross_page_continue):
+# endif
+ /* Load first VEC regardless. */
+ VPCMP $0, (VEC_SIZE)(%rdi), %YMMMATCH, %k0
+ kmovd %k0, %eax
+# ifndef USE_AS_RAWMEMCHR
+ /* Adjust length. If near end handle specially. */
+ subq %rsi, %rdx
+ jbe L(last_4x_vec_or_less)
+# endif
testl %eax, %eax
jnz L(first_vec_x1)
- VPCMP $0, (VEC_SIZE * 2)(%rdi), %YMMMATCH, %k1
- kmovd %k1, %eax
+ VPCMP $0, (VEC_SIZE * 2)(%rdi), %YMMMATCH, %k0
+ kmovd %k0, %eax
testl %eax, %eax
jnz L(first_vec_x2)
- VPCMP $0, (VEC_SIZE * 3)(%rdi), %YMMMATCH, %k1
- kmovd %k1, %eax
+ VPCMP $0, (VEC_SIZE * 3)(%rdi), %YMMMATCH, %k0
+ kmovd %k0, %eax
testl %eax, %eax
jnz L(first_vec_x3)
- addq $(VEC_SIZE * 4), %rdi
+ VPCMP $0, (VEC_SIZE * 4)(%rdi), %YMMMATCH, %k0
+ kmovd %k0, %eax
+ testl %eax, %eax
+ jnz L(first_vec_x4)
+
# ifndef USE_AS_RAWMEMCHR
- subq $(VEC_SIZE * 4), %rdx
- jbe L(last_4x_vec_or_less)
-# endif
+ /* Check if at last CHAR_PER_VEC * 4 length. */
+ subq $(CHAR_PER_VEC * 4), %rdx
+ jbe L(last_4x_vec_or_less_cmpeq)
+ addq $VEC_SIZE, %rdi
- /* Align data to 4 * VEC_SIZE. */
- movq %rdi, %rcx
- andl $(4 * VEC_SIZE - 1), %ecx
+ /* Align data to VEC_SIZE * 4 for the loop and readjust length.
+ */
+# ifdef USE_AS_WMEMCHR
+ movl %edi, %ecx
andq $-(4 * VEC_SIZE), %rdi
-
-# ifndef USE_AS_RAWMEMCHR
- /* Adjust length. */
+ andl $(VEC_SIZE * 4 - 1), %ecx
+ /* NB: Divide bytes by 4 to get the wchar_t count. */
+ sarl $2, %ecx
addq %rcx, %rdx
+# else
+ addq %rdi, %rdx
+ andq $-(4 * VEC_SIZE), %rdi
+ subq %rdi, %rdx
+# endif
+# else
+ addq $VEC_SIZE, %rdi
+ andq $-(4 * VEC_SIZE), %rdi
# endif
+ vpxorq %XMMZERO, %XMMZERO, %XMMZERO
+
+ /* Compare 4 * VEC at a time forward. */
.p2align 4
L(loop_4x_vec):
- /* Compare 4 * VEC at a time forward. */
- VPCMP $0, (%rdi), %YMMMATCH, %k1
- VPCMP $0, VEC_SIZE(%rdi), %YMMMATCH, %k2
- kord %k1, %k2, %k5
- VPCMP $0, (VEC_SIZE * 2)(%rdi), %YMMMATCH, %k3
- VPCMP $0, (VEC_SIZE * 3)(%rdi), %YMMMATCH, %k4
-
- kord %k3, %k4, %k6
- kortestd %k5, %k6
- jnz L(4x_vec_end)
-
- addq $(VEC_SIZE * 4), %rdi
-
+ /* It would be possible to save some instructions using 4x VPCMP
+ but bottleneck on port 5 makes it not woth it. */
+ VPCMP $4, (VEC_SIZE * 4)(%rdi), %YMMMATCH, %k1
+ /* xor will set bytes match esi to zero. */
+ vpxorq (VEC_SIZE * 5)(%rdi), %YMMMATCH, %YMM2
+ vpxorq (VEC_SIZE * 6)(%rdi), %YMMMATCH, %YMM3
+ VPCMP $0, (VEC_SIZE * 7)(%rdi), %YMMMATCH, %k3
+ /* Reduce VEC2 / VEC3 with min and VEC1 with zero mask. */
+ VPMINU %YMM2, %YMM3, %YMM3 {%k1} {z}
+ VPCMP $0, %YMM3, %YMMZERO, %k2
# ifdef USE_AS_RAWMEMCHR
- jmp L(loop_4x_vec)
+ subq $-(VEC_SIZE * 4), %rdi
+ kortestd %k2, %k3
+ jz L(loop_4x_vec)
# else
- subq $(VEC_SIZE * 4), %rdx
+ kortestd %k2, %k3
+ jnz L(loop_4x_vec_end)
+
+ subq $-(VEC_SIZE * 4), %rdi
+
+ subq $(CHAR_PER_VEC * 4), %rdx
ja L(loop_4x_vec)
+ /* Fall through into less than 4 remaining vectors of length case.
+ */
+ VPCMP $0, (VEC_SIZE * 4)(%rdi), %YMMMATCH, %k0
+ kmovd %k0, %eax
+ addq $(VEC_SIZE * 3), %rdi
+ .p2align 4
L(last_4x_vec_or_less):
- /* Less than 4 * VEC and aligned to VEC_SIZE. */
- addl $(VEC_SIZE * 2), %edx
- jle L(last_2x_vec)
-
- VPCMP $0, (%rdi), %YMMMATCH, %k1
- kmovd %k1, %eax
+ /* Check if first VEC contained match. */
testl %eax, %eax
- jnz L(first_vec_x0)
+ jnz L(first_vec_x1_check)
- VPCMP $0, VEC_SIZE(%rdi), %YMMMATCH, %k1
- kmovd %k1, %eax
- testl %eax, %eax
- jnz L(first_vec_x1)
+ /* If remaining length > CHAR_PER_VEC * 2. */
+ addl $(CHAR_PER_VEC * 2), %edx
+ jg L(last_4x_vec)
- VPCMP $0, (VEC_SIZE * 2)(%rdi), %YMMMATCH, %k1
- kmovd %k1, %eax
- testl %eax, %eax
+L(last_2x_vec):
+ /* If remaining length < CHAR_PER_VEC. */
+ addl $CHAR_PER_VEC, %edx
+ jle L(zero_end)
- jnz L(first_vec_x2_check)
- subl $VEC_SIZE, %edx
- jle L(zero)
+ /* Check VEC2 and compare any match with remaining length. */
+ VPCMP $0, (VEC_SIZE * 2)(%rdi), %YMMMATCH, %k0
+ kmovd %k0, %eax
+ tzcntl %eax, %eax
+ cmpl %eax, %edx
+ jbe L(set_zero_end)
+ leaq (VEC_SIZE * 2)(%rdi, %rax, CHAR_SIZE), %rax
+L(zero_end):
+ ret
- VPCMP $0, (VEC_SIZE * 3)(%rdi), %YMMMATCH, %k1
- kmovd %k1, %eax
- testl %eax, %eax
- jnz L(first_vec_x3_check)
+ .p2align 4
+L(first_vec_x1_check):
+ tzcntl %eax, %eax
+ /* Adjust length. */
+ subl $-(CHAR_PER_VEC * 4), %edx
+ /* Check if match within remaining length. */
+ cmpl %eax, %edx
+ jbe L(set_zero_end)
+ /* NB: Multiply bytes by CHAR_SIZE to get the wchar_t count. */
+ leaq VEC_SIZE(%rdi, %rax, CHAR_SIZE), %rax
+ ret
+L(set_zero_end):
xorl %eax, %eax
ret
.p2align 4
-L(last_2x_vec):
- addl $(VEC_SIZE * 2), %edx
- VPCMP $0, (%rdi), %YMMMATCH, %k1
+L(loop_4x_vec_end):
+# endif
+ /* rawmemchr will fall through into this if match was found in
+ loop. */
+
+ /* k1 has not of matches with VEC1. */
kmovd %k1, %eax
- testl %eax, %eax
+# ifdef USE_AS_WMEMCHR
+ subl $((1 << CHAR_PER_VEC) - 1), %eax
+# else
+ incl %eax
+# endif
+ jnz L(last_vec_x1_return)
- jnz L(first_vec_x0_check)
- subl $VEC_SIZE, %edx
- jle L(zero)
+ VPCMP $0, %YMM2, %YMMZERO, %k0
+ kmovd %k0, %eax
+ testl %eax, %eax
+ jnz L(last_vec_x2_return)
- VPCMP $0, VEC_SIZE(%rdi), %YMMMATCH, %k1
- kmovd %k1, %eax
+ kmovd %k2, %eax
testl %eax, %eax
- jnz L(first_vec_x1_check)
- xorl %eax, %eax
- ret
+ jnz L(last_vec_x3_return)
- .p2align 4
-L(first_vec_x0_check):
+ kmovd %k3, %eax
tzcntl %eax, %eax
-# ifdef USE_AS_WMEMCHR
- /* NB: Multiply wchar_t count by 4 to get the number of bytes. */
- sall $2, %eax
+# ifdef USE_AS_RAWMEMCHR
+ leaq (VEC_SIZE * 3)(%rdi, %rax, CHAR_SIZE), %rax
+# else
+ leaq (VEC_SIZE * 7)(%rdi, %rax, CHAR_SIZE), %rax
# endif
- /* Check the end of data. */
- cmpq %rax, %rdx
- jbe L(zero)
- addq %rdi, %rax
ret
.p2align 4
-L(first_vec_x1_check):
+L(last_vec_x1_return):
tzcntl %eax, %eax
-# ifdef USE_AS_WMEMCHR
- /* NB: Multiply wchar_t count by 4 to get the number of bytes. */
- sall $2, %eax
-# endif
- /* Check the end of data. */
- cmpq %rax, %rdx
- jbe L(zero)
- addq $VEC_SIZE, %rax
+# ifdef USE_AS_RAWMEMCHR
+# ifdef USE_AS_WMEMCHR
+ /* NB: Multiply bytes by CHAR_SIZE to get the wchar_t count. */
+ leaq (%rdi, %rax, CHAR_SIZE), %rax
+# else
addq %rdi, %rax
- ret
-
- .p2align 4
-L(first_vec_x2_check):
- tzcntl %eax, %eax
-# ifdef USE_AS_WMEMCHR
- /* NB: Multiply wchar_t count by 4 to get the number of bytes. */
- sall $2, %eax
+# endif
+# else
+ /* NB: Multiply bytes by CHAR_SIZE to get the wchar_t count. */
+ leaq (VEC_SIZE * 4)(%rdi, %rax, CHAR_SIZE), %rax
# endif
- /* Check the end of data. */
- cmpq %rax, %rdx
- jbe L(zero)
- addq $(VEC_SIZE * 2), %rax
- addq %rdi, %rax
ret
.p2align 4
-L(first_vec_x3_check):
+L(last_vec_x2_return):
tzcntl %eax, %eax
-# ifdef USE_AS_WMEMCHR
- /* NB: Multiply wchar_t count by 4 to get the number of bytes. */
- sall $2, %eax
+# ifdef USE_AS_RAWMEMCHR
+ /* NB: Multiply bytes by CHAR_SIZE to get the wchar_t count. */
+ leaq VEC_SIZE(%rdi, %rax, CHAR_SIZE), %rax
+# else
+ /* NB: Multiply bytes by CHAR_SIZE to get the wchar_t count. */
+ leaq (VEC_SIZE * 5)(%rdi, %rax, CHAR_SIZE), %rax
# endif
- /* Check the end of data. */
- cmpq %rax, %rdx
- jbe L(zero)
- addq $(VEC_SIZE * 3), %rax
- addq %rdi, %rax
ret
.p2align 4
-L(zero):
- xorl %eax, %eax
- ret
-# endif
-
- .p2align 4
-L(first_vec_x0):
+L(last_vec_x3_return):
tzcntl %eax, %eax
-# ifdef USE_AS_WMEMCHR
- /* NB: Multiply wchar_t count by 4 to get the number of bytes. */
- leaq (%rdi, %rax, 4), %rax
+# ifdef USE_AS_RAWMEMCHR
+ /* NB: Multiply bytes by CHAR_SIZE to get the wchar_t count. */
+ leaq (VEC_SIZE * 2)(%rdi, %rax, CHAR_SIZE), %rax
# else
- addq %rdi, %rax
+ /* NB: Multiply bytes by CHAR_SIZE to get the wchar_t count. */
+ leaq (VEC_SIZE * 6)(%rdi, %rax, CHAR_SIZE), %rax
# endif
ret
+
+# ifndef USE_AS_RAWMEMCHR
+L(last_4x_vec_or_less_cmpeq):
+ VPCMP $0, (VEC_SIZE * 5)(%rdi), %YMMMATCH, %k0
+ kmovd %k0, %eax
+ subq $-(VEC_SIZE * 4), %rdi
+ /* Check first VEC regardless. */
+ testl %eax, %eax
+ jnz L(first_vec_x1_check)
+
+ /* If remaining length <= CHAR_PER_VEC * 2. */
+ addl $(CHAR_PER_VEC * 2), %edx
+ jle L(last_2x_vec)
+
.p2align 4
-L(first_vec_x1):
+L(last_4x_vec):
+ VPCMP $0, (VEC_SIZE * 2)(%rdi), %YMMMATCH, %k0
+ kmovd %k0, %eax
+ testl %eax, %eax
+ jnz L(last_vec_x2)
+
+
+ VPCMP $0, (VEC_SIZE * 3)(%rdi), %YMMMATCH, %k0
+ kmovd %k0, %eax
+ /* Create mask for possible matches within remaining length. */
+# ifdef USE_AS_WMEMCHR
+ movl $((1 << (CHAR_PER_VEC * 2)) - 1), %ecx
+ bzhil %edx, %ecx, %ecx
+# else
+ movq $-1, %rcx
+ bzhiq %rdx, %rcx, %rcx
+# endif
+ /* Test matches in data against length match. */
+ andl %ecx, %eax
+ jnz L(last_vec_x3)
+
+ /* if remaining length <= CHAR_PER_VEC * 3 (Note this is after
+ remaining length was found to be > CHAR_PER_VEC * 2. */
+ subl $CHAR_PER_VEC, %edx
+ jbe L(zero_end2)
+
+
+ VPCMP $0, (VEC_SIZE * 4)(%rdi), %YMMMATCH, %k0
+ kmovd %k0, %eax
+ /* Shift remaining length mask for last VEC. */
+# ifdef USE_AS_WMEMCHR
+ shrl $CHAR_PER_VEC, %ecx
+# else
+ shrq $CHAR_PER_VEC, %rcx
+# endif
+ andl %ecx, %eax
+ jz L(zero_end2)
tzcntl %eax, %eax
-# ifdef USE_AS_WMEMCHR
- /* NB: Multiply wchar_t count by 4 to get the number of bytes. */
- leaq VEC_SIZE(%rdi, %rax, 4), %rax
-# else
- addq $VEC_SIZE, %rax
- addq %rdi, %rax
-# endif
+ leaq (VEC_SIZE * 4)(%rdi, %rax, CHAR_SIZE), %rax
+L(zero_end2):
ret
- .p2align 4
-L(first_vec_x2):
+L(last_vec_x2):
tzcntl %eax, %eax
-# ifdef USE_AS_WMEMCHR
- /* NB: Multiply wchar_t count by 4 to get the number of bytes. */
- leaq (VEC_SIZE * 2)(%rdi, %rax, 4), %rax
-# else
- addq $(VEC_SIZE * 2), %rax
- addq %rdi, %rax
-# endif
+ leaq (VEC_SIZE * 2)(%rdi, %rax, CHAR_SIZE), %rax
ret
.p2align 4
-L(4x_vec_end):
- kmovd %k1, %eax
- testl %eax, %eax
- jnz L(first_vec_x0)
- kmovd %k2, %eax
- testl %eax, %eax
- jnz L(first_vec_x1)
- kmovd %k3, %eax
- testl %eax, %eax
- jnz L(first_vec_x2)
- kmovd %k4, %eax
- testl %eax, %eax
-L(first_vec_x3):
+L(last_vec_x3):
tzcntl %eax, %eax
-# ifdef USE_AS_WMEMCHR
- /* NB: Multiply wchar_t count by 4 to get the number of bytes. */
- leaq (VEC_SIZE * 3)(%rdi, %rax, 4), %rax
-# else
- addq $(VEC_SIZE * 3), %rax
- addq %rdi, %rax
-# endif
+ leaq (VEC_SIZE * 3)(%rdi, %rax, CHAR_SIZE), %rax
ret
+# endif
END (MEMCHR)
#endif
--
GitLab

View File

@ -1,30 +0,0 @@
From 6ea916adfa0ab9af6e7dc6adcf6f977dfe017835 Mon Sep 17 00:00:00 2001
From: Alice Xu <alice.d.xu@gmail.com>
Date: Fri, 7 May 2021 19:03:21 -0700
Subject: [PATCH] x86-64: Fix an unknown vector operation in memchr-evex.S
Content-type: text/plain; charset=UTF-8
An unknown vector operation occurred in commit 2a76821c308. Fixed it
by using "ymm{k1}{z}" but not "ymm {k1} {z}".
Reviewed-by: H.J. Lu <hjl.tools@gmail.com>
---
sysdeps/x86_64/multiarch/memchr-evex.S | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/sysdeps/x86_64/multiarch/memchr-evex.S b/sysdeps/x86_64/multiarch/memchr-evex.S
index 81d5cd64..f3fdad4f 100644
--- a/sysdeps/x86_64/multiarch/memchr-evex.S
+++ b/sysdeps/x86_64/multiarch/memchr-evex.S
@@ -271,7 +271,7 @@ L(loop_4x_vec):
vpxorq (VEC_SIZE * 6)(%rdi), %YMMMATCH, %YMM3
VPCMP $0, (VEC_SIZE * 7)(%rdi), %YMMMATCH, %k3
/* Reduce VEC2 / VEC3 with min and VEC1 with zero mask. */
- VPMINU %YMM2, %YMM3, %YMM3 {%k1} {z}
+ VPMINU %YMM2, %YMM3, %YMM3{%k1}{z}
VPCMP $0, %YMM3, %YMMZERO, %k2
# ifdef USE_AS_RAWMEMCHR
subq $-(VEC_SIZE * 4), %rdi
--
GitLab

View File

@ -1,566 +0,0 @@
From a0db678071c60b6c47c468d231dd0b3694ba7a98 Mon Sep 17 00:00:00 2001
From: "H.J. Lu" <hjl.tools@gmail.com>
Date: Tue, 22 Jun 2021 20:42:10 -0700
Subject: [PATCH] x86-64: Move strlen.S to multiarch/strlen-vec.S
Content-type: text/plain; charset=UTF-8
Since strlen.S contains SSE2 version of strlen/strnlen and SSE4.1
version of wcslen/wcsnlen, move strlen.S to multiarch/strlen-vec.S
and include multiarch/strlen-vec.S from SSE2 and SSE4.1 variants.
This also removes the unused symbols, __GI___strlen_sse2 and
__GI___wcsnlen_sse4_1.
---
sysdeps/x86_64/multiarch/strlen-sse2.S | 2 +-
sysdeps/x86_64/multiarch/strlen-vec.S | 257 ++++++++++++++++++++++
sysdeps/x86_64/multiarch/wcsnlen-sse4_1.S | 2 +-
sysdeps/x86_64/strlen.S | 243 +-------------------
4 files changed, 262 insertions(+), 242 deletions(-)
create mode 100644 sysdeps/x86_64/multiarch/strlen-vec.S
Conflicts:
sysdeps/x86_64/strlen.S b/sysdeps/x86_64/strlen.S
(Copyright dates, URL)
diff --git a/sysdeps/x86_64/multiarch/strlen-sse2.S b/sysdeps/x86_64/multiarch/strlen-sse2.S
index 7bc57b8d..449c8a7f 100644
--- a/sysdeps/x86_64/multiarch/strlen-sse2.S
+++ b/sysdeps/x86_64/multiarch/strlen-sse2.S
@@ -20,4 +20,4 @@
# define strlen __strlen_sse2
#endif
-#include "../strlen.S"
+#include "strlen-vec.S"
diff --git a/sysdeps/x86_64/multiarch/strlen-vec.S b/sysdeps/x86_64/multiarch/strlen-vec.S
new file mode 100644
index 00000000..8f660bb9
--- /dev/null
+++ b/sysdeps/x86_64/multiarch/strlen-vec.S
@@ -0,0 +1,257 @@
+/* SSE2 version of strlen and SSE4.1 version of wcslen.
+ Copyright (C) 2012-2021 Free Software Foundation, Inc.
+ This file is part of the GNU C Library.
+
+ The GNU C Library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ The GNU C Library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with the GNU C Library; if not, see
+ <https://www.gnu.org/licenses/>. */
+
+#include <sysdep.h>
+
+#ifdef AS_WCSLEN
+# define PMINU pminud
+# define PCMPEQ pcmpeqd
+# define SHIFT_RETURN shrq $2, %rax
+#else
+# define PMINU pminub
+# define PCMPEQ pcmpeqb
+# define SHIFT_RETURN
+#endif
+
+/* Long lived register in strlen(s), strnlen(s, n) are:
+
+ %xmm3 - zero
+ %rdi - s
+ %r10 (s+n) & (~(64-1))
+ %r11 s+n
+*/
+
+
+.text
+ENTRY(strlen)
+
+/* Test 64 bytes from %rax for zero. Save result as bitmask in %rdx. */
+#define FIND_ZERO \
+ PCMPEQ (%rax), %xmm0; \
+ PCMPEQ 16(%rax), %xmm1; \
+ PCMPEQ 32(%rax), %xmm2; \
+ PCMPEQ 48(%rax), %xmm3; \
+ pmovmskb %xmm0, %esi; \
+ pmovmskb %xmm1, %edx; \
+ pmovmskb %xmm2, %r8d; \
+ pmovmskb %xmm3, %ecx; \
+ salq $16, %rdx; \
+ salq $16, %rcx; \
+ orq %rsi, %rdx; \
+ orq %r8, %rcx; \
+ salq $32, %rcx; \
+ orq %rcx, %rdx;
+
+#ifdef AS_STRNLEN
+/* Do not read anything when n==0. */
+ test %RSI_LP, %RSI_LP
+ jne L(n_nonzero)
+ xor %rax, %rax
+ ret
+L(n_nonzero):
+# ifdef AS_WCSLEN
+ shl $2, %RSI_LP
+# endif
+
+/* Initialize long lived registers. */
+
+ add %RDI_LP, %RSI_LP
+ mov %RSI_LP, %R10_LP
+ and $-64, %R10_LP
+ mov %RSI_LP, %R11_LP
+#endif
+
+ pxor %xmm0, %xmm0
+ pxor %xmm1, %xmm1
+ pxor %xmm2, %xmm2
+ pxor %xmm3, %xmm3
+ movq %rdi, %rax
+ movq %rdi, %rcx
+ andq $4095, %rcx
+/* Offsets 4032-4047 will be aligned into 4032 thus fit into page. */
+ cmpq $4047, %rcx
+/* We cannot unify this branching as it would be ~6 cycles slower. */
+ ja L(cross_page)
+
+#ifdef AS_STRNLEN
+/* Test if end is among first 64 bytes. */
+# define STRNLEN_PROLOG \
+ mov %r11, %rsi; \
+ subq %rax, %rsi; \
+ andq $-64, %rax; \
+ testq $-64, %rsi; \
+ je L(strnlen_ret)
+#else
+# define STRNLEN_PROLOG andq $-64, %rax;
+#endif
+
+/* Ignore bits in mask that come before start of string. */
+#define PROLOG(lab) \
+ movq %rdi, %rcx; \
+ xorq %rax, %rcx; \
+ STRNLEN_PROLOG; \
+ sarq %cl, %rdx; \
+ test %rdx, %rdx; \
+ je L(lab); \
+ bsfq %rdx, %rax; \
+ SHIFT_RETURN; \
+ ret
+
+#ifdef AS_STRNLEN
+ andq $-16, %rax
+ FIND_ZERO
+#else
+ /* Test first 16 bytes unaligned. */
+ movdqu (%rax), %xmm4
+ PCMPEQ %xmm0, %xmm4
+ pmovmskb %xmm4, %edx
+ test %edx, %edx
+ je L(next48_bytes)
+ bsf %edx, %eax /* If eax is zeroed 16bit bsf can be used. */
+ SHIFT_RETURN
+ ret
+
+L(next48_bytes):
+/* Same as FIND_ZERO except we do not check first 16 bytes. */
+ andq $-16, %rax
+ PCMPEQ 16(%rax), %xmm1
+ PCMPEQ 32(%rax), %xmm2
+ PCMPEQ 48(%rax), %xmm3
+ pmovmskb %xmm1, %edx
+ pmovmskb %xmm2, %r8d
+ pmovmskb %xmm3, %ecx
+ salq $16, %rdx
+ salq $16, %rcx
+ orq %r8, %rcx
+ salq $32, %rcx
+ orq %rcx, %rdx
+#endif
+
+ /* When no zero byte is found xmm1-3 are zero so we do not have to
+ zero them. */
+ PROLOG(loop)
+
+ .p2align 4
+L(cross_page):
+ andq $-64, %rax
+ FIND_ZERO
+ PROLOG(loop_init)
+
+#ifdef AS_STRNLEN
+/* We must do this check to correctly handle strnlen (s, -1). */
+L(strnlen_ret):
+ bts %rsi, %rdx
+ sarq %cl, %rdx
+ test %rdx, %rdx
+ je L(loop_init)
+ bsfq %rdx, %rax
+ SHIFT_RETURN
+ ret
+#endif
+ .p2align 4
+L(loop_init):
+ pxor %xmm1, %xmm1
+ pxor %xmm2, %xmm2
+ pxor %xmm3, %xmm3
+#ifdef AS_STRNLEN
+ .p2align 4
+L(loop):
+
+ addq $64, %rax
+ cmpq %rax, %r10
+ je L(exit_end)
+
+ movdqa (%rax), %xmm0
+ PMINU 16(%rax), %xmm0
+ PMINU 32(%rax), %xmm0
+ PMINU 48(%rax), %xmm0
+ PCMPEQ %xmm3, %xmm0
+ pmovmskb %xmm0, %edx
+ testl %edx, %edx
+ jne L(exit)
+ jmp L(loop)
+
+ .p2align 4
+L(exit_end):
+ cmp %rax, %r11
+ je L(first) /* Do not read when end is at page boundary. */
+ pxor %xmm0, %xmm0
+ FIND_ZERO
+
+L(first):
+ bts %r11, %rdx
+ bsfq %rdx, %rdx
+ addq %rdx, %rax
+ subq %rdi, %rax
+ SHIFT_RETURN
+ ret
+
+ .p2align 4
+L(exit):
+ pxor %xmm0, %xmm0
+ FIND_ZERO
+
+ bsfq %rdx, %rdx
+ addq %rdx, %rax
+ subq %rdi, %rax
+ SHIFT_RETURN
+ ret
+
+#else
+
+ /* Main loop. Unrolled twice to improve L2 cache performance on core2. */
+ .p2align 4
+L(loop):
+
+ movdqa 64(%rax), %xmm0
+ PMINU 80(%rax), %xmm0
+ PMINU 96(%rax), %xmm0
+ PMINU 112(%rax), %xmm0
+ PCMPEQ %xmm3, %xmm0
+ pmovmskb %xmm0, %edx
+ testl %edx, %edx
+ jne L(exit64)
+
+ subq $-128, %rax
+
+ movdqa (%rax), %xmm0
+ PMINU 16(%rax), %xmm0
+ PMINU 32(%rax), %xmm0
+ PMINU 48(%rax), %xmm0
+ PCMPEQ %xmm3, %xmm0
+ pmovmskb %xmm0, %edx
+ testl %edx, %edx
+ jne L(exit0)
+ jmp L(loop)
+
+ .p2align 4
+L(exit64):
+ addq $64, %rax
+L(exit0):
+ pxor %xmm0, %xmm0
+ FIND_ZERO
+
+ bsfq %rdx, %rdx
+ addq %rdx, %rax
+ subq %rdi, %rax
+ SHIFT_RETURN
+ ret
+
+#endif
+
+END(strlen)
diff --git a/sysdeps/x86_64/multiarch/wcsnlen-sse4_1.S b/sysdeps/x86_64/multiarch/wcsnlen-sse4_1.S
index a8cab0cb..5fa51fe0 100644
--- a/sysdeps/x86_64/multiarch/wcsnlen-sse4_1.S
+++ b/sysdeps/x86_64/multiarch/wcsnlen-sse4_1.S
@@ -2,4 +2,4 @@
#define AS_STRNLEN
#define strlen __wcsnlen_sse4_1
-#include "../strlen.S"
+#include "strlen-vec.S"
diff --git a/sysdeps/x86_64/strlen.S b/sysdeps/x86_64/strlen.S
index f845f3d4..ad047d84 100644
--- a/sysdeps/x86_64/strlen.S
+++ b/sysdeps/x86_64/strlen.S
@@ -1,5 +1,5 @@
-/* SSE2 version of strlen/wcslen.
- Copyright (C) 2012-2018 Free Software Foundation, Inc.
+/* SSE2 version of strlen.
+ Copyright (C) 2021 Free Software Foundation, Inc.
This file is part of the GNU C Library.
The GNU C Library is free software; you can redistribute it and/or
@@ -16,243 +16,6 @@
License along with the GNU C Library; if not, see
<http://www.gnu.org/licenses/>. */
-#include <sysdep.h>
+#include "multiarch/strlen-vec.S"
-#ifdef AS_WCSLEN
-# define PMINU pminud
-# define PCMPEQ pcmpeqd
-# define SHIFT_RETURN shrq $2, %rax
-#else
-# define PMINU pminub
-# define PCMPEQ pcmpeqb
-# define SHIFT_RETURN
-#endif
-
-/* Long lived register in strlen(s), strnlen(s, n) are:
-
- %xmm3 - zero
- %rdi - s
- %r10 (s+n) & (~(64-1))
- %r11 s+n
-*/
-
-
-.text
-ENTRY(strlen)
-
-/* Test 64 bytes from %rax for zero. Save result as bitmask in %rdx. */
-#define FIND_ZERO \
- PCMPEQ (%rax), %xmm0; \
- PCMPEQ 16(%rax), %xmm1; \
- PCMPEQ 32(%rax), %xmm2; \
- PCMPEQ 48(%rax), %xmm3; \
- pmovmskb %xmm0, %esi; \
- pmovmskb %xmm1, %edx; \
- pmovmskb %xmm2, %r8d; \
- pmovmskb %xmm3, %ecx; \
- salq $16, %rdx; \
- salq $16, %rcx; \
- orq %rsi, %rdx; \
- orq %r8, %rcx; \
- salq $32, %rcx; \
- orq %rcx, %rdx;
-
-#ifdef AS_STRNLEN
-/* Do not read anything when n==0. */
- test %RSI_LP, %RSI_LP
- jne L(n_nonzero)
- xor %rax, %rax
- ret
-L(n_nonzero):
-# ifdef AS_WCSLEN
- shl $2, %RSI_LP
-# endif
-
-/* Initialize long lived registers. */
-
- add %RDI_LP, %RSI_LP
- mov %RSI_LP, %R10_LP
- and $-64, %R10_LP
- mov %RSI_LP, %R11_LP
-#endif
-
- pxor %xmm0, %xmm0
- pxor %xmm1, %xmm1
- pxor %xmm2, %xmm2
- pxor %xmm3, %xmm3
- movq %rdi, %rax
- movq %rdi, %rcx
- andq $4095, %rcx
-/* Offsets 4032-4047 will be aligned into 4032 thus fit into page. */
- cmpq $4047, %rcx
-/* We cannot unify this branching as it would be ~6 cycles slower. */
- ja L(cross_page)
-
-#ifdef AS_STRNLEN
-/* Test if end is among first 64 bytes. */
-# define STRNLEN_PROLOG \
- mov %r11, %rsi; \
- subq %rax, %rsi; \
- andq $-64, %rax; \
- testq $-64, %rsi; \
- je L(strnlen_ret)
-#else
-# define STRNLEN_PROLOG andq $-64, %rax;
-#endif
-
-/* Ignore bits in mask that come before start of string. */
-#define PROLOG(lab) \
- movq %rdi, %rcx; \
- xorq %rax, %rcx; \
- STRNLEN_PROLOG; \
- sarq %cl, %rdx; \
- test %rdx, %rdx; \
- je L(lab); \
- bsfq %rdx, %rax; \
- SHIFT_RETURN; \
- ret
-
-#ifdef AS_STRNLEN
- andq $-16, %rax
- FIND_ZERO
-#else
- /* Test first 16 bytes unaligned. */
- movdqu (%rax), %xmm4
- PCMPEQ %xmm0, %xmm4
- pmovmskb %xmm4, %edx
- test %edx, %edx
- je L(next48_bytes)
- bsf %edx, %eax /* If eax is zeroed 16bit bsf can be used. */
- SHIFT_RETURN
- ret
-
-L(next48_bytes):
-/* Same as FIND_ZERO except we do not check first 16 bytes. */
- andq $-16, %rax
- PCMPEQ 16(%rax), %xmm1
- PCMPEQ 32(%rax), %xmm2
- PCMPEQ 48(%rax), %xmm3
- pmovmskb %xmm1, %edx
- pmovmskb %xmm2, %r8d
- pmovmskb %xmm3, %ecx
- salq $16, %rdx
- salq $16, %rcx
- orq %r8, %rcx
- salq $32, %rcx
- orq %rcx, %rdx
-#endif
-
- /* When no zero byte is found xmm1-3 are zero so we do not have to
- zero them. */
- PROLOG(loop)
-
- .p2align 4
-L(cross_page):
- andq $-64, %rax
- FIND_ZERO
- PROLOG(loop_init)
-
-#ifdef AS_STRNLEN
-/* We must do this check to correctly handle strnlen (s, -1). */
-L(strnlen_ret):
- bts %rsi, %rdx
- sarq %cl, %rdx
- test %rdx, %rdx
- je L(loop_init)
- bsfq %rdx, %rax
- SHIFT_RETURN
- ret
-#endif
- .p2align 4
-L(loop_init):
- pxor %xmm1, %xmm1
- pxor %xmm2, %xmm2
- pxor %xmm3, %xmm3
-#ifdef AS_STRNLEN
- .p2align 4
-L(loop):
-
- addq $64, %rax
- cmpq %rax, %r10
- je L(exit_end)
-
- movdqa (%rax), %xmm0
- PMINU 16(%rax), %xmm0
- PMINU 32(%rax), %xmm0
- PMINU 48(%rax), %xmm0
- PCMPEQ %xmm3, %xmm0
- pmovmskb %xmm0, %edx
- testl %edx, %edx
- jne L(exit)
- jmp L(loop)
-
- .p2align 4
-L(exit_end):
- cmp %rax, %r11
- je L(first) /* Do not read when end is at page boundary. */
- pxor %xmm0, %xmm0
- FIND_ZERO
-
-L(first):
- bts %r11, %rdx
- bsfq %rdx, %rdx
- addq %rdx, %rax
- subq %rdi, %rax
- SHIFT_RETURN
- ret
-
- .p2align 4
-L(exit):
- pxor %xmm0, %xmm0
- FIND_ZERO
-
- bsfq %rdx, %rdx
- addq %rdx, %rax
- subq %rdi, %rax
- SHIFT_RETURN
- ret
-
-#else
-
- /* Main loop. Unrolled twice to improve L2 cache performance on core2. */
- .p2align 4
-L(loop):
-
- movdqa 64(%rax), %xmm0
- PMINU 80(%rax), %xmm0
- PMINU 96(%rax), %xmm0
- PMINU 112(%rax), %xmm0
- PCMPEQ %xmm3, %xmm0
- pmovmskb %xmm0, %edx
- testl %edx, %edx
- jne L(exit64)
-
- subq $-128, %rax
-
- movdqa (%rax), %xmm0
- PMINU 16(%rax), %xmm0
- PMINU 32(%rax), %xmm0
- PMINU 48(%rax), %xmm0
- PCMPEQ %xmm3, %xmm0
- pmovmskb %xmm0, %edx
- testl %edx, %edx
- jne L(exit0)
- jmp L(loop)
-
- .p2align 4
-L(exit64):
- addq $64, %rax
-L(exit0):
- pxor %xmm0, %xmm0
- FIND_ZERO
-
- bsfq %rdx, %rdx
- addq %rdx, %rax
- subq %rdi, %rax
- SHIFT_RETURN
- ret
-
-#endif
-
-END(strlen)
libc_hidden_builtin_def (strlen)
--
GitLab

View File

@ -1,181 +0,0 @@
From 6f573a27b6c8b4236445810a44660612323f5a73 Mon Sep 17 00:00:00 2001
From: Noah Goldstein <goldstein.w.n@gmail.com>
Date: Wed, 23 Jun 2021 01:19:34 -0400
Subject: [PATCH] x86-64: Add wcslen optimize for sse4.1
Content-type: text/plain; charset=UTF-8
No bug. This comment adds the ifunc / build infrastructure
necessary for wcslen to prefer the sse4.1 implementation
in strlen-vec.S. test-wcslen.c is passing.
Signed-off-by: Noah Goldstein <goldstein.w.n@gmail.com>
Reviewed-by: H.J. Lu <hjl.tools@gmail.com>
---
sysdeps/x86_64/multiarch/Makefile | 4 +-
sysdeps/x86_64/multiarch/ifunc-impl-list.c | 3 ++
sysdeps/x86_64/multiarch/ifunc-wcslen.h | 52 ++++++++++++++++++++++
sysdeps/x86_64/multiarch/wcslen-sse4_1.S | 4 ++
sysdeps/x86_64/multiarch/wcslen.c | 2 +-
sysdeps/x86_64/multiarch/wcsnlen.c | 34 +-------------
6 files changed, 63 insertions(+), 36 deletions(-)
create mode 100644 sysdeps/x86_64/multiarch/ifunc-wcslen.h
create mode 100644 sysdeps/x86_64/multiarch/wcslen-sse4_1.S
diff --git a/sysdeps/x86_64/multiarch/Makefile b/sysdeps/x86_64/multiarch/Makefile
index 491c7698..65fde4eb 100644
--- a/sysdeps/x86_64/multiarch/Makefile
+++ b/sysdeps/x86_64/multiarch/Makefile
@@ -93,8 +93,8 @@ sysdep_routines += wmemcmp-sse4 wmemcmp-ssse3 wmemcmp-c \
wcscpy-ssse3 wcscpy-c \
wcschr-sse2 wcschr-avx2 \
wcsrchr-sse2 wcsrchr-avx2 \
- wcsnlen-sse4_1 wcsnlen-c \
- wcslen-sse2 wcslen-avx2 wcsnlen-avx2 \
+ wcslen-sse2 wcslen-sse4_1 wcslen-avx2 \
+ wcsnlen-c wcsnlen-sse4_1 wcsnlen-avx2 \
wcschr-avx2-rtm \
wcscmp-avx2-rtm \
wcslen-avx2-rtm \
diff --git a/sysdeps/x86_64/multiarch/ifunc-impl-list.c b/sysdeps/x86_64/multiarch/ifunc-impl-list.c
index f1a6460a..580913ca 100644
--- a/sysdeps/x86_64/multiarch/ifunc-impl-list.c
+++ b/sysdeps/x86_64/multiarch/ifunc-impl-list.c
@@ -657,6 +657,9 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,
&& CPU_FEATURE_USABLE (AVX512BW)
&& CPU_FEATURE_USABLE (BMI2)),
__wcslen_evex)
+ IFUNC_IMPL_ADD (array, i, wcsnlen,
+ CPU_FEATURE_USABLE (SSE4_1),
+ __wcsnlen_sse4_1)
IFUNC_IMPL_ADD (array, i, wcslen, 1, __wcslen_sse2))
/* Support sysdeps/x86_64/multiarch/wcsnlen.c. */
diff --git a/sysdeps/x86_64/multiarch/ifunc-wcslen.h b/sysdeps/x86_64/multiarch/ifunc-wcslen.h
new file mode 100644
index 00000000..39e33473
--- /dev/null
+++ b/sysdeps/x86_64/multiarch/ifunc-wcslen.h
@@ -0,0 +1,52 @@
+/* Common definition for ifunc selections for wcslen and wcsnlen
+ All versions must be listed in ifunc-impl-list.c.
+ Copyright (C) 2017-2021 Free Software Foundation, Inc.
+ This file is part of the GNU C Library.
+
+ The GNU C Library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ The GNU C Library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with the GNU C Library; if not, see
+ <https://www.gnu.org/licenses/>. */
+
+#include <init-arch.h>
+
+extern __typeof (REDIRECT_NAME) OPTIMIZE (sse2) attribute_hidden;
+extern __typeof (REDIRECT_NAME) OPTIMIZE (sse4_1) attribute_hidden;
+extern __typeof (REDIRECT_NAME) OPTIMIZE (avx2) attribute_hidden;
+extern __typeof (REDIRECT_NAME) OPTIMIZE (avx2_rtm) attribute_hidden;
+extern __typeof (REDIRECT_NAME) OPTIMIZE (evex) attribute_hidden;
+
+static inline void *
+IFUNC_SELECTOR (void)
+{
+ const struct cpu_features* cpu_features = __get_cpu_features ();
+
+ if (CPU_FEATURE_USABLE_P (cpu_features, AVX2)
+ && CPU_FEATURE_USABLE_P (cpu_features, BMI2)
+ && CPU_FEATURES_ARCH_P (cpu_features, AVX_Fast_Unaligned_Load))
+ {
+ if (CPU_FEATURE_USABLE_P (cpu_features, AVX512VL)
+ && CPU_FEATURE_USABLE_P (cpu_features, AVX512BW))
+ return OPTIMIZE (evex);
+
+ if (CPU_FEATURE_USABLE_P (cpu_features, RTM))
+ return OPTIMIZE (avx2_rtm);
+
+ if (!CPU_FEATURES_ARCH_P (cpu_features, Prefer_No_VZEROUPPER))
+ return OPTIMIZE (avx2);
+ }
+
+ if (CPU_FEATURE_USABLE_P (cpu_features, SSE4_1))
+ return OPTIMIZE (sse4_1);
+
+ return OPTIMIZE (sse2);
+}
diff --git a/sysdeps/x86_64/multiarch/wcslen-sse4_1.S b/sysdeps/x86_64/multiarch/wcslen-sse4_1.S
new file mode 100644
index 00000000..7e62621a
--- /dev/null
+++ b/sysdeps/x86_64/multiarch/wcslen-sse4_1.S
@@ -0,0 +1,4 @@
+#define AS_WCSLEN
+#define strlen __wcslen_sse4_1
+
+#include "strlen-vec.S"
diff --git a/sysdeps/x86_64/multiarch/wcslen.c b/sysdeps/x86_64/multiarch/wcslen.c
index 6d06e47c..3b04b75b 100644
--- a/sysdeps/x86_64/multiarch/wcslen.c
+++ b/sysdeps/x86_64/multiarch/wcslen.c
@@ -24,7 +24,7 @@
# undef __wcslen
# define SYMBOL_NAME wcslen
-# include "ifunc-avx2.h"
+# include "ifunc-wcslen.h"
libc_ifunc_redirected (__redirect_wcslen, __wcslen, IFUNC_SELECTOR ());
weak_alias (__wcslen, wcslen);
diff --git a/sysdeps/x86_64/multiarch/wcsnlen.c b/sysdeps/x86_64/multiarch/wcsnlen.c
index 20b731ae..06736410 100644
--- a/sysdeps/x86_64/multiarch/wcsnlen.c
+++ b/sysdeps/x86_64/multiarch/wcsnlen.c
@@ -24,39 +24,7 @@
# undef __wcsnlen
# define SYMBOL_NAME wcsnlen
-# include <init-arch.h>
-
-extern __typeof (REDIRECT_NAME) OPTIMIZE (sse2) attribute_hidden;
-extern __typeof (REDIRECT_NAME) OPTIMIZE (sse4_1) attribute_hidden;
-extern __typeof (REDIRECT_NAME) OPTIMIZE (avx2) attribute_hidden;
-extern __typeof (REDIRECT_NAME) OPTIMIZE (avx2_rtm) attribute_hidden;
-extern __typeof (REDIRECT_NAME) OPTIMIZE (evex) attribute_hidden;
-
-static inline void *
-IFUNC_SELECTOR (void)
-{
- const struct cpu_features* cpu_features = __get_cpu_features ();
-
- if (CPU_FEATURE_USABLE_P (cpu_features, AVX2)
- && CPU_FEATURES_ARCH_P (cpu_features, AVX_Fast_Unaligned_Load))
- {
- if (CPU_FEATURE_USABLE_P (cpu_features, AVX512VL)
- && CPU_FEATURE_USABLE_P (cpu_features, AVX512BW)
- && CPU_FEATURE_USABLE_P (cpu_features, BMI2))
- return OPTIMIZE (evex);
-
- if (CPU_FEATURE_USABLE_P (cpu_features, RTM))
- return OPTIMIZE (avx2_rtm);
-
- if (!CPU_FEATURES_ARCH_P (cpu_features, Prefer_No_VZEROUPPER))
- return OPTIMIZE (avx2);
- }
-
- if (CPU_FEATURE_USABLE_P (cpu_features, SSE4_1))
- return OPTIMIZE (sse4_1);
-
- return OPTIMIZE (sse2);
-}
+# include "ifunc-wcslen.h"
libc_ifunc_redirected (__redirect_wcsnlen, __wcsnlen, IFUNC_SELECTOR ());
weak_alias (__wcsnlen, wcsnlen);
--
GitLab

View File

@ -1,396 +0,0 @@
From 231c56760c1e2ded21ad96bbb860b1f08c556c7a Mon Sep 17 00:00:00 2001
From: "H.J. Lu" <hjl.tools@gmail.com>
Date: Mon, 21 Jan 2019 11:27:25 -0800
Subject: [PATCH] x86-64 memcpy: Properly handle the length parameter [BZ#
24097]
Content-type: text/plain; charset=UTF-8
On x32, the size_t parameter may be passed in the lower 32 bits of a
64-bit register with the non-zero upper 32 bits. The string/memory
functions written in assembly can only use the lower 32 bits of a
64-bit register as length or must clear the upper 32 bits before using
the full 64-bit register for length.
This pach fixes memcpy for x32. Tested on x86-64 and x32. On x86-64,
libc.so is the same with and withou the fix.
[BZ# 24097]
CVE-2019-6488
* sysdeps/x86_64/multiarch/memcpy-ssse3-back.S: Use RDX_LP for
length. Clear the upper 32 bits of RDX register.
* sysdeps/x86_64/multiarch/memcpy-ssse3.S: Likewise.
* sysdeps/x86_64/multiarch/memmove-avx512-no-vzeroupper.S:
Likewise.
* sysdeps/x86_64/multiarch/memmove-vec-unaligned-erms.S:
Likewise.
* sysdeps/x86_64/x32/Makefile (tests): Add tst-size_t-memcpy.
tst-size_t-wmemchr.
* sysdeps/x86_64/x32/tst-size_t-memcpy.c: New file.
---
sysdeps/x86_64/multiarch/memcpy-ssse3-back.S | 17 ++++--
sysdeps/x86_64/multiarch/memcpy-ssse3.S | 17 ++++--
.../multiarch/memmove-avx512-no-vzeroupper.S | 16 +++--
.../multiarch/memmove-vec-unaligned-erms.S | 54 +++++++++--------
sysdeps/x86_64/x32/Makefile | 2 +-
sysdeps/x86_64/x32/tst-size_t-memcpy.c | 58 +++++++++++++++++++
6 files changed, 122 insertions(+), 42 deletions(-)
create mode 100644 sysdeps/x86_64/x32/tst-size_t-memcpy.c
Conflicts:
ChangeLog
(removed)
diff --git a/sysdeps/x86_64/multiarch/memcpy-ssse3-back.S b/sysdeps/x86_64/multiarch/memcpy-ssse3-back.S
index 3cd11233..568eebd3 100644
--- a/sysdeps/x86_64/multiarch/memcpy-ssse3-back.S
+++ b/sysdeps/x86_64/multiarch/memcpy-ssse3-back.S
@@ -45,28 +45,33 @@
.section .text.ssse3,"ax",@progbits
#if !defined USE_AS_MEMPCPY && !defined USE_AS_MEMMOVE
ENTRY (MEMPCPY_CHK)
- cmpq %rdx, %rcx
+ cmp %RDX_LP, %RCX_LP
jb HIDDEN_JUMPTARGET (__chk_fail)
END (MEMPCPY_CHK)
ENTRY (MEMPCPY)
- movq %rdi, %rax
- addq %rdx, %rax
+ mov %RDI_LP, %RAX_LP
+ add %RDX_LP, %RAX_LP
jmp L(start)
END (MEMPCPY)
#endif
#if !defined USE_AS_BCOPY
ENTRY (MEMCPY_CHK)
- cmpq %rdx, %rcx
+ cmp %RDX_LP, %RCX_LP
jb HIDDEN_JUMPTARGET (__chk_fail)
END (MEMCPY_CHK)
#endif
ENTRY (MEMCPY)
- mov %rdi, %rax
+ mov %RDI_LP, %RAX_LP
#ifdef USE_AS_MEMPCPY
- add %rdx, %rax
+ add %RDX_LP, %RAX_LP
+#endif
+
+#ifdef __ILP32__
+ /* Clear the upper 32 bits. */
+ mov %edx, %edx
#endif
#ifdef USE_AS_MEMMOVE
diff --git a/sysdeps/x86_64/multiarch/memcpy-ssse3.S b/sysdeps/x86_64/multiarch/memcpy-ssse3.S
index 0240bfa3..0bd5ee99 100644
--- a/sysdeps/x86_64/multiarch/memcpy-ssse3.S
+++ b/sysdeps/x86_64/multiarch/memcpy-ssse3.S
@@ -45,28 +45,33 @@
.section .text.ssse3,"ax",@progbits
#if !defined USE_AS_MEMPCPY && !defined USE_AS_MEMMOVE
ENTRY (MEMPCPY_CHK)
- cmpq %rdx, %rcx
+ cmp %RDX_LP, %RCX_LP
jb HIDDEN_JUMPTARGET (__chk_fail)
END (MEMPCPY_CHK)
ENTRY (MEMPCPY)
- movq %rdi, %rax
- addq %rdx, %rax
+ mov %RDI_LP, %RAX_LP
+ add %RDX_LP, %RAX_LP
jmp L(start)
END (MEMPCPY)
#endif
#if !defined USE_AS_BCOPY
ENTRY (MEMCPY_CHK)
- cmpq %rdx, %rcx
+ cmp %RDX_LP, %RCX_LP
jb HIDDEN_JUMPTARGET (__chk_fail)
END (MEMCPY_CHK)
#endif
ENTRY (MEMCPY)
- mov %rdi, %rax
+ mov %RDI_LP, %RAX_LP
#ifdef USE_AS_MEMPCPY
- add %rdx, %rax
+ add %RDX_LP, %RAX_LP
+#endif
+
+#ifdef __ILP32__
+ /* Clear the upper 32 bits. */
+ mov %edx, %edx
#endif
#ifdef USE_AS_MEMMOVE
diff --git a/sysdeps/x86_64/multiarch/memmove-avx512-no-vzeroupper.S b/sysdeps/x86_64/multiarch/memmove-avx512-no-vzeroupper.S
index effc3ac2..6ca2bbc9 100644
--- a/sysdeps/x86_64/multiarch/memmove-avx512-no-vzeroupper.S
+++ b/sysdeps/x86_64/multiarch/memmove-avx512-no-vzeroupper.S
@@ -24,27 +24,31 @@
.section .text.avx512,"ax",@progbits
ENTRY (__mempcpy_chk_avx512_no_vzeroupper)
- cmpq %rdx, %rcx
+ cmp %RDX_LP, %RCX_LP
jb HIDDEN_JUMPTARGET (__chk_fail)
END (__mempcpy_chk_avx512_no_vzeroupper)
ENTRY (__mempcpy_avx512_no_vzeroupper)
- movq %rdi, %rax
- addq %rdx, %rax
+ mov %RDI_LP, %RAX_LP
+ add %RDX_LP, %RAX_LP
jmp L(start)
END (__mempcpy_avx512_no_vzeroupper)
ENTRY (__memmove_chk_avx512_no_vzeroupper)
- cmpq %rdx, %rcx
+ cmp %RDX_LP, %RCX_LP
jb HIDDEN_JUMPTARGET (__chk_fail)
END (__memmove_chk_avx512_no_vzeroupper)
ENTRY (__memmove_avx512_no_vzeroupper)
- mov %rdi, %rax
+ mov %RDI_LP, %RAX_LP
# ifdef USE_AS_MEMPCPY
- add %rdx, %rax
+ add %RDX_LP, %RAX_LP
# endif
L(start):
+# ifdef __ILP32__
+ /* Clear the upper 32 bits. */
+ mov %edx, %edx
+# endif
lea (%rsi, %rdx), %rcx
lea (%rdi, %rdx), %r9
cmp $512, %rdx
diff --git a/sysdeps/x86_64/multiarch/memmove-vec-unaligned-erms.S b/sysdeps/x86_64/multiarch/memmove-vec-unaligned-erms.S
index c952576c..274aa1c7 100644
--- a/sysdeps/x86_64/multiarch/memmove-vec-unaligned-erms.S
+++ b/sysdeps/x86_64/multiarch/memmove-vec-unaligned-erms.S
@@ -95,20 +95,20 @@
.section SECTION(.text),"ax",@progbits
#if defined SHARED && IS_IN (libc)
ENTRY (MEMMOVE_CHK_SYMBOL (__mempcpy_chk, unaligned))
- cmpq %rdx, %rcx
+ cmp %RDX_LP, %RCX_LP
jb HIDDEN_JUMPTARGET (__chk_fail)
END (MEMMOVE_CHK_SYMBOL (__mempcpy_chk, unaligned))
#endif
ENTRY (MEMPCPY_SYMBOL (__mempcpy, unaligned))
- movq %rdi, %rax
- addq %rdx, %rax
+ mov %RDI_LP, %RAX_LP
+ add %RDX_LP, %RAX_LP
jmp L(start)
END (MEMPCPY_SYMBOL (__mempcpy, unaligned))
#if defined SHARED && IS_IN (libc)
ENTRY (MEMMOVE_CHK_SYMBOL (__memmove_chk, unaligned))
- cmpq %rdx, %rcx
+ cmp %RDX_LP, %RCX_LP
jb HIDDEN_JUMPTARGET (__chk_fail)
END (MEMMOVE_CHK_SYMBOL (__memmove_chk, unaligned))
#endif
@@ -116,9 +116,13 @@ END (MEMMOVE_CHK_SYMBOL (__memmove_chk, unaligned))
ENTRY (MEMMOVE_SYMBOL (__memmove, unaligned))
movq %rdi, %rax
L(start):
- cmpq $VEC_SIZE, %rdx
+# ifdef __ILP32__
+ /* Clear the upper 32 bits. */
+ movl %edx, %edx
+# endif
+ cmp $VEC_SIZE, %RDX_LP
jb L(less_vec)
- cmpq $(VEC_SIZE * 2), %rdx
+ cmp $(VEC_SIZE * 2), %RDX_LP
ja L(more_2x_vec)
#if !defined USE_MULTIARCH || !IS_IN (libc)
L(last_2x_vec):
@@ -138,38 +142,38 @@ END (MEMMOVE_SYMBOL (__memmove, unaligned))
# if VEC_SIZE == 16
ENTRY (__mempcpy_chk_erms)
- cmpq %rdx, %rcx
+ cmp %RDX_LP, %RCX_LP
jb HIDDEN_JUMPTARGET (__chk_fail)
END (__mempcpy_chk_erms)
/* Only used to measure performance of REP MOVSB. */
ENTRY (__mempcpy_erms)
- movq %rdi, %rax
+ mov %RDI_LP, %RAX_LP
/* Skip zero length. */
- testq %rdx, %rdx
+ test %RDX_LP, %RDX_LP
jz 2f
- addq %rdx, %rax
+ add %RDX_LP, %RAX_LP
jmp L(start_movsb)
END (__mempcpy_erms)
ENTRY (__memmove_chk_erms)
- cmpq %rdx, %rcx
+ cmp %RDX_LP, %RCX_LP
jb HIDDEN_JUMPTARGET (__chk_fail)
END (__memmove_chk_erms)
ENTRY (__memmove_erms)
movq %rdi, %rax
/* Skip zero length. */
- testq %rdx, %rdx
+ test %RDX_LP, %RDX_LP
jz 2f
L(start_movsb):
- movq %rdx, %rcx
- cmpq %rsi, %rdi
+ mov %RDX_LP, %RCX_LP
+ cmp %RSI_LP, %RDI_LP
jb 1f
/* Source == destination is less common. */
je 2f
- leaq (%rsi,%rcx), %rdx
- cmpq %rdx, %rdi
+ lea (%rsi,%rcx), %RDX_LP
+ cmp %RDX_LP, %RDI_LP
jb L(movsb_backward)
1:
rep movsb
@@ -189,20 +193,20 @@ strong_alias (__memmove_chk_erms, __memcpy_chk_erms)
# ifdef SHARED
ENTRY (MEMMOVE_CHK_SYMBOL (__mempcpy_chk, unaligned_erms))
- cmpq %rdx, %rcx
+ cmp %RDX_LP, %RCX_LP
jb HIDDEN_JUMPTARGET (__chk_fail)
END (MEMMOVE_CHK_SYMBOL (__mempcpy_chk, unaligned_erms))
# endif
ENTRY (MEMMOVE_SYMBOL (__mempcpy, unaligned_erms))
- movq %rdi, %rax
- addq %rdx, %rax
+ mov %RDI_LP, %RAX_LP
+ add %RDX_LP, %RAX_LP
jmp L(start_erms)
END (MEMMOVE_SYMBOL (__mempcpy, unaligned_erms))
# ifdef SHARED
ENTRY (MEMMOVE_CHK_SYMBOL (__memmove_chk, unaligned_erms))
- cmpq %rdx, %rcx
+ cmp %RDX_LP, %RCX_LP
jb HIDDEN_JUMPTARGET (__chk_fail)
END (MEMMOVE_CHK_SYMBOL (__memmove_chk, unaligned_erms))
# endif
@@ -210,9 +214,13 @@ END (MEMMOVE_CHK_SYMBOL (__memmove_chk, unaligned_erms))
ENTRY (MEMMOVE_SYMBOL (__memmove, unaligned_erms))
movq %rdi, %rax
L(start_erms):
- cmpq $VEC_SIZE, %rdx
+# ifdef __ILP32__
+ /* Clear the upper 32 bits. */
+ movl %edx, %edx
+# endif
+ cmp $VEC_SIZE, %RDX_LP
jb L(less_vec)
- cmpq $(VEC_SIZE * 2), %rdx
+ cmp $(VEC_SIZE * 2), %RDX_LP
ja L(movsb_more_2x_vec)
L(last_2x_vec):
/* From VEC and to 2 * VEC. No branch when size == VEC_SIZE. */
@@ -236,7 +244,7 @@ L(movsb):
/* Avoid slow backward REP MOVSB. */
jb L(more_8x_vec_backward)
1:
- movq %rdx, %rcx
+ mov %RDX_LP, %RCX_LP
rep movsb
L(nop):
ret
diff --git a/sysdeps/x86_64/x32/Makefile b/sysdeps/x86_64/x32/Makefile
index ddec7f04..2fe1e5ac 100644
--- a/sysdeps/x86_64/x32/Makefile
+++ b/sysdeps/x86_64/x32/Makefile
@@ -6,7 +6,7 @@ CFLAGS-s_llround.c += -fno-builtin-lround
endif
ifeq ($(subdir),string)
-tests += tst-size_t-memchr tst-size_t-memcmp
+tests += tst-size_t-memchr tst-size_t-memcmp tst-size_t-memcpy
endif
ifeq ($(subdir),wcsmbs)
diff --git a/sysdeps/x86_64/x32/tst-size_t-memcpy.c b/sysdeps/x86_64/x32/tst-size_t-memcpy.c
new file mode 100644
index 00000000..66b71e17
--- /dev/null
+++ b/sysdeps/x86_64/x32/tst-size_t-memcpy.c
@@ -0,0 +1,58 @@
+/* Test memcpy with size_t in the lower 32 bits of 64-bit register.
+ Copyright (C) 2019 Free Software Foundation, Inc.
+ This file is part of the GNU C Library.
+
+ The GNU C Library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ The GNU C Library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with the GNU C Library; if not, see
+ <http://www.gnu.org/licenses/>. */
+
+#define TEST_NAME "memcpy"
+#include "test-size_t.h"
+
+IMPL (memcpy, 1)
+
+typedef void *(*proto_t) (void *, const void *, size_t);
+
+static void *
+__attribute__ ((noinline, noclone))
+do_memcpy (parameter_t a, parameter_t b)
+{
+ return CALL (&b, a.p, b.p, a.len);
+}
+
+static int
+test_main (void)
+{
+ test_init ();
+
+ parameter_t dest = { { page_size }, buf1 };
+ parameter_t src = { { 0 }, buf2 };
+
+ int ret = 0;
+ FOR_EACH_IMPL (impl, 0)
+ {
+ src.fn = impl->fn;
+ do_memcpy (dest, src);
+ int res = memcmp (dest.p, src.p, dest.len);
+ if (res)
+ {
+ error (0, 0, "Wrong result in function %s: %i != 0",
+ impl->name, res);
+ ret = 1;
+ }
+ }
+
+ return ret ? EXIT_FAILURE : EXIT_SUCCESS;
+}
+
+#include <support/test-driver.c>
--
GitLab

View File

@ -1,497 +0,0 @@
From a775a7a3eb1e85b54af0b4ee5ff4dcf66772a1fb Mon Sep 17 00:00:00 2001
From: Noah Goldstein <goldstein.w.n@gmail.com>
Date: Wed, 23 Jun 2021 01:56:29 -0400
Subject: [PATCH] x86: Fix overflow bug in wcsnlen-sse4_1 and wcsnlen-avx2 [BZ
#27974]
Content-type: text/plain; charset=UTF-8
This commit fixes the bug mentioned in the previous commit.
The previous implementations of wmemchr in these files relied
on maxlen * sizeof(wchar_t) which was not guranteed by the standard.
The new overflow tests added in the previous commit now
pass (As well as all the other tests).
Signed-off-by: Noah Goldstein <goldstein.w.n@gmail.com>
Reviewed-by: H.J. Lu <hjl.tools@gmail.com>
---
sysdeps/x86_64/multiarch/strlen-avx2.S | 130 ++++++++++++++++++-------
sysdeps/x86_64/multiarch/strlen-vec.S | 15 ++-
2 files changed, 107 insertions(+), 38 deletions(-)
diff --git a/sysdeps/x86_64/multiarch/strlen-avx2.S b/sysdeps/x86_64/multiarch/strlen-avx2.S
index be8a5db5..37688966 100644
--- a/sysdeps/x86_64/multiarch/strlen-avx2.S
+++ b/sysdeps/x86_64/multiarch/strlen-avx2.S
@@ -44,21 +44,21 @@
# define VEC_SIZE 32
# define PAGE_SIZE 4096
+# define CHAR_PER_VEC (VEC_SIZE / CHAR_SIZE)
.section SECTION(.text),"ax",@progbits
ENTRY (STRLEN)
# ifdef USE_AS_STRNLEN
/* Check zero length. */
+# ifdef __ILP32__
+ /* Clear upper bits. */
+ and %RSI_LP, %RSI_LP
+# else
test %RSI_LP, %RSI_LP
+# endif
jz L(zero)
/* Store max len in R8_LP before adjusting if using WCSLEN. */
mov %RSI_LP, %R8_LP
-# ifdef USE_AS_WCSLEN
- shl $2, %RSI_LP
-# elif defined __ILP32__
- /* Clear the upper 32 bits. */
- movl %esi, %esi
-# endif
# endif
movl %edi, %eax
movq %rdi, %rdx
@@ -72,10 +72,10 @@ ENTRY (STRLEN)
/* Check the first VEC_SIZE bytes. */
VPCMPEQ (%rdi), %ymm0, %ymm1
- vpmovmskb %ymm1, %eax
+ vpmovmskb %ymm1, %eax
# ifdef USE_AS_STRNLEN
/* If length < VEC_SIZE handle special. */
- cmpq $VEC_SIZE, %rsi
+ cmpq $CHAR_PER_VEC, %rsi
jbe L(first_vec_x0)
# endif
/* If empty continue to aligned_more. Otherwise return bit
@@ -84,6 +84,7 @@ ENTRY (STRLEN)
jz L(aligned_more)
tzcntl %eax, %eax
# ifdef USE_AS_WCSLEN
+ /* NB: Divide bytes by 4 to get wchar_t count. */
shrl $2, %eax
# endif
VZEROUPPER_RETURN
@@ -97,9 +98,14 @@ L(zero):
L(first_vec_x0):
/* Set bit for max len so that tzcnt will return min of max len
and position of first match. */
+# ifdef USE_AS_WCSLEN
+ /* NB: Multiply length by 4 to get byte count. */
+ sall $2, %esi
+# endif
btsq %rsi, %rax
tzcntl %eax, %eax
# ifdef USE_AS_WCSLEN
+ /* NB: Divide bytes by 4 to get wchar_t count. */
shrl $2, %eax
# endif
VZEROUPPER_RETURN
@@ -113,14 +119,19 @@ L(first_vec_x1):
# ifdef USE_AS_STRNLEN
/* Use ecx which was computed earlier to compute correct value.
*/
+# ifdef USE_AS_WCSLEN
+ leal -(VEC_SIZE * 4 + 1)(%rax, %rcx, 4), %eax
+# else
subl $(VEC_SIZE * 4 + 1), %ecx
addl %ecx, %eax
+# endif
# else
subl %edx, %edi
incl %edi
addl %edi, %eax
# endif
# ifdef USE_AS_WCSLEN
+ /* NB: Divide bytes by 4 to get wchar_t count. */
shrl $2, %eax
# endif
VZEROUPPER_RETURN
@@ -133,14 +144,19 @@ L(first_vec_x2):
# ifdef USE_AS_STRNLEN
/* Use ecx which was computed earlier to compute correct value.
*/
+# ifdef USE_AS_WCSLEN
+ leal -(VEC_SIZE * 3 + 1)(%rax, %rcx, 4), %eax
+# else
subl $(VEC_SIZE * 3 + 1), %ecx
addl %ecx, %eax
+# endif
# else
subl %edx, %edi
addl $(VEC_SIZE + 1), %edi
addl %edi, %eax
# endif
# ifdef USE_AS_WCSLEN
+ /* NB: Divide bytes by 4 to get wchar_t count. */
shrl $2, %eax
# endif
VZEROUPPER_RETURN
@@ -153,14 +169,19 @@ L(first_vec_x3):
# ifdef USE_AS_STRNLEN
/* Use ecx which was computed earlier to compute correct value.
*/
+# ifdef USE_AS_WCSLEN
+ leal -(VEC_SIZE * 2 + 1)(%rax, %rcx, 4), %eax
+# else
subl $(VEC_SIZE * 2 + 1), %ecx
addl %ecx, %eax
+# endif
# else
subl %edx, %edi
addl $(VEC_SIZE * 2 + 1), %edi
addl %edi, %eax
# endif
# ifdef USE_AS_WCSLEN
+ /* NB: Divide bytes by 4 to get wchar_t count. */
shrl $2, %eax
# endif
VZEROUPPER_RETURN
@@ -173,14 +194,19 @@ L(first_vec_x4):
# ifdef USE_AS_STRNLEN
/* Use ecx which was computed earlier to compute correct value.
*/
+# ifdef USE_AS_WCSLEN
+ leal -(VEC_SIZE * 1 + 1)(%rax, %rcx, 4), %eax
+# else
subl $(VEC_SIZE + 1), %ecx
addl %ecx, %eax
+# endif
# else
subl %edx, %edi
addl $(VEC_SIZE * 3 + 1), %edi
addl %edi, %eax
# endif
# ifdef USE_AS_WCSLEN
+ /* NB: Divide bytes by 4 to get wchar_t count. */
shrl $2, %eax
# endif
VZEROUPPER_RETURN
@@ -195,10 +221,14 @@ L(cross_page_continue):
/* Check the first 4 * VEC_SIZE. Only one VEC_SIZE at a time
since data is only aligned to VEC_SIZE. */
# ifdef USE_AS_STRNLEN
- /* + 1 because rdi is aligned to VEC_SIZE - 1. + CHAR_SIZE because
- it simplies the logic in last_4x_vec_or_less. */
+ /* + 1 because rdi is aligned to VEC_SIZE - 1. + CHAR_SIZE
+ because it simplies the logic in last_4x_vec_or_less. */
leaq (VEC_SIZE * 4 + CHAR_SIZE + 1)(%rdi), %rcx
subq %rdx, %rcx
+# ifdef USE_AS_WCSLEN
+ /* NB: Divide bytes by 4 to get the wchar_t count. */
+ sarl $2, %ecx
+# endif
# endif
/* Load first VEC regardless. */
VPCMPEQ 1(%rdi), %ymm0, %ymm1
@@ -207,34 +237,38 @@ L(cross_page_continue):
subq %rcx, %rsi
jb L(last_4x_vec_or_less)
# endif
- vpmovmskb %ymm1, %eax
+ vpmovmskb %ymm1, %eax
testl %eax, %eax
jnz L(first_vec_x1)
VPCMPEQ (VEC_SIZE + 1)(%rdi), %ymm0, %ymm1
- vpmovmskb %ymm1, %eax
+ vpmovmskb %ymm1, %eax
testl %eax, %eax
jnz L(first_vec_x2)
VPCMPEQ (VEC_SIZE * 2 + 1)(%rdi), %ymm0, %ymm1
- vpmovmskb %ymm1, %eax
+ vpmovmskb %ymm1, %eax
testl %eax, %eax
jnz L(first_vec_x3)
VPCMPEQ (VEC_SIZE * 3 + 1)(%rdi), %ymm0, %ymm1
- vpmovmskb %ymm1, %eax
+ vpmovmskb %ymm1, %eax
testl %eax, %eax
jnz L(first_vec_x4)
/* Align data to VEC_SIZE * 4 - 1. */
# ifdef USE_AS_STRNLEN
/* Before adjusting length check if at last VEC_SIZE * 4. */
- cmpq $(VEC_SIZE * 4 - 1), %rsi
+ cmpq $(CHAR_PER_VEC * 4 - 1), %rsi
jbe L(last_4x_vec_or_less_load)
incq %rdi
movl %edi, %ecx
orq $(VEC_SIZE * 4 - 1), %rdi
andl $(VEC_SIZE * 4 - 1), %ecx
+# ifdef USE_AS_WCSLEN
+ /* NB: Divide bytes by 4 to get the wchar_t count. */
+ sarl $2, %ecx
+# endif
/* Readjust length. */
addq %rcx, %rsi
# else
@@ -246,13 +280,13 @@ L(cross_page_continue):
L(loop_4x_vec):
# ifdef USE_AS_STRNLEN
/* Break if at end of length. */
- subq $(VEC_SIZE * 4), %rsi
+ subq $(CHAR_PER_VEC * 4), %rsi
jb L(last_4x_vec_or_less_cmpeq)
# endif
- /* Save some code size by microfusing VPMINU with the load. Since
- the matches in ymm2/ymm4 can only be returned if there where no
- matches in ymm1/ymm3 respectively there is no issue with overlap.
- */
+ /* Save some code size by microfusing VPMINU with the load.
+ Since the matches in ymm2/ymm4 can only be returned if there
+ where no matches in ymm1/ymm3 respectively there is no issue
+ with overlap. */
vmovdqa 1(%rdi), %ymm1
VPMINU (VEC_SIZE + 1)(%rdi), %ymm1, %ymm2
vmovdqa (VEC_SIZE * 2 + 1)(%rdi), %ymm3
@@ -260,7 +294,7 @@ L(loop_4x_vec):
VPMINU %ymm2, %ymm4, %ymm5
VPCMPEQ %ymm5, %ymm0, %ymm5
- vpmovmskb %ymm5, %ecx
+ vpmovmskb %ymm5, %ecx
subq $-(VEC_SIZE * 4), %rdi
testl %ecx, %ecx
@@ -268,27 +302,28 @@ L(loop_4x_vec):
VPCMPEQ %ymm1, %ymm0, %ymm1
- vpmovmskb %ymm1, %eax
+ vpmovmskb %ymm1, %eax
subq %rdx, %rdi
testl %eax, %eax
jnz L(last_vec_return_x0)
VPCMPEQ %ymm2, %ymm0, %ymm2
- vpmovmskb %ymm2, %eax
+ vpmovmskb %ymm2, %eax
testl %eax, %eax
jnz L(last_vec_return_x1)
/* Combine last 2 VEC. */
VPCMPEQ %ymm3, %ymm0, %ymm3
- vpmovmskb %ymm3, %eax
- /* rcx has combined result from all 4 VEC. It will only be used if
- the first 3 other VEC all did not contain a match. */
+ vpmovmskb %ymm3, %eax
+ /* rcx has combined result from all 4 VEC. It will only be used
+ if the first 3 other VEC all did not contain a match. */
salq $32, %rcx
orq %rcx, %rax
tzcntq %rax, %rax
subq $(VEC_SIZE * 2 - 1), %rdi
addq %rdi, %rax
# ifdef USE_AS_WCSLEN
+ /* NB: Divide bytes by 4 to get wchar_t count. */
shrq $2, %rax
# endif
VZEROUPPER_RETURN
@@ -297,15 +332,19 @@ L(loop_4x_vec):
# ifdef USE_AS_STRNLEN
.p2align 4
L(last_4x_vec_or_less_load):
- /* Depending on entry adjust rdi / prepare first VEC in ymm1. */
+ /* Depending on entry adjust rdi / prepare first VEC in ymm1.
+ */
subq $-(VEC_SIZE * 4), %rdi
L(last_4x_vec_or_less_cmpeq):
VPCMPEQ 1(%rdi), %ymm0, %ymm1
L(last_4x_vec_or_less):
-
- vpmovmskb %ymm1, %eax
- /* If remaining length > VEC_SIZE * 2. This works if esi is off by
- VEC_SIZE * 4. */
+# ifdef USE_AS_WCSLEN
+ /* NB: Multiply length by 4 to get byte count. */
+ sall $2, %esi
+# endif
+ vpmovmskb %ymm1, %eax
+ /* If remaining length > VEC_SIZE * 2. This works if esi is off
+ by VEC_SIZE * 4. */
testl $(VEC_SIZE * 2), %esi
jnz L(last_4x_vec)
@@ -320,7 +359,7 @@ L(last_4x_vec_or_less):
jb L(max)
VPCMPEQ (VEC_SIZE + 1)(%rdi), %ymm0, %ymm1
- vpmovmskb %ymm1, %eax
+ vpmovmskb %ymm1, %eax
tzcntl %eax, %eax
/* Check the end of data. */
cmpl %eax, %esi
@@ -329,6 +368,7 @@ L(last_4x_vec_or_less):
addl $(VEC_SIZE + 1), %eax
addq %rdi, %rax
# ifdef USE_AS_WCSLEN
+ /* NB: Divide bytes by 4 to get wchar_t count. */
shrq $2, %rax
# endif
VZEROUPPER_RETURN
@@ -340,6 +380,7 @@ L(last_vec_return_x0):
subq $(VEC_SIZE * 4 - 1), %rdi
addq %rdi, %rax
# ifdef USE_AS_WCSLEN
+ /* NB: Divide bytes by 4 to get wchar_t count. */
shrq $2, %rax
# endif
VZEROUPPER_RETURN
@@ -350,6 +391,7 @@ L(last_vec_return_x1):
subq $(VEC_SIZE * 3 - 1), %rdi
addq %rdi, %rax
# ifdef USE_AS_WCSLEN
+ /* NB: Divide bytes by 4 to get wchar_t count. */
shrq $2, %rax
# endif
VZEROUPPER_RETURN
@@ -366,6 +408,7 @@ L(last_vec_x1_check):
incl %eax
addq %rdi, %rax
# ifdef USE_AS_WCSLEN
+ /* NB: Divide bytes by 4 to get wchar_t count. */
shrq $2, %rax
# endif
VZEROUPPER_RETURN
@@ -381,14 +424,14 @@ L(last_4x_vec):
jnz L(last_vec_x1)
VPCMPEQ (VEC_SIZE + 1)(%rdi), %ymm0, %ymm1
- vpmovmskb %ymm1, %eax
+ vpmovmskb %ymm1, %eax
testl %eax, %eax
jnz L(last_vec_x2)
/* Normalize length. */
andl $(VEC_SIZE * 4 - 1), %esi
VPCMPEQ (VEC_SIZE * 2 + 1)(%rdi), %ymm0, %ymm1
- vpmovmskb %ymm1, %eax
+ vpmovmskb %ymm1, %eax
testl %eax, %eax
jnz L(last_vec_x3)
@@ -396,7 +439,7 @@ L(last_4x_vec):
jb L(max)
VPCMPEQ (VEC_SIZE * 3 + 1)(%rdi), %ymm0, %ymm1
- vpmovmskb %ymm1, %eax
+ vpmovmskb %ymm1, %eax
tzcntl %eax, %eax
/* Check the end of data. */
cmpl %eax, %esi
@@ -405,6 +448,7 @@ L(last_4x_vec):
addl $(VEC_SIZE * 3 + 1), %eax
addq %rdi, %rax
# ifdef USE_AS_WCSLEN
+ /* NB: Divide bytes by 4 to get wchar_t count. */
shrq $2, %rax
# endif
VZEROUPPER_RETURN
@@ -419,6 +463,7 @@ L(last_vec_x1):
incl %eax
addq %rdi, %rax
# ifdef USE_AS_WCSLEN
+ /* NB: Divide bytes by 4 to get wchar_t count. */
shrq $2, %rax
# endif
VZEROUPPER_RETURN
@@ -432,6 +477,7 @@ L(last_vec_x2):
addl $(VEC_SIZE + 1), %eax
addq %rdi, %rax
# ifdef USE_AS_WCSLEN
+ /* NB: Divide bytes by 4 to get wchar_t count. */
shrq $2, %rax
# endif
VZEROUPPER_RETURN
@@ -447,6 +493,7 @@ L(last_vec_x3):
addl $(VEC_SIZE * 2 + 1), %eax
addq %rdi, %rax
# ifdef USE_AS_WCSLEN
+ /* NB: Divide bytes by 4 to get wchar_t count. */
shrq $2, %rax
# endif
VZEROUPPER_RETURN
@@ -455,13 +502,13 @@ L(max_end):
VZEROUPPER_RETURN
# endif
- /* Cold case for crossing page with first load. */
+ /* Cold case for crossing page with first load. */
.p2align 4
L(cross_page_boundary):
/* Align data to VEC_SIZE - 1. */
orq $(VEC_SIZE - 1), %rdi
VPCMPEQ -(VEC_SIZE - 1)(%rdi), %ymm0, %ymm1
- vpmovmskb %ymm1, %eax
+ vpmovmskb %ymm1, %eax
/* Remove the leading bytes. sarxl only uses bits [5:0] of COUNT
so no need to manually mod rdx. */
sarxl %edx, %eax, %eax
@@ -470,6 +517,10 @@ L(cross_page_boundary):
jnz L(cross_page_less_vec)
leaq 1(%rdi), %rcx
subq %rdx, %rcx
+# ifdef USE_AS_WCSLEN
+ /* NB: Divide bytes by 4 to get wchar_t count. */
+ shrl $2, %ecx
+# endif
/* Check length. */
cmpq %rsi, %rcx
jb L(cross_page_continue)
@@ -479,6 +530,7 @@ L(cross_page_boundary):
jz L(cross_page_continue)
tzcntl %eax, %eax
# ifdef USE_AS_WCSLEN
+ /* NB: Divide length by 4 to get wchar_t count. */
shrl $2, %eax
# endif
# endif
@@ -489,6 +541,10 @@ L(return_vzeroupper):
.p2align 4
L(cross_page_less_vec):
tzcntl %eax, %eax
+# ifdef USE_AS_WCSLEN
+ /* NB: Multiply length by 4 to get byte count. */
+ sall $2, %esi
+# endif
cmpq %rax, %rsi
cmovb %esi, %eax
# ifdef USE_AS_WCSLEN
diff --git a/sysdeps/x86_64/multiarch/strlen-vec.S b/sysdeps/x86_64/multiarch/strlen-vec.S
index 8f660bb9..439e486a 100644
--- a/sysdeps/x86_64/multiarch/strlen-vec.S
+++ b/sysdeps/x86_64/multiarch/strlen-vec.S
@@ -65,12 +65,25 @@ ENTRY(strlen)
ret
L(n_nonzero):
# ifdef AS_WCSLEN
- shl $2, %RSI_LP
+/* Check for overflow from maxlen * sizeof(wchar_t). If it would
+ overflow the only way this program doesn't have undefined behavior
+ is if there is a null terminator in valid memory so wcslen will
+ suffice. */
+ mov %RSI_LP, %R10_LP
+ sar $62, %R10_LP
+ test %R10_LP, %R10_LP
+ jnz __wcslen_sse4_1
+ sal $2, %RSI_LP
# endif
+
/* Initialize long lived registers. */
add %RDI_LP, %RSI_LP
+# ifdef AS_WCSLEN
+/* Check for overflow again from s + maxlen * sizeof(wchar_t). */
+ jbe __wcslen_sse4_1
+# endif
mov %RSI_LP, %R10_LP
and $-64, %R10_LP
mov %RSI_LP, %R11_LP
--
GitLab

View File

@ -1,745 +0,0 @@
From 4ba65586847751372520a36757c17f114588794e Mon Sep 17 00:00:00 2001
From: Noah Goldstein <goldstein.w.n@gmail.com>
Date: Mon, 19 Apr 2021 19:36:06 -0400
Subject: [PATCH] x86: Optimize strlen-evex.S
Content-type: text/plain; charset=UTF-8
No bug. This commit optimizes strlen-evex.S. The
optimizations are mostly small things but they add up to roughly
10-30% performance improvement for strlen. The results for strnlen are
bit more ambiguous. test-strlen, test-strnlen, test-wcslen, and
test-wcsnlen are all passing.
Signed-off-by: Noah Goldstein <goldstein.w.n@gmail.com>
---
sysdeps/x86_64/multiarch/strlen-evex.S | 581 ++++++++++++++-----------
1 file changed, 317 insertions(+), 264 deletions(-)
diff --git a/sysdeps/x86_64/multiarch/strlen-evex.S b/sysdeps/x86_64/multiarch/strlen-evex.S
index 05838190..4bf6874b 100644
--- a/sysdeps/x86_64/multiarch/strlen-evex.S
+++ b/sysdeps/x86_64/multiarch/strlen-evex.S
@@ -29,11 +29,13 @@
# ifdef USE_AS_WCSLEN
# define VPCMP vpcmpd
# define VPMINU vpminud
-# define SHIFT_REG r9d
+# define SHIFT_REG ecx
+# define CHAR_SIZE 4
# else
# define VPCMP vpcmpb
# define VPMINU vpminub
-# define SHIFT_REG ecx
+# define SHIFT_REG edx
+# define CHAR_SIZE 1
# endif
# define XMMZERO xmm16
@@ -46,132 +48,165 @@
# define YMM6 ymm22
# define VEC_SIZE 32
+# define PAGE_SIZE 4096
+# define CHAR_PER_VEC (VEC_SIZE / CHAR_SIZE)
.section .text.evex,"ax",@progbits
ENTRY (STRLEN)
# ifdef USE_AS_STRNLEN
- /* Check for zero length. */
+ /* Check zero length. */
test %RSI_LP, %RSI_LP
jz L(zero)
-# ifdef USE_AS_WCSLEN
- shl $2, %RSI_LP
-# elif defined __ILP32__
+# ifdef __ILP32__
/* Clear the upper 32 bits. */
movl %esi, %esi
# endif
mov %RSI_LP, %R8_LP
# endif
- movl %edi, %ecx
- movq %rdi, %rdx
+ movl %edi, %eax
vpxorq %XMMZERO, %XMMZERO, %XMMZERO
-
+ /* Clear high bits from edi. Only keeping bits relevant to page
+ cross check. */
+ andl $(PAGE_SIZE - 1), %eax
/* Check if we may cross page boundary with one vector load. */
- andl $(2 * VEC_SIZE - 1), %ecx
- cmpl $VEC_SIZE, %ecx
- ja L(cros_page_boundary)
+ cmpl $(PAGE_SIZE - VEC_SIZE), %eax
+ ja L(cross_page_boundary)
/* Check the first VEC_SIZE bytes. Each bit in K0 represents a
null byte. */
VPCMP $0, (%rdi), %YMMZERO, %k0
kmovd %k0, %eax
- testl %eax, %eax
-
# ifdef USE_AS_STRNLEN
- jnz L(first_vec_x0_check)
- /* Adjust length and check the end of data. */
- subq $VEC_SIZE, %rsi
- jbe L(max)
-# else
- jnz L(first_vec_x0)
+ /* If length < CHAR_PER_VEC handle special. */
+ cmpq $CHAR_PER_VEC, %rsi
+ jbe L(first_vec_x0)
# endif
-
- /* Align data for aligned loads in the loop. */
- addq $VEC_SIZE, %rdi
- andl $(VEC_SIZE - 1), %ecx
- andq $-VEC_SIZE, %rdi
-
+ testl %eax, %eax
+ jz L(aligned_more)
+ tzcntl %eax, %eax
+ ret
# ifdef USE_AS_STRNLEN
- /* Adjust length. */
- addq %rcx, %rsi
+L(zero):
+ xorl %eax, %eax
+ ret
- subq $(VEC_SIZE * 4), %rsi
- jbe L(last_4x_vec_or_less)
+ .p2align 4
+L(first_vec_x0):
+ /* Set bit for max len so that tzcnt will return min of max len
+ and position of first match. */
+ btsq %rsi, %rax
+ tzcntl %eax, %eax
+ ret
# endif
- jmp L(more_4x_vec)
.p2align 4
-L(cros_page_boundary):
- andl $(VEC_SIZE - 1), %ecx
- andq $-VEC_SIZE, %rdi
-
-# ifdef USE_AS_WCSLEN
- /* NB: Divide shift count by 4 since each bit in K0 represent 4
- bytes. */
- movl %ecx, %SHIFT_REG
- sarl $2, %SHIFT_REG
+L(first_vec_x1):
+ tzcntl %eax, %eax
+ /* Safe to use 32 bit instructions as these are only called for
+ size = [1, 159]. */
+# ifdef USE_AS_STRNLEN
+ /* Use ecx which was computed earlier to compute correct value.
+ */
+ leal -(CHAR_PER_VEC * 4 + 1)(%rcx, %rax), %eax
+# else
+ subl %edx, %edi
+# ifdef USE_AS_WCSLEN
+ /* NB: Divide bytes by 4 to get the wchar_t count. */
+ sarl $2, %edi
+# endif
+ leal CHAR_PER_VEC(%rdi, %rax), %eax
# endif
- VPCMP $0, (%rdi), %YMMZERO, %k0
- kmovd %k0, %eax
+ ret
- /* Remove the leading bytes. */
- sarxl %SHIFT_REG, %eax, %eax
- testl %eax, %eax
- jz L(aligned_more)
+ .p2align 4
+L(first_vec_x2):
tzcntl %eax, %eax
-# ifdef USE_AS_WCSLEN
- /* NB: Multiply wchar_t count by 4 to get the number of bytes. */
- sall $2, %eax
-# endif
+ /* Safe to use 32 bit instructions as these are only called for
+ size = [1, 159]. */
# ifdef USE_AS_STRNLEN
- /* Check the end of data. */
- cmpq %rax, %rsi
- jbe L(max)
-# endif
- addq %rdi, %rax
- addq %rcx, %rax
- subq %rdx, %rax
-# ifdef USE_AS_WCSLEN
- shrq $2, %rax
+ /* Use ecx which was computed earlier to compute correct value.
+ */
+ leal -(CHAR_PER_VEC * 3 + 1)(%rcx, %rax), %eax
+# else
+ subl %edx, %edi
+# ifdef USE_AS_WCSLEN
+ /* NB: Divide bytes by 4 to get the wchar_t count. */
+ sarl $2, %edi
+# endif
+ leal (CHAR_PER_VEC * 2)(%rdi, %rax), %eax
# endif
ret
.p2align 4
-L(aligned_more):
+L(first_vec_x3):
+ tzcntl %eax, %eax
+ /* Safe to use 32 bit instructions as these are only called for
+ size = [1, 159]. */
# ifdef USE_AS_STRNLEN
- /* "rcx" is less than VEC_SIZE. Calculate "rdx + rcx - VEC_SIZE"
- with "rdx - (VEC_SIZE - rcx)" instead of "(rdx + rcx) - VEC_SIZE"
- to void possible addition overflow. */
- negq %rcx
- addq $VEC_SIZE, %rcx
-
- /* Check the end of data. */
- subq %rcx, %rsi
- jbe L(max)
+ /* Use ecx which was computed earlier to compute correct value.
+ */
+ leal -(CHAR_PER_VEC * 2 + 1)(%rcx, %rax), %eax
+# else
+ subl %edx, %edi
+# ifdef USE_AS_WCSLEN
+ /* NB: Divide bytes by 4 to get the wchar_t count. */
+ sarl $2, %edi
+# endif
+ leal (CHAR_PER_VEC * 3)(%rdi, %rax), %eax
# endif
+ ret
- addq $VEC_SIZE, %rdi
-
+ .p2align 4
+L(first_vec_x4):
+ tzcntl %eax, %eax
+ /* Safe to use 32 bit instructions as these are only called for
+ size = [1, 159]. */
# ifdef USE_AS_STRNLEN
- subq $(VEC_SIZE * 4), %rsi
- jbe L(last_4x_vec_or_less)
+ /* Use ecx which was computed earlier to compute correct value.
+ */
+ leal -(CHAR_PER_VEC + 1)(%rcx, %rax), %eax
+# else
+ subl %edx, %edi
+# ifdef USE_AS_WCSLEN
+ /* NB: Divide bytes by 4 to get the wchar_t count. */
+ sarl $2, %edi
+# endif
+ leal (CHAR_PER_VEC * 4)(%rdi, %rax), %eax
# endif
+ ret
-L(more_4x_vec):
+ .p2align 5
+L(aligned_more):
+ movq %rdi, %rdx
+ /* Align data to VEC_SIZE. */
+ andq $-(VEC_SIZE), %rdi
+L(cross_page_continue):
/* Check the first 4 * VEC_SIZE. Only one VEC_SIZE at a time
since data is only aligned to VEC_SIZE. */
- VPCMP $0, (%rdi), %YMMZERO, %k0
- kmovd %k0, %eax
- testl %eax, %eax
- jnz L(first_vec_x0)
-
+# ifdef USE_AS_STRNLEN
+ /* + CHAR_SIZE because it simplies the logic in
+ last_4x_vec_or_less. */
+ leaq (VEC_SIZE * 5 + CHAR_SIZE)(%rdi), %rcx
+ subq %rdx, %rcx
+# ifdef USE_AS_WCSLEN
+ /* NB: Divide bytes by 4 to get the wchar_t count. */
+ sarl $2, %ecx
+# endif
+# endif
+ /* Load first VEC regardless. */
VPCMP $0, VEC_SIZE(%rdi), %YMMZERO, %k0
+# ifdef USE_AS_STRNLEN
+ /* Adjust length. If near end handle specially. */
+ subq %rcx, %rsi
+ jb L(last_4x_vec_or_less)
+# endif
kmovd %k0, %eax
testl %eax, %eax
jnz L(first_vec_x1)
VPCMP $0, (VEC_SIZE * 2)(%rdi), %YMMZERO, %k0
kmovd %k0, %eax
- testl %eax, %eax
+ test %eax, %eax
jnz L(first_vec_x2)
VPCMP $0, (VEC_SIZE * 3)(%rdi), %YMMZERO, %k0
@@ -179,258 +214,276 @@ L(more_4x_vec):
testl %eax, %eax
jnz L(first_vec_x3)
- addq $(VEC_SIZE * 4), %rdi
-
-# ifdef USE_AS_STRNLEN
- subq $(VEC_SIZE * 4), %rsi
- jbe L(last_4x_vec_or_less)
-# endif
-
- /* Align data to 4 * VEC_SIZE. */
- movq %rdi, %rcx
- andl $(4 * VEC_SIZE - 1), %ecx
- andq $-(4 * VEC_SIZE), %rdi
+ VPCMP $0, (VEC_SIZE * 4)(%rdi), %YMMZERO, %k0
+ kmovd %k0, %eax
+ testl %eax, %eax
+ jnz L(first_vec_x4)
+ addq $VEC_SIZE, %rdi
# ifdef USE_AS_STRNLEN
- /* Adjust length. */
+ /* Check if at last VEC_SIZE * 4 length. */
+ cmpq $(CHAR_PER_VEC * 4 - 1), %rsi
+ jbe L(last_4x_vec_or_less_load)
+ movl %edi, %ecx
+ andl $(VEC_SIZE * 4 - 1), %ecx
+# ifdef USE_AS_WCSLEN
+ /* NB: Divide bytes by 4 to get the wchar_t count. */
+ sarl $2, %ecx
+# endif
+ /* Readjust length. */
addq %rcx, %rsi
# endif
+ /* Align data to VEC_SIZE * 4. */
+ andq $-(VEC_SIZE * 4), %rdi
+ /* Compare 4 * VEC at a time forward. */
.p2align 4
L(loop_4x_vec):
- /* Compare 4 * VEC at a time forward. */
- VMOVA (%rdi), %YMM1
- VMOVA VEC_SIZE(%rdi), %YMM2
- VMOVA (VEC_SIZE * 2)(%rdi), %YMM3
- VMOVA (VEC_SIZE * 3)(%rdi), %YMM4
-
- VPMINU %YMM1, %YMM2, %YMM5
- VPMINU %YMM3, %YMM4, %YMM6
+ /* Load first VEC regardless. */
+ VMOVA (VEC_SIZE * 4)(%rdi), %YMM1
+# ifdef USE_AS_STRNLEN
+ /* Break if at end of length. */
+ subq $(CHAR_PER_VEC * 4), %rsi
+ jb L(last_4x_vec_or_less_cmpeq)
+# endif
+ /* Save some code size by microfusing VPMINU with the load. Since
+ the matches in ymm2/ymm4 can only be returned if there where no
+ matches in ymm1/ymm3 respectively there is no issue with overlap.
+ */
+ VPMINU (VEC_SIZE * 5)(%rdi), %YMM1, %YMM2
+ VMOVA (VEC_SIZE * 6)(%rdi), %YMM3
+ VPMINU (VEC_SIZE * 7)(%rdi), %YMM3, %YMM4
+
+ VPCMP $0, %YMM2, %YMMZERO, %k0
+ VPCMP $0, %YMM4, %YMMZERO, %k1
+ subq $-(VEC_SIZE * 4), %rdi
+ kortestd %k0, %k1
+ jz L(loop_4x_vec)
+
+ /* Check if end was in first half. */
+ kmovd %k0, %eax
+ subq %rdx, %rdi
+# ifdef USE_AS_WCSLEN
+ shrq $2, %rdi
+# endif
+ testl %eax, %eax
+ jz L(second_vec_return)
- VPMINU %YMM5, %YMM6, %YMM5
- VPCMP $0, %YMM5, %YMMZERO, %k0
- ktestd %k0, %k0
- jnz L(4x_vec_end)
+ VPCMP $0, %YMM1, %YMMZERO, %k2
+ kmovd %k2, %edx
+ /* Combine VEC1 matches (edx) with VEC2 matches (eax). */
+# ifdef USE_AS_WCSLEN
+ sall $CHAR_PER_VEC, %eax
+ orl %edx, %eax
+ tzcntl %eax, %eax
+# else
+ salq $CHAR_PER_VEC, %rax
+ orq %rdx, %rax
+ tzcntq %rax, %rax
+# endif
+ addq %rdi, %rax
+ ret
- addq $(VEC_SIZE * 4), %rdi
-# ifndef USE_AS_STRNLEN
- jmp L(loop_4x_vec)
-# else
- subq $(VEC_SIZE * 4), %rsi
- ja L(loop_4x_vec)
+# ifdef USE_AS_STRNLEN
+L(last_4x_vec_or_less_load):
+ /* Depending on entry adjust rdi / prepare first VEC in YMM1. */
+ VMOVA (VEC_SIZE * 4)(%rdi), %YMM1
+L(last_4x_vec_or_less_cmpeq):
+ VPCMP $0, %YMM1, %YMMZERO, %k0
+ addq $(VEC_SIZE * 3), %rdi
L(last_4x_vec_or_less):
- /* Less than 4 * VEC and aligned to VEC_SIZE. */
- addl $(VEC_SIZE * 2), %esi
- jle L(last_2x_vec)
-
- VPCMP $0, (%rdi), %YMMZERO, %k0
kmovd %k0, %eax
+ /* If remaining length > VEC_SIZE * 2. This works if esi is off by
+ VEC_SIZE * 4. */
+ testl $(CHAR_PER_VEC * 2), %esi
+ jnz L(last_4x_vec)
+
+ /* length may have been negative or positive by an offset of
+ CHAR_PER_VEC * 4 depending on where this was called from. This
+ fixes that. */
+ andl $(CHAR_PER_VEC * 4 - 1), %esi
testl %eax, %eax
- jnz L(first_vec_x0)
+ jnz L(last_vec_x1_check)
- VPCMP $0, VEC_SIZE(%rdi), %YMMZERO, %k0
- kmovd %k0, %eax
- testl %eax, %eax
- jnz L(first_vec_x1)
+ /* Check the end of data. */
+ subl $CHAR_PER_VEC, %esi
+ jb L(max)
VPCMP $0, (VEC_SIZE * 2)(%rdi), %YMMZERO, %k0
kmovd %k0, %eax
- testl %eax, %eax
- jnz L(first_vec_x2_check)
- subl $VEC_SIZE, %esi
- jle L(max)
+ tzcntl %eax, %eax
+ /* Check the end of data. */
+ cmpl %eax, %esi
+ jb L(max)
- VPCMP $0, (VEC_SIZE * 3)(%rdi), %YMMZERO, %k0
- kmovd %k0, %eax
- testl %eax, %eax
- jnz L(first_vec_x3_check)
+ subq %rdx, %rdi
+# ifdef USE_AS_WCSLEN
+ /* NB: Divide bytes by 4 to get the wchar_t count. */
+ sarq $2, %rdi
+# endif
+ leaq (CHAR_PER_VEC * 2)(%rdi, %rax), %rax
+ ret
+L(max):
movq %r8, %rax
+ ret
+# endif
+
+ /* Placed here in strnlen so that the jcc L(last_4x_vec_or_less)
+ in the 4x VEC loop can use 2 byte encoding. */
+ .p2align 4
+L(second_vec_return):
+ VPCMP $0, %YMM3, %YMMZERO, %k0
+ /* Combine YMM3 matches (k0) with YMM4 matches (k1). */
+# ifdef USE_AS_WCSLEN
+ kunpckbw %k0, %k1, %k0
+ kmovd %k0, %eax
+ tzcntl %eax, %eax
+# else
+ kunpckdq %k0, %k1, %k0
+ kmovq %k0, %rax
+ tzcntq %rax, %rax
+# endif
+ leaq (CHAR_PER_VEC * 2)(%rdi, %rax), %rax
+ ret
+
+
+# ifdef USE_AS_STRNLEN
+L(last_vec_x1_check):
+ tzcntl %eax, %eax
+ /* Check the end of data. */
+ cmpl %eax, %esi
+ jb L(max)
+ subq %rdx, %rdi
# ifdef USE_AS_WCSLEN
- shrq $2, %rax
+ /* NB: Divide bytes by 4 to get the wchar_t count. */
+ sarq $2, %rdi
# endif
+ leaq (CHAR_PER_VEC)(%rdi, %rax), %rax
ret
.p2align 4
-L(last_2x_vec):
- addl $(VEC_SIZE * 2), %esi
+L(last_4x_vec):
+ /* Test first 2x VEC normally. */
+ testl %eax, %eax
+ jnz L(last_vec_x1)
- VPCMP $0, (%rdi), %YMMZERO, %k0
+ VPCMP $0, (VEC_SIZE * 2)(%rdi), %YMMZERO, %k0
kmovd %k0, %eax
testl %eax, %eax
- jnz L(first_vec_x0_check)
- subl $VEC_SIZE, %esi
- jle L(max)
+ jnz L(last_vec_x2)
- VPCMP $0, VEC_SIZE(%rdi), %YMMZERO, %k0
+ /* Normalize length. */
+ andl $(CHAR_PER_VEC * 4 - 1), %esi
+ VPCMP $0, (VEC_SIZE * 3)(%rdi), %YMMZERO, %k0
kmovd %k0, %eax
testl %eax, %eax
- jnz L(first_vec_x1_check)
- movq %r8, %rax
-# ifdef USE_AS_WCSLEN
- shrq $2, %rax
-# endif
- ret
+ jnz L(last_vec_x3)
- .p2align 4
-L(first_vec_x0_check):
+ /* Check the end of data. */
+ subl $(CHAR_PER_VEC * 3), %esi
+ jb L(max)
+
+ VPCMP $0, (VEC_SIZE * 4)(%rdi), %YMMZERO, %k0
+ kmovd %k0, %eax
tzcntl %eax, %eax
-# ifdef USE_AS_WCSLEN
- /* NB: Multiply wchar_t count by 4 to get the number of bytes. */
- sall $2, %eax
-# endif
/* Check the end of data. */
- cmpq %rax, %rsi
- jbe L(max)
- addq %rdi, %rax
- subq %rdx, %rax
+ cmpl %eax, %esi
+ jb L(max_end)
+
+ subq %rdx, %rdi
# ifdef USE_AS_WCSLEN
- shrq $2, %rax
+ /* NB: Divide bytes by 4 to get the wchar_t count. */
+ sarq $2, %rdi
# endif
+ leaq (CHAR_PER_VEC * 4)(%rdi, %rax), %rax
ret
.p2align 4
-L(first_vec_x1_check):
+L(last_vec_x1):
tzcntl %eax, %eax
+ subq %rdx, %rdi
# ifdef USE_AS_WCSLEN
- /* NB: Multiply wchar_t count by 4 to get the number of bytes. */
- sall $2, %eax
-# endif
- /* Check the end of data. */
- cmpq %rax, %rsi
- jbe L(max)
- addq $VEC_SIZE, %rax
- addq %rdi, %rax
- subq %rdx, %rax
-# ifdef USE_AS_WCSLEN
- shrq $2, %rax
+ /* NB: Divide bytes by 4 to get the wchar_t count. */
+ sarq $2, %rdi
# endif
+ leaq (CHAR_PER_VEC)(%rdi, %rax), %rax
ret
.p2align 4
-L(first_vec_x2_check):
+L(last_vec_x2):
tzcntl %eax, %eax
+ subq %rdx, %rdi
# ifdef USE_AS_WCSLEN
- /* NB: Multiply wchar_t count by 4 to get the number of bytes. */
- sall $2, %eax
-# endif
- /* Check the end of data. */
- cmpq %rax, %rsi
- jbe L(max)
- addq $(VEC_SIZE * 2), %rax
- addq %rdi, %rax
- subq %rdx, %rax
-# ifdef USE_AS_WCSLEN
- shrq $2, %rax
+ /* NB: Divide bytes by 4 to get the wchar_t count. */
+ sarq $2, %rdi
# endif
+ leaq (CHAR_PER_VEC * 2)(%rdi, %rax), %rax
ret
.p2align 4
-L(first_vec_x3_check):
+L(last_vec_x3):
tzcntl %eax, %eax
-# ifdef USE_AS_WCSLEN
- /* NB: Multiply wchar_t count by 4 to get the number of bytes. */
- sall $2, %eax
-# endif
+ subl $(CHAR_PER_VEC * 2), %esi
/* Check the end of data. */
- cmpq %rax, %rsi
- jbe L(max)
- addq $(VEC_SIZE * 3), %rax
- addq %rdi, %rax
- subq %rdx, %rax
+ cmpl %eax, %esi
+ jb L(max_end)
+ subq %rdx, %rdi
# ifdef USE_AS_WCSLEN
- shrq $2, %rax
+ /* NB: Divide bytes by 4 to get the wchar_t count. */
+ sarq $2, %rdi
# endif
+ leaq (CHAR_PER_VEC * 3)(%rdi, %rax), %rax
ret
-
- .p2align 4
-L(max):
+L(max_end):
movq %r8, %rax
-# ifdef USE_AS_WCSLEN
- shrq $2, %rax
-# endif
- ret
-
- .p2align 4
-L(zero):
- xorl %eax, %eax
ret
# endif
+ /* Cold case for crossing page with first load. */
.p2align 4
-L(first_vec_x0):
- tzcntl %eax, %eax
-# ifdef USE_AS_WCSLEN
- /* NB: Multiply wchar_t count by 4 to get the number of bytes. */
- sall $2, %eax
-# endif
- addq %rdi, %rax
- subq %rdx, %rax
+L(cross_page_boundary):
+ movq %rdi, %rdx
+ /* Align data to VEC_SIZE. */
+ andq $-VEC_SIZE, %rdi
+ VPCMP $0, (%rdi), %YMMZERO, %k0
+ kmovd %k0, %eax
+ /* Remove the leading bytes. */
# ifdef USE_AS_WCSLEN
- shrq $2, %rax
+ /* NB: Divide shift count by 4 since each bit in K0 represent 4
+ bytes. */
+ movl %edx, %ecx
+ shrl $2, %ecx
+ andl $(CHAR_PER_VEC - 1), %ecx
# endif
- ret
-
- .p2align 4
-L(first_vec_x1):
+ /* SHIFT_REG is ecx for USE_AS_WCSLEN and edx otherwise. */
+ sarxl %SHIFT_REG, %eax, %eax
+ testl %eax, %eax
+# ifndef USE_AS_STRNLEN
+ jz L(cross_page_continue)
tzcntl %eax, %eax
-# ifdef USE_AS_WCSLEN
- /* NB: Multiply wchar_t count by 4 to get the number of bytes. */
- sall $2, %eax
-# endif
- addq $VEC_SIZE, %rax
- addq %rdi, %rax
- subq %rdx, %rax
-# ifdef USE_AS_WCSLEN
- shrq $2, %rax
-# endif
ret
-
- .p2align 4
-L(first_vec_x2):
- tzcntl %eax, %eax
-# ifdef USE_AS_WCSLEN
- /* NB: Multiply wchar_t count by 4 to get the number of bytes. */
- sall $2, %eax
-# endif
- addq $(VEC_SIZE * 2), %rax
- addq %rdi, %rax
- subq %rdx, %rax
-# ifdef USE_AS_WCSLEN
- shrq $2, %rax
-# endif
+# else
+ jnz L(cross_page_less_vec)
+# ifndef USE_AS_WCSLEN
+ movl %edx, %ecx
+ andl $(CHAR_PER_VEC - 1), %ecx
+# endif
+ movl $CHAR_PER_VEC, %eax
+ subl %ecx, %eax
+ /* Check the end of data. */
+ cmpq %rax, %rsi
+ ja L(cross_page_continue)
+ movl %esi, %eax
ret
-
- .p2align 4
-L(4x_vec_end):
- VPCMP $0, %YMM1, %YMMZERO, %k0
- kmovd %k0, %eax
- testl %eax, %eax
- jnz L(first_vec_x0)
- VPCMP $0, %YMM2, %YMMZERO, %k1
- kmovd %k1, %eax
- testl %eax, %eax
- jnz L(first_vec_x1)
- VPCMP $0, %YMM3, %YMMZERO, %k2
- kmovd %k2, %eax
- testl %eax, %eax
- jnz L(first_vec_x2)
- VPCMP $0, %YMM4, %YMMZERO, %k3
- kmovd %k3, %eax
-L(first_vec_x3):
+L(cross_page_less_vec):
tzcntl %eax, %eax
-# ifdef USE_AS_WCSLEN
- /* NB: Multiply wchar_t count by 4 to get the number of bytes. */
- sall $2, %eax
-# endif
- addq $(VEC_SIZE * 3), %rax
- addq %rdi, %rax
- subq %rdx, %rax
-# ifdef USE_AS_WCSLEN
- shrq $2, %rax
-# endif
+ /* Select min of length and position of first null. */
+ cmpq %rax, %rsi
+ cmovb %esi, %eax
ret
+# endif
END (STRLEN)
#endif
--
GitLab

View File

@ -1,158 +0,0 @@
From ea8e465a6b8d0f26c72bcbe453a854de3abf68ec Mon Sep 17 00:00:00 2001
From: "H.J. Lu" <hjl.tools@gmail.com>
Date: Wed, 30 Jun 2021 10:47:06 -0700
Subject: [PATCH] x86: Check RTM_ALWAYS_ABORT for RTM [BZ #28033]
Content-type: text/plain; charset=UTF-8
From
https://www.intel.com/content/www/us/en/support/articles/000059422/processors.html
* Intel TSX will be disabled by default.
* The processor will force abort all Restricted Transactional Memory (RTM)
transactions by default.
* A new CPUID bit CPUID.07H.0H.EDX[11](RTM_ALWAYS_ABORT) will be enumerated,
which is set to indicate to updated software that the loaded microcode is
forcing RTM abort.
* On processors that enumerate support for RTM, the CPUID enumeration bits
for Intel TSX (CPUID.07H.0H.EBX[11] and CPUID.07H.0H.EBX[4]) continue to
be set by default after microcode update.
* Workloads that were benefited from Intel TSX might experience a change
in performance.
* System software may use a new bit in Model-Specific Register (MSR) 0x10F
TSX_FORCE_ABORT[TSX_CPUID_CLEAR] functionality to clear the Hardware Lock
Elision (HLE) and RTM bits to indicate to software that Intel TSX is
disabled.
1. Add RTM_ALWAYS_ABORT to CPUID features.
2. Set RTM usable only if RTM_ALWAYS_ABORT isn't set. This skips the
string/tst-memchr-rtm etc. testcases on the affected processors, which
always fail after a microcde update.
3. Check RTM feature, instead of usability, against /proc/cpuinfo.
This fixes BZ #28033.
---
manual/platform.texi | 3 +++
sysdeps/x86/cpu-features.c | 5 ++++-
sysdeps/x86/sys/platform/x86.h | 6 +++---
sysdeps/x86/tst-cpu-features-supports.c | 2 +-
sysdeps/x86/tst-get-cpu-features.c | 2 ++
5 files changed, 13 insertions(+), 5 deletions(-)
Conflicts:
sysdeps/x86/bits/platform/x86.h
(doesn't exist)
sysdeps/x86/bits/platform/x86.h
(account for lack of upstream renames)
diff --git a/manual/platform.texi b/manual/platform.texi
index 8fec2933..b7e8aef7 100644
--- a/manual/platform.texi
+++ b/manual/platform.texi
@@ -510,6 +510,9 @@ capability.
@item
@code{RTM} -- RTM instruction extensions.
+@item
+@code{RTM_ALWAYS_ABORT} -- Transactions always abort, making RTM unusable.
+
@item
@code{SDBG} -- IA32_DEBUG_INTERFACE MSR for silicon debug.
diff --git a/sysdeps/x86/cpu-features.c b/sysdeps/x86/cpu-features.c
index 3610ee5c..4889f062 100644
--- a/sysdeps/x86/cpu-features.c
+++ b/sysdeps/x86/cpu-features.c
@@ -74,7 +74,6 @@ update_usable (struct cpu_features *cpu_features)
CPU_FEATURE_SET_USABLE (cpu_features, HLE);
CPU_FEATURE_SET_USABLE (cpu_features, BMI2);
CPU_FEATURE_SET_USABLE (cpu_features, ERMS);
- CPU_FEATURE_SET_USABLE (cpu_features, RTM);
CPU_FEATURE_SET_USABLE (cpu_features, RDSEED);
CPU_FEATURE_SET_USABLE (cpu_features, ADX);
CPU_FEATURE_SET_USABLE (cpu_features, CLFLUSHOPT);
@@ -90,6 +89,7 @@ update_usable (struct cpu_features *cpu_features)
CPU_FEATURE_SET_USABLE (cpu_features, MOVDIRI);
CPU_FEATURE_SET_USABLE (cpu_features, MOVDIR64B);
CPU_FEATURE_SET_USABLE (cpu_features, FSRM);
+ CPU_FEATURE_SET_USABLE (cpu_features, RTM_ALWAYS_ABORT);
CPU_FEATURE_SET_USABLE (cpu_features, SERIALIZE);
CPU_FEATURE_SET_USABLE (cpu_features, TSXLDTRK);
CPU_FEATURE_SET_USABLE (cpu_features, LAHF64_SAHF64);
@@ -779,6 +779,9 @@ no_cpuid:
GLRO(dl_platform) = "i586";
#endif
+ if (!CPU_FEATURES_CPU_P (cpu_features, RTM_ALWAYS_ABORT))
+ CPU_FEATURE_SET_USABLE (cpu_features, RTM);
+
#if CET_ENABLED
# if HAVE_TUNABLES
TUNABLE_GET (x86_ibt, tunable_val_t *,
diff --git a/sysdeps/x86/sys/platform/x86.h b/sysdeps/x86/sys/platform/x86.h
index e5cc7c68..7a434926 100644
--- a/sysdeps/x86/sys/platform/x86.h
+++ b/sysdeps/x86/sys/platform/x86.h
@@ -247,7 +247,7 @@ extern const struct cpu_features *__x86_get_cpu_features (unsigned int)
#define bit_cpu_AVX512_VP2INTERSECT (1u << 8)
#define bit_cpu_INDEX_7_EDX_9 (1u << 9)
#define bit_cpu_MD_CLEAR (1u << 10)
-#define bit_cpu_INDEX_7_EDX_11 (1u << 11)
+#define bit_cpu_RTM_ALWAYS_ABORT (1u << 11)
#define bit_cpu_INDEX_7_EDX_12 (1u << 12)
#define bit_cpu_INDEX_7_EDX_13 (1u << 13)
#define bit_cpu_SERIALIZE (1u << 14)
@@ -471,7 +471,7 @@ extern const struct cpu_features *__x86_get_cpu_features (unsigned int)
#define index_cpu_AVX512_VP2INTERSECT COMMON_CPUID_INDEX_7
#define index_cpu_INDEX_7_EDX_9 COMMON_CPUID_INDEX_7
#define index_cpu_MD_CLEAR COMMON_CPUID_INDEX_7
-#define index_cpu_INDEX_7_EDX_11 COMMON_CPUID_INDEX_7
+#define index_cpu_RTM_ALWAYS_ABORT COMMON_CPUID_INDEX_7
#define index_cpu_INDEX_7_EDX_12 COMMON_CPUID_INDEX_7
#define index_cpu_INDEX_7_EDX_13 COMMON_CPUID_INDEX_7
#define index_cpu_SERIALIZE COMMON_CPUID_INDEX_7
@@ -695,7 +695,7 @@ extern const struct cpu_features *__x86_get_cpu_features (unsigned int)
#define reg_AVX512_VP2INTERSECT edx
#define reg_INDEX_7_EDX_9 edx
#define reg_MD_CLEAR edx
-#define reg_INDEX_7_EDX_11 edx
+#define reg_RTM_ALWAYS_ABORT edx
#define reg_INDEX_7_EDX_12 edx
#define reg_INDEX_7_EDX_13 edx
#define reg_SERIALIZE edx
diff --git a/sysdeps/x86/tst-cpu-features-supports.c b/sysdeps/x86/tst-cpu-features-supports.c
index 287cf01f..8100a319 100644
--- a/sysdeps/x86/tst-cpu-features-supports.c
+++ b/sysdeps/x86/tst-cpu-features-supports.c
@@ -152,7 +152,7 @@ do_test (int argc, char **argv)
fails += CHECK_SUPPORTS (rdpid, RDPID);
fails += CHECK_SUPPORTS (rdrnd, RDRAND);
fails += CHECK_SUPPORTS (rdseed, RDSEED);
- fails += CHECK_SUPPORTS (rtm, RTM);
+ fails += CHECK_CPU_SUPPORTS (rtm, RTM);
fails += CHECK_SUPPORTS (serialize, SERIALIZE);
fails += CHECK_SUPPORTS (sha, SHA);
fails += CHECK_CPU_SUPPORTS (shstk, SHSTK);
diff --git a/sysdeps/x86/tst-get-cpu-features.c b/sysdeps/x86/tst-get-cpu-features.c
index 2763deb6..0717e5d8 100644
--- a/sysdeps/x86/tst-get-cpu-features.c
+++ b/sysdeps/x86/tst-get-cpu-features.c
@@ -183,6 +183,7 @@ do_test (void)
CHECK_CPU_FEATURE (UINTR);
CHECK_CPU_FEATURE (AVX512_VP2INTERSECT);
CHECK_CPU_FEATURE (MD_CLEAR);
+ CHECK_CPU_FEATURE (RTM_ALWAYS_ABORT);
CHECK_CPU_FEATURE (SERIALIZE);
CHECK_CPU_FEATURE (HYBRID);
CHECK_CPU_FEATURE (TSXLDTRK);
@@ -344,6 +345,7 @@ do_test (void)
CHECK_CPU_FEATURE_USABLE (FSRM);
CHECK_CPU_FEATURE_USABLE (AVX512_VP2INTERSECT);
CHECK_CPU_FEATURE_USABLE (MD_CLEAR);
+ CHECK_CPU_FEATURE_USABLE (RTM_ALWAYS_ABORT);
CHECK_CPU_FEATURE_USABLE (SERIALIZE);
CHECK_CPU_FEATURE_USABLE (HYBRID);
CHECK_CPU_FEATURE_USABLE (TSXLDTRK);
--
GitLab

View File

@ -1,51 +0,0 @@
From 0679442defedf7e52a94264975880ab8674736b2 Mon Sep 17 00:00:00 2001
From: Noah Goldstein <goldstein.w.n@gmail.com>
Date: Thu, 8 Jul 2021 16:13:19 -0400
Subject: [PATCH] x86: Remove wcsnlen-sse4_1 from wcslen ifunc-impl-list [BZ
#28064]
Content-type: text/plain; charset=UTF-8
The following commit
commit 6f573a27b6c8b4236445810a44660612323f5a73
Author: Noah Goldstein <goldstein.w.n@gmail.com>
Date: Wed Jun 23 01:19:34 2021 -0400
x86-64: Add wcslen optimize for sse4.1
Added wcsnlen-sse4.1 to the wcslen ifunc implementation list and did
not add wcslen-sse4.1 to wcslen ifunc implementation list. This commit
fixes that by removing wcsnlen-sse4.1 from the wcslen ifunc
implementation list and adding wcslen-sse4.1 to the ifunc
implementation list.
Testing:
test-wcslen.c, test-rsi-wcslen.c, and test-rsi-strlen.c are passing as
well as all other tests in wcsmbs and string.
Signed-off-by: Noah Goldstein <goldstein.w.n@gmail.com>
Reviewed-by: H.J. Lu <hjl.tools@gmail.com>
Reviewed-by: H.J. Lu <hjl.tools@gmail.com>
---
sysdeps/x86_64/multiarch/ifunc-impl-list.c | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/sysdeps/x86_64/multiarch/ifunc-impl-list.c b/sysdeps/x86_64/multiarch/ifunc-impl-list.c
index 580913ca..695cdba6 100644
--- a/sysdeps/x86_64/multiarch/ifunc-impl-list.c
+++ b/sysdeps/x86_64/multiarch/ifunc-impl-list.c
@@ -657,9 +657,9 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,
&& CPU_FEATURE_USABLE (AVX512BW)
&& CPU_FEATURE_USABLE (BMI2)),
__wcslen_evex)
- IFUNC_IMPL_ADD (array, i, wcsnlen,
+ IFUNC_IMPL_ADD (array, i, wcslen,
CPU_FEATURE_USABLE (SSE4_1),
- __wcsnlen_sse4_1)
+ __wcslen_sse4_1)
IFUNC_IMPL_ADD (array, i, wcslen, 1, __wcslen_sse2))
/* Support sysdeps/x86_64/multiarch/wcsnlen.c. */
--
GitLab

View File

@ -1,51 +0,0 @@
From 55c7bcc71b84123d5d4bd2814366a6b05fcf8ebd Mon Sep 17 00:00:00 2001
From: "H.J. Lu" <hjl.tools@gmail.com>
Date: Sat, 9 May 2020 12:04:23 -0700
Subject: [PATCH] x86-64: Use RDX_LP on __x86_shared_non_temporal_threshold [BZ
#25966]
Content-type: text/plain; charset=UTF-8
Since __x86_shared_non_temporal_threshold is defined as
long int __x86_shared_non_temporal_threshold;
and long int is 4 bytes for x32, use RDX_LP to compare against
__x86_shared_non_temporal_threshold in assembly code.
---
sysdeps/x86_64/multiarch/memmove-vec-unaligned-erms.S | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/sysdeps/x86_64/multiarch/memmove-vec-unaligned-erms.S b/sysdeps/x86_64/multiarch/memmove-vec-unaligned-erms.S
index 71f5954d..673b73aa 100644
--- a/sysdeps/x86_64/multiarch/memmove-vec-unaligned-erms.S
+++ b/sysdeps/x86_64/multiarch/memmove-vec-unaligned-erms.S
@@ -245,7 +245,7 @@ L(return):
#endif
L(movsb):
- cmpq __x86_shared_non_temporal_threshold(%rip), %rdx
+ cmp __x86_shared_non_temporal_threshold(%rip), %RDX_LP
jae L(more_8x_vec)
cmpq %rsi, %rdi
jb 1f
@@ -397,7 +397,7 @@ L(more_8x_vec):
addq %r8, %rdx
#if (defined USE_MULTIARCH || VEC_SIZE == 16) && IS_IN (libc)
/* Check non-temporal store threshold. */
- cmpq __x86_shared_non_temporal_threshold(%rip), %rdx
+ cmp __x86_shared_non_temporal_threshold(%rip), %RDX_LP
ja L(large_forward)
#endif
L(loop_4x_vec_forward):
@@ -448,7 +448,7 @@ L(more_8x_vec_backward):
subq %r8, %rdx
#if (defined USE_MULTIARCH || VEC_SIZE == 16) && IS_IN (libc)
/* Check non-temporal store threshold. */
- cmpq __x86_shared_non_temporal_threshold(%rip), %rdx
+ cmp __x86_shared_non_temporal_threshold(%rip), %RDX_LP
ja L(large_backward)
#endif
L(loop_4x_vec_backward):
--
GitLab

View File

@ -1,44 +0,0 @@
From a35a59036ebae3efcdf5e8167610e0656fca9770 Mon Sep 17 00:00:00 2001
From: "H.J. Lu" <hjl.tools@gmail.com>
Date: Thu, 11 Jun 2020 12:41:18 -0700
Subject: [PATCH] x86_64: Use %xmmN with vpxor to clear a vector register
Content-type: text/plain; charset=UTF-8
Since "vpxor %xmmN, %xmmN, %xmmN" clears the whole vector register, use
%xmmN, instead of %ymmN, with vpxor to clear a vector register.
---
sysdeps/x86_64/multiarch/strcmp-avx2.S | 4 ++--
sysdeps/x86_64/multiarch/strrchr-avx2.S | 2 +-
2 files changed, 3 insertions(+), 3 deletions(-)
diff --git a/sysdeps/x86_64/multiarch/strcmp-avx2.S b/sysdeps/x86_64/multiarch/strcmp-avx2.S
index 433ae047..70d8499b 100644
--- a/sysdeps/x86_64/multiarch/strcmp-avx2.S
+++ b/sysdeps/x86_64/multiarch/strcmp-avx2.S
@@ -105,8 +105,8 @@ ENTRY (STRCMP)
# endif
movl %edi, %eax
xorl %edx, %edx
- /* Make %ymm7 all zeros in this function. */
- vpxor %ymm7, %ymm7, %ymm7
+ /* Make %xmm7 (%ymm7) all zeros in this function. */
+ vpxor %xmm7, %xmm7, %xmm7
orl %esi, %eax
andl $(PAGE_SIZE - 1), %eax
cmpl $(PAGE_SIZE - (VEC_SIZE * 4)), %eax
diff --git a/sysdeps/x86_64/multiarch/strrchr-avx2.S b/sysdeps/x86_64/multiarch/strrchr-avx2.S
index 9f22a15e..c949410b 100644
--- a/sysdeps/x86_64/multiarch/strrchr-avx2.S
+++ b/sysdeps/x86_64/multiarch/strrchr-avx2.S
@@ -48,7 +48,7 @@ ENTRY (STRRCHR)
movl %edi, %ecx
/* Broadcast CHAR to YMM4. */
VPBROADCAST %xmm4, %ymm4
- vpxor %ymm0, %ymm0, %ymm0
+ vpxor %xmm0, %xmm0, %xmm0
/* Check if we may cross page boundary with one vector load. */
andl $(2 * VEC_SIZE - 1), %ecx
--
GitLab

View File

@ -1,359 +0,0 @@
From 1f745ecc2109890886b161d4791e1406fdfc29b8 Mon Sep 17 00:00:00 2001
From: noah <goldstein.w.n@gmail.com>
Date: Wed, 3 Feb 2021 00:38:59 -0500
Subject: [PATCH] x86-64: Refactor and improve performance of strchr-avx2.S
Content-type: text/plain; charset=UTF-8
No bug. Just seemed the performance could be improved a bit. Observed
and expected behavior are unchanged. Optimized body of main
loop. Updated page cross logic and optimized accordingly. Made a few
minor instruction selection modifications. No regressions in test
suite. Both test-strchrnul and test-strchr passed.
---
sysdeps/x86_64/multiarch/strchr-avx2.S | 225 ++++++++++++-------------
sysdeps/x86_64/multiarch/strchr.c | 4 +-
2 files changed, 114 insertions(+), 115 deletions(-)
Conflicts:
sysdeps/x86_64/multiarch/strchr.c
(account for missing upstream macros)
diff --git a/sysdeps/x86_64/multiarch/strchr-avx2.S b/sysdeps/x86_64/multiarch/strchr-avx2.S
index da7d2620..919d256c 100644
--- a/sysdeps/x86_64/multiarch/strchr-avx2.S
+++ b/sysdeps/x86_64/multiarch/strchr-avx2.S
@@ -27,10 +27,12 @@
# ifdef USE_AS_WCSCHR
# define VPBROADCAST vpbroadcastd
# define VPCMPEQ vpcmpeqd
+# define VPMINU vpminud
# define CHAR_REG esi
# else
# define VPBROADCAST vpbroadcastb
# define VPCMPEQ vpcmpeqb
+# define VPMINU vpminub
# define CHAR_REG sil
# endif
@@ -43,71 +45,54 @@
# endif
# define VEC_SIZE 32
+# define PAGE_SIZE 4096
.section SECTION(.text),"ax",@progbits
ENTRY (STRCHR)
movl %edi, %ecx
- /* Broadcast CHAR to YMM0. */
+# ifndef USE_AS_STRCHRNUL
+ xorl %edx, %edx
+# endif
+
+ /* Broadcast CHAR to YMM0. */
vmovd %esi, %xmm0
vpxor %xmm9, %xmm9, %xmm9
VPBROADCAST %xmm0, %ymm0
- /* Check if we may cross page boundary with one vector load. */
- andl $(2 * VEC_SIZE - 1), %ecx
- cmpl $VEC_SIZE, %ecx
- ja L(cros_page_boundary)
- /* Check the first VEC_SIZE bytes. Search for both CHAR and the
- null byte. */
- vmovdqu (%rdi), %ymm8
- VPCMPEQ %ymm8, %ymm0, %ymm1
- VPCMPEQ %ymm8, %ymm9, %ymm2
- vpor %ymm1, %ymm2, %ymm1
- vpmovmskb %ymm1, %eax
- testl %eax, %eax
- jnz L(first_vec_x0)
+ /* Check if we cross page boundary with one vector load. */
+ andl $(PAGE_SIZE - 1), %ecx
+ cmpl $(PAGE_SIZE - VEC_SIZE), %ecx
+ ja L(cross_page_boundary)
- /* Align data for aligned loads in the loop. */
- addq $VEC_SIZE, %rdi
- andl $(VEC_SIZE - 1), %ecx
- andq $-VEC_SIZE, %rdi
-
- jmp L(more_4x_vec)
-
- .p2align 4
-L(cros_page_boundary):
- andl $(VEC_SIZE - 1), %ecx
- andq $-VEC_SIZE, %rdi
+ /* Check the first VEC_SIZE bytes. Search for both CHAR and the
+ null byte. */
vmovdqu (%rdi), %ymm8
VPCMPEQ %ymm8, %ymm0, %ymm1
VPCMPEQ %ymm8, %ymm9, %ymm2
vpor %ymm1, %ymm2, %ymm1
vpmovmskb %ymm1, %eax
- /* Remove the leading bytes. */
- sarl %cl, %eax
testl %eax, %eax
- jz L(aligned_more)
- /* Found CHAR or the null byte. */
+ jz L(more_vecs)
tzcntl %eax, %eax
- addq %rcx, %rax
-# ifdef USE_AS_STRCHRNUL
+ /* Found CHAR or the null byte. */
addq %rdi, %rax
-# else
- xorl %edx, %edx
- leaq (%rdi, %rax), %rax
- cmp (%rax), %CHAR_REG
+# ifndef USE_AS_STRCHRNUL
+ cmp (%rax), %CHAR_REG
cmovne %rdx, %rax
# endif
L(return_vzeroupper):
ZERO_UPPER_VEC_REGISTERS_RETURN
.p2align 4
+L(more_vecs):
+ /* Align data for aligned loads in the loop. */
+ andq $-VEC_SIZE, %rdi
L(aligned_more):
- addq $VEC_SIZE, %rdi
-L(more_4x_vec):
- /* Check the first 4 * VEC_SIZE. Only one VEC_SIZE at a time
- since data is only aligned to VEC_SIZE. */
- vmovdqa (%rdi), %ymm8
+ /* Check the next 4 * VEC_SIZE. Only one VEC_SIZE at a time
+ since data is only aligned to VEC_SIZE. */
+ vmovdqa VEC_SIZE(%rdi), %ymm8
+ addq $VEC_SIZE, %rdi
VPCMPEQ %ymm8, %ymm0, %ymm1
VPCMPEQ %ymm8, %ymm9, %ymm2
vpor %ymm1, %ymm2, %ymm1
@@ -137,61 +122,24 @@ L(more_4x_vec):
vpor %ymm1, %ymm2, %ymm1
vpmovmskb %ymm1, %eax
testl %eax, %eax
- jnz L(first_vec_x3)
-
- addq $(VEC_SIZE * 4), %rdi
-
- /* Align data to 4 * VEC_SIZE. */
- movq %rdi, %rcx
- andl $(4 * VEC_SIZE - 1), %ecx
- andq $-(4 * VEC_SIZE), %rdi
-
- .p2align 4
-L(loop_4x_vec):
- /* Compare 4 * VEC at a time forward. */
- vmovdqa (%rdi), %ymm5
- vmovdqa VEC_SIZE(%rdi), %ymm6
- vmovdqa (VEC_SIZE * 2)(%rdi), %ymm7
- vmovdqa (VEC_SIZE * 3)(%rdi), %ymm8
-
- VPCMPEQ %ymm5, %ymm0, %ymm1
- VPCMPEQ %ymm6, %ymm0, %ymm2
- VPCMPEQ %ymm7, %ymm0, %ymm3
- VPCMPEQ %ymm8, %ymm0, %ymm4
-
- VPCMPEQ %ymm5, %ymm9, %ymm5
- VPCMPEQ %ymm6, %ymm9, %ymm6
- VPCMPEQ %ymm7, %ymm9, %ymm7
- VPCMPEQ %ymm8, %ymm9, %ymm8
-
- vpor %ymm1, %ymm5, %ymm1
- vpor %ymm2, %ymm6, %ymm2
- vpor %ymm3, %ymm7, %ymm3
- vpor %ymm4, %ymm8, %ymm4
-
- vpor %ymm1, %ymm2, %ymm5
- vpor %ymm3, %ymm4, %ymm6
-
- vpor %ymm5, %ymm6, %ymm5
-
- vpmovmskb %ymm5, %eax
- testl %eax, %eax
- jnz L(4x_vec_end)
-
- addq $(VEC_SIZE * 4), %rdi
+ jz L(prep_loop_4x)
- jmp L(loop_4x_vec)
+ tzcntl %eax, %eax
+ leaq (VEC_SIZE * 3)(%rdi, %rax), %rax
+# ifndef USE_AS_STRCHRNUL
+ cmp (%rax), %CHAR_REG
+ cmovne %rdx, %rax
+# endif
+ VZEROUPPER
+ ret
.p2align 4
L(first_vec_x0):
- /* Found CHAR or the null byte. */
tzcntl %eax, %eax
-# ifdef USE_AS_STRCHRNUL
+ /* Found CHAR or the null byte. */
addq %rdi, %rax
-# else
- xorl %edx, %edx
- leaq (%rdi, %rax), %rax
- cmp (%rax), %CHAR_REG
+# ifndef USE_AS_STRCHRNUL
+ cmp (%rax), %CHAR_REG
cmovne %rdx, %rax
# endif
VZEROUPPER_RETURN
@@ -199,13 +147,9 @@ L(first_vec_x0):
.p2align 4
L(first_vec_x1):
tzcntl %eax, %eax
-# ifdef USE_AS_STRCHRNUL
- addq $VEC_SIZE, %rax
- addq %rdi, %rax
-# else
- xorl %edx, %edx
leaq VEC_SIZE(%rdi, %rax), %rax
- cmp (%rax), %CHAR_REG
+# ifndef USE_AS_STRCHRNUL
+ cmp (%rax), %CHAR_REG
cmovne %rdx, %rax
# endif
VZEROUPPER_RETURN
@@ -213,42 +157,97 @@ L(first_vec_x1):
.p2align 4
L(first_vec_x2):
tzcntl %eax, %eax
-# ifdef USE_AS_STRCHRNUL
- addq $(VEC_SIZE * 2), %rax
- addq %rdi, %rax
-# else
- xorl %edx, %edx
+ /* Found CHAR or the null byte. */
leaq (VEC_SIZE * 2)(%rdi, %rax), %rax
- cmp (%rax), %CHAR_REG
+# ifndef USE_AS_STRCHRNUL
+ cmp (%rax), %CHAR_REG
cmovne %rdx, %rax
# endif
VZEROUPPER_RETURN
+L(prep_loop_4x):
+ /* Align data to 4 * VEC_SIZE. */
+ andq $-(VEC_SIZE * 4), %rdi
+
.p2align 4
-L(4x_vec_end):
+L(loop_4x_vec):
+ /* Compare 4 * VEC at a time forward. */
+ vmovdqa (VEC_SIZE * 4)(%rdi), %ymm5
+ vmovdqa (VEC_SIZE * 5)(%rdi), %ymm6
+ vmovdqa (VEC_SIZE * 6)(%rdi), %ymm7
+ vmovdqa (VEC_SIZE * 7)(%rdi), %ymm8
+
+ /* Leaves only CHARS matching esi as 0. */
+ vpxor %ymm5, %ymm0, %ymm1
+ vpxor %ymm6, %ymm0, %ymm2
+ vpxor %ymm7, %ymm0, %ymm3
+ vpxor %ymm8, %ymm0, %ymm4
+
+ VPMINU %ymm1, %ymm5, %ymm1
+ VPMINU %ymm2, %ymm6, %ymm2
+ VPMINU %ymm3, %ymm7, %ymm3
+ VPMINU %ymm4, %ymm8, %ymm4
+
+ VPMINU %ymm1, %ymm2, %ymm5
+ VPMINU %ymm3, %ymm4, %ymm6
+
+ VPMINU %ymm5, %ymm6, %ymm5
+
+ VPCMPEQ %ymm5, %ymm9, %ymm5
+ vpmovmskb %ymm5, %eax
+
+ addq $(VEC_SIZE * 4), %rdi
+ testl %eax, %eax
+ jz L(loop_4x_vec)
+
+ VPCMPEQ %ymm1, %ymm9, %ymm1
vpmovmskb %ymm1, %eax
testl %eax, %eax
jnz L(first_vec_x0)
+
+ VPCMPEQ %ymm2, %ymm9, %ymm2
vpmovmskb %ymm2, %eax
testl %eax, %eax
jnz L(first_vec_x1)
- vpmovmskb %ymm3, %eax
- testl %eax, %eax
- jnz L(first_vec_x2)
+
+ VPCMPEQ %ymm3, %ymm9, %ymm3
+ VPCMPEQ %ymm4, %ymm9, %ymm4
+ vpmovmskb %ymm3, %ecx
vpmovmskb %ymm4, %eax
+ salq $32, %rax
+ orq %rcx, %rax
+ tzcntq %rax, %rax
+ leaq (VEC_SIZE * 2)(%rdi, %rax), %rax
+# ifndef USE_AS_STRCHRNUL
+ cmp (%rax), %CHAR_REG
+ cmovne %rdx, %rax
+# endif
+ VZEROUPPER
+ ret
+
+ /* Cold case for crossing page with first load. */
+ .p2align 4
+L(cross_page_boundary):
+ andq $-VEC_SIZE, %rdi
+ andl $(VEC_SIZE - 1), %ecx
+
+ vmovdqa (%rdi), %ymm8
+ VPCMPEQ %ymm8, %ymm0, %ymm1
+ VPCMPEQ %ymm8, %ymm9, %ymm2
+ vpor %ymm1, %ymm2, %ymm1
+ vpmovmskb %ymm1, %eax
+ /* Remove the leading bits. */
+ sarxl %ecx, %eax, %eax
testl %eax, %eax
-L(first_vec_x3):
+ jz L(aligned_more)
tzcntl %eax, %eax
-# ifdef USE_AS_STRCHRNUL
- addq $(VEC_SIZE * 3), %rax
+ addq %rcx, %rdi
addq %rdi, %rax
-# else
- xorl %edx, %edx
- leaq (VEC_SIZE * 3)(%rdi, %rax), %rax
- cmp (%rax), %CHAR_REG
+# ifndef USE_AS_STRCHRNUL
+ cmp (%rax), %CHAR_REG
cmovne %rdx, %rax
# endif
VZEROUPPER_RETURN
END (STRCHR)
-#endif
+# endif
diff --git a/sysdeps/x86_64/multiarch/strchr.c b/sysdeps/x86_64/multiarch/strchr.c
index 7e582f02..5225bd4f 100644
--- a/sysdeps/x86_64/multiarch/strchr.c
+++ b/sysdeps/x86_64/multiarch/strchr.c
@@ -38,11 +38,11 @@ IFUNC_SELECTOR (void)
const struct cpu_features* cpu_features = __get_cpu_features ();
if (CPU_FEATURE_USABLE_P (cpu_features, AVX2)
+ && CPU_FEATURE_USABLE_P (cpu_features, BMI2)
&& CPU_FEATURES_ARCH_P (cpu_features, AVX_Fast_Unaligned_Load))
{
if (CPU_FEATURE_USABLE_P (cpu_features, AVX512VL)
- && CPU_FEATURE_USABLE_P (cpu_features, AVX512BW)
- && CPU_FEATURE_USABLE_P (cpu_features, BMI2))
+ && CPU_FEATURE_USABLE_P (cpu_features, AVX512BW))
return OPTIMIZE (evex);
if (CPU_FEATURE_USABLE_P (cpu_features, RTM))
--
GitLab

View File

@ -1,67 +0,0 @@
From 3ec5d83d2a237d39e7fd6ef7a0bc8ac4c171a4a5 Mon Sep 17 00:00:00 2001
From: "H.J. Lu" <hjl.tools@gmail.com>
Date: Sat, 25 Jan 2020 14:19:40 -0800
Subject: [PATCH] x86-64: Avoid rep movsb with short distance [BZ #27130]
Content-type: text/plain; charset=UTF-8
When copying with "rep movsb", if the distance between source and
destination is N*4GB + [1..63] with N >= 0, performance may be very
slow. This patch updates memmove-vec-unaligned-erms.S for AVX and
AVX512 versions with the distance in RCX:
cmpl $63, %ecx
// Don't use "rep movsb" if ECX <= 63
jbe L(Don't use rep movsb")
Use "rep movsb"
Benchtests data with bench-memcpy, bench-memcpy-large, bench-memcpy-random
and bench-memcpy-walk on Skylake, Ice Lake and Tiger Lake show that its
performance impact is within noise range as "rep movsb" is only used for
data size >= 4KB.
---
.../multiarch/memmove-vec-unaligned-erms.S | 21 +++++++++++++++++++
1 file changed, 21 insertions(+)
diff --git a/sysdeps/x86_64/multiarch/memmove-vec-unaligned-erms.S b/sysdeps/x86_64/multiarch/memmove-vec-unaligned-erms.S
index 673b73aa..c475fed4 100644
--- a/sysdeps/x86_64/multiarch/memmove-vec-unaligned-erms.S
+++ b/sysdeps/x86_64/multiarch/memmove-vec-unaligned-erms.S
@@ -64,6 +64,13 @@
# endif
#endif
+/* Avoid short distance rep movsb only with non-SSE vector. */
+#ifndef AVOID_SHORT_DISTANCE_REP_MOVSB
+# define AVOID_SHORT_DISTANCE_REP_MOVSB (VEC_SIZE > 16)
+#else
+# define AVOID_SHORT_DISTANCE_REP_MOVSB 0
+#endif
+
#ifndef PREFETCH
# define PREFETCH(addr) prefetcht0 addr
#endif
@@ -255,7 +262,21 @@ L(movsb):
cmpq %r9, %rdi
/* Avoid slow backward REP MOVSB. */
jb L(more_8x_vec_backward)
+# if AVOID_SHORT_DISTANCE_REP_MOVSB
+ movq %rdi, %rcx
+ subq %rsi, %rcx
+ jmp 2f
+# endif
1:
+# if AVOID_SHORT_DISTANCE_REP_MOVSB
+ movq %rsi, %rcx
+ subq %rdi, %rcx
+2:
+/* Avoid "rep movsb" if RCX, the distance between source and destination,
+ is N*4GB + [1..63] with N >= 0. */
+ cmpl $63, %ecx
+ jbe L(more_2x_vec) /* Avoid "rep movsb" if ECX <= 63. */
+# endif
mov %RDX_LP, %RCX_LP
rep movsb
L(nop):
--
GitLab

View File

@ -1,449 +0,0 @@
From 1a8605b6cd257e8a74e29b5b71c057211f5fb847 Mon Sep 17 00:00:00 2001
From: noah <goldstein.w.n@gmail.com>
Date: Sat, 3 Apr 2021 04:12:15 -0400
Subject: [PATCH] x86: Update large memcpy case in memmove-vec-unaligned-erms.S
Content-type: text/plain; charset=UTF-8
No Bug. This commit updates the large memcpy case (no overlap). The
update is to perform memcpy on either 2 or 4 contiguous pages at
once. This 1) helps to alleviate the affects of false memory aliasing
when destination and source have a close 4k alignment and 2) In most
cases and for most DRAM units is a modestly more efficient access
pattern. These changes are a clear performance improvement for
VEC_SIZE =16/32, though more ambiguous for VEC_SIZE=64. test-memcpy,
test-memccpy, test-mempcpy, test-memmove, and tst-memmove-overflow all
pass.
Signed-off-by: Noah Goldstein <goldstein.w.n@gmail.com>
---
.../multiarch/memmove-vec-unaligned-erms.S | 338 ++++++++++++++----
1 file changed, 265 insertions(+), 73 deletions(-)
Conflicts:
sysdeps/x86_64/multiarch/memmove-vec-unaligned-erms.S
(different number of sections)
diff --git a/sysdeps/x86_64/multiarch/memmove-vec-unaligned-erms.S b/sysdeps/x86_64/multiarch/memmove-vec-unaligned-erms.S
index c475fed4..3e2dd6bc 100644
--- a/sysdeps/x86_64/multiarch/memmove-vec-unaligned-erms.S
+++ b/sysdeps/x86_64/multiarch/memmove-vec-unaligned-erms.S
@@ -32,7 +32,16 @@
overlapping addresses.
6. If size >= __x86_shared_non_temporal_threshold and there is no
overlap between destination and source, use non-temporal store
- instead of aligned store. */
+ instead of aligned store copying from either 2 or 4 pages at
+ once.
+ 8. For point 7) if size < 16 * __x86_shared_non_temporal_threshold
+ and source and destination do not page alias, copy from 2 pages
+ at once using non-temporal stores. Page aliasing in this case is
+ considered true if destination's page alignment - sources' page
+ alignment is less than 8 * VEC_SIZE.
+ 9. If size >= 16 * __x86_shared_non_temporal_threshold or source
+ and destination do page alias copy from 4 pages at once using
+ non-temporal stores. */
#include <sysdep.h>
@@ -64,6 +73,34 @@
# endif
#endif
+#ifndef PAGE_SIZE
+# define PAGE_SIZE 4096
+#endif
+
+#if PAGE_SIZE != 4096
+# error Unsupported PAGE_SIZE
+#endif
+
+#ifndef LOG_PAGE_SIZE
+# define LOG_PAGE_SIZE 12
+#endif
+
+#if PAGE_SIZE != (1 << LOG_PAGE_SIZE)
+# error Invalid LOG_PAGE_SIZE
+#endif
+
+/* Byte per page for large_memcpy inner loop. */
+#if VEC_SIZE == 64
+# define LARGE_LOAD_SIZE (VEC_SIZE * 2)
+#else
+# define LARGE_LOAD_SIZE (VEC_SIZE * 4)
+#endif
+
+/* Amount to shift rdx by to compare for memcpy_large_4x. */
+#ifndef LOG_4X_MEMCPY_THRESH
+# define LOG_4X_MEMCPY_THRESH 4
+#endif
+
/* Avoid short distance rep movsb only with non-SSE vector. */
#ifndef AVOID_SHORT_DISTANCE_REP_MOVSB
# define AVOID_SHORT_DISTANCE_REP_MOVSB (VEC_SIZE > 16)
@@ -103,6 +140,28 @@
# error Unsupported PREFETCH_SIZE!
#endif
+#if LARGE_LOAD_SIZE == (VEC_SIZE * 2)
+# define LOAD_ONE_SET(base, offset, vec0, vec1, ...) \
+ VMOVU (offset)base, vec0; \
+ VMOVU ((offset) + VEC_SIZE)base, vec1;
+# define STORE_ONE_SET(base, offset, vec0, vec1, ...) \
+ VMOVNT vec0, (offset)base; \
+ VMOVNT vec1, ((offset) + VEC_SIZE)base;
+#elif LARGE_LOAD_SIZE == (VEC_SIZE * 4)
+# define LOAD_ONE_SET(base, offset, vec0, vec1, vec2, vec3) \
+ VMOVU (offset)base, vec0; \
+ VMOVU ((offset) + VEC_SIZE)base, vec1; \
+ VMOVU ((offset) + VEC_SIZE * 2)base, vec2; \
+ VMOVU ((offset) + VEC_SIZE * 3)base, vec3;
+# define STORE_ONE_SET(base, offset, vec0, vec1, vec2, vec3) \
+ VMOVNT vec0, (offset)base; \
+ VMOVNT vec1, ((offset) + VEC_SIZE)base; \
+ VMOVNT vec2, ((offset) + VEC_SIZE * 2)base; \
+ VMOVNT vec3, ((offset) + VEC_SIZE * 3)base;
+#else
+# error Invalid LARGE_LOAD_SIZE
+#endif
+
#ifndef SECTION
# error SECTION is not defined!
#endif
@@ -390,6 +449,15 @@ L(last_4x_vec):
VZEROUPPER_RETURN
L(more_8x_vec):
+ /* Check if non-temporal move candidate. */
+#if (defined USE_MULTIARCH || VEC_SIZE == 16) && IS_IN (libc)
+ /* Check non-temporal store threshold. */
+ cmp __x86_shared_non_temporal_threshold(%rip), %RDX_LP
+ ja L(large_memcpy_2x)
+#endif
+ /* Entry if rdx is greater than non-temporal threshold but there
+ is overlap. */
+L(more_8x_vec_check):
cmpq %rsi, %rdi
ja L(more_8x_vec_backward)
/* Source == destination is less common. */
@@ -416,24 +484,21 @@ L(more_8x_vec):
subq %r8, %rdi
/* Adjust length. */
addq %r8, %rdx
-#if (defined USE_MULTIARCH || VEC_SIZE == 16) && IS_IN (libc)
- /* Check non-temporal store threshold. */
- cmp __x86_shared_non_temporal_threshold(%rip), %RDX_LP
- ja L(large_forward)
-#endif
+
+ .p2align 4
L(loop_4x_vec_forward):
/* Copy 4 * VEC a time forward. */
VMOVU (%rsi), %VEC(0)
VMOVU VEC_SIZE(%rsi), %VEC(1)
VMOVU (VEC_SIZE * 2)(%rsi), %VEC(2)
VMOVU (VEC_SIZE * 3)(%rsi), %VEC(3)
- addq $(VEC_SIZE * 4), %rsi
- subq $(VEC_SIZE * 4), %rdx
+ subq $-(VEC_SIZE * 4), %rsi
+ addq $-(VEC_SIZE * 4), %rdx
VMOVA %VEC(0), (%rdi)
VMOVA %VEC(1), VEC_SIZE(%rdi)
VMOVA %VEC(2), (VEC_SIZE * 2)(%rdi)
VMOVA %VEC(3), (VEC_SIZE * 3)(%rdi)
- addq $(VEC_SIZE * 4), %rdi
+ subq $-(VEC_SIZE * 4), %rdi
cmpq $(VEC_SIZE * 4), %rdx
ja L(loop_4x_vec_forward)
/* Store the last 4 * VEC. */
@@ -467,24 +532,21 @@ L(more_8x_vec_backward):
subq %r8, %r9
/* Adjust length. */
subq %r8, %rdx
-#if (defined USE_MULTIARCH || VEC_SIZE == 16) && IS_IN (libc)
- /* Check non-temporal store threshold. */
- cmp __x86_shared_non_temporal_threshold(%rip), %RDX_LP
- ja L(large_backward)
-#endif
+
+ .p2align 4
L(loop_4x_vec_backward):
/* Copy 4 * VEC a time backward. */
VMOVU (%rcx), %VEC(0)
VMOVU -VEC_SIZE(%rcx), %VEC(1)
VMOVU -(VEC_SIZE * 2)(%rcx), %VEC(2)
VMOVU -(VEC_SIZE * 3)(%rcx), %VEC(3)
- subq $(VEC_SIZE * 4), %rcx
- subq $(VEC_SIZE * 4), %rdx
+ addq $-(VEC_SIZE * 4), %rcx
+ addq $-(VEC_SIZE * 4), %rdx
VMOVA %VEC(0), (%r9)
VMOVA %VEC(1), -VEC_SIZE(%r9)
VMOVA %VEC(2), -(VEC_SIZE * 2)(%r9)
VMOVA %VEC(3), -(VEC_SIZE * 3)(%r9)
- subq $(VEC_SIZE * 4), %r9
+ addq $-(VEC_SIZE * 4), %r9
cmpq $(VEC_SIZE * 4), %rdx
ja L(loop_4x_vec_backward)
/* Store the first 4 * VEC. */
@@ -497,72 +559,202 @@ L(loop_4x_vec_backward):
VZEROUPPER_RETURN
#if (defined USE_MULTIARCH || VEC_SIZE == 16) && IS_IN (libc)
-L(large_forward):
+ .p2align 4
+L(large_memcpy_2x):
+ /* Compute absolute value of difference between source and
+ destination. */
+ movq %rdi, %r9
+ subq %rsi, %r9
+ movq %r9, %r8
+ leaq -1(%r9), %rcx
+ sarq $63, %r8
+ xorq %r8, %r9
+ subq %r8, %r9
/* Don't use non-temporal store if there is overlap between
- destination and source since destination may be in cache
- when source is loaded. */
- leaq (%rdi, %rdx), %r10
- cmpq %r10, %rsi
- jb L(loop_4x_vec_forward)
-L(loop_large_forward):
+ destination and source since destination may be in cache when
+ source is loaded. */
+ cmpq %r9, %rdx
+ ja L(more_8x_vec_check)
+
+ /* Cache align destination. First store the first 64 bytes then
+ adjust alignments. */
+ VMOVU (%rsi), %VEC(8)
+#if VEC_SIZE < 64
+ VMOVU VEC_SIZE(%rsi), %VEC(9)
+#if VEC_SIZE < 32
+ VMOVU (VEC_SIZE * 2)(%rsi), %VEC(10)
+ VMOVU (VEC_SIZE * 3)(%rsi), %VEC(11)
+#endif
+#endif
+ VMOVU %VEC(8), (%rdi)
+#if VEC_SIZE < 64
+ VMOVU %VEC(9), VEC_SIZE(%rdi)
+#if VEC_SIZE < 32
+ VMOVU %VEC(10), (VEC_SIZE * 2)(%rdi)
+ VMOVU %VEC(11), (VEC_SIZE * 3)(%rdi)
+#endif
+#endif
+ /* Adjust source, destination, and size. */
+ movq %rdi, %r8
+ andq $63, %r8
+ /* Get the negative of offset for alignment. */
+ subq $64, %r8
+ /* Adjust source. */
+ subq %r8, %rsi
+ /* Adjust destination which should be aligned now. */
+ subq %r8, %rdi
+ /* Adjust length. */
+ addq %r8, %rdx
+
+ /* Test if source and destination addresses will alias. If they do
+ the larger pipeline in large_memcpy_4x alleviated the
+ performance drop. */
+ testl $(PAGE_SIZE - VEC_SIZE * 8), %ecx
+ jz L(large_memcpy_4x)
+
+ movq %rdx, %r10
+ shrq $LOG_4X_MEMCPY_THRESH, %r10
+ cmp __x86_shared_non_temporal_threshold(%rip), %r10
+ jae L(large_memcpy_4x)
+
+ /* edx will store remainder size for copying tail. */
+ andl $(PAGE_SIZE * 2 - 1), %edx
+ /* r10 stores outer loop counter. */
+ shrq $((LOG_PAGE_SIZE + 1) - LOG_4X_MEMCPY_THRESH), %r10
+ /* Copy 4x VEC at a time from 2 pages. */
+ .p2align 4
+L(loop_large_memcpy_2x_outer):
+ /* ecx stores inner loop counter. */
+ movl $(PAGE_SIZE / LARGE_LOAD_SIZE), %ecx
+L(loop_large_memcpy_2x_inner):
+ PREFETCH_ONE_SET(1, (%rsi), PREFETCHED_LOAD_SIZE)
+ PREFETCH_ONE_SET(1, (%rsi), PREFETCHED_LOAD_SIZE * 2)
+ PREFETCH_ONE_SET(1, (%rsi), PAGE_SIZE + PREFETCHED_LOAD_SIZE)
+ PREFETCH_ONE_SET(1, (%rsi), PAGE_SIZE + PREFETCHED_LOAD_SIZE * 2)
+ /* Load vectors from rsi. */
+ LOAD_ONE_SET((%rsi), 0, %VEC(0), %VEC(1), %VEC(2), %VEC(3))
+ LOAD_ONE_SET((%rsi), PAGE_SIZE, %VEC(4), %VEC(5), %VEC(6), %VEC(7))
+ subq $-LARGE_LOAD_SIZE, %rsi
+ /* Non-temporal store vectors to rdi. */
+ STORE_ONE_SET((%rdi), 0, %VEC(0), %VEC(1), %VEC(2), %VEC(3))
+ STORE_ONE_SET((%rdi), PAGE_SIZE, %VEC(4), %VEC(5), %VEC(6), %VEC(7))
+ subq $-LARGE_LOAD_SIZE, %rdi
+ decl %ecx
+ jnz L(loop_large_memcpy_2x_inner)
+ addq $PAGE_SIZE, %rdi
+ addq $PAGE_SIZE, %rsi
+ decq %r10
+ jne L(loop_large_memcpy_2x_outer)
+ sfence
+
+ /* Check if only last 4 loads are needed. */
+ cmpl $(VEC_SIZE * 4), %edx
+ jbe L(large_memcpy_2x_end)
+
+ /* Handle the last 2 * PAGE_SIZE bytes. */
+L(loop_large_memcpy_2x_tail):
/* Copy 4 * VEC a time forward with non-temporal stores. */
- PREFETCH_ONE_SET (1, (%rsi), PREFETCHED_LOAD_SIZE * 2)
- PREFETCH_ONE_SET (1, (%rsi), PREFETCHED_LOAD_SIZE * 3)
+ PREFETCH_ONE_SET (1, (%rsi), PREFETCHED_LOAD_SIZE)
+ PREFETCH_ONE_SET (1, (%rdi), PREFETCHED_LOAD_SIZE)
VMOVU (%rsi), %VEC(0)
VMOVU VEC_SIZE(%rsi), %VEC(1)
VMOVU (VEC_SIZE * 2)(%rsi), %VEC(2)
VMOVU (VEC_SIZE * 3)(%rsi), %VEC(3)
- addq $PREFETCHED_LOAD_SIZE, %rsi
- subq $PREFETCHED_LOAD_SIZE, %rdx
- VMOVNT %VEC(0), (%rdi)
- VMOVNT %VEC(1), VEC_SIZE(%rdi)
- VMOVNT %VEC(2), (VEC_SIZE * 2)(%rdi)
- VMOVNT %VEC(3), (VEC_SIZE * 3)(%rdi)
- addq $PREFETCHED_LOAD_SIZE, %rdi
- cmpq $PREFETCHED_LOAD_SIZE, %rdx
- ja L(loop_large_forward)
- sfence
+ subq $-(VEC_SIZE * 4), %rsi
+ addl $-(VEC_SIZE * 4), %edx
+ VMOVA %VEC(0), (%rdi)
+ VMOVA %VEC(1), VEC_SIZE(%rdi)
+ VMOVA %VEC(2), (VEC_SIZE * 2)(%rdi)
+ VMOVA %VEC(3), (VEC_SIZE * 3)(%rdi)
+ subq $-(VEC_SIZE * 4), %rdi
+ cmpl $(VEC_SIZE * 4), %edx
+ ja L(loop_large_memcpy_2x_tail)
+
+L(large_memcpy_2x_end):
/* Store the last 4 * VEC. */
- VMOVU %VEC(5), (%rcx)
- VMOVU %VEC(6), -VEC_SIZE(%rcx)
- VMOVU %VEC(7), -(VEC_SIZE * 2)(%rcx)
- VMOVU %VEC(8), -(VEC_SIZE * 3)(%rcx)
- /* Store the first VEC. */
- VMOVU %VEC(4), (%r11)
+ VMOVU -(VEC_SIZE * 4)(%rsi, %rdx), %VEC(0)
+ VMOVU -(VEC_SIZE * 3)(%rsi, %rdx), %VEC(1)
+ VMOVU -(VEC_SIZE * 2)(%rsi, %rdx), %VEC(2)
+ VMOVU -VEC_SIZE(%rsi, %rdx), %VEC(3)
+
+ VMOVU %VEC(0), -(VEC_SIZE * 4)(%rdi, %rdx)
+ VMOVU %VEC(1), -(VEC_SIZE * 3)(%rdi, %rdx)
+ VMOVU %VEC(2), -(VEC_SIZE * 2)(%rdi, %rdx)
+ VMOVU %VEC(3), -VEC_SIZE(%rdi, %rdx)
VZEROUPPER_RETURN
-L(large_backward):
- /* Don't use non-temporal store if there is overlap between
- destination and source since destination may be in cache
- when source is loaded. */
- leaq (%rcx, %rdx), %r10
- cmpq %r10, %r9
- jb L(loop_4x_vec_backward)
-L(loop_large_backward):
- /* Copy 4 * VEC a time backward with non-temporal stores. */
- PREFETCH_ONE_SET (-1, (%rcx), -PREFETCHED_LOAD_SIZE * 2)
- PREFETCH_ONE_SET (-1, (%rcx), -PREFETCHED_LOAD_SIZE * 3)
- VMOVU (%rcx), %VEC(0)
- VMOVU -VEC_SIZE(%rcx), %VEC(1)
- VMOVU -(VEC_SIZE * 2)(%rcx), %VEC(2)
- VMOVU -(VEC_SIZE * 3)(%rcx), %VEC(3)
- subq $PREFETCHED_LOAD_SIZE, %rcx
- subq $PREFETCHED_LOAD_SIZE, %rdx
- VMOVNT %VEC(0), (%r9)
- VMOVNT %VEC(1), -VEC_SIZE(%r9)
- VMOVNT %VEC(2), -(VEC_SIZE * 2)(%r9)
- VMOVNT %VEC(3), -(VEC_SIZE * 3)(%r9)
- subq $PREFETCHED_LOAD_SIZE, %r9
- cmpq $PREFETCHED_LOAD_SIZE, %rdx
- ja L(loop_large_backward)
+ .p2align 4
+L(large_memcpy_4x):
+ movq %rdx, %r10
+ /* edx will store remainder size for copying tail. */
+ andl $(PAGE_SIZE * 4 - 1), %edx
+ /* r10 stores outer loop counter. */
+ shrq $(LOG_PAGE_SIZE + 2), %r10
+ /* Copy 4x VEC at a time from 4 pages. */
+ .p2align 4
+L(loop_large_memcpy_4x_outer):
+ /* ecx stores inner loop counter. */
+ movl $(PAGE_SIZE / LARGE_LOAD_SIZE), %ecx
+L(loop_large_memcpy_4x_inner):
+ /* Only one prefetch set per page as doing 4 pages give more time
+ for prefetcher to keep up. */
+ PREFETCH_ONE_SET(1, (%rsi), PREFETCHED_LOAD_SIZE)
+ PREFETCH_ONE_SET(1, (%rsi), PAGE_SIZE + PREFETCHED_LOAD_SIZE)
+ PREFETCH_ONE_SET(1, (%rsi), PAGE_SIZE * 2 + PREFETCHED_LOAD_SIZE)
+ PREFETCH_ONE_SET(1, (%rsi), PAGE_SIZE * 3 + PREFETCHED_LOAD_SIZE)
+ /* Load vectors from rsi. */
+ LOAD_ONE_SET((%rsi), 0, %VEC(0), %VEC(1), %VEC(2), %VEC(3))
+ LOAD_ONE_SET((%rsi), PAGE_SIZE, %VEC(4), %VEC(5), %VEC(6), %VEC(7))
+ LOAD_ONE_SET((%rsi), PAGE_SIZE * 2, %VEC(8), %VEC(9), %VEC(10), %VEC(11))
+ LOAD_ONE_SET((%rsi), PAGE_SIZE * 3, %VEC(12), %VEC(13), %VEC(14), %VEC(15))
+ subq $-LARGE_LOAD_SIZE, %rsi
+ /* Non-temporal store vectors to rdi. */
+ STORE_ONE_SET((%rdi), 0, %VEC(0), %VEC(1), %VEC(2), %VEC(3))
+ STORE_ONE_SET((%rdi), PAGE_SIZE, %VEC(4), %VEC(5), %VEC(6), %VEC(7))
+ STORE_ONE_SET((%rdi), PAGE_SIZE * 2, %VEC(8), %VEC(9), %VEC(10), %VEC(11))
+ STORE_ONE_SET((%rdi), PAGE_SIZE * 3, %VEC(12), %VEC(13), %VEC(14), %VEC(15))
+ subq $-LARGE_LOAD_SIZE, %rdi
+ decl %ecx
+ jnz L(loop_large_memcpy_4x_inner)
+ addq $(PAGE_SIZE * 3), %rdi
+ addq $(PAGE_SIZE * 3), %rsi
+ decq %r10
+ jne L(loop_large_memcpy_4x_outer)
sfence
- /* Store the first 4 * VEC. */
- VMOVU %VEC(4), (%rdi)
- VMOVU %VEC(5), VEC_SIZE(%rdi)
- VMOVU %VEC(6), (VEC_SIZE * 2)(%rdi)
- VMOVU %VEC(7), (VEC_SIZE * 3)(%rdi)
- /* Store the last VEC. */
- VMOVU %VEC(8), (%r11)
+ /* Check if only last 4 loads are needed. */
+ cmpl $(VEC_SIZE * 4), %edx
+ jbe L(large_memcpy_4x_end)
+
+ /* Handle the last 4 * PAGE_SIZE bytes. */
+L(loop_large_memcpy_4x_tail):
+ /* Copy 4 * VEC a time forward with non-temporal stores. */
+ PREFETCH_ONE_SET (1, (%rsi), PREFETCHED_LOAD_SIZE)
+ PREFETCH_ONE_SET (1, (%rdi), PREFETCHED_LOAD_SIZE)
+ VMOVU (%rsi), %VEC(0)
+ VMOVU VEC_SIZE(%rsi), %VEC(1)
+ VMOVU (VEC_SIZE * 2)(%rsi), %VEC(2)
+ VMOVU (VEC_SIZE * 3)(%rsi), %VEC(3)
+ subq $-(VEC_SIZE * 4), %rsi
+ addl $-(VEC_SIZE * 4), %edx
+ VMOVA %VEC(0), (%rdi)
+ VMOVA %VEC(1), VEC_SIZE(%rdi)
+ VMOVA %VEC(2), (VEC_SIZE * 2)(%rdi)
+ VMOVA %VEC(3), (VEC_SIZE * 3)(%rdi)
+ subq $-(VEC_SIZE * 4), %rdi
+ cmpl $(VEC_SIZE * 4), %edx
+ ja L(loop_large_memcpy_4x_tail)
+
+L(large_memcpy_4x_end):
+ /* Store the last 4 * VEC. */
+ VMOVU -(VEC_SIZE * 4)(%rsi, %rdx), %VEC(0)
+ VMOVU -(VEC_SIZE * 3)(%rsi, %rdx), %VEC(1)
+ VMOVU -(VEC_SIZE * 2)(%rsi, %rdx), %VEC(2)
+ VMOVU -VEC_SIZE(%rsi, %rdx), %VEC(3)
+
+ VMOVU %VEC(0), -(VEC_SIZE * 4)(%rdi, %rdx)
+ VMOVU %VEC(1), -(VEC_SIZE * 3)(%rdi, %rdx)
+ VMOVU %VEC(2), -(VEC_SIZE * 2)(%rdi, %rdx)
+ VMOVU %VEC(3), -VEC_SIZE(%rdi, %rdx)
VZEROUPPER_RETURN
#endif
END (MEMMOVE_SYMBOL (__memmove, unaligned_erms))
--
GitLab

View File

@ -1,151 +0,0 @@
From ecd8b842cf37ea112e59cd9085ff1f1b6e208ae0 Mon Sep 17 00:00:00 2001
From: "H.J. Lu" <hjl.tools@gmail.com>
Date: Mon, 21 Jan 2019 11:29:58 -0800
Subject: [PATCH] x86-64 memrchr: Properly handle the length parameter [BZ#
24097]
Content-type: text/plain; charset=UTF-8
On x32, the size_t parameter may be passed in the lower 32 bits of a
64-bit register with the non-zero upper 32 bits. The string/memory
functions written in assembly can only use the lower 32 bits of a
64-bit register as length or must clear the upper 32 bits before using
the full 64-bit register for length.
This pach fixes memrchr for x32. Tested on x86-64 and x32. On x86-64,
libc.so is the same with and withou the fix.
[BZ# 24097]
CVE-2019-6488
* sysdeps/x86_64/memrchr.S: Use RDX_LP for length.
* sysdeps/x86_64/multiarch/memrchr-avx2.S: Likewise.
* sysdeps/x86_64/x32/Makefile (tests): Add tst-size_t-memrchr.
* sysdeps/x86_64/x32/tst-size_t-memrchr.c: New file.
---
sysdeps/x86_64/memrchr.S | 4 +-
sysdeps/x86_64/multiarch/memrchr-avx2.S | 4 +-
sysdeps/x86_64/x32/Makefile | 3 +-
sysdeps/x86_64/x32/tst-size_t-memrchr.c | 57 +++++++++++++++++++++++++
4 files changed, 63 insertions(+), 5 deletions(-)
create mode 100644 sysdeps/x86_64/x32/tst-size_t-memrchr.c
Conflicts:
ChangeLog
(removed)
diff --git a/sysdeps/x86_64/memrchr.S b/sysdeps/x86_64/memrchr.S
index b8e3fa1d..dc82f8f7 100644
--- a/sysdeps/x86_64/memrchr.S
+++ b/sysdeps/x86_64/memrchr.S
@@ -24,13 +24,13 @@
ENTRY (__memrchr)
movd %esi, %xmm1
- sub $16, %rdx
+ sub $16, %RDX_LP
jbe L(length_less16)
punpcklbw %xmm1, %xmm1
punpcklbw %xmm1, %xmm1
- add %rdx, %rdi
+ add %RDX_LP, %RDI_LP
pshufd $0, %xmm1, %xmm1
movdqu (%rdi), %xmm0
diff --git a/sysdeps/x86_64/multiarch/memrchr-avx2.S b/sysdeps/x86_64/multiarch/memrchr-avx2.S
index b41a58bc..ce488dd9 100644
--- a/sysdeps/x86_64/multiarch/memrchr-avx2.S
+++ b/sysdeps/x86_64/multiarch/memrchr-avx2.S
@@ -32,10 +32,10 @@ ENTRY (__memrchr_avx2)
vmovd %esi, %xmm0
vpbroadcastb %xmm0, %ymm0
- subq $VEC_SIZE, %rdx
+ sub $VEC_SIZE, %RDX_LP
jbe L(last_vec_or_less)
- addq %rdx, %rdi
+ add %RDX_LP, %RDI_LP
/* Check the last VEC_SIZE bytes. */
vpcmpeqb (%rdi), %ymm0, %ymm1
diff --git a/sysdeps/x86_64/x32/Makefile b/sysdeps/x86_64/x32/Makefile
index 2fe1e5ac..e99dbd7c 100644
--- a/sysdeps/x86_64/x32/Makefile
+++ b/sysdeps/x86_64/x32/Makefile
@@ -6,7 +6,8 @@ CFLAGS-s_llround.c += -fno-builtin-lround
endif
ifeq ($(subdir),string)
-tests += tst-size_t-memchr tst-size_t-memcmp tst-size_t-memcpy
+tests += tst-size_t-memchr tst-size_t-memcmp tst-size_t-memcpy \
+ tst-size_t-memrchr
endif
ifeq ($(subdir),wcsmbs)
diff --git a/sysdeps/x86_64/x32/tst-size_t-memrchr.c b/sysdeps/x86_64/x32/tst-size_t-memrchr.c
new file mode 100644
index 00000000..c83699c0
--- /dev/null
+++ b/sysdeps/x86_64/x32/tst-size_t-memrchr.c
@@ -0,0 +1,57 @@
+/* Test memrchr with size_t in the lower 32 bits of 64-bit register.
+ Copyright (C) 2019 Free Software Foundation, Inc.
+ This file is part of the GNU C Library.
+
+ The GNU C Library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ The GNU C Library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with the GNU C Library; if not, see
+ <http://www.gnu.org/licenses/>. */
+
+#define TEST_NAME "memrchr"
+#include "test-size_t.h"
+
+IMPL (memchr, 1)
+
+typedef void * (*proto_t) (const void *, int, size_t);
+
+static void *
+__attribute__ ((noinline, noclone))
+do_memrchr (parameter_t a, parameter_t b)
+{
+ return CALL (&b, a.p, (uintptr_t) b.p, a.len);
+}
+
+static int
+test_main (void)
+{
+ test_init ();
+
+ parameter_t src = { { page_size }, buf2 };
+ parameter_t c = { { 0 }, (void *) (uintptr_t) 0x12 };
+
+ int ret = 0;
+ FOR_EACH_IMPL (impl, 0)
+ {
+ c.fn = impl->fn;
+ void * res = do_memrchr (src, c);
+ if (res)
+ {
+ error (0, 0, "Wrong result in function %s: %p != NULL",
+ impl->name, res);
+ ret = 1;
+ }
+ }
+
+ return ret ? EXIT_FAILURE : EXIT_SUCCESS;
+}
+
+#include <support/test-driver.c>
--
GitLab

View File

@ -1,92 +0,0 @@
From 83c5b368226c34a2f0a5287df40fc290b2b34359 Mon Sep 17 00:00:00 2001
From: "H.J. Lu" <hjl.tools@gmail.com>
Date: Mon, 19 Apr 2021 10:45:07 -0700
Subject: [PATCH] x86-64: Require BMI2 for strchr-avx2.S
Content-type: text/plain; charset=UTF-8
Since strchr-avx2.S updated by
commit 1f745ecc2109890886b161d4791e1406fdfc29b8
Author: noah <goldstein.w.n@gmail.com>
Date: Wed Feb 3 00:38:59 2021 -0500
x86-64: Refactor and improve performance of strchr-avx2.S
uses sarx:
c4 e2 72 f7 c0 sarx %ecx,%eax,%eax
for strchr-avx2 family functions, require BMI2 in ifunc-impl-list.c and
ifunc-avx2.h.
---
sysdeps/x86_64/multiarch/ifunc-avx2.h | 4 ++--
sysdeps/x86_64/multiarch/ifunc-impl-list.c | 12 +++++++++---
2 files changed, 11 insertions(+), 5 deletions(-)
diff --git a/sysdeps/x86_64/multiarch/ifunc-avx2.h b/sysdeps/x86_64/multiarch/ifunc-avx2.h
index e0f30e61..ef72b73f 100644
--- a/sysdeps/x86_64/multiarch/ifunc-avx2.h
+++ b/sysdeps/x86_64/multiarch/ifunc-avx2.h
@@ -30,11 +30,11 @@ IFUNC_SELECTOR (void)
const struct cpu_features* cpu_features = __get_cpu_features ();
if (CPU_FEATURE_USABLE_P (cpu_features, AVX2)
+ && CPU_FEATURE_USABLE_P (cpu_features, BMI2)
&& CPU_FEATURES_ARCH_P (cpu_features, AVX_Fast_Unaligned_Load))
{
if (CPU_FEATURE_USABLE_P (cpu_features, AVX512VL)
- && CPU_FEATURE_USABLE_P (cpu_features, AVX512BW)
- && CPU_FEATURE_USABLE_P (cpu_features, BMI2))
+ && CPU_FEATURE_USABLE_P (cpu_features, AVX512BW))
return OPTIMIZE (evex);
if (CPU_FEATURE_USABLE_P (cpu_features, RTM))
diff --git a/sysdeps/x86_64/multiarch/ifunc-impl-list.c b/sysdeps/x86_64/multiarch/ifunc-impl-list.c
index 695cdba6..85b8863a 100644
--- a/sysdeps/x86_64/multiarch/ifunc-impl-list.c
+++ b/sysdeps/x86_64/multiarch/ifunc-impl-list.c
@@ -400,10 +400,12 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,
/* Support sysdeps/x86_64/multiarch/strchr.c. */
IFUNC_IMPL (i, name, strchr,
IFUNC_IMPL_ADD (array, i, strchr,
- CPU_FEATURE_USABLE (AVX2),
+ (CPU_FEATURE_USABLE (AVX2)
+ && CPU_FEATURE_USABLE (BMI2)),
__strchr_avx2)
IFUNC_IMPL_ADD (array, i, strchr,
(CPU_FEATURE_USABLE (AVX2)
+ && CPU_FEATURE_USABLE (BMI2)
&& CPU_FEATURE_USABLE (RTM)),
__strchr_avx2_rtm)
IFUNC_IMPL_ADD (array, i, strchr,
@@ -417,10 +419,12 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,
/* Support sysdeps/x86_64/multiarch/strchrnul.c. */
IFUNC_IMPL (i, name, strchrnul,
IFUNC_IMPL_ADD (array, i, strchrnul,
- CPU_FEATURE_USABLE (AVX2),
+ (CPU_FEATURE_USABLE (AVX2)
+ && CPU_FEATURE_USABLE (BMI2)),
__strchrnul_avx2)
IFUNC_IMPL_ADD (array, i, strchrnul,
(CPU_FEATURE_USABLE (AVX2)
+ && CPU_FEATURE_USABLE (BMI2)
&& CPU_FEATURE_USABLE (RTM)),
__strchrnul_avx2_rtm)
IFUNC_IMPL_ADD (array, i, strchrnul,
@@ -574,10 +578,12 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,
/* Support sysdeps/x86_64/multiarch/wcschr.c. */
IFUNC_IMPL (i, name, wcschr,
IFUNC_IMPL_ADD (array, i, wcschr,
- CPU_FEATURE_USABLE (AVX2),
+ (CPU_FEATURE_USABLE (AVX2)
+ && CPU_FEATURE_USABLE (BMI2)),
__wcschr_avx2)
IFUNC_IMPL_ADD (array, i, wcschr,
(CPU_FEATURE_USABLE (AVX2)
+ && CPU_FEATURE_USABLE (BMI2)
&& CPU_FEATURE_USABLE (RTM)),
__wcschr_avx2_rtm)
IFUNC_IMPL_ADD (array, i, wcschr,
--
GitLab

View File

@ -1,265 +0,0 @@
From f53790272ce7bdc5ecd14b45f65d0464d2a61a3a Mon Sep 17 00:00:00 2001
From: Noah Goldstein <goldstein.w.n@gmail.com>
Date: Mon, 19 Apr 2021 17:48:10 -0400
Subject: [PATCH] x86: Optimize less_vec evex and avx512
memset-vec-unaligned-erms.S
Content-type: text/plain; charset=UTF-8
No bug. This commit adds optimized cased for less_vec memset case that
uses the avx512vl/avx512bw mask store avoiding the excessive
branches. test-memset and test-wmemset are passing.
Signed-off-by: Noah Goldstein <goldstein.w.n@gmail.com>
---
sysdeps/x86_64/multiarch/ifunc-impl-list.c | 40 ++++++++++-----
sysdeps/x86_64/multiarch/ifunc-memset.h | 6 ++-
.../multiarch/memset-avx512-unaligned-erms.S | 2 +-
.../multiarch/memset-evex-unaligned-erms.S | 2 +-
.../multiarch/memset-vec-unaligned-erms.S | 51 +++++++++++++++----
5 files changed, 74 insertions(+), 27 deletions(-)
diff --git a/sysdeps/x86_64/multiarch/ifunc-impl-list.c b/sysdeps/x86_64/multiarch/ifunc-impl-list.c
index 85b8863a..d59d65f8 100644
--- a/sysdeps/x86_64/multiarch/ifunc-impl-list.c
+++ b/sysdeps/x86_64/multiarch/ifunc-impl-list.c
@@ -204,19 +204,23 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,
__memset_chk_avx2_unaligned_erms_rtm)
IFUNC_IMPL_ADD (array, i, __memset_chk,
(CPU_FEATURE_USABLE (AVX512VL)
- && CPU_FEATURE_USABLE (AVX512BW)),
+ && CPU_FEATURE_USABLE (AVX512BW)
+ && CPU_FEATURE_USABLE (BMI2)),
__memset_chk_evex_unaligned)
IFUNC_IMPL_ADD (array, i, __memset_chk,
(CPU_FEATURE_USABLE (AVX512VL)
- && CPU_FEATURE_USABLE (AVX512BW)),
+ && CPU_FEATURE_USABLE (AVX512BW)
+ && CPU_FEATURE_USABLE (BMI2)),
__memset_chk_evex_unaligned_erms)
IFUNC_IMPL_ADD (array, i, __memset_chk,
(CPU_FEATURE_USABLE (AVX512VL)
- && CPU_FEATURE_USABLE (AVX512BW)),
+ && CPU_FEATURE_USABLE (AVX512BW)
+ && CPU_FEATURE_USABLE (BMI2)),
__memset_chk_avx512_unaligned_erms)
IFUNC_IMPL_ADD (array, i, __memset_chk,
(CPU_FEATURE_USABLE (AVX512VL)
- && CPU_FEATURE_USABLE (AVX512BW)),
+ && CPU_FEATURE_USABLE (AVX512BW)
+ && CPU_FEATURE_USABLE (BMI2)),
__memset_chk_avx512_unaligned)
IFUNC_IMPL_ADD (array, i, __memset_chk,
CPU_FEATURE_USABLE (AVX512F),
@@ -247,19 +251,23 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,
__memset_avx2_unaligned_erms_rtm)
IFUNC_IMPL_ADD (array, i, memset,
(CPU_FEATURE_USABLE (AVX512VL)
- && CPU_FEATURE_USABLE (AVX512BW)),
+ && CPU_FEATURE_USABLE (AVX512BW)
+ && CPU_FEATURE_USABLE (BMI2)),
__memset_evex_unaligned)
IFUNC_IMPL_ADD (array, i, memset,
(CPU_FEATURE_USABLE (AVX512VL)
- && CPU_FEATURE_USABLE (AVX512BW)),
+ && CPU_FEATURE_USABLE (AVX512BW)
+ && CPU_FEATURE_USABLE (BMI2)),
__memset_evex_unaligned_erms)
IFUNC_IMPL_ADD (array, i, memset,
(CPU_FEATURE_USABLE (AVX512VL)
- && CPU_FEATURE_USABLE (AVX512BW)),
+ && CPU_FEATURE_USABLE (AVX512BW)
+ && CPU_FEATURE_USABLE (BMI2)),
__memset_avx512_unaligned_erms)
IFUNC_IMPL_ADD (array, i, memset,
(CPU_FEATURE_USABLE (AVX512VL)
- && CPU_FEATURE_USABLE (AVX512BW)),
+ && CPU_FEATURE_USABLE (AVX512BW)
+ && CPU_FEATURE_USABLE (BMI2)),
__memset_avx512_unaligned)
IFUNC_IMPL_ADD (array, i, memset,
CPU_FEATURE_USABLE (AVX512F),
@@ -739,10 +747,14 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,
&& CPU_FEATURE_USABLE (RTM)),
__wmemset_avx2_unaligned_rtm)
IFUNC_IMPL_ADD (array, i, wmemset,
- CPU_FEATURE_USABLE (AVX512VL),
+ (CPU_FEATURE_USABLE (AVX512VL)
+ && CPU_FEATURE_USABLE (AVX512BW)
+ && CPU_FEATURE_USABLE (BMI2)),
__wmemset_evex_unaligned)
IFUNC_IMPL_ADD (array, i, wmemset,
- CPU_FEATURE_USABLE (AVX512VL),
+ (CPU_FEATURE_USABLE (AVX512VL)
+ && CPU_FEATURE_USABLE (AVX512BW)
+ && CPU_FEATURE_USABLE (BMI2)),
__wmemset_avx512_unaligned))
#ifdef SHARED
@@ -946,10 +958,14 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,
CPU_FEATURE_USABLE (AVX2),
__wmemset_chk_avx2_unaligned)
IFUNC_IMPL_ADD (array, i, __wmemset_chk,
- CPU_FEATURE_USABLE (AVX512VL),
+ (CPU_FEATURE_USABLE (AVX512VL)
+ && CPU_FEATURE_USABLE (AVX512BW)
+ && CPU_FEATURE_USABLE (BMI2)),
__wmemset_chk_evex_unaligned)
IFUNC_IMPL_ADD (array, i, __wmemset_chk,
- CPU_FEATURE_USABLE (AVX512F),
+ (CPU_FEATURE_USABLE (AVX512VL)
+ && CPU_FEATURE_USABLE (AVX512BW)
+ && CPU_FEATURE_USABLE (BMI2)),
__wmemset_chk_avx512_unaligned))
#endif
diff --git a/sysdeps/x86_64/multiarch/ifunc-memset.h b/sysdeps/x86_64/multiarch/ifunc-memset.h
index 19795938..100e3707 100644
--- a/sysdeps/x86_64/multiarch/ifunc-memset.h
+++ b/sysdeps/x86_64/multiarch/ifunc-memset.h
@@ -54,7 +54,8 @@ IFUNC_SELECTOR (void)
&& !CPU_FEATURES_ARCH_P (cpu_features, Prefer_No_AVX512))
{
if (CPU_FEATURE_USABLE_P (cpu_features, AVX512VL)
- && CPU_FEATURE_USABLE_P (cpu_features, AVX512BW))
+ && CPU_FEATURE_USABLE_P (cpu_features, AVX512BW)
+ && CPU_FEATURE_USABLE_P (cpu_features, BMI2))
{
if (CPU_FEATURE_USABLE_P (cpu_features, ERMS))
return OPTIMIZE (avx512_unaligned_erms);
@@ -68,7 +69,8 @@ IFUNC_SELECTOR (void)
if (CPU_FEATURE_USABLE_P (cpu_features, AVX2))
{
if (CPU_FEATURE_USABLE_P (cpu_features, AVX512VL)
- && CPU_FEATURE_USABLE_P (cpu_features, AVX512BW))
+ && CPU_FEATURE_USABLE_P (cpu_features, AVX512BW)
+ && CPU_FEATURE_USABLE_P (cpu_features, BMI2))
{
if (CPU_FEATURE_USABLE_P (cpu_features, ERMS))
return OPTIMIZE (evex_unaligned_erms);
diff --git a/sysdeps/x86_64/multiarch/memset-avx512-unaligned-erms.S b/sysdeps/x86_64/multiarch/memset-avx512-unaligned-erms.S
index 22e7b187..8ad842fc 100644
--- a/sysdeps/x86_64/multiarch/memset-avx512-unaligned-erms.S
+++ b/sysdeps/x86_64/multiarch/memset-avx512-unaligned-erms.S
@@ -19,6 +19,6 @@
# define SECTION(p) p##.evex512
# define MEMSET_SYMBOL(p,s) p##_avx512_##s
# define WMEMSET_SYMBOL(p,s) p##_avx512_##s
-
+# define USE_LESS_VEC_MASK_STORE 1
# include "memset-vec-unaligned-erms.S"
#endif
diff --git a/sysdeps/x86_64/multiarch/memset-evex-unaligned-erms.S b/sysdeps/x86_64/multiarch/memset-evex-unaligned-erms.S
index ae0a4d6e..640f0929 100644
--- a/sysdeps/x86_64/multiarch/memset-evex-unaligned-erms.S
+++ b/sysdeps/x86_64/multiarch/memset-evex-unaligned-erms.S
@@ -19,6 +19,6 @@
# define SECTION(p) p##.evex
# define MEMSET_SYMBOL(p,s) p##_evex_##s
# define WMEMSET_SYMBOL(p,s) p##_evex_##s
-
+# define USE_LESS_VEC_MASK_STORE 1
# include "memset-vec-unaligned-erms.S"
#endif
diff --git a/sysdeps/x86_64/multiarch/memset-vec-unaligned-erms.S b/sysdeps/x86_64/multiarch/memset-vec-unaligned-erms.S
index bae5cba4..f877ac9d 100644
--- a/sysdeps/x86_64/multiarch/memset-vec-unaligned-erms.S
+++ b/sysdeps/x86_64/multiarch/memset-vec-unaligned-erms.S
@@ -63,6 +63,8 @@
# endif
#endif
+#define PAGE_SIZE 4096
+
#ifndef SECTION
# error SECTION is not defined!
#endif
@@ -213,11 +215,38 @@ L(loop):
cmpq %rcx, %rdx
jne L(loop)
VZEROUPPER_SHORT_RETURN
+
+ .p2align 4
L(less_vec):
/* Less than 1 VEC. */
# if VEC_SIZE != 16 && VEC_SIZE != 32 && VEC_SIZE != 64
# error Unsupported VEC_SIZE!
# endif
+# ifdef USE_LESS_VEC_MASK_STORE
+ /* Clear high bits from edi. Only keeping bits relevant to page
+ cross check. Note that we are using rax which is set in
+ MEMSET_VDUP_TO_VEC0_AND_SET_RETURN as ptr from here on out.
+ */
+ andl $(PAGE_SIZE - 1), %edi
+ /* Check if VEC_SIZE store cross page. Mask stores suffer serious
+ performance degradation when it has to fault supress. */
+ cmpl $(PAGE_SIZE - VEC_SIZE), %edi
+ ja L(cross_page)
+# if VEC_SIZE > 32
+ movq $-1, %rcx
+ bzhiq %rdx, %rcx, %rcx
+ kmovq %rcx, %k1
+# else
+ movl $-1, %ecx
+ bzhil %edx, %ecx, %ecx
+ kmovd %ecx, %k1
+# endif
+ vmovdqu8 %VEC(0), (%rax) {%k1}
+ VZEROUPPER_RETURN
+
+ .p2align 4
+L(cross_page):
+# endif
# if VEC_SIZE > 32
cmpb $32, %dl
jae L(between_32_63)
@@ -234,36 +263,36 @@ L(less_vec):
cmpb $1, %dl
ja L(between_2_3)
jb 1f
- movb %cl, (%rdi)
+ movb %cl, (%rax)
1:
VZEROUPPER_RETURN
# if VEC_SIZE > 32
/* From 32 to 63. No branch when size == 32. */
L(between_32_63):
- VMOVU %YMM0, -32(%rdi,%rdx)
- VMOVU %YMM0, (%rdi)
+ VMOVU %YMM0, -32(%rax,%rdx)
+ VMOVU %YMM0, (%rax)
VZEROUPPER_RETURN
# endif
# if VEC_SIZE > 16
/* From 16 to 31. No branch when size == 16. */
L(between_16_31):
- VMOVU %XMM0, -16(%rdi,%rdx)
- VMOVU %XMM0, (%rdi)
+ VMOVU %XMM0, -16(%rax,%rdx)
+ VMOVU %XMM0, (%rax)
VZEROUPPER_RETURN
# endif
/* From 8 to 15. No branch when size == 8. */
L(between_8_15):
- movq %rcx, -8(%rdi,%rdx)
- movq %rcx, (%rdi)
+ movq %rcx, -8(%rax,%rdx)
+ movq %rcx, (%rax)
VZEROUPPER_RETURN
L(between_4_7):
/* From 4 to 7. No branch when size == 4. */
- movl %ecx, -4(%rdi,%rdx)
- movl %ecx, (%rdi)
+ movl %ecx, -4(%rax,%rdx)
+ movl %ecx, (%rax)
VZEROUPPER_RETURN
L(between_2_3):
/* From 2 to 3. No branch when size == 2. */
- movw %cx, -2(%rdi,%rdx)
- movw %cx, (%rdi)
+ movw %cx, -2(%rax,%rdx)
+ movw %cx, (%rax)
VZEROUPPER_RETURN
END (MEMSET_SYMBOL (__memset, unaligned_erms))
--
GitLab

View File

@ -1,396 +0,0 @@
From ccabe7971f508709d034b63b8672f6f751a3d356 Mon Sep 17 00:00:00 2001
From: Noah Goldstein <goldstein.w.n@gmail.com>
Date: Fri, 23 Apr 2021 15:56:24 -0400
Subject: [PATCH] x86: Optimize strchr-avx2.S
Content-type: text/plain; charset=UTF-8
No bug. This commit optimizes strchr-avx2.S. The optimizations are all
small things such as save an ALU in the alignment process, saving a
few instructions in the loop return, saving some bytes in the main
loop, and increasing the ILP in the return cases. test-strchr,
test-strchrnul, test-wcschr, and test-wcschrnul are all passing.
Signed-off-by: Noah Goldstein <goldstein.w.n@gmail.com>
---
sysdeps/x86_64/multiarch/strchr-avx2.S | 290 +++++++++++++++----------
1 file changed, 170 insertions(+), 120 deletions(-)
Conflics:
sysdeps/x86_64/multiarch/strchr-avx2.S
(rearranged to account for branch changes)
diff --git a/sysdeps/x86_64/multiarch/strchr-avx2.S b/sysdeps/x86_64/multiarch/strchr-avx2.S
index 919d256c..5884726b 100644
--- a/sysdeps/x86_64/multiarch/strchr-avx2.S
+++ b/sysdeps/x86_64/multiarch/strchr-avx2.S
@@ -49,133 +49,144 @@
.section SECTION(.text),"ax",@progbits
ENTRY (STRCHR)
- movl %edi, %ecx
-# ifndef USE_AS_STRCHRNUL
- xorl %edx, %edx
-# endif
-
/* Broadcast CHAR to YMM0. */
vmovd %esi, %xmm0
+ movl %edi, %eax
+ andl $(PAGE_SIZE - 1), %eax
+ VPBROADCAST %xmm0, %ymm0
vpxor %xmm9, %xmm9, %xmm9
- VPBROADCAST %xmm0, %ymm0
/* Check if we cross page boundary with one vector load. */
- andl $(PAGE_SIZE - 1), %ecx
- cmpl $(PAGE_SIZE - VEC_SIZE), %ecx
- ja L(cross_page_boundary)
+ cmpl $(PAGE_SIZE - VEC_SIZE), %eax
+ ja L(cross_page_boundary)
/* Check the first VEC_SIZE bytes. Search for both CHAR and the
null byte. */
vmovdqu (%rdi), %ymm8
- VPCMPEQ %ymm8, %ymm0, %ymm1
- VPCMPEQ %ymm8, %ymm9, %ymm2
+ VPCMPEQ %ymm8, %ymm0, %ymm1
+ VPCMPEQ %ymm8, %ymm9, %ymm2
vpor %ymm1, %ymm2, %ymm1
vpmovmskb %ymm1, %eax
testl %eax, %eax
- jz L(more_vecs)
+ jz L(aligned_more)
tzcntl %eax, %eax
+# ifndef USE_AS_STRCHRNUL
/* Found CHAR or the null byte. */
+ cmp (%rdi, %rax), %CHAR_REG
+ jne L(zero)
+# endif
addq %rdi, %rax
+ VZEROUPPER_RETURN
+
+ /* .p2align 5 helps keep performance more consistent if ENTRY()
+ alignment % 32 was either 16 or 0. As well this makes the
+ alignment % 32 of the loop_4x_vec fixed which makes tuning it
+ easier. */
+ .p2align 5
+L(first_vec_x4):
+ tzcntl %eax, %eax
+ addq $(VEC_SIZE * 3 + 1), %rdi
# ifndef USE_AS_STRCHRNUL
- cmp (%rax), %CHAR_REG
- cmovne %rdx, %rax
+ /* Found CHAR or the null byte. */
+ cmp (%rdi, %rax), %CHAR_REG
+ jne L(zero)
# endif
-L(return_vzeroupper):
- ZERO_UPPER_VEC_REGISTERS_RETURN
-
- .p2align 4
-L(more_vecs):
- /* Align data for aligned loads in the loop. */
- andq $-VEC_SIZE, %rdi
-L(aligned_more):
-
- /* Check the next 4 * VEC_SIZE. Only one VEC_SIZE at a time
- since data is only aligned to VEC_SIZE. */
- vmovdqa VEC_SIZE(%rdi), %ymm8
- addq $VEC_SIZE, %rdi
- VPCMPEQ %ymm8, %ymm0, %ymm1
- VPCMPEQ %ymm8, %ymm9, %ymm2
- vpor %ymm1, %ymm2, %ymm1
- vpmovmskb %ymm1, %eax
- testl %eax, %eax
- jnz L(first_vec_x0)
-
- vmovdqa VEC_SIZE(%rdi), %ymm8
- VPCMPEQ %ymm8, %ymm0, %ymm1
- VPCMPEQ %ymm8, %ymm9, %ymm2
- vpor %ymm1, %ymm2, %ymm1
- vpmovmskb %ymm1, %eax
- testl %eax, %eax
- jnz L(first_vec_x1)
-
- vmovdqa (VEC_SIZE * 2)(%rdi), %ymm8
- VPCMPEQ %ymm8, %ymm0, %ymm1
- VPCMPEQ %ymm8, %ymm9, %ymm2
- vpor %ymm1, %ymm2, %ymm1
- vpmovmskb %ymm1, %eax
- testl %eax, %eax
- jnz L(first_vec_x2)
-
- vmovdqa (VEC_SIZE * 3)(%rdi), %ymm8
- VPCMPEQ %ymm8, %ymm0, %ymm1
- VPCMPEQ %ymm8, %ymm9, %ymm2
- vpor %ymm1, %ymm2, %ymm1
- vpmovmskb %ymm1, %eax
- testl %eax, %eax
- jz L(prep_loop_4x)
+ addq %rdi, %rax
+ VZEROUPPER_RETURN
- tzcntl %eax, %eax
- leaq (VEC_SIZE * 3)(%rdi, %rax), %rax
# ifndef USE_AS_STRCHRNUL
- cmp (%rax), %CHAR_REG
- cmovne %rdx, %rax
+L(zero):
+ xorl %eax, %eax
+ VZEROUPPER_RETURN
# endif
- VZEROUPPER
- ret
+
.p2align 4
-L(first_vec_x0):
+L(first_vec_x1):
tzcntl %eax, %eax
- /* Found CHAR or the null byte. */
- addq %rdi, %rax
+ incq %rdi
# ifndef USE_AS_STRCHRNUL
- cmp (%rax), %CHAR_REG
- cmovne %rdx, %rax
+ /* Found CHAR or the null byte. */
+ cmp (%rdi, %rax), %CHAR_REG
+ jne L(zero)
# endif
+ addq %rdi, %rax
VZEROUPPER_RETURN
.p2align 4
-L(first_vec_x1):
+L(first_vec_x2):
tzcntl %eax, %eax
- leaq VEC_SIZE(%rdi, %rax), %rax
+ addq $(VEC_SIZE + 1), %rdi
# ifndef USE_AS_STRCHRNUL
- cmp (%rax), %CHAR_REG
- cmovne %rdx, %rax
+ /* Found CHAR or the null byte. */
+ cmp (%rdi, %rax), %CHAR_REG
+ jne L(zero)
# endif
+ addq %rdi, %rax
VZEROUPPER_RETURN
.p2align 4
-L(first_vec_x2):
+L(first_vec_x3):
tzcntl %eax, %eax
- /* Found CHAR or the null byte. */
- leaq (VEC_SIZE * 2)(%rdi, %rax), %rax
+ addq $(VEC_SIZE * 2 + 1), %rdi
# ifndef USE_AS_STRCHRNUL
- cmp (%rax), %CHAR_REG
- cmovne %rdx, %rax
+ /* Found CHAR or the null byte. */
+ cmp (%rdi, %rax), %CHAR_REG
+ jne L(zero)
# endif
+ addq %rdi, %rax
VZEROUPPER_RETURN
-L(prep_loop_4x):
- /* Align data to 4 * VEC_SIZE. */
- andq $-(VEC_SIZE * 4), %rdi
+ .p2align 4
+L(aligned_more):
+ /* Align data to VEC_SIZE - 1. This is the same number of
+ instructions as using andq -VEC_SIZE but saves 4 bytes of code
+ on x4 check. */
+ orq $(VEC_SIZE - 1), %rdi
+L(cross_page_continue):
+ /* Check the next 4 * VEC_SIZE. Only one VEC_SIZE at a time
+ since data is only aligned to VEC_SIZE. */
+ vmovdqa 1(%rdi), %ymm8
+ VPCMPEQ %ymm8, %ymm0, %ymm1
+ VPCMPEQ %ymm8, %ymm9, %ymm2
+ vpor %ymm1, %ymm2, %ymm1
+ vpmovmskb %ymm1, %eax
+ testl %eax, %eax
+ jnz L(first_vec_x1)
+
+ vmovdqa (VEC_SIZE + 1)(%rdi), %ymm8
+ VPCMPEQ %ymm8, %ymm0, %ymm1
+ VPCMPEQ %ymm8, %ymm9, %ymm2
+ vpor %ymm1, %ymm2, %ymm1
+ vpmovmskb %ymm1, %eax
+ testl %eax, %eax
+ jnz L(first_vec_x2)
+
+ vmovdqa (VEC_SIZE * 2 + 1)(%rdi), %ymm8
+ VPCMPEQ %ymm8, %ymm0, %ymm1
+ VPCMPEQ %ymm8, %ymm9, %ymm2
+ vpor %ymm1, %ymm2, %ymm1
+ vpmovmskb %ymm1, %eax
+ testl %eax, %eax
+ jnz L(first_vec_x3)
+ vmovdqa (VEC_SIZE * 3 + 1)(%rdi), %ymm8
+ VPCMPEQ %ymm8, %ymm0, %ymm1
+ VPCMPEQ %ymm8, %ymm9, %ymm2
+ vpor %ymm1, %ymm2, %ymm1
+ vpmovmskb %ymm1, %eax
+ testl %eax, %eax
+ jnz L(first_vec_x4)
+ /* Align data to VEC_SIZE * 4 - 1. */
+ addq $(VEC_SIZE * 4 + 1), %rdi
+ andq $-(VEC_SIZE * 4), %rdi
.p2align 4
L(loop_4x_vec):
/* Compare 4 * VEC at a time forward. */
- vmovdqa (VEC_SIZE * 4)(%rdi), %ymm5
- vmovdqa (VEC_SIZE * 5)(%rdi), %ymm6
- vmovdqa (VEC_SIZE * 6)(%rdi), %ymm7
- vmovdqa (VEC_SIZE * 7)(%rdi), %ymm8
+ vmovdqa (%rdi), %ymm5
+ vmovdqa (VEC_SIZE)(%rdi), %ymm6
+ vmovdqa (VEC_SIZE * 2)(%rdi), %ymm7
+ vmovdqa (VEC_SIZE * 3)(%rdi), %ymm8
/* Leaves only CHARS matching esi as 0. */
vpxor %ymm5, %ymm0, %ymm1
@@ -191,63 +202,102 @@ L(loop_4x_vec):
VPMINU %ymm1, %ymm2, %ymm5
VPMINU %ymm3, %ymm4, %ymm6
- VPMINU %ymm5, %ymm6, %ymm5
+ VPMINU %ymm5, %ymm6, %ymm6
- VPCMPEQ %ymm5, %ymm9, %ymm5
- vpmovmskb %ymm5, %eax
+ VPCMPEQ %ymm6, %ymm9, %ymm6
+ vpmovmskb %ymm6, %ecx
+ subq $-(VEC_SIZE * 4), %rdi
+ testl %ecx, %ecx
+ jz L(loop_4x_vec)
- addq $(VEC_SIZE * 4), %rdi
- testl %eax, %eax
- jz L(loop_4x_vec)
- VPCMPEQ %ymm1, %ymm9, %ymm1
+ VPCMPEQ %ymm1, %ymm9, %ymm1
vpmovmskb %ymm1, %eax
testl %eax, %eax
- jnz L(first_vec_x0)
+ jnz L(last_vec_x0)
+
- VPCMPEQ %ymm2, %ymm9, %ymm2
+ VPCMPEQ %ymm5, %ymm9, %ymm2
vpmovmskb %ymm2, %eax
testl %eax, %eax
- jnz L(first_vec_x1)
+ jnz L(last_vec_x1)
+
+ VPCMPEQ %ymm3, %ymm9, %ymm3
+ vpmovmskb %ymm3, %eax
+ /* rcx has combined result from all 4 VEC. It will only be used
+ if the first 3 other VEC all did not contain a match. */
+ salq $32, %rcx
+ orq %rcx, %rax
+ tzcntq %rax, %rax
+ subq $(VEC_SIZE * 2), %rdi
+# ifndef USE_AS_STRCHRNUL
+ /* Found CHAR or the null byte. */
+ cmp (%rdi, %rax), %CHAR_REG
+ jne L(zero_end)
+# endif
+ addq %rdi, %rax
+ VZEROUPPER_RETURN
+
+
+ .p2align 4
+L(last_vec_x0):
+ tzcntl %eax, %eax
+ addq $-(VEC_SIZE * 4), %rdi
+# ifndef USE_AS_STRCHRNUL
+ /* Found CHAR or the null byte. */
+ cmp (%rdi, %rax), %CHAR_REG
+ jne L(zero_end)
+# endif
+ addq %rdi, %rax
+ VZEROUPPER_RETURN
- VPCMPEQ %ymm3, %ymm9, %ymm3
- VPCMPEQ %ymm4, %ymm9, %ymm4
- vpmovmskb %ymm3, %ecx
- vpmovmskb %ymm4, %eax
- salq $32, %rax
- orq %rcx, %rax
- tzcntq %rax, %rax
- leaq (VEC_SIZE * 2)(%rdi, %rax), %rax
# ifndef USE_AS_STRCHRNUL
- cmp (%rax), %CHAR_REG
- cmovne %rdx, %rax
+L(zero_end):
+ xorl %eax, %eax
+ VZEROUPPER_RETURN
# endif
- VZEROUPPER
- ret
+
+ .p2align 4
+L(last_vec_x1):
+ tzcntl %eax, %eax
+ subq $(VEC_SIZE * 3), %rdi
+# ifndef USE_AS_STRCHRNUL
+ /* Found CHAR or the null byte. */
+ cmp (%rdi, %rax), %CHAR_REG
+ jne L(zero_end)
+# endif
+ addq %rdi, %rax
+ VZEROUPPER_RETURN
+
/* Cold case for crossing page with first load. */
.p2align 4
L(cross_page_boundary):
- andq $-VEC_SIZE, %rdi
- andl $(VEC_SIZE - 1), %ecx
-
- vmovdqa (%rdi), %ymm8
- VPCMPEQ %ymm8, %ymm0, %ymm1
- VPCMPEQ %ymm8, %ymm9, %ymm2
+ movq %rdi, %rdx
+ /* Align rdi to VEC_SIZE - 1. */
+ orq $(VEC_SIZE - 1), %rdi
+ vmovdqa -(VEC_SIZE - 1)(%rdi), %ymm8
+ VPCMPEQ %ymm8, %ymm0, %ymm1
+ VPCMPEQ %ymm8, %ymm9, %ymm2
vpor %ymm1, %ymm2, %ymm1
vpmovmskb %ymm1, %eax
- /* Remove the leading bits. */
- sarxl %ecx, %eax, %eax
+ /* Remove the leading bytes. sarxl only uses bits [5:0] of COUNT
+ so no need to manually mod edx. */
+ sarxl %edx, %eax, %eax
testl %eax, %eax
- jz L(aligned_more)
+ jz L(cross_page_continue)
tzcntl %eax, %eax
- addq %rcx, %rdi
- addq %rdi, %rax
# ifndef USE_AS_STRCHRNUL
- cmp (%rax), %CHAR_REG
- cmovne %rdx, %rax
+ xorl %ecx, %ecx
+ /* Found CHAR or the null byte. */
+ cmp (%rdx, %rax), %CHAR_REG
+ leaq (%rdx, %rax), %rax
+ cmovne %rcx, %rax
+# else
+ addq %rdx, %rax
# endif
- VZEROUPPER_RETURN
+L(return_vzeroupper):
+ ZERO_UPPER_VEC_REGISTERS_RETURN
END (STRCHR)
# endif
--
GitLab

View File

@ -1,532 +0,0 @@
From 7f3e7c262cab4e2401e4331a6ef29c428de02044 Mon Sep 17 00:00:00 2001
From: Noah Goldstein <goldstein.w.n@gmail.com>
Date: Fri, 23 Apr 2021 15:56:25 -0400
Subject: [PATCH] x86: Optimize strchr-evex.S
Content-type: text/plain; charset=UTF-8
No bug. This commit optimizes strchr-evex.S. The optimizations are
mostly small things such as save an ALU in the alignment process,
saving a few instructions in the loop return. The one significant
change is saving 2 instructions in the 4x loop. test-strchr,
test-strchrnul, test-wcschr, and test-wcschrnul are all passing.
Signed-off-by: Noah Goldstein <goldstein.w.n@gmail.com>
---
sysdeps/x86_64/multiarch/strchr-evex.S | 392 ++++++++++++++-----------
1 file changed, 218 insertions(+), 174 deletions(-)
diff --git a/sysdeps/x86_64/multiarch/strchr-evex.S b/sysdeps/x86_64/multiarch/strchr-evex.S
index ddc86a70..7f9d4ee4 100644
--- a/sysdeps/x86_64/multiarch/strchr-evex.S
+++ b/sysdeps/x86_64/multiarch/strchr-evex.S
@@ -32,13 +32,15 @@
# define VPCMP vpcmpd
# define VPMINU vpminud
# define CHAR_REG esi
-# define SHIFT_REG r8d
+# define SHIFT_REG ecx
+# define CHAR_SIZE 4
# else
# define VPBROADCAST vpbroadcastb
# define VPCMP vpcmpb
# define VPMINU vpminub
# define CHAR_REG sil
-# define SHIFT_REG ecx
+# define SHIFT_REG edx
+# define CHAR_SIZE 1
# endif
# define XMMZERO xmm16
@@ -56,23 +58,20 @@
# define VEC_SIZE 32
# define PAGE_SIZE 4096
+# define CHAR_PER_VEC (VEC_SIZE / CHAR_SIZE)
.section .text.evex,"ax",@progbits
ENTRY (STRCHR)
- movl %edi, %ecx
-# ifndef USE_AS_STRCHRNUL
- xorl %edx, %edx
-# endif
-
/* Broadcast CHAR to YMM0. */
- VPBROADCAST %esi, %YMM0
-
+ VPBROADCAST %esi, %YMM0
+ movl %edi, %eax
+ andl $(PAGE_SIZE - 1), %eax
vpxorq %XMMZERO, %XMMZERO, %XMMZERO
- /* Check if we cross page boundary with one vector load. */
- andl $(PAGE_SIZE - 1), %ecx
- cmpl $(PAGE_SIZE - VEC_SIZE), %ecx
- ja L(cross_page_boundary)
+ /* Check if we cross page boundary with one vector load.
+ Otherwise it is safe to use an unaligned load. */
+ cmpl $(PAGE_SIZE - VEC_SIZE), %eax
+ ja L(cross_page_boundary)
/* Check the first VEC_SIZE bytes. Search for both CHAR and the
null bytes. */
@@ -83,251 +82,296 @@ ENTRY (STRCHR)
VPMINU %YMM2, %YMM1, %YMM2
/* Each bit in K0 represents a CHAR or a null byte in YMM1. */
VPCMP $0, %YMMZERO, %YMM2, %k0
- ktestd %k0, %k0
- jz L(more_vecs)
kmovd %k0, %eax
+ testl %eax, %eax
+ jz L(aligned_more)
tzcntl %eax, %eax
- /* Found CHAR or the null byte. */
# ifdef USE_AS_WCSCHR
- /* NB: Multiply wchar_t count by 4 to get the number of bytes. */
- leaq (%rdi, %rax, 4), %rax
+ /* NB: Multiply wchar_t count by 4 to get the number of bytes.
+ */
+ leaq (%rdi, %rax, CHAR_SIZE), %rax
# else
addq %rdi, %rax
# endif
# ifndef USE_AS_STRCHRNUL
- cmp (%rax), %CHAR_REG
- cmovne %rdx, %rax
+ /* Found CHAR or the null byte. */
+ cmp (%rax), %CHAR_REG
+ jne L(zero)
# endif
ret
- .p2align 4
-L(more_vecs):
- /* Align data for aligned loads in the loop. */
- andq $-VEC_SIZE, %rdi
-L(aligned_more):
-
- /* Check the next 4 * VEC_SIZE. Only one VEC_SIZE at a time
- since data is only aligned to VEC_SIZE. */
- VMOVA VEC_SIZE(%rdi), %YMM1
- addq $VEC_SIZE, %rdi
-
- /* Leaves only CHARS matching esi as 0. */
- vpxorq %YMM1, %YMM0, %YMM2
- VPMINU %YMM2, %YMM1, %YMM2
- /* Each bit in K0 represents a CHAR or a null byte in YMM1. */
- VPCMP $0, %YMMZERO, %YMM2, %k0
- kmovd %k0, %eax
- testl %eax, %eax
- jnz L(first_vec_x0)
-
- VMOVA VEC_SIZE(%rdi), %YMM1
- /* Leaves only CHARS matching esi as 0. */
- vpxorq %YMM1, %YMM0, %YMM2
- VPMINU %YMM2, %YMM1, %YMM2
- /* Each bit in K0 represents a CHAR or a null byte in YMM1. */
- VPCMP $0, %YMMZERO, %YMM2, %k0
- kmovd %k0, %eax
- testl %eax, %eax
- jnz L(first_vec_x1)
-
- VMOVA (VEC_SIZE * 2)(%rdi), %YMM1
- /* Leaves only CHARS matching esi as 0. */
- vpxorq %YMM1, %YMM0, %YMM2
- VPMINU %YMM2, %YMM1, %YMM2
- /* Each bit in K0 represents a CHAR or a null byte in YMM1. */
- VPCMP $0, %YMMZERO, %YMM2, %k0
- kmovd %k0, %eax
- testl %eax, %eax
- jnz L(first_vec_x2)
-
- VMOVA (VEC_SIZE * 3)(%rdi), %YMM1
- /* Leaves only CHARS matching esi as 0. */
- vpxorq %YMM1, %YMM0, %YMM2
- VPMINU %YMM2, %YMM1, %YMM2
- /* Each bit in K0 represents a CHAR or a null byte in YMM1. */
- VPCMP $0, %YMMZERO, %YMM2, %k0
- ktestd %k0, %k0
- jz L(prep_loop_4x)
-
- kmovd %k0, %eax
+ /* .p2align 5 helps keep performance more consistent if ENTRY()
+ alignment % 32 was either 16 or 0. As well this makes the
+ alignment % 32 of the loop_4x_vec fixed which makes tuning it
+ easier. */
+ .p2align 5
+L(first_vec_x3):
tzcntl %eax, %eax
+# ifndef USE_AS_STRCHRNUL
/* Found CHAR or the null byte. */
-# ifdef USE_AS_WCSCHR
- /* NB: Multiply wchar_t count by 4 to get the number of bytes. */
- leaq (VEC_SIZE * 3)(%rdi, %rax, 4), %rax
-# else
- leaq (VEC_SIZE * 3)(%rdi, %rax), %rax
+ cmp (VEC_SIZE * 3)(%rdi, %rax, CHAR_SIZE), %CHAR_REG
+ jne L(zero)
# endif
+ /* NB: Multiply sizeof char type (1 or 4) to get the number of
+ bytes. */
+ leaq (VEC_SIZE * 3)(%rdi, %rax, CHAR_SIZE), %rax
+ ret
+
# ifndef USE_AS_STRCHRNUL
- cmp (%rax), %CHAR_REG
- cmovne %rdx, %rax
-# endif
+L(zero):
+ xorl %eax, %eax
ret
+# endif
.p2align 4
-L(first_vec_x0):
+L(first_vec_x4):
+# ifndef USE_AS_STRCHRNUL
+ /* Check to see if first match was CHAR (k0) or null (k1). */
+ kmovd %k0, %eax
tzcntl %eax, %eax
- /* Found CHAR or the null byte. */
-# ifdef USE_AS_WCSCHR
- /* NB: Multiply wchar_t count by 4 to get the number of bytes. */
- leaq (%rdi, %rax, 4), %rax
+ kmovd %k1, %ecx
+ /* bzhil will not be 0 if first match was null. */
+ bzhil %eax, %ecx, %ecx
+ jne L(zero)
# else
- addq %rdi, %rax
-# endif
-# ifndef USE_AS_STRCHRNUL
- cmp (%rax), %CHAR_REG
- cmovne %rdx, %rax
+ /* Combine CHAR and null matches. */
+ kord %k0, %k1, %k0
+ kmovd %k0, %eax
+ tzcntl %eax, %eax
# endif
+ /* NB: Multiply sizeof char type (1 or 4) to get the number of
+ bytes. */
+ leaq (VEC_SIZE * 4)(%rdi, %rax, CHAR_SIZE), %rax
ret
.p2align 4
L(first_vec_x1):
tzcntl %eax, %eax
- /* Found CHAR or the null byte. */
-# ifdef USE_AS_WCSCHR
- /* NB: Multiply wchar_t count by 4 to get the number of bytes. */
- leaq VEC_SIZE(%rdi, %rax, 4), %rax
-# else
- leaq VEC_SIZE(%rdi, %rax), %rax
-# endif
# ifndef USE_AS_STRCHRNUL
- cmp (%rax), %CHAR_REG
- cmovne %rdx, %rax
+ /* Found CHAR or the null byte. */
+ cmp (VEC_SIZE)(%rdi, %rax, CHAR_SIZE), %CHAR_REG
+ jne L(zero)
+
# endif
+ /* NB: Multiply sizeof char type (1 or 4) to get the number of
+ bytes. */
+ leaq (VEC_SIZE)(%rdi, %rax, CHAR_SIZE), %rax
ret
.p2align 4
L(first_vec_x2):
+# ifndef USE_AS_STRCHRNUL
+ /* Check to see if first match was CHAR (k0) or null (k1). */
+ kmovd %k0, %eax
tzcntl %eax, %eax
- /* Found CHAR or the null byte. */
-# ifdef USE_AS_WCSCHR
- /* NB: Multiply wchar_t count by 4 to get the number of bytes. */
- leaq (VEC_SIZE * 2)(%rdi, %rax, 4), %rax
+ kmovd %k1, %ecx
+ /* bzhil will not be 0 if first match was null. */
+ bzhil %eax, %ecx, %ecx
+ jne L(zero)
# else
- leaq (VEC_SIZE * 2)(%rdi, %rax), %rax
-# endif
-# ifndef USE_AS_STRCHRNUL
- cmp (%rax), %CHAR_REG
- cmovne %rdx, %rax
+ /* Combine CHAR and null matches. */
+ kord %k0, %k1, %k0
+ kmovd %k0, %eax
+ tzcntl %eax, %eax
# endif
+ /* NB: Multiply sizeof char type (1 or 4) to get the number of
+ bytes. */
+ leaq (VEC_SIZE * 2)(%rdi, %rax, CHAR_SIZE), %rax
ret
-L(prep_loop_4x):
- /* Align data to 4 * VEC_SIZE. */
+ .p2align 4
+L(aligned_more):
+ /* Align data to VEC_SIZE. */
+ andq $-VEC_SIZE, %rdi
+L(cross_page_continue):
+ /* Check the next 4 * VEC_SIZE. Only one VEC_SIZE at a time since
+ data is only aligned to VEC_SIZE. Use two alternating methods
+ for checking VEC to balance latency and port contention. */
+
+ /* This method has higher latency but has better port
+ distribution. */
+ VMOVA (VEC_SIZE)(%rdi), %YMM1
+ /* Leaves only CHARS matching esi as 0. */
+ vpxorq %YMM1, %YMM0, %YMM2
+ VPMINU %YMM2, %YMM1, %YMM2
+ /* Each bit in K0 represents a CHAR or a null byte in YMM1. */
+ VPCMP $0, %YMMZERO, %YMM2, %k0
+ kmovd %k0, %eax
+ testl %eax, %eax
+ jnz L(first_vec_x1)
+
+ /* This method has higher latency but has better port
+ distribution. */
+ VMOVA (VEC_SIZE * 2)(%rdi), %YMM1
+ /* Each bit in K0 represents a CHAR in YMM1. */
+ VPCMP $0, %YMM1, %YMM0, %k0
+ /* Each bit in K1 represents a CHAR in YMM1. */
+ VPCMP $0, %YMM1, %YMMZERO, %k1
+ kortestd %k0, %k1
+ jnz L(first_vec_x2)
+
+ VMOVA (VEC_SIZE * 3)(%rdi), %YMM1
+ /* Leaves only CHARS matching esi as 0. */
+ vpxorq %YMM1, %YMM0, %YMM2
+ VPMINU %YMM2, %YMM1, %YMM2
+ /* Each bit in K0 represents a CHAR or a null byte in YMM1. */
+ VPCMP $0, %YMMZERO, %YMM2, %k0
+ kmovd %k0, %eax
+ testl %eax, %eax
+ jnz L(first_vec_x3)
+
+ VMOVA (VEC_SIZE * 4)(%rdi), %YMM1
+ /* Each bit in K0 represents a CHAR in YMM1. */
+ VPCMP $0, %YMM1, %YMM0, %k0
+ /* Each bit in K1 represents a CHAR in YMM1. */
+ VPCMP $0, %YMM1, %YMMZERO, %k1
+ kortestd %k0, %k1
+ jnz L(first_vec_x4)
+
+ /* Align data to VEC_SIZE * 4 for the loop. */
+ addq $VEC_SIZE, %rdi
andq $-(VEC_SIZE * 4), %rdi
.p2align 4
L(loop_4x_vec):
- /* Compare 4 * VEC at a time forward. */
+ /* Check 4x VEC at a time. No penalty to imm32 offset with evex
+ encoding. */
VMOVA (VEC_SIZE * 4)(%rdi), %YMM1
VMOVA (VEC_SIZE * 5)(%rdi), %YMM2
VMOVA (VEC_SIZE * 6)(%rdi), %YMM3
VMOVA (VEC_SIZE * 7)(%rdi), %YMM4
- /* Leaves only CHARS matching esi as 0. */
+ /* For YMM1 and YMM3 use xor to set the CHARs matching esi to
+ zero. */
vpxorq %YMM1, %YMM0, %YMM5
- vpxorq %YMM2, %YMM0, %YMM6
+ /* For YMM2 and YMM4 cmp not equals to CHAR and store result in
+ k register. Its possible to save either 1 or 2 instructions
+ using cmp no equals method for either YMM1 or YMM1 and YMM3
+ respectively but bottleneck on p5 makes it not worth it. */
+ VPCMP $4, %YMM0, %YMM2, %k2
vpxorq %YMM3, %YMM0, %YMM7
- vpxorq %YMM4, %YMM0, %YMM8
-
- VPMINU %YMM5, %YMM1, %YMM5
- VPMINU %YMM6, %YMM2, %YMM6
- VPMINU %YMM7, %YMM3, %YMM7
- VPMINU %YMM8, %YMM4, %YMM8
-
- VPMINU %YMM5, %YMM6, %YMM1
- VPMINU %YMM7, %YMM8, %YMM2
-
- VPMINU %YMM1, %YMM2, %YMM1
-
- /* Each bit in K0 represents a CHAR or a null byte. */
- VPCMP $0, %YMMZERO, %YMM1, %k0
-
- addq $(VEC_SIZE * 4), %rdi
-
- ktestd %k0, %k0
+ VPCMP $4, %YMM0, %YMM4, %k4
+
+ /* Use min to select all zeros from either xor or end of string).
+ */
+ VPMINU %YMM1, %YMM5, %YMM1
+ VPMINU %YMM3, %YMM7, %YMM3
+
+ /* Use min + zeromask to select for zeros. Since k2 and k4 will
+ have 0 as positions that matched with CHAR which will set
+ zero in the corresponding destination bytes in YMM2 / YMM4.
+ */
+ VPMINU %YMM1, %YMM2, %YMM2{%k2}{z}
+ VPMINU %YMM3, %YMM4, %YMM4
+ VPMINU %YMM2, %YMM4, %YMM4{%k4}{z}
+
+ VPCMP $0, %YMMZERO, %YMM4, %k1
+ kmovd %k1, %ecx
+ subq $-(VEC_SIZE * 4), %rdi
+ testl %ecx, %ecx
jz L(loop_4x_vec)
- /* Each bit in K0 represents a CHAR or a null byte in YMM1. */
- VPCMP $0, %YMMZERO, %YMM5, %k0
+ VPCMP $0, %YMMZERO, %YMM1, %k0
kmovd %k0, %eax
testl %eax, %eax
- jnz L(first_vec_x0)
+ jnz L(last_vec_x1)
- /* Each bit in K1 represents a CHAR or a null byte in YMM2. */
- VPCMP $0, %YMMZERO, %YMM6, %k1
- kmovd %k1, %eax
+ VPCMP $0, %YMMZERO, %YMM2, %k0
+ kmovd %k0, %eax
testl %eax, %eax
- jnz L(first_vec_x1)
-
- /* Each bit in K2 represents a CHAR or a null byte in YMM3. */
- VPCMP $0, %YMMZERO, %YMM7, %k2
- /* Each bit in K3 represents a CHAR or a null byte in YMM4. */
- VPCMP $0, %YMMZERO, %YMM8, %k3
+ jnz L(last_vec_x2)
+ VPCMP $0, %YMMZERO, %YMM3, %k0
+ kmovd %k0, %eax
+ /* Combine YMM3 matches (eax) with YMM4 matches (ecx). */
# ifdef USE_AS_WCSCHR
- /* NB: Each bit in K2/K3 represents 4-byte element. */
- kshiftlw $8, %k3, %k1
+ sall $8, %ecx
+ orl %ecx, %eax
+ tzcntl %eax, %eax
# else
- kshiftlq $32, %k3, %k1
+ salq $32, %rcx
+ orq %rcx, %rax
+ tzcntq %rax, %rax
# endif
+# ifndef USE_AS_STRCHRNUL
+ /* Check if match was CHAR or null. */
+ cmp (VEC_SIZE * 2)(%rdi, %rax, CHAR_SIZE), %CHAR_REG
+ jne L(zero_end)
+# endif
+ /* NB: Multiply sizeof char type (1 or 4) to get the number of
+ bytes. */
+ leaq (VEC_SIZE * 2)(%rdi, %rax, CHAR_SIZE), %rax
+ ret
- /* Each bit in K1 represents a NULL or a mismatch. */
- korq %k1, %k2, %k1
- kmovq %k1, %rax
+# ifndef USE_AS_STRCHRNUL
+L(zero_end):
+ xorl %eax, %eax
+ ret
+# endif
- tzcntq %rax, %rax
-# ifdef USE_AS_WCSCHR
- /* NB: Multiply wchar_t count by 4 to get the number of bytes. */
- leaq (VEC_SIZE * 2)(%rdi, %rax, 4), %rax
-# else
- leaq (VEC_SIZE * 2)(%rdi, %rax), %rax
+ .p2align 4
+L(last_vec_x1):
+ tzcntl %eax, %eax
+# ifndef USE_AS_STRCHRNUL
+ /* Check if match was null. */
+ cmp (%rdi, %rax, CHAR_SIZE), %CHAR_REG
+ jne L(zero_end)
# endif
+ /* NB: Multiply sizeof char type (1 or 4) to get the number of
+ bytes. */
+ leaq (%rdi, %rax, CHAR_SIZE), %rax
+ ret
+
+ .p2align 4
+L(last_vec_x2):
+ tzcntl %eax, %eax
# ifndef USE_AS_STRCHRNUL
- cmp (%rax), %CHAR_REG
- cmovne %rdx, %rax
+ /* Check if match was null. */
+ cmp (VEC_SIZE)(%rdi, %rax, CHAR_SIZE), %CHAR_REG
+ jne L(zero_end)
# endif
+ /* NB: Multiply sizeof char type (1 or 4) to get the number of
+ bytes. */
+ leaq (VEC_SIZE)(%rdi, %rax, CHAR_SIZE), %rax
ret
/* Cold case for crossing page with first load. */
.p2align 4
L(cross_page_boundary):
+ movq %rdi, %rdx
+ /* Align rdi. */
andq $-VEC_SIZE, %rdi
- andl $(VEC_SIZE - 1), %ecx
-
VMOVA (%rdi), %YMM1
-
/* Leaves only CHARS matching esi as 0. */
vpxorq %YMM1, %YMM0, %YMM2
VPMINU %YMM2, %YMM1, %YMM2
/* Each bit in K0 represents a CHAR or a null byte in YMM1. */
VPCMP $0, %YMMZERO, %YMM2, %k0
kmovd %k0, %eax
- testl %eax, %eax
-
+ /* Remove the leading bits. */
# ifdef USE_AS_WCSCHR
+ movl %edx, %SHIFT_REG
/* NB: Divide shift count by 4 since each bit in K1 represent 4
bytes. */
- movl %ecx, %SHIFT_REG
- sarl $2, %SHIFT_REG
+ sarl $2, %SHIFT_REG
+ andl $(CHAR_PER_VEC - 1), %SHIFT_REG
# endif
-
- /* Remove the leading bits. */
sarxl %SHIFT_REG, %eax, %eax
+ /* If eax is zero continue. */
testl %eax, %eax
-
- jz L(aligned_more)
+ jz L(cross_page_continue)
tzcntl %eax, %eax
- addq %rcx, %rdi
+# ifndef USE_AS_STRCHRNUL
+ /* Check to see if match was CHAR or null. */
+ cmp (%rdx, %rax, CHAR_SIZE), %CHAR_REG
+ jne L(zero_end)
+# endif
# ifdef USE_AS_WCSCHR
- /* NB: Multiply wchar_t count by 4 to get the number of bytes. */
- leaq (%rdi, %rax, 4), %rax
+ /* NB: Multiply wchar_t count by 4 to get the number of
+ bytes. */
+ leaq (%rdx, %rax, CHAR_SIZE), %rax
# else
- addq %rdi, %rax
-# endif
-# ifndef USE_AS_STRCHRNUL
- cmp (%rax), %CHAR_REG
- cmovne %rdx, %rax
+ addq %rdx, %rax
# endif
ret
--
GitLab

View File

@ -1,536 +0,0 @@
From 104c7b1967c3e78435c6f7eab5e225a7eddf9c6e Mon Sep 17 00:00:00 2001
From: Noah Goldstein <goldstein.w.n@gmail.com>
Date: Tue, 4 May 2021 19:02:40 -0400
Subject: [PATCH] x86: Add EVEX optimized memchr family not safe for RTM
Content-type: text/plain; charset=UTF-8
No bug.
This commit adds a new implementation for EVEX memchr that is not safe
for RTM because it uses vzeroupper. The benefit is that by using
ymm0-ymm15 it can use vpcmpeq and vpternlogd in the 4x loop which is
faster than the RTM safe version which cannot use vpcmpeq because
there is no EVEX encoding for the instruction. All parts of the
implementation aside from the 4x loop are the same for the two
versions and the optimization is only relevant for large sizes.
Tigerlake:
size , algn , Pos , Cur T , New T , Win , Dif
512 , 6 , 192 , 9.2 , 9.04 , no-RTM , 0.16
512 , 7 , 224 , 9.19 , 8.98 , no-RTM , 0.21
2048 , 0 , 256 , 10.74 , 10.54 , no-RTM , 0.2
2048 , 0 , 512 , 14.81 , 14.87 , RTM , 0.06
2048 , 0 , 1024 , 22.97 , 22.57 , no-RTM , 0.4
2048 , 0 , 2048 , 37.49 , 34.51 , no-RTM , 2.98 <--
Icelake:
size , algn , Pos , Cur T , New T , Win , Dif
512 , 6 , 192 , 7.6 , 7.3 , no-RTM , 0.3
512 , 7 , 224 , 7.63 , 7.27 , no-RTM , 0.36
2048 , 0 , 256 , 8.48 , 8.38 , no-RTM , 0.1
2048 , 0 , 512 , 11.57 , 11.42 , no-RTM , 0.15
2048 , 0 , 1024 , 17.92 , 17.38 , no-RTM , 0.54
2048 , 0 , 2048 , 30.37 , 27.34 , no-RTM , 3.03 <--
test-memchr, test-wmemchr, and test-rawmemchr are all passing.
Signed-off-by: Noah Goldstein <goldstein.w.n@gmail.com>
Reviewed-by: H.J. Lu <hjl.tools@gmail.com>
---
sysdeps/x86_64/multiarch/Makefile | 7 +-
sysdeps/x86_64/multiarch/ifunc-evex.h | 55 ++++++
sysdeps/x86_64/multiarch/ifunc-impl-list.c | 15 ++
sysdeps/x86_64/multiarch/memchr-evex-rtm.S | 8 +
sysdeps/x86_64/multiarch/memchr-evex.S | 161 ++++++++++++++----
sysdeps/x86_64/multiarch/memchr.c | 2 +-
sysdeps/x86_64/multiarch/rawmemchr-evex-rtm.S | 3 +
sysdeps/x86_64/multiarch/rawmemchr.c | 2 +-
sysdeps/x86_64/multiarch/wmemchr-evex-rtm.S | 3 +
sysdeps/x86_64/multiarch/wmemchr.c | 2 +-
10 files changed, 217 insertions(+), 41 deletions(-)
create mode 100644 sysdeps/x86_64/multiarch/ifunc-evex.h
create mode 100644 sysdeps/x86_64/multiarch/memchr-evex-rtm.S
create mode 100644 sysdeps/x86_64/multiarch/rawmemchr-evex-rtm.S
create mode 100644 sysdeps/x86_64/multiarch/wmemchr-evex-rtm.S
diff --git a/sysdeps/x86_64/multiarch/Makefile b/sysdeps/x86_64/multiarch/Makefile
index 65fde4eb..26be4095 100644
--- a/sysdeps/x86_64/multiarch/Makefile
+++ b/sysdeps/x86_64/multiarch/Makefile
@@ -77,7 +77,9 @@ sysdep_routines += strncat-c stpncpy-c strncpy-c \
strncmp-evex \
strncpy-evex \
strnlen-evex \
- strrchr-evex
+ strrchr-evex \
+ memchr-evex-rtm \
+ rawmemchr-evex-rtm
CFLAGS-varshift.c += -msse4
CFLAGS-strcspn-c.c += -msse4
CFLAGS-strpbrk-c.c += -msse4
@@ -110,7 +112,8 @@ sysdep_routines += wmemcmp-sse4 wmemcmp-ssse3 wmemcmp-c \
wcsnlen-evex \
wcsrchr-evex \
wmemchr-evex \
- wmemcmp-evex-movbe
+ wmemcmp-evex-movbe \
+ wmemchr-evex-rtm
endif
ifeq ($(subdir),debug)
diff --git a/sysdeps/x86_64/multiarch/ifunc-evex.h b/sysdeps/x86_64/multiarch/ifunc-evex.h
new file mode 100644
index 00000000..fc391edb
--- /dev/null
+++ b/sysdeps/x86_64/multiarch/ifunc-evex.h
@@ -0,0 +1,55 @@
+/* Common definition for ifunc selection optimized with EVEX.
+ All versions must be listed in ifunc-impl-list.c.
+ Copyright (C) 2017-2021 Free Software Foundation, Inc.
+ This file is part of the GNU C Library.
+
+ The GNU C Library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ The GNU C Library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with the GNU C Library; if not, see
+ <https://www.gnu.org/licenses/>. */
+
+#include <init-arch.h>
+
+extern __typeof (REDIRECT_NAME) OPTIMIZE (sse2) attribute_hidden;
+extern __typeof (REDIRECT_NAME) OPTIMIZE (avx2) attribute_hidden;
+extern __typeof (REDIRECT_NAME) OPTIMIZE (avx2_rtm) attribute_hidden;
+extern __typeof (REDIRECT_NAME) OPTIMIZE (evex) attribute_hidden;
+extern __typeof (REDIRECT_NAME) OPTIMIZE (evex_rtm) attribute_hidden;
+
+
+static inline void *
+IFUNC_SELECTOR (void)
+{
+ const struct cpu_features* cpu_features = __get_cpu_features ();
+
+ if (CPU_FEATURE_USABLE_P (cpu_features, AVX2)
+ && CPU_FEATURE_USABLE_P (cpu_features, BMI2)
+ && CPU_FEATURES_ARCH_P (cpu_features, AVX_Fast_Unaligned_Load))
+ {
+ if (CPU_FEATURE_USABLE_P (cpu_features, AVX512VL)
+ && CPU_FEATURE_USABLE_P (cpu_features, AVX512BW))
+ {
+ if (CPU_FEATURE_USABLE_P (cpu_features, RTM))
+ return OPTIMIZE (evex_rtm);
+
+ return OPTIMIZE (evex);
+ }
+
+ if (CPU_FEATURE_USABLE_P (cpu_features, RTM))
+ return OPTIMIZE (avx2_rtm);
+
+ if (!CPU_FEATURES_ARCH_P (cpu_features, Prefer_No_VZEROUPPER))
+ return OPTIMIZE (avx2);
+ }
+
+ return OPTIMIZE (sse2);
+}
diff --git a/sysdeps/x86_64/multiarch/ifunc-impl-list.c b/sysdeps/x86_64/multiarch/ifunc-impl-list.c
index d59d65f8..ac097e8d 100644
--- a/sysdeps/x86_64/multiarch/ifunc-impl-list.c
+++ b/sysdeps/x86_64/multiarch/ifunc-impl-list.c
@@ -52,6 +52,11 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,
&& CPU_FEATURE_USABLE (AVX512BW)
&& CPU_FEATURE_USABLE (BMI2)),
__memchr_evex)
+ IFUNC_IMPL_ADD (array, i, memchr,
+ (CPU_FEATURE_USABLE (AVX512VL)
+ && CPU_FEATURE_USABLE (AVX512BW)
+ && CPU_FEATURE_USABLE (BMI2)),
+ __memchr_evex_rtm)
IFUNC_IMPL_ADD (array, i, memchr, 1, __memchr_sse2))
/* Support sysdeps/x86_64/multiarch/memcmp.c. */
@@ -288,6 +293,11 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,
&& CPU_FEATURE_USABLE (AVX512BW)
&& CPU_FEATURE_USABLE (BMI2)),
__rawmemchr_evex)
+ IFUNC_IMPL_ADD (array, i, rawmemchr,
+ (CPU_FEATURE_USABLE (AVX512VL)
+ && CPU_FEATURE_USABLE (AVX512BW)
+ && CPU_FEATURE_USABLE (BMI2)),
+ __rawmemchr_evex_rtm)
IFUNC_IMPL_ADD (array, i, rawmemchr, 1, __rawmemchr_sse2))
/* Support sysdeps/x86_64/multiarch/strlen.c. */
@@ -711,6 +721,11 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,
&& CPU_FEATURE_USABLE (AVX512BW)
&& CPU_FEATURE_USABLE (BMI2)),
__wmemchr_evex)
+ IFUNC_IMPL_ADD (array, i, wmemchr,
+ (CPU_FEATURE_USABLE (AVX512VL)
+ && CPU_FEATURE_USABLE (AVX512BW)
+ && CPU_FEATURE_USABLE (BMI2)),
+ __wmemchr_evex_rtm)
IFUNC_IMPL_ADD (array, i, wmemchr, 1, __wmemchr_sse2))
/* Support sysdeps/x86_64/multiarch/wmemcmp.c. */
diff --git a/sysdeps/x86_64/multiarch/memchr-evex-rtm.S b/sysdeps/x86_64/multiarch/memchr-evex-rtm.S
new file mode 100644
index 00000000..19871882
--- /dev/null
+++ b/sysdeps/x86_64/multiarch/memchr-evex-rtm.S
@@ -0,0 +1,8 @@
+#ifndef MEMCHR
+# define MEMCHR __memchr_evex_rtm
+#endif
+
+#define USE_IN_RTM 1
+#define SECTION(p) p##.evex.rtm
+
+#include "memchr-evex.S"
diff --git a/sysdeps/x86_64/multiarch/memchr-evex.S b/sysdeps/x86_64/multiarch/memchr-evex.S
index f3fdad4f..4d0ed6d1 100644
--- a/sysdeps/x86_64/multiarch/memchr-evex.S
+++ b/sysdeps/x86_64/multiarch/memchr-evex.S
@@ -38,10 +38,32 @@
# define CHAR_SIZE 1
# endif
+ /* In the 4x loop the RTM and non-RTM versions have data pointer
+ off by VEC_SIZE * 4 with RTM version being VEC_SIZE * 4 greater.
+ This is represented by BASE_OFFSET. As well because the RTM
+ version uses vpcmp which stores a bit per element compared where
+ the non-RTM version uses vpcmpeq which stores a bit per byte
+ compared RET_SCALE of CHAR_SIZE is only relevant for the RTM
+ version. */
+# ifdef USE_IN_RTM
+# define VZEROUPPER
+# define BASE_OFFSET (VEC_SIZE * 4)
+# define RET_SCALE CHAR_SIZE
+# else
+# define VZEROUPPER vzeroupper
+# define BASE_OFFSET 0
+# define RET_SCALE 1
+# endif
+
+ /* In the return from 4x loop memchr and rawmemchr versions have
+ data pointers off by VEC_SIZE * 4 with memchr version being
+ VEC_SIZE * 4 greater. */
# ifdef USE_AS_RAWMEMCHR
+# define RET_OFFSET (BASE_OFFSET - (VEC_SIZE * 4))
# define RAW_PTR_REG rcx
# define ALGN_PTR_REG rdi
# else
+# define RET_OFFSET BASE_OFFSET
# define RAW_PTR_REG rdi
# define ALGN_PTR_REG rcx
# endif
@@ -57,11 +79,15 @@
# define YMM5 ymm21
# define YMM6 ymm22
+# ifndef SECTION
+# define SECTION(p) p##.evex
+# endif
+
# define VEC_SIZE 32
# define CHAR_PER_VEC (VEC_SIZE / CHAR_SIZE)
# define PAGE_SIZE 4096
- .section .text.evex,"ax",@progbits
+ .section SECTION(.text),"ax",@progbits
ENTRY (MEMCHR)
# ifndef USE_AS_RAWMEMCHR
/* Check for zero length. */
@@ -237,14 +263,15 @@ L(cross_page_continue):
/* Check if at last CHAR_PER_VEC * 4 length. */
subq $(CHAR_PER_VEC * 4), %rdx
jbe L(last_4x_vec_or_less_cmpeq)
- addq $VEC_SIZE, %rdi
+ /* +VEC_SIZE if USE_IN_RTM otherwise +VEC_SIZE * 5. */
+ addq $(VEC_SIZE + (VEC_SIZE * 4 - BASE_OFFSET)), %rdi
/* Align data to VEC_SIZE * 4 for the loop and readjust length.
*/
# ifdef USE_AS_WMEMCHR
movl %edi, %ecx
andq $-(4 * VEC_SIZE), %rdi
- andl $(VEC_SIZE * 4 - 1), %ecx
+ subl %edi, %ecx
/* NB: Divide bytes by 4 to get the wchar_t count. */
sarl $2, %ecx
addq %rcx, %rdx
@@ -254,15 +281,28 @@ L(cross_page_continue):
subq %rdi, %rdx
# endif
# else
- addq $VEC_SIZE, %rdi
+ addq $(VEC_SIZE + (VEC_SIZE * 4 - BASE_OFFSET)), %rdi
andq $-(4 * VEC_SIZE), %rdi
# endif
-
+# ifdef USE_IN_RTM
vpxorq %XMMZERO, %XMMZERO, %XMMZERO
+# else
+ /* copy ymmmatch to ymm0 so we can use vpcmpeq which is not
+ encodable with EVEX registers (ymm16-ymm31). */
+ vmovdqa64 %YMMMATCH, %ymm0
+# endif
/* Compare 4 * VEC at a time forward. */
.p2align 4
L(loop_4x_vec):
+ /* Two versions of the loop. One that does not require
+ vzeroupper by not using ymm0-ymm15 and another does that require
+ vzeroupper because it uses ymm0-ymm15. The reason why ymm0-ymm15
+ is used at all is because there is no EVEX encoding vpcmpeq and
+ with vpcmpeq this loop can be performed more efficiently. The
+ non-vzeroupper version is safe for RTM while the vzeroupper
+ version should be prefered if RTM are not supported. */
+# ifdef USE_IN_RTM
/* It would be possible to save some instructions using 4x VPCMP
but bottleneck on port 5 makes it not woth it. */
VPCMP $4, (VEC_SIZE * 4)(%rdi), %YMMMATCH, %k1
@@ -273,12 +313,55 @@ L(loop_4x_vec):
/* Reduce VEC2 / VEC3 with min and VEC1 with zero mask. */
VPMINU %YMM2, %YMM3, %YMM3{%k1}{z}
VPCMP $0, %YMM3, %YMMZERO, %k2
+# else
+ /* Since vptern can only take 3x vectors fastest to do 1 vec
+ seperately with EVEX vpcmp. */
+# ifdef USE_AS_WMEMCHR
+ /* vptern can only accept masks for epi32/epi64 so can only save
+ instruction using not equals mask on vptern with wmemchr. */
+ VPCMP $4, (%rdi), %YMMMATCH, %k1
+# else
+ VPCMP $0, (%rdi), %YMMMATCH, %k1
+# endif
+ /* Compare 3x with vpcmpeq and or them all together with vptern.
+ */
+ VPCMPEQ VEC_SIZE(%rdi), %ymm0, %ymm2
+ VPCMPEQ (VEC_SIZE * 2)(%rdi), %ymm0, %ymm3
+ VPCMPEQ (VEC_SIZE * 3)(%rdi), %ymm0, %ymm4
+# ifdef USE_AS_WMEMCHR
+ /* This takes the not of or between ymm2, ymm3, ymm4 as well as
+ combines result from VEC0 with zero mask. */
+ vpternlogd $1, %ymm2, %ymm3, %ymm4{%k1}{z}
+ vpmovmskb %ymm4, %ecx
+# else
+ /* 254 is mask for oring ymm2, ymm3, ymm4 into ymm4. */
+ vpternlogd $254, %ymm2, %ymm3, %ymm4
+ vpmovmskb %ymm4, %ecx
+ kmovd %k1, %eax
+# endif
+# endif
+
# ifdef USE_AS_RAWMEMCHR
subq $-(VEC_SIZE * 4), %rdi
+# endif
+# ifdef USE_IN_RTM
kortestd %k2, %k3
+# else
+# ifdef USE_AS_WMEMCHR
+ /* ecx contains not of matches. All 1s means no matches. incl will
+ overflow and set zeroflag if that is the case. */
+ incl %ecx
+# else
+ /* If either VEC1 (eax) or VEC2-VEC4 (ecx) are not zero. Adding
+ to ecx is not an issue because if eax is non-zero it will be
+ used for returning the match. If it is zero the add does
+ nothing. */
+ addq %rax, %rcx
+# endif
+# endif
+# ifdef USE_AS_RAWMEMCHR
jz L(loop_4x_vec)
# else
- kortestd %k2, %k3
jnz L(loop_4x_vec_end)
subq $-(VEC_SIZE * 4), %rdi
@@ -288,10 +371,11 @@ L(loop_4x_vec):
/* Fall through into less than 4 remaining vectors of length case.
*/
- VPCMP $0, (VEC_SIZE * 4)(%rdi), %YMMMATCH, %k0
+ VPCMP $0, BASE_OFFSET(%rdi), %YMMMATCH, %k0
+ addq $(BASE_OFFSET - VEC_SIZE), %rdi
kmovd %k0, %eax
- addq $(VEC_SIZE * 3), %rdi
- .p2align 4
+ VZEROUPPER
+
L(last_4x_vec_or_less):
/* Check if first VEC contained match. */
testl %eax, %eax
@@ -338,73 +422,78 @@ L(loop_4x_vec_end):
/* rawmemchr will fall through into this if match was found in
loop. */
+# if defined USE_IN_RTM || defined USE_AS_WMEMCHR
/* k1 has not of matches with VEC1. */
kmovd %k1, %eax
-# ifdef USE_AS_WMEMCHR
+# ifdef USE_AS_WMEMCHR
subl $((1 << CHAR_PER_VEC) - 1), %eax
-# else
+# else
incl %eax
+# endif
+# else
+ /* eax already has matches for VEC1. */
+ testl %eax, %eax
# endif
jnz L(last_vec_x1_return)
+# ifdef USE_IN_RTM
VPCMP $0, %YMM2, %YMMZERO, %k0
kmovd %k0, %eax
+# else
+ vpmovmskb %ymm2, %eax
+# endif
testl %eax, %eax
jnz L(last_vec_x2_return)
+# ifdef USE_IN_RTM
kmovd %k2, %eax
testl %eax, %eax
jnz L(last_vec_x3_return)
kmovd %k3, %eax
tzcntl %eax, %eax
-# ifdef USE_AS_RAWMEMCHR
- leaq (VEC_SIZE * 3)(%rdi, %rax, CHAR_SIZE), %rax
+ leaq (VEC_SIZE * 3 + RET_OFFSET)(%rdi, %rax, CHAR_SIZE), %rax
# else
- leaq (VEC_SIZE * 7)(%rdi, %rax, CHAR_SIZE), %rax
+ vpmovmskb %ymm3, %eax
+ /* Combine matches in VEC3 (eax) with matches in VEC4 (ecx). */
+ salq $VEC_SIZE, %rcx
+ orq %rcx, %rax
+ tzcntq %rax, %rax
+ leaq (VEC_SIZE * 2 + RET_OFFSET)(%rdi, %rax), %rax
+ VZEROUPPER
# endif
ret
.p2align 4
L(last_vec_x1_return):
tzcntl %eax, %eax
-# ifdef USE_AS_RAWMEMCHR
-# ifdef USE_AS_WMEMCHR
+# if defined USE_AS_WMEMCHR || RET_OFFSET != 0
/* NB: Multiply bytes by CHAR_SIZE to get the wchar_t count. */
- leaq (%rdi, %rax, CHAR_SIZE), %rax
-# else
- addq %rdi, %rax
-# endif
+ leaq RET_OFFSET(%rdi, %rax, CHAR_SIZE), %rax
# else
- /* NB: Multiply bytes by CHAR_SIZE to get the wchar_t count. */
- leaq (VEC_SIZE * 4)(%rdi, %rax, CHAR_SIZE), %rax
+ addq %rdi, %rax
# endif
+ VZEROUPPER
ret
.p2align 4
L(last_vec_x2_return):
tzcntl %eax, %eax
-# ifdef USE_AS_RAWMEMCHR
- /* NB: Multiply bytes by CHAR_SIZE to get the wchar_t count. */
- leaq VEC_SIZE(%rdi, %rax, CHAR_SIZE), %rax
-# else
- /* NB: Multiply bytes by CHAR_SIZE to get the wchar_t count. */
- leaq (VEC_SIZE * 5)(%rdi, %rax, CHAR_SIZE), %rax
-# endif
+ /* NB: Multiply bytes by RET_SCALE to get the wchar_t count
+ if relevant (RET_SCALE = CHAR_SIZE if USE_AS_WMEMCHAR and
+ USE_IN_RTM are both defined. Otherwise RET_SCALE = 1. */
+ leaq (VEC_SIZE + RET_OFFSET)(%rdi, %rax, RET_SCALE), %rax
+ VZEROUPPER
ret
+# ifdef USE_IN_RTM
.p2align 4
L(last_vec_x3_return):
tzcntl %eax, %eax
-# ifdef USE_AS_RAWMEMCHR
- /* NB: Multiply bytes by CHAR_SIZE to get the wchar_t count. */
- leaq (VEC_SIZE * 2)(%rdi, %rax, CHAR_SIZE), %rax
-# else
/* NB: Multiply bytes by CHAR_SIZE to get the wchar_t count. */
- leaq (VEC_SIZE * 6)(%rdi, %rax, CHAR_SIZE), %rax
-# endif
+ leaq (VEC_SIZE * 2 + RET_OFFSET)(%rdi, %rax, CHAR_SIZE), %rax
ret
-
+# endif
# ifndef USE_AS_RAWMEMCHR
L(last_4x_vec_or_less_cmpeq):
diff --git a/sysdeps/x86_64/multiarch/memchr.c b/sysdeps/x86_64/multiarch/memchr.c
index 016f5784..f28aea77 100644
--- a/sysdeps/x86_64/multiarch/memchr.c
+++ b/sysdeps/x86_64/multiarch/memchr.c
@@ -24,7 +24,7 @@
# undef memchr
# define SYMBOL_NAME memchr
-# include "ifunc-avx2.h"
+# include "ifunc-evex.h"
libc_ifunc_redirected (__redirect_memchr, memchr, IFUNC_SELECTOR ());
strong_alias (memchr, __memchr)
diff --git a/sysdeps/x86_64/multiarch/rawmemchr-evex-rtm.S b/sysdeps/x86_64/multiarch/rawmemchr-evex-rtm.S
new file mode 100644
index 00000000..deda1ca3
--- /dev/null
+++ b/sysdeps/x86_64/multiarch/rawmemchr-evex-rtm.S
@@ -0,0 +1,3 @@
+#define MEMCHR __rawmemchr_evex_rtm
+#define USE_AS_RAWMEMCHR 1
+#include "memchr-evex-rtm.S"
diff --git a/sysdeps/x86_64/multiarch/rawmemchr.c b/sysdeps/x86_64/multiarch/rawmemchr.c
index 8a0bc313..1f764f35 100644
--- a/sysdeps/x86_64/multiarch/rawmemchr.c
+++ b/sysdeps/x86_64/multiarch/rawmemchr.c
@@ -26,7 +26,7 @@
# undef __rawmemchr
# define SYMBOL_NAME rawmemchr
-# include "ifunc-avx2.h"
+# include "ifunc-evex.h"
libc_ifunc_redirected (__redirect_rawmemchr, __rawmemchr,
IFUNC_SELECTOR ());
diff --git a/sysdeps/x86_64/multiarch/wmemchr-evex-rtm.S b/sysdeps/x86_64/multiarch/wmemchr-evex-rtm.S
new file mode 100644
index 00000000..a346cd35
--- /dev/null
+++ b/sysdeps/x86_64/multiarch/wmemchr-evex-rtm.S
@@ -0,0 +1,3 @@
+#define MEMCHR __wmemchr_evex_rtm
+#define USE_AS_WMEMCHR 1
+#include "memchr-evex-rtm.S"
diff --git a/sysdeps/x86_64/multiarch/wmemchr.c b/sysdeps/x86_64/multiarch/wmemchr.c
index 6d833702..f9c91915 100644
--- a/sysdeps/x86_64/multiarch/wmemchr.c
+++ b/sysdeps/x86_64/multiarch/wmemchr.c
@@ -26,7 +26,7 @@
# undef __wmemchr
# define SYMBOL_NAME wmemchr
-# include "ifunc-avx2.h"
+# include "ifunc-evex.h"
libc_ifunc_redirected (__redirect_wmemchr, __wmemchr, IFUNC_SELECTOR ());
weak_alias (__wmemchr, wmemchr)
--
GitLab

View File

@ -1,873 +0,0 @@
From 16d12015c57701b08d7bbed6ec536641bcafb428 Mon Sep 17 00:00:00 2001
From: Noah Goldstein <goldstein.w.n@gmail.com>
Date: Mon, 17 May 2021 13:56:52 -0400
Subject: [PATCH] x86: Optimize memcmp-avx2-movbe.S
Content-type: text/plain; charset=UTF-8
No bug. This commit optimizes memcmp-avx2.S. The optimizations include
adding a new vec compare path for small sizes, reorganizing the entry
control flow, and removing some unnecissary ALU instructions from the
main loop. test-memcmp and test-wmemcmp are both passing.
Signed-off-by: Noah Goldstein <goldstein.w.n@gmail.com>
Reviewed-by: H.J. Lu <hjl.tools@gmail.com>
---
sysdeps/x86_64/multiarch/ifunc-impl-list.c | 6 +
sysdeps/x86_64/multiarch/ifunc-memcmp.h | 1 +
sysdeps/x86_64/multiarch/memcmp-avx2-movbe.S | 676 +++++++++++--------
3 files changed, 402 insertions(+), 281 deletions(-)
diff --git a/sysdeps/x86_64/multiarch/ifunc-impl-list.c b/sysdeps/x86_64/multiarch/ifunc-impl-list.c
index ac097e8d..8be0d78a 100644
--- a/sysdeps/x86_64/multiarch/ifunc-impl-list.c
+++ b/sysdeps/x86_64/multiarch/ifunc-impl-list.c
@@ -63,16 +63,19 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,
IFUNC_IMPL (i, name, memcmp,
IFUNC_IMPL_ADD (array, i, memcmp,
(CPU_FEATURE_USABLE (AVX2)
+ && CPU_FEATURE_USABLE (BMI2)
&& CPU_FEATURE_USABLE (MOVBE)),
__memcmp_avx2_movbe)
IFUNC_IMPL_ADD (array, i, memcmp,
(CPU_FEATURE_USABLE (AVX2)
+ && CPU_FEATURE_USABLE (BMI2)
&& CPU_FEATURE_USABLE (MOVBE)
&& CPU_FEATURE_USABLE (RTM)),
__memcmp_avx2_movbe_rtm)
IFUNC_IMPL_ADD (array, i, memcmp,
(CPU_FEATURE_USABLE (AVX512VL)
&& CPU_FEATURE_USABLE (AVX512BW)
+ && CPU_FEATURE_USABLE (BMI2)
&& CPU_FEATURE_USABLE (MOVBE)),
__memcmp_evex_movbe)
IFUNC_IMPL_ADD (array, i, memcmp, CPU_FEATURE_USABLE (SSE4_1),
@@ -732,16 +735,19 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,
IFUNC_IMPL (i, name, wmemcmp,
IFUNC_IMPL_ADD (array, i, wmemcmp,
(CPU_FEATURE_USABLE (AVX2)
+ && CPU_FEATURE_USABLE (BMI2)
&& CPU_FEATURE_USABLE (MOVBE)),
__wmemcmp_avx2_movbe)
IFUNC_IMPL_ADD (array, i, wmemcmp,
(CPU_FEATURE_USABLE (AVX2)
+ && CPU_FEATURE_USABLE (BMI2)
&& CPU_FEATURE_USABLE (MOVBE)
&& CPU_FEATURE_USABLE (RTM)),
__wmemcmp_avx2_movbe_rtm)
IFUNC_IMPL_ADD (array, i, wmemcmp,
(CPU_FEATURE_USABLE (AVX512VL)
&& CPU_FEATURE_USABLE (AVX512BW)
+ && CPU_FEATURE_USABLE (BMI2)
&& CPU_FEATURE_USABLE (MOVBE)),
__wmemcmp_evex_movbe)
IFUNC_IMPL_ADD (array, i, wmemcmp, CPU_FEATURE_USABLE (SSE4_1),
diff --git a/sysdeps/x86_64/multiarch/ifunc-memcmp.h b/sysdeps/x86_64/multiarch/ifunc-memcmp.h
index 8043c635..690dffe8 100644
--- a/sysdeps/x86_64/multiarch/ifunc-memcmp.h
+++ b/sysdeps/x86_64/multiarch/ifunc-memcmp.h
@@ -33,6 +33,7 @@ IFUNC_SELECTOR (void)
if (CPU_FEATURE_USABLE_P (cpu_features, AVX2)
&& CPU_FEATURE_USABLE_P (cpu_features, MOVBE)
+ && CPU_FEATURE_USABLE_P (cpu_features, BMI2)
&& CPU_FEATURES_ARCH_P (cpu_features, AVX_Fast_Unaligned_Load))
{
if (CPU_FEATURE_USABLE_P (cpu_features, AVX512VL)
diff --git a/sysdeps/x86_64/multiarch/memcmp-avx2-movbe.S b/sysdeps/x86_64/multiarch/memcmp-avx2-movbe.S
index 9d5c9c72..16fc673e 100644
--- a/sysdeps/x86_64/multiarch/memcmp-avx2-movbe.S
+++ b/sysdeps/x86_64/multiarch/memcmp-avx2-movbe.S
@@ -19,17 +19,23 @@
#if IS_IN (libc)
/* memcmp/wmemcmp is implemented as:
- 1. For size from 2 to 7 bytes, load as big endian with movbe and bswap
- to avoid branches.
- 2. Use overlapping compare to avoid branch.
- 3. Use vector compare when size >= 4 bytes for memcmp or size >= 8
- bytes for wmemcmp.
- 4. If size is 8 * VEC_SIZE or less, unroll the loop.
- 5. Compare 4 * VEC_SIZE at a time with the aligned first memory
+ 1. Use ymm vector compares when possible. The only case where
+ vector compares is not possible for when size < VEC_SIZE
+ and loading from either s1 or s2 would cause a page cross.
+ 2. For size from 2 to 7 bytes on page cross, load as big endian
+ with movbe and bswap to avoid branches.
+ 3. Use xmm vector compare when size >= 4 bytes for memcmp or
+ size >= 8 bytes for wmemcmp.
+ 4. Optimistically compare up to first 4 * VEC_SIZE one at a
+ to check for early mismatches. Only do this if its guranteed the
+ work is not wasted.
+ 5. If size is 8 * VEC_SIZE or less, unroll the loop.
+ 6. Compare 4 * VEC_SIZE at a time with the aligned first memory
area.
- 6. Use 2 vector compares when size is 2 * VEC_SIZE or less.
- 7. Use 4 vector compares when size is 4 * VEC_SIZE or less.
- 8. Use 8 vector compares when size is 8 * VEC_SIZE or less. */
+ 7. Use 2 vector compares when size is 2 * VEC_SIZE or less.
+ 8. Use 4 vector compares when size is 4 * VEC_SIZE or less.
+ 9. Use 8 vector compares when size is 8 * VEC_SIZE or less. */
+
# include <sysdep.h>
@@ -38,8 +44,10 @@
# endif
# ifdef USE_AS_WMEMCMP
+# define CHAR_SIZE 4
# define VPCMPEQ vpcmpeqd
# else
+# define CHAR_SIZE 1
# define VPCMPEQ vpcmpeqb
# endif
@@ -52,7 +60,7 @@
# endif
# define VEC_SIZE 32
-# define VEC_MASK ((1 << VEC_SIZE) - 1)
+# define PAGE_SIZE 4096
/* Warning!
wmemcmp has to use SIGNED comparison for elements.
@@ -71,136 +79,359 @@ ENTRY (MEMCMP)
jb L(less_vec)
/* From VEC to 2 * VEC. No branch when size == VEC_SIZE. */
- vmovdqu (%rsi), %ymm2
- VPCMPEQ (%rdi), %ymm2, %ymm2
- vpmovmskb %ymm2, %eax
- subl $VEC_MASK, %eax
- jnz L(first_vec)
+ vmovdqu (%rsi), %ymm1
+ VPCMPEQ (%rdi), %ymm1, %ymm1
+ vpmovmskb %ymm1, %eax
+ /* NB: eax must be destination register if going to
+ L(return_vec_[0,2]). For L(return_vec_3 destination register
+ must be ecx. */
+ incl %eax
+ jnz L(return_vec_0)
cmpq $(VEC_SIZE * 2), %rdx
- jbe L(last_vec)
-
- VPCMPEQ %ymm0, %ymm0, %ymm0
- /* More than 2 * VEC. */
- cmpq $(VEC_SIZE * 8), %rdx
- ja L(more_8x_vec)
- cmpq $(VEC_SIZE * 4), %rdx
- jb L(last_4x_vec)
-
- /* From 4 * VEC to 8 * VEC, inclusively. */
- vmovdqu (%rsi), %ymm1
- VPCMPEQ (%rdi), %ymm1, %ymm1
+ jbe L(last_1x_vec)
+ /* Check second VEC no matter what. */
vmovdqu VEC_SIZE(%rsi), %ymm2
- VPCMPEQ VEC_SIZE(%rdi), %ymm2, %ymm2
+ VPCMPEQ VEC_SIZE(%rdi), %ymm2, %ymm2
+ vpmovmskb %ymm2, %eax
+ /* If all 4 VEC where equal eax will be all 1s so incl will
+ overflow and set zero flag. */
+ incl %eax
+ jnz L(return_vec_1)
- vmovdqu (VEC_SIZE * 2)(%rsi), %ymm3
- VPCMPEQ (VEC_SIZE * 2)(%rdi), %ymm3, %ymm3
+ /* Less than 4 * VEC. */
+ cmpq $(VEC_SIZE * 4), %rdx
+ jbe L(last_2x_vec)
+ /* Check third and fourth VEC no matter what. */
+ vmovdqu (VEC_SIZE * 2)(%rsi), %ymm3
+ VPCMPEQ (VEC_SIZE * 2)(%rdi), %ymm3, %ymm3
+ vpmovmskb %ymm3, %eax
+ incl %eax
+ jnz L(return_vec_2)
vmovdqu (VEC_SIZE * 3)(%rsi), %ymm4
- VPCMPEQ (VEC_SIZE * 3)(%rdi), %ymm4, %ymm4
+ VPCMPEQ (VEC_SIZE * 3)(%rdi), %ymm4, %ymm4
+ vpmovmskb %ymm4, %ecx
+ incl %ecx
+ jnz L(return_vec_3)
- vpand %ymm1, %ymm2, %ymm5
- vpand %ymm3, %ymm4, %ymm6
- vpand %ymm5, %ymm6, %ymm5
+ /* Go to 4x VEC loop. */
+ cmpq $(VEC_SIZE * 8), %rdx
+ ja L(more_8x_vec)
- vptest %ymm0, %ymm5
- jnc L(4x_vec_end)
+ /* Handle remainder of size = 4 * VEC + 1 to 8 * VEC without any
+ branches. */
+ /* Load first two VEC from s2 before adjusting addresses. */
+ vmovdqu -(VEC_SIZE * 4)(%rsi, %rdx), %ymm1
+ vmovdqu -(VEC_SIZE * 3)(%rsi, %rdx), %ymm2
leaq -(4 * VEC_SIZE)(%rdi, %rdx), %rdi
leaq -(4 * VEC_SIZE)(%rsi, %rdx), %rsi
- vmovdqu (%rsi), %ymm1
- VPCMPEQ (%rdi), %ymm1, %ymm1
- vmovdqu VEC_SIZE(%rsi), %ymm2
- VPCMPEQ VEC_SIZE(%rdi), %ymm2, %ymm2
- vpand %ymm2, %ymm1, %ymm5
+ /* Wait to load from s1 until addressed adjust due to
+ unlamination of microfusion with complex address mode. */
+ VPCMPEQ (%rdi), %ymm1, %ymm1
+ VPCMPEQ (VEC_SIZE)(%rdi), %ymm2, %ymm2
vmovdqu (VEC_SIZE * 2)(%rsi), %ymm3
- VPCMPEQ (VEC_SIZE * 2)(%rdi), %ymm3, %ymm3
- vpand %ymm3, %ymm5, %ymm5
-
+ VPCMPEQ (VEC_SIZE * 2)(%rdi), %ymm3, %ymm3
vmovdqu (VEC_SIZE * 3)(%rsi), %ymm4
- VPCMPEQ (VEC_SIZE * 3)(%rdi), %ymm4, %ymm4
- vpand %ymm4, %ymm5, %ymm5
+ VPCMPEQ (VEC_SIZE * 3)(%rdi), %ymm4, %ymm4
- vptest %ymm0, %ymm5
- jnc L(4x_vec_end)
- xorl %eax, %eax
+ /* Reduce VEC0 - VEC4. */
+ vpand %ymm1, %ymm2, %ymm5
+ vpand %ymm3, %ymm4, %ymm6
+ vpand %ymm5, %ymm6, %ymm7
+ vpmovmskb %ymm7, %ecx
+ incl %ecx
+ jnz L(return_vec_0_1_2_3)
+ /* NB: eax must be zero to reach here. */
+ VZEROUPPER_RETURN
+
+ .p2align 4
+L(return_vec_0):
+ tzcntl %eax, %eax
+# ifdef USE_AS_WMEMCMP
+ movl (%rdi, %rax), %ecx
+ xorl %edx, %edx
+ cmpl (%rsi, %rax), %ecx
+ /* NB: no partial register stall here because xorl zero idiom
+ above. */
+ setg %dl
+ leal -1(%rdx, %rdx), %eax
+# else
+ movzbl (%rsi, %rax), %ecx
+ movzbl (%rdi, %rax), %eax
+ subl %ecx, %eax
+# endif
L(return_vzeroupper):
ZERO_UPPER_VEC_REGISTERS_RETURN
.p2align 4
-L(last_2x_vec):
- /* From VEC to 2 * VEC. No branch when size == VEC_SIZE. */
- vmovdqu (%rsi), %ymm2
- VPCMPEQ (%rdi), %ymm2, %ymm2
- vpmovmskb %ymm2, %eax
- subl $VEC_MASK, %eax
- jnz L(first_vec)
+L(return_vec_1):
+ tzcntl %eax, %eax
+# ifdef USE_AS_WMEMCMP
+ movl VEC_SIZE(%rdi, %rax), %ecx
+ xorl %edx, %edx
+ cmpl VEC_SIZE(%rsi, %rax), %ecx
+ setg %dl
+ leal -1(%rdx, %rdx), %eax
+# else
+ movzbl VEC_SIZE(%rsi, %rax), %ecx
+ movzbl VEC_SIZE(%rdi, %rax), %eax
+ subl %ecx, %eax
+# endif
+ VZEROUPPER_RETURN
+
+ .p2align 4
+L(return_vec_2):
+ tzcntl %eax, %eax
+# ifdef USE_AS_WMEMCMP
+ movl (VEC_SIZE * 2)(%rdi, %rax), %ecx
+ xorl %edx, %edx
+ cmpl (VEC_SIZE * 2)(%rsi, %rax), %ecx
+ setg %dl
+ leal -1(%rdx, %rdx), %eax
+# else
+ movzbl (VEC_SIZE * 2)(%rsi, %rax), %ecx
+ movzbl (VEC_SIZE * 2)(%rdi, %rax), %eax
+ subl %ecx, %eax
+# endif
+ VZEROUPPER_RETURN
+
+ /* NB: p2align 5 here to ensure 4x loop is 32 byte aligned. */
+ .p2align 5
+L(8x_return_vec_0_1_2_3):
+ /* Returning from L(more_8x_vec) requires restoring rsi. */
+ addq %rdi, %rsi
+L(return_vec_0_1_2_3):
+ vpmovmskb %ymm1, %eax
+ incl %eax
+ jnz L(return_vec_0)
-L(last_vec):
- /* Use overlapping loads to avoid branches. */
- leaq -VEC_SIZE(%rdi, %rdx), %rdi
- leaq -VEC_SIZE(%rsi, %rdx), %rsi
- vmovdqu (%rsi), %ymm2
- VPCMPEQ (%rdi), %ymm2, %ymm2
vpmovmskb %ymm2, %eax
- subl $VEC_MASK, %eax
- jnz L(first_vec)
+ incl %eax
+ jnz L(return_vec_1)
+
+ vpmovmskb %ymm3, %eax
+ incl %eax
+ jnz L(return_vec_2)
+L(return_vec_3):
+ tzcntl %ecx, %ecx
+# ifdef USE_AS_WMEMCMP
+ movl (VEC_SIZE * 3)(%rdi, %rcx), %eax
+ xorl %edx, %edx
+ cmpl (VEC_SIZE * 3)(%rsi, %rcx), %eax
+ setg %dl
+ leal -1(%rdx, %rdx), %eax
+# else
+ movzbl (VEC_SIZE * 3)(%rdi, %rcx), %eax
+ movzbl (VEC_SIZE * 3)(%rsi, %rcx), %ecx
+ subl %ecx, %eax
+# endif
+ VZEROUPPER_RETURN
+
+ .p2align 4
+L(more_8x_vec):
+ /* Set end of s1 in rdx. */
+ leaq -(VEC_SIZE * 4)(%rdi, %rdx), %rdx
+ /* rsi stores s2 - s1. This allows loop to only update one
+ pointer. */
+ subq %rdi, %rsi
+ /* Align s1 pointer. */
+ andq $-VEC_SIZE, %rdi
+ /* Adjust because first 4x vec where check already. */
+ subq $-(VEC_SIZE * 4), %rdi
+ .p2align 4
+L(loop_4x_vec):
+ /* rsi has s2 - s1 so get correct address by adding s1 (in rdi).
+ */
+ vmovdqu (%rsi, %rdi), %ymm1
+ VPCMPEQ (%rdi), %ymm1, %ymm1
+
+ vmovdqu VEC_SIZE(%rsi, %rdi), %ymm2
+ VPCMPEQ VEC_SIZE(%rdi), %ymm2, %ymm2
+
+ vmovdqu (VEC_SIZE * 2)(%rsi, %rdi), %ymm3
+ VPCMPEQ (VEC_SIZE * 2)(%rdi), %ymm3, %ymm3
+
+ vmovdqu (VEC_SIZE * 3)(%rsi, %rdi), %ymm4
+ VPCMPEQ (VEC_SIZE * 3)(%rdi), %ymm4, %ymm4
+
+ vpand %ymm1, %ymm2, %ymm5
+ vpand %ymm3, %ymm4, %ymm6
+ vpand %ymm5, %ymm6, %ymm7
+ vpmovmskb %ymm7, %ecx
+ incl %ecx
+ jnz L(8x_return_vec_0_1_2_3)
+ subq $-(VEC_SIZE * 4), %rdi
+ /* Check if s1 pointer at end. */
+ cmpq %rdx, %rdi
+ jb L(loop_4x_vec)
+
+ subq %rdx, %rdi
+ /* rdi has 4 * VEC_SIZE - remaining length. */
+ cmpl $(VEC_SIZE * 3), %edi
+ jae L(8x_last_1x_vec)
+ /* Load regardless of branch. */
+ vmovdqu (VEC_SIZE * 2)(%rsi, %rdx), %ymm3
+ cmpl $(VEC_SIZE * 2), %edi
+ jae L(8x_last_2x_vec)
+
+ /* Check last 4 VEC. */
+ vmovdqu (%rsi, %rdx), %ymm1
+ VPCMPEQ (%rdx), %ymm1, %ymm1
+
+ vmovdqu VEC_SIZE(%rsi, %rdx), %ymm2
+ VPCMPEQ VEC_SIZE(%rdx), %ymm2, %ymm2
+
+ VPCMPEQ (VEC_SIZE * 2)(%rdx), %ymm3, %ymm3
+
+ vmovdqu (VEC_SIZE * 3)(%rsi, %rdx), %ymm4
+ VPCMPEQ (VEC_SIZE * 3)(%rdx), %ymm4, %ymm4
+
+ vpand %ymm1, %ymm2, %ymm5
+ vpand %ymm3, %ymm4, %ymm6
+ vpand %ymm5, %ymm6, %ymm7
+ vpmovmskb %ymm7, %ecx
+ /* Restore s1 pointer to rdi. */
+ movq %rdx, %rdi
+ incl %ecx
+ jnz L(8x_return_vec_0_1_2_3)
+ /* NB: eax must be zero to reach here. */
+ VZEROUPPER_RETURN
+
+ /* Only entry is from L(more_8x_vec). */
+ .p2align 4
+L(8x_last_2x_vec):
+ /* Check second to last VEC. rdx store end pointer of s1 and
+ ymm3 has already been loaded with second to last VEC from s2.
+ */
+ VPCMPEQ (VEC_SIZE * 2)(%rdx), %ymm3, %ymm3
+ vpmovmskb %ymm3, %eax
+ incl %eax
+ jnz L(8x_return_vec_2)
+ /* Check last VEC. */
+ .p2align 4
+L(8x_last_1x_vec):
+ vmovdqu (VEC_SIZE * 3)(%rsi, %rdx), %ymm4
+ VPCMPEQ (VEC_SIZE * 3)(%rdx), %ymm4, %ymm4
+ vpmovmskb %ymm4, %eax
+ incl %eax
+ jnz L(8x_return_vec_3)
VZEROUPPER_RETURN
.p2align 4
-L(first_vec):
- /* A byte or int32 is different within 16 or 32 bytes. */
- tzcntl %eax, %ecx
+L(last_2x_vec):
+ /* Check second to last VEC. */
+ vmovdqu -(VEC_SIZE * 2)(%rsi, %rdx), %ymm1
+ VPCMPEQ -(VEC_SIZE * 2)(%rdi, %rdx), %ymm1, %ymm1
+ vpmovmskb %ymm1, %eax
+ incl %eax
+ jnz L(return_vec_1_end)
+ /* Check last VEC. */
+L(last_1x_vec):
+ vmovdqu -(VEC_SIZE * 1)(%rsi, %rdx), %ymm1
+ VPCMPEQ -(VEC_SIZE * 1)(%rdi, %rdx), %ymm1, %ymm1
+ vpmovmskb %ymm1, %eax
+ incl %eax
+ jnz L(return_vec_0_end)
+ VZEROUPPER_RETURN
+
+ .p2align 4
+L(8x_return_vec_2):
+ subq $VEC_SIZE, %rdx
+L(8x_return_vec_3):
+ tzcntl %eax, %eax
+ addq %rdx, %rax
# ifdef USE_AS_WMEMCMP
- xorl %eax, %eax
- movl (%rdi, %rcx), %edx
- cmpl (%rsi, %rcx), %edx
-L(wmemcmp_return):
- setl %al
- negl %eax
- orl $1, %eax
+ movl (VEC_SIZE * 3)(%rax), %ecx
+ xorl %edx, %edx
+ cmpl (VEC_SIZE * 3)(%rsi, %rax), %ecx
+ setg %dl
+ leal -1(%rdx, %rdx), %eax
# else
- movzbl (%rdi, %rcx), %eax
- movzbl (%rsi, %rcx), %edx
- sub %edx, %eax
+ movzbl (VEC_SIZE * 3)(%rsi, %rax), %ecx
+ movzbl (VEC_SIZE * 3)(%rax), %eax
+ subl %ecx, %eax
# endif
VZEROUPPER_RETURN
-# ifdef USE_AS_WMEMCMP
.p2align 4
-L(4):
- xorl %eax, %eax
- movl (%rdi), %edx
- cmpl (%rsi), %edx
- jne L(wmemcmp_return)
- ret
+L(return_vec_1_end):
+ tzcntl %eax, %eax
+ addl %edx, %eax
+# ifdef USE_AS_WMEMCMP
+ movl -(VEC_SIZE * 2)(%rdi, %rax), %ecx
+ xorl %edx, %edx
+ cmpl -(VEC_SIZE * 2)(%rsi, %rax), %ecx
+ setg %dl
+ leal -1(%rdx, %rdx), %eax
# else
+ movzbl -(VEC_SIZE * 2)(%rsi, %rax), %ecx
+ movzbl -(VEC_SIZE * 2)(%rdi, %rax), %eax
+ subl %ecx, %eax
+# endif
+ VZEROUPPER_RETURN
+
.p2align 4
-L(between_4_7):
- /* Load as big endian with overlapping movbe to avoid branches. */
- movbe (%rdi), %eax
- movbe (%rsi), %ecx
- shlq $32, %rax
- shlq $32, %rcx
- movbe -4(%rdi, %rdx), %edi
- movbe -4(%rsi, %rdx), %esi
- orq %rdi, %rax
- orq %rsi, %rcx
- subq %rcx, %rax
- je L(exit)
- sbbl %eax, %eax
- orl $1, %eax
- ret
+L(return_vec_0_end):
+ tzcntl %eax, %eax
+ addl %edx, %eax
+# ifdef USE_AS_WMEMCMP
+ movl -VEC_SIZE(%rdi, %rax), %ecx
+ xorl %edx, %edx
+ cmpl -VEC_SIZE(%rsi, %rax), %ecx
+ setg %dl
+ leal -1(%rdx, %rdx), %eax
+# else
+ movzbl -VEC_SIZE(%rsi, %rax), %ecx
+ movzbl -VEC_SIZE(%rdi, %rax), %eax
+ subl %ecx, %eax
+# endif
+ VZEROUPPER_RETURN
.p2align 4
-L(exit):
- ret
+L(less_vec):
+ /* Check if one or less CHAR. This is necessary for size = 0 but
+ is also faster for size = CHAR_SIZE. */
+ cmpl $CHAR_SIZE, %edx
+ jbe L(one_or_less)
+
+ /* Check if loading one VEC from either s1 or s2 could cause a
+ page cross. This can have false positives but is by far the
+ fastest method. */
+ movl %edi, %eax
+ orl %esi, %eax
+ andl $(PAGE_SIZE - 1), %eax
+ cmpl $(PAGE_SIZE - VEC_SIZE), %eax
+ jg L(page_cross_less_vec)
+
+ /* No page cross possible. */
+ vmovdqu (%rsi), %ymm2
+ VPCMPEQ (%rdi), %ymm2, %ymm2
+ vpmovmskb %ymm2, %eax
+ incl %eax
+ /* Result will be zero if s1 and s2 match. Otherwise first set
+ bit will be first mismatch. */
+ bzhil %edx, %eax, %edx
+ jnz L(return_vec_0)
+ xorl %eax, %eax
+ VZEROUPPER_RETURN
.p2align 4
-L(between_2_3):
+L(page_cross_less_vec):
+ /* if USE_AS_WMEMCMP it can only be 0, 4, 8, 12, 16, 20, 24, 28
+ bytes. */
+ cmpl $16, %edx
+ jae L(between_16_31)
+# ifndef USE_AS_WMEMCMP
+ cmpl $8, %edx
+ jae L(between_8_15)
+ cmpl $4, %edx
+ jae L(between_4_7)
+
/* Load as big endian to avoid branches. */
movzwl (%rdi), %eax
movzwl (%rsi), %ecx
@@ -208,223 +439,106 @@ L(between_2_3):
shll $8, %ecx
bswap %eax
bswap %ecx
- movb -1(%rdi, %rdx), %al
- movb -1(%rsi, %rdx), %cl
+ movzbl -1(%rdi, %rdx), %edi
+ movzbl -1(%rsi, %rdx), %esi
+ orl %edi, %eax
+ orl %esi, %ecx
/* Subtraction is okay because the upper 8 bits are zero. */
subl %ecx, %eax
+ /* No ymm register was touched. */
ret
.p2align 4
-L(1):
- movzbl (%rdi), %eax
+L(one_or_less):
+ jb L(zero)
movzbl (%rsi), %ecx
+ movzbl (%rdi), %eax
subl %ecx, %eax
- ret
-# endif
-
- .p2align 4
-L(zero):
- xorl %eax, %eax
+ /* No ymm register was touched. */
ret
.p2align 4
-L(less_vec):
-# ifdef USE_AS_WMEMCMP
- /* It can only be 0, 4, 8, 12, 16, 20, 24, 28 bytes. */
- cmpb $4, %dl
- je L(4)
- jb L(zero)
-# else
- cmpb $1, %dl
- je L(1)
- jb L(zero)
- cmpb $4, %dl
- jb L(between_2_3)
- cmpb $8, %dl
- jb L(between_4_7)
+L(between_8_15):
# endif
- cmpb $16, %dl
- jae L(between_16_31)
- /* It is between 8 and 15 bytes. */
+ /* If USE_AS_WMEMCMP fall through into 8-15 byte case. */
vmovq (%rdi), %xmm1
vmovq (%rsi), %xmm2
- VPCMPEQ %xmm1, %xmm2, %xmm2
+ VPCMPEQ %xmm1, %xmm2, %xmm2
vpmovmskb %xmm2, %eax
- subl $0xffff, %eax
- jnz L(first_vec)
+ subl $0xffff, %eax
+ jnz L(return_vec_0)
/* Use overlapping loads to avoid branches. */
leaq -8(%rdi, %rdx), %rdi
leaq -8(%rsi, %rdx), %rsi
vmovq (%rdi), %xmm1
vmovq (%rsi), %xmm2
- VPCMPEQ %xmm1, %xmm2, %xmm2
+ VPCMPEQ %xmm1, %xmm2, %xmm2
vpmovmskb %xmm2, %eax
- subl $0xffff, %eax
- jnz L(first_vec)
+ subl $0xffff, %eax
+ jnz L(return_vec_0)
+ /* No ymm register was touched. */
+ ret
+
+ .p2align 4
+L(zero):
+ xorl %eax, %eax
ret
.p2align 4
L(between_16_31):
/* From 16 to 31 bytes. No branch when size == 16. */
vmovdqu (%rsi), %xmm2
- VPCMPEQ (%rdi), %xmm2, %xmm2
+ VPCMPEQ (%rdi), %xmm2, %xmm2
vpmovmskb %xmm2, %eax
- subl $0xffff, %eax
- jnz L(first_vec)
+ subl $0xffff, %eax
+ jnz L(return_vec_0)
/* Use overlapping loads to avoid branches. */
+
+ vmovdqu -16(%rsi, %rdx), %xmm2
leaq -16(%rdi, %rdx), %rdi
leaq -16(%rsi, %rdx), %rsi
- vmovdqu (%rsi), %xmm2
- VPCMPEQ (%rdi), %xmm2, %xmm2
+ VPCMPEQ (%rdi), %xmm2, %xmm2
vpmovmskb %xmm2, %eax
- subl $0xffff, %eax
- jnz L(first_vec)
+ subl $0xffff, %eax
+ jnz L(return_vec_0)
+ /* No ymm register was touched. */
ret
- .p2align 4
-L(more_8x_vec):
- /* More than 8 * VEC. Check the first VEC. */
- vmovdqu (%rsi), %ymm2
- VPCMPEQ (%rdi), %ymm2, %ymm2
- vpmovmskb %ymm2, %eax
- subl $VEC_MASK, %eax
- jnz L(first_vec)
-
- /* Align the first memory area for aligned loads in the loop.
- Compute how much the first memory area is misaligned. */
- movq %rdi, %rcx
- andl $(VEC_SIZE - 1), %ecx
- /* Get the negative of offset for alignment. */
- subq $VEC_SIZE, %rcx
- /* Adjust the second memory area. */
- subq %rcx, %rsi
- /* Adjust the first memory area which should be aligned now. */
- subq %rcx, %rdi
- /* Adjust length. */
- addq %rcx, %rdx
-
-L(loop_4x_vec):
- /* Compare 4 * VEC at a time forward. */
- vmovdqu (%rsi), %ymm1
- VPCMPEQ (%rdi), %ymm1, %ymm1
-
- vmovdqu VEC_SIZE(%rsi), %ymm2
- VPCMPEQ VEC_SIZE(%rdi), %ymm2, %ymm2
- vpand %ymm2, %ymm1, %ymm5
-
- vmovdqu (VEC_SIZE * 2)(%rsi), %ymm3
- VPCMPEQ (VEC_SIZE * 2)(%rdi), %ymm3, %ymm3
- vpand %ymm3, %ymm5, %ymm5
-
- vmovdqu (VEC_SIZE * 3)(%rsi), %ymm4
- VPCMPEQ (VEC_SIZE * 3)(%rdi), %ymm4, %ymm4
- vpand %ymm4, %ymm5, %ymm5
-
- vptest %ymm0, %ymm5
- jnc L(4x_vec_end)
-
- addq $(VEC_SIZE * 4), %rdi
- addq $(VEC_SIZE * 4), %rsi
-
- subq $(VEC_SIZE * 4), %rdx
- cmpq $(VEC_SIZE * 4), %rdx
- jae L(loop_4x_vec)
-
- /* Less than 4 * VEC. */
- cmpq $VEC_SIZE, %rdx
- jbe L(last_vec)
- cmpq $(VEC_SIZE * 2), %rdx
- jbe L(last_2x_vec)
-
-L(last_4x_vec):
- /* From 2 * VEC to 4 * VEC. */
- vmovdqu (%rsi), %ymm2
- VPCMPEQ (%rdi), %ymm2, %ymm2
- vpmovmskb %ymm2, %eax
- subl $VEC_MASK, %eax
- jnz L(first_vec)
-
- addq $VEC_SIZE, %rdi
- addq $VEC_SIZE, %rsi
- vmovdqu (%rsi), %ymm2
- VPCMPEQ (%rdi), %ymm2, %ymm2
- vpmovmskb %ymm2, %eax
- subl $VEC_MASK, %eax
- jnz L(first_vec)
-
- /* Use overlapping loads to avoid branches. */
- leaq -(3 * VEC_SIZE)(%rdi, %rdx), %rdi
- leaq -(3 * VEC_SIZE)(%rsi, %rdx), %rsi
- vmovdqu (%rsi), %ymm2
- VPCMPEQ (%rdi), %ymm2, %ymm2
- vpmovmskb %ymm2, %eax
- subl $VEC_MASK, %eax
- jnz L(first_vec)
-
- addq $VEC_SIZE, %rdi
- addq $VEC_SIZE, %rsi
- vmovdqu (%rsi), %ymm2
- VPCMPEQ (%rdi), %ymm2, %ymm2
- vpmovmskb %ymm2, %eax
- subl $VEC_MASK, %eax
- jnz L(first_vec)
- VZEROUPPER_RETURN
-
- .p2align 4
-L(4x_vec_end):
- vpmovmskb %ymm1, %eax
- subl $VEC_MASK, %eax
- jnz L(first_vec)
- vpmovmskb %ymm2, %eax
- subl $VEC_MASK, %eax
- jnz L(first_vec_x1)
- vpmovmskb %ymm3, %eax
- subl $VEC_MASK, %eax
- jnz L(first_vec_x2)
- vpmovmskb %ymm4, %eax
- subl $VEC_MASK, %eax
- tzcntl %eax, %ecx
# ifdef USE_AS_WMEMCMP
- xorl %eax, %eax
- movl (VEC_SIZE * 3)(%rdi, %rcx), %edx
- cmpl (VEC_SIZE * 3)(%rsi, %rcx), %edx
- jmp L(wmemcmp_return)
-# else
- movzbl (VEC_SIZE * 3)(%rdi, %rcx), %eax
- movzbl (VEC_SIZE * 3)(%rsi, %rcx), %edx
- sub %edx, %eax
-# endif
- VZEROUPPER_RETURN
-
.p2align 4
-L(first_vec_x1):
- tzcntl %eax, %ecx
-# ifdef USE_AS_WMEMCMP
- xorl %eax, %eax
- movl VEC_SIZE(%rdi, %rcx), %edx
- cmpl VEC_SIZE(%rsi, %rcx), %edx
- jmp L(wmemcmp_return)
+L(one_or_less):
+ jb L(zero)
+ movl (%rdi), %ecx
+ xorl %edx, %edx
+ cmpl (%rsi), %ecx
+ je L(zero)
+ setg %dl
+ leal -1(%rdx, %rdx), %eax
+ /* No ymm register was touched. */
+ ret
# else
- movzbl VEC_SIZE(%rdi, %rcx), %eax
- movzbl VEC_SIZE(%rsi, %rcx), %edx
- sub %edx, %eax
-# endif
- VZEROUPPER_RETURN
.p2align 4
-L(first_vec_x2):
- tzcntl %eax, %ecx
-# ifdef USE_AS_WMEMCMP
- xorl %eax, %eax
- movl (VEC_SIZE * 2)(%rdi, %rcx), %edx
- cmpl (VEC_SIZE * 2)(%rsi, %rcx), %edx
- jmp L(wmemcmp_return)
-# else
- movzbl (VEC_SIZE * 2)(%rdi, %rcx), %eax
- movzbl (VEC_SIZE * 2)(%rsi, %rcx), %edx
- sub %edx, %eax
+L(between_4_7):
+ /* Load as big endian with overlapping movbe to avoid branches.
+ */
+ movbe (%rdi), %eax
+ movbe (%rsi), %ecx
+ shlq $32, %rax
+ shlq $32, %rcx
+ movbe -4(%rdi, %rdx), %edi
+ movbe -4(%rsi, %rdx), %esi
+ orq %rdi, %rax
+ orq %rsi, %rcx
+ subq %rcx, %rax
+ jz L(zero_4_7)
+ sbbl %eax, %eax
+ orl $1, %eax
+L(zero_4_7):
+ /* No ymm register was touched. */
+ ret
# endif
- VZEROUPPER_RETURN
+
END (MEMCMP)
#endif
--
GitLab

View File

@ -1,851 +0,0 @@
From 4ad473e97acdc5f6d811755b67c09f2128a644ce Mon Sep 17 00:00:00 2001
From: Noah Goldstein <goldstein.w.n@gmail.com>
Date: Mon, 17 May 2021 13:57:24 -0400
Subject: [PATCH] x86: Optimize memcmp-evex-movbe.S
Content-type: text/plain; charset=UTF-8
No bug. This commit optimizes memcmp-evex.S. The optimizations include
adding a new vec compare path for small sizes, reorganizing the entry
control flow, removing some unnecissary ALU instructions from the main
loop, and most importantly replacing the heavy use of vpcmp + kand
logic with vpxor + vptern. test-memcmp and test-wmemcmp are both
passing.
Signed-off-by: Noah Goldstein <goldstein.w.n@gmail.com>
Reviewed-by: H.J. Lu <hjl.tools@gmail.com>
---
sysdeps/x86_64/multiarch/memcmp-evex-movbe.S | 710 +++++++++++--------
1 file changed, 408 insertions(+), 302 deletions(-)
diff --git a/sysdeps/x86_64/multiarch/memcmp-evex-movbe.S b/sysdeps/x86_64/multiarch/memcmp-evex-movbe.S
index 9c093972..654dc7ac 100644
--- a/sysdeps/x86_64/multiarch/memcmp-evex-movbe.S
+++ b/sysdeps/x86_64/multiarch/memcmp-evex-movbe.S
@@ -19,17 +19,22 @@
#if IS_IN (libc)
/* memcmp/wmemcmp is implemented as:
- 1. For size from 2 to 7 bytes, load as big endian with movbe and bswap
- to avoid branches.
- 2. Use overlapping compare to avoid branch.
- 3. Use vector compare when size >= 4 bytes for memcmp or size >= 8
- bytes for wmemcmp.
- 4. If size is 8 * VEC_SIZE or less, unroll the loop.
- 5. Compare 4 * VEC_SIZE at a time with the aligned first memory
+ 1. Use ymm vector compares when possible. The only case where
+ vector compares is not possible for when size < CHAR_PER_VEC
+ and loading from either s1 or s2 would cause a page cross.
+ 2. For size from 2 to 7 bytes on page cross, load as big endian
+ with movbe and bswap to avoid branches.
+ 3. Use xmm vector compare when size >= 4 bytes for memcmp or
+ size >= 8 bytes for wmemcmp.
+ 4. Optimistically compare up to first 4 * CHAR_PER_VEC one at a
+ to check for early mismatches. Only do this if its guranteed the
+ work is not wasted.
+ 5. If size is 8 * VEC_SIZE or less, unroll the loop.
+ 6. Compare 4 * VEC_SIZE at a time with the aligned first memory
area.
- 6. Use 2 vector compares when size is 2 * VEC_SIZE or less.
- 7. Use 4 vector compares when size is 4 * VEC_SIZE or less.
- 8. Use 8 vector compares when size is 8 * VEC_SIZE or less. */
+ 7. Use 2 vector compares when size is 2 * CHAR_PER_VEC or less.
+ 8. Use 4 vector compares when size is 4 * CHAR_PER_VEC or less.
+ 9. Use 8 vector compares when size is 8 * CHAR_PER_VEC or less. */
# include <sysdep.h>
@@ -40,11 +45,21 @@
# define VMOVU vmovdqu64
# ifdef USE_AS_WMEMCMP
-# define VPCMPEQ vpcmpeqd
+# define CHAR_SIZE 4
+# define VPCMP vpcmpd
# else
-# define VPCMPEQ vpcmpeqb
+# define CHAR_SIZE 1
+# define VPCMP vpcmpub
# endif
+# define VEC_SIZE 32
+# define PAGE_SIZE 4096
+# define CHAR_PER_VEC (VEC_SIZE / CHAR_SIZE)
+
+# define XMM0 xmm16
+# define XMM1 xmm17
+# define XMM2 xmm18
+# define YMM0 ymm16
# define XMM1 xmm17
# define XMM2 xmm18
# define YMM1 ymm17
@@ -54,15 +69,6 @@
# define YMM5 ymm21
# define YMM6 ymm22
-# define VEC_SIZE 32
-# ifdef USE_AS_WMEMCMP
-# define VEC_MASK 0xff
-# define XMM_MASK 0xf
-# else
-# define VEC_MASK 0xffffffff
-# define XMM_MASK 0xffff
-# endif
-
/* Warning!
wmemcmp has to use SIGNED comparison for elements.
memcmp has to use UNSIGNED comparison for elemnts.
@@ -70,145 +76,370 @@
.section .text.evex,"ax",@progbits
ENTRY (MEMCMP)
-# ifdef USE_AS_WMEMCMP
- shl $2, %RDX_LP
-# elif defined __ILP32__
+# ifdef __ILP32__
/* Clear the upper 32 bits. */
movl %edx, %edx
# endif
- cmp $VEC_SIZE, %RDX_LP
+ cmp $CHAR_PER_VEC, %RDX_LP
jb L(less_vec)
/* From VEC to 2 * VEC. No branch when size == VEC_SIZE. */
- VMOVU (%rsi), %YMM2
- VPCMPEQ (%rdi), %YMM2, %k1
+ VMOVU (%rsi), %YMM1
+ /* Use compare not equals to directly check for mismatch. */
+ VPCMP $4, (%rdi), %YMM1, %k1
kmovd %k1, %eax
- subl $VEC_MASK, %eax
- jnz L(first_vec)
-
- cmpq $(VEC_SIZE * 2), %rdx
- jbe L(last_vec)
-
- /* More than 2 * VEC. */
- cmpq $(VEC_SIZE * 8), %rdx
- ja L(more_8x_vec)
- cmpq $(VEC_SIZE * 4), %rdx
- jb L(last_4x_vec)
+ /* NB: eax must be destination register if going to
+ L(return_vec_[0,2]). For L(return_vec_3 destination register
+ must be ecx. */
+ testl %eax, %eax
+ jnz L(return_vec_0)
- /* From 4 * VEC to 8 * VEC, inclusively. */
- VMOVU (%rsi), %YMM1
- VPCMPEQ (%rdi), %YMM1, %k1
+ cmpq $(CHAR_PER_VEC * 2), %rdx
+ jbe L(last_1x_vec)
+ /* Check second VEC no matter what. */
VMOVU VEC_SIZE(%rsi), %YMM2
- VPCMPEQ VEC_SIZE(%rdi), %YMM2, %k2
+ VPCMP $4, VEC_SIZE(%rdi), %YMM2, %k1
+ kmovd %k1, %eax
+ testl %eax, %eax
+ jnz L(return_vec_1)
+
+ /* Less than 4 * VEC. */
+ cmpq $(CHAR_PER_VEC * 4), %rdx
+ jbe L(last_2x_vec)
+ /* Check third and fourth VEC no matter what. */
VMOVU (VEC_SIZE * 2)(%rsi), %YMM3
- VPCMPEQ (VEC_SIZE * 2)(%rdi), %YMM3, %k3
+ VPCMP $4, (VEC_SIZE * 2)(%rdi), %YMM3, %k1
+ kmovd %k1, %eax
+ testl %eax, %eax
+ jnz L(return_vec_2)
VMOVU (VEC_SIZE * 3)(%rsi), %YMM4
- VPCMPEQ (VEC_SIZE * 3)(%rdi), %YMM4, %k4
+ VPCMP $4, (VEC_SIZE * 3)(%rdi), %YMM4, %k1
+ kmovd %k1, %ecx
+ testl %ecx, %ecx
+ jnz L(return_vec_3)
- kandd %k1, %k2, %k5
- kandd %k3, %k4, %k6
- kandd %k5, %k6, %k6
+ /* Zero YMM0. 4x VEC reduction is done with vpxor + vtern so
+ compare with zero to get a mask is needed. */
+ vpxorq %XMM0, %XMM0, %XMM0
- kmovd %k6, %eax
- cmpl $VEC_MASK, %eax
- jne L(4x_vec_end)
+ /* Go to 4x VEC loop. */
+ cmpq $(CHAR_PER_VEC * 8), %rdx
+ ja L(more_8x_vec)
- leaq -(4 * VEC_SIZE)(%rdi, %rdx), %rdi
- leaq -(4 * VEC_SIZE)(%rsi, %rdx), %rsi
- VMOVU (%rsi), %YMM1
- VPCMPEQ (%rdi), %YMM1, %k1
+ /* Handle remainder of size = 4 * VEC + 1 to 8 * VEC without any
+ branches. */
- VMOVU VEC_SIZE(%rsi), %YMM2
- VPCMPEQ VEC_SIZE(%rdi), %YMM2, %k2
- kandd %k1, %k2, %k5
+ /* Load first two VEC from s2 before adjusting addresses. */
+ VMOVU -(VEC_SIZE * 4)(%rsi, %rdx, CHAR_SIZE), %YMM1
+ VMOVU -(VEC_SIZE * 3)(%rsi, %rdx, CHAR_SIZE), %YMM2
+ leaq -(4 * VEC_SIZE)(%rdi, %rdx, CHAR_SIZE), %rdi
+ leaq -(4 * VEC_SIZE)(%rsi, %rdx, CHAR_SIZE), %rsi
+
+ /* Wait to load from s1 until addressed adjust due to
+ unlamination of microfusion with complex address mode. */
+
+ /* vpxor will be all 0s if s1 and s2 are equal. Otherwise it
+ will have some 1s. */
+ vpxorq (%rdi), %YMM1, %YMM1
+ vpxorq (VEC_SIZE)(%rdi), %YMM2, %YMM2
VMOVU (VEC_SIZE * 2)(%rsi), %YMM3
- VPCMPEQ (VEC_SIZE * 2)(%rdi), %YMM3, %k3
- kandd %k3, %k5, %k5
+ vpxorq (VEC_SIZE * 2)(%rdi), %YMM3, %YMM3
+ /* Or together YMM1, YMM2, and YMM3 into YMM3. */
+ vpternlogd $0xfe, %YMM1, %YMM2, %YMM3
VMOVU (VEC_SIZE * 3)(%rsi), %YMM4
- VPCMPEQ (VEC_SIZE * 3)(%rdi), %YMM4, %k4
- kandd %k4, %k5, %k5
+ /* Ternary logic to xor (VEC_SIZE * 3)(%rdi) with YMM4 while
+ oring with YMM3. Result is stored in YMM4. */
+ vpternlogd $0xde, (VEC_SIZE * 3)(%rdi), %YMM3, %YMM4
+ /* Compare YMM4 with 0. If any 1s s1 and s2 don't match. */
+ VPCMP $4, %YMM4, %YMM0, %k1
+ kmovd %k1, %ecx
+ testl %ecx, %ecx
+ jnz L(return_vec_0_1_2_3)
+ /* NB: eax must be zero to reach here. */
+ ret
- kmovd %k5, %eax
- cmpl $VEC_MASK, %eax
- jne L(4x_vec_end)
- xorl %eax, %eax
+ /* NB: aligning 32 here allows for the rest of the jump targets
+ to be tuned for 32 byte alignment. Most important this ensures
+ the L(more_8x_vec) loop is 32 byte aligned. */
+ .p2align 5
+L(less_vec):
+ /* Check if one or less CHAR. This is necessary for size = 0 but
+ is also faster for size = CHAR_SIZE. */
+ cmpl $1, %edx
+ jbe L(one_or_less)
+
+ /* Check if loading one VEC from either s1 or s2 could cause a
+ page cross. This can have false positives but is by far the
+ fastest method. */
+ movl %edi, %eax
+ orl %esi, %eax
+ andl $(PAGE_SIZE - 1), %eax
+ cmpl $(PAGE_SIZE - VEC_SIZE), %eax
+ jg L(page_cross_less_vec)
+
+ /* No page cross possible. */
+ VMOVU (%rsi), %YMM2
+ VPCMP $4, (%rdi), %YMM2, %k1
+ kmovd %k1, %eax
+ /* Create mask in ecx for potentially in bound matches. */
+ bzhil %edx, %eax, %eax
+ jnz L(return_vec_0)
ret
.p2align 4
-L(last_2x_vec):
- /* From VEC to 2 * VEC. No branch when size == VEC_SIZE. */
- VMOVU (%rsi), %YMM2
- VPCMPEQ (%rdi), %YMM2, %k2
- kmovd %k2, %eax
- subl $VEC_MASK, %eax
- jnz L(first_vec)
+L(return_vec_0):
+ tzcntl %eax, %eax
+# ifdef USE_AS_WMEMCMP
+ movl (%rdi, %rax, CHAR_SIZE), %ecx
+ xorl %edx, %edx
+ cmpl (%rsi, %rax, CHAR_SIZE), %ecx
+ /* NB: no partial register stall here because xorl zero idiom
+ above. */
+ setg %dl
+ leal -1(%rdx, %rdx), %eax
+# else
+ movzbl (%rsi, %rax), %ecx
+ movzbl (%rdi, %rax), %eax
+ subl %ecx, %eax
+# endif
+ ret
-L(last_vec):
- /* Use overlapping loads to avoid branches. */
- leaq -VEC_SIZE(%rdi, %rdx), %rdi
- leaq -VEC_SIZE(%rsi, %rdx), %rsi
- VMOVU (%rsi), %YMM2
- VPCMPEQ (%rdi), %YMM2, %k2
- kmovd %k2, %eax
- subl $VEC_MASK, %eax
- jnz L(first_vec)
+ /* NB: No p2align necessary. Alignment % 16 is naturally 1
+ which is good enough for a target not in a loop. */
+L(return_vec_1):
+ tzcntl %eax, %eax
+# ifdef USE_AS_WMEMCMP
+ movl VEC_SIZE(%rdi, %rax, CHAR_SIZE), %ecx
+ xorl %edx, %edx
+ cmpl VEC_SIZE(%rsi, %rax, CHAR_SIZE), %ecx
+ setg %dl
+ leal -1(%rdx, %rdx), %eax
+# else
+ movzbl VEC_SIZE(%rsi, %rax), %ecx
+ movzbl VEC_SIZE(%rdi, %rax), %eax
+ subl %ecx, %eax
+# endif
ret
- .p2align 4
-L(first_vec):
- /* A byte or int32 is different within 16 or 32 bytes. */
- tzcntl %eax, %ecx
+ /* NB: No p2align necessary. Alignment % 16 is naturally 2
+ which is good enough for a target not in a loop. */
+L(return_vec_2):
+ tzcntl %eax, %eax
# ifdef USE_AS_WMEMCMP
- xorl %eax, %eax
- movl (%rdi, %rcx, 4), %edx
- cmpl (%rsi, %rcx, 4), %edx
-L(wmemcmp_return):
- setl %al
- negl %eax
- orl $1, %eax
+ movl (VEC_SIZE * 2)(%rdi, %rax, CHAR_SIZE), %ecx
+ xorl %edx, %edx
+ cmpl (VEC_SIZE * 2)(%rsi, %rax, CHAR_SIZE), %ecx
+ setg %dl
+ leal -1(%rdx, %rdx), %eax
# else
- movzbl (%rdi, %rcx), %eax
- movzbl (%rsi, %rcx), %edx
- sub %edx, %eax
+ movzbl (VEC_SIZE * 2)(%rsi, %rax), %ecx
+ movzbl (VEC_SIZE * 2)(%rdi, %rax), %eax
+ subl %ecx, %eax
# endif
ret
+ .p2align 4
+L(8x_return_vec_0_1_2_3):
+ /* Returning from L(more_8x_vec) requires restoring rsi. */
+ addq %rdi, %rsi
+L(return_vec_0_1_2_3):
+ VPCMP $4, %YMM1, %YMM0, %k0
+ kmovd %k0, %eax
+ testl %eax, %eax
+ jnz L(return_vec_0)
+
+ VPCMP $4, %YMM2, %YMM0, %k0
+ kmovd %k0, %eax
+ testl %eax, %eax
+ jnz L(return_vec_1)
+
+ VPCMP $4, %YMM3, %YMM0, %k0
+ kmovd %k0, %eax
+ testl %eax, %eax
+ jnz L(return_vec_2)
+L(return_vec_3):
+ tzcntl %ecx, %ecx
# ifdef USE_AS_WMEMCMP
+ movl (VEC_SIZE * 3)(%rdi, %rcx, CHAR_SIZE), %eax
+ xorl %edx, %edx
+ cmpl (VEC_SIZE * 3)(%rsi, %rcx, CHAR_SIZE), %eax
+ setg %dl
+ leal -1(%rdx, %rdx), %eax
+# else
+ movzbl (VEC_SIZE * 3)(%rdi, %rcx), %eax
+ movzbl (VEC_SIZE * 3)(%rsi, %rcx), %ecx
+ subl %ecx, %eax
+# endif
+ ret
+
.p2align 4
-L(4):
- xorl %eax, %eax
- movl (%rdi), %edx
- cmpl (%rsi), %edx
- jne L(wmemcmp_return)
+L(more_8x_vec):
+ /* Set end of s1 in rdx. */
+ leaq -(VEC_SIZE * 4)(%rdi, %rdx, CHAR_SIZE), %rdx
+ /* rsi stores s2 - s1. This allows loop to only update one
+ pointer. */
+ subq %rdi, %rsi
+ /* Align s1 pointer. */
+ andq $-VEC_SIZE, %rdi
+ /* Adjust because first 4x vec where check already. */
+ subq $-(VEC_SIZE * 4), %rdi
+ .p2align 4
+L(loop_4x_vec):
+ VMOVU (%rsi, %rdi), %YMM1
+ vpxorq (%rdi), %YMM1, %YMM1
+
+ VMOVU VEC_SIZE(%rsi, %rdi), %YMM2
+ vpxorq VEC_SIZE(%rdi), %YMM2, %YMM2
+
+ VMOVU (VEC_SIZE * 2)(%rsi, %rdi), %YMM3
+ vpxorq (VEC_SIZE * 2)(%rdi), %YMM3, %YMM3
+ vpternlogd $0xfe, %YMM1, %YMM2, %YMM3
+
+ VMOVU (VEC_SIZE * 3)(%rsi, %rdi), %YMM4
+ vpternlogd $0xde, (VEC_SIZE * 3)(%rdi), %YMM3, %YMM4
+ VPCMP $4, %YMM4, %YMM0, %k1
+ kmovd %k1, %ecx
+ testl %ecx, %ecx
+ jnz L(8x_return_vec_0_1_2_3)
+ subq $-(VEC_SIZE * 4), %rdi
+ cmpq %rdx, %rdi
+ jb L(loop_4x_vec)
+
+ subq %rdx, %rdi
+ /* rdi has 4 * VEC_SIZE - remaining length. */
+ cmpl $(VEC_SIZE * 3), %edi
+ jae L(8x_last_1x_vec)
+ /* Load regardless of branch. */
+ VMOVU (VEC_SIZE * 2)(%rsi, %rdx), %YMM3
+ cmpl $(VEC_SIZE * 2), %edi
+ jae L(8x_last_2x_vec)
+
+ VMOVU (%rsi, %rdx), %YMM1
+ vpxorq (%rdx), %YMM1, %YMM1
+
+ VMOVU VEC_SIZE(%rsi, %rdx), %YMM2
+ vpxorq VEC_SIZE(%rdx), %YMM2, %YMM2
+
+ vpxorq (VEC_SIZE * 2)(%rdx), %YMM3, %YMM3
+ vpternlogd $0xfe, %YMM1, %YMM2, %YMM3
+
+ VMOVU (VEC_SIZE * 3)(%rsi, %rdx), %YMM4
+ vpternlogd $0xde, (VEC_SIZE * 3)(%rdx), %YMM3, %YMM4
+ VPCMP $4, %YMM4, %YMM0, %k1
+ kmovd %k1, %ecx
+ /* Restore s1 pointer to rdi. */
+ movq %rdx, %rdi
+ testl %ecx, %ecx
+ jnz L(8x_return_vec_0_1_2_3)
+ /* NB: eax must be zero to reach here. */
+ ret
+
+ /* Only entry is from L(more_8x_vec). */
+ .p2align 4
+L(8x_last_2x_vec):
+ VPCMP $4, (VEC_SIZE * 2)(%rdx), %YMM3, %k1
+ kmovd %k1, %eax
+ testl %eax, %eax
+ jnz L(8x_return_vec_2)
+ /* Naturally aligned to 16 bytes. */
+L(8x_last_1x_vec):
+ VMOVU (VEC_SIZE * 3)(%rsi, %rdx), %YMM1
+ VPCMP $4, (VEC_SIZE * 3)(%rdx), %YMM1, %k1
+ kmovd %k1, %eax
+ testl %eax, %eax
+ jnz L(8x_return_vec_3)
+ ret
+
+ .p2align 4
+L(last_2x_vec):
+ /* Check second to last VEC. */
+ VMOVU -(VEC_SIZE * 2)(%rsi, %rdx, CHAR_SIZE), %YMM1
+ VPCMP $4, -(VEC_SIZE * 2)(%rdi, %rdx, CHAR_SIZE), %YMM1, %k1
+ kmovd %k1, %eax
+ testl %eax, %eax
+ jnz L(return_vec_1_end)
+
+ /* Check last VEC. */
+ .p2align 4
+L(last_1x_vec):
+ VMOVU -(VEC_SIZE * 1)(%rsi, %rdx, CHAR_SIZE), %YMM1
+ VPCMP $4, -(VEC_SIZE * 1)(%rdi, %rdx, CHAR_SIZE), %YMM1, %k1
+ kmovd %k1, %eax
+ testl %eax, %eax
+ jnz L(return_vec_0_end)
ret
+
+ .p2align 4
+L(8x_return_vec_2):
+ subq $VEC_SIZE, %rdx
+L(8x_return_vec_3):
+ tzcntl %eax, %eax
+# ifdef USE_AS_WMEMCMP
+ leaq (%rdx, %rax, CHAR_SIZE), %rax
+ movl (VEC_SIZE * 3)(%rax), %ecx
+ xorl %edx, %edx
+ cmpl (VEC_SIZE * 3)(%rsi, %rax), %ecx
+ setg %dl
+ leal -1(%rdx, %rdx), %eax
# else
+ addq %rdx, %rax
+ movzbl (VEC_SIZE * 3)(%rsi, %rax), %ecx
+ movzbl (VEC_SIZE * 3)(%rax), %eax
+ subl %ecx, %eax
+# endif
+ ret
+
.p2align 4
-L(between_4_7):
- /* Load as big endian with overlapping movbe to avoid branches. */
- movbe (%rdi), %eax
- movbe (%rsi), %ecx
- shlq $32, %rax
- shlq $32, %rcx
- movbe -4(%rdi, %rdx), %edi
- movbe -4(%rsi, %rdx), %esi
- orq %rdi, %rax
- orq %rsi, %rcx
- subq %rcx, %rax
- je L(exit)
- sbbl %eax, %eax
- orl $1, %eax
+L(return_vec_0_end):
+ tzcntl %eax, %eax
+ addl %edx, %eax
+# ifdef USE_AS_WMEMCMP
+ movl -VEC_SIZE(%rdi, %rax, CHAR_SIZE), %ecx
+ xorl %edx, %edx
+ cmpl -VEC_SIZE(%rsi, %rax, CHAR_SIZE), %ecx
+ setg %dl
+ leal -1(%rdx, %rdx), %eax
+# else
+ movzbl -VEC_SIZE(%rsi, %rax), %ecx
+ movzbl -VEC_SIZE(%rdi, %rax), %eax
+ subl %ecx, %eax
+# endif
ret
.p2align 4
-L(exit):
+L(return_vec_1_end):
+ tzcntl %eax, %eax
+ addl %edx, %eax
+# ifdef USE_AS_WMEMCMP
+ movl -(VEC_SIZE * 2)(%rdi, %rax, CHAR_SIZE), %ecx
+ xorl %edx, %edx
+ cmpl -(VEC_SIZE * 2)(%rsi, %rax, CHAR_SIZE), %ecx
+ setg %dl
+ leal -1(%rdx, %rdx), %eax
+# else
+ movzbl -(VEC_SIZE * 2)(%rsi, %rax), %ecx
+ movzbl -(VEC_SIZE * 2)(%rdi, %rax), %eax
+ subl %ecx, %eax
+# endif
ret
+
.p2align 4
+L(page_cross_less_vec):
+ /* if USE_AS_WMEMCMP it can only be 0, 4, 8, 12, 16, 20, 24, 28
+ bytes. */
+ cmpl $(16 / CHAR_SIZE), %edx
+ jae L(between_16_31)
+# ifndef USE_AS_WMEMCMP
+ cmpl $8, %edx
+ jae L(between_8_15)
+ cmpl $4, %edx
+ jae L(between_4_7)
L(between_2_3):
/* Load as big endian to avoid branches. */
movzwl (%rdi), %eax
@@ -217,224 +448,99 @@ L(between_2_3):
shll $8, %ecx
bswap %eax
bswap %ecx
- movb -1(%rdi, %rdx), %al
- movb -1(%rsi, %rdx), %cl
+ movzbl -1(%rdi, %rdx), %edi
+ movzbl -1(%rsi, %rdx), %esi
+ orl %edi, %eax
+ orl %esi, %ecx
/* Subtraction is okay because the upper 8 bits are zero. */
subl %ecx, %eax
ret
-
.p2align 4
-L(1):
- movzbl (%rdi), %eax
+L(one_or_less):
+ jb L(zero)
movzbl (%rsi), %ecx
+ movzbl (%rdi), %eax
subl %ecx, %eax
ret
-# endif
-
- .p2align 4
-L(zero):
- xorl %eax, %eax
- ret
.p2align 4
-L(less_vec):
-# ifdef USE_AS_WMEMCMP
- /* It can only be 0, 4, 8, 12, 16, 20, 24, 28 bytes. */
- cmpb $4, %dl
- je L(4)
- jb L(zero)
-# else
- cmpb $1, %dl
- je L(1)
- jb L(zero)
- cmpb $4, %dl
- jb L(between_2_3)
- cmpb $8, %dl
- jb L(between_4_7)
+L(between_8_15):
# endif
- cmpb $16, %dl
- jae L(between_16_31)
- /* It is between 8 and 15 bytes. */
+ /* If USE_AS_WMEMCMP fall through into 8-15 byte case. */
vmovq (%rdi), %XMM1
vmovq (%rsi), %XMM2
- VPCMPEQ %XMM1, %XMM2, %k2
- kmovw %k2, %eax
- subl $XMM_MASK, %eax
- jnz L(first_vec)
+ VPCMP $4, %XMM1, %XMM2, %k1
+ kmovd %k1, %eax
+ testl %eax, %eax
+ jnz L(return_vec_0)
/* Use overlapping loads to avoid branches. */
- leaq -8(%rdi, %rdx), %rdi
- leaq -8(%rsi, %rdx), %rsi
+ leaq -8(%rdi, %rdx, CHAR_SIZE), %rdi
+ leaq -8(%rsi, %rdx, CHAR_SIZE), %rsi
vmovq (%rdi), %XMM1
vmovq (%rsi), %XMM2
- VPCMPEQ %XMM1, %XMM2, %k2
- kmovw %k2, %eax
- subl $XMM_MASK, %eax
- jnz L(first_vec)
+ VPCMP $4, %XMM1, %XMM2, %k1
+ kmovd %k1, %eax
+ testl %eax, %eax
+ jnz L(return_vec_0)
ret
.p2align 4
-L(between_16_31):
- /* From 16 to 31 bytes. No branch when size == 16. */
- VMOVU (%rsi), %XMM2
- VPCMPEQ (%rdi), %XMM2, %k2
- kmovw %k2, %eax
- subl $XMM_MASK, %eax
- jnz L(first_vec)
-
- /* Use overlapping loads to avoid branches. */
- leaq -16(%rdi, %rdx), %rdi
- leaq -16(%rsi, %rdx), %rsi
- VMOVU (%rsi), %XMM2
- VPCMPEQ (%rdi), %XMM2, %k2
- kmovw %k2, %eax
- subl $XMM_MASK, %eax
- jnz L(first_vec)
+L(zero):
+ xorl %eax, %eax
ret
.p2align 4
-L(more_8x_vec):
- /* More than 8 * VEC. Check the first VEC. */
- VMOVU (%rsi), %YMM2
- VPCMPEQ (%rdi), %YMM2, %k2
- kmovd %k2, %eax
- subl $VEC_MASK, %eax
- jnz L(first_vec)
-
- /* Align the first memory area for aligned loads in the loop.
- Compute how much the first memory area is misaligned. */
- movq %rdi, %rcx
- andl $(VEC_SIZE - 1), %ecx
- /* Get the negative of offset for alignment. */
- subq $VEC_SIZE, %rcx
- /* Adjust the second memory area. */
- subq %rcx, %rsi
- /* Adjust the first memory area which should be aligned now. */
- subq %rcx, %rdi
- /* Adjust length. */
- addq %rcx, %rdx
-
-L(loop_4x_vec):
- /* Compare 4 * VEC at a time forward. */
- VMOVU (%rsi), %YMM1
- VPCMPEQ (%rdi), %YMM1, %k1
-
- VMOVU VEC_SIZE(%rsi), %YMM2
- VPCMPEQ VEC_SIZE(%rdi), %YMM2, %k2
- kandd %k2, %k1, %k5
-
- VMOVU (VEC_SIZE * 2)(%rsi), %YMM3
- VPCMPEQ (VEC_SIZE * 2)(%rdi), %YMM3, %k3
- kandd %k3, %k5, %k5
-
- VMOVU (VEC_SIZE * 3)(%rsi), %YMM4
- VPCMPEQ (VEC_SIZE * 3)(%rdi), %YMM4, %k4
- kandd %k4, %k5, %k5
-
- kmovd %k5, %eax
- cmpl $VEC_MASK, %eax
- jne L(4x_vec_end)
-
- addq $(VEC_SIZE * 4), %rdi
- addq $(VEC_SIZE * 4), %rsi
-
- subq $(VEC_SIZE * 4), %rdx
- cmpq $(VEC_SIZE * 4), %rdx
- jae L(loop_4x_vec)
-
- /* Less than 4 * VEC. */
- cmpq $VEC_SIZE, %rdx
- jbe L(last_vec)
- cmpq $(VEC_SIZE * 2), %rdx
- jbe L(last_2x_vec)
-
-L(last_4x_vec):
- /* From 2 * VEC to 4 * VEC. */
- VMOVU (%rsi), %YMM2
- VPCMPEQ (%rdi), %YMM2, %k2
- kmovd %k2, %eax
- subl $VEC_MASK, %eax
- jnz L(first_vec)
-
- addq $VEC_SIZE, %rdi
- addq $VEC_SIZE, %rsi
- VMOVU (%rsi), %YMM2
- VPCMPEQ (%rdi), %YMM2, %k2
- kmovd %k2, %eax
- subl $VEC_MASK, %eax
- jnz L(first_vec)
+L(between_16_31):
+ /* From 16 to 31 bytes. No branch when size == 16. */
+ VMOVU (%rsi), %XMM2
+ VPCMP $4, (%rdi), %XMM2, %k1
+ kmovd %k1, %eax
+ testl %eax, %eax
+ jnz L(return_vec_0)
/* Use overlapping loads to avoid branches. */
- leaq -(3 * VEC_SIZE)(%rdi, %rdx), %rdi
- leaq -(3 * VEC_SIZE)(%rsi, %rdx), %rsi
- VMOVU (%rsi), %YMM2
- VPCMPEQ (%rdi), %YMM2, %k2
- kmovd %k2, %eax
- subl $VEC_MASK, %eax
- jnz L(first_vec)
- addq $VEC_SIZE, %rdi
- addq $VEC_SIZE, %rsi
- VMOVU (%rsi), %YMM2
- VPCMPEQ (%rdi), %YMM2, %k2
- kmovd %k2, %eax
- subl $VEC_MASK, %eax
- jnz L(first_vec)
- ret
-
- .p2align 4
-L(4x_vec_end):
+ VMOVU -16(%rsi, %rdx, CHAR_SIZE), %XMM2
+ leaq -16(%rdi, %rdx, CHAR_SIZE), %rdi
+ leaq -16(%rsi, %rdx, CHAR_SIZE), %rsi
+ VPCMP $4, (%rdi), %XMM2, %k1
kmovd %k1, %eax
- subl $VEC_MASK, %eax
- jnz L(first_vec)
- kmovd %k2, %eax
- subl $VEC_MASK, %eax
- jnz L(first_vec_x1)
- kmovd %k3, %eax
- subl $VEC_MASK, %eax
- jnz L(first_vec_x2)
- kmovd %k4, %eax
- subl $VEC_MASK, %eax
- tzcntl %eax, %ecx
-# ifdef USE_AS_WMEMCMP
- xorl %eax, %eax
- movl (VEC_SIZE * 3)(%rdi, %rcx, 4), %edx
- cmpl (VEC_SIZE * 3)(%rsi, %rcx, 4), %edx
- jmp L(wmemcmp_return)
-# else
- movzbl (VEC_SIZE * 3)(%rdi, %rcx), %eax
- movzbl (VEC_SIZE * 3)(%rsi, %rcx), %edx
- sub %edx, %eax
-# endif
+ testl %eax, %eax
+ jnz L(return_vec_0)
ret
- .p2align 4
-L(first_vec_x1):
- tzcntl %eax, %ecx
# ifdef USE_AS_WMEMCMP
- xorl %eax, %eax
- movl VEC_SIZE(%rdi, %rcx, 4), %edx
- cmpl VEC_SIZE(%rsi, %rcx, 4), %edx
- jmp L(wmemcmp_return)
-# else
- movzbl VEC_SIZE(%rdi, %rcx), %eax
- movzbl VEC_SIZE(%rsi, %rcx), %edx
- sub %edx, %eax
-# endif
+ .p2align 4
+L(one_or_less):
+ jb L(zero)
+ movl (%rdi), %ecx
+ xorl %edx, %edx
+ cmpl (%rsi), %ecx
+ je L(zero)
+ setg %dl
+ leal -1(%rdx, %rdx), %eax
ret
+# else
.p2align 4
-L(first_vec_x2):
- tzcntl %eax, %ecx
-# ifdef USE_AS_WMEMCMP
- xorl %eax, %eax
- movl (VEC_SIZE * 2)(%rdi, %rcx, 4), %edx
- cmpl (VEC_SIZE * 2)(%rsi, %rcx, 4), %edx
- jmp L(wmemcmp_return)
-# else
- movzbl (VEC_SIZE * 2)(%rdi, %rcx), %eax
- movzbl (VEC_SIZE * 2)(%rsi, %rcx), %edx
- sub %edx, %eax
-# endif
+L(between_4_7):
+ /* Load as big endian with overlapping movbe to avoid branches.
+ */
+ movbe (%rdi), %eax
+ movbe (%rsi), %ecx
+ shlq $32, %rax
+ shlq $32, %rcx
+ movbe -4(%rdi, %rdx), %edi
+ movbe -4(%rsi, %rdx), %esi
+ orq %rdi, %rax
+ orq %rsi, %rcx
+ subq %rcx, %rax
+ jz L(zero_4_7)
+ sbbl %eax, %eax
+ orl $1, %eax
+L(zero_4_7):
ret
+# endif
+
END (MEMCMP)
#endif
--
GitLab

View File

@ -1,104 +0,0 @@
From 6abf27980a947f9b6e514d6b33b83059d39566ae Mon Sep 17 00:00:00 2001
From: Noah Goldstein <goldstein.w.n@gmail.com>
Date: Thu, 20 May 2021 13:13:51 -0400
Subject: [PATCH] x86: Improve memset-vec-unaligned-erms.S
Content-type: text/plain; charset=UTF-8
No bug. This commit makes a few small improvements to
memset-vec-unaligned-erms.S. The changes are 1) only aligning to 64
instead of 128. Either alignment will perform equally well in a loop
and 128 just increases the odds of having to do an extra iteration
which can be significant overhead for small values. 2) Align some
targets and the loop. 3) Remove an ALU from the alignment process. 4)
Reorder the last 4x VEC so that they are stored after the loop. 5)
Move the condition for leq 8x VEC to before the alignment
process. test-memset and test-wmemset are both passing.
Signed-off-by: Noah Goldstein <goldstein.w.n@gmail.com>
Reviewed-by: H.J. Lu <hjl.tools@gmail.com>
---
.../multiarch/memset-vec-unaligned-erms.S | 50 +++++++++++--------
1 file changed, 28 insertions(+), 22 deletions(-)
diff --git a/sysdeps/x86_64/multiarch/memset-vec-unaligned-erms.S b/sysdeps/x86_64/multiarch/memset-vec-unaligned-erms.S
index f877ac9d..909c33f6 100644
--- a/sysdeps/x86_64/multiarch/memset-vec-unaligned-erms.S
+++ b/sysdeps/x86_64/multiarch/memset-vec-unaligned-erms.S
@@ -173,17 +173,22 @@ ENTRY (MEMSET_SYMBOL (__memset, unaligned_erms))
VMOVU %VEC(0), (%rdi)
VZEROUPPER_RETURN
+ .p2align 4
L(stosb_more_2x_vec):
cmp __x86_rep_stosb_threshold(%rip), %RDX_LP
ja L(stosb)
+#else
+ .p2align 4
#endif
L(more_2x_vec):
- cmpq $(VEC_SIZE * 4), %rdx
- ja L(loop_start)
+ /* Stores to first 2x VEC before cmp as any path forward will
+ require it. */
VMOVU %VEC(0), (%rdi)
VMOVU %VEC(0), VEC_SIZE(%rdi)
- VMOVU %VEC(0), -VEC_SIZE(%rdi,%rdx)
+ cmpq $(VEC_SIZE * 4), %rdx
+ ja L(loop_start)
VMOVU %VEC(0), -(VEC_SIZE * 2)(%rdi,%rdx)
+ VMOVU %VEC(0), -VEC_SIZE(%rdi,%rdx)
L(return):
#if VEC_SIZE > 16
ZERO_UPPER_VEC_REGISTERS_RETURN
@@ -192,28 +197,29 @@ L(return):
#endif
L(loop_start):
- leaq (VEC_SIZE * 4)(%rdi), %rcx
- VMOVU %VEC(0), (%rdi)
- andq $-(VEC_SIZE * 4), %rcx
- VMOVU %VEC(0), -VEC_SIZE(%rdi,%rdx)
- VMOVU %VEC(0), VEC_SIZE(%rdi)
- VMOVU %VEC(0), -(VEC_SIZE * 2)(%rdi,%rdx)
VMOVU %VEC(0), (VEC_SIZE * 2)(%rdi)
- VMOVU %VEC(0), -(VEC_SIZE * 3)(%rdi,%rdx)
VMOVU %VEC(0), (VEC_SIZE * 3)(%rdi)
- VMOVU %VEC(0), -(VEC_SIZE * 4)(%rdi,%rdx)
- addq %rdi, %rdx
- andq $-(VEC_SIZE * 4), %rdx
- cmpq %rdx, %rcx
- je L(return)
+ cmpq $(VEC_SIZE * 8), %rdx
+ jbe L(loop_end)
+ andq $-(VEC_SIZE * 2), %rdi
+ subq $-(VEC_SIZE * 4), %rdi
+ leaq -(VEC_SIZE * 4)(%rax, %rdx), %rcx
+ .p2align 4
L(loop):
- VMOVA %VEC(0), (%rcx)
- VMOVA %VEC(0), VEC_SIZE(%rcx)
- VMOVA %VEC(0), (VEC_SIZE * 2)(%rcx)
- VMOVA %VEC(0), (VEC_SIZE * 3)(%rcx)
- addq $(VEC_SIZE * 4), %rcx
- cmpq %rcx, %rdx
- jne L(loop)
+ VMOVA %VEC(0), (%rdi)
+ VMOVA %VEC(0), VEC_SIZE(%rdi)
+ VMOVA %VEC(0), (VEC_SIZE * 2)(%rdi)
+ VMOVA %VEC(0), (VEC_SIZE * 3)(%rdi)
+ subq $-(VEC_SIZE * 4), %rdi
+ cmpq %rcx, %rdi
+ jb L(loop)
+L(loop_end):
+ /* NB: rax is set as ptr in MEMSET_VDUP_TO_VEC0_AND_SET_RETURN.
+ rdx as length is also unchanged. */
+ VMOVU %VEC(0), -(VEC_SIZE * 4)(%rax, %rdx)
+ VMOVU %VEC(0), -(VEC_SIZE * 3)(%rax, %rdx)
+ VMOVU %VEC(0), -(VEC_SIZE * 2)(%rax, %rdx)
+ VMOVU %VEC(0), -VEC_SIZE(%rax, %rdx)
VZEROUPPER_SHORT_RETURN
.p2align 4
--
GitLab

View File

@ -1,84 +0,0 @@
From 1b992204f68af851e905c16016756fd4421e1934 Mon Sep 17 00:00:00 2001
From: Noah Goldstein <goldstein.w.n@gmail.com>
Date: Sun, 23 May 2021 19:43:24 -0400
Subject: [PATCH] x86: Improve memmove-vec-unaligned-erms.S
Content-type: text/plain; charset=UTF-8
This patch changes the condition for copy 4x VEC so that if length is
exactly equal to 4 * VEC_SIZE it will use the 4x VEC case instead of
8x VEC case.
Results For Skylake memcpy-avx2-erms
size, al1 , al2 , Cur T , New T , Win , New / Cur
128 , 0 , 0 , 9.137 , 6.873 , New , 75.22
128 , 7 , 0 , 12.933 , 7.732 , New , 59.79
128 , 0 , 7 , 11.852 , 6.76 , New , 57.04
128 , 7 , 7 , 12.587 , 6.808 , New , 54.09
Results For Icelake memcpy-evex-erms
size, al1 , al2 , Cur T , New T , Win , New / Cur
128 , 0 , 0 , 9.963 , 5.416 , New , 54.36
128 , 7 , 0 , 16.467 , 8.061 , New , 48.95
128 , 0 , 7 , 14.388 , 7.644 , New , 53.13
128 , 7 , 7 , 14.546 , 7.642 , New , 52.54
Results For Tigerlake memcpy-evex-erms
size, al1 , al2 , Cur T , New T , Win , New / Cur
128 , 0 , 0 , 8.979 , 4.95 , New , 55.13
128 , 7 , 0 , 14.245 , 7.122 , New , 50.0
128 , 0 , 7 , 12.668 , 6.675 , New , 52.69
128 , 7 , 7 , 13.042 , 6.802 , New , 52.15
Results For Skylake memmove-avx2-erms
size, al1 , al2 , Cur T , New T , Win , New / Cur
128 , 0 , 32 , 6.181 , 5.691 , New , 92.07
128 , 32 , 0 , 6.165 , 5.752 , New , 93.3
128 , 0 , 7 , 13.923 , 9.37 , New , 67.3
128 , 7 , 0 , 12.049 , 10.182 , New , 84.5
Results For Icelake memmove-evex-erms
size, al1 , al2 , Cur T , New T , Win , New / Cur
128 , 0 , 32 , 5.479 , 4.889 , New , 89.23
128 , 32 , 0 , 5.127 , 4.911 , New , 95.79
128 , 0 , 7 , 18.885 , 13.547 , New , 71.73
128 , 7 , 0 , 15.565 , 14.436 , New , 92.75
Results For Tigerlake memmove-evex-erms
size, al1 , al2 , Cur T , New T , Win , New / Cur
128 , 0 , 32 , 5.275 , 4.815 , New , 91.28
128 , 32 , 0 , 5.376 , 4.565 , New , 84.91
128 , 0 , 7 , 19.426 , 14.273 , New , 73.47
128 , 7 , 0 , 15.924 , 14.951 , New , 93.89
Signed-off-by: Noah Goldstein <goldstein.w.n@gmail.com>
---
sysdeps/x86_64/multiarch/memmove-vec-unaligned-erms.S | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/sysdeps/x86_64/multiarch/memmove-vec-unaligned-erms.S b/sysdeps/x86_64/multiarch/memmove-vec-unaligned-erms.S
index 3e2dd6bc..572cef04 100644
--- a/sysdeps/x86_64/multiarch/memmove-vec-unaligned-erms.S
+++ b/sysdeps/x86_64/multiarch/memmove-vec-unaligned-erms.S
@@ -417,8 +417,8 @@ L(more_2x_vec):
cmpq $(VEC_SIZE * 8), %rdx
ja L(more_8x_vec)
cmpq $(VEC_SIZE * 4), %rdx
- jb L(last_4x_vec)
- /* Copy from 4 * VEC to 8 * VEC, inclusively. */
+ jbe L(last_4x_vec)
+ /* Copy from 4 * VEC + 1 to 8 * VEC, inclusively. */
VMOVU (%rsi), %VEC(0)
VMOVU VEC_SIZE(%rsi), %VEC(1)
VMOVU (VEC_SIZE * 2)(%rsi), %VEC(2)
@@ -437,7 +437,7 @@ L(more_2x_vec):
VMOVU %VEC(7), -(VEC_SIZE * 4)(%rdi,%rdx)
VZEROUPPER_RETURN
L(last_4x_vec):
- /* Copy from 2 * VEC to 4 * VEC. */
+ /* Copy from 2 * VEC + 1 to 4 * VEC, inclusively. */
VMOVU (%rsi), %VEC(0)
VMOVU VEC_SIZE(%rsi), %VEC(1)
VMOVU -VEC_SIZE(%rsi,%rdx), %VEC(2)
--
GitLab

View File

@ -1,55 +0,0 @@
From 08cbcd4dbc686bb38ec3093aff2f919fbff5ec17 Mon Sep 17 00:00:00 2001
From: Noah Goldstein <goldstein.w.n@gmail.com>
Date: Wed, 23 Jun 2021 19:19:34 -0400
Subject: [PATCH] x86: Remove unnecessary overflow check from wcsnlen-sse4_1.S
Content-type: text/plain; charset=UTF-8
No bug. The way wcsnlen will check if near the end of maxlen
is the following macro:
mov %r11, %rsi; \
subq %rax, %rsi; \
andq $-64, %rax; \
testq $-64, %rsi; \
je L(strnlen_ret)
Which words independently of s + maxlen overflowing. So the
second overflow check is unnecissary for correctness and
just extra overhead in the common no overflow case.
test-strlen.c, test-wcslen.c, test-strnlen.c and test-wcsnlen.c are
all passing
Signed-off-by: Noah Goldstein <goldstein.w.n@gmail.com>
Reviewed-by: H.J. Lu <hjl.tools@gmail.com>
---
sysdeps/x86_64/multiarch/strlen-vec.S | 7 -------
1 file changed, 7 deletions(-)
diff --git a/sysdeps/x86_64/multiarch/strlen-vec.S b/sysdeps/x86_64/multiarch/strlen-vec.S
index 439e486a..b7657282 100644
--- a/sysdeps/x86_64/multiarch/strlen-vec.S
+++ b/sysdeps/x86_64/multiarch/strlen-vec.S
@@ -71,19 +71,12 @@ L(n_nonzero):
suffice. */
mov %RSI_LP, %R10_LP
sar $62, %R10_LP
- test %R10_LP, %R10_LP
jnz __wcslen_sse4_1
sal $2, %RSI_LP
# endif
-
/* Initialize long lived registers. */
-
add %RDI_LP, %RSI_LP
-# ifdef AS_WCSLEN
-/* Check for overflow again from s + maxlen * sizeof(wchar_t). */
- jbe __wcslen_sse4_1
-# endif
mov %RSI_LP, %R10_LP
and $-64, %R10_LP
mov %RSI_LP, %R11_LP
--
GitLab

View File

@ -1,290 +0,0 @@
From 82d0b4a4d76db554eb6757acb790fcea30b19965 Mon Sep 17 00:00:00 2001
From: "H.J. Lu" <hjl.tools@gmail.com>
Date: Mon, 21 Jan 2019 11:32:24 -0800
Subject: [PATCH] x86-64 memset/wmemset: Properly handle the length parameter
[BZ# 24097]
Content-type: text/plain; charset=UTF-8
On x32, the size_t parameter may be passed in the lower 32 bits of a
64-bit register with the non-zero upper 32 bits. The string/memory
functions written in assembly can only use the lower 32 bits of a
64-bit register as length or must clear the upper 32 bits before using
the full 64-bit register for length.
This pach fixes memset/wmemset for x32. Tested on x86-64 and x32. On
x86-64, libc.so is the same with and withou the fix.
[BZ# 24097]
CVE-2019-6488
* sysdeps/x86_64/multiarch/memset-avx512-no-vzeroupper.S: Use
RDX_LP for length. Clear the upper 32 bits of RDX register.
* sysdeps/x86_64/multiarch/memset-vec-unaligned-erms.S: Likewise.
* sysdeps/x86_64/x32/Makefile (tests): Add tst-size_t-wmemset.
* sysdeps/x86_64/x32/tst-size_t-memset.c: New file.
* sysdeps/x86_64/x32/tst-size_t-wmemset.c: Likewise.
---
.../multiarch/memset-avx512-no-vzeroupper.S | 6 +-
.../multiarch/memset-vec-unaligned-erms.S | 34 +++++----
sysdeps/x86_64/x32/Makefile | 4 +-
sysdeps/x86_64/x32/tst-size_t-memset.c | 73 +++++++++++++++++++
sysdeps/x86_64/x32/tst-size_t-wmemset.c | 20 +++++
5 files changed, 121 insertions(+), 16 deletions(-)
create mode 100644 sysdeps/x86_64/x32/tst-size_t-memset.c
create mode 100644 sysdeps/x86_64/x32/tst-size_t-wmemset.c
Conflicts:
ChangeLog
(removed)
diff --git a/sysdeps/x86_64/multiarch/memset-avx512-no-vzeroupper.S b/sysdeps/x86_64/multiarch/memset-avx512-no-vzeroupper.S
index 689cc119..99e25519 100644
--- a/sysdeps/x86_64/multiarch/memset-avx512-no-vzeroupper.S
+++ b/sysdeps/x86_64/multiarch/memset-avx512-no-vzeroupper.S
@@ -29,12 +29,16 @@
.section .text.avx512,"ax",@progbits
#if defined PIC
ENTRY (MEMSET_CHK)
- cmpq %rdx, %rcx
+ cmp %RDX_LP, %RCX_LP
jb HIDDEN_JUMPTARGET (__chk_fail)
END (MEMSET_CHK)
#endif
ENTRY (MEMSET)
+# ifdef __ILP32__
+ /* Clear the upper 32 bits. */
+ mov %edx, %edx
+# endif
vpxor %xmm0, %xmm0, %xmm0
vmovd %esi, %xmm1
lea (%rdi, %rdx), %rsi
diff --git a/sysdeps/x86_64/multiarch/memset-vec-unaligned-erms.S b/sysdeps/x86_64/multiarch/memset-vec-unaligned-erms.S
index 270a1d49..9a0fd818 100644
--- a/sysdeps/x86_64/multiarch/memset-vec-unaligned-erms.S
+++ b/sysdeps/x86_64/multiarch/memset-vec-unaligned-erms.S
@@ -65,8 +65,8 @@
.section SECTION(.text),"ax",@progbits
#if VEC_SIZE == 16 && IS_IN (libc)
ENTRY (__bzero)
- movq %rdi, %rax /* Set return value. */
- movq %rsi, %rdx /* Set n. */
+ mov %RDI_LP, %RAX_LP /* Set return value. */
+ mov %RSI_LP, %RDX_LP /* Set n. */
pxor %xmm0, %xmm0
jmp L(entry_from_bzero)
END (__bzero)
@@ -76,13 +76,13 @@ weak_alias (__bzero, bzero)
#if IS_IN (libc)
# if defined SHARED
ENTRY_CHK (WMEMSET_CHK_SYMBOL (__wmemset_chk, unaligned))
- cmpq %rdx, %rcx
+ cmp %RDX_LP, %RCX_LP
jb HIDDEN_JUMPTARGET (__chk_fail)
END_CHK (WMEMSET_CHK_SYMBOL (__wmemset_chk, unaligned))
# endif
ENTRY (WMEMSET_SYMBOL (__wmemset, unaligned))
- shlq $2, %rdx
+ shl $2, %RDX_LP
WMEMSET_VDUP_TO_VEC0_AND_SET_RETURN (%esi, %rdi)
jmp L(entry_from_bzero)
END (WMEMSET_SYMBOL (__wmemset, unaligned))
@@ -90,13 +90,17 @@ END (WMEMSET_SYMBOL (__wmemset, unaligned))
#if defined SHARED && IS_IN (libc)
ENTRY_CHK (MEMSET_CHK_SYMBOL (__memset_chk, unaligned))
- cmpq %rdx, %rcx
+ cmp %RDX_LP, %RCX_LP
jb HIDDEN_JUMPTARGET (__chk_fail)
END_CHK (MEMSET_CHK_SYMBOL (__memset_chk, unaligned))
#endif
ENTRY (MEMSET_SYMBOL (__memset, unaligned))
MEMSET_VDUP_TO_VEC0_AND_SET_RETURN (%esi, %rdi)
+# ifdef __ILP32__
+ /* Clear the upper 32 bits. */
+ mov %edx, %edx
+# endif
L(entry_from_bzero):
cmpq $VEC_SIZE, %rdx
jb L(less_vec)
@@ -112,14 +116,14 @@ END (MEMSET_SYMBOL (__memset, unaligned))
# if VEC_SIZE == 16
ENTRY (__memset_chk_erms)
- cmpq %rdx, %rcx
+ cmp %RDX_LP, %RCX_LP
jb HIDDEN_JUMPTARGET (__chk_fail)
END (__memset_chk_erms)
/* Only used to measure performance of REP STOSB. */
ENTRY (__memset_erms)
/* Skip zero length. */
- testq %rdx, %rdx
+ test %RDX_LP, %RDX_LP
jnz L(stosb)
movq %rdi, %rax
ret
@@ -131,11 +135,11 @@ ENTRY (MEMSET_SYMBOL (__memset, erms))
L(stosb):
/* Issue vzeroupper before rep stosb. */
VZEROUPPER
- movq %rdx, %rcx
+ mov %RDX_LP, %RCX_LP
movzbl %sil, %eax
- movq %rdi, %rdx
+ mov %RDI_LP, %RDX_LP
rep stosb
- movq %rdx, %rax
+ mov %RDX_LP, %RAX_LP
ret
# if VEC_SIZE == 16
END (__memset_erms)
@@ -145,16 +149,20 @@ END (MEMSET_SYMBOL (__memset, erms))
# if defined SHARED && IS_IN (libc)
ENTRY_CHK (MEMSET_CHK_SYMBOL (__memset_chk, unaligned_erms))
- cmpq %rdx, %rcx
+ cmp %RDX_LP, %RCX_LP
jb HIDDEN_JUMPTARGET (__chk_fail)
END_CHK (MEMSET_CHK_SYMBOL (__memset_chk, unaligned_erms))
# endif
ENTRY (MEMSET_SYMBOL (__memset, unaligned_erms))
MEMSET_VDUP_TO_VEC0_AND_SET_RETURN (%esi, %rdi)
- cmpq $VEC_SIZE, %rdx
+# ifdef __ILP32__
+ /* Clear the upper 32 bits. */
+ mov %edx, %edx
+# endif
+ cmp $VEC_SIZE, %RDX_LP
jb L(less_vec)
- cmpq $(VEC_SIZE * 2), %rdx
+ cmp $(VEC_SIZE * 2), %RDX_LP
ja L(stosb_more_2x_vec)
/* From VEC and to 2 * VEC. No branch when size == VEC_SIZE. */
VMOVU %VEC(0), -VEC_SIZE(%rdi,%rdx)
diff --git a/sysdeps/x86_64/x32/Makefile b/sysdeps/x86_64/x32/Makefile
index e99dbd7c..98bd9ae9 100644
--- a/sysdeps/x86_64/x32/Makefile
+++ b/sysdeps/x86_64/x32/Makefile
@@ -7,9 +7,9 @@ endif
ifeq ($(subdir),string)
tests += tst-size_t-memchr tst-size_t-memcmp tst-size_t-memcpy \
- tst-size_t-memrchr
+ tst-size_t-memrchr tst-size_t-memset
endif
ifeq ($(subdir),wcsmbs)
-tests += tst-size_t-wmemchr tst-size_t-wmemcmp
+tests += tst-size_t-wmemchr tst-size_t-wmemcmp tst-size_t-wmemset
endif
diff --git a/sysdeps/x86_64/x32/tst-size_t-memset.c b/sysdeps/x86_64/x32/tst-size_t-memset.c
new file mode 100644
index 00000000..2c367af6
--- /dev/null
+++ b/sysdeps/x86_64/x32/tst-size_t-memset.c
@@ -0,0 +1,73 @@
+/* Test memset with size_t in the lower 32 bits of 64-bit register.
+ Copyright (C) 2019 Free Software Foundation, Inc.
+ This file is part of the GNU C Library.
+
+ The GNU C Library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ The GNU C Library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with the GNU C Library; if not, see
+ <http://www.gnu.org/licenses/>. */
+
+#ifdef WIDE
+# define TEST_NAME "wmemset"
+#else
+# define TEST_NAME "memset"
+#endif /* WIDE */
+
+#include "test-size_t.h"
+
+#ifdef WIDE
+# include <wchar.h>
+# define MEMSET wmemset
+# define CHAR wchar_t
+#else
+# define MEMSET memset
+# define CHAR char
+#endif /* WIDE */
+
+IMPL (MEMSET, 1)
+
+typedef CHAR *(*proto_t) (CHAR *, int, size_t);
+
+static void *
+__attribute__ ((noinline, noclone))
+do_memset (parameter_t a, parameter_t b)
+{
+ return CALL (&b, a.p, (uintptr_t) b.p, a.len);
+}
+
+static int
+test_main (void)
+{
+ test_init ();
+
+ CHAR ch = 0x23;
+ parameter_t src = { { page_size / sizeof (CHAR) }, buf2 };
+ parameter_t c = { { 0 }, (void *) (uintptr_t) ch };
+
+ int ret = 0;
+ FOR_EACH_IMPL (impl, 0)
+ {
+ c.fn = impl->fn;
+ CHAR *p = (CHAR *) do_memset (src, c);
+ size_t i;
+ for (i = 0; i < src.len; i++)
+ if (p[i] != ch)
+ {
+ error (0, 0, "Wrong result in function %s", impl->name);
+ ret = 1;
+ }
+ }
+
+ return ret ? EXIT_FAILURE : EXIT_SUCCESS;
+}
+
+#include <support/test-driver.c>
diff --git a/sysdeps/x86_64/x32/tst-size_t-wmemset.c b/sysdeps/x86_64/x32/tst-size_t-wmemset.c
new file mode 100644
index 00000000..955eb488
--- /dev/null
+++ b/sysdeps/x86_64/x32/tst-size_t-wmemset.c
@@ -0,0 +1,20 @@
+/* Test wmemset with size_t in the lower 32 bits of 64-bit register.
+ Copyright (C) 2019 Free Software Foundation, Inc.
+ This file is part of the GNU C Library.
+
+ The GNU C Library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ The GNU C Library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with the GNU C Library; if not, see
+ <http://www.gnu.org/licenses/>. */
+
+#define WIDE 1
+#include "tst-size_t-memset.c"
--
GitLab

View File

@ -1,43 +0,0 @@
From 447954a206837b5f153869cfeeeab44631c3fac9 Mon Sep 17 00:00:00 2001
Author: Shen-Ta Hsieh <ibmibmibm.tw@gmail.com> 2021-05-23 21:43:10
Committer: H.J. Lu <hjl.tools@gmail.com> 2021-06-27 10:56:57
Parent: 2c16cb88a6e5ace0fb7cedca86860ea7bde522a7 (Linux: Move timer helper routines from librt to libc)
Child: 1683249d17e14827b6579529742eb895027dfa84 (x86_64: roundeven with sse4.1 support)
Branches: master, remotes/origin/master and many more (41)
Follows: glibc-2.33.9000
Precedes: glibc-2.34
math: redirect roundeven function
This patch redirect roundeven function for futhermore changes.
Signed-off-by: Shen-Ta Hsieh <ibmibmibm.tw@gmail.com>
Reviewed-by: H.J. Lu <hjl.tools@gmail.com>
Conflicts:
*
(rewritten for older branch)
diff --git a/sysdeps/ieee754/dbl-64/wordsize-64/s_roundeven.c b/sysdeps/ieee754/dbl-64/wordsize-64/s_roundeven.c
index 7bbbb2dc..8728d0f2 100644
--- a/sysdeps/ieee754/dbl-64/wordsize-64/s_roundeven.c
+++ b/sysdeps/ieee754/dbl-64/wordsize-64/s_roundeven.c
@@ -17,6 +17,7 @@
License along with the GNU C Library; if not, see
<http://www.gnu.org/licenses/>. */
+#define NO_MATH_REDIRECT
#include <math.h>
#include <math_private.h>
#include <libm-alias-double.h>
@@ -67,5 +68,6 @@ __roundeven (double x)
INSERT_WORDS64 (x, ix);
return x;
}
-hidden_def (__roundeven)
+#ifndef __roundeven
libm_alias_double (__roundeven, roundeven)
+#endif
--
GitLab

View File

@ -1,118 +0,0 @@
From 447954a206837b5f153869cfeeeab44631c3fac9 Mon Sep 17 00:00:00 2001
From: Shen-Ta Hsieh <ibmibmibm.tw@gmail.com>
Date: Mon, 24 May 2021 09:43:10 +0800
Subject: [PATCH] math: redirect roundeven function
Content-type: text/plain; charset=UTF-8
This patch redirect roundeven function for futhermore changes.
Signed-off-by: Shen-Ta Hsieh <ibmibmibm.tw@gmail.com>
Reviewed-by: H.J. Lu <hjl.tools@gmail.com>
---
include/math.h | 3 ++-
sysdeps/ieee754/dbl-64/s_roundeven.c | 4 +++-
sysdeps/ieee754/float128/s_roundevenf128.c | 1 +
sysdeps/ieee754/flt-32/s_roundevenf.c | 3 +++
sysdeps/ieee754/ldbl-128/s_roundevenl.c | 1 +
sysdeps/ieee754/ldbl-96/s_roundevenl.c | 1 +
6 files changed, 11 insertions(+), 2 deletions(-)
Conflicts:
include/math.h
(missing MATH_REDIRECT macros)
diff --git a/include/math.h b/include/math.h
index e21d34b8..1f9f9a54 100644
--- a/include/math.h
+++ b/include/math.h
@@ -38,7 +38,6 @@ libm_hidden_proto (__issignaling)
libm_hidden_proto (__issignalingf)
libm_hidden_proto (__exp)
libm_hidden_proto (__expf)
-libm_hidden_proto (__roundeven)
# ifndef __NO_LONG_DOUBLE_MATH
libm_hidden_proto (__fpclassifyl)
@@ -56,6 +55,8 @@ libm_hidden_proto (__expm1f128)
# if !(defined __FINITE_MATH_ONLY__ && __FINITE_MATH_ONLY__ > 0)
# ifndef NO_MATH_REDIRECT
+float (roundevenf) (float) asm ("__roundevenf");
+double (roundeven) (double) asm ("__roundeven");
/* Declare sqrt for use within GLIBC. Compilers typically inline sqrt as a
single instruction. Use an asm to avoid use of PLTs if it doesn't. */
float (sqrtf) (float) asm ("__ieee754_sqrtf");
diff --git a/sysdeps/ieee754/dbl-64/s_roundeven.c b/sysdeps/ieee754/dbl-64/s_roundeven.c
index 1438e81d..61962184 100644
--- a/sysdeps/ieee754/dbl-64/s_roundeven.c
+++ b/sysdeps/ieee754/dbl-64/s_roundeven.c
@@ -17,6 +17,7 @@
License along with the GNU C Library; if not, see
<http://www.gnu.org/licenses/>. */
+#define NO_MATH_REDIRECT
#include <math.h>
#include <math_private.h>
#include <libm-alias-double.h>
@@ -101,5 +102,6 @@ __roundeven (double x)
INSERT_WORDS (x, hx, lx);
return x;
}
-hidden_def (__roundeven)
+#ifndef __roundeven
libm_alias_double (__roundeven, roundeven)
+#endif
diff --git a/sysdeps/ieee754/float128/s_roundevenf128.c b/sysdeps/ieee754/float128/s_roundevenf128.c
index 5a9b3f39..e0faf727 100644
--- a/sysdeps/ieee754/float128/s_roundevenf128.c
+++ b/sysdeps/ieee754/float128/s_roundevenf128.c
@@ -1,2 +1,3 @@
+#define NO_MATH_REDIRECT
#include <float128_private.h>
#include "../ldbl-128/s_roundevenl.c"
diff --git a/sysdeps/ieee754/flt-32/s_roundevenf.c b/sysdeps/ieee754/flt-32/s_roundevenf.c
index 90f991d5..a661875e 100644
--- a/sysdeps/ieee754/flt-32/s_roundevenf.c
+++ b/sysdeps/ieee754/flt-32/s_roundevenf.c
@@ -17,6 +17,7 @@
License along with the GNU C Library; if not, see
<http://www.gnu.org/licenses/>. */
+#define NO_MATH_REDIRECT
#include <math.h>
#include <math_private.h>
#include <libm-alias-float.h>
@@ -67,4 +68,6 @@ __roundevenf (float x)
SET_FLOAT_WORD (x, ix);
return x;
}
+#ifndef __roundevenf
libm_alias_float (__roundeven, roundeven)
+#endif
diff --git a/sysdeps/ieee754/ldbl-128/s_roundevenl.c b/sysdeps/ieee754/ldbl-128/s_roundevenl.c
index 5fc59af4..b9375b6c 100644
--- a/sysdeps/ieee754/ldbl-128/s_roundevenl.c
+++ b/sysdeps/ieee754/ldbl-128/s_roundevenl.c
@@ -17,6 +17,7 @@
License along with the GNU C Library; if not, see
<http://www.gnu.org/licenses/>. */
+#define NO_MATH_REDIRECT
#include <math.h>
#include <math_private.h>
#include <libm-alias-ldouble.h>
diff --git a/sysdeps/ieee754/ldbl-96/s_roundevenl.c b/sysdeps/ieee754/ldbl-96/s_roundevenl.c
index be2e4fa4..65031ab7 100644
--- a/sysdeps/ieee754/ldbl-96/s_roundevenl.c
+++ b/sysdeps/ieee754/ldbl-96/s_roundevenl.c
@@ -17,6 +17,7 @@
License along with the GNU C Library; if not, see
<http://www.gnu.org/licenses/>. */
+#define NO_MATH_REDIRECT
#include <math.h>
#include <math_private.h>
#include <libm-alias-ldouble.h>
--
GitLab

View File

@ -1,242 +0,0 @@
From 1683249d17e14827b6579529742eb895027dfa84 Mon Sep 17 00:00:00 2001
From: Shen-Ta Hsieh <ibmibmibm.tw@gmail.com>
Date: Mon, 24 May 2021 09:43:11 +0800
Subject: [PATCH] x86_64: roundeven with sse4.1 support
Content-type: text/plain; charset=UTF-8
This patch adds support for the sse4.1 hardware floating point
roundeven.
Here is some benchmark results on my systems:
=AMD Ryzen 9 3900X 12-Core Processor=
* benchmark result before this commit
| | roundeven | roundevenf |
|------------|--------------|--------------|
| duration | 3.75587e+09 | 3.75114e+09 |
| iterations | 3.93053e+08 | 4.35402e+08 |
| max | 52.592 | 58.71 |
| min | 7.98 | 7.22 |
| mean | 9.55563 | 8.61535 |
* benchmark result after this commit
| | roundeven | roundevenf |
|------------|---------------|--------------|
| duration | 3.73815e+09 | 3.73738e+09 |
| iterations | 5.82692e+08 | 5.91498e+08 |
| max | 56.468 | 51.642 |
| min | 6.27 | 6.156 |
| mean | 6.41532 | 6.3185 |
=Intel(R) Pentium(R) CPU D1508 @ 2.20GHz=
* benchmark result before this commit
| | roundeven | roundevenf |
|------------|--------------|--------------|
| duration | 2.18208e+09 | 2.18258e+09 |
| iterations | 2.39932e+08 | 2.46924e+08 |
| max | 96.378 | 98.035 |
| min | 6.776 | 5.94 |
| mean | 9.09456 | 8.83907 |
* benchmark result after this commit
| | roundeven | roundevenf |
|------------|--------------|--------------|
| duration | 2.17415e+09 | 2.17005e+09 |
| iterations | 3.56193e+08 | 4.09824e+08 |
| max | 51.693 | 97.192 |
| min | 5.926 | 5.093 |
| mean | 6.10385 | 5.29507 |
Signed-off-by: Shen-Ta Hsieh <ibmibmibm.tw@gmail.com>
Reviewed-by: H.J. Lu <hjl.tools@gmail.com>
---
sysdeps/x86_64/fpu/multiarch/Makefile | 5 +--
sysdeps/x86_64/fpu/multiarch/s_roundeven-c.c | 2 ++
.../x86_64/fpu/multiarch/s_roundeven-sse4_1.S | 24 ++++++++++++++
sysdeps/x86_64/fpu/multiarch/s_roundeven.c | 31 +++++++++++++++++++
sysdeps/x86_64/fpu/multiarch/s_roundevenf-c.c | 3 ++
.../fpu/multiarch/s_roundevenf-sse4_1.S | 24 ++++++++++++++
sysdeps/x86_64/fpu/multiarch/s_roundevenf.c | 31 +++++++++++++++++++
7 files changed, 118 insertions(+), 2 deletions(-)
create mode 100644 sysdeps/x86_64/fpu/multiarch/s_roundeven-c.c
create mode 100644 sysdeps/x86_64/fpu/multiarch/s_roundeven-sse4_1.S
create mode 100644 sysdeps/x86_64/fpu/multiarch/s_roundeven.c
create mode 100644 sysdeps/x86_64/fpu/multiarch/s_roundevenf-c.c
create mode 100644 sysdeps/x86_64/fpu/multiarch/s_roundevenf-sse4_1.S
create mode 100644 sysdeps/x86_64/fpu/multiarch/s_roundevenf.c
diff --git a/sysdeps/x86_64/fpu/multiarch/Makefile b/sysdeps/x86_64/fpu/multiarch/Makefile
index 9f387248..6ddd1c01 100644
--- a/sysdeps/x86_64/fpu/multiarch/Makefile
+++ b/sysdeps/x86_64/fpu/multiarch/Makefile
@@ -1,11 +1,12 @@
ifeq ($(subdir),math)
libm-sysdep_routines += s_floor-c s_ceil-c s_floorf-c s_ceilf-c \
s_rint-c s_rintf-c s_nearbyint-c s_nearbyintf-c \
- s_trunc-c s_truncf-c
+ s_roundeven-c s_roundevenf-c s_trunc-c s_truncf-c
libm-sysdep_routines += s_ceil-sse4_1 s_ceilf-sse4_1 s_floor-sse4_1 \
s_floorf-sse4_1 s_nearbyint-sse4_1 \
- s_nearbyintf-sse4_1 s_rint-sse4_1 s_rintf-sse4_1 \
+ s_nearbyintf-sse4_1 s_roundeven-sse4_1 \
+ s_roundevenf-sse4_1 s_rint-sse4_1 s_rintf-sse4_1 \
s_trunc-sse4_1 s_truncf-sse4_1
libm-sysdep_routines += e_exp-fma e_log-fma e_pow-fma s_atan-fma \
diff --git a/sysdeps/x86_64/fpu/multiarch/s_roundeven-c.c b/sysdeps/x86_64/fpu/multiarch/s_roundeven-c.c
new file mode 100644
index 00000000..c7be43cb
--- /dev/null
+++ b/sysdeps/x86_64/fpu/multiarch/s_roundeven-c.c
@@ -0,0 +1,2 @@
+#define __roundeven __roundeven_c
+#include <sysdeps/ieee754/dbl-64/s_roundeven.c>
diff --git a/sysdeps/x86_64/fpu/multiarch/s_roundeven-sse4_1.S b/sysdeps/x86_64/fpu/multiarch/s_roundeven-sse4_1.S
new file mode 100644
index 00000000..6ae8f6b1
--- /dev/null
+++ b/sysdeps/x86_64/fpu/multiarch/s_roundeven-sse4_1.S
@@ -0,0 +1,24 @@
+/* Copyright (C) 2021 Free Software Foundation, Inc.
+ This file is part of the GNU C Library.
+
+ The GNU C Library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ The GNU C Library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with the GNU C Library; if not, see
+ <https://www.gnu.org/licenses/>. */
+
+#include <sysdep.h>
+
+ .section .text.sse4.1,"ax",@progbits
+ENTRY(__roundeven_sse41)
+ roundsd $8, %xmm0, %xmm0
+ ret
+END(__roundeven_sse41)
diff --git a/sysdeps/x86_64/fpu/multiarch/s_roundeven.c b/sysdeps/x86_64/fpu/multiarch/s_roundeven.c
new file mode 100644
index 00000000..d92eda65
--- /dev/null
+++ b/sysdeps/x86_64/fpu/multiarch/s_roundeven.c
@@ -0,0 +1,31 @@
+/* Multiple versions of __roundeven.
+ Copyright (C) 2021 Free Software Foundation, Inc.
+ This file is part of the GNU C Library.
+
+ The GNU C Library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ The GNU C Library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with the GNU C Library; if not, see
+ <https://www.gnu.org/licenses/>. */
+
+#include <libm-alias-double.h>
+
+#define roundeven __redirect_roundeven
+#define __roundeven __redirect___roundeven
+#include <math.h>
+#undef roundeven
+#undef __roundeven
+
+#define SYMBOL_NAME roundeven
+#include "ifunc-sse4_1.h"
+
+libc_ifunc_redirected (__redirect_roundeven, __roundeven, IFUNC_SELECTOR ());
+libm_alias_double (__roundeven, roundeven)
diff --git a/sysdeps/x86_64/fpu/multiarch/s_roundevenf-c.c b/sysdeps/x86_64/fpu/multiarch/s_roundevenf-c.c
new file mode 100644
index 00000000..72a6e7d1
--- /dev/null
+++ b/sysdeps/x86_64/fpu/multiarch/s_roundevenf-c.c
@@ -0,0 +1,3 @@
+#undef __roundevenf
+#define __roundevenf __roundevenf_c
+#include <sysdeps/ieee754/flt-32/s_roundevenf.c>
diff --git a/sysdeps/x86_64/fpu/multiarch/s_roundevenf-sse4_1.S b/sysdeps/x86_64/fpu/multiarch/s_roundevenf-sse4_1.S
new file mode 100644
index 00000000..a76e1080
--- /dev/null
+++ b/sysdeps/x86_64/fpu/multiarch/s_roundevenf-sse4_1.S
@@ -0,0 +1,24 @@
+/* Copyright (C) 2021 Free Software Foundation, Inc.
+ This file is part of the GNU C Library.
+
+ The GNU C Library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ The GNU C Library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with the GNU C Library; if not, see
+ <https://www.gnu.org/licenses/>. */
+
+#include <sysdep.h>
+
+ .section .text.sse4.1,"ax",@progbits
+ENTRY(__roundevenf_sse41)
+ roundss $8, %xmm0, %xmm0
+ ret
+END(__roundevenf_sse41)
diff --git a/sysdeps/x86_64/fpu/multiarch/s_roundevenf.c b/sysdeps/x86_64/fpu/multiarch/s_roundevenf.c
new file mode 100644
index 00000000..2ee196e6
--- /dev/null
+++ b/sysdeps/x86_64/fpu/multiarch/s_roundevenf.c
@@ -0,0 +1,31 @@
+/* Multiple versions of __roundevenf.
+ Copyright (C) 2021 Free Software Foundation, Inc.
+ This file is part of the GNU C Library.
+
+ The GNU C Library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ The GNU C Library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with the GNU C Library; if not, see
+ <https://www.gnu.org/licenses/>. */
+
+#include <libm-alias-float.h>
+
+#define roundevenf __redirect_roundevenf
+#define __roundevenf __redirect___roundevenf
+#include <math.h>
+#undef roundevenf
+#undef __roundevenf
+
+#define SYMBOL_NAME roundevenf
+#include "ifunc-sse4_1.h"
+
+libc_ifunc_redirected (__redirect_roundevenf, __roundevenf, IFUNC_SELECTOR ());
+libm_alias_float (__roundeven, roundeven)
--
GitLab

View File

@ -1,41 +0,0 @@
From 7e08db3359c86c94918feb33a1182cd0ff3bb10b Mon Sep 17 00:00:00 2001
From: Noah Goldstein <goldstein.w.n@gmail.com>
Date: Sun, 9 Jan 2022 16:02:28 -0600
Subject: [PATCH] x86: Fix __wcsncmp_evex in strcmp-evex.S [BZ# 28755]
Content-type: text/plain; charset=UTF-8
Fixes [BZ# 28755] for wcsncmp by redirecting length >= 2^56 to
__wcscmp_evex. For x86_64 this covers the entire address range so any
length larger could not possibly be used to bound `s1` or `s2`.
test-strcmp, test-strncmp, test-wcscmp, and test-wcsncmp all pass.
Signed-off-by: Noah Goldstein <goldstein.w.n@gmail.com>
---
sysdeps/x86_64/multiarch/strcmp-evex.S | 10 ++++++++++
1 file changed, 10 insertions(+)
diff --git a/sysdeps/x86_64/multiarch/strcmp-evex.S b/sysdeps/x86_64/multiarch/strcmp-evex.S
index 459eeed0..d5aa6daa 100644
--- a/sysdeps/x86_64/multiarch/strcmp-evex.S
+++ b/sysdeps/x86_64/multiarch/strcmp-evex.S
@@ -97,6 +97,16 @@ ENTRY (STRCMP)
je L(char0)
jb L(zero)
# ifdef USE_AS_WCSCMP
+# ifndef __ILP32__
+ movq %rdx, %rcx
+ /* Check if length could overflow when multiplied by
+ sizeof(wchar_t). Checking top 8 bits will cover all potential
+ overflow cases as well as redirect cases where its impossible to
+ length to bound a valid memory region. In these cases just use
+ 'wcscmp'. */
+ shrq $56, %rcx
+ jnz __wcscmp_evex
+# endif
/* Convert units: from wide to byte char. */
shl $2, %RDX_LP
# endif
--
GitLab

View File

@ -1,45 +0,0 @@
From bad852b61b79503fcb3c5fc379c70f768df3e1fb Mon Sep 17 00:00:00 2001
From: Noah Goldstein <goldstein.w.n@gmail.com>
Date: Sat, 23 Oct 2021 01:26:47 -0400
Subject: [PATCH] x86: Replace sse2 instructions with avx in
memcmp-evex-movbe.S
Content-type: text/plain; charset=UTF-8
This commit replaces two usages of SSE2 'movups' with AVX 'vmovdqu'.
it could potentially be dangerous to use SSE2 if this function is ever
called without using 'vzeroupper' beforehand. While compilers appear
to use 'vzeroupper' before function calls if AVX2 has been used, using
SSE2 here is more brittle. Since it is not absolutely necessary it
should be avoided.
It costs 2-extra bytes but the extra bytes should only eat into
alignment padding.
Reviewed-by: H.J. Lu <hjl.tools@gmail.com>
---
sysdeps/x86_64/multiarch/memcmp-evex-movbe.S | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/sysdeps/x86_64/multiarch/memcmp-evex-movbe.S b/sysdeps/x86_64/multiarch/memcmp-evex-movbe.S
index 2761b54f..640f6757 100644
--- a/sysdeps/x86_64/multiarch/memcmp-evex-movbe.S
+++ b/sysdeps/x86_64/multiarch/memcmp-evex-movbe.S
@@ -561,13 +561,13 @@ L(between_16_31):
/* From 16 to 31 bytes. No branch when size == 16. */
/* Use movups to save code size. */
- movups (%rsi), %xmm2
+ vmovdqu (%rsi), %xmm2
VPCMP $4, (%rdi), %xmm2, %k1
kmovd %k1, %eax
testl %eax, %eax
jnz L(return_vec_0_lv)
/* Use overlapping loads to avoid branches. */
- movups -16(%rsi, %rdx, CHAR_SIZE), %xmm2
+ vmovdqu -16(%rsi, %rdx, CHAR_SIZE), %xmm2
VPCMP $4, -16(%rdi, %rdx, CHAR_SIZE), %xmm2, %k1
addl $(CHAR_PER_VEC - (16 / CHAR_SIZE)), %edx
kmovd %k1, %eax
--
GitLab

View File

@ -1,300 +0,0 @@
From ee915088a0231cd421054dbd8abab7aadf331153 Mon Sep 17 00:00:00 2001
From: "H.J. Lu" <hjl.tools@gmail.com>
Date: Mon, 21 Jan 2019 11:33:52 -0800
Subject: [PATCH] x86-64 strncmp family: Properly handle the length parameter
[BZ# 24097]
Content-type: text/plain; charset=UTF-8
On x32, the size_t parameter may be passed in the lower 32 bits of a
64-bit register with the non-zero upper 32 bits. The string/memory
functions written in assembly can only use the lower 32 bits of a
64-bit register as length or must clear the upper 32 bits before using
the full 64-bit register for length.
This pach fixes the strncmp family for x32. Tested on x86-64 and x32.
On x86-64, libc.so is the same with and withou the fix.
[BZ# 24097]
CVE-2019-6488
* sysdeps/x86_64/multiarch/strcmp-avx2.S: Use RDX_LP for length.
* sysdeps/x86_64/multiarch/strcmp-sse42.S: Likewise.
* sysdeps/x86_64/strcmp.S: Likewise.
* sysdeps/x86_64/x32/Makefile (tests): Add tst-size_t-strncasecmp,
tst-size_t-strncmp and tst-size_t-wcsncmp.
* sysdeps/x86_64/x32/tst-size_t-strncasecmp.c: New file.
* sysdeps/x86_64/x32/tst-size_t-strncmp.c: Likewise.
* sysdeps/x86_64/x32/tst-size_t-wcsncmp.c: Likewise.
---
sysdeps/x86_64/multiarch/strcmp-avx2.S | 6 +-
sysdeps/x86_64/multiarch/strcmp-sse42.S | 6 +-
sysdeps/x86_64/strcmp.S | 6 +-
sysdeps/x86_64/x32/Makefile | 6 +-
sysdeps/x86_64/x32/tst-size_t-strncasecmp.c | 59 ++++++++++++++++
sysdeps/x86_64/x32/tst-size_t-strncmp.c | 78 +++++++++++++++++++++
sysdeps/x86_64/x32/tst-size_t-wcsncmp.c | 20 ++++++
7 files changed, 170 insertions(+), 11 deletions(-)
create mode 100644 sysdeps/x86_64/x32/tst-size_t-strncasecmp.c
create mode 100644 sysdeps/x86_64/x32/tst-size_t-strncmp.c
create mode 100644 sysdeps/x86_64/x32/tst-size_t-wcsncmp.c
Conflicts:
ChangeLog
(removed)
diff --git a/sysdeps/x86_64/multiarch/strcmp-avx2.S b/sysdeps/x86_64/multiarch/strcmp-avx2.S
index 327e3d87..156c1949 100644
--- a/sysdeps/x86_64/multiarch/strcmp-avx2.S
+++ b/sysdeps/x86_64/multiarch/strcmp-avx2.S
@@ -79,15 +79,15 @@
ENTRY (STRCMP)
# ifdef USE_AS_STRNCMP
/* Check for simple cases (0 or 1) in offset. */
- cmp $1, %rdx
+ cmp $1, %RDX_LP
je L(char0)
jb L(zero)
# ifdef USE_AS_WCSCMP
/* Convert units: from wide to byte char. */
- shl $2, %rdx
+ shl $2, %RDX_LP
# endif
/* Register %r11 tracks the maximum offset. */
- movq %rdx, %r11
+ mov %RDX_LP, %R11_LP
# endif
movl %edi, %eax
xorl %edx, %edx
diff --git a/sysdeps/x86_64/multiarch/strcmp-sse42.S b/sysdeps/x86_64/multiarch/strcmp-sse42.S
index d3c07bd2..a1ebea46 100644
--- a/sysdeps/x86_64/multiarch/strcmp-sse42.S
+++ b/sysdeps/x86_64/multiarch/strcmp-sse42.S
@@ -156,11 +156,11 @@ STRCMP_SSE42:
#endif
#if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L
- test %rdx, %rdx
+ test %RDX_LP, %RDX_LP
je LABEL(strcmp_exitz)
- cmp $1, %rdx
+ cmp $1, %RDX_LP
je LABEL(Byte0)
- mov %rdx, %r11
+ mov %RDX_LP, %R11_LP
#endif
mov %esi, %ecx
mov %edi, %eax
diff --git a/sysdeps/x86_64/strcmp.S b/sysdeps/x86_64/strcmp.S
index e16945b9..f47c8ad4 100644
--- a/sysdeps/x86_64/strcmp.S
+++ b/sysdeps/x86_64/strcmp.S
@@ -135,11 +135,11 @@ ENTRY (STRCMP)
* This implementation uses SSE to compare up to 16 bytes at a time.
*/
#if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L
- test %rdx, %rdx
+ test %RDX_LP, %RDX_LP
je LABEL(strcmp_exitz)
- cmp $1, %rdx
+ cmp $1, %RDX_LP
je LABEL(Byte0)
- mov %rdx, %r11
+ mov %RDX_LP, %R11_LP
#endif
mov %esi, %ecx
mov %edi, %eax
diff --git a/sysdeps/x86_64/x32/Makefile b/sysdeps/x86_64/x32/Makefile
index 98bd9ae9..db302839 100644
--- a/sysdeps/x86_64/x32/Makefile
+++ b/sysdeps/x86_64/x32/Makefile
@@ -7,9 +7,11 @@ endif
ifeq ($(subdir),string)
tests += tst-size_t-memchr tst-size_t-memcmp tst-size_t-memcpy \
- tst-size_t-memrchr tst-size_t-memset
+ tst-size_t-memrchr tst-size_t-memset tst-size_t-strncasecmp \
+ tst-size_t-strncmp
endif
ifeq ($(subdir),wcsmbs)
-tests += tst-size_t-wmemchr tst-size_t-wmemcmp tst-size_t-wmemset
+tests += tst-size_t-wmemchr tst-size_t-wmemcmp tst-size_t-wmemset \
+ tst-size_t-wcsncmp
endif
diff --git a/sysdeps/x86_64/x32/tst-size_t-strncasecmp.c b/sysdeps/x86_64/x32/tst-size_t-strncasecmp.c
new file mode 100644
index 00000000..86233593
--- /dev/null
+++ b/sysdeps/x86_64/x32/tst-size_t-strncasecmp.c
@@ -0,0 +1,59 @@
+/* Test strncaecmp with size_t in the lower 32 bits of 64-bit register.
+ Copyright (C) 2019 Free Software Foundation, Inc.
+ This file is part of the GNU C Library.
+
+ The GNU C Library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ The GNU C Library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with the GNU C Library; if not, see
+ <http://www.gnu.org/licenses/>. */
+
+#define TEST_NAME "strncasecmp"
+#include "test-size_t.h"
+
+IMPL (strncasecmp, 1)
+
+typedef int (*proto_t) (const char *, const char *, size_t);
+
+static int
+__attribute__ ((noinline, noclone))
+do_strncasecmp (parameter_t a, parameter_t b)
+{
+ return CALL (&b, a.p, b.p, a.len);
+}
+
+static int
+test_main (void)
+{
+ test_init ();
+
+ parameter_t dest = { { page_size }, buf1 };
+ parameter_t src = { { 0 }, buf2 };
+
+ strncpy ((char *) buf1, (const char *) buf2, page_size);
+
+ int ret = 0;
+ FOR_EACH_IMPL (impl, 0)
+ {
+ src.fn = impl->fn;
+ int res = do_strncasecmp (dest, src);
+ if (res)
+ {
+ error (0, 0, "Wrong result in function %s: %i != 0",
+ impl->name, res);
+ ret = 1;
+ }
+ }
+
+ return ret ? EXIT_FAILURE : EXIT_SUCCESS;
+}
+
+#include <support/test-driver.c>
diff --git a/sysdeps/x86_64/x32/tst-size_t-strncmp.c b/sysdeps/x86_64/x32/tst-size_t-strncmp.c
new file mode 100644
index 00000000..54e6bd83
--- /dev/null
+++ b/sysdeps/x86_64/x32/tst-size_t-strncmp.c
@@ -0,0 +1,78 @@
+/* Test strncmp with size_t in the lower 32 bits of 64-bit register.
+ Copyright (C) 2019 Free Software Foundation, Inc.
+ This file is part of the GNU C Library.
+
+ The GNU C Library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ The GNU C Library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with the GNU C Library; if not, see
+ <http://www.gnu.org/licenses/>. */
+
+#ifdef WIDE
+# define TEST_NAME "wcsncmp"
+#else
+# define TEST_NAME "strncmp"
+#endif
+
+#include "test-size_t.h"
+
+#ifdef WIDE
+# include <wchar.h>
+
+# define STRNCMP wcsncmp
+# define STRNCPY wcsncpy
+# define CHAR wchar_t
+#else
+# define STRNCMP strncmp
+# define STRNCPY strncpy
+# define CHAR char
+#endif
+
+IMPL (STRNCMP, 1)
+
+typedef int (*proto_t) (const CHAR *, const CHAR *, size_t);
+
+
+static int
+__attribute__ ((noinline, noclone))
+do_strncmp (parameter_t a, parameter_t b)
+{
+ return CALL (&b, a.p, b.p, a.len);
+}
+
+static int
+test_main (void)
+{
+ test_init ();
+
+ size_t size = page_size / sizeof (CHAR);
+ parameter_t dest = { { size }, buf1 };
+ parameter_t src = { { 0 }, buf2 };
+
+ STRNCPY ((CHAR *) buf1, (const CHAR *) buf2, size);
+
+ int ret = 0;
+ FOR_EACH_IMPL (impl, 0)
+ {
+ src.fn = impl->fn;
+ int res = do_strncmp (dest, src);
+ if (res)
+ {
+ error (0, 0, "Wrong result in function %s: %i != 0",
+ impl->name, res);
+ ret = 1;
+ }
+ }
+
+ return ret ? EXIT_FAILURE : EXIT_SUCCESS;
+}
+
+#include <support/test-driver.c>
diff --git a/sysdeps/x86_64/x32/tst-size_t-wcsncmp.c b/sysdeps/x86_64/x32/tst-size_t-wcsncmp.c
new file mode 100644
index 00000000..4829647c
--- /dev/null
+++ b/sysdeps/x86_64/x32/tst-size_t-wcsncmp.c
@@ -0,0 +1,20 @@
+/* Test wcsncmp with size_t in the lower 32 bits of 64-bit register.
+ Copyright (C) 2019 Free Software Foundation, Inc.
+ This file is part of the GNU C Library.
+
+ The GNU C Library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ The GNU C Library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with the GNU C Library; if not, see
+ <http://www.gnu.org/licenses/>. */
+
+#define WIDE 1
+#include "tst-size_t-strncmp.c"
--
GitLab

View File

@ -1,56 +0,0 @@
From cf2c57526ba4b57e6863ad4db8a868e2678adce8 Mon Sep 17 00:00:00 2001
From: "H.J. Lu" <hjl.tools@gmail.com>
Date: Fri, 30 Apr 2021 05:58:59 -0700
Subject: [PATCH] x86: Set rep_movsb_threshold to 2112 on processors with FSRM
Content-type: text/plain; charset=UTF-8
The glibc memcpy benchmark on Intel Core i7-1065G7 (Ice Lake) showed
that REP MOVSB became faster after 2112 bytes:
Vector Move REP MOVSB
length=2112, align1=0, align2=0: 24.20 24.40
length=2112, align1=1, align2=0: 26.07 23.13
length=2112, align1=0, align2=1: 27.18 28.13
length=2112, align1=1, align2=1: 26.23 25.16
length=2176, align1=0, align2=0: 23.18 22.52
length=2176, align1=2, align2=0: 25.45 22.52
length=2176, align1=0, align2=2: 27.14 27.82
length=2176, align1=2, align2=2: 22.73 25.56
length=2240, align1=0, align2=0: 24.62 24.25
length=2240, align1=3, align2=0: 29.77 27.15
length=2240, align1=0, align2=3: 35.55 29.93
length=2240, align1=3, align2=3: 34.49 25.15
length=2304, align1=0, align2=0: 34.75 26.64
length=2304, align1=4, align2=0: 32.09 22.63
length=2304, align1=0, align2=4: 28.43 31.24
Use REP MOVSB for data size > 2112 bytes in memcpy on processors with
fast short REP MOVSB (FSRM).
* sysdeps/x86/dl-cacheinfo.h (dl_init_cacheinfo): Set
rep_movsb_threshold to 2112 on processors with fast short REP
MOVSB (FSRM).
---
sysdeps/x86/cacheinfo.h | 6 ++++++
1 file changed, 6 insertions(+)
diff --git a/sysdeps/x86/cacheinfo.h b/sysdeps/x86/cacheinfo.h
index f72f634a..cc3941d3 100644
--- a/sysdeps/x86/cacheinfo.h
+++ b/sysdeps/x86/cacheinfo.h
@@ -430,6 +430,12 @@ init_cacheinfo (void)
rep_movsb_threshold = 2048 * (16 / 16);
minimum_rep_movsb_threshold = 16 * 8;
}
+
+ /* NB: The default REP MOVSB threshold is 2112 on processors with fast
+ short REP MOVSB (FSRM). */
+ if (CPU_FEATURE_USABLE_P (cpu_features, FSRM))
+ rep_movsb_threshold = 2112;
+
if (cpu_features->rep_movsb_threshold > minimum_rep_movsb_threshold)
__x86_rep_movsb_threshold = cpu_features->rep_movsb_threshold;
else
--
GitLab

View File

@ -1,51 +0,0 @@
From d672a98a1af106bd68deb15576710cd61363f7a6 Mon Sep 17 00:00:00 2001
From: "H.J. Lu" <hjl.tools@gmail.com>
Date: Tue, 2 Nov 2021 18:33:07 -0700
Subject: [PATCH] Add LLL_MUTEX_READ_LOCK [BZ #28537]
Content-type: text/plain; charset=UTF-8
CAS instruction is expensive. From the x86 CPU's point of view, getting
a cache line for writing is more expensive than reading. See Appendix
A.2 Spinlock in:
https://www.intel.com/content/dam/www/public/us/en/documents/white-papers/xeon-lock-scaling-analysis-paper.pdf
The full compare and swap will grab the cache line exclusive and cause
excessive cache line bouncing.
Add LLL_MUTEX_READ_LOCK to do an atomic load and skip CAS in spinlock
loop if compare may fail to reduce cache line bouncing on contended locks.
Reviewed-by: Szabolcs Nagy <szabolcs.nagy@arm.com>
---
nptl/pthread_mutex_lock.c | 7 +++++++
1 file changed, 7 insertions(+)
diff --git a/nptl/pthread_mutex_lock.c b/nptl/pthread_mutex_lock.c
index 60ada70d..eb4d8baa 100644
--- a/nptl/pthread_mutex_lock.c
+++ b/nptl/pthread_mutex_lock.c
@@ -56,6 +56,11 @@
#define FORCE_ELISION(m, s)
#endif
+#ifndef LLL_MUTEX_READ_LOCK
+# define LLL_MUTEX_READ_LOCK(mutex) \
+ atomic_load_relaxed (&(mutex)->__data.__lock)
+#endif
+
static int __pthread_mutex_lock_full (pthread_mutex_t *mutex)
__attribute_noinline__;
@@ -136,6 +141,8 @@ __pthread_mutex_lock (pthread_mutex_t *mutex)
break;
}
atomic_spin_nop ();
+ if (LLL_MUTEX_READ_LOCK (mutex) != 0)
+ continue;
}
while (LLL_MUTEX_TRYLOCK (mutex) != 0);
--
GitLab

View File

@ -1,71 +0,0 @@
From 120ac6d238825452e8024e2f627da33b2508dfd3 Mon Sep 17 00:00:00 2001
From: "H.J. Lu" <hjl.tools@gmail.com>
Date: Fri, 12 Nov 2021 11:47:42 -0800
Subject: [PATCH] Move assignment out of the CAS condition
Content-type: text/plain; charset=UTF-8
Update
commit 49302b8fdf9103b6fc0a398678668a22fa19574c
Author: H.J. Lu <hjl.tools@gmail.com>
Date: Thu Nov 11 06:54:01 2021 -0800
Avoid extra load with CAS in __pthread_mutex_clocklock_common [BZ #28537]
Replace boolean CAS with value CAS to avoid the extra load.
and
commit 0b82747dc48d5bf0871bdc6da8cb6eec1256355f
Author: H.J. Lu <hjl.tools@gmail.com>
Date: Thu Nov 11 06:31:51 2021 -0800
Avoid extra load with CAS in __pthread_mutex_lock_full [BZ #28537]
Replace boolean CAS with value CAS to avoid the extra load.
by moving assignment out of the CAS condition.
---
nptl/pthread_mutex_lock.c | 7 +++----
nptl/pthread_mutex_timedlock.c | 7 +++----
2 files changed, 6 insertions(+), 8 deletions(-)
diff --git a/nptl/pthread_mutex_lock.c b/nptl/pthread_mutex_lock.c
index eb4d8baa..a633d95e 100644
--- a/nptl/pthread_mutex_lock.c
+++ b/nptl/pthread_mutex_lock.c
@@ -299,10 +299,9 @@ __pthread_mutex_lock_full (pthread_mutex_t *mutex)
meantime. */
if ((oldval & FUTEX_WAITERS) == 0)
{
- int val;
- if ((val = atomic_compare_and_exchange_val_acq
- (&mutex->__data.__lock, oldval | FUTEX_WAITERS,
- oldval)) != oldval)
+ int val = atomic_compare_and_exchange_val_acq
+ (&mutex->__data.__lock, oldval | FUTEX_WAITERS, oldval);
+ if (val != oldval)
{
oldval = val;
continue;
diff --git a/nptl/pthread_mutex_timedlock.c b/nptl/pthread_mutex_timedlock.c
index c4627ef6..a76c30b7 100644
--- a/nptl/pthread_mutex_timedlock.c
+++ b/nptl/pthread_mutex_timedlock.c
@@ -269,10 +269,9 @@ __pthread_mutex_timedlock (pthread_mutex_t *mutex,
meantime. */
if ((oldval & FUTEX_WAITERS) == 0)
{
- int val;
- if ((val = atomic_compare_and_exchange_val_acq
- (&mutex->__data.__lock, oldval | FUTEX_WAITERS,
- oldval)) != oldval)
+ int val = atomic_compare_and_exchange_val_acq
+ (&mutex->__data.__lock, oldval | FUTEX_WAITERS, oldval);
+ if (val != oldval)
{
oldval = val;
continue;
--
GitLab

View File

@ -1,60 +0,0 @@
From 4df1fa6ddc8925a75f3da644d5da3bb16eb33f02 Mon Sep 17 00:00:00 2001
From: Noah Goldstein <goldstein.w.n@gmail.com>
Date: Fri, 3 Dec 2021 15:29:25 -0800
Subject: [PATCH] x86-64: Use notl in EVEX strcmp [BZ #28646]
Content-type: text/plain; charset=UTF-8
Must use notl %edi here as lower bits are for CHAR comparisons
potentially out of range thus can be 0 without indicating mismatch.
This fixes BZ #28646.
Co-Authored-By: H.J. Lu <hjl.tools@gmail.com>
---
sysdeps/x86_64/multiarch/strcmp-evex.S | 14 ++++++++------
1 file changed, 8 insertions(+), 6 deletions(-)
Conflicts:
string/test-strcmp.c
(new check omitted)
diff --git a/sysdeps/x86_64/multiarch/strcmp-evex.S b/sysdeps/x86_64/multiarch/strcmp-evex.S
index 82f12ac8..6f5c4bf9 100644
--- a/sysdeps/x86_64/multiarch/strcmp-evex.S
+++ b/sysdeps/x86_64/multiarch/strcmp-evex.S
@@ -656,12 +656,13 @@ L(loop_cross_page):
in YMM3 and 32 bytes at VEC_SIZE(%rdx, %r10). */
VPCMP $0, VEC_SIZE(%rdx, %r10), %YMM3, %k3{%k4}
kmovd %k3, %edi
+ /* Must use notl %edi here as lower bits are for CHAR
+ comparisons potentially out of range thus can be 0 without
+ indicating mismatch. */
+ notl %edi
# ifdef USE_AS_WCSCMP
/* Don't use subl since it is the upper 8 bits of EDI below. */
- notl %edi
andl $0xff, %edi
-# else
- incl %edi
# endif
# ifdef USE_AS_WCSCMP
@@ -743,12 +744,13 @@ L(loop_cross_page_2_vec):
in YMM1 and 32 bytes at (VEC_SIZE * 3)(%rdx, %r10). */
VPCMP $0, (VEC_SIZE * 3)(%rdx, %r10), %YMM1, %k3{%k4}
kmovd %k3, %edi
+ /* Must use notl %edi here as lower bits are for CHAR
+ comparisons potentially out of range thus can be 0 without
+ indicating mismatch. */
+ notl %edi
# ifdef USE_AS_WCSCMP
/* Don't use subl since it is the upper 8 bits of EDI below. */
- notl %edi
andl $0xff, %edi
-# else
- incl %edi
# endif
# ifdef USE_AS_WCSCMP
--
GitLab

View File

@ -1,153 +0,0 @@
From c7c54f65b080affb87a1513dee449c8ad6143c8b Mon Sep 17 00:00:00 2001
From: "H.J. Lu" <hjl.tools@gmail.com>
Date: Mon, 21 Jan 2019 11:35:18 -0800
Subject: [PATCH] x86-64 strncpy: Properly handle the length parameter [BZ#
24097]
Content-type: text/plain; charset=UTF-8
On x32, the size_t parameter may be passed in the lower 32 bits of a
64-bit register with the non-zero upper 32 bits. The string/memory
functions written in assembly can only use the lower 32 bits of a
64-bit register as length or must clear the upper 32 bits before using
the full 64-bit register for length.
This pach fixes strncpy for x32. Tested on x86-64 and x32. On x86-64,
libc.so is the same with and withou the fix.
[BZ# 24097]
CVE-2019-6488
* sysdeps/x86_64/multiarch/strcpy-avx2.S: Use RDX_LP for length.
* sysdeps/x86_64/multiarch/strcpy-sse2-unaligned.S: Likewise.
* sysdeps/x86_64/multiarch/strcpy-ssse3.S: Likewise.
* sysdeps/x86_64/x32/Makefile (tests): Add tst-size_t-strncpy.
* sysdeps/x86_64/x32/tst-size_t-strncpy.c: New file.
---
.../x86_64/multiarch/strcpy-sse2-unaligned.S | 4 +-
sysdeps/x86_64/multiarch/strcpy-ssse3.S | 6 +-
sysdeps/x86_64/x32/Makefile | 2 +-
sysdeps/x86_64/x32/tst-size_t-strncpy.c | 58 +++++++++++++++++++
4 files changed, 64 insertions(+), 6 deletions(-)
create mode 100644 sysdeps/x86_64/x32/tst-size_t-strncpy.c
Conflicts:
ChangeLog
(removed)
sysdeps/x86_64/multiarch/strcpy-avx2.S
(skipped, only needed for x32 arch)
diff --git a/sysdeps/x86_64/multiarch/strcpy-sse2-unaligned.S b/sysdeps/x86_64/multiarch/strcpy-sse2-unaligned.S
index 72bf7e85..50aca22d 100644
--- a/sysdeps/x86_64/multiarch/strcpy-sse2-unaligned.S
+++ b/sysdeps/x86_64/multiarch/strcpy-sse2-unaligned.S
@@ -40,8 +40,8 @@
.text
ENTRY (STRCPY)
# ifdef USE_AS_STRNCPY
- mov %rdx, %r8
- test %r8, %r8
+ mov %RDX_LP, %R8_LP
+ test %R8_LP, %R8_LP
jz L(ExitZero)
# endif
mov %rsi, %rcx
diff --git a/sysdeps/x86_64/multiarch/strcpy-ssse3.S b/sysdeps/x86_64/multiarch/strcpy-ssse3.S
index 9858d0c4..0a62814a 100644
--- a/sysdeps/x86_64/multiarch/strcpy-ssse3.S
+++ b/sysdeps/x86_64/multiarch/strcpy-ssse3.S
@@ -31,13 +31,13 @@ ENTRY (STRCPY)
mov %rsi, %rcx
# ifdef USE_AS_STRNCPY
- mov %rdx, %r8
+ mov %RDX_LP, %R8_LP
# endif
mov %rdi, %rdx
# ifdef USE_AS_STRNCPY
- test %r8, %r8
+ test %R8_LP, %R8_LP
jz L(Exit0)
- cmp $8, %r8
+ cmp $8, %R8_LP
jbe L(StrncpyExit8Bytes)
# endif
cmpb $0, (%rcx)
diff --git a/sysdeps/x86_64/x32/Makefile b/sysdeps/x86_64/x32/Makefile
index db302839..2a9e20a9 100644
--- a/sysdeps/x86_64/x32/Makefile
+++ b/sysdeps/x86_64/x32/Makefile
@@ -8,7 +8,7 @@ endif
ifeq ($(subdir),string)
tests += tst-size_t-memchr tst-size_t-memcmp tst-size_t-memcpy \
tst-size_t-memrchr tst-size_t-memset tst-size_t-strncasecmp \
- tst-size_t-strncmp
+ tst-size_t-strncmp tst-size_t-strncpy
endif
ifeq ($(subdir),wcsmbs)
diff --git a/sysdeps/x86_64/x32/tst-size_t-strncpy.c b/sysdeps/x86_64/x32/tst-size_t-strncpy.c
new file mode 100644
index 00000000..4dec71e6
--- /dev/null
+++ b/sysdeps/x86_64/x32/tst-size_t-strncpy.c
@@ -0,0 +1,58 @@
+/* Test strncpy with size_t in the lower 32 bits of 64-bit register.
+ Copyright (C) 2019 Free Software Foundation, Inc.
+ This file is part of the GNU C Library.
+
+ The GNU C Library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ The GNU C Library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with the GNU C Library; if not, see
+ <http://www.gnu.org/licenses/>. */
+
+#define TEST_NAME "strncpy"
+#include "test-size_t.h"
+
+IMPL (strncpy, 1)
+
+typedef char *(*proto_t) (char *, const char*, size_t);
+
+static void *
+__attribute__ ((noinline, noclone))
+do_strncpy (parameter_t a, parameter_t b)
+{
+ return CALL (&b, a.p, b.p, a.len);
+}
+
+static int
+test_main (void)
+{
+ test_init ();
+
+ parameter_t dest = { { page_size }, buf1 };
+ parameter_t src = { { 0 }, buf2 };
+
+ int ret = 0;
+ FOR_EACH_IMPL (impl, 0)
+ {
+ src.fn = impl->fn;
+ do_strncpy (dest, src);
+ int res = strncmp (dest.p, src.p, dest.len);
+ if (res)
+ {
+ error (0, 0, "Wrong result in function %s: %i != 0",
+ impl->name, res);
+ ret = 1;
+ }
+ }
+
+ return ret ? EXIT_FAILURE : EXIT_SUCCESS;
+}
+
+#include <support/test-driver.c>
--
GitLab

View File

@ -1,43 +0,0 @@
From 6b8dbbd03ac88f169b65b5c7d7278576a11d2e44 Mon Sep 17 00:00:00 2001
From: Jangwoong Kim <6812skiii@gmail.com>
Date: Tue, 14 Dec 2021 21:30:51 +0900
Subject: [PATCH] nptl: Effectively skip CAS in spinlock loop
Content-type: text/plain; charset=UTF-8
The commit:
"Add LLL_MUTEX_READ_LOCK [BZ #28537]"
SHA1: d672a98a1af106bd68deb15576710cd61363f7a6
introduced LLL_MUTEX_READ_LOCK, to skip CAS in spinlock loop
if atomic load fails. But, "continue" inside of do-while loop
does not skip the evaluation of escape expression, thus CAS
is not skipped.
Replace do-while with while and skip LLL_MUTEX_TRYLOCK if
LLL_MUTEX_READ_LOCK fails.
Reviewed-by: H.J. Lu <hjl.tools@gmail.com>
---
nptl/pthread_mutex_lock.c | 5 ++---
1 file changed, 2 insertions(+), 3 deletions(-)
diff --git a/nptl/pthread_mutex_lock.c b/nptl/pthread_mutex_lock.c
index a633d95e..d96a9933 100644
--- a/nptl/pthread_mutex_lock.c
+++ b/nptl/pthread_mutex_lock.c
@@ -141,10 +141,9 @@ __pthread_mutex_lock (pthread_mutex_t *mutex)
break;
}
atomic_spin_nop ();
- if (LLL_MUTEX_READ_LOCK (mutex) != 0)
- continue;
}
- while (LLL_MUTEX_TRYLOCK (mutex) != 0);
+ while (LLL_MUTEX_READ_LOCK (mutex) != 0
+ || LLL_MUTEX_TRYLOCK (mutex) != 0);
mutex->__data.__spins += (cnt - mutex->__data.__spins) / 8;
}
--
GitLab

View File

@ -1,37 +0,0 @@
From b98d0bbf747f39770e0caba7e984ce9f8f900330 Mon Sep 17 00:00:00 2001
From: Noah Goldstein <goldstein.w.n@gmail.com>
Date: Fri, 18 Feb 2022 17:00:25 -0600
Subject: [PATCH] x86: Fix TEST_NAME to make it a string in tst-strncmp-rtm.c
Content-type: text/plain; charset=UTF-8
Previously TEST_NAME was passing a function pointer. This didn't fail
because of the -Wno-error flag (to allow for overflow sizes passed
to strncmp/wcsncmp)
Reviewed-by: H.J. Lu <hjl.tools@gmail.com>
---
sysdeps/x86/tst-strncmp-rtm.c | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/sysdeps/x86/tst-strncmp-rtm.c b/sysdeps/x86/tst-strncmp-rtm.c
index 4e9f094f..aef9866c 100644
--- a/sysdeps/x86/tst-strncmp-rtm.c
+++ b/sysdeps/x86/tst-strncmp-rtm.c
@@ -23,12 +23,12 @@
# define CHAR wchar_t
# define MEMSET wmemset
# define STRNCMP wcsncmp
-# define TEST_NAME wcsncmp
+# define TEST_NAME "wcsncmp"
#else /* !WIDE */
# define CHAR char
# define MEMSET memset
# define STRNCMP strncmp
-# define TEST_NAME strncmp
+# define TEST_NAME "strncmp"
#endif /* !WIDE */
--
GitLab

View File

@ -1,33 +0,0 @@
From c15efd011cea3d8f0494269eb539583215a1feed Mon Sep 17 00:00:00 2001
From: "H.J. Lu" <hjl.tools@gmail.com>
Date: Fri, 4 Feb 2022 11:09:10 -0800
Subject: [PATCH] x86-64: Fix strcmp-avx2.S
Content-type: text/plain; charset=UTF-8
Change "movl %edx, %rdx" to "movl %edx, %edx" in:
commit b77b06e0e296f1a2276c27a67e1d44f2cfa38d45
Author: Noah Goldstein <goldstein.w.n@gmail.com>
Date: Mon Jan 10 15:35:38 2022 -0600
x86: Optimize strcmp-avx2.S
---
sysdeps/x86_64/multiarch/strcmp-avx2.S | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/sysdeps/x86_64/multiarch/strcmp-avx2.S b/sysdeps/x86_64/multiarch/strcmp-avx2.S
index 554ffe4c..04675aa4 100644
--- a/sysdeps/x86_64/multiarch/strcmp-avx2.S
+++ b/sysdeps/x86_64/multiarch/strcmp-avx2.S
@@ -106,7 +106,7 @@ ENTRY(STRCMP)
# ifdef USE_AS_STRNCMP
# ifdef __ILP32__
/* Clear the upper 32 bits. */
- movl %edx, %rdx
+ movl %edx, %edx
# endif
cmp $1, %RDX_LP
/* Signed comparison intentional. We use this branch to also
--
GitLab

View File

@ -1,33 +0,0 @@
From 0e0199a9e02ebe42e2b36958964d63f03573c382 Mon Sep 17 00:00:00 2001
From: "H.J. Lu" <hjl.tools@gmail.com>
Date: Fri, 4 Feb 2022 11:11:08 -0800
Subject: [PATCH] x86-64: Fix strcmp-evex.S
Content-type: text/plain; charset=UTF-8
Change "movl %edx, %rdx" to "movl %edx, %edx" in:
commit 8418eb3ff4b781d31c4ed5dc6c0bd7356bc45db9
Author: Noah Goldstein <goldstein.w.n@gmail.com>
Date: Mon Jan 10 15:35:39 2022 -0600
x86: Optimize strcmp-evex.S
---
sysdeps/x86_64/multiarch/strcmp-evex.S | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/sysdeps/x86_64/multiarch/strcmp-evex.S b/sysdeps/x86_64/multiarch/strcmp-evex.S
index 99d8409a..ed56af8e 100644
--- a/sysdeps/x86_64/multiarch/strcmp-evex.S
+++ b/sysdeps/x86_64/multiarch/strcmp-evex.S
@@ -116,7 +116,7 @@ ENTRY(STRCMP)
# ifdef USE_AS_STRNCMP
# ifdef __ILP32__
/* Clear the upper 32 bits. */
- movl %edx, %rdx
+ movl %edx, %edx
# endif
cmp $1, %RDX_LP
/* Signed comparison intentional. We use this branch to also
--
GitLab

View File

@ -1,40 +0,0 @@
From 1b0c60f95bbe2eded80b2bb5be75c0e45b11cde1 Mon Sep 17 00:00:00 2001
From: Noah Goldstein <goldstein.w.n@gmail.com>
Date: Mon, 7 Feb 2022 00:32:23 -0600
Subject: [PATCH] x86: Remove SSSE3 instruction for broadcast in memset.S (SSE2
Only)
Content-type: text/plain; charset=UTF-8
commit b62ace2740a106222e124cc86956448fa07abf4d
Author: Noah Goldstein <goldstein.w.n@gmail.com>
Date: Sun Feb 6 00:54:18 2022 -0600
x86: Improve vec generation in memset-vec-unaligned-erms.S
Revert usage of 'pshufb' in broadcast logic as it is an SSSE3
instruction and memset.S is restricted to only SSE2 instructions.
---
sysdeps/x86_64/memset.S | 7 ++++---
1 file changed, 4 insertions(+), 3 deletions(-)
diff --git a/sysdeps/x86_64/memset.S b/sysdeps/x86_64/memset.S
index 27debd2b..4cb4aa71 100644
--- a/sysdeps/x86_64/memset.S
+++ b/sysdeps/x86_64/memset.S
@@ -30,9 +30,10 @@
# define MEMSET_SET_VEC0_AND_SET_RETURN(d, r) \
movd d, %xmm0; \
- pxor %xmm1, %xmm1; \
- pshufb %xmm1, %xmm0; \
- movq r, %rax
+ movq r, %rax; \
+ punpcklbw %xmm0, %xmm0; \
+ punpcklwd %xmm0, %xmm0; \
+ pshufd $0, %xmm0, %xmm0
# define WMEMSET_SET_VEC0_AND_SET_RETURN(d, r) \
movd d, %xmm0; \
--
GitLab

View File

@ -1,218 +0,0 @@
From 5165de69c0908e28a380cbd4bb054e55ea4abc95 Mon Sep 17 00:00:00 2001
From: "H.J. Lu" <hjl.tools@gmail.com>
Date: Mon, 21 Jan 2019 11:36:36 -0800
Subject: [PATCH] x86-64 strnlen/wcsnlen: Properly handle the length parameter
[BZ# 24097]
Content-type: text/plain; charset=UTF-8
On x32, the size_t parameter may be passed in the lower 32 bits of a
64-bit register with the non-zero upper 32 bits. The string/memory
functions written in assembly can only use the lower 32 bits of a
64-bit register as length or must clear the upper 32 bits before using
the full 64-bit register for length.
This pach fixes strnlen/wcsnlen for x32. Tested on x86-64 and x32. On
x86-64, libc.so is the same with and withou the fix.
[BZ# 24097]
CVE-2019-6488
* sysdeps/x86_64/multiarch/strlen-avx2.S: Use RSI_LP for length.
Clear the upper 32 bits of RSI register.
* sysdeps/x86_64/strlen.S: Use RSI_LP for length.
* sysdeps/x86_64/x32/Makefile (tests): Add tst-size_t-strnlen
and tst-size_t-wcsnlen.
* sysdeps/x86_64/x32/tst-size_t-strnlen.c: New file.
* sysdeps/x86_64/x32/tst-size_t-wcsnlen.c: Likewise.
---
sysdeps/x86_64/multiarch/strlen-avx2.S | 9 ++--
sysdeps/x86_64/strlen.S | 12 ++---
sysdeps/x86_64/x32/Makefile | 4 +-
sysdeps/x86_64/x32/tst-size_t-strnlen.c | 72 +++++++++++++++++++++++++
sysdeps/x86_64/x32/tst-size_t-wcsnlen.c | 20 +++++++
5 files changed, 106 insertions(+), 11 deletions(-)
create mode 100644 sysdeps/x86_64/x32/tst-size_t-strnlen.c
create mode 100644 sysdeps/x86_64/x32/tst-size_t-wcsnlen.c
Conflicts:
ChangeLog
(removed)
diff --git a/sysdeps/x86_64/multiarch/strlen-avx2.S b/sysdeps/x86_64/multiarch/strlen-avx2.S
index fb2418cd..645e0446 100644
--- a/sysdeps/x86_64/multiarch/strlen-avx2.S
+++ b/sysdeps/x86_64/multiarch/strlen-avx2.S
@@ -42,12 +42,15 @@
ENTRY (STRLEN)
# ifdef USE_AS_STRNLEN
/* Check for zero length. */
- testq %rsi, %rsi
+ test %RSI_LP, %RSI_LP
jz L(zero)
# ifdef USE_AS_WCSLEN
- shl $2, %rsi
+ shl $2, %RSI_LP
+# elif defined __ILP32__
+ /* Clear the upper 32 bits. */
+ movl %esi, %esi
# endif
- movq %rsi, %r8
+ mov %RSI_LP, %R8_LP
# endif
movl %edi, %ecx
movq %rdi, %rdx
diff --git a/sysdeps/x86_64/strlen.S b/sysdeps/x86_64/strlen.S
index 01cb5fa8..f845f3d4 100644
--- a/sysdeps/x86_64/strlen.S
+++ b/sysdeps/x86_64/strlen.S
@@ -59,21 +59,21 @@ ENTRY(strlen)
#ifdef AS_STRNLEN
/* Do not read anything when n==0. */
- test %rsi, %rsi
+ test %RSI_LP, %RSI_LP
jne L(n_nonzero)
xor %rax, %rax
ret
L(n_nonzero):
# ifdef AS_WCSLEN
- shlq $2, %rsi
+ shl $2, %RSI_LP
# endif
/* Initialize long lived registers. */
- add %rdi, %rsi
- mov %rsi, %r10
- and $-64, %r10
- mov %rsi, %r11
+ add %RDI_LP, %RSI_LP
+ mov %RSI_LP, %R10_LP
+ and $-64, %R10_LP
+ mov %RSI_LP, %R11_LP
#endif
pxor %xmm0, %xmm0
diff --git a/sysdeps/x86_64/x32/Makefile b/sysdeps/x86_64/x32/Makefile
index 2a9e20a9..1557724b 100644
--- a/sysdeps/x86_64/x32/Makefile
+++ b/sysdeps/x86_64/x32/Makefile
@@ -8,10 +8,10 @@ endif
ifeq ($(subdir),string)
tests += tst-size_t-memchr tst-size_t-memcmp tst-size_t-memcpy \
tst-size_t-memrchr tst-size_t-memset tst-size_t-strncasecmp \
- tst-size_t-strncmp tst-size_t-strncpy
+ tst-size_t-strncmp tst-size_t-strncpy tst-size_t-strnlen
endif
ifeq ($(subdir),wcsmbs)
tests += tst-size_t-wmemchr tst-size_t-wmemcmp tst-size_t-wmemset \
- tst-size_t-wcsncmp
+ tst-size_t-wcsncmp tst-size_t-wcsnlen
endif
diff --git a/sysdeps/x86_64/x32/tst-size_t-strnlen.c b/sysdeps/x86_64/x32/tst-size_t-strnlen.c
new file mode 100644
index 00000000..690a4a8a
--- /dev/null
+++ b/sysdeps/x86_64/x32/tst-size_t-strnlen.c
@@ -0,0 +1,72 @@
+/* Test strnlen with size_t in the lower 32 bits of 64-bit register.
+ Copyright (C) 2019 Free Software Foundation, Inc.
+ This file is part of the GNU C Library.
+
+ The GNU C Library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ The GNU C Library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with the GNU C Library; if not, see
+ <http://www.gnu.org/licenses/>. */
+
+#ifdef WIDE
+# define TEST_NAME "wcsnlen"
+#else
+# define TEST_NAME "strnlen"
+#endif /* WIDE */
+
+#include "test-size_t.h"
+
+#ifdef WIDE
+# include <wchar.h>
+# define STRNLEN wcsnlen
+# define CHAR wchar_t
+#else
+# define STRNLEN strnlen
+# define CHAR char
+#endif /* WIDE */
+
+IMPL (STRNLEN, 1)
+
+typedef size_t (*proto_t) (const CHAR *, size_t);
+
+static size_t
+__attribute__ ((noinline, noclone))
+do_strnlen (parameter_t a, parameter_t b)
+{
+ return CALL (&a, a.p, b.len);
+}
+
+static int
+test_main (void)
+{
+ test_init ();
+
+ size_t size = page_size / sizeof (CHAR);
+ parameter_t src = { { 0 }, buf2 };
+ parameter_t c = { { size }, (void *) (uintptr_t) 'a' };
+
+ int ret = 0;
+ FOR_EACH_IMPL (impl, 0)
+ {
+ src.fn = impl->fn;
+ size_t res = do_strnlen (src, c);
+ if (res != size)
+ {
+ error (0, 0, "Wrong result in function %s: 0x%x != 0x%x",
+ impl->name, res, size);
+ ret = 1;
+ }
+ }
+
+ return ret ? EXIT_FAILURE : EXIT_SUCCESS;
+}
+
+#include <support/test-driver.c>
diff --git a/sysdeps/x86_64/x32/tst-size_t-wcsnlen.c b/sysdeps/x86_64/x32/tst-size_t-wcsnlen.c
new file mode 100644
index 00000000..093b4bbe
--- /dev/null
+++ b/sysdeps/x86_64/x32/tst-size_t-wcsnlen.c
@@ -0,0 +1,20 @@
+/* Test wcsnlen with size_t in the lower 32 bits of 64-bit register.
+ Copyright (C) 2019 Free Software Foundation, Inc.
+ This file is part of the GNU C Library.
+
+ The GNU C Library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ The GNU C Library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with the GNU C Library; if not, see
+ <http://www.gnu.org/licenses/>. */
+
+#define WIDE 1
+#include "tst-size_t-strnlen.c"
--
GitLab

View File

@ -1,33 +0,0 @@
From 7912236f4a597deb092650ca79f33504ddb4af28 Mon Sep 17 00:00:00 2001
From: Noah Goldstein <goldstein.w.n@gmail.com>
Date: Sat, 12 Feb 2022 00:45:00 -0600
Subject: [PATCH] x86: Set .text section in memset-vec-unaligned-erms
Content-type: text/plain; charset=UTF-8
commit 3d9f171bfb5325bd5f427e9fc386453358c6e840
Author: H.J. Lu <hjl.tools@gmail.com>
Date: Mon Feb 7 05:55:15 2022 -0800
x86-64: Optimize bzero
Remove setting the .text section for the code. This commit
adds that back.
---
sysdeps/x86_64/multiarch/memset-vec-unaligned-erms.S | 1 +
1 file changed, 1 insertion(+)
diff --git a/sysdeps/x86_64/multiarch/memset-vec-unaligned-erms.S b/sysdeps/x86_64/multiarch/memset-vec-unaligned-erms.S
index 06f5f5d7..4fb475c0 100644
--- a/sysdeps/x86_64/multiarch/memset-vec-unaligned-erms.S
+++ b/sysdeps/x86_64/multiarch/memset-vec-unaligned-erms.S
@@ -114,6 +114,7 @@
# error SECTION is not defined!
#endif
+ .section SECTION(.text), "ax", @progbits
#if IS_IN (libc)
# if defined SHARED
ENTRY_CHK (WMEMSET_CHK_SYMBOL (__wmemset_chk, unaligned))
--
GitLab

View File

@ -1,36 +0,0 @@
From 0fb8800029d230b3711bf722b2a47db92d0e273f Mon Sep 17 00:00:00 2001
From: "H.J. Lu" <hjl.tools@gmail.com>
Date: Thu, 10 Feb 2022 11:52:50 -0800
Subject: [PATCH] x86-64: Remove bzero weak alias in SS2 memset
Content-type: text/plain; charset=UTF-8
commit 3d9f171bfb5325bd5f427e9fc386453358c6e840
Author: H.J. Lu <hjl.tools@gmail.com>
Date: Mon Feb 7 05:55:15 2022 -0800
x86-64: Optimize bzero
added the optimized bzero. Remove bzero weak alias in SS2 memset to
avoid undefined __bzero in memset-sse2-unaligned-erms.
---
sysdeps/x86_64/multiarch/memset-sse2-unaligned-erms.S | 4 +---
1 file changed, 1 insertion(+), 3 deletions(-)
diff --git a/sysdeps/x86_64/multiarch/memset-sse2-unaligned-erms.S b/sysdeps/x86_64/multiarch/memset-sse2-unaligned-erms.S
index 8f579ad6..af51362b 100644
--- a/sysdeps/x86_64/multiarch/memset-sse2-unaligned-erms.S
+++ b/sysdeps/x86_64/multiarch/memset-sse2-unaligned-erms.S
@@ -31,9 +31,7 @@
# endif
# undef weak_alias
-# define weak_alias(original, alias) \
- .weak bzero; bzero = __bzero
-
+# define weak_alias(original, alias)
# undef strong_alias
# define strong_alias(ignored1, ignored2)
#endif
--
GitLab

View File

@ -1,29 +0,0 @@
From bf92893a14ebc161b08b28acc24fa06ae6be19cb Mon Sep 17 00:00:00 2001
From: Adhemerval Zanella <adhemerval.zanella@linaro.org>
Date: Thu, 10 Feb 2022 11:23:24 -0300
Subject: [PATCH] x86_64: Remove bcopy optimizations
Content-type: text/plain; charset=UTF-8
The symbols is not present in current POSIX specification and compiler
already generates memmove call.
---
sysdeps/x86_64/multiarch/bcopy.S | 7 -------
1 file changed, 7 deletions(-)
delete mode 100644 sysdeps/x86_64/multiarch/bcopy.S
diff --git a/sysdeps/x86_64/multiarch/bcopy.S b/sysdeps/x86_64/multiarch/bcopy.S
deleted file mode 100644
index 639f02bd..00000000
--- a/sysdeps/x86_64/multiarch/bcopy.S
+++ /dev/null
@@ -1,7 +0,0 @@
-#include <sysdep.h>
-
- .text
-ENTRY(bcopy)
- xchg %rdi, %rsi
- jmp __libc_memmove /* Branch to IFUNC memmove. */
-END(bcopy)
--
GitLab

View File

@ -1,206 +0,0 @@
From 3f635fb43389b54f682fc9ed2acc0b2aaf4a923d Mon Sep 17 00:00:00 2001
From: "H.J. Lu" <hjl.tools@gmail.com>
Date: Mon, 4 Feb 2019 06:31:01 -0800
Subject: [PATCH] x86-64 memcmp: Use unsigned Jcc instructions on size [BZ
#24155]
Content-type: text/plain; charset=UTF-8
Since the size argument is unsigned. we should use unsigned Jcc
instructions, instead of signed, to check size.
Tested on x86-64 and x32, with and without --disable-multi-arch.
[BZ #24155]
CVE-2019-7309
* NEWS: Updated for CVE-2019-7309.
* sysdeps/x86_64/memcmp.S: Use RDX_LP for size. Clear the
upper 32 bits of RDX register for x32. Use unsigned Jcc
instructions, instead of signed.
* sysdeps/x86_64/x32/Makefile (tests): Add tst-size_t-memcmp-2.
* sysdeps/x86_64/x32/tst-size_t-memcmp-2.c: New test.
---
sysdeps/x86_64/memcmp.S | 20 +++---
sysdeps/x86_64/x32/Makefile | 3 +-
sysdeps/x86_64/x32/tst-size_t-memcmp-2.c | 79 ++++++++++++++++++++++++
3 files changed, 93 insertions(+), 9 deletions(-)
create mode 100644 sysdeps/x86_64/x32/tst-size_t-memcmp-2.c
Conflics:
ChangeLog
(removed)
NEWS
(removed)
diff --git a/sysdeps/x86_64/memcmp.S b/sysdeps/x86_64/memcmp.S
index bcb4a2e8..45918d37 100644
--- a/sysdeps/x86_64/memcmp.S
+++ b/sysdeps/x86_64/memcmp.S
@@ -21,14 +21,18 @@
.text
ENTRY (memcmp)
- test %rdx, %rdx
+#ifdef __ILP32__
+ /* Clear the upper 32 bits. */
+ movl %edx, %edx
+#endif
+ test %RDX_LP, %RDX_LP
jz L(finz)
cmpq $1, %rdx
- jle L(finr1b)
+ jbe L(finr1b)
subq %rdi, %rsi
movq %rdx, %r10
cmpq $32, %r10
- jge L(gt32)
+ jae L(gt32)
/* Handle small chunks and last block of less than 32 bytes. */
L(small):
testq $1, %r10
@@ -156,7 +160,7 @@ L(A32):
movq %r11, %r10
andq $-32, %r10
cmpq %r10, %rdi
- jge L(mt16)
+ jae L(mt16)
/* Pre-unroll to be ready for unrolled 64B loop. */
testq $32, %rdi
jz L(A64)
@@ -178,7 +182,7 @@ L(A64):
movq %r11, %r10
andq $-64, %r10
cmpq %r10, %rdi
- jge L(mt32)
+ jae L(mt32)
L(A64main):
movdqu (%rdi,%rsi), %xmm0
@@ -216,7 +220,7 @@ L(mt32):
movq %r11, %r10
andq $-32, %r10
cmpq %r10, %rdi
- jge L(mt16)
+ jae L(mt16)
L(A32main):
movdqu (%rdi,%rsi), %xmm0
@@ -254,7 +258,7 @@ L(ATR):
movq %r11, %r10
andq $-32, %r10
cmpq %r10, %rdi
- jge L(mt16)
+ jae L(mt16)
testq $16, %rdi
jz L(ATR32)
@@ -325,7 +329,7 @@ L(ATR64main):
movq %r11, %r10
andq $-32, %r10
cmpq %r10, %rdi
- jge L(mt16)
+ jae L(mt16)
L(ATR32res):
movdqa (%rdi,%rsi), %xmm0
diff --git a/sysdeps/x86_64/x32/Makefile b/sysdeps/x86_64/x32/Makefile
index 1557724b..87489565 100644
--- a/sysdeps/x86_64/x32/Makefile
+++ b/sysdeps/x86_64/x32/Makefile
@@ -8,7 +8,8 @@ endif
ifeq ($(subdir),string)
tests += tst-size_t-memchr tst-size_t-memcmp tst-size_t-memcpy \
tst-size_t-memrchr tst-size_t-memset tst-size_t-strncasecmp \
- tst-size_t-strncmp tst-size_t-strncpy tst-size_t-strnlen
+ tst-size_t-strncmp tst-size_t-strncpy tst-size_t-strnlen \
+ tst-size_t-memcmp-2
endif
ifeq ($(subdir),wcsmbs)
diff --git a/sysdeps/x86_64/x32/tst-size_t-memcmp-2.c b/sysdeps/x86_64/x32/tst-size_t-memcmp-2.c
new file mode 100644
index 00000000..d8ae1a08
--- /dev/null
+++ b/sysdeps/x86_64/x32/tst-size_t-memcmp-2.c
@@ -0,0 +1,79 @@
+/* Test memcmp with size_t in the lower 32 bits of 64-bit register.
+ Copyright (C) 2019 Free Software Foundation, Inc.
+ This file is part of the GNU C Library.
+
+ The GNU C Library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ The GNU C Library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with the GNU C Library; if not, see
+ <http://www.gnu.org/licenses/>. */
+
+#define TEST_MAIN
+#ifdef WIDE
+# define TEST_NAME "wmemcmp"
+#else
+# define TEST_NAME "memcmp"
+#endif
+
+#include "test-size_t.h"
+
+#ifdef WIDE
+# include <inttypes.h>
+# include <wchar.h>
+
+# define MEMCMP wmemcmp
+# define CHAR wchar_t
+#else
+# define MEMCMP memcmp
+# define CHAR char
+#endif
+
+IMPL (MEMCMP, 1)
+
+typedef int (*proto_t) (const CHAR *, const CHAR *, size_t);
+
+static int
+__attribute__ ((noinline, noclone))
+do_memcmp (parameter_t a, parameter_t b)
+{
+ return CALL (&b, a.p, b.p, a.len);
+}
+
+static int
+test_main (void)
+{
+ test_init ();
+
+ parameter_t dest = { { page_size / sizeof (CHAR) }, buf1 };
+ parameter_t src = { { 0 }, buf2 };
+
+ memcpy (buf1, buf2, page_size);
+
+ CHAR *p = (CHAR *) buf1;
+ p[page_size / sizeof (CHAR) - 1] = (CHAR) 1;
+
+ int ret = 0;
+ FOR_EACH_IMPL (impl, 0)
+ {
+ src.fn = impl->fn;
+ int res = do_memcmp (dest, src);
+ if (res >= 0)
+ {
+ error (0, 0, "Wrong result in function %s: %i >= 0",
+ impl->name, res);
+ ret = 1;
+ }
+ }
+
+ return ret ? EXIT_FAILURE : EXIT_SUCCESS;
+}
+
+#include <support/test-driver.c>
--
GitLab

View File

@ -1,77 +0,0 @@
commit 7ac6fad987c28dc5e0e7964084260f163179be68
Author: Florian Weimer <fweimer@redhat.com>
Date: Wed May 15 07:16:47 2019 +0200
iconv: Remove public declaration of __gconv_transliterate
Commit ba7b4d294b01870ce3497971e9d07ee261cdc540 ("Complete the
removal of __gconv_translit_find") added a declaration of the
GLIBC_PRIVATE function, __gconv_transliterate, to the installed
header <gconv.h>. It should have been added to the internal
<gconv_int.h> header.
Reviewed-by: Carlos O'Donell <carlos@redhat.com>
diff --git a/iconv/gconv.h b/iconv/gconv.h
index e53f4dba0e3530fc..9dcc509c12c4a343 100644
--- a/iconv/gconv.h
+++ b/iconv/gconv.h
@@ -142,13 +142,4 @@ typedef struct __gconv_info
__extension__ struct __gconv_step_data __data[0];
} *__gconv_t;
-/* Transliteration using the locale's data. */
-extern int __gconv_transliterate (struct __gconv_step *step,
- struct __gconv_step_data *step_data,
- const unsigned char *inbufstart,
- const unsigned char **inbufp,
- const unsigned char *inbufend,
- unsigned char **outbufstart,
- size_t *irreversible);
-
#endif /* gconv.h */
diff --git a/iconv/gconv_int.h b/iconv/gconv_int.h
index 8067a341b0903e1b..6f908f283720520a 100644
--- a/iconv/gconv_int.h
+++ b/iconv/gconv_int.h
@@ -276,6 +276,14 @@ extern void __gconv_get_builtin_trans (const char *name,
struct __gconv_step *step)
attribute_hidden;
+/* Transliteration using the locale's data. */
+extern int __gconv_transliterate (struct __gconv_step *step,
+ struct __gconv_step_data *step_data,
+ const unsigned char *inbufstart,
+ const unsigned char **inbufp,
+ const unsigned char *inbufend,
+ unsigned char **outbufstart,
+ size_t *irreversible);
libc_hidden_proto (__gconv_transliterate)
/* If NAME is an codeset alias expand it. */
diff --git a/iconv/loop.c b/iconv/loop.c
index d571b593c7b8da93..4adbe04ae3698411 100644
--- a/iconv/loop.c
+++ b/iconv/loop.c
@@ -49,7 +49,7 @@
#include <assert.h>
#include <endian.h>
-#include <gconv.h>
+#include <iconv/gconv_int.h>
#include <stdint.h>
#include <string.h>
#include <wchar.h>
diff --git a/iconv/skeleton.c b/iconv/skeleton.c
index fa799305236001c7..9eb0200cc50a0afc 100644
--- a/iconv/skeleton.c
+++ b/iconv/skeleton.c
@@ -133,7 +133,7 @@
*/
#include <assert.h>
-#include <gconv.h>
+#include <iconv/gconv_int.h>
#include <string.h>
#define __need_size_t
#define __need_NULL

View File

@ -1,369 +0,0 @@
commit 5729e0e9af590807df66a3db688008f9547bce9f
Author: Adhemerval Zanella <adhemerval.zanella@linaro.org>
Date: Fri Feb 10 14:09:10 2023 -0300
iconv: Remove _STRING_ARCH_unaligned usage for get/set macros
And use a packed structure instead. The compiler generates optimized
unaligned code if the architecture supports it.
Checked on x86_64-linux-gnu and i686-linux-gnu.
Reviewed-by: Wilco Dijkstra <Wilco.Dijkstra@arm.com>
diff --git a/iconv/gconv_int.h b/iconv/gconv_int.h
index 6f908f283720520a..ebaf3518a95219f0 100644
--- a/iconv/gconv_int.h
+++ b/iconv/gconv_int.h
@@ -27,6 +27,34 @@
__BEGIN_DECLS
+/* We have to provide support for machines which are not able to handled
+ unaligned memory accesses. Some of the character encodings have
+ representations with a fixed width of 2 or 4 bytes. */
+#define get16(addr) \
+({ \
+ const struct { uint16_t r; } __attribute__ ((__packed__)) *__ptr \
+ = (__typeof(__ptr))(addr); \
+ __ptr->r; \
+})
+#define get32(addr) \
+({ \
+ const struct { uint32_t r; } __attribute__ ((__packed__)) *__ptr \
+ = (__typeof(__ptr))(addr); \
+ __ptr->r; \
+})
+
+#define put16(addr, val) \
+do { \
+ struct { uint16_t r; } __attribute__ ((__packed__)) *__ptr \
+ = (__typeof(__ptr))(addr); \
+ __ptr->r = val; \
+} while (0)
+#define put32(addr, val) \
+do { \
+ struct { uint32_t r; } __attribute__ ((__packed__)) *__ptr \
+ = (__typeof(__ptr))(addr); \
+ __ptr->r = val; \
+} while (0)
/* Type to represent search path. */
struct path_elem
diff --git a/iconv/loop.c b/iconv/loop.c
index 4adbe04ae3698411..a9aa201eadc07bd9 100644
--- a/iconv/loop.c
+++ b/iconv/loop.c
@@ -58,75 +58,10 @@
#include <stddef.h>
#include <libc-diag.h>
-/* We have to provide support for machines which are not able to handled
- unaligned memory accesses. Some of the character encodings have
- representations with a fixed width of 2 or 4 bytes. But if we cannot
- access unaligned memory we still have to read byte-wise. */
#undef FCTNAME2
#if _STRING_ARCH_unaligned || !defined DEFINE_UNALIGNED
-/* We can handle unaligned memory access. */
-# define get16(addr) *((const uint16_t *) (addr))
-# define get32(addr) *((const uint32_t *) (addr))
-
-/* We need no special support for writing values either. */
-# define put16(addr, val) *((uint16_t *) (addr)) = (val)
-# define put32(addr, val) *((uint32_t *) (addr)) = (val)
-
# define FCTNAME2(name) name
#else
-/* Distinguish between big endian and little endian. */
-# if __BYTE_ORDER == __LITTLE_ENDIAN
-# define get16(addr) \
- (((const unsigned char *) (addr))[1] << 8 \
- | ((const unsigned char *) (addr))[0])
-# define get32(addr) \
- (((((const unsigned char *) (addr))[3] << 8 \
- | ((const unsigned char *) (addr))[2]) << 8 \
- | ((const unsigned char *) (addr))[1]) << 8 \
- | ((const unsigned char *) (addr))[0])
-
-# define put16(addr, val) \
- ({ uint16_t __val = (val); \
- ((unsigned char *) (addr))[0] = __val; \
- ((unsigned char *) (addr))[1] = __val >> 8; \
- (void) 0; })
-# define put32(addr, val) \
- ({ uint32_t __val = (val); \
- ((unsigned char *) (addr))[0] = __val; \
- __val >>= 8; \
- ((unsigned char *) (addr))[1] = __val; \
- __val >>= 8; \
- ((unsigned char *) (addr))[2] = __val; \
- __val >>= 8; \
- ((unsigned char *) (addr))[3] = __val; \
- (void) 0; })
-# else
-# define get16(addr) \
- (((const unsigned char *) (addr))[0] << 8 \
- | ((const unsigned char *) (addr))[1])
-# define get32(addr) \
- (((((const unsigned char *) (addr))[0] << 8 \
- | ((const unsigned char *) (addr))[1]) << 8 \
- | ((const unsigned char *) (addr))[2]) << 8 \
- | ((const unsigned char *) (addr))[3])
-
-# define put16(addr, val) \
- ({ uint16_t __val = (val); \
- ((unsigned char *) (addr))[1] = __val; \
- ((unsigned char *) (addr))[0] = __val >> 8; \
- (void) 0; })
-# define put32(addr, val) \
- ({ uint32_t __val = (val); \
- ((unsigned char *) (addr))[3] = __val; \
- __val >>= 8; \
- ((unsigned char *) (addr))[2] = __val; \
- __val >>= 8; \
- ((unsigned char *) (addr))[1] = __val; \
- __val >>= 8; \
- ((unsigned char *) (addr))[0] = __val; \
- (void) 0; })
-# endif
-
# define FCTNAME2(name) name##_unaligned
#endif
#define FCTNAME(name) FCTNAME2(name)
@@ -350,10 +285,6 @@ FCTNAME (LOOPFCT) (struct __gconv_step *step,
#if !defined DEFINE_UNALIGNED && !_STRING_ARCH_unaligned \
&& MIN_NEEDED_INPUT != 1 && MAX_NEEDED_INPUT % MIN_NEEDED_INPUT == 0 \
&& MIN_NEEDED_OUTPUT != 1 && MAX_NEEDED_OUTPUT % MIN_NEEDED_OUTPUT == 0
-# undef get16
-# undef get32
-# undef put16
-# undef put32
# undef unaligned
# define DEFINE_UNALIGNED
@@ -527,8 +458,4 @@ gconv_btowc (struct __gconv_step *step, unsigned char c)
#undef LOOP_NEED_STATE
#undef LOOP_NEED_FLAGS
#undef LOOP_NEED_DATA
-#undef get16
-#undef get32
-#undef put16
-#undef put32
#undef unaligned
diff --git a/iconv/skeleton.c b/iconv/skeleton.c
index 9eb0200cc50a0afc..4f20a009d7ffb1db 100644
--- a/iconv/skeleton.c
+++ b/iconv/skeleton.c
@@ -200,73 +200,6 @@
#endif
-/* Define macros which can access unaligned buffers. These macros are
- supposed to be used only in code outside the inner loops. For the inner
- loops we have other definitions which allow optimized access. */
-#if _STRING_ARCH_unaligned
-/* We can handle unaligned memory access. */
-# define get16u(addr) *((const uint16_t *) (addr))
-# define get32u(addr) *((const uint32_t *) (addr))
-
-/* We need no special support for writing values either. */
-# define put16u(addr, val) *((uint16_t *) (addr)) = (val)
-# define put32u(addr, val) *((uint32_t *) (addr)) = (val)
-#else
-/* Distinguish between big endian and little endian. */
-# if __BYTE_ORDER == __LITTLE_ENDIAN
-# define get16u(addr) \
- (((const unsigned char *) (addr))[1] << 8 \
- | ((const unsigned char *) (addr))[0])
-# define get32u(addr) \
- (((((const unsigned char *) (addr))[3] << 8 \
- | ((const unsigned char *) (addr))[2]) << 8 \
- | ((const unsigned char *) (addr))[1]) << 8 \
- | ((const unsigned char *) (addr))[0])
-
-# define put16u(addr, val) \
- ({ uint16_t __val = (val); \
- ((unsigned char *) (addr))[0] = __val; \
- ((unsigned char *) (addr))[1] = __val >> 8; \
- (void) 0; })
-# define put32u(addr, val) \
- ({ uint32_t __val = (val); \
- ((unsigned char *) (addr))[0] = __val; \
- __val >>= 8; \
- ((unsigned char *) (addr))[1] = __val; \
- __val >>= 8; \
- ((unsigned char *) (addr))[2] = __val; \
- __val >>= 8; \
- ((unsigned char *) (addr))[3] = __val; \
- (void) 0; })
-# else
-# define get16u(addr) \
- (((const unsigned char *) (addr))[0] << 8 \
- | ((const unsigned char *) (addr))[1])
-# define get32u(addr) \
- (((((const unsigned char *) (addr))[0] << 8 \
- | ((const unsigned char *) (addr))[1]) << 8 \
- | ((const unsigned char *) (addr))[2]) << 8 \
- | ((const unsigned char *) (addr))[3])
-
-# define put16u(addr, val) \
- ({ uint16_t __val = (val); \
- ((unsigned char *) (addr))[1] = __val; \
- ((unsigned char *) (addr))[0] = __val >> 8; \
- (void) 0; })
-# define put32u(addr, val) \
- ({ uint32_t __val = (val); \
- ((unsigned char *) (addr))[3] = __val; \
- __val >>= 8; \
- ((unsigned char *) (addr))[2] = __val; \
- __val >>= 8; \
- ((unsigned char *) (addr))[1] = __val; \
- __val >>= 8; \
- ((unsigned char *) (addr))[0] = __val; \
- (void) 0; })
-# endif
-#endif
-
-
/* For conversions from a fixed width character set to another fixed width
character set we can define RESET_INPUT_BUFFER in a very fast way. */
#if !defined RESET_INPUT_BUFFER && !defined SAVE_RESET_STATE
diff --git a/iconvdata/iso-2022-jp-3.c b/iconvdata/iso-2022-jp-3.c
index a2b33b171e56392a..122e4593c5f6a0b4 100644
--- a/iconvdata/iso-2022-jp-3.c
+++ b/iconvdata/iso-2022-jp-3.c
@@ -93,7 +93,7 @@ enum
if (__glibc_likely (outbuf + 4 <= outend)) \
{ \
/* Write out the last character. */ \
- put32u (outbuf, ch); \
+ put32 (outbuf, ch); \
outbuf += 4; \
data->__statep->__count &= 7; \
data->__statep->__count |= ASCII_set; \
diff --git a/iconvdata/unicode.c b/iconvdata/unicode.c
index d0db4239093fd33f..9b17e3a355e7f8a6 100644
--- a/iconvdata/unicode.c
+++ b/iconvdata/unicode.c
@@ -52,10 +52,10 @@
return (inptr == inend \
? __GCONV_EMPTY_INPUT : __GCONV_INCOMPLETE_INPUT); \
\
- if (get16u (inptr) == BOM) \
+ if (get16 (inptr) == BOM) \
/* Simply ignore the BOM character. */ \
*inptrp = inptr += 2; \
- else if (get16u (inptr) == BOM_OE) \
+ else if (get16 (inptr) == BOM_OE) \
{ \
data->__flags |= __GCONV_SWAP; \
*inptrp = inptr += 2; \
@@ -68,7 +68,7 @@
if (__glibc_unlikely (outbuf + 2 > outend)) \
return __GCONV_FULL_OUTPUT; \
\
- put16u (outbuf, BOM); \
+ put16 (outbuf, BOM); \
outbuf += 2; \
} \
swap = data->__flags & __GCONV_SWAP;
diff --git a/iconvdata/utf-16.c b/iconvdata/utf-16.c
index ed5f5b2f1f350ddf..6e714928fee8bdca 100644
--- a/iconvdata/utf-16.c
+++ b/iconvdata/utf-16.c
@@ -56,10 +56,10 @@
return (inptr == inend \
? __GCONV_EMPTY_INPUT : __GCONV_INCOMPLETE_INPUT); \
\
- if (get16u (inptr) == BOM) \
+ if (get16 (inptr) == BOM) \
/* Simply ignore the BOM character. */ \
*inptrp = inptr += 2; \
- else if (get16u (inptr) == BOM_OE) \
+ else if (get16 (inptr) == BOM_OE) \
{ \
data->__flags |= __GCONV_SWAP; \
*inptrp = inptr += 2; \
@@ -71,7 +71,7 @@
if (__glibc_unlikely (outbuf + 2 > outend)) \
return __GCONV_FULL_OUTPUT; \
\
- put16u (outbuf, BOM); \
+ put16 (outbuf, BOM); \
outbuf += 2; \
} \
} \
diff --git a/iconvdata/utf-32.c b/iconvdata/utf-32.c
index 7ed0e9bdec672683..dada94f2cca3b752 100644
--- a/iconvdata/utf-32.c
+++ b/iconvdata/utf-32.c
@@ -52,10 +52,10 @@
return (inptr == inend \
? __GCONV_EMPTY_INPUT : __GCONV_INCOMPLETE_INPUT); \
\
- if (get32u (inptr) == BOM) \
+ if (get32 (inptr) == BOM) \
/* Simply ignore the BOM character. */ \
*inptrp = inptr += 4; \
- else if (get32u (inptr) == BOM_OE) \
+ else if (get32 (inptr) == BOM_OE) \
{ \
data->__flags |= __GCONV_SWAP; \
*inptrp = inptr += 4; \
@@ -69,7 +69,7 @@
if (__glibc_unlikely (outbuf + 4 > outend)) \
return __GCONV_FULL_OUTPUT; \
\
- put32u (outbuf, BOM); \
+ put32 (outbuf, BOM); \
outbuf += 4; \
} \
else if (__builtin_expect (data->__invocation_counter == 0, 0) \
diff --git a/sysdeps/s390/utf16-utf32-z9.c b/sysdeps/s390/utf16-utf32-z9.c
index 27086d3e8a1f44ed..4f0d09ab94da59b5 100644
--- a/sysdeps/s390/utf16-utf32-z9.c
+++ b/sysdeps/s390/utf16-utf32-z9.c
@@ -177,7 +177,7 @@ gconv_end (struct __gconv_step *data)
if (__glibc_unlikely (outbuf + 2 > outend)) \
return __GCONV_FULL_OUTPUT; \
\
- put16u (outbuf, BOM_UTF16); \
+ put16 (outbuf, BOM_UTF16); \
outbuf += 2; \
} \
else \
@@ -186,7 +186,7 @@ gconv_end (struct __gconv_step *data)
if (__glibc_unlikely (outbuf + 4 > outend)) \
return __GCONV_FULL_OUTPUT; \
\
- put32u (outbuf, BOM_UTF32); \
+ put32 (outbuf, BOM_UTF32); \
outbuf += 4; \
} \
}
diff --git a/sysdeps/s390/utf8-utf16-z9.c b/sysdeps/s390/utf8-utf16-z9.c
index 409d64c578baa623..8e361890e4ea4ba4 100644
--- a/sysdeps/s390/utf8-utf16-z9.c
+++ b/sysdeps/s390/utf8-utf16-z9.c
@@ -217,7 +217,7 @@ gconv_end (struct __gconv_step *data)
if (__glibc_unlikely (outbuf + 2 > outend)) \
return __GCONV_FULL_OUTPUT; \
\
- put16u (outbuf, BOM_UTF16); \
+ put16 (outbuf, BOM_UTF16); \
outbuf += 2; \
}
diff --git a/sysdeps/s390/utf8-utf32-z9.c b/sysdeps/s390/utf8-utf32-z9.c
index c09d9b5bbde46e4d..422a999bfb690187 100644
--- a/sysdeps/s390/utf8-utf32-z9.c
+++ b/sysdeps/s390/utf8-utf32-z9.c
@@ -217,7 +217,7 @@ gconv_end (struct __gconv_step *data)
if (__glibc_unlikely (outbuf + 4 > outend)) \
return __GCONV_FULL_OUTPUT; \
\
- put32u (outbuf, BOM); \
+ put32 (outbuf, BOM); \
outbuf += 4; \
}

View File

@ -1,700 +0,0 @@
commit 3e20ddade31d9c392d8ccf7ec902172f4bb01c2b
Author: Adhemerval Zanella <adhemerval.zanella@linaro.org>
Date: Fri Feb 10 16:37:36 2023 -0300
iconv: Remove _STRING_ARCH_unaligned usage
Use put/get macros __builtin_bswap32 instead. It allows to remove
the unaligned routines, the compiler will generate unaligned access
if the ABI allows it.
Checked on x86_64-linux-gnu and i686-linux-gnu.
Reviewed-by: Wilco Dijkstra <Wilco.Dijkstra@arm.com>
Conflicts:
iconv/gconv_simple.c
(fixup context)
iconv/loop.c
(fixup context)
iconv/skeleton.c
(fixup context)
diff --git a/iconv/gconv_simple.c b/iconv/gconv_simple.c
index 506c92caf228d61f..d95a5e75e3cd80f1 100644
--- a/iconv/gconv_simple.c
+++ b/iconv/gconv_simple.c
@@ -87,68 +87,22 @@ internal_ucs4_loop (struct __gconv_step *step,
#if __BYTE_ORDER == __LITTLE_ENDIAN
/* Sigh, we have to do some real work. */
size_t cnt;
- uint32_t *outptr32 = (uint32_t *) outptr;
-
- for (cnt = 0; cnt < n_convert; ++cnt, inptr += 4)
- *outptr32++ = bswap_32 (*(const uint32_t *) inptr);
-
- *inptrp = inptr;
- *outptrp = (unsigned char *) outptr32;
-#elif __BYTE_ORDER == __BIG_ENDIAN
- /* Simply copy the data. */
- *inptrp = inptr + n_convert * 4;
- *outptrp = __mempcpy (outptr, inptr, n_convert * 4);
-#else
-# error "This endianess is not supported."
-#endif
-
- /* Determine the status. */
- if (*inptrp == inend)
- result = __GCONV_EMPTY_INPUT;
- else if (*outptrp + 4 > outend)
- result = __GCONV_FULL_OUTPUT;
- else
- result = __GCONV_INCOMPLETE_INPUT;
-
- return result;
-}
-
-#if !_STRING_ARCH_unaligned
-static inline int
-__attribute ((always_inline))
-internal_ucs4_loop_unaligned (struct __gconv_step *step,
- struct __gconv_step_data *step_data,
- const unsigned char **inptrp,
- const unsigned char *inend,
- unsigned char **outptrp, unsigned char *outend,
- size_t *irreversible)
-{
- const unsigned char *inptr = *inptrp;
- unsigned char *outptr = *outptrp;
- size_t n_convert = MIN (inend - inptr, outend - outptr) / 4;
- int result;
-
-# if __BYTE_ORDER == __LITTLE_ENDIAN
- /* Sigh, we have to do some real work. */
- size_t cnt;
for (cnt = 0; cnt < n_convert; ++cnt, inptr += 4, outptr += 4)
{
- outptr[0] = inptr[3];
- outptr[1] = inptr[2];
- outptr[2] = inptr[1];
- outptr[3] = inptr[0];
+ uint32_t val = get32 (inptr);
+ put32 (outptr, __builtin_bswap32 (val));
}
*inptrp = inptr;
*outptrp = outptr;
-# elif __BYTE_ORDER == __BIG_ENDIAN
+#elif __BYTE_ORDER == __BIG_ENDIAN
/* Simply copy the data. */
*inptrp = inptr + n_convert * 4;
*outptrp = __mempcpy (outptr, inptr, n_convert * 4);
-# else
-# error "This endianess is not supported."
-# endif
+#else
+# error "This endianess is not supported."
+#endif
/* Determine the status. */
if (*inptrp == inend)
@@ -160,8 +114,6 @@ internal_ucs4_loop_unaligned (struct __gconv_step *step,
return result;
}
-#endif
-
static inline int
__attribute ((always_inline))
@@ -243,12 +195,9 @@ ucs4_internal_loop (struct __gconv_step *step,
for (cnt = 0; cnt < n_convert; ++cnt, inptr += 4)
{
- uint32_t inval;
-
+ uint32_t inval = get32 (inptr);
#if __BYTE_ORDER == __LITTLE_ENDIAN
- inval = bswap_32 (*(const uint32_t *) inptr);
-#else
- inval = *(const uint32_t *) inptr;
+ inval = __builtin_bswap32 (inval);
#endif
if (__glibc_unlikely (inval > 0x7fffffff))
@@ -273,7 +222,7 @@ ucs4_internal_loop (struct __gconv_step *step,
return __GCONV_ILLEGAL_INPUT;
}
- *((uint32_t *) outptr) = inval;
+ put32 (outptr, inval);
outptr += sizeof (uint32_t);
}
@@ -291,77 +240,6 @@ ucs4_internal_loop (struct __gconv_step *step,
return result;
}
-#if !_STRING_ARCH_unaligned
-static inline int
-__attribute ((always_inline))
-ucs4_internal_loop_unaligned (struct __gconv_step *step,
- struct __gconv_step_data *step_data,
- const unsigned char **inptrp,
- const unsigned char *inend,
- unsigned char **outptrp, unsigned char *outend,
- size_t *irreversible)
-{
- int flags = step_data->__flags;
- const unsigned char *inptr = *inptrp;
- unsigned char *outptr = *outptrp;
- size_t n_convert = MIN (inend - inptr, outend - outptr) / 4;
- int result;
- size_t cnt;
-
- for (cnt = 0; cnt < n_convert; ++cnt, inptr += 4)
- {
- if (__glibc_unlikely (inptr[0] > 0x80))
- {
- /* The value is too large. We don't try transliteration here since
- this is not an error because of the lack of possibilities to
- represent the result. This is a genuine bug in the input since
- UCS4 does not allow such values. */
- if (irreversible == NULL)
- /* We are transliterating, don't try to correct anything. */
- return __GCONV_ILLEGAL_INPUT;
-
- if (flags & __GCONV_IGNORE_ERRORS)
- {
- /* Just ignore this character. */
- ++*irreversible;
- continue;
- }
-
- *inptrp = inptr;
- *outptrp = outptr;
- return __GCONV_ILLEGAL_INPUT;
- }
-
-# if __BYTE_ORDER == __LITTLE_ENDIAN
- outptr[3] = inptr[0];
- outptr[2] = inptr[1];
- outptr[1] = inptr[2];
- outptr[0] = inptr[3];
-# else
- outptr[0] = inptr[0];
- outptr[1] = inptr[1];
- outptr[2] = inptr[2];
- outptr[3] = inptr[3];
-# endif
- outptr += 4;
- }
-
- *inptrp = inptr;
- *outptrp = outptr;
-
- /* Determine the status. */
- if (*inptrp == inend)
- result = __GCONV_EMPTY_INPUT;
- else if (*outptrp + 4 > outend)
- result = __GCONV_FULL_OUTPUT;
- else
- result = __GCONV_INCOMPLETE_INPUT;
-
- return result;
-}
-#endif
-
-
static inline int
__attribute ((always_inline))
ucs4_internal_loop_single (struct __gconv_step *step,
@@ -454,11 +332,12 @@ internal_ucs4le_loop (struct __gconv_step *step,
#if __BYTE_ORDER == __BIG_ENDIAN
/* Sigh, we have to do some real work. */
size_t cnt;
- uint32_t *outptr32 = (uint32_t *) outptr;
- for (cnt = 0; cnt < n_convert; ++cnt, inptr += 4)
- *outptr32++ = bswap_32 (*(const uint32_t *) inptr);
- outptr = (unsigned char *) outptr32;
+ for (cnt = 0; cnt < n_convert; ++cnt, inptr += 4, outptr += 4)
+ {
+ uint32_t val = get32 (inptr);
+ put32 (outptr, __builtin_bswap32 (val));
+ }
*inptrp = inptr;
*outptrp = outptr;
@@ -481,59 +360,6 @@ internal_ucs4le_loop (struct __gconv_step *step,
return result;
}
-#if !_STRING_ARCH_unaligned
-static inline int
-__attribute ((always_inline))
-internal_ucs4le_loop_unaligned (struct __gconv_step *step,
- struct __gconv_step_data *step_data,
- const unsigned char **inptrp,
- const unsigned char *inend,
- unsigned char **outptrp, unsigned char *outend,
- size_t *irreversible)
-{
- const unsigned char *inptr = *inptrp;
- unsigned char *outptr = *outptrp;
- size_t n_convert = MIN (inend - inptr, outend - outptr) / 4;
- int result;
-
-# if __BYTE_ORDER == __BIG_ENDIAN
- /* Sigh, we have to do some real work. */
- size_t cnt;
-
- for (cnt = 0; cnt < n_convert; ++cnt, inptr += 4, outptr += 4)
- {
- outptr[0] = inptr[3];
- outptr[1] = inptr[2];
- outptr[2] = inptr[1];
- outptr[3] = inptr[0];
- }
-
- *inptrp = inptr;
- *outptrp = outptr;
-# elif __BYTE_ORDER == __LITTLE_ENDIAN
- /* Simply copy the data. */
- *inptrp = inptr + n_convert * 4;
- *outptrp = __mempcpy (outptr, inptr, n_convert * 4);
-# else
-# error "This endianess is not supported."
-# endif
-
- /* Determine the status. */
- if (*inptrp == inend)
- result = __GCONV_EMPTY_INPUT;
- else if (*inptrp + 4 > inend)
- result = __GCONV_INCOMPLETE_INPUT;
- else
- {
- assert (*outptrp + 4 > outend);
- result = __GCONV_FULL_OUTPUT;
- }
-
- return result;
-}
-#endif
-
-
static inline int
__attribute ((always_inline))
internal_ucs4le_loop_single (struct __gconv_step *step,
@@ -613,12 +439,9 @@ ucs4le_internal_loop (struct __gconv_step *step,
for (cnt = 0; cnt < n_convert; ++cnt, inptr += 4)
{
- uint32_t inval;
-
+ uint32_t inval = get32 (inptr);
#if __BYTE_ORDER == __BIG_ENDIAN
- inval = bswap_32 (*(const uint32_t *) inptr);
-#else
- inval = *(const uint32_t *) inptr;
+ inval = __builtin_bswap32 (inval);
#endif
if (__glibc_unlikely (inval > 0x7fffffff))
@@ -643,7 +466,7 @@ ucs4le_internal_loop (struct __gconv_step *step,
return __GCONV_ILLEGAL_INPUT;
}
- *((uint32_t *) outptr) = inval;
+ put32 (outptr, inval);
outptr += sizeof (uint32_t);
}
@@ -664,81 +487,6 @@ ucs4le_internal_loop (struct __gconv_step *step,
return result;
}
-#if !_STRING_ARCH_unaligned
-static inline int
-__attribute ((always_inline))
-ucs4le_internal_loop_unaligned (struct __gconv_step *step,
- struct __gconv_step_data *step_data,
- const unsigned char **inptrp,
- const unsigned char *inend,
- unsigned char **outptrp, unsigned char *outend,
- size_t *irreversible)
-{
- int flags = step_data->__flags;
- const unsigned char *inptr = *inptrp;
- unsigned char *outptr = *outptrp;
- size_t n_convert = MIN (inend - inptr, outend - outptr) / 4;
- int result;
- size_t cnt;
-
- for (cnt = 0; cnt < n_convert; ++cnt, inptr += 4)
- {
- if (__glibc_unlikely (inptr[3] > 0x80))
- {
- /* The value is too large. We don't try transliteration here since
- this is not an error because of the lack of possibilities to
- represent the result. This is a genuine bug in the input since
- UCS4 does not allow such values. */
- if (irreversible == NULL)
- /* We are transliterating, don't try to correct anything. */
- return __GCONV_ILLEGAL_INPUT;
-
- if (flags & __GCONV_IGNORE_ERRORS)
- {
- /* Just ignore this character. */
- ++*irreversible;
- continue;
- }
-
- *inptrp = inptr;
- *outptrp = outptr;
- return __GCONV_ILLEGAL_INPUT;
- }
-
-# if __BYTE_ORDER == __BIG_ENDIAN
- outptr[3] = inptr[0];
- outptr[2] = inptr[1];
- outptr[1] = inptr[2];
- outptr[0] = inptr[3];
-# else
- outptr[0] = inptr[0];
- outptr[1] = inptr[1];
- outptr[2] = inptr[2];
- outptr[3] = inptr[3];
-# endif
-
- outptr += 4;
- }
-
- *inptrp = inptr;
- *outptrp = outptr;
-
- /* Determine the status. */
- if (*inptrp == inend)
- result = __GCONV_EMPTY_INPUT;
- else if (*inptrp + 4 > inend)
- result = __GCONV_INCOMPLETE_INPUT;
- else
- {
- assert (*outptrp + 4 > outend);
- result = __GCONV_FULL_OUTPUT;
- }
-
- return result;
-}
-#endif
-
-
static inline int
__attribute ((always_inline))
ucs4le_internal_loop_single (struct __gconv_step *step,
diff --git a/iconv/loop.c b/iconv/loop.c
index a9aa201eadc07bd9..9ef2657f4cbe051c 100644
--- a/iconv/loop.c
+++ b/iconv/loop.c
@@ -59,12 +59,7 @@
#include <libc-diag.h>
#undef FCTNAME2
-#if _STRING_ARCH_unaligned || !defined DEFINE_UNALIGNED
-# define FCTNAME2(name) name
-#else
-# define FCTNAME2(name) name##_unaligned
-#endif
-#define FCTNAME(name) FCTNAME2(name)
+#define FCTNAME(name) name
/* We need at least one byte for the next round. */
@@ -280,20 +275,9 @@ FCTNAME (LOOPFCT) (struct __gconv_step *step,
}
-/* Include the file a second time to define the function to handle
- unaligned access. */
-#if !defined DEFINE_UNALIGNED && !_STRING_ARCH_unaligned \
- && MIN_NEEDED_INPUT != 1 && MAX_NEEDED_INPUT % MIN_NEEDED_INPUT == 0 \
- && MIN_NEEDED_OUTPUT != 1 && MAX_NEEDED_OUTPUT % MIN_NEEDED_OUTPUT == 0
-# undef unaligned
-
-# define DEFINE_UNALIGNED
-# include "loop.c"
-# undef DEFINE_UNALIGNED
-#else
-# if MAX_NEEDED_INPUT > 1
-# define SINGLE(fct) SINGLE2 (fct)
-# define SINGLE2(fct) fct##_single
+#if MAX_NEEDED_INPUT > 1
+# define SINGLE(fct) SINGLE2 (fct)
+# define SINGLE2(fct) fct##_single
static inline int
__attribute ((always_inline))
SINGLE(LOOPFCT) (struct __gconv_step *step,
@@ -303,37 +287,37 @@ SINGLE(LOOPFCT) (struct __gconv_step *step,
size_t *irreversible EXTRA_LOOP_DECLS)
{
mbstate_t *state = step_data->__statep;
-# ifdef LOOP_NEED_FLAGS
+# ifdef LOOP_NEED_FLAGS
int flags = step_data->__flags;
-# endif
-# ifdef LOOP_NEED_DATA
+# endif
+# ifdef LOOP_NEED_DATA
void *data = step->__data;
-# endif
+# endif
int result = __GCONV_OK;
unsigned char bytebuf[MAX_NEEDED_INPUT];
const unsigned char *inptr = *inptrp;
unsigned char *outptr = *outptrp;
size_t inlen;
-# ifdef INIT_PARAMS
+# ifdef INIT_PARAMS
INIT_PARAMS;
-# endif
+# endif
-# ifdef UNPACK_BYTES
+# ifdef UNPACK_BYTES
UNPACK_BYTES
-# else
+# else
/* Add the bytes from the state to the input buffer. */
assert ((state->__count & 7) <= sizeof (state->__value));
for (inlen = 0; inlen < (size_t) (state->__count & 7); ++inlen)
bytebuf[inlen] = state->__value.__wchb[inlen];
-# endif
+# endif
/* Are there enough bytes in the input buffer? */
if (MIN_NEEDED_INPUT > 1
&& __builtin_expect (inptr + (MIN_NEEDED_INPUT - inlen) > inend, 0))
{
*inptrp = inend;
-# ifdef STORE_REST
+# ifdef STORE_REST
/* Building with -O3 GCC emits a `array subscript is above array
bounds' warning. GCC BZ #64739 has been opened for this. */
@@ -348,12 +332,12 @@ SINGLE(LOOPFCT) (struct __gconv_step *step,
inend = &bytebuf[inlen];
STORE_REST
-# else
+# else
/* We don't have enough input for another complete input
character. */
while (inptr < inend)
state->__value.__wchb[inlen++] = *inptr++;
-# endif
+# endif
return __GCONV_INCOMPLETE_INPUT;
}
@@ -393,11 +377,11 @@ SINGLE(LOOPFCT) (struct __gconv_step *step,
result = __GCONV_OK;
/* Clear the state buffer. */
-# ifdef CLEAR_STATE
+# ifdef CLEAR_STATE
CLEAR_STATE;
-# else
+# else
state->__count &= ~7;
-# endif
+# endif
}
else if (result == __GCONV_INCOMPLETE_INPUT)
{
@@ -406,11 +390,11 @@ SINGLE(LOOPFCT) (struct __gconv_step *step,
assert (inend != &bytebuf[MAX_NEEDED_INPUT]);
*inptrp += inend - bytebuf - (state->__count & 7);
-# ifdef STORE_REST
+# ifdef STORE_REST
inptrp = &inptr;
STORE_REST
-# else
+# else
/* We don't have enough input for another complete input
character. */
assert (inend - inptr > (state->__count & ~7));
@@ -419,14 +403,13 @@ SINGLE(LOOPFCT) (struct __gconv_step *step,
inlen = 0;
while (inptr < inend)
state->__value.__wchb[inlen++] = *inptr++;
-# endif
+# endif
}
return result;
}
-# undef SINGLE
-# undef SINGLE2
-# endif
+# undef SINGLE
+# undef SINGLE2
# ifdef ONEBYTE_BODY
@@ -458,4 +441,3 @@ gconv_btowc (struct __gconv_step *step, unsigned char c)
#undef LOOP_NEED_STATE
#undef LOOP_NEED_FLAGS
#undef LOOP_NEED_DATA
-#undef unaligned
diff --git a/iconv/skeleton.c b/iconv/skeleton.c
index 4f20a009d7ffb1db..bf2855c84f0270cd 100644
--- a/iconv/skeleton.c
+++ b/iconv/skeleton.c
@@ -446,33 +446,6 @@ FUNCTION_NAME (struct __gconv_step *step, struct __gconv_step_data *data,
size_t lirreversible = 0;
size_t *lirreversiblep = irreversible ? &lirreversible : NULL;
- /* The following assumes that encodings, which have a variable length
- what might unalign a buffer even though it is an aligned in the
- beginning, either don't have the minimal number of bytes as a divisor
- of the maximum length or have a minimum length of 1. This is true
- for all known and supported encodings.
- We use && instead of || to combine the subexpression for the FROM
- encoding and for the TO encoding, because usually one of them is
- INTERNAL, for which the subexpression evaluates to 1, but INTERNAL
- buffers are always aligned correctly. */
-#define POSSIBLY_UNALIGNED \
- (!_STRING_ARCH_unaligned \
- && (((FROM_LOOP_MIN_NEEDED_FROM != 1 \
- && FROM_LOOP_MAX_NEEDED_FROM % FROM_LOOP_MIN_NEEDED_FROM == 0) \
- && (FROM_LOOP_MIN_NEEDED_TO != 1 \
- && FROM_LOOP_MAX_NEEDED_TO % FROM_LOOP_MIN_NEEDED_TO == 0)) \
- || ((TO_LOOP_MIN_NEEDED_FROM != 1 \
- && TO_LOOP_MAX_NEEDED_FROM % TO_LOOP_MIN_NEEDED_FROM == 0) \
- && (TO_LOOP_MIN_NEEDED_TO != 1 \
- && TO_LOOP_MAX_NEEDED_TO % TO_LOOP_MIN_NEEDED_TO == 0))))
-#if POSSIBLY_UNALIGNED
- int unaligned;
-# define GEN_unaligned(name) GEN_unaligned2 (name)
-# define GEN_unaligned2(name) name##_unaligned
-#else
-# define unaligned 0
-#endif
-
#ifdef PREPARE_LOOP
PREPARE_LOOP
#endif
@@ -512,18 +485,6 @@ FUNCTION_NAME (struct __gconv_step *step, struct __gconv_step_data *data,
}
#endif
-#if POSSIBLY_UNALIGNED
- unaligned =
- ((FROM_DIRECTION
- && ((uintptr_t) inptr % FROM_LOOP_MIN_NEEDED_FROM != 0
- || ((data->__flags & __GCONV_IS_LAST)
- && (uintptr_t) outbuf % FROM_LOOP_MIN_NEEDED_TO != 0)))
- || (!FROM_DIRECTION
- && (((data->__flags & __GCONV_IS_LAST)
- && (uintptr_t) outbuf % TO_LOOP_MIN_NEEDED_TO != 0)
- || (uintptr_t) inptr % TO_LOOP_MIN_NEEDED_FROM != 0)));
-#endif
-
while (1)
{
/* Remember the start value for this round. */
@@ -535,34 +496,14 @@ FUNCTION_NAME (struct __gconv_step *step, struct __gconv_step_data *data,
SAVE_RESET_STATE (1);
#endif
- if (__glibc_likely (!unaligned))
- {
- if (FROM_DIRECTION)
- /* Run the conversion loop. */
- status = FROM_LOOP (step, data, inptrp, inend, &outbuf, outend,
- lirreversiblep EXTRA_LOOP_ARGS);
- else
- /* Run the conversion loop. */
- status = TO_LOOP (step, data, inptrp, inend, &outbuf, outend,
- lirreversiblep EXTRA_LOOP_ARGS);
- }
-#if POSSIBLY_UNALIGNED
+ if (FROM_DIRECTION)
+ /* Run the conversion loop. */
+ status = FROM_LOOP (step, data, inptrp, inend, &outbuf, outend,
+ lirreversiblep EXTRA_LOOP_ARGS);
else
- {
- if (FROM_DIRECTION)
- /* Run the conversion loop. */
- status = GEN_unaligned (FROM_LOOP) (step, data, inptrp, inend,
- &outbuf, outend,
- lirreversiblep
- EXTRA_LOOP_ARGS);
- else
- /* Run the conversion loop. */
- status = GEN_unaligned (TO_LOOP) (step, data, inptrp, inend,
- &outbuf, outend,
- lirreversiblep
- EXTRA_LOOP_ARGS);
- }
-#endif
+ /* Run the conversion loop. */
+ status = TO_LOOP (step, data, inptrp, inend, &outbuf, outend,
+ lirreversiblep EXTRA_LOOP_ARGS);
/* If we were called as part of an error handling module we
don't do anything else here. */
@@ -619,41 +560,14 @@ FUNCTION_NAME (struct __gconv_step *step, struct __gconv_step_data *data,
SAVE_RESET_STATE (0);
# endif
- if (__glibc_likely (!unaligned))
- {
- if (FROM_DIRECTION)
- /* Run the conversion loop. */
- nstatus = FROM_LOOP (step, data, inptrp, inend,
- &outbuf, outerr,
- lirreversiblep
- EXTRA_LOOP_ARGS);
- else
- /* Run the conversion loop. */
- nstatus = TO_LOOP (step, data, inptrp, inend,
- &outbuf, outerr,
- lirreversiblep
- EXTRA_LOOP_ARGS);
- }
-# if POSSIBLY_UNALIGNED
+ if (FROM_DIRECTION)
+ /* Run the conversion loop. */
+ nstatus = FROM_LOOP (step, data, inptrp, inend,
+ &outbuf, outerr, lirreversiblep EXTRA_LOOP_ARGS);
else
- {
- if (FROM_DIRECTION)
- /* Run the conversion loop. */
- nstatus = GEN_unaligned (FROM_LOOP) (step, data,
- inptrp, inend,
- &outbuf,
- outerr,
- lirreversiblep
- EXTRA_LOOP_ARGS);
- else
- /* Run the conversion loop. */
- nstatus = GEN_unaligned (TO_LOOP) (step, data,
- inptrp, inend,
- &outbuf, outerr,
- lirreversiblep
- EXTRA_LOOP_ARGS);
- }
-# endif
+ /* Run the conversion loop. */
+ nstatus = TO_LOOP (step, data, inptrp, inend, &outbuf,
+ outerr, lirreversiblep EXTRA_LOOP_ARGS);
/* We must run out of output buffer space in this
rerun. */

View File

@ -1,325 +0,0 @@
commit d6f08d1cf027f4eb2ba289a6cc66853722d4badc
Author: Florian Weimer <fweimer@redhat.com>
Date: Thu Apr 16 19:13:43 2026 +0200
Use pending character state in IBM1390, IBM1399 character sets (CVE-2026-4046)
Follow the example in iso-2022-jp-3.c and use the __count state
variable to store the pending character. This avoids restarting
the conversion if the output buffer ends between two 4-byte UCS-4
code points, so that the assert reported in the bug can no longer
happen.
Even though the fix is applied to ibm1364.c, the change is only
effective for the two HAS_COMBINED codecs for IBM1390, IBM1399.
The test case was mostly auto-generated using
claude-4.6-opus-high-thinking, and composer-2-fast shows up in the
log as well. During review, gpt-5.4-xhigh flagged that the original
version of the test case was not exercising the new character
flush logic.
This fixes bug 33980.
Assisted-by: LLM
Reviewed-by: Carlos O'Donell <carlos@redhat.com>
Conflicts:
iconvdata/Makefile
(fixup context)
diff --git a/iconvdata/Makefile b/iconvdata/Makefile
index c959758a90ed954f..c4a7159488bbfdf2 100644
--- a/iconvdata/Makefile
+++ b/iconvdata/Makefile
@@ -75,7 +75,7 @@ ifeq (yes,$(build-shared))
tests = bug-iconv1 bug-iconv2 tst-loading tst-e2big tst-iconv4 bug-iconv4 \
tst-iconv6 bug-iconv5 bug-iconv6 tst-iconv7 bug-iconv8 bug-iconv9 \
bug-iconv10 bug-iconv11 bug-iconv12 bug-iconv13 bug-iconv14 \
- bug-iconv15 tst-iconv-iso-2022-cn-ext
+ bug-iconv15 tst-iconv-iso-2022-cn-ext tst-bug33980
ifeq ($(have-thread-library),yes)
tests += bug-iconv3
endif
@@ -327,6 +327,8 @@ $(objpfx)bug-iconv15.out: $(addprefix $(objpfx), $(gconv-modules)) \
$(addprefix $(objpfx),$(modules.so))
$(objpfx)tst-iconv-iso-2022-cn-ext.out: $(addprefix $(objpfx), $(gconv-modules)) \
$(addprefix $(objpfx),$(modules.so))
+$(objpfx)tst-bug33980.out: $(addprefix $(objpfx), $(gconv-modules)) \
+ $(addprefix $(objpfx),$(modules.so))
$(objpfx)iconv-test.out: run-iconv-test.sh \
$(addprefix $(objpfx), $(gconv-modules)) \
diff --git a/iconvdata/ibm1364.c b/iconvdata/ibm1364.c
index ecc3f8ddddbdbc8c..5edd9d01262a1c0d 100644
--- a/iconvdata/ibm1364.c
+++ b/iconvdata/ibm1364.c
@@ -68,12 +68,29 @@
/* Since this is a stateful encoding we have to provide code which resets
the output state to the initial state. This has to be done during the
- flushing. */
+ flushing. For the to-internal direction (FROM_DIRECTION is true),
+ there may be a pending character that needs flushing. */
#define EMIT_SHIFT_TO_INIT \
if ((data->__statep->__count & ~7) != sb) \
{ \
if (FROM_DIRECTION) \
- data->__statep->__count &= 7; \
+ { \
+ uint32_t ch = data->__statep->__count >> 7; \
+ if (__glibc_unlikely (ch != 0)) \
+ { \
+ if (__glibc_unlikely (outend - outbuf < 4)) \
+ status = __GCONV_FULL_OUTPUT; \
+ else \
+ { \
+ put32 (outbuf, ch); \
+ outbuf += 4; \
+ /* Clear character and db bit. */ \
+ data->__statep->__count &= 7; \
+ } \
+ } \
+ else \
+ data->__statep->__count &= 7; \
+ } \
else \
{ \
/* We are not in the initial state. To switch back we have \
@@ -100,11 +117,13 @@
*curcsp = save_curcs
-/* Current codeset type. */
+/* Current codeset type. The bit is stored in the __count variable of
+ the conversion state. If the db bit is set, bit 7 and above store
+ a pending UCS-4 code point if non-zero. */
enum
{
- sb = 0,
- db = 64
+ sb = 0, /* Single byte mode. */
+ db = 64 /* Double byte mode. */
};
@@ -120,21 +139,29 @@ enum
} \
else \
{ \
- /* This is a combined character. Make sure we have room. */ \
- if (__glibc_unlikely (outptr + 8 > outend)) \
- { \
- result = __GCONV_FULL_OUTPUT; \
- break; \
- } \
- \
const struct divide *cmbp \
= &DB_TO_UCS4_COMB[ch - __TO_UCS4_COMBINED_MIN]; \
assert (cmbp->res1 != 0 && cmbp->res2 != 0); \
\
put32 (outptr, cmbp->res1); \
outptr += 4; \
- put32 (outptr, cmbp->res2); \
- outptr += 4; \
+ \
+ /* See whether we have room for the second character. */ \
+ if (outend - outptr >= 4) \
+ { \
+ put32 (outptr, cmbp->res2); \
+ outptr += 4; \
+ } \
+ else \
+ { \
+ /* Otherwise store only the first character now, and \
+ put the second one into the queue. */ \
+ curcs |= cmbp->res2 << 7; \
+ inptr += 2; \
+ /* Tell the caller why we terminate the loop. */ \
+ result = __GCONV_FULL_OUTPUT; \
+ break; \
+ } \
} \
}
#else
@@ -154,7 +181,20 @@ enum
#define LOOPFCT FROM_LOOP
#define BODY \
{ \
- uint32_t ch = *inptr; \
+ uint32_t ch; \
+ \
+ ch = curcs >> 7; \
+ if (__glibc_unlikely (ch != 0)) \
+ { \
+ put32 (outptr, ch); \
+ outptr += 4; \
+ /* Remove the pending character, but preserve state bits. */ \
+ curcs &= (1 << 7) - 1; \
+ continue; \
+ } \
+ \
+ /* Otherwise read the next input byte. */ \
+ ch = *inptr; \
\
if (__builtin_expect (ch, 0) == SO) \
{ \
diff --git a/iconvdata/tst-bug33980.c b/iconvdata/tst-bug33980.c
new file mode 100644
index 0000000000000000..c9693e0efebe4eae
--- /dev/null
+++ b/iconvdata/tst-bug33980.c
@@ -0,0 +1,153 @@
+/* Test for bug 33980: combining characters in IBM1390/IBM1399.
+ Copyright (C) 2026 Free Software Foundation, Inc.
+ This file is part of the GNU C Library.
+
+ The GNU C Library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ The GNU C Library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with the GNU C Library; if not, see
+ <https://www.gnu.org/licenses/>. */
+
+#include <alloc_buffer.h>
+#include <errno.h>
+#include <iconv.h>
+#include <stdbool.h>
+#include <string.h>
+
+#include <support/check.h>
+#include <support/next_to_fault.h>
+#include <support/support.h>
+
+/* Run iconv in a loop with a small output buffer of OUTBUFSIZE bytes
+ starting at OUTBUF. OUTBUF should be right before an unmapped page
+ so that writing past the end will fault. Skip SHIFT bytes at the
+ start of the input and output, to exercise different buffer
+ alignment. TRUNCATE indicates skipped bytes at the end of
+ input (0 and 1 a valid). */
+static void
+test_one (const char *encoding, unsigned int shift, unsigned int truncate,
+ char *outbuf, size_t outbufsize)
+{
+ /* In IBM1390 and IBM1399, the DBCS code 0xECB5 expands to two
+ Unicode code points when translated. */
+ static char input[] =
+ {
+ /* 8 letters X. */
+ 0xe7, 0xe7, 0xe7, 0xe7, 0xe7, 0xe7, 0xe7, 0xe7,
+ /* SO, 0xECB5, SI: shift to DBCS, special character, shift back. */
+ 0x0e, 0xec, 0xb5, 0x0f
+ };
+
+ /* Expected output after UTF-8 conversion. */
+ static char expected[] =
+ {
+ 'X', 'X', 'X', 'X', 'X', 'X', 'X', 'X',
+ /* U+304B (HIRAGANA LETTER KA). */
+ 0xe3, 0x81, 0x8b,
+ /* U+309A (COMBINING KATAKANA-HIRAGANA SEMI-VOICED SOUND MARK). */
+ 0xe3, 0x82, 0x9a
+ };
+
+ iconv_t cd = iconv_open ("UTF-8", encoding);
+ TEST_VERIFY_EXIT (cd != (iconv_t) -1);
+
+ char result_storage[64];
+ struct alloc_buffer result_buf
+ = alloc_buffer_create (result_storage, sizeof (result_storage));
+
+ char *inptr = &input[shift];
+ size_t inleft = sizeof (input) - shift - truncate;
+
+ while (inleft > 0)
+ {
+ char *outptr = outbuf;
+ size_t outleft = outbufsize;
+ size_t inleft_before = inleft;
+
+ size_t ret = iconv (cd, &inptr, &inleft, &outptr, &outleft);
+ size_t produced = outptr - outbuf;
+ alloc_buffer_copy_bytes (&result_buf, outbuf, produced);
+
+ if (ret == (size_t) -1 && errno == E2BIG)
+ {
+ if (produced == 0 && inleft == inleft_before)
+ {
+ /* Output buffer too small to make progress. This is
+ expected for very small output buffer sizes. */
+ TEST_VERIFY_EXIT (outbufsize < 3);
+ break;
+ }
+ continue;
+ }
+ if (ret == (size_t) -1)
+ FAIL_EXIT1 ("%s (outbufsize %zu): iconv: %m", encoding, outbufsize);
+ break;
+ }
+
+ /* Flush any pending state (e.g. a buffered combined character).
+ With outbufsize < 3, we could not store the first character, so
+ the second character did not become pending, and there is nothing
+ to flush. */
+ {
+ char *outptr = outbuf;
+ size_t outleft = outbufsize;
+
+ size_t ret = iconv (cd, NULL, NULL, &outptr, &outleft);
+ TEST_VERIFY_EXIT (ret == 0);
+ size_t produced = outptr - outbuf;
+ alloc_buffer_copy_bytes (&result_buf, outbuf, produced);
+
+ /* Second flush does not provide more data. */
+ outptr = outbuf;
+ outleft = outbufsize;
+ ret = iconv (cd, NULL, NULL, &outptr, &outleft);
+ TEST_VERIFY_EXIT (ret == 0);
+ TEST_VERIFY (outptr == outbuf);
+ }
+
+ TEST_VERIFY_EXIT (!alloc_buffer_has_failed (&result_buf));
+ size_t result_used
+ = sizeof (result_storage) - alloc_buffer_size (&result_buf);
+
+ if (outbufsize >= 3)
+ {
+ TEST_COMPARE (inleft, 0);
+ TEST_COMPARE (result_used, sizeof (expected) - shift);
+ TEST_COMPARE_BLOB (result_storage, result_used,
+ &expected[shift], sizeof (expected) - shift);
+ }
+ else
+ /* If the buffer is too small, only the leading X could be converted. */
+ TEST_COMPARE (result_used, 8 - shift);
+
+ TEST_VERIFY_EXIT (iconv_close (cd) == 0);
+}
+
+static int
+do_test (void)
+{
+ struct support_next_to_fault ntf
+ = support_next_to_fault_allocate (8);
+
+ for (int shift = 0; shift <= 8; ++shift)
+ for (int truncate = 0; truncate < 2; ++truncate)
+ for (size_t outbufsize = 1; outbufsize <= 8; outbufsize++)
+ {
+ char *outbuf = ntf.buffer + ntf.length - outbufsize;
+ test_one ("IBM1390", shift, truncate, outbuf, outbufsize);
+ test_one ("IBM1399", shift, truncate, outbuf, outbufsize);
+ }
+
+ support_next_to_fault_free (&ntf);
+ return 0;
+}
+
+#include <support/test-driver.c>

View File

@ -1,181 +0,0 @@
commit d8997716a1ca22cf038eac86ed286830ba9818cc
Author: DJ Delorie <dj@redhat.com>
Date: Wed Apr 1 17:52:25 2026 -0400
nss: fix __get_default_domain logic
Fix logic bug in __nss_get_default_domain that prevents
proper initialization.
Because this function is not exposed, the test case must link
against the object directly.
Bug origin commit: 64d1e08ea822bf47cb2796ad0f727136227f983c
Co-authored-by: Florian Weimer <fweimer@redhat.com>
Reviewed-by: Frédéric Bérat <fberat@redhat.com>
Conflicts:
nss/Makefile
(fixup context)
diff --git a/nss/Makefile b/nss/Makefile
index 05fcadf60fd0d771..da0f0fdc0f70ff69 100644
--- a/nss/Makefile
+++ b/nss/Makefile
@@ -55,6 +55,7 @@ extra-objs += $(makedb-modules:=.o)
tests-static = tst-field
tests-internal = tst-field
tests = test-netdb test-digits-dots tst-nss-getpwent bug17079 \
+ tst-default-domain \
tst-nss-test1 \
tst-nss-test2 \
tst-nss-test3 \
@@ -212,3 +213,5 @@ $(objpfx)tst-nss-files-hosts-multi: $(libdl)
$(objpfx)tst-nss-files-hosts-getent: $(libdl)
$(objpfx)tst-nss-files-alias-leak: $(libdl)
$(objpfx)tst-nss-files-alias-leak.out: $(objpfx)libnss_files.so
+
+$(objpfx)tst-default-domain: $(objpfx)nisdomain.os
diff --git a/nss/nss_compat/nisdomain.c b/nss/nss_compat/nisdomain.c
index facce925b36d9418..def681af8f2363fb 100644
--- a/nss/nss_compat/nisdomain.c
+++ b/nss/nss_compat/nisdomain.c
@@ -36,7 +36,7 @@ __nss_get_default_domain (char **outdomain)
__libc_lock_lock (domainname_lock);
- if (domainname[0] != '\0')
+ if (domainname[0] == '\0')
{
if (getdomainname (domainname, MAXDOMAINNAMELEN) < 0)
result = errno;
diff --git a/nss/tst-default-domain.c b/nss/tst-default-domain.c
new file mode 100644
index 0000000000000000..2fa9153d8802d4c3
--- /dev/null
+++ b/nss/tst-default-domain.c
@@ -0,0 +1,123 @@
+/* Basic test of __nss_get_default_domain
+ Copyright (C) 2026 Free Software Foundation, Inc.
+ This file is part of the GNU C Library.
+
+ The GNU C Library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ The GNU C Library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with the GNU C Library; if not, see
+ <https://www.gnu.org/licenses/>. */
+
+#include <unistd.h>
+#include <string.h>
+#include <dlfcn.h>
+
+#include "nss_compat/nisdomain.h"
+
+#include <support/test-driver.h>
+#include <support/support.h>
+#include <support/namespace.h>
+#include <support/check.h>
+
+char unset_domain[] = "unset_domain";
+char new_domain[] = "new_domain";
+
+/* This function checks the __nss_get_default_domain() function in
+ nss_compat/nssdomain.c. Because this is an internal function to
+ libnss_compat.so, the Makefile will link that object to this test
+ case directly. */
+
+static int
+do_test (void)
+{
+ char *domain_name;
+ char buf[1024];
+
+ /* We need to be in a network namespace so we can change the domain
+ name without interfering with the host system. */
+ support_become_root ();
+ support_enter_network_namespace ();
+ if (!support_in_uts_namespace ())
+ return EXIT_UNSUPPORTED;
+
+ /* First pass: set an empty domain and make sure it's returned
+ correctly. This should not be cached. */
+
+ /* Set the domain name to a known value. */
+ TEST_VERIFY (setdomainname ("", 0) == 0);
+
+ /* Make sure it got set. */
+ TEST_VERIFY (getdomainname (buf, sizeof(buf)) == 0);
+ TEST_COMPARE_STRING (buf, "");
+
+ /* Set this to a known "unknown" value so we can detect if it's not
+ changed. */
+ domain_name = unset_domain;
+
+ /* This is the function we're testing. */
+ TEST_VERIFY (__nss_get_default_domain (& domain_name) == 0);
+
+ /* Make sure the correct domain name is returned. */
+ TEST_VERIFY (domain_name != NULL);
+ TEST_COMPARE_STRING (domain_name, "");
+
+ /* Second pass: set a non-empty domain and make sure it's returned
+ correctly. This works because the empty domain is not
+ cached. */
+
+ /* Set the domain name to a known value. */
+ TEST_VERIFY (setdomainname (new_domain, strlen (new_domain)) == 0);
+
+ /* Make sure it got set. */
+ TEST_VERIFY (getdomainname (buf, sizeof(buf)) == 0);
+ TEST_COMPARE_STRING (buf, new_domain);
+
+ /* Set this to a known "unknown" value so we can detect if it's not
+ changed. */
+ domain_name = unset_domain;
+
+ /* This is the function we're testing. */
+ TEST_VERIFY (__nss_get_default_domain (& domain_name) == 0);
+
+ /* Make sure the correct domain name is returned. */
+ TEST_VERIFY (domain_name != NULL);
+ TEST_COMPARE_STRING (domain_name, new_domain);
+
+ /* The function caches the name, so check it twice. */
+ TEST_VERIFY (__nss_get_default_domain (& domain_name) == 0);
+
+ TEST_VERIFY (domain_name != NULL);
+ TEST_COMPARE_STRING (domain_name, new_domain);
+
+ /* Third pass: set an empty domain again but expect the cached
+ value. */
+
+ /* Set the domain name to a known value. */
+ TEST_VERIFY (setdomainname ("", 0) == 0);
+
+ /* Make sure it got set. */
+ TEST_VERIFY (getdomainname (buf, sizeof(buf)) == 0);
+ TEST_COMPARE_STRING (buf, "");
+
+ /* Set this to a known "unknown" value so we can detect if it's not
+ changed. */
+ domain_name = unset_domain;
+
+ /* This is the function we're testing. */
+ TEST_VERIFY (__nss_get_default_domain (& domain_name) == 0);
+
+ TEST_VERIFY (domain_name != NULL);
+ TEST_COMPARE_STRING (domain_name, new_domain);
+
+ return 0;
+}
+
+#include <support/test-driver.c>

View File

@ -1,224 +0,0 @@
Test-only backport of test for CVE-2026-4437 that does not affect
glibc-2.28. Test extracted from the following upstream commit:
commit 5663ab0b833df187b15e7bb4b18173e22beb8bd1
Author: Carlos O'Donell <carlos@redhat.com>
Date: Fri Mar 20 16:43:33 2026 -0400
resolv: Count records correctly (CVE-2026-4437)
The answer section boundary was previously ignored, and the code in
getanswer_ptr would iterate past the last resource record, but not
beyond the end of the returned data. This could lead to subsequent data
being interpreted as answer records, thus violating the DNS
specification. Such resource records could be maliciously crafted and
hidden from other tooling, but processed by the glibc stub resolver and
acted upon by the application. While we trust the data returned by the
configured recursive resolvers, we should not trust its format and
should validate it as required. It is a security issue to incorrectly
process the DNS protocol.
A regression test is added for response section crossing.
No regressions on x86_64-linux-gnu.
Reviewed-by: Collin Funk <collin.funk1@gmail.com>
(cherry picked from commit 9f5f18aab40ec6b61fa49a007615e6077e9a979b)
diff --git a/resolv/Makefile b/resolv/Makefile
index f3feaf072a3ec66d..9837a8cfa145e1da 100644
--- a/resolv/Makefile
+++ b/resolv/Makefile
@@ -55,6 +55,7 @@ tests += \
tst-res_use_inet6 \
tst-resolv-basic \
tst-resolv-binary \
+ tst-resolv-dns-section \
tst-resolv-edns \
tst-resolv-network \
tst-resolv-noaaaa \
@@ -65,6 +66,7 @@ tests += \
tst-resolv-semi-failure \
tst-resolv-short-response \
tst-resolv-trailing \
+ # tests
# This test calls __res_context_send directly, which is not exported
# from libresolv.
@@ -197,6 +199,8 @@ $(objpfx)tst-resolv-ai_idn-nolibidn2.out: \
$(gen-locales) $(objpfx)tst-no-libidn2.so
$(objpfx)tst-resolv-basic: $(objpfx)libresolv.so $(shared-thread-library)
$(objpfx)tst-resolv-binary: $(objpfx)libresolv.so $(shared-thread-library)
+$(objpfx)tst-resolv-dns-section: $(objpfx)libresolv.so \
+ $(shared-thread-library)
$(objpfx)tst-resolv-edns: $(objpfx)libresolv.so $(shared-thread-library)
$(objpfx)tst-resolv-network: $(objpfx)libresolv.so $(shared-thread-library)
$(objpfx)tst-resolv-res_init: $(libdl) $(objpfx)libresolv.so
diff --git a/resolv/tst-resolv-dns-section.c b/resolv/tst-resolv-dns-section.c
new file mode 100644
index 0000000000000000..1171baef51e3cc36
--- /dev/null
+++ b/resolv/tst-resolv-dns-section.c
@@ -0,0 +1,162 @@
+/* Test handling of invalid section transitions (bug 34014).
+ Copyright (C) 2022-2026 Free Software Foundation, Inc.
+ This file is part of the GNU C Library.
+
+ The GNU C Library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ The GNU C Library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with the GNU C Library; if not, see
+ <https://www.gnu.org/licenses/>. */
+
+#include <array_length.h>
+#include <errno.h>
+#include <netdb.h>
+#include <resolv.h>
+#include <stdlib.h>
+#include <string.h>
+#include <support/check.h>
+#include <support/format_nss.h>
+#include <support/resolv_test.h>
+#include <support/support.h>
+
+/* Name of test, and the second section type. */
+struct item {
+ const char *test;
+ int ns_section;
+};
+
+static const struct item test_items[] =
+ {
+ { "Test crossing from ns_s_an to ns_s_ar.", ns_s_ar },
+ { "Test crossing from ns_s_an to ns_s_an.", ns_s_ns },
+
+ { NULL, 0 },
+ };
+
+/* The response is designed to contain the following:
+ - An Answer section with one T_PTR record that is skipped.
+ - A second section with a semantically invalid T_PTR record.
+ The original defect is that the response parsing would cross
+ section boundaries and handle the additional section T_PTR
+ as if it were an answer. A conforming implementation would
+ stop as soon as it reaches the end of the section. */
+static void
+response (const struct resolv_response_context *ctx,
+ struct resolv_response_builder *b,
+ const char *qname, uint16_t qclass, uint16_t qtype)
+{
+ TEST_COMPARE (qclass, C_IN);
+
+ /* We only test PTR. */
+ TEST_COMPARE (qtype, T_PTR);
+
+ unsigned int count;
+ char *tail = NULL;
+
+ if (strstr (qname, "in-addr.arpa") != NULL
+ && sscanf (qname, "%u.%ms", &count, &tail) == 2)
+ TEST_COMPARE_STRING (tail, "0.168.192.in-addr.arpa");
+ else if (sscanf (qname, "%x.%ms", &count, &tail) == 2)
+ {
+ TEST_COMPARE_STRING (tail, "\
+0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.8.b.d.0.1.0.0.2.ip6.arpa");
+ }
+ else
+ FAIL_EXIT1 ("invalid QNAME: %s\n", qname);
+ free (tail);
+
+ /* We have a bounded number of possible tests. */
+ TEST_VERIFY (count >= 0);
+ TEST_VERIFY (count <= 15);
+
+ struct resolv_response_flags flags = {};
+ resolv_response_init (b, flags);
+ resolv_response_add_question (b, qname, qclass, qtype);
+ resolv_response_section (b, ns_s_an);
+
+ /* Actual answer record, but the wrong name (skipped). */
+ resolv_response_open_record (b, "1.0.0.10.in-addr.arpa", qclass, qtype, 60);
+
+ /* Record the answer. */
+ resolv_response_add_name (b, "test.ptr.example.net");
+ resolv_response_close_record (b);
+
+ /* Add a second section to test section boundary crossing. */
+ resolv_response_section (b, test_items[count].ns_section);
+ /* Semantically incorrect, but hide a T_PTR entry. */
+ resolv_response_open_record (b, qname, qclass, qtype, 60);
+ resolv_response_add_name (b, "wrong.ptr.example.net");
+ resolv_response_close_record (b);
+}
+
+
+/* Perform one check using a reverse lookup. */
+static void
+check_reverse (int af, int count)
+{
+ TEST_VERIFY (af == AF_INET || af == AF_INET6);
+ TEST_VERIFY (count < array_length (test_items));
+
+ char addr[sizeof (struct in6_addr)] = { 0 };
+ socklen_t addrlen;
+ if (af == AF_INET)
+ {
+ addr[0] = (char) 192;
+ addr[1] = (char) 168;
+ addr[2] = (char) 0;
+ addr[3] = (char) count;
+ addrlen = 4;
+ }
+ else
+ {
+ addr[0] = 0x20;
+ addr[1] = 0x01;
+ addr[2] = 0x0d;
+ addr[3] = 0xb8;
+ addr[4] = addr[5] = addr[6] = addr[7] = 0x0;
+ addr[8] = addr[9] = addr[10] = addr[11] = 0x0;
+ addr[12] = 0x0;
+ addr[13] = 0x0;
+ addr[14] = 0x0;
+ addr[15] = count;
+ addrlen = 16;
+ }
+
+ h_errno = 0;
+ struct hostent *answer = gethostbyaddr (addr, addrlen, af);
+ TEST_VERIFY (answer == NULL);
+ TEST_VERIFY (h_errno == NO_RECOVERY);
+ if (answer != NULL)
+ printf ("error: unexpected success: %s\n",
+ support_format_hostent (answer));
+}
+
+static int
+do_test (void)
+{
+ struct resolv_test *obj = resolv_test_start
+ ((struct resolv_redirect_config)
+ {
+ .response_callback = response
+ });
+
+ for (int i = 0; test_items[i].test != NULL; i++)
+ {
+ check_reverse (AF_INET, i);
+ check_reverse (AF_INET6, i);
+ }
+
+ resolv_test_end (obj);
+
+ return 0;
+}
+
+#include <support/test-driver.c>

View File

@ -1,304 +0,0 @@
Test-only backport of test for CVE-2026-4438 that does not affect
glibc-2.28. Test extracted from the following upstream commit:
commit 9344c796f7a4ac8f2c59de63f3e1e936b51e817d
Author: Carlos O'Donell <carlos@redhat.com>
Date: Fri Mar 20 17:14:33 2026 -0400
resolv: Check hostname for validity (CVE-2026-4438)
The processed hostname in getanswer_ptr should be correctly checked to
avoid invalid characters from being allowed, including shell
metacharacters. It is a security issue to fail to check the returned
hostname for validity.
A regression test is added for invalid metacharacters and other cases
of invalid or valid characters.
No regressions on x86_64-linux-gnu.
Reviewed-by: Adhemerval Zanella <adhemerval.zanella@linaro.org>
(cherry picked from commit e10977481f4db4b2a3ce34fa4c3a1e26651ae312)
diff --git a/resolv/Makefile b/resolv/Makefile
index 9837a8cfa145e1da..359e8a0a9b22aff9 100644
--- a/resolv/Makefile
+++ b/resolv/Makefile
@@ -57,6 +57,7 @@ tests += \
tst-resolv-binary \
tst-resolv-dns-section \
tst-resolv-edns \
+ tst-resolv-invalid-ptr \
tst-resolv-network \
tst-resolv-noaaaa \
tst-resolv-noaaaa-vc \
@@ -208,6 +209,8 @@ $(objpfx)tst-resolv-res_init-multi: $(objpfx)libresolv.so \
$(shared-thread-library)
$(objpfx)tst-resolv-res_init-thread: $(libdl) $(objpfx)libresolv.so \
$(shared-thread-library)
+$(objpfx)tst-resolv-invalid-ptr: $(objpfx)libresolv.so \
+ $(shared-thread-library)
$(objpfx)tst-resolv-noaaaa: $(objpfx)libresolv.so $(shared-thread-library)
$(objpfx)tst-resolv-noaaaa-vc: $(objpfx)libresolv.so $(shared-thread-library)
$(objpfx)tst-resolv-nondecimal: $(objpfx)libresolv.so $(shared-thread-library)
diff --git a/resolv/tst-resolv-invalid-ptr.c b/resolv/tst-resolv-invalid-ptr.c
new file mode 100644
index 0000000000000000..0c802ab96780efb0
--- /dev/null
+++ b/resolv/tst-resolv-invalid-ptr.c
@@ -0,0 +1,255 @@
+/* Test handling of invalid T_PTR results (bug 34015).
+ Copyright (C) 2022-2026 Free Software Foundation, Inc.
+ This file is part of the GNU C Library.
+
+ The GNU C Library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ The GNU C Library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with the GNU C Library; if not, see
+ <https://www.gnu.org/licenses/>. */
+
+#include <array_length.h>
+#include <errno.h>
+#include <netdb.h>
+#include <resolv.h>
+#include <stdlib.h>
+#include <string.h>
+#include <support/check.h>
+#include <support/format_nss.h>
+#include <support/resolv_test.h>
+#include <support/support.h>
+
+/* Name of test, the answer, the expected error return, and if we
+ expect the call to fail. */
+struct item {
+ const char *test;
+ const char *answer;
+ int expected;
+ bool fail;
+};
+
+static const struct item test_items[] =
+ {
+ /* Test for invalid characters. */
+ { "Invalid use of \"|\"",
+ "test.|.ptr.example", NO_RECOVERY, true },
+ { "Invalid use of \"&\"",
+ "test.&.ptr.example", NO_RECOVERY, true },
+ { "Invalid use of \";\"",
+ "test.;.ptr.example", NO_RECOVERY, true },
+ { "Invalid use of \"<\"",
+ "test.<.ptr.example", NO_RECOVERY, true },
+ { "Invalid use of \">\"",
+ "test.>.ptr.example", NO_RECOVERY, true },
+ { "Invalid use of \"(\"",
+ "test.(.ptr.example", NO_RECOVERY, true },
+ { "Invalid use of \")\"",
+ "test.).ptr.example", NO_RECOVERY, true },
+ { "Invalid use of \"$\"",
+ "test.$.ptr.example", NO_RECOVERY, true },
+ { "Invalid use of \"`\"",
+ "test.`.ptr.example", NO_RECOVERY, true },
+ { "Invalid use of \"\\\"",
+ "test.\\.ptr.example", NO_RECOVERY, true },
+ { "Invalid use of \"\'\"",
+ "test.'.ptr.example", NO_RECOVERY, true },
+ { "Invalid use of \"\"\"",
+ "test.\".ptr.example", NO_RECOVERY, true },
+ { "Invalid use of \" \"",
+ "test. .ptr.example", NO_RECOVERY, true },
+ { "Invalid use of \"\\t\"",
+ "test.\t.ptr.example", NO_RECOVERY, true },
+ { "Invalid use of \"\\n\"",
+ "test.\n.ptr.example", NO_RECOVERY, true },
+ { "Invalid use of \"\\r\"",
+ "test.\r.ptr.example", NO_RECOVERY, true },
+ { "Invalid use of \"*\"",
+ "test.*.ptr.example", NO_RECOVERY, true },
+ { "Invalid use of \"?\"",
+ "test.?.ptr.example", NO_RECOVERY, true },
+ { "Invalid use of \"[\"",
+ "test.[.ptr.example", NO_RECOVERY, true },
+ { "Invalid use of \"]\"",
+ "test.].ptr.example", NO_RECOVERY, true },
+ { "Invalid use of \",\"",
+ "test.,.ptr.example", NO_RECOVERY, true },
+ { "Invalid use of \"~\"",
+ "test.~.ptr.example", NO_RECOVERY, true },
+ { "Invalid use of \":\"",
+ "test.:.ptr.example", NO_RECOVERY, true },
+ { "Invalid use of \"!\"",
+ "test.!.ptr.example", NO_RECOVERY, true },
+ { "Invalid use of \"@\"",
+ "test.@.ptr.example", NO_RECOVERY, true },
+ { "Invalid use of \"#\"",
+ "test.#.ptr.example", NO_RECOVERY, true },
+ { "Invalid use of \"%\"",
+ "test.%%.ptr.example", NO_RECOVERY, true },
+ { "Invalid use of \"^\"",
+ "test.^.ptr.example", NO_RECOVERY, true },
+
+ /* Test for invalid UTF-8 characters (2-byte, 4-byte, 6-byte). */
+ { "Invalid use of UTF-8 (2-byte, U+00C0-U+00C2)",
+ "ÁÂÃ.test.ptr.example", NO_RECOVERY, true },
+ { "Invalid use of UTF-8 (4-byte, U+0750-U+0752)",
+ "ݐݑݒ.test.ptr.example", NO_RECOVERY, true },
+ { "Invalid use of UTF-8 (6-byte, U+0904-U+0906)",
+ "ऄअआ.test.ptr.example", NO_RECOVERY, true },
+
+ /* Test for "-" which may be valid depending on position. */
+ { "Invalid leading \"-\"",
+ "-test.ptr.example", NO_RECOVERY, true },
+ { "Valid trailing \"-\"",
+ "test-.ptr.example", 0, false },
+ { "Valid mid-label use of \"-\"",
+ "te-st.ptr.example", 0, false },
+
+ /* Test for "_" which is always valid in any position. */
+ { "Valid leading use of \"_\"",
+ "_test.ptr.example", 0, false },
+ { "Valid mid-label use of \"_\"",
+ "te_st.ptr.example", 0, false },
+ { "Valid trailing use of \"_\"",
+ "test_.ptr.example", 0, false },
+
+ /* Sanity test the broader set [A-Za-z0-9_-] of valid characters. */
+ { "Valid \"[A-Z]\"",
+ "test.ABCDEFGHIJKLMNOPQRSTUVWXYZ.ptr.example", 0, false },
+ { "Valid \"[a-z]\"",
+ "test.abcdefghijklmnopqrstuvwxyz.ptr.example", 0, false },
+ { "Valid \"[0-9]\"",
+ "test.0123456789.ptr.example", 0, false },
+ { "Valid mixed use of \"[A-Za-z0-9_-]\"",
+ "test.012abcABZ_-.ptr.example", 0, false },
+ };
+
+static void
+response (const struct resolv_response_context *ctx,
+ struct resolv_response_builder *b,
+ const char *qname, uint16_t qclass, uint16_t qtype)
+{
+ TEST_COMPARE (qclass, C_IN);
+
+ /* We only test PTR. */
+ TEST_COMPARE (qtype, T_PTR);
+
+ unsigned int count, count1;
+ char *tail = NULL;
+
+ /* The test implementation can handle up to 255 tests. */
+ if (strstr (qname, "in-addr.arpa") != NULL
+ && sscanf (qname, "%u.%ms", &count, &tail) == 2)
+ TEST_COMPARE_STRING (tail, "0.168.192.in-addr.arpa");
+ else if (sscanf (qname, "%x.%x.%ms", &count, &count1, &tail) == 3)
+ {
+ TEST_COMPARE_STRING (tail, "\
+0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.8.b.d.0.1.0.0.2.ip6.arpa");
+ count |= count1 << 4;
+ }
+ else
+ FAIL_EXIT1 ("invalid QNAME: %s\n", qname);
+ free (tail);
+
+ /* Cross check. Count has a fixed bound (soft limit). */
+ TEST_VERIFY (count >= 0 && count <= 255);
+
+ /* We have a fixed number of tests (hard limit). */
+ TEST_VERIFY_EXIT (count < array_length (test_items));
+
+ struct resolv_response_flags flags = {};
+ resolv_response_init (b, flags);
+ resolv_response_add_question (b, qname, qclass, qtype);
+ resolv_response_section (b, ns_s_an);
+
+ /* Actual answer record. */
+ resolv_response_open_record (b, qname, qclass, qtype, 60);
+
+ /* Record the answer. */
+ resolv_response_add_name (b, test_items[count].answer);
+ resolv_response_close_record (b);
+}
+
+/* Perform one check using a reverse lookup. */
+static void
+check_reverse (int af, int count)
+{
+ TEST_VERIFY (af == AF_INET || af == AF_INET6);
+ TEST_VERIFY_EXIT (count < array_length (test_items));
+
+ /* Generate an address to query for each test. */
+ char addr[sizeof (struct in6_addr)] = { 0 };
+ socklen_t addrlen;
+ if (af == AF_INET)
+ {
+ addr[0] = (char) 192;
+ addr[1] = (char) 168;
+ addr[2] = (char) 0;
+ addr[3] = (char) count;
+ addrlen = 4;
+ }
+ else
+ {
+ addr[0] = 0x20;
+ addr[1] = 0x01;
+ addr[2] = 0x0d;
+ addr[3] = 0xb8;
+ addr[4] = addr[5] = addr[6] = addr[7] = 0x0;
+ addr[8] = addr[9] = addr[10] = addr[11] = 0x0;
+ addr[12] = 0x0;
+ addr[13] = 0x0;
+ addr[14] = 0x0;
+ addr[15] = (char) count;
+ addrlen = 16;
+ }
+
+ h_errno = 0;
+ struct hostent *answer = gethostbyaddr (addr, addrlen, af);
+
+ /* Verify h_errno is as expected. */
+ TEST_COMPARE (h_errno, test_items[count].expected);
+ if (h_errno != test_items[count].expected)
+ /* And print more information if it's not. */
+ printf ("INFO: %s\n", test_items[count].test);
+
+ if (test_items[count].fail)
+ {
+ /* We expected a failure so verify answer is NULL. */
+ TEST_VERIFY (answer == NULL);
+ /* If it's not NULL we should print out what we received. */
+ if (answer != NULL)
+ printf ("error: unexpected success: %s\n",
+ support_format_hostent (answer));
+ }
+ else
+ /* We don't expect a failure so answer must be valid. */
+ TEST_COMPARE_STRING (answer->h_name, test_items[count].answer);
+}
+
+static int
+do_test (void)
+{
+ struct resolv_test *obj = resolv_test_start
+ ((struct resolv_redirect_config)
+ {
+ .response_callback = response
+ });
+
+ for (int i = 0; i < array_length (test_items); i++)
+ {
+ check_reverse (AF_INET, i);
+ check_reverse (AF_INET6, i);
+ }
+ resolv_test_end (obj);
+
+ return 0;
+}
+
+#include <support/test-driver.c>

View File

@ -1,31 +0,0 @@
Based on the following commit, adjusted for glibc-2.28 in RHEL-8:
commit 5eabdb6a6ac1599d23dd5966a37417215950245f
Author: Andreas Schwab <schwab@suse.de>
Date: Wed Dec 6 14:48:22 2023 +0100
getaddrinfo: translate ENOMEM to EAI_MEMORY (bug 31163)
When __resolv_context_get returns NULL due to out of memory, translate it
to a return value of EAI_MEMORY.
diff --git a/sysdeps/posix/getaddrinfo.c b/sysdeps/posix/getaddrinfo.c
index 46046504a6858f2e..d0708f3e84e20025 100644
--- a/sysdeps/posix/getaddrinfo.c
+++ b/sysdeps/posix/getaddrinfo.c
@@ -777,7 +777,14 @@ gaih_inet (const char *name, const struct gaih_service *service,
res_ctx = __resolv_context_get ();
res_enable_inet6 = __resolv_context_disable_inet6 (res_ctx);
if (res_ctx == NULL)
- no_more = 1;
+ {
+ if (errno == ENOMEM)
+ {
+ result = -EAI_MEMORY;
+ goto free_and_return;
+ }
+ no_more = 1;
+ }
while (!no_more)
{

View File

@ -1,112 +0,0 @@
This downstream-only patch compensates for the missing backport of
commit 2d651eb9265d1366d7b9e881bfddd46db9c1ecc4 ("x86: Move
x86 processor cache info to cpu_features"). Without it,
ld.so --list-diagnostics prints values that have not been properly
initalized from CPUID data.
diff --git a/sysdeps/x86/cacheinfo.h b/sysdeps/x86/cacheinfo.h
index 10ebadd819d9efff..d8421fab83ab08ac 100644
--- a/sysdeps/x86/cacheinfo.h
+++ b/sysdeps/x86/cacheinfo.h
@@ -19,31 +19,42 @@
#include <assert.h>
#include <unistd.h>
+/* When building ld.so, do not export any of the variables. They are
+ only used for diagnostics and are not initialized during regular
+ operation. */
+#if IS_IN (rtld)
+# define CACHEINFO_VARIABLE(name, initializer) \
+ static long int name = initializer
+#else
+# define CACHEINFO_VARIABLE(name, initializer) \
+ long int name attribute_hidden = initializer
+#endif
+
/* Data cache size for use in memory and string routines, typically
L1 size, rounded to multiple of 256 bytes. */
-long int __x86_data_cache_size_half attribute_hidden = 32 * 1024 / 2;
-long int __x86_data_cache_size attribute_hidden = 32 * 1024;
+CACHEINFO_VARIABLE (__x86_data_cache_size_half, 32 * 1024 / 2);
+CACHEINFO_VARIABLE (__x86_data_cache_size, 32 * 1024);
/* Similar to __x86_data_cache_size_half, but not rounded. */
-long int __x86_raw_data_cache_size_half attribute_hidden = 32 * 1024 / 2;
+CACHEINFO_VARIABLE (__x86_raw_data_cache_size_half, 32 * 1024 / 2);
/* Similar to __x86_data_cache_size, but not rounded. */
-long int __x86_raw_data_cache_size attribute_hidden = 32 * 1024;
+CACHEINFO_VARIABLE (__x86_raw_data_cache_size, 32 * 1024);
/* Shared cache size for use in memory and string routines, typically
L2 or L3 size, rounded to multiple of 256 bytes. */
-long int __x86_shared_cache_size_half attribute_hidden = 1024 * 1024 / 2;
-long int __x86_shared_cache_size attribute_hidden = 1024 * 1024;
+CACHEINFO_VARIABLE (__x86_shared_cache_size_half, 1024 * 1024 / 2);
+CACHEINFO_VARIABLE (__x86_shared_cache_size, 1024 * 1024);
/* Similar to __x86_shared_cache_size_half, but not rounded. */
-long int __x86_raw_shared_cache_size_half attribute_hidden = 1024 * 1024 / 2;
+CACHEINFO_VARIABLE (__x86_raw_shared_cache_size_half, 1024 * 1024 / 2);
/* Similar to __x86_shared_cache_size, but not rounded. */
-long int __x86_raw_shared_cache_size attribute_hidden = 1024 * 1024;
+CACHEINFO_VARIABLE (__x86_raw_shared_cache_size, 1024 * 1024);
/* Threshold to use non temporal store. */
-long int __x86_shared_non_temporal_threshold attribute_hidden;
+CACHEINFO_VARIABLE (__x86_shared_non_temporal_threshold, 0);
/* Threshold to use Enhanced REP MOVSB. */
-long int __x86_rep_movsb_threshold attribute_hidden = 2048;
+CACHEINFO_VARIABLE (__x86_rep_movsb_threshold, 2048);
/* Threshold to use Enhanced REP STOSB. */
-long int __x86_rep_stosb_threshold attribute_hidden = 2048;
+CACHEINFO_VARIABLE (__x86_rep_stosb_threshold, 2048);
static void
get_common_cache_info (long int *shared_ptr, long int * shared_per_thread_ptr, unsigned int *threads_ptr,
diff --git a/sysdeps/x86/dl-diagnostics-cpu.c b/sysdeps/x86/dl-diagnostics-cpu.c
index 0ba286a828b69937..9215604ecf22344c 100644
--- a/sysdeps/x86/dl-diagnostics-cpu.c
+++ b/sysdeps/x86/dl-diagnostics-cpu.c
@@ -19,6 +19,13 @@
#include <dl-diagnostics.h>
#include <ldsodefs.h>
+#include <assert.h>
+#include <unistd.h>
+#include <cpu-features.h>
+#include <cpuid.h>
+#include <dl-cacheinfo.h>
+#include <cacheinfo.h>
+
static void
print_cpu_features_value (const char *label, uint64_t value)
{
@@ -81,19 +88,21 @@ _dl_diagnostics_cpu (void)
#include "cpu-features-preferred_feature_index_1.def"
#undef BIT
+ /* The cache information variables are only used for diagnostics and
+ are not initialized during startup. The values used at run time
+ are only in libc.so.6. */
+ init_cacheinfo ();
+
print_cpu_features_value ("xsave_state_size",
cpu_features->xsave_state_size);
print_cpu_features_value ("xsave_state_full_size",
cpu_features->xsave_state_full_size);
- print_cpu_features_value ("data_cache_size", cpu_features->data_cache_size);
- print_cpu_features_value ("shared_cache_size",
- cpu_features->shared_cache_size);
+ print_cpu_features_value ("data_cache_size", __x86_data_cache_size);
+ print_cpu_features_value ("shared_cache_size", __x86_shared_cache_size);
print_cpu_features_value ("non_temporal_threshold",
- cpu_features->non_temporal_threshold);
- print_cpu_features_value ("rep_movsb_threshold",
- cpu_features->rep_movsb_threshold);
- print_cpu_features_value ("rep_stosb_threshold",
- cpu_features->rep_stosb_threshold);
+ __x86_shared_non_temporal_threshold);
+ print_cpu_features_value ("rep_movsb_threshold", __x86_rep_movsb_threshold);
+ print_cpu_features_value ("rep_stosb_threshold", __x86_rep_stosb_threshold);
_Static_assert (offsetof (struct cpu_features, rep_stosb_threshold)
+ sizeof (cpu_features->rep_stosb_threshold)
== sizeof (*cpu_features),

View File

@ -1,229 +0,0 @@
Downstream-only patch to add arc4random to support/ for use in qsort
testing.
The arc4random implementation is up-to-date with upstream commit
2642002380aafb71a1d3b569b6d7ebeab3284816, with minor changes to keep
everything self-contained within support infrastructure. Unlike the
upstream version, this implementation is a cancellation point.
Included atomic.h needed by arc4random.
diff -Nrup a/support/Makefile b/support/Makefile
--- a/support/Makefile 2026-03-30 09:17:27.259848607 -0400
+++ b/support/Makefile 2026-03-30 09:15:31.748426860 -0400
@@ -39,6 +39,8 @@ libsupport-routines = \
resolv_response_context_free \
resolv_test \
set_fortify_handler \
+ support-arc4random \
+ support-arc4random_uniform \
support-xfstat \
support-xstat \
support_become_root \
diff -Nrup a/support/support-arc4random.c b/support/support-arc4random.c
--- a/support/support-arc4random.c 1969-12-31 19:00:00.000000000 -0500
+++ b/support/support-arc4random.c 2026-03-30 12:05:14.721241229 -0400
@@ -0,0 +1,100 @@
+/* Pseudo Random Number Generator
+ Copyright (C) 2022-2025 Free Software Foundation, Inc.
+ This file is part of the GNU C Library.
+
+ The GNU C Library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ The GNU C Library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with the GNU C Library; if not, see
+ <https://www.gnu.org/licenses/>. */
+
+#include <errno.h>
+#include <not-cancel.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <sys/mman.h>
+#include <sys/param.h>
+#include <sys/random.h>
+#include <poll.h>
+#include <atomic.h>
+
+static void
+arc4random_getrandom_failure (void)
+{
+ __libc_fatal ("Fatal glibc error: cannot get entropy for arc4random\n");
+}
+
+void
+arc4random_buf (void *p, size_t n)
+{
+ static int seen_initialized;
+ ssize_t l;
+ int fd;
+
+ if (n == 0)
+ return;
+
+ for (;;)
+ {
+ l = TEMP_FAILURE_RETRY (getrandom (p, n, 0));
+ if (l > 0)
+ {
+ if ((size_t) l == n)
+ return; /* Done reading, success. */
+ p = (uint8_t *) p + l;
+ n -= l;
+ continue; /* Interrupted by a signal; keep going. */
+ }
+ else if (l < 0 && errno == ENOSYS)
+ break; /* No syscall, so fallback to /dev/urandom. */
+ arc4random_getrandom_failure ();
+ }
+
+ if (atomic_load_relaxed (&seen_initialized) == 0)
+ {
+ /* Poll /dev/random as an approximation of RNG initialization. */
+ struct pollfd pfd = { .events = POLLIN };
+ pfd.fd = TEMP_FAILURE_RETRY (
+ __open64_nocancel ("/dev/random", O_RDONLY | O_CLOEXEC | O_NOCTTY));
+ if (pfd.fd < 0)
+ arc4random_getrandom_failure ();
+ if (TEMP_FAILURE_RETRY (poll (&pfd, 1, -1)) < 0)
+ arc4random_getrandom_failure ();
+ if (__close_nocancel (pfd.fd) < 0)
+ arc4random_getrandom_failure ();
+ atomic_store_relaxed (&seen_initialized, 1);
+ }
+
+ fd = TEMP_FAILURE_RETRY (
+ __open64_nocancel ("/dev/urandom", O_RDONLY | O_CLOEXEC | O_NOCTTY));
+ if (fd < 0)
+ arc4random_getrandom_failure ();
+ for (;;)
+ {
+ l = TEMP_FAILURE_RETRY (__read_nocancel (fd, p, n));
+ if (l <= 0)
+ arc4random_getrandom_failure ();
+ if ((size_t) l == n)
+ break; /* Done reading, success. */
+ p = (uint8_t *) p + l;
+ n -= l;
+ }
+ if (__close_nocancel (fd) < 0)
+ arc4random_getrandom_failure ();
+}
+
+uint32_t
+arc4random (void)
+{
+ uint32_t r;
+ arc4random_buf (&r, sizeof (r));
+ return r;
+}
diff -Nrup a/support/support-arc4random_uniform.c b/support/support-arc4random_uniform.c
--- a/support/support-arc4random_uniform.c 1969-12-31 19:00:00.000000000 -0500
+++ b/support/support-arc4random_uniform.c 2026-03-30 09:15:31.748993540 -0400
@@ -0,0 +1,70 @@
+/* Random pseudo generator number which returns a single 32 bit value
+ uniformly distributed but with an upper_bound.
+ Copyright (C) 2022-2025 Free Software Foundation, Inc.
+ This file is part of the GNU C Library.
+
+ The GNU C Library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ The GNU C Library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with the GNU C Library; if not, see
+ <https://www.gnu.org/licenses/>. */
+
+#include <stdlib.h>
+#include <sys/param.h>
+#include <support/support.h>
+
+/* Return a uniformly distributed random number less than N. The algorithm
+ calculates a mask being the lowest power of two bounding the upper bound
+ N, successively queries new random values, and rejects values outside of
+ the request range.
+
+ For reject values, it also tries if the remaining entropy could fit on
+ the asked range after range adjustment.
+
+ The algorithm avoids modulo and divide operations, which might be costly
+ depending on the architecture. */
+uint32_t
+arc4random_uniform (uint32_t n)
+{
+ if (n <= 1)
+ /* There is no valid return value for a zero limit, and 0 is the
+ only possible result for limit 1. */
+ return 0;
+
+ /* Powers of two are easy. */
+ if (powerof2 (n))
+ return arc4random () & (n - 1);
+
+ /* mask is the smallest power of 2 minus 1 number larger than n. */
+ int z = __builtin_clz (n);
+ uint32_t mask = ~UINT32_C(0) >> z;
+ int bits = CHAR_BIT * sizeof (uint32_t) - z;
+
+ while (1)
+ {
+ uint32_t value = arc4random ();
+
+ /* Return if the lower power of 2 minus 1 satisfy the condition. */
+ uint32_t r = value & mask;
+ if (r < n)
+ return r;
+
+ /* Otherwise check if remaining bits of entropy provides fits in the
+ bound. */
+ for (int bits_left = z; bits_left >= bits; bits_left -= bits)
+ {
+ value >>= bits;
+ r = value & mask;
+ if (r < n)
+ return r;
+ }
+ }
+}
diff -Nrup a/support/support.h b/support/support.h
--- a/support/support.h 2026-03-30 09:17:27.259975554 -0400
+++ b/support/support.h 2026-03-30 09:19:25.546023987 -0400
@@ -24,6 +24,7 @@
#define SUPPORT_H
#include <stdbool.h>
+#include <stdint.h>
#include <stddef.h>
#include <sys/cdefs.h>
/* For mode_t. */
@@ -130,6 +131,18 @@ extern void support_copy_file (const cha
extern ssize_t support_copy_file_range (int, off64_t *, int, off64_t *,
size_t, unsigned int);
+/* Return a random integer between zero and 2**32-1 (inclusive). */
+extern uint32_t arc4random (void)
+ __THROW __wur;
+
+/* Fill the buffer with random data. */
+extern void arc4random_buf (void *__buf, size_t __size)
+ __THROW __nonnull ((1));
+
+/* Return a random number between zero (inclusive) and the specified
+ limit (exclusive). */
+extern uint32_t arc4random_uniform (__uint32_t __upper_bound)
+ __THROW __wur;
__END_DECLS

View File

@ -1,43 +0,0 @@
commit f8cfb6836e8d91bb789b2e7fd65338d6f5bd459c
Author: Florian Weimer <fweimer@redhat.com>
Date: Wed Nov 8 15:18:02 2023 +0100
stdlib: Avoid element self-comparisons in qsort
This improves compatibility with applications which assume that qsort
does not invoke the comparison function with equal pointer arguments.
The newly introduced branches should be predictable, as leading to a
call to the comparison function. If the prediction fails, we avoid
calling the function.
Reviewed-by: Adhemerval Zanella <adhemerval.zanella@linaro.org>
diff --git a/stdlib/qsort.c b/stdlib/qsort.c
index cb1619aa0ae7de72..2ee39e2c492f792e 100644
--- a/stdlib/qsort.c
+++ b/stdlib/qsort.c
@@ -137,7 +137,7 @@ siftdown (void *base, size_t size, size_t k, size_t n,
if (j < n && cmp (base + (j * size), base + ((j + 1) * size), arg) < 0)
j++;
- if (cmp (base + (k * size), base + (j * size), arg) >= 0)
+ if (j == k || cmp (base + (k * size), base + (j * size), arg) >= 0)
break;
do_swap (base + (size * j), base + (k * size), size, swap_type);
@@ -333,10 +333,12 @@ __qsort_r (void *const pbase, size_t total_elems, size_t size,
that this algorithm runs much faster than others. */
do
{
- while ((*cmp) ((void *) left_ptr, (void *) mid, arg) < 0)
+ while (left_ptr != mid
+ && (*cmp) ((void *) left_ptr, (void *) mid, arg) < 0)
left_ptr += size;
- while ((*cmp) ((void *) mid, (void *) right_ptr, arg) < 0)
+ while (right_ptr != mid
+ && (*cmp) ((void *) mid, (void *) right_ptr, arg) < 0)
right_ptr -= size;
if (left_ptr < right_ptr)

View File

@ -1,25 +0,0 @@
commit e4d8117b82065dc72e8df80097360e7c05a349b9
Author: Florian Weimer <fweimer@redhat.com>
Date: Tue Nov 21 16:45:35 2023 +0100
stdlib: Avoid another self-comparison in qsort
In the insertion phase, we could run off the start of the array if the
comparison function never runs zero. In that case, it never finds the
initial element that terminates the iteration.
Reviewed-by: Adhemerval Zanella <adhemerval.zanella@linaro.org>
diff --git a/stdlib/qsort.c b/stdlib/qsort.c
index 2ee39e2c492f792e..0d5f8b92e8072965 100644
--- a/stdlib/qsort.c
+++ b/stdlib/qsort.c
@@ -218,7 +218,7 @@ insertion_sort_qsort_partitions (void *const pbase, size_t total_elems,
while ((run_ptr += size) <= end_ptr)
{
tmp_ptr = run_ptr - size;
- while (cmp (run_ptr, tmp_ptr, arg) < 0)
+ while (run_ptr != tmp_ptr && cmp (run_ptr, tmp_ptr, arg) < 0)
tmp_ptr -= size;
tmp_ptr += size;

View File

@ -1,273 +0,0 @@
commit 55364e1f7dfab372f0710513c4d1c967c4965f71
Author: Florian Weimer <fweimer@redhat.com>
Date: Tue Nov 21 16:45:35 2023 +0100
stdlib: Handle various corner cases in the fallback heapsort for qsort
The previous implementation did not consistently apply the rule that
the child nodes of node K are at 2 * K + 1 and 2 * K + 2, or
that the parent node is at (K - 1) / 2.
Add an internal test that targets the heapsort implementation
directly.
Reported-by: Stepan Golosunov <stepan@golosunov.pp.ru>
Reviewed-by: Adhemerval Zanella <adhemerval.zanella@linaro.org>
diff --git a/stdlib/Makefile b/stdlib/Makefile
index 4039e5395eeea2b0..ee005ce8caa48abe 100644
--- a/stdlib/Makefile
+++ b/stdlib/Makefile
@@ -254,6 +254,7 @@ tests := \
# tests
tests-internal := \
+ tst-qsort4 \
tst-strtod1i \
tst-strtod3 \
tst-strtod4 \
diff --git a/stdlib/qsort.c b/stdlib/qsort.c
index 0d5f8b92e8072965..b207c12d2f0a38cc 100644
--- a/stdlib/qsort.c
+++ b/stdlib/qsort.c
@@ -126,29 +126,44 @@ pop (stack_node *top, char **lo, char **hi, size_t *depth)
return top;
}
-/* NB: N is inclusive bound for BASE. */
+/* Establish the heap condition at index K, that is, the key at K will
+ not be less than either of its children, at 2 * K + 1 and 2 * K + 2
+ (if they exist). N is the last valid index. */
static inline void
siftdown (void *base, size_t size, size_t k, size_t n,
enum swap_type_t swap_type, __compar_d_fn_t cmp, void *arg)
{
- while (k <= n / 2)
+ /* There can only be a heap condition violation if there are
+ children. */
+ while (2 * k + 1 <= n)
{
- size_t j = 2 * k;
+ /* Left child. */
+ size_t j = 2 * k + 1;
+ /* If the right child is larger, use it. */
if (j < n && cmp (base + (j * size), base + ((j + 1) * size), arg) < 0)
j++;
+ /* If k is already >= to its children, we are done. */
if (j == k || cmp (base + (k * size), base + (j * size), arg) >= 0)
break;
+ /* Heal the violation. */
do_swap (base + (size * j), base + (k * size), size, swap_type);
+
+ /* Swapping with j may have introduced a violation at j. Fix
+ it in the next loop iteration. */
k = j;
}
}
+/* Establish the heap condition for the indices 0 to N (inclusive). */
static inline void
heapify (void *base, size_t size, size_t n, enum swap_type_t swap_type,
__compar_d_fn_t cmp, void *arg)
{
+ /* If n is odd, k = n / 2 has a left child at n, so this is the
+ largest index that can have a heap condition violation regarding
+ its children. */
size_t k = n / 2;
while (1)
{
@@ -158,32 +173,38 @@ heapify (void *base, size_t size, size_t n, enum swap_type_t swap_type,
}
}
-/* A non-recursive heapsort, used on introsort implementation as a fallback
- routine with worst-case performance of O(nlog n) and worst-case space
- complexity of O(1). It sorts the array starting at BASE and ending at
- END, with each element of SIZE bytes. The SWAP_TYPE is the callback
- function used to swap elements, and CMP is the function used to compare
- elements. */
+/* A non-recursive heapsort, used on introsort implementation as a
+ fallback routine with worst-case performance of O(nlog n) and
+ worst-case space complexity of O(1). It sorts the array starting
+ at BASE and ending at END (inclusive), with each element of SIZE
+ bytes. The SWAP_TYPE is the callback function used to swap
+ elements, and CMP is the function used to compare elements. */
static void
heapsort_r (void *base, void *end, size_t size, enum swap_type_t swap_type,
__compar_d_fn_t cmp, void *arg)
{
- const size_t count = ((uintptr_t) end - (uintptr_t) base) / size;
-
- if (count < 2)
+ size_t n = ((uintptr_t) end - (uintptr_t) base) / size;
+ if (n <= 1)
+ /* Handled by insertion sort. */
return;
- size_t n = count - 1;
-
/* Build the binary heap, largest value at the base[0]. */
heapify (base, size, n, swap_type, cmp, arg);
- /* On each iteration base[0:n] is the binary heap, while base[n:count]
- is sorted. */
- while (n > 0)
+ while (true)
{
+ /* Indices 0 .. n contain the binary heap. Extract the largest
+ element put it into the final position in the array. */
do_swap (base, base + (n * size), size, swap_type);
+
+ /* The heap is now one element shorter. */
n--;
+ if (n == 0)
+ break;
+
+ /* By swapping in elements 0 and the previous value of n (now at
+ n + 1), we likely introduced a heap condition violation. Fix
+ it for the reduced heap. */
siftdown (base, size, 0, n, swap_type, cmp, arg);
}
}
diff --git a/stdlib/tst-qsort4.c b/stdlib/tst-qsort4.c
new file mode 100644
index 0000000000000000..a7abaa1a37461666
--- /dev/null
+++ b/stdlib/tst-qsort4.c
@@ -0,0 +1,134 @@
+/* Test the heapsort implementation behind qsort.
+ Copyright (C) 2023 Free Software Foundation, Inc.
+ This file is part of the GNU C Library.
+
+ The GNU C Library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ The GNU C Library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with the GNU C Library; if not, see
+ <http://www.gnu.org/licenses/>. */
+
+#include "qsort.c"
+
+#include <stdio.h>
+#include <support/check.h>
+#include <support/support.h>
+
+static int
+cmp (const void *a1, const void *b1, void *closure)
+{
+ const signed char *a = a1;
+ const signed char *b = b1;
+ return *a - *b;
+}
+
+/* Wrapper around heapsort_r that set ups the required variables. */
+static void
+heapsort_wrapper (void *const pbase, size_t total_elems, size_t size,
+ __compar_d_fn_t cmp, void *arg)
+{
+ char *base_ptr = (char *) pbase;
+ char *lo = base_ptr;
+ char *hi = &lo[size * (total_elems - 1)];
+
+ if (total_elems <= 1)
+ /* Avoid lossage with unsigned arithmetic below. */
+ return;
+
+ enum swap_type_t swap_type;
+ if (is_aligned (pbase, size, 8))
+ swap_type = SWAP_WORDS_64;
+ else if (is_aligned (pbase, size, 4))
+ swap_type = SWAP_WORDS_32;
+ else
+ swap_type = SWAP_BYTES;
+ heapsort_r (lo, hi, size, swap_type, cmp, arg);
+}
+
+static void
+check_one_sort (signed char *array, int length)
+{
+ signed char *copy = xmalloc (length);
+ memcpy (copy, array, length);
+ heapsort_wrapper (copy, length, 1, cmp, NULL);
+
+ /* Verify that the result is sorted. */
+ for (int i = 1; i < length; ++i)
+ if (copy[i] < copy[i - 1])
+ {
+ support_record_failure ();
+ printf ("error: sorting failure for length %d at offset %d\n",
+ length, i - 1);
+ printf ("input:");
+ for (int i = 0; i < length; ++i)
+ printf (" %d", array[i]);
+ printf ("\noutput:");
+ for (int i = 0; i < length; ++i)
+ printf (" %d", copy[i]);
+ putchar ('\n');
+ break;
+ }
+
+ /* Verify that no elements went away or were added. */
+ {
+ int expected_counts[256];
+ for (int i = 0; i < length; ++i)
+ ++expected_counts[array[i] & 0xff];
+ int actual_counts[256];
+ for (int i = 0; i < length; ++i)
+ ++actual_counts[copy[i] & 0xff];
+ for (int i = 0; i < 256; ++i)
+ TEST_COMPARE (expected_counts[i], expected_counts[i]);
+ }
+
+ free (copy);
+}
+
+/* Enumerate all possible combinations of LENGTH elements. */
+static void
+check_combinations (int length, signed char *start, int offset)
+{
+ if (offset == length)
+ check_one_sort (start, length);
+ else
+ for (int i = 0; i < length; ++i)
+ {
+ start[offset] = i;
+ check_combinations(length, start, offset + 1);
+ }
+}
+
+static int
+do_test (void)
+{
+ /* A random permutation of 20 values. */
+ check_one_sort ((signed char[20]) {5, 12, 16, 10, 14, 11, 9, 13, 8, 15,
+ 0, 17, 3, 7, 1, 18, 2, 19, 4, 6}, 20);
+
+
+ /* A permutation that appeared during adversarial testing for the
+ quicksort pass. */
+ check_one_sort ((signed char[16]) {15, 3, 4, 2, 1, 0, 8, 7, 6, 5, 14,
+ 13, 12, 11, 10, 9}, 16);
+
+ /* Array lengths 2 and less are not handled by heapsort_r and
+ deferred to insertion sort. */
+ for (int i = 3; i <= 8; ++i)
+ {
+ signed char *buf = xmalloc (i);
+ check_combinations (i, buf, 0);
+ free (buf);
+ }
+
+ return 0;
+}
+
+#include <support/test-driver.c>

View File

@ -1,243 +0,0 @@
commit 64e4acf24da15c11cb83f933947df3b2e8a700cd
Author: Florian Weimer <fweimer@redhat.com>
Date: Tue Nov 21 16:45:35 2023 +0100
stdlib: The qsort implementation needs to use heapsort in more cases
The existing logic avoided internal stack overflow. To avoid
a denial-of-service condition with adversarial input, it is necessary
to fall over to heapsort if tail-recursing deeply, too, which does
not result in a deep stack of pending partitions.
The new test stdlib/tst-qsort5 is based on Douglas McIlroy's paper
on this subject.
Reviewed-by: Adhemerval Zanella <adhemerval.zanella@linaro.org>
Conflicts:
stdlib/Makefile
(Adjust for missing backport of getenv tests.)
diff -Nrup a/stdlib/Makefile b/stdlib/Makefile
--- a/stdlib/Makefile 2026-03-25 13:56:09.898523597 -0400
+++ b/stdlib/Makefile 2026-03-25 14:24:03.909095717 -0400
@@ -184,6 +184,7 @@ tests := \
tst-qsort \
tst-qsort2 \
tst-qsort3 \
+ tst-qsort5 \
testmb2 \
bug-strtod2 \
tst-atof1 \
@@ -456,4 +457,5 @@ $(objpfx)tst-setcontext3.out: tst-setcon
$(common-objpfx)stdlib/ > $@; \
$(evaluate-test)
+$(objpfx)tst-qsort5: $(libm)
$(objpfx)tst-makecontext: $(libdl)
diff -Nrup a/stdlib/qsort.c b/stdlib/qsort.c
--- a/stdlib/qsort.c 2026-03-25 13:56:09.898857270 -0400
+++ b/stdlib/qsort.c 2026-03-25 13:37:08.329722448 -0400
@@ -390,14 +390,23 @@ __qsort_r (void *const pbase, size_t tot
{
if ((size_t) (hi - left_ptr) <= max_thresh)
/* Ignore both small partitions. */
- top = pop (top, &lo, &hi, &depth);
+ {
+ top = pop (top, &lo, &hi, &depth);
+ --depth;
+ }
else
- /* Ignore small left partition. */
- lo = left_ptr;
+ {
+ /* Ignore small left partition. */
+ lo = left_ptr;
+ --depth;
+ }
}
else if ((size_t) (hi - left_ptr) <= max_thresh)
/* Ignore small right partition. */
- hi = right_ptr;
+ {
+ hi = right_ptr;
+ --depth;
+ }
else if ((right_ptr - lo) > (hi - left_ptr))
{
/* Push larger left partition indices. */
diff -Nrup a/stdlib/tst-qsort5.c b/stdlib/tst-qsort5.c
--- a/stdlib/tst-qsort5.c 1969-12-31 19:00:00.000000000 -0500
+++ b/stdlib/tst-qsort5.c 2026-03-25 13:37:08.329928112 -0400
@@ -0,0 +1,171 @@
+/* Adversarial test for qsort_r.
+ Copyright (C) 2023 Free Software Foundation, Inc.
+ This file is part of the GNU C Library.
+
+ The GNU C Library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ The GNU C Library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with the GNU C Library; if not, see
+ <http://www.gnu.org/licenses/>. */
+
+/* The approach follows Douglas McIlroy, A Killer Adversary for
+ Quicksort. Software—Practice and Experience 29 (1999) 341-344.
+ Downloaded <http://www.cs.dartmouth.edu/~doug/mdmspe.pdf>
+ (2023-11-17). */
+
+#include <math.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <support/check.h>
+#include <support/support.h>
+
+struct context
+{
+ /* Called the gas value in the paper. This value is larger than all
+ other values (length minus one will do), so comparison with any
+ decided value has a known result. */
+ int undecided_value;
+
+ /* If comparing undecided values, one of them as to be assigned a
+ value to ensure consistency with future comparisons. This is the
+ value that will be used. Starts out at zero. */
+ int next_decided;
+
+ /* Used to trick pivot selection. Deciding the value for the last
+ seen undcided value in a decided/undecided comparison happens
+ to trick the many qsort implementations. */
+ int last_undecided_index;
+
+ /* This array contains the actually asigned values. The call to
+ qsort_r sorts a different array that contains indices into this
+ array. */
+ int *decided_values;
+};
+
+static int
+compare_opponent (const void *l1, const void *r1, void *ctx1)
+{
+ const int *l = l1;
+ const int *r = r1;
+ struct context *ctx = ctx1;
+ int rvalue = ctx->decided_values[*r];
+ int lvalue = ctx->decided_values[*l];
+
+ if (lvalue == ctx->undecided_value)
+ {
+ if (rvalue == ctx->undecided_value)
+ {
+ /* Both values are undecided. In this case, make a decision
+ for the last-used undecided value. This is tweak is very
+ specific to quicksort. */
+ if (*l == ctx->last_undecided_index)
+ {
+ ctx->decided_values[*l] = ctx->next_decided;
+ ++ctx->next_decided;
+ /* The undecided value or *r is greater. */
+ return -1;
+ }
+ else
+ {
+ ctx->decided_values[*r] = ctx->next_decided;
+ ++ctx->next_decided;
+ /* The undecided value for *l is greater. */
+ return 1;
+ }
+ }
+ else
+ {
+ ctx->last_undecided_index = *l;
+ return 1;
+ }
+ }
+ else
+ {
+ /* *l is a decided value. */
+ if (rvalue == ctx->undecided_value)
+ {
+ ctx->last_undecided_index = *r;
+ /* The undecided value for *r is greater. */
+ return -1;
+ }
+ else
+ return lvalue - rvalue;
+ }
+}
+
+/* Return a pointer to the adversarial permutation of length N. */
+static int *
+create_permutation (size_t n)
+{
+ struct context ctx =
+ {
+ .undecided_value = n - 1, /* Larger than all other values. */
+ .decided_values = xcalloc (n, sizeof (int)),
+ };
+ for (size_t i = 0; i < n; ++i)
+ ctx.decided_values[i] = ctx.undecided_value;
+ int *scratch = xcalloc (n, sizeof (int));
+ for (size_t i = 0; i < n; ++i)
+ scratch[i] = i;
+ qsort_r (scratch, n, sizeof (*scratch), compare_opponent, &ctx);
+ free (scratch);
+ return ctx.decided_values;
+}
+
+/* Callback function for qsort which counts the number of invocations
+ in *CLOSURE. */
+static int
+compare_counter (const void *l1, const void *r1, void *closure)
+{
+ const int *l = l1;
+ const int *r = r1;
+ unsigned long long int *counter = closure;
+ ++*counter;
+ return *l - *r;
+}
+
+/* Count the comparisons required for an adversarial permutation of
+ length N. */
+static unsigned long long int
+count_comparisons (size_t n)
+{
+ int *array = create_permutation (n);
+ unsigned long long int counter = 0;
+ qsort_r (array, n, sizeof (*array), compare_counter, &counter);
+ free (array);
+ return counter;
+}
+
+/* Check the scaling factor for one adversarial permutation of length
+ N, and report some statistics. */
+static void
+check_one_n (size_t n)
+{
+ unsigned long long int count = count_comparisons (n);
+ double factor = count / (n * log (count));
+ printf ("info: length %zu: %llu comparisons ~ %f * n * log (n)\n",
+ n, count, factor);
+ /* This is an arbitrary factor which is true for the current
+ implementation across a wide range of sizes. */
+ TEST_VERIFY (factor <= 4.5);
+}
+
+static int
+do_test (void)
+{
+ check_one_n (100);
+ check_one_n (1000);
+ for (int i = 1; i <= 15; ++i)
+ check_one_n (i * 10 * 1000);
+ return 0;
+}
+
+#include <support/test-driver.c>

View File

@ -1,105 +0,0 @@
commit b9390ba93676c4b1e87e218af5e7e4bb596312ac
Author: Florian Weimer <fweimer@redhat.com>
Date: Mon Dec 4 06:35:56 2023 +0100
stdlib: Fix array bounds protection in insertion sort phase of qsort
The previous check did not do anything because tmp_ptr already
points before run_ptr due to the way it is initialized.
Fixes commit e4d8117b82065dc72e8df80097360e7c05a349b9
("stdlib: Avoid another self-comparison in qsort").
Reviewed-by: Adhemerval Zanella <adhemerval.zanella@linaro.org>
Conflicts:
stdlib/Makefile
( Usual Makefile differences.)
diff -Nrup a/stdlib/Makefile b/stdlib/Makefile
--- a/stdlib/Makefile 2026-03-25 14:51:00.600156599 -0400
+++ b/stdlib/Makefile 2026-03-25 14:52:13.822994373 -0400
@@ -185,6 +185,7 @@ tests := \
tst-qsort2 \
tst-qsort3 \
tst-qsort5 \
+ tst-qsort6 \
testmb2 \
bug-strtod2 \
tst-atof1 \
diff -Nrup a/stdlib/qsort.c b/stdlib/qsort.c
--- a/stdlib/qsort.c 2026-03-25 14:51:00.600987843 -0400
+++ b/stdlib/qsort.c 2026-03-25 14:49:55.308712656 -0400
@@ -239,7 +239,7 @@ insertion_sort_qsort_partitions (void *c
while ((run_ptr += size) <= end_ptr)
{
tmp_ptr = run_ptr - size;
- while (run_ptr != tmp_ptr && cmp (run_ptr, tmp_ptr, arg) < 0)
+ while (tmp_ptr != base_ptr && cmp (run_ptr, tmp_ptr, arg) < 0)
tmp_ptr -= size;
tmp_ptr += size;
diff -Nrup a/stdlib/tst-qsort6.c b/stdlib/tst-qsort6.c
--- a/stdlib/tst-qsort6.c 1969-12-31 19:00:00.000000000 -0500
+++ b/stdlib/tst-qsort6.c 2026-03-25 14:49:55.309005934 -0400
@@ -0,0 +1,60 @@
+/* Test qsort with invalid comparison functions.
+ Copyright (C) 2023 Free Software Foundation, Inc.
+ This file is part of the GNU C Library.
+
+ The GNU C Library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ The GNU C Library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with the GNU C Library; if not, see
+ <http://www.gnu.org/licenses/>. */
+
+#include <array_length.h>
+#include <stdlib.h>
+#include <support/check.h>
+
+/* Invalid comparison function that always returns -1. */
+static int
+invalid_compare_1 (const void *a1, const void *b1)
+{
+ const int *a = a1;
+ const int *b = b1;
+ /* Check that the marker value matches, which means that we are
+ likely within the array. */
+ TEST_COMPARE (*a, 842523635);
+ TEST_COMPARE (*b, 842523635);
+ TEST_VERIFY_EXIT (*a == 842523635);
+ TEST_VERIFY_EXIT (*b == 842523635);
+ return -1;
+}
+
+/* Invalid comparison function that always returns 1. */
+static int
+invalid_compare_2 (const void *a1, const void *b1)
+{
+ const int *a = a1;
+ const int *b = b1;
+ TEST_COMPARE (*a, 842523635);
+ TEST_COMPARE (*b, 842523635);
+ TEST_VERIFY_EXIT (*a == 842523635);
+ TEST_VERIFY_EXIT (*b == 842523635);
+ return 1;
+}
+
+static int
+do_test (void)
+{
+ int array[] = {842523635, 842523635, 842523635, 842523635, 842523635};
+ qsort (array, array_length (array), sizeof (array[0]), invalid_compare_1);
+ qsort (array, array_length (array), sizeof (array[0]), invalid_compare_2);
+ return 0;
+}
+
+#include <support/test-driver.c>

View File

@ -1,871 +0,0 @@
commit 709fbd3ec3595f2d1076b4fec09a739327459288
Author: Adhemerval Zanella <adhemerval.zanella@linaro.org>
Date: Mon Jan 15 11:07:21 2024 -0300
stdlib: Reinstate stable mergesort implementation on qsort
The mergesort removal from qsort implementation (commit 03bf8357e8)
had the side-effect of making sorting nonstable. Although neither
POSIX nor C standard specify that qsort should be stable, it seems
that it has become an instance of Hyrum's law where multiple programs
expect it.
Also, the resulting introsort implementation is not faster than
the previous mergesort (which makes the change even less appealing).
This patch restores the previous mergesort implementation, with the
exception of machinery that checks the resulting allocation against
the _SC_PHYS_PAGES (it only adds complexity and the heuristic not
always make sense depending on the system configuration and load).
The alloca usage was replaced with a fixed-size buffer.
For the fallback mechanism, the implementation uses heapsort. It is
simpler than quicksort, and it does not suffer from adversarial
inputs. With memory overcommit, it should be rarely triggered.
The drawback is mergesort requires O(n) extra space, and since it is
allocated with malloc the function is AS-signal-unsafe. It should be
feasible to change it to use mmap, although I am not sure how urgent
it is. The heapsort is also nonstable, so programs that require a
stable sort would still be subject to this latent issue.
The tst-qsort5 is removed since it will not create quicksort adversarial
inputs with the current qsort_r implementation.
Checked on x86_64-linux-gnu and aarch64-linux-gnu.
Reviewed-by: Florian Weimer <fweimer@redhat.com>
Conflicts:
stdlib/Makefile
(Usual Makefile differences.)
diff -Nrup a/manual/argp.texi b/manual/argp.texi
--- a/manual/argp.texi 2026-03-25 15:00:34.008145868 -0400
+++ b/manual/argp.texi 2026-03-25 14:58:51.499679969 -0400
@@ -735,7 +735,7 @@ for options, bad phase of the moon, etc.
@c hol_set_group ok
@c hol_find_entry ok
@c hol_sort @mtslocale @acucorrupt
-@c qsort dup
+@c qsort dup @acucorrupt
@c hol_entry_qcmp @mtslocale
@c hol_entry_cmp @mtslocale
@c group_cmp ok
diff -Nrup a/manual/locale.texi b/manual/locale.texi
--- a/manual/locale.texi 2026-03-25 15:00:34.009516509 -0400
+++ b/manual/locale.texi 2026-03-25 14:58:51.500013812 -0400
@@ -253,7 +253,7 @@ The symbols in this section are defined
@c calculate_head_size ok
@c __munmap ok
@c compute_hashval ok
-@c qsort dup
+@c qsort dup @acucorrupt
@c rangecmp ok
@c malloc @ascuheap @acsmem
@c strdup @ascuheap @acsmem
diff -Nrup a/manual/search.texi b/manual/search.texi
--- a/manual/search.texi 2026-03-25 15:00:34.010463389 -0400
+++ b/manual/search.texi 2026-03-25 14:58:51.500247569 -0400
@@ -159,7 +159,7 @@ To sort an array using an arbitrary comp
@deftypefun void qsort (void *@var{array}, size_t @var{count}, size_t @var{size}, comparison_fn_t @var{compare})
@standards{ISO, stdlib.h}
-@safety{@prelim{}@mtsafe{}@assafe{}@acsafe{}}
+@safety{@prelim{}@mtsafe{}@assafe{}@acunsafe{@acucorrupt{}}}
The @code{qsort} function sorts the array @var{array}. The array
contains @var{count} elements, each of which is of size @var{size}.
@@ -199,8 +199,9 @@ Functions}):
The @code{qsort} function derives its name from the fact that it was
originally implemented using the ``quick sort'' algorithm.
-The implementation of @code{qsort} in this library is an in-place sort
-and uses a constant extra space (allocated on the stack).
+The implementation of @code{qsort} attempts to allocate auxiliary storage
+and use the merge sort algorithm, without violating C standard requirement
+that arguments passed to the comparison function point within the array.
@end deftypefun
@node Search/Sort Example
diff -Nrup a/stdlib/Makefile b/stdlib/Makefile
--- a/stdlib/Makefile 2026-03-25 15:00:34.144468914 -0400
+++ b/stdlib/Makefile 2026-03-25 15:14:19.775090144 -0400
@@ -184,7 +184,6 @@ tests := \
tst-qsort \
tst-qsort2 \
tst-qsort3 \
- tst-qsort5 \
tst-qsort6 \
testmb2 \
bug-strtod2 \
diff -Nrup a/stdlib/qsort.c b/stdlib/qsort.c
--- a/stdlib/qsort.c 2026-03-25 15:00:34.145271423 -0400
+++ b/stdlib/qsort.c 2026-03-25 14:58:51.500702490 -0400
@@ -20,6 +20,7 @@
Engineering a sort function; Jon Bentley and M. Douglas McIlroy;
Software - Practice and Experience; Vol. 23 (11), 1249-1265, 1993. */
+#include <errno.h>
#include <limits.h>
#include <memswap.h>
#include <stdlib.h>
@@ -33,9 +34,13 @@ enum swap_type_t
{
SWAP_WORDS_64,
SWAP_WORDS_32,
+ SWAP_VOID_ARG,
SWAP_BYTES
};
+typedef uint32_t __attribute__ ((__may_alias__)) u32_alias_t;
+typedef uint64_t __attribute__ ((__may_alias__)) u64_alias_t;
+
/* If this function returns true, elements can be safely copied using word
loads and stores. Otherwise, it might not be safe. BASE (as an integer)
must be a multiple of the word alignment. SIZE must be a multiple of
@@ -52,7 +57,6 @@ is_aligned (const void *base, size_t siz
static inline void
swap_words_64 (void * restrict a, void * restrict b, size_t n)
{
- typedef uint64_t __attribute__ ((__may_alias__)) u64_alias_t;
do
{
n -= 8;
@@ -65,7 +69,6 @@ swap_words_64 (void * restrict a, void *
static inline void
swap_words_32 (void * restrict a, void * restrict b, size_t n)
{
- typedef uint32_t __attribute__ ((__may_alias__)) u32_alias_t;
do
{
n -= 4;
@@ -89,43 +92,6 @@ do_swap (void * restrict a, void * restr
__memswap (a, b, size);
}
-/* Discontinue quicksort algorithm when partition gets below this size.
- This particular magic number was chosen to work best on a Sun 4/260. */
-#define MAX_THRESH 4
-
-/* Stack node declarations used to store unfulfilled partition obligations. */
-typedef struct
- {
- char *lo;
- char *hi;
- size_t depth;
- } stack_node;
-
-/* The stack needs log (total_elements) entries (we could even subtract
- log(MAX_THRESH)). Since total_elements has type size_t, we get as
- upper bound for log (total_elements):
- bits per byte (CHAR_BIT) * sizeof(size_t). */
-enum { STACK_SIZE = CHAR_BIT * sizeof (size_t) };
-
-static inline stack_node *
-push (stack_node *top, char *lo, char *hi, size_t depth)
-{
- top->lo = lo;
- top->hi = hi;
- top->depth = depth;
- return ++top;
-}
-
-static inline stack_node *
-pop (stack_node *top, char **lo, char **hi, size_t *depth)
-{
- --top;
- *lo = top->lo;
- *hi = top->hi;
- *depth = top->depth;
- return top;
-}
-
/* Establish the heap condition at index K, that is, the key at K will
not be less than either of its children, at 2 * K + 1 and 2 * K + 2
(if they exist). N is the last valid index. */
@@ -173,21 +139,35 @@ heapify (void *base, size_t size, size_t
}
}
-/* A non-recursive heapsort, used on introsort implementation as a
- fallback routine with worst-case performance of O(nlog n) and
- worst-case space complexity of O(1). It sorts the array starting
- at BASE and ending at END (inclusive), with each element of SIZE
- bytes. The SWAP_TYPE is the callback function used to swap
- elements, and CMP is the function used to compare elements. */
+static enum swap_type_t
+get_swap_type (void *const pbase, size_t size)
+{
+ if ((size & (sizeof (uint32_t) - 1)) == 0
+ && ((uintptr_t) pbase) % __alignof__ (uint32_t) == 0)
+ {
+ if (size == sizeof (uint32_t))
+ return SWAP_WORDS_32;
+ else if (size == sizeof (uint64_t)
+ && ((uintptr_t) pbase) % __alignof__ (uint64_t) == 0)
+ return SWAP_WORDS_64;
+ }
+ return SWAP_BYTES;
+}
+
+
+/* A non-recursive heapsort with worst-case performance of O(nlog n) and
+ worst-case space complexity of O(1). It sorts the array starting at
+ BASE with n + 1 elements of SIZE bytes. The SWAP_TYPE is the callback
+ function used to swap elements, and CMP is the function used to compare
+ elements. */
static void
-heapsort_r (void *base, void *end, size_t size, enum swap_type_t swap_type,
- __compar_d_fn_t cmp, void *arg)
+heapsort_r (void *base, size_t n, size_t size, __compar_d_fn_t cmp, void *arg)
{
- size_t n = ((uintptr_t) end - (uintptr_t) base) / size;
if (n <= 1)
- /* Handled by insertion sort. */
return;
+ enum swap_type_t swap_type = get_swap_type (base, size);
+
/* Build the binary heap, largest value at the base[0]. */
heapify (base, size, n, swap_type, cmp, arg);
@@ -209,226 +189,226 @@ heapsort_r (void *base, void *end, size_
}
}
-static inline void
-insertion_sort_qsort_partitions (void *const pbase, size_t total_elems,
- size_t size, enum swap_type_t swap_type,
- __compar_d_fn_t cmp, void *arg)
-{
- char *base_ptr = (char *) pbase;
- char *const end_ptr = &base_ptr[size * (total_elems - 1)];
- char *tmp_ptr = base_ptr;
-#define min(x, y) ((x) < (y) ? (x) : (y))
- const size_t max_thresh = MAX_THRESH * size;
- char *thresh = min(end_ptr, base_ptr + max_thresh);
- char *run_ptr;
-
- /* Find smallest element in first threshold and place it at the
- array's beginning. This is the smallest array element,
- and the operation speeds up insertion sort's inner loop. */
-
- for (run_ptr = tmp_ptr + size; run_ptr <= thresh; run_ptr += size)
- if (cmp (run_ptr, tmp_ptr, arg) < 0)
- tmp_ptr = run_ptr;
+/* The maximum size in bytes required by mergesort that will be provided
+ through a buffer allocated in the stack. */
+#define QSORT_STACK_SIZE 1024
+
+/* Elements larger than this value will be sorted through indirect sorting
+ to minimize the need to memory swap calls. */
+#define INDIRECT_SORT_SIZE_THRES 32
+
+struct msort_param
+{
+ size_t s;
+ enum swap_type_t var;
+ __compar_d_fn_t cmp;
+ void *arg;
+ char *t;
+};
- if (tmp_ptr != base_ptr)
- do_swap (tmp_ptr, base_ptr, size, swap_type);
+static void
+msort_with_tmp (const struct msort_param *p, void *b, size_t n)
+{
+ char *b1, *b2;
+ size_t n1, n2;
- /* Insertion sort, running from left-hand-side up to right-hand-side. */
+ if (n <= 1)
+ return;
- run_ptr = base_ptr + size;
- while ((run_ptr += size) <= end_ptr)
+ n1 = n / 2;
+ n2 = n - n1;
+ b1 = b;
+ b2 = (char *) b + (n1 * p->s);
+
+ msort_with_tmp (p, b1, n1);
+ msort_with_tmp (p, b2, n2);
+
+ char *tmp = p->t;
+ const size_t s = p->s;
+ __compar_d_fn_t cmp = p->cmp;
+ void *arg = p->arg;
+ switch (p->var)
{
- tmp_ptr = run_ptr - size;
- while (tmp_ptr != base_ptr && cmp (run_ptr, tmp_ptr, arg) < 0)
- tmp_ptr -= size;
-
- tmp_ptr += size;
- if (tmp_ptr != run_ptr)
- {
- char *trav;
-
- trav = run_ptr + size;
- while (--trav >= run_ptr)
- {
- char c = *trav;
- char *hi, *lo;
-
- for (hi = lo = trav; (lo -= size) >= tmp_ptr; hi = lo)
- *hi = *lo;
- *hi = c;
- }
- }
+ case SWAP_WORDS_32:
+ while (n1 > 0 && n2 > 0)
+ {
+ if (cmp (b1, b2, arg) <= 0)
+ {
+ *(u32_alias_t *) tmp = *(u32_alias_t *) b1;
+ b1 += sizeof (u32_alias_t);
+ --n1;
+ }
+ else
+ {
+ *(u32_alias_t *) tmp = *(u32_alias_t *) b2;
+ b2 += sizeof (u32_alias_t);
+ --n2;
+ }
+ tmp += sizeof (u32_alias_t);
+ }
+ break;
+ case SWAP_WORDS_64:
+ while (n1 > 0 && n2 > 0)
+ {
+ if (cmp (b1, b2, arg) <= 0)
+ {
+ *(u64_alias_t *) tmp = *(u64_alias_t *) b1;
+ b1 += sizeof (u64_alias_t);
+ --n1;
+ }
+ else
+ {
+ *(u64_alias_t *) tmp = *(u64_alias_t *) b2;
+ b2 += sizeof (u64_alias_t);
+ --n2;
+ }
+ tmp += sizeof (u64_alias_t);
+ }
+ break;
+ case SWAP_VOID_ARG:
+ while (n1 > 0 && n2 > 0)
+ {
+ if ((*cmp) (*(const void **) b1, *(const void **) b2, arg) <= 0)
+ {
+ *(void **) tmp = *(void **) b1;
+ b1 += sizeof (void *);
+ --n1;
+ }
+ else
+ {
+ *(void **) tmp = *(void **) b2;
+ b2 += sizeof (void *);
+ --n2;
+ }
+ tmp += sizeof (void *);
+ }
+ break;
+ default:
+ while (n1 > 0 && n2 > 0)
+ {
+ if (cmp (b1, b2, arg) <= 0)
+ {
+ tmp = (char *) __mempcpy (tmp, b1, s);
+ b1 += s;
+ --n1;
+ }
+ else
+ {
+ tmp = (char *) __mempcpy (tmp, b2, s);
+ b2 += s;
+ --n2;
+ }
+ }
+ break;
}
+
+ if (n1 > 0)
+ memcpy (tmp, b1, n1 * s);
+ memcpy (b, p->t, (n - n2) * s);
}
-/* Order size using quicksort. This implementation incorporates
- four optimizations discussed in Sedgewick:
+static void
+__attribute_used__
+indirect_msort_with_tmp (const struct msort_param *p, void *b, size_t n,
+ size_t s)
+{
+ /* Indirect sorting. */
+ char *ip = (char *) b;
+ void **tp = (void **) (p->t + n * sizeof (void *));
+ void **t = tp;
+ void *tmp_storage = (void *) (tp + n);
- 1. Non-recursive, using an explicit stack of pointer that store the
- next array partition to sort. To save time, this maximum amount
- of space required to store an array of SIZE_MAX is allocated on the
- stack. Assuming a 32-bit (64 bit) integer for size_t, this needs
- only 32 * sizeof(stack_node) == 256 bytes (for 64 bit: 1024 bytes).
- Pretty cheap, actually.
-
- 2. Chose the pivot element using a median-of-three decision tree.
- This reduces the probability of selecting a bad pivot value and
- eliminates certain extraneous comparisons.
-
- 3. Only quicksorts TOTAL_ELEMS / MAX_THRESH partitions, leaving
- insertion sort to order the MAX_THRESH items within each partition.
- This is a big win, since insertion sort is faster for small, mostly
- sorted array segments.
-
- 4. The larger of the two sub-partitions is always pushed onto the
- stack first, with the algorithm then concentrating on the
- smaller partition. This *guarantees* no more than log (total_elems)
- stack size is needed (actually O(1) in this case)! */
+ while ((void *) t < tmp_storage)
+ {
+ *t++ = ip;
+ ip += s;
+ }
+ msort_with_tmp (p, p->t + n * sizeof (void *), n);
+
+ /* tp[0] .. tp[n - 1] is now sorted, copy around entries of
+ the original array. Knuth vol. 3 (2nd ed.) exercise 5.2-10. */
+ char *kp;
+ size_t i;
+ for (i = 0, ip = (char *) b; i < n; i++, ip += s)
+ if ((kp = tp[i]) != ip)
+ {
+ size_t j = i;
+ char *jp = ip;
+ memcpy (tmp_storage, ip, s);
+
+ do
+ {
+ size_t k = (kp - (char *) b) / s;
+ tp[j] = jp;
+ memcpy (jp, kp, s);
+ j = k;
+ jp = kp;
+ kp = tp[k];
+ }
+ while (kp != ip);
+
+ tp[j] = jp;
+ memcpy (jp, tmp_storage, s);
+ }
+}
void
__qsort_r (void *const pbase, size_t total_elems, size_t size,
__compar_d_fn_t cmp, void *arg)
{
- char *base_ptr = (char *) pbase;
-
- const size_t max_thresh = MAX_THRESH * size;
-
if (total_elems <= 1)
- /* Avoid lossage with unsigned arithmetic below. */
return;
- enum swap_type_t swap_type;
- if (is_aligned (pbase, size, 8))
- swap_type = SWAP_WORDS_64;
- else if (is_aligned (pbase, size, 4))
- swap_type = SWAP_WORDS_32;
- else
- swap_type = SWAP_BYTES;
+ /* Align to the maximum size used by the swap optimization. */
+ _Alignas (uint64_t) char tmp[QSORT_STACK_SIZE];
+ size_t total_size = total_elems * size;
+ char *buf;
- /* Maximum depth before quicksort switches to heapsort. */
- size_t depth = 2 * (sizeof (size_t) * CHAR_BIT - 1
- - __builtin_clzl (total_elems));
+ if (size > INDIRECT_SORT_SIZE_THRES)
+ total_size = 2 * total_elems * sizeof (void *) + size;
- if (total_elems > MAX_THRESH)
+ if (total_size < sizeof buf)
+ buf = tmp;
+ else
{
- char *lo = base_ptr;
- char *hi = &lo[size * (total_elems - 1)];
- stack_node stack[STACK_SIZE];
- stack_node *top = push (stack, NULL, NULL, depth);
-
- while (stack < top)
- {
- if (depth == 0)
- {
- heapsort_r (lo, hi, size, swap_type, cmp, arg);
- top = pop (top, &lo, &hi, &depth);
- continue;
- }
-
- char *left_ptr;
- char *right_ptr;
-
- /* Select median value from among LO, MID, and HI. Rearrange
- LO and HI so the three values are sorted. This lowers the
- probability of picking a pathological pivot value and
- skips a comparison for both the LEFT_PTR and RIGHT_PTR in
- the while loops. */
-
- char *mid = lo + size * ((hi - lo) / size >> 1);
-
- if ((*cmp) ((void *) mid, (void *) lo, arg) < 0)
- do_swap (mid, lo, size, swap_type);
- if ((*cmp) ((void *) hi, (void *) mid, arg) < 0)
- do_swap (mid, hi, size, swap_type);
- else
- goto jump_over;
- if ((*cmp) ((void *) mid, (void *) lo, arg) < 0)
- do_swap (mid, lo, size, swap_type);
- jump_over:;
-
- left_ptr = lo + size;
- right_ptr = hi - size;
-
- /* Here's the famous ``collapse the walls'' section of quicksort.
- Gotta like those tight inner loops! They are the main reason
- that this algorithm runs much faster than others. */
- do
- {
- while (left_ptr != mid
- && (*cmp) ((void *) left_ptr, (void *) mid, arg) < 0)
- left_ptr += size;
-
- while (right_ptr != mid
- && (*cmp) ((void *) mid, (void *) right_ptr, arg) < 0)
- right_ptr -= size;
-
- if (left_ptr < right_ptr)
- {
- do_swap (left_ptr, right_ptr, size, swap_type);
- if (mid == left_ptr)
- mid = right_ptr;
- else if (mid == right_ptr)
- mid = left_ptr;
- left_ptr += size;
- right_ptr -= size;
- }
- else if (left_ptr == right_ptr)
- {
- left_ptr += size;
- right_ptr -= size;
- break;
- }
- }
- while (left_ptr <= right_ptr);
+ int save = errno;
+ buf = malloc (total_size);
+ __set_errno (save);
+ if (buf == NULL)
+ {
+ /* Fallback to heapsort in case of memory failure. */
+ heapsort_r (pbase, total_elems - 1, size, cmp, arg);
+ return;
+ }
+ }
- /* Set up pointers for next iteration. First determine whether
- left and right partitions are below the threshold size. If so,
- ignore one or both. Otherwise, push the larger partition's
- bounds on the stack and continue sorting the smaller one. */
-
- if ((size_t) (right_ptr - lo) <= max_thresh)
- {
- if ((size_t) (hi - left_ptr) <= max_thresh)
- /* Ignore both small partitions. */
- {
- top = pop (top, &lo, &hi, &depth);
- --depth;
- }
- else
- {
- /* Ignore small left partition. */
- lo = left_ptr;
- --depth;
- }
- }
- else if ((size_t) (hi - left_ptr) <= max_thresh)
- /* Ignore small right partition. */
- {
- hi = right_ptr;
- --depth;
- }
- else if ((right_ptr - lo) > (hi - left_ptr))
- {
- /* Push larger left partition indices. */
- top = push (top, lo, right_ptr, depth - 1);
- lo = left_ptr;
- }
- else
- {
- /* Push larger right partition indices. */
- top = push (top, left_ptr, hi, depth - 1);
- hi = right_ptr;
- }
- }
+ if (size > INDIRECT_SORT_SIZE_THRES)
+ {
+ const struct msort_param msort_param =
+ {
+ .s = sizeof (void *),
+ .cmp = cmp,
+ .arg = arg,
+ .var = SWAP_VOID_ARG,
+ .t = buf,
+ };
+ indirect_msort_with_tmp (&msort_param, pbase, total_elems, size);
+ }
+ else
+ {
+ const struct msort_param msort_param =
+ {
+ .s = size,
+ .cmp = cmp,
+ .arg = arg,
+ .var = get_swap_type (pbase, size),
+ .t = buf,
+ };
+ msort_with_tmp (&msort_param, pbase, total_elems);
}
- /* Once the BASE_PTR array is partially sorted by quicksort the rest
- is completely sorted using insertion sort, since this is efficient
- for partitions below MAX_THRESH size. BASE_PTR points to the beginning
- of the array to sort, and END_PTR points at the very last element in
- the array (*not* one beyond it!). */
- insertion_sort_qsort_partitions (pbase, total_elems, size, swap_type, cmp,
- arg);
+ if (buf != tmp)
+ free (buf);
}
libc_hidden_def (__qsort_r)
weak_alias (__qsort_r, qsort_r)
diff -Nrup a/stdlib/tst-qsort4.c b/stdlib/tst-qsort4.c
--- a/stdlib/tst-qsort4.c 2026-03-25 15:00:34.096307458 -0400
+++ b/stdlib/tst-qsort4.c 2026-03-25 14:58:51.500894548 -0400
@@ -30,35 +30,12 @@ cmp (const void *a1, const void *b1, voi
return *a - *b;
}
-/* Wrapper around heapsort_r that set ups the required variables. */
-static void
-heapsort_wrapper (void *const pbase, size_t total_elems, size_t size,
- __compar_d_fn_t cmp, void *arg)
-{
- char *base_ptr = (char *) pbase;
- char *lo = base_ptr;
- char *hi = &lo[size * (total_elems - 1)];
-
- if (total_elems <= 1)
- /* Avoid lossage with unsigned arithmetic below. */
- return;
-
- enum swap_type_t swap_type;
- if (is_aligned (pbase, size, 8))
- swap_type = SWAP_WORDS_64;
- else if (is_aligned (pbase, size, 4))
- swap_type = SWAP_WORDS_32;
- else
- swap_type = SWAP_BYTES;
- heapsort_r (lo, hi, size, swap_type, cmp, arg);
-}
-
static void
check_one_sort (signed char *array, int length)
{
signed char *copy = xmalloc (length);
memcpy (copy, array, length);
- heapsort_wrapper (copy, length, 1, cmp, NULL);
+ heapsort_r (copy, length - 1, 1, cmp, NULL);
/* Verify that the result is sorted. */
for (int i = 1; i < length; ++i)
diff -Nrup a/stdlib/tst-qsort5.c b/stdlib/tst-qsort5.c
--- a/stdlib/tst-qsort5.c 2026-03-25 15:00:34.121480227 -0400
+++ b/stdlib/tst-qsort5.c 1969-12-31 19:00:00.000000000 -0500
@@ -1,171 +0,0 @@
-/* Adversarial test for qsort_r.
- Copyright (C) 2023 Free Software Foundation, Inc.
- This file is part of the GNU C Library.
-
- The GNU C Library is free software; you can redistribute it and/or
- modify it under the terms of the GNU Lesser General Public
- License as published by the Free Software Foundation; either
- version 2.1 of the License, or (at your option) any later version.
-
- The GNU C Library is distributed in the hope that it will be useful,
- but WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- Lesser General Public License for more details.
-
- You should have received a copy of the GNU Lesser General Public
- License along with the GNU C Library; if not, see
- <http://www.gnu.org/licenses/>. */
-
-/* The approach follows Douglas McIlroy, A Killer Adversary for
- Quicksort. Software—Practice and Experience 29 (1999) 341-344.
- Downloaded <http://www.cs.dartmouth.edu/~doug/mdmspe.pdf>
- (2023-11-17). */
-
-#include <math.h>
-#include <stdlib.h>
-#include <stdio.h>
-#include <support/check.h>
-#include <support/support.h>
-
-struct context
-{
- /* Called the gas value in the paper. This value is larger than all
- other values (length minus one will do), so comparison with any
- decided value has a known result. */
- int undecided_value;
-
- /* If comparing undecided values, one of them as to be assigned a
- value to ensure consistency with future comparisons. This is the
- value that will be used. Starts out at zero. */
- int next_decided;
-
- /* Used to trick pivot selection. Deciding the value for the last
- seen undcided value in a decided/undecided comparison happens
- to trick the many qsort implementations. */
- int last_undecided_index;
-
- /* This array contains the actually asigned values. The call to
- qsort_r sorts a different array that contains indices into this
- array. */
- int *decided_values;
-};
-
-static int
-compare_opponent (const void *l1, const void *r1, void *ctx1)
-{
- const int *l = l1;
- const int *r = r1;
- struct context *ctx = ctx1;
- int rvalue = ctx->decided_values[*r];
- int lvalue = ctx->decided_values[*l];
-
- if (lvalue == ctx->undecided_value)
- {
- if (rvalue == ctx->undecided_value)
- {
- /* Both values are undecided. In this case, make a decision
- for the last-used undecided value. This is tweak is very
- specific to quicksort. */
- if (*l == ctx->last_undecided_index)
- {
- ctx->decided_values[*l] = ctx->next_decided;
- ++ctx->next_decided;
- /* The undecided value or *r is greater. */
- return -1;
- }
- else
- {
- ctx->decided_values[*r] = ctx->next_decided;
- ++ctx->next_decided;
- /* The undecided value for *l is greater. */
- return 1;
- }
- }
- else
- {
- ctx->last_undecided_index = *l;
- return 1;
- }
- }
- else
- {
- /* *l is a decided value. */
- if (rvalue == ctx->undecided_value)
- {
- ctx->last_undecided_index = *r;
- /* The undecided value for *r is greater. */
- return -1;
- }
- else
- return lvalue - rvalue;
- }
-}
-
-/* Return a pointer to the adversarial permutation of length N. */
-static int *
-create_permutation (size_t n)
-{
- struct context ctx =
- {
- .undecided_value = n - 1, /* Larger than all other values. */
- .decided_values = xcalloc (n, sizeof (int)),
- };
- for (size_t i = 0; i < n; ++i)
- ctx.decided_values[i] = ctx.undecided_value;
- int *scratch = xcalloc (n, sizeof (int));
- for (size_t i = 0; i < n; ++i)
- scratch[i] = i;
- qsort_r (scratch, n, sizeof (*scratch), compare_opponent, &ctx);
- free (scratch);
- return ctx.decided_values;
-}
-
-/* Callback function for qsort which counts the number of invocations
- in *CLOSURE. */
-static int
-compare_counter (const void *l1, const void *r1, void *closure)
-{
- const int *l = l1;
- const int *r = r1;
- unsigned long long int *counter = closure;
- ++*counter;
- return *l - *r;
-}
-
-/* Count the comparisons required for an adversarial permutation of
- length N. */
-static unsigned long long int
-count_comparisons (size_t n)
-{
- int *array = create_permutation (n);
- unsigned long long int counter = 0;
- qsort_r (array, n, sizeof (*array), compare_counter, &counter);
- free (array);
- return counter;
-}
-
-/* Check the scaling factor for one adversarial permutation of length
- N, and report some statistics. */
-static void
-check_one_n (size_t n)
-{
- unsigned long long int count = count_comparisons (n);
- double factor = count / (n * log (count));
- printf ("info: length %zu: %llu comparisons ~ %f * n * log (n)\n",
- n, count, factor);
- /* This is an arbitrary factor which is true for the current
- implementation across a wide range of sizes. */
- TEST_VERIFY (factor <= 4.5);
-}
-
-static int
-do_test (void)
-{
- check_one_n (100);
- check_one_n (1000);
- for (int i = 1; i <= 15; ++i)
- check_one_n (i * 10 * 1000);
- return 0;
-}
-
-#include <support/test-driver.c>

Some files were not shown because too many files have changed in this diff Show More