From 2ca0b0c57e6c6d64509aba47b74c5fe134a9df0f Mon Sep 17 00:00:00 2001 From: Florian Weimer Date: Thu, 13 Feb 2025 16:42:58 +0100 Subject: [PATCH] POWER10 string function optimizations (RHEL-24740) Resolves: RHEL-24740 --- glibc-RHEL-24740-1.patch | 315 ++++++++++++++++++++++++++++ glibc-RHEL-24740-2.patch | 435 +++++++++++++++++++++++++++++++++++++++ glibc-RHEL-24740-3.patch | 218 ++++++++++++++++++++ glibc.spec | 8 +- 4 files changed, 975 insertions(+), 1 deletion(-) create mode 100644 glibc-RHEL-24740-1.patch create mode 100644 glibc-RHEL-24740-2.patch create mode 100644 glibc-RHEL-24740-3.patch diff --git a/glibc-RHEL-24740-1.patch b/glibc-RHEL-24740-1.patch new file mode 100644 index 0000000..7fd8043 --- /dev/null +++ b/glibc-RHEL-24740-1.patch @@ -0,0 +1,315 @@ +commit 3367d8e180848030d1646f088759f02b8dfe0d6f +Author: Amrita H S +Date: Wed Dec 6 11:43:11 2023 -0500 + + powerpc: Optimized strcmp for power10 + + This patch is based on __strcmp_power9 and __strlen_power10. + + Improvements from __strcmp_power9: + + 1. Uses new POWER10 instructions + - This code uses lxvp to decrease contention on load + by loading 32 bytes per instruction. + + 2. Performance implication + - This version has around 30% better performance on average. + - Performance regression is seen for a specific combination + of sizes and alignments. Some of them is observed without + changes also, while rest may be induced by the patch. + + Signed-off-by: Amrita H S + Reviewed-by: Paul E. Murphy + +diff --git a/sysdeps/powerpc/powerpc64/le/power10/strcmp.S b/sysdeps/powerpc/powerpc64/le/power10/strcmp.S +new file mode 100644 +index 0000000000000000..a3c1adad539978e0 +--- /dev/null ++++ b/sysdeps/powerpc/powerpc64/le/power10/strcmp.S +@@ -0,0 +1,204 @@ ++/* Optimized strcmp implementation for PowerPC64/POWER10. ++ Copyright (C) 2021-2023 Free Software Foundation, Inc. ++ This file is part of the GNU C Library. ++ ++ The GNU C Library is free software; you can redistribute it and/or ++ modify it under the terms of the GNU Lesser General Public ++ License as published by the Free Software Foundation; either ++ version 2.1 of the License, or (at your option) any later version. ++ ++ The GNU C Library is distributed in the hope that it will be useful, ++ but WITHOUT ANY WARRANTY; without even the implied warranty of ++ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ++ Lesser General Public License for more details. ++ ++ You should have received a copy of the GNU Lesser General Public ++ License along with the GNU C Library; if not, see ++ . */ ++#include ++ ++#ifndef STRCMP ++# define STRCMP strcmp ++#endif ++ ++/* Implements the function ++ int [r3] strcmp (const char *s1 [r3], const char *s2 [r4]). */ ++ ++/* TODO: Change this to actual instructions when minimum binutils is upgraded ++ to 2.27. Macros are defined below for these newer instructions in order ++ to maintain compatibility. */ ++ ++#define LXVP(xtp,dq,ra) \ ++ .long(((6)<<(32-6)) \ ++ | ((((xtp)-32)>>1)<<(32-10)) \ ++ | ((1)<<(32-11)) \ ++ | ((ra)<<(32-16)) \ ++ | dq) ++ ++#define COMPARE_16(vreg1,vreg2,offset) \ ++ lxv vreg1+32,offset(r3); \ ++ lxv vreg2+32,offset(r4); \ ++ vcmpnezb. v7,vreg1,vreg2; \ ++ bne cr6,L(different); \ ++ ++#define COMPARE_32(vreg1,vreg2,offset,label1,label2) \ ++ LXVP(vreg1+32,offset,r3); \ ++ LXVP(vreg2+32,offset,r4); \ ++ vcmpnezb. v7,vreg1+1,vreg2+1; \ ++ bne cr6,L(label1); \ ++ vcmpnezb. v7,vreg1,vreg2; \ ++ bne cr6,L(label2); \ ++ ++#define TAIL(vreg1,vreg2) \ ++ vctzlsbb r6,v7; \ ++ vextubrx r5,r6,vreg1; \ ++ vextubrx r4,r6,vreg2; \ ++ subf r3,r4,r5; \ ++ blr; \ ++ ++#define CHECK_N_BYTES(reg1,reg2,len_reg) \ ++ sldi r0,len_reg,56; \ ++ lxvl 32+v4,reg1,r0; \ ++ lxvl 32+v5,reg2,r0; \ ++ add reg1,reg1,len_reg; \ ++ add reg2,reg2,len_reg; \ ++ vcmpnezb. v7,v4,v5; \ ++ vctzlsbb r6,v7; \ ++ cmpld cr7,r6,len_reg; \ ++ blt cr7,L(different); \ ++ ++ /* TODO: change this to .machine power10 when the minimum required ++ binutils allows it. */ ++ ++ .machine power9 ++ENTRY_TOCLESS (STRCMP, 4) ++ li r11,16 ++ /* eq bit of cr1 used as swap status flag to indicate if ++ source pointers were swapped. */ ++ crclr 4*cr1+eq ++ vspltisb v19,-1 ++ andi. r7,r3,15 ++ sub r7,r11,r7 /* r7(nalign1) = 16 - (str1 & 15). */ ++ andi. r9,r4,15 ++ sub r5,r11,r9 /* r5(nalign2) = 16 - (str2 & 15). */ ++ cmpld cr7,r7,r5 ++ beq cr7,L(same_aligned) ++ blt cr7,L(nalign1_min) ++ /* Swap r3 and r4, and r7 and r5 such that r3 and r7 hold the ++ pointer which is closer to the next 16B boundary so that only ++ one CHECK_N_BYTES is needed before entering the loop below. */ ++ mr r8,r4 ++ mr r4,r3 ++ mr r3,r8 ++ mr r12,r7 ++ mr r7,r5 ++ mr r5,r12 ++ crset 4*cr1+eq /* Set bit on swapping source pointers. */ ++ ++ .p2align 5 ++L(nalign1_min): ++ CHECK_N_BYTES(r3,r4,r7) ++ ++ .p2align 5 ++L(s1_aligned): ++ /* r9 and r5 is number of bytes to be read after and before ++ page boundary correspondingly. */ ++ sub r5,r5,r7 ++ subfic r9,r5,16 ++ /* Now let r7 hold the count of quadwords which can be ++ checked without crossing a page boundary. quadword offset is ++ (str2>>4)&0xFF. */ ++ rlwinm r7,r4,28,0xFF ++ /* Below check is required only for first iteration. For second ++ iteration and beyond, the new loop counter is always 255. */ ++ cmpldi r7,255 ++ beq L(L3) ++ /* Get the initial loop count by 255-((str2>>4)&0xFF). */ ++ subfic r11,r7,255 ++ ++ .p2align 5 ++L(L1): ++ mtctr r11 ++ ++ .p2align 5 ++L(L2): ++ COMPARE_16(v4,v5,0) /* Load 16B blocks using lxv. */ ++ addi r3,r3,16 ++ addi r4,r4,16 ++ bdnz L(L2) ++ /* Cross the page boundary of s2, carefully. */ ++ ++ .p2align 5 ++L(L3): ++ CHECK_N_BYTES(r3,r4,r5) ++ CHECK_N_BYTES(r3,r4,r9) ++ li r11,255 /* Load the new loop counter. */ ++ b L(L1) ++ ++ .p2align 5 ++L(same_aligned): ++ CHECK_N_BYTES(r3,r4,r7) ++ /* Align s1 to 32B and adjust s2 address. ++ Use lxvp only if both s1 and s2 are 32B aligned. */ ++ COMPARE_16(v4,v5,0) ++ COMPARE_16(v4,v5,16) ++ COMPARE_16(v4,v5,32) ++ COMPARE_16(v4,v5,48) ++ addi r3,r3,64 ++ addi r4,r4,64 ++ COMPARE_16(v4,v5,0) ++ COMPARE_16(v4,v5,16) ++ ++ clrldi r6,r3,59 ++ subfic r5,r6,32 ++ add r3,r3,r5 ++ add r4,r4,r5 ++ andi. r5,r4,0x1F ++ beq cr0,L(32B_aligned_loop) ++ ++ .p2align 5 ++L(16B_aligned_loop): ++ COMPARE_16(v4,v5,0) ++ COMPARE_16(v4,v5,16) ++ COMPARE_16(v4,v5,32) ++ COMPARE_16(v4,v5,48) ++ addi r3,r3,64 ++ addi r4,r4,64 ++ b L(16B_aligned_loop) ++ ++ /* Calculate and return the difference. */ ++L(different): ++ vctzlsbb r6,v7 ++ vextubrx r5,r6,v4 ++ vextubrx r4,r6,v5 ++ bt 4*cr1+eq,L(swapped) ++ subf r3,r4,r5 ++ blr ++ ++ /* If src pointers were swapped, then swap the ++ indices and calculate the return value. */ ++L(swapped): ++ subf r3,r5,r4 ++ blr ++ ++ .p2align 5 ++L(32B_aligned_loop): ++ COMPARE_32(v14,v16,0,tail1,tail2) ++ COMPARE_32(v18,v20,32,tail3,tail4) ++ COMPARE_32(v22,v24,64,tail5,tail6) ++ COMPARE_32(v26,v28,96,tail7,tail8) ++ addi r3,r3,128 ++ addi r4,r4,128 ++ b L(32B_aligned_loop) ++ ++L(tail1): TAIL(v15,v17) ++L(tail2): TAIL(v14,v16) ++L(tail3): TAIL(v19,v21) ++L(tail4): TAIL(v18,v20) ++L(tail5): TAIL(v23,v25) ++L(tail6): TAIL(v22,v24) ++L(tail7): TAIL(v27,v29) ++L(tail8): TAIL(v26,v28) ++ ++END (STRCMP) ++libc_hidden_builtin_def (strcmp) +diff --git a/sysdeps/powerpc/powerpc64/multiarch/Makefile b/sysdeps/powerpc/powerpc64/multiarch/Makefile +index 0ee7ce39d6470d80..91ed88a9c716800d 100644 +--- a/sysdeps/powerpc/powerpc64/multiarch/Makefile ++++ b/sysdeps/powerpc/powerpc64/multiarch/Makefile +@@ -33,7 +33,8 @@ sysdep_routines += memcpy-power8-cached memcpy-power7 memcpy-a2 memcpy-power6 \ + ifneq (,$(filter %le,$(config-machine))) + sysdep_routines += memcmp-power10 memcpy-power10 memmove-power10 memset-power10 \ + rawmemchr-power9 rawmemchr-power10 \ +- strcmp-power9 strncmp-power9 strcpy-power9 stpcpy-power9 \ ++ strcmp-power9 strcmp-power10 strncmp-power9 \ ++ strcpy-power9 stpcpy-power9 \ + strlen-power9 strncpy-power9 stpncpy-power9 strlen-power10 + endif + CFLAGS-strncase-power7.c += -mcpu=power7 -funroll-loops +diff --git a/sysdeps/powerpc/powerpc64/multiarch/ifunc-impl-list.c b/sysdeps/powerpc/powerpc64/multiarch/ifunc-impl-list.c +index 2c84d287ee76a7ea..caec2047ab10d209 100644 +--- a/sysdeps/powerpc/powerpc64/multiarch/ifunc-impl-list.c ++++ b/sysdeps/powerpc/powerpc64/multiarch/ifunc-impl-list.c +@@ -416,6 +416,10 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array, + /* Support sysdeps/powerpc/powerpc64/multiarch/strcmp.c. */ + IFUNC_IMPL (i, name, strcmp, + #ifdef __LITTLE_ENDIAN__ ++ IFUNC_IMPL_ADD (array, i, strcmp, ++ (hwcap2 & PPC_FEATURE2_ARCH_3_1) ++ && (hwcap & PPC_FEATURE_HAS_VSX), ++ __strcmp_power10) + IFUNC_IMPL_ADD (array, i, strcmp, + hwcap2 & PPC_FEATURE2_ARCH_3_00 + && hwcap & PPC_FEATURE_HAS_ALTIVEC, +diff --git a/sysdeps/powerpc/powerpc64/multiarch/strcmp-power10.S b/sysdeps/powerpc/powerpc64/multiarch/strcmp-power10.S +new file mode 100644 +index 0000000000000000..c80067ce3305de81 +--- /dev/null ++++ b/sysdeps/powerpc/powerpc64/multiarch/strcmp-power10.S +@@ -0,0 +1,26 @@ ++/* Optimized strcmp implementation for POWER10/PPC64. ++ Copyright (C) 2021-2023 Free Software Foundation, Inc. ++ This file is part of the GNU C Library. ++ ++ The GNU C Library is free software; you can redistribute it and/or ++ modify it under the terms of the GNU Lesser General Public ++ License as published by the Free Software Foundation; either ++ version 2.1 of the License, or (at your option) any later version. ++ ++ The GNU C Library is distributed in the hope that it will be useful, ++ but WITHOUT ANY WARRANTY; without even the implied warranty of ++ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ++ Lesser General Public License for more details. ++ ++ You should have received a copy of the GNU Lesser General Public ++ License along with the GNU C Library; if not, see ++ . */ ++ ++#if defined __LITTLE_ENDIAN__ && IS_IN (libc) ++#define STRCMP __strcmp_power10 ++ ++#undef libc_hidden_builtin_def ++#define libc_hidden_builtin_def(name) ++ ++#include ++#endif /* __LITTLE_ENDIAN__ && IS_IN (libc) */ +diff --git a/sysdeps/powerpc/powerpc64/multiarch/strcmp.c b/sysdeps/powerpc/powerpc64/multiarch/strcmp.c +index 8132682a992edb7a..4e77005117525edb 100644 +--- a/sysdeps/powerpc/powerpc64/multiarch/strcmp.c ++++ b/sysdeps/powerpc/powerpc64/multiarch/strcmp.c +@@ -29,12 +29,16 @@ extern __typeof (strcmp) __strcmp_power7 attribute_hidden; + extern __typeof (strcmp) __strcmp_power8 attribute_hidden; + # ifdef __LITTLE_ENDIAN__ + extern __typeof (strcmp) __strcmp_power9 attribute_hidden; ++extern __typeof (strcmp) __strcmp_power10 attribute_hidden; + # endif + + # undef strcmp + + libc_ifunc_redirected (__redirect_strcmp, strcmp, + # ifdef __LITTLE_ENDIAN__ ++ (hwcap2 & PPC_FEATURE2_ARCH_3_1 ++ && hwcap & PPC_FEATURE_HAS_VSX) ++ ? __strcmp_power10 : + (hwcap2 & PPC_FEATURE2_ARCH_3_00 + && hwcap & PPC_FEATURE_HAS_ALTIVEC) + ? __strcmp_power9 : diff --git a/glibc-RHEL-24740-2.patch b/glibc-RHEL-24740-2.patch new file mode 100644 index 0000000..11f9510 --- /dev/null +++ b/glibc-RHEL-24740-2.patch @@ -0,0 +1,435 @@ +commit b9182c793caa05df5d697427c0538936e6396d4b +Author: MAHESH BODAPATI +Date: Tue Dec 12 08:52:45 2023 -0600 + + powerpc : Add optimized memchr for POWER10 + + Optimized memchr for POWER10 based on existing rawmemchr and strlen. + Reordering instructions and loop unrolling helped in getting better performance. + Reviewed-by: Rajalakshmi Srinivasaraghavan + +diff --git a/sysdeps/powerpc/powerpc64/le/power10/memchr.S b/sysdeps/powerpc/powerpc64/le/power10/memchr.S +new file mode 100644 +index 0000000000000000..faf293f3447e6fc6 +--- /dev/null ++++ b/sysdeps/powerpc/powerpc64/le/power10/memchr.S +@@ -0,0 +1,315 @@ ++/* Optimized memchr implementation for POWER10 LE. ++ Copyright (C) 2021-2023 Free Software Foundation, Inc. ++ This file is part of the GNU C Library. ++ ++ The GNU C Library is free software; you can redistribute it and/or ++ modify it under the terms of the GNU Lesser General Public ++ License as published by the Free Software Foundation; either ++ version 2.1 of the License, or (at your option) any later version. ++ ++ The GNU C Library is distributed in the hope that it will be useful, ++ but WITHOUT ANY WARRANTY; without even the implied warranty of ++ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ++ Lesser General Public License for more details. ++ ++ You should have received a copy of the GNU Lesser General Public ++ License along with the GNU C Library; if not, see ++ . */ ++ ++#include ++ ++# ifndef MEMCHR ++# define MEMCHR __memchr ++# endif ++# define M_VREG_ZERO v20 ++# define M_OFF_START_LOOP 256 ++# define MEMCHR_SUBTRACT_VECTORS \ ++ vsububm v4,v4,v18; \ ++ vsububm v5,v5,v18; \ ++ vsububm v6,v6,v18; \ ++ vsububm v7,v7,v18; ++# define M_TAIL(vreg,increment) \ ++ vctzlsbb r4,vreg; \ ++ cmpld r5,r4; \ ++ ble L(null); \ ++ addi r4,r4,increment; \ ++ add r3,r6,r4; \ ++ blr ++ ++/* TODO: Replace macros by the actual instructions when minimum binutils becomes ++ >= 2.35. This is used to keep compatibility with older versions. */ ++#define M_VEXTRACTBM(rt,vrb) \ ++ .long(((4)<<(32-6)) \ ++ | ((rt)<<(32-11)) \ ++ | ((8)<<(32-16)) \ ++ | ((vrb)<<(32-21)) \ ++ | 1602) ++ ++#define M_LXVP(xtp,dq,ra) \ ++ .long(((6)<<(32-6)) \ ++ | ((((xtp)-32)>>1)<<(32-10)) \ ++ | ((1)<<(32-11)) \ ++ | ((ra)<<(32-16)) \ ++ | dq) ++ ++#define CHECK16B(vreg,offset,addr,label) \ ++ lxv vreg+32,offset(addr); \ ++ vcmpequb. vreg,vreg,v18; \ ++ bne cr6,L(label); \ ++ cmpldi r5,16; \ ++ ble L(null); \ ++ addi r5,r5,-16; ++ ++/* Load 4 quadwords, merge into one VR for speed and check for NULLs. r6 has # ++ of bytes already checked. */ ++#define CHECK64B(offset,addr,label) \ ++ M_LXVP(v4+32,offset,addr); \ ++ M_LXVP(v6+32,offset+32,addr); \ ++ MEMCHR_SUBTRACT_VECTORS; \ ++ vminub v14,v4,v5; \ ++ vminub v15,v6,v7; \ ++ vminub v16,v14,v15; \ ++ vcmpequb. v0,v16,M_VREG_ZERO; \ ++ beq cr6,$+12; \ ++ li r7,offset; \ ++ b L(label); \ ++ cmpldi r5,64; \ ++ ble L(null); \ ++ addi r5,r5,-64 ++ ++/* Implements the function ++ void *[r3] memchr (const void *s [r3], int c [r4], size_t n [r5]). */ ++ ++ .machine power9 ++ ++ENTRY_TOCLESS (MEMCHR) ++ CALL_MCOUNT 3 ++ ++ cmpldi r5,0 ++ beq L(null) ++ mr r0,r5 ++ xori r6,r4,0xff ++ ++ mtvsrd v18+32,r4 /* matching char in v18 */ ++ mtvsrd v19+32,r6 /* non matching char in v19 */ ++ ++ vspltb v18,v18,7 /* replicate */ ++ vspltb v19,v19,7 /* replicate */ ++ vspltisb M_VREG_ZERO,0 ++ ++ /* Next 16B-aligned address. Prepare address for L(aligned). */ ++ addi r6,r3,16 ++ clrrdi r6,r6,4 ++ ++ /* Align data and fill bytes not loaded with non matching char. */ ++ lvx v0,0,r3 ++ lvsr v1,0,r3 ++ vperm v0,v19,v0,v1 ++ ++ vcmpequb. v6,v0,v18 ++ bne cr6,L(found) ++ sub r4,r6,r3 ++ cmpld r5,r4 ++ ble L(null) ++ sub r5,r5,r4 ++ ++ /* Test up to OFF_START_LOOP-16 bytes in 16B chunks. The main loop is ++ optimized for longer strings, so checking the first bytes in 16B ++ chunks benefits a lot small strings. */ ++ .p2align 5 ++L(aligned): ++ cmpldi r5,0 ++ beq L(null) ++ ++ CHECK16B(v0,0,r6,tail1) ++ CHECK16B(v1,16,r6,tail2) ++ CHECK16B(v2,32,r6,tail3) ++ CHECK16B(v3,48,r6,tail4) ++ CHECK16B(v4,64,r6,tail5) ++ CHECK16B(v5,80,r6,tail6) ++ CHECK16B(v6,96,r6,tail7) ++ CHECK16B(v7,112,r6,tail8) ++ CHECK16B(v8,128,r6,tail9) ++ CHECK16B(v9,144,r6,tail10) ++ CHECK16B(v10,160,r6,tail11) ++ CHECK16B(v0,176,r6,tail12) ++ CHECK16B(v1,192,r6,tail13) ++ CHECK16B(v2,208,r6,tail14) ++ CHECK16B(v3,224,r6,tail15) ++ ++ cmpdi cr5,r4,0 /* Check if c == 0. This will be useful to ++ choose how we will perform the main loop. */ ++ ++ /* Prepare address for the loop. */ ++ addi r4,r3,M_OFF_START_LOOP ++ clrrdi r4,r4,6 ++ sub r6,r4,r3 ++ sub r5,r0,r6 ++ addi r6,r4,128 ++ ++ /* If c == 0, use the loop without the vsububm. */ ++ beq cr5,L(loop) ++ ++ /* This is very similar to the block after L(loop), the difference is ++ that here MEMCHR_SUBTRACT_VECTORS is not empty, and we subtract ++ each byte loaded by the char we are looking for, this way we can keep ++ using vminub to merge the results and checking for nulls. */ ++ .p2align 5 ++L(memchr_loop): ++ CHECK64B(0,r4,pre_tail_64b) ++ CHECK64B(64,r4,pre_tail_64b) ++ addi r4,r4,256 ++ ++ CHECK64B(0,r6,tail_64b) ++ CHECK64B(64,r6,tail_64b) ++ addi r6,r6,256 ++ ++ CHECK64B(0,r4,pre_tail_64b) ++ CHECK64B(64,r4,pre_tail_64b) ++ addi r4,r4,256 ++ ++ CHECK64B(0,r6,tail_64b) ++ CHECK64B(64,r6,tail_64b) ++ addi r6,r6,256 ++ ++ b L(memchr_loop) ++ /* Switch to a more aggressive approach checking 64B each time. Use 2 ++ pointers 128B apart and unroll the loop once to make the pointer ++ updates and usages separated enough to avoid stalls waiting for ++ address calculation. */ ++ .p2align 5 ++L(loop): ++#undef MEMCHR_SUBTRACT_VECTORS ++#define MEMCHR_SUBTRACT_VECTORS /* nothing */ ++ CHECK64B(0,r4,pre_tail_64b) ++ CHECK64B(64,r4,pre_tail_64b) ++ addi r4,r4,256 ++ ++ CHECK64B(0,r6,tail_64b) ++ CHECK64B(64,r6,tail_64b) ++ addi r6,r6,256 ++ ++ CHECK64B(0,r4,pre_tail_64b) ++ CHECK64B(64,r4,pre_tail_64b) ++ addi r4,r4,256 ++ ++ CHECK64B(0,r6,tail_64b) ++ CHECK64B(64,r6,tail_64b) ++ addi r6,r6,256 ++ ++ b L(loop) ++ ++ .p2align 5 ++L(pre_tail_64b): ++ mr r6,r4 ++L(tail_64b): ++ /* OK, we found a null byte. Let's look for it in the current 64-byte ++ block and mark it in its corresponding VR. lxvp vx,0(ry) puts the ++ low 16B bytes into vx+1, and the high into vx, so the order here is ++ v5, v4, v7, v6. */ ++ vcmpequb v1,v5,M_VREG_ZERO ++ vcmpequb v2,v4,M_VREG_ZERO ++ vcmpequb v3,v7,M_VREG_ZERO ++ vcmpequb v4,v6,M_VREG_ZERO ++ ++ /* Take into account the other 64B blocks we had already checked. */ ++ add r6,r6,r7 ++ /* Extract first bit of each byte. */ ++ M_VEXTRACTBM(r8,v1) ++ M_VEXTRACTBM(r9,v2) ++ M_VEXTRACTBM(r10,v3) ++ M_VEXTRACTBM(r11,v4) ++ ++ /* Shift each value into their corresponding position. */ ++ sldi r9,r9,16 ++ sldi r10,r10,32 ++ sldi r11,r11,48 ++ ++ /* Merge the results. */ ++ or r8,r8,r9 ++ or r9,r10,r11 ++ or r11,r9,r8 ++ ++ cnttzd r0,r11 /* Count trailing zeros before the match. */ ++ cmpld r5,r0 ++ ble L(null) ++ add r3,r6,r0 /* Compute final address. */ ++ blr ++ ++ .p2align 5 ++L(tail1): ++ M_TAIL(v0,0) ++ ++ .p2align 5 ++L(tail2): ++ M_TAIL(v1,16) ++ ++ .p2align 5 ++L(tail3): ++ M_TAIL(v2,32) ++ ++ .p2align 5 ++L(tail4): ++ M_TAIL(v3,48) ++ ++ .p2align 5 ++L(tail5): ++ M_TAIL(v4,64) ++ ++ .p2align 5 ++L(tail6): ++ M_TAIL(v5,80) ++ ++ .p2align 5 ++L(tail7): ++ M_TAIL(v6,96) ++ ++ .p2align 5 ++L(tail8): ++ M_TAIL(v7,112) ++ ++ .p2align 5 ++L(tail9): ++ M_TAIL(v8,128) ++ ++ .p2align 5 ++L(tail10): ++ M_TAIL(v9,144) ++ ++ .p2align 5 ++L(tail11): ++ M_TAIL(v10,160) ++ ++ .p2align 5 ++L(tail12): ++ M_TAIL(v0,176) ++ ++ .p2align 5 ++L(tail13): ++ M_TAIL(v1,192) ++ ++ .p2align 5 ++L(tail14): ++ M_TAIL(v2,208) ++ ++ .p2align 5 ++L(tail15): ++ M_TAIL(v3,224) ++ ++ .p2align 5 ++L(found): ++ vctzlsbb r7,v6 ++ cmpld r5,r7 ++ ble L(null) ++ add r3,r3,r7 ++ blr ++ ++ .p2align 5 ++L(null): ++ li r3,0 ++ blr ++ ++END (MEMCHR) ++ ++weak_alias (__memchr, memchr) ++libc_hidden_builtin_def (memchr) +diff --git a/sysdeps/powerpc/powerpc64/multiarch/Makefile b/sysdeps/powerpc/powerpc64/multiarch/Makefile +index 91ed88a9c716800d..b4251932de1854c2 100644 +--- a/sysdeps/powerpc/powerpc64/multiarch/Makefile ++++ b/sysdeps/powerpc/powerpc64/multiarch/Makefile +@@ -31,10 +31,10 @@ sysdep_routines += memcpy-power8-cached memcpy-power7 memcpy-a2 memcpy-power6 \ + strncase-power8 + + ifneq (,$(filter %le,$(config-machine))) +-sysdep_routines += memcmp-power10 memcpy-power10 memmove-power10 memset-power10 \ +- rawmemchr-power9 rawmemchr-power10 \ +- strcmp-power9 strcmp-power10 strncmp-power9 \ +- strcpy-power9 stpcpy-power9 \ ++sysdep_routines += memchr-power10 memcmp-power10 memcpy-power10 \ ++ memmove-power10 memset-power10 rawmemchr-power9 \ ++ rawmemchr-power10 strcmp-power9 strcmp-power10 \ ++ strncmp-power9 strcpy-power9 stpcpy-power9 \ + strlen-power9 strncpy-power9 stpncpy-power9 strlen-power10 + endif + CFLAGS-strncase-power7.c += -mcpu=power7 -funroll-loops +diff --git a/sysdeps/powerpc/powerpc64/multiarch/ifunc-impl-list.c b/sysdeps/powerpc/powerpc64/multiarch/ifunc-impl-list.c +index caec2047ab10d209..e8a38fd4d5e1357e 100644 +--- a/sysdeps/powerpc/powerpc64/multiarch/ifunc-impl-list.c ++++ b/sysdeps/powerpc/powerpc64/multiarch/ifunc-impl-list.c +@@ -265,6 +265,12 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array, + + /* Support sysdeps/powerpc/powerpc64/multiarch/memchr.c. */ + IFUNC_IMPL (i, name, memchr, ++#ifdef __LITTLE_ENDIAN__ ++ IFUNC_IMPL_ADD (array, i, memchr, ++ hwcap2 & PPC_FEATURE2_ARCH_3_1 ++ && hwcap & PPC_FEATURE_HAS_VSX, ++ __memchr_power10) ++#endif + IFUNC_IMPL_ADD (array, i, memchr, + hwcap2 & PPC_FEATURE2_ARCH_2_07 + && hwcap & PPC_FEATURE_HAS_ALTIVEC, +diff --git a/sysdeps/powerpc/powerpc64/multiarch/memchr-power10.S b/sysdeps/powerpc/powerpc64/multiarch/memchr-power10.S +new file mode 100644 +index 0000000000000000..b9ed7926762e2b6f +--- /dev/null ++++ b/sysdeps/powerpc/powerpc64/multiarch/memchr-power10.S +@@ -0,0 +1,28 @@ ++/* Optimized memchr implementation for POWER10/PPC64. ++ Copyright (C) 2016-2023 Free Software Foundation, Inc. ++ This file is part of the GNU C Library. ++ ++ The GNU C Library is free software; you can redistribute it and/or ++ modify it under the terms of the GNU Lesser General Public ++ License as published by the Free Software Foundation; either ++ version 2.1 of the License, or (at your option) any later version. ++ ++ The GNU C Library is distributed in the hope that it will be useful, ++ but WITHOUT ANY WARRANTY; without even the implied warranty of ++ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ++ Lesser General Public License for more details. ++ ++ You should have received a copy of the GNU Lesser General Public ++ License along with the GNU C Library; if not, see ++ . */ ++ ++#if defined __LITTLE_ENDIAN__ && IS_IN (libc) ++#define MEMCHR __memchr_power10 ++ ++#undef libc_hidden_builtin_def ++#define libc_hidden_builtin_def(name) ++#undef weak_alias ++#define weak_alias(name,alias) ++ ++#include ++#endif +diff --git a/sysdeps/powerpc/powerpc64/multiarch/memchr.c b/sysdeps/powerpc/powerpc64/multiarch/memchr.c +index f40013e06113096f..389d5f18683c2dfc 100644 +--- a/sysdeps/powerpc/powerpc64/multiarch/memchr.c ++++ b/sysdeps/powerpc/powerpc64/multiarch/memchr.c +@@ -25,15 +25,23 @@ extern __typeof (__memchr) __memchr_ppc attribute_hidden; + extern __typeof (__memchr) __memchr_power7 attribute_hidden; + extern __typeof (__memchr) __memchr_power8 attribute_hidden; + ++# ifdef __LITTLE_ENDIAN__ ++extern __typeof (__memchr) __memchr_power10 attribute_hidden; ++# endif + /* Avoid DWARF definition DIE on ifunc symbol so that GDB can handle + ifunc symbol properly. */ + libc_ifunc (__memchr, +- (hwcap2 & PPC_FEATURE2_ARCH_2_07 +- && hwcap & PPC_FEATURE_HAS_ALTIVEC) +- ? __memchr_power8 : +- (hwcap & PPC_FEATURE_ARCH_2_06) +- ? __memchr_power7 +- : __memchr_ppc); ++# ifdef __LITTLE_ENDIAN__ ++ (hwcap2 & PPC_FEATURE2_ARCH_3_1 ++ && hwcap & PPC_FEATURE_HAS_VSX) ++ ? __memchr_power10 : ++# endif ++ (hwcap2 & PPC_FEATURE2_ARCH_2_07 ++ && hwcap & PPC_FEATURE_HAS_ALTIVEC) ++ ? __memchr_power8 : ++ (hwcap & PPC_FEATURE_ARCH_2_06) ++ ? __memchr_power7 ++ : __memchr_ppc); + + weak_alias (__memchr, memchr) + libc_hidden_builtin_def (memchr) diff --git a/glibc-RHEL-24740-3.patch b/glibc-RHEL-24740-3.patch new file mode 100644 index 0000000..353966c --- /dev/null +++ b/glibc-RHEL-24740-3.patch @@ -0,0 +1,218 @@ +commit 90bcc8721ef82b7378d2b080141228660e862d56 +Author: Amrita H S +Date: Fri Dec 15 11:48:17 2023 -0500 + + powerpc: Fix performance issues of strcmp power10 + + Current implementation of strcmp for power10 has + performance regression for multiple small sizes + and alignment combination. + + Most of these performance issues are fixed by this + patch. The compare loop is unrolled and page crosses + of unrolled loop is handled. + + Thanks to Paul E. Murphy for helping in fixing the + performance issues. + + Signed-off-by: Amrita H S + Co-Authored-By: Paul E. Murphy + Reviewed-by: Rajalakshmi Srinivasaraghavan + +diff --git a/sysdeps/powerpc/powerpc64/le/power10/strcmp.S b/sysdeps/powerpc/powerpc64/le/power10/strcmp.S +index a3c1adad539978e0..3406f4f26a214270 100644 +--- a/sysdeps/powerpc/powerpc64/le/power10/strcmp.S ++++ b/sysdeps/powerpc/powerpc64/le/power10/strcmp.S +@@ -62,7 +62,7 @@ + lxvl 32+v5,reg2,r0; \ + add reg1,reg1,len_reg; \ + add reg2,reg2,len_reg; \ +- vcmpnezb. v7,v4,v5; \ ++ vcmpnezb v7,v4,v5; \ + vctzlsbb r6,v7; \ + cmpld cr7,r6,len_reg; \ + blt cr7,L(different); \ +@@ -72,70 +72,110 @@ + + .machine power9 + ENTRY_TOCLESS (STRCMP, 4) +- li r11,16 +- /* eq bit of cr1 used as swap status flag to indicate if +- source pointers were swapped. */ +- crclr 4*cr1+eq +- vspltisb v19,-1 +- andi. r7,r3,15 +- sub r7,r11,r7 /* r7(nalign1) = 16 - (str1 & 15). */ +- andi. r9,r4,15 +- sub r5,r11,r9 /* r5(nalign2) = 16 - (str2 & 15). */ +- cmpld cr7,r7,r5 +- beq cr7,L(same_aligned) +- blt cr7,L(nalign1_min) +- /* Swap r3 and r4, and r7 and r5 such that r3 and r7 hold the +- pointer which is closer to the next 16B boundary so that only +- one CHECK_N_BYTES is needed before entering the loop below. */ +- mr r8,r4 +- mr r4,r3 +- mr r3,r8 +- mr r12,r7 +- mr r7,r5 +- mr r5,r12 +- crset 4*cr1+eq /* Set bit on swapping source pointers. */ ++ andi. r7,r3,4095 ++ andi. r8,r4,4095 ++ cmpldi cr0,r7,4096-16 ++ cmpldi cr1,r8,4096-16 ++ bgt cr0,L(crosses) ++ bgt cr1,L(crosses) ++ COMPARE_16(v4,v5,0) + +- .p2align 5 ++L(crosses): ++ andi. r7,r3,15 ++ subfic r7,r7,16 /* r7(nalign1) = 16 - (str1 & 15). */ ++ andi. r9,r4,15 ++ subfic r5,r9,16 /* r5(nalign2) = 16 - (str2 & 15). */ ++ cmpld cr7,r7,r5 ++ beq cr7,L(same_aligned) ++ blt cr7,L(nalign1_min) ++ ++ /* nalign2 is minimum and s2 pointer is aligned. */ ++ CHECK_N_BYTES(r3,r4,r5) ++ /* Are we on the 64B hunk which crosses a page? */ ++ andi. r10,r3,63 /* Determine offset into 64B hunk. */ ++ andi. r8,r3,15 /* The offset into the 16B hunk. */ ++ neg r7,r3 ++ andi. r9,r7,15 /* Number of bytes after a 16B cross. */ ++ rlwinm. r7,r7,26,0x3F /* ((r3-4096))>>6&63. */ ++ beq L(compare_64_pagecross) ++ mtctr r7 ++ b L(compare_64B_unaligned) ++ ++ /* nalign1 is minimum and s1 pointer is aligned. */ + L(nalign1_min): + CHECK_N_BYTES(r3,r4,r7) ++ /* Are we on the 64B hunk which crosses a page? */ ++ andi. r10,r4,63 /* Determine offset into 64B hunk. */ ++ andi. r8,r4,15 /* The offset into the 16B hunk. */ ++ neg r7,r4 ++ andi. r9,r7,15 /* Number of bytes after a 16B cross. */ ++ rlwinm. r7,r7,26,0x3F /* ((r4-4096))>>6&63. */ ++ beq L(compare_64_pagecross) ++ mtctr r7 + + .p2align 5 +-L(s1_aligned): +- /* r9 and r5 is number of bytes to be read after and before +- page boundary correspondingly. */ +- sub r5,r5,r7 +- subfic r9,r5,16 +- /* Now let r7 hold the count of quadwords which can be +- checked without crossing a page boundary. quadword offset is +- (str2>>4)&0xFF. */ +- rlwinm r7,r4,28,0xFF +- /* Below check is required only for first iteration. For second +- iteration and beyond, the new loop counter is always 255. */ +- cmpldi r7,255 +- beq L(L3) +- /* Get the initial loop count by 255-((str2>>4)&0xFF). */ +- subfic r11,r7,255 ++L(compare_64B_unaligned): ++ COMPARE_16(v4,v5,0) ++ COMPARE_16(v4,v5,16) ++ COMPARE_16(v4,v5,32) ++ COMPARE_16(v4,v5,48) ++ addi r3,r3,64 ++ addi r4,r4,64 ++ bdnz L(compare_64B_unaligned) + +- .p2align 5 +-L(L1): ++ /* Cross the page boundary of s2, carefully. Only for first ++ iteration we have to get the count of 64B blocks to be checked. ++ From second iteration and beyond, loop counter is always 63. */ ++L(compare_64_pagecross): ++ li r11, 63 + mtctr r11 +- +- .p2align 5 +-L(L2): +- COMPARE_16(v4,v5,0) /* Load 16B blocks using lxv. */ ++ cmpldi r10,16 ++ ble L(cross_4) ++ cmpldi r10,32 ++ ble L(cross_3) ++ cmpldi r10,48 ++ ble L(cross_2) ++L(cross_1): ++ CHECK_N_BYTES(r3,r4,r9) ++ CHECK_N_BYTES(r3,r4,r8) ++ COMPARE_16(v4,v5,0) ++ COMPARE_16(v4,v5,16) ++ COMPARE_16(v4,v5,32) ++ addi r3,r3,48 ++ addi r4,r4,48 ++ b L(compare_64B_unaligned) ++L(cross_2): ++ COMPARE_16(v4,v5,0) + addi r3,r3,16 + addi r4,r4,16 +- bdnz L(L2) +- /* Cross the page boundary of s2, carefully. */ +- +- .p2align 5 +-L(L3): +- CHECK_N_BYTES(r3,r4,r5) + CHECK_N_BYTES(r3,r4,r9) +- li r11,255 /* Load the new loop counter. */ +- b L(L1) ++ CHECK_N_BYTES(r3,r4,r8) ++ COMPARE_16(v4,v5,0) ++ COMPARE_16(v4,v5,16) ++ addi r3,r3,32 ++ addi r4,r4,32 ++ b L(compare_64B_unaligned) ++L(cross_3): ++ COMPARE_16(v4,v5,0) ++ COMPARE_16(v4,v5,16) ++ addi r3,r3,32 ++ addi r4,r4,32 ++ CHECK_N_BYTES(r3,r4,r9) ++ CHECK_N_BYTES(r3,r4,r8) ++ COMPARE_16(v4,v5,0) ++ addi r3,r3,16 ++ addi r4,r4,16 ++ b L(compare_64B_unaligned) ++L(cross_4): ++ COMPARE_16(v4,v5,0) ++ COMPARE_16(v4,v5,16) ++ COMPARE_16(v4,v5,32) ++ addi r3,r3,48 ++ addi r4,r4,48 ++ CHECK_N_BYTES(r3,r4,r9) ++ CHECK_N_BYTES(r3,r4,r8) ++ b L(compare_64B_unaligned) + +- .p2align 5 + L(same_aligned): + CHECK_N_BYTES(r3,r4,r7) + /* Align s1 to 32B and adjust s2 address. +@@ -168,18 +208,7 @@ L(16B_aligned_loop): + + /* Calculate and return the difference. */ + L(different): +- vctzlsbb r6,v7 +- vextubrx r5,r6,v4 +- vextubrx r4,r6,v5 +- bt 4*cr1+eq,L(swapped) +- subf r3,r4,r5 +- blr +- +- /* If src pointers were swapped, then swap the +- indices and calculate the return value. */ +-L(swapped): +- subf r3,r5,r4 +- blr ++ TAIL(v4,v5) + + .p2align 5 + L(32B_aligned_loop): diff --git a/glibc.spec b/glibc.spec index b223573..77369db 100644 --- a/glibc.spec +++ b/glibc.spec @@ -157,7 +157,7 @@ end \ Summary: The GNU libc libraries Name: glibc Version: %{glibcversion} -Release: 166%{?dist} +Release: 167%{?dist} # In general, GPLv2+ is used by programs, LGPLv2+ is used for # libraries. @@ -1110,6 +1110,9 @@ Patch802: glibc-RHEL-2419-9.patch Patch803: glibc-RHEL-2419-10.patch Patch804: glibc-RHEL-46738-5.patch Patch805: glibc-RHEL-46761-6.patch +Patch806: glibc-RHEL-24740-1.patch +Patch807: glibc-RHEL-24740-2.patch +Patch808: glibc-RHEL-24740-3.patch ############################################################################## # Continued list of core "glibc" package information: @@ -3103,6 +3106,9 @@ update_gconv_modules_cache () %endif %changelog +* Thu Feb 13 2025 Florian Weimer - 2.34-167 +- POWER10 string function optimizations (RHEL-24740) + * Tue Feb 11 2025 Arjun Shankar - 2.34-166 - Revert: Backport: debug: Add regression tests for BZ 30932 (RHEL-46761)