rpms/kernel
9
4
Fork 6
Code Issues Pull Requests Releases Activity
a10
kernel/1340-arm64-scripts-sorttable-implement-sorting-mcount-loc-at-boot.patch
Andrew Lukoshko 0eab958d1f Recreate RHEL 6.12.0-211.22.1 from CS10/upstream backports 
Add the RHEL 211.21.1..211.22.1 backports (1288-1351) from centos-stream-10 and
upstream stable, on top of 211.20.1. Bump pkgrelease and specrelease to 211.22.1.
(The redhat/ automotive rebuild-changelog tooling change is omitted: it patches
redhat/scripts not present in this build base and does not affect the kernel.)
2026-06-11 11:06:20 +00:00

374 lines
11 KiB
Diff
Raw Permalink Blame History

From 4b605df96d9249250134a81573caa2ab136e0ce8 Mon Sep 17 00:00:00 2001
From: Jerome Marchand <jmarchan@redhat.com>
Date: Mon, 1 Jun 2026 15:55:45 +0200
Subject: [PATCH] arm64: scripts/sorttable: Implement sorting mcount_loc at
boot for arm64
JIRA: https://redhat.atlassian.net/browse/RHEL-180193
Conflicts: Context change from missing commit a762e9267dca ("ftrace:
Add CONFIG_HAVE_FTRACE_GRAPH_FUNC")
commit b3d09d06e052e1d754645acea4e4d1e96f81c934
Author: Steven Rostedt <rostedt@goodmis.org>
Date: Tue Feb 18 14:59:19 2025 -0500
arm64: scripts/sorttable: Implement sorting mcount_loc at boot for arm64
The mcount_loc section holds the addresses of the functions that get
patched by ftrace when enabling function callbacks. It can contain tens of
thousands of entries. These addresses must be sorted. If they are not
sorted at compile time, they are sorted at boot. Sorting at boot does take
some time and does have a small impact on boot performance.
x86 and arm32 have the addresses in the mcount_loc section of the ELF
file. But for arm64, the section just contains zeros. The .rela.dyn
Elf_Rela section holds the addresses and they get patched at boot during
the relocation phase.
In order to sort these addresses, the Elf_Rela needs to be updated instead
of the location in the binary that holds the mcount_loc section. Have the
sorttable code, allocate an array to hold the functions, load the
addresses from the Elf_Rela entries, sort them, then put them back in
order into the Elf_rela entries so that they will be sorted at boot up
without having to sort them during boot up.
Cc: bpf <bpf@vger.kernel.org>
Cc: Masami Hiramatsu <mhiramat@kernel.org>
Cc: Mark Rutland <mark.rutland@arm.com>
Cc: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
Cc: Andrew Morton <akpm@linux-foundation.org>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Masahiro Yamada <masahiroy@kernel.org>
Cc: Nathan Chancellor <nathan@kernel.org>
Cc: Nicolas Schier <nicolas@fjasle.eu>
Cc: Zheng Yejian <zhengyejian1@huawei.com>
Cc: Martin Kelly <martin.kelly@crowdstrike.com>
Cc: Christophe Leroy <christophe.leroy@csgroup.eu>
Cc: Josh Poimboeuf <jpoimboe@redhat.com>
Cc: Heiko Carstens <hca@linux.ibm.com>
Cc: Will Deacon <will@kernel.org>
Cc: Vasily Gorbik <gor@linux.ibm.com>
Cc: Alexander Gordeev <agordeev@linux.ibm.com>
Link: https://lore.kernel.org/20250218200022.373319428@goodmis.org
Acked-by: Catalin Marinas <catalin.marinas@arm.com>
Signed-off-by: Steven Rostedt (Google) <rostedt@goodmis.org>
(cherry picked from commit b3d09d06e052e1d754645acea4e4d1e96f81c934)
Assisted-by: Patchpal
Signed-off-by: Jerome Marchand <jmarchan@redhat.com>
diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig
index 534daec407ef..add6005c5e8b 100644
--- a/arch/arm64/Kconfig
+++ b/arch/arm64/Kconfig
@@ -219,6 +219,7 @@ config ARM64
if DYNAMIC_FTRACE_WITH_ARGS
select HAVE_SAMPLE_FTRACE_DIRECT
select HAVE_SAMPLE_FTRACE_DIRECT_MULTI
+ select HAVE_BUILDTIME_MCOUNT_SORT
select HAVE_EFFICIENT_UNALIGNED_ACCESS
select HAVE_GUP_FAST
select HAVE_FTRACE_MCOUNT_RECORD
diff --git a/scripts/sorttable.c b/scripts/sorttable.c
index 9f41575afd7a..4a34c275123e 100644
--- a/scripts/sorttable.c
+++ b/scripts/sorttable.c
@@ -28,6 +28,7 @@
#include <fcntl.h>
#include <stdio.h>
#include <stdlib.h>
+#include <stdbool.h>
#include <string.h>
#include <unistd.h>
#include <errno.h>
@@ -79,10 +80,16 @@ typedef union {
Elf64_Sym e64;
} Elf_Sym;
+typedef union {
+ Elf32_Rela e32;
+ Elf64_Rela e64;
+} Elf_Rela;
+
static uint32_t (*r)(const uint32_t *);
static uint16_t (*r2)(const uint16_t *);
static uint64_t (*r8)(const uint64_t *);
static void (*w)(uint32_t, uint32_t *);
+static void (*w8)(uint64_t, uint64_t *);
typedef void (*table_sort_t)(char *, int);
static struct elf_funcs {
@@ -102,6 +109,10 @@ static struct elf_funcs {
uint32_t (*sym_name)(Elf_Sym *sym);
uint64_t (*sym_value)(Elf_Sym *sym);
uint16_t (*sym_shndx)(Elf_Sym *sym);
+ uint64_t (*rela_offset)(Elf_Rela *rela);
+ uint64_t (*rela_info)(Elf_Rela *rela);
+ uint64_t (*rela_addend)(Elf_Rela *rela);
+ void (*rela_write_addend)(Elf_Rela *rela, uint64_t val);
} e;
static uint64_t ehdr64_shoff(Elf_Ehdr *ehdr)
@@ -262,6 +273,38 @@ SYM_ADDR(value)
SYM_WORD(name)
SYM_HALF(shndx)
+#define __maybe_unused __attribute__((__unused__))
+
+#define RELA_ADDR(fn_name) \
+static uint64_t rela64_##fn_name(Elf_Rela *rela) \
+{ \
+ return r8((uint64_t *)&rela->e64.r_##fn_name); \
+} \
+ \
+static uint64_t rela32_##fn_name(Elf_Rela *rela) \
+{ \
+ return r((uint32_t *)&rela->e32.r_##fn_name); \
+} \
+ \
+static uint64_t __maybe_unused rela_##fn_name(Elf_Rela *rela) \
+{ \
+ return e.rela_##fn_name(rela); \
+}
+
+RELA_ADDR(offset)
+RELA_ADDR(info)
+RELA_ADDR(addend)
+
+static void rela64_write_addend(Elf_Rela *rela, uint64_t val)
+{
+ w8(val, (uint64_t *)&rela->e64.r_addend);
+}
+
+static void rela32_write_addend(Elf_Rela *rela, uint64_t val)
+{
+ w(val, (uint32_t *)&rela->e32.r_addend);
+}
+
/*
* Get the whole file as a programming convenience in order to avoid
* malloc+lseek+read+free of many pieces. If successful, then mmap
@@ -341,6 +384,16 @@ static void wle(uint32_t val, uint32_t *x)
put_unaligned_le32(val, x);
}
+static void w8be(uint64_t val, uint64_t *x)
+{
+ put_unaligned_be64(val, x);
+}
+
+static void w8le(uint64_t val, uint64_t *x)
+{
+ put_unaligned_le64(val, x);
+}
+
/*
* Move reserved section indices SHN_LORESERVE..SHN_HIRESERVE out of
* the way to -256..-1, to avoid conflicting with real section
@@ -398,13 +451,12 @@ static inline void *get_index(void *start, int entsize, int index)
static int extable_ent_size;
static int long_size;
+#define ERRSTR_MAXSZ 256
#ifdef UNWINDER_ORC_ENABLED
/* ORC unwinder only support X86_64 */
#include <asm/orc_types.h>
-#define ERRSTR_MAXSZ 256
-
static char g_err[ERRSTR_MAXSZ];
static int *g_orc_ip_table;
static struct orc_entry *g_orc_table;
@@ -499,7 +551,19 @@ static void *sort_orctable(void *arg)
#endif
#ifdef MCOUNT_SORT_ENABLED
+
+/* Only used for sorting mcount table */
+static void rela_write_addend(Elf_Rela *rela, uint64_t val)
+{
+ e.rela_write_addend(rela, val);
+}
+
static pthread_t mcount_sort_thread;
+static bool sort_reloc;
+
+static long rela_type;
+
+static char m_err[ERRSTR_MAXSZ];
struct elf_mcount_loc {
Elf_Ehdr *ehdr;
@@ -508,6 +572,103 @@ struct elf_mcount_loc {
uint64_t stop_mcount_loc;
};
+/* Sort the relocations not the address itself */
+static void *sort_relocs(Elf_Ehdr *ehdr, uint64_t start_loc, uint64_t size)
+{
+ Elf_Shdr *shdr_start;
+ Elf_Rela *rel;
+ unsigned int shnum;
+ unsigned int count;
+ int shentsize;
+ void *vals;
+ void *ptr;
+
+ shdr_start = (Elf_Shdr *)((char *)ehdr + ehdr_shoff(ehdr));
+ shentsize = ehdr_shentsize(ehdr);
+
+ vals = malloc(long_size * size);
+ if (!vals) {
+ snprintf(m_err, ERRSTR_MAXSZ, "Failed to allocate sort array");
+ pthread_exit(m_err);
+ return NULL;
+ }
+
+ ptr = vals;
+
+ shnum = ehdr_shnum(ehdr);
+ if (shnum == SHN_UNDEF)
+ shnum = shdr_size(shdr_start);
+
+ for (int i = 0; i < shnum; i++) {
+ Elf_Shdr *shdr = get_index(shdr_start, shentsize, i);
+ void *end;
+
+ if (shdr_type(shdr) != SHT_RELA)
+ continue;
+
+ rel = (void *)ehdr + shdr_offset(shdr);
+ end = (void *)rel + shdr_size(shdr);
+
+ for (; (void *)rel < end; rel = (void *)rel + shdr_entsize(shdr)) {
+ uint64_t offset = rela_offset(rel);
+
+ if (offset >= start_loc && offset < start_loc + size) {
+ if (ptr + long_size > vals + size) {
+ free(vals);
+ snprintf(m_err, ERRSTR_MAXSZ,
+ "Too many relocations");
+ pthread_exit(m_err);
+ return NULL;
+ }
+
+ /* Make sure this has the correct type */
+ if (rela_info(rel) != rela_type) {
+ free(vals);
+ snprintf(m_err, ERRSTR_MAXSZ,
+ "rela has type %lx but expected %lx\n",
+ (long)rela_info(rel), rela_type);
+ pthread_exit(m_err);
+ return NULL;
+ }
+
+ if (long_size == 4)
+ *(uint32_t *)ptr = rela_addend(rel);
+ else
+ *(uint64_t *)ptr = rela_addend(rel);
+ ptr += long_size;
+ }
+ }
+ }
+ count = ptr - vals;
+ qsort(vals, count / long_size, long_size, compare_extable);
+
+ ptr = vals;
+ for (int i = 0; i < shnum; i++) {
+ Elf_Shdr *shdr = get_index(shdr_start, shentsize, i);
+ void *end;
+
+ if (shdr_type(shdr) != SHT_RELA)
+ continue;
+
+ rel = (void *)ehdr + shdr_offset(shdr);
+ end = (void *)rel + shdr_size(shdr);
+
+ for (; (void *)rel < end; rel = (void *)rel + shdr_entsize(shdr)) {
+ uint64_t offset = rela_offset(rel);
+
+ if (offset >= start_loc && offset < start_loc + size) {
+ if (long_size == 4)
+ rela_write_addend(rel, *(uint32_t *)ptr);
+ else
+ rela_write_addend(rel, *(uint64_t *)ptr);
+ ptr += long_size;
+ }
+ }
+ }
+ free(vals);
+ return NULL;
+}
+
/* Sort the addresses stored between __start_mcount_loc to __stop_mcount_loc in vmlinux */
static void *sort_mcount_loc(void *arg)
{
@@ -517,6 +678,9 @@ static void *sort_mcount_loc(void *arg)
uint64_t count = emloc->stop_mcount_loc - emloc->start_mcount_loc;
unsigned char *start_loc = (void *)emloc->ehdr + offset;
+ if (sort_reloc)
+ return sort_relocs(emloc->ehdr, emloc->start_mcount_loc, count);
+
qsort(start_loc, count/long_size, long_size, compare_extable);
return NULL;
}
@@ -866,12 +1030,14 @@ static int do_file(char const *const fname, void *addr)
r2 = r2le;
r8 = r8le;
w = wle;
+ w8 = w8le;
break;
case ELFDATA2MSB:
r = rbe;
r2 = r2be;
r8 = r8be;
w = wbe;
+ w8 = w8be;
break;
default:
fprintf(stderr, "unrecognized ELF data encoding %d: %s\n",
@@ -887,8 +1053,13 @@ static int do_file(char const *const fname, void *addr)
}
switch (r2(&ehdr->e32.e_machine)) {
- case EM_386:
case EM_AARCH64:
+#ifdef MCOUNT_SORT_ENABLED
+ sort_reloc = true;
+ rela_type = 0x403;
+#endif
+ /* fallthrough */
+ case EM_386:
case EM_LOONGARCH:
case EM_RISCV:
case EM_S390:
@@ -932,6 +1103,10 @@ static int do_file(char const *const fname, void *addr)
.sym_name = sym32_name,
.sym_value = sym32_value,
.sym_shndx = sym32_shndx,
+ .rela_offset = rela32_offset,
+ .rela_info = rela32_info,
+ .rela_addend = rela32_addend,
+ .rela_write_addend = rela32_write_addend,
};
e = efuncs;
@@ -965,6 +1140,10 @@ static int do_file(char const *const fname, void *addr)
.sym_name = sym64_name,
.sym_value = sym64_value,
.sym_shndx = sym64_shndx,
+ .rela_offset = rela64_offset,
+ .rela_info = rela64_info,
+ .rela_addend = rela64_addend,
+ .rela_write_addend = rela64_write_addend,
};
e = efuncs;
--
2.50.1 (Apple Git-155)
Reference in New Issue View Git Blame Copy Permalink