glibc/v2-0012-x86-cet-Sync-with-the-kernel-shadow-stack-interfa.patch

From fc14bc65a7e0d3fbe5d939272f25d2e81100e44a Mon Sep 17 00:00:00 2001
From: "H.J. Lu" <hjl.tools@gmail.com>
Date: Sat, 15 Aug 2020 08:04:23 -0700
Subject: [PATCH v2 12/15] x86/cet: Sync with the kernel shadow stack interface

Sync with the upstream shadow stack kernel.  Although shadow stack is
only supported for x86-64, the i386 shadow stack support is also adjusted.
Don't enable CET for i386 since CET won't work on i386.

1. When the shadow stack base in TCB is unset, the default shadow stack
is in use.  Use the current shadow stack pointer as the marker for the
default shadow stack.
2. Allocate shadow stack with the map_shadow_stack syscall.
3. Rename arch_prctl CET commands to ARCH_SHSTK_XXX.
4. Rewrite the CET control functions with the current kernel shadow stack
interface.

Since CET is no longer enabled by kernel, a separate patch will enable
shadow stack during startup.
---
 sysdeps/i386/nptl/tls.h                       |  2 +-
 sysdeps/unix/sysv/linux/i386/getcontext.S     | 28 +-----
 sysdeps/unix/sysv/linux/i386/makecontext.S    | 88 +++++++------------
 sysdeps/unix/sysv/linux/i386/swapcontext.S    | 22 +----
 sysdeps/unix/sysv/linux/x86/Makefile          |  1 +
 .../sysv/linux/x86/allocate-shadow-stack.c    | 54 ++++++++++++
 .../sysv/linux/x86/allocate-shadow-stack.h    | 27 ++++++
 sysdeps/unix/sysv/linux/x86/bits/mman.h       |  5 ++
 sysdeps/unix/sysv/linux/x86/cpu-features.c    | 13 ++-
 sysdeps/unix/sysv/linux/x86/dl-cet.h          | 16 +++-
 .../unix/sysv/linux/x86/include/asm/prctl.h   | 37 ++++----
 .../sysv/linux/x86/tst-cet-setcontext-1.c     | 17 ++--
 .../unix/sysv/linux/x86_64/__start_context.S  | 38 ++------
 sysdeps/unix/sysv/linux/x86_64/getcontext.S   | 30 ++-----
 sysdeps/unix/sysv/linux/x86_64/makecontext.c  | 29 +++---
 sysdeps/unix/sysv/linux/x86_64/swapcontext.S  | 22 +----
 sysdeps/x86/cpu-features.c                    | 15 ++--
 sysdeps/x86/dl-cet.c                          |  2 +-
 sysdeps/x86_64/nptl/tls.h                     |  2 +-
 19 files changed, 218 insertions(+), 230 deletions(-)
 create mode 100644 sysdeps/unix/sysv/linux/x86/allocate-shadow-stack.c
 create mode 100644 sysdeps/unix/sysv/linux/x86/allocate-shadow-stack.h

diff --git a/sysdeps/i386/nptl/tls.h b/sysdeps/i386/nptl/tls.h
index 95e7c5be51..da4b83dd49 100644
--- a/sysdeps/i386/nptl/tls.h
+++ b/sysdeps/i386/nptl/tls.h
@@ -49,7 +49,7 @@ typedef struct
   void *__private_tm[3];
   /* GCC split stack support.  */
   void *__private_ss;
-  /* The lowest address of shadow stack,  */
+  /* The marker for the current shadow stack.  */
   unsigned long ssp_base;
 } tcbhead_t;
 
diff --git a/sysdeps/unix/sysv/linux/i386/getcontext.S b/sysdeps/unix/sysv/linux/i386/getcontext.S
index b69a73847b..49ece43dd2 100644
--- a/sysdeps/unix/sysv/linux/i386/getcontext.S
+++ b/sysdeps/unix/sysv/linux/i386/getcontext.S
@@ -54,31 +54,11 @@ ENTRY(__getcontext)
 	cmpl	%gs:SSP_BASE_OFFSET, %eax
 	jnz	L(shadow_stack_bound_recorded)
 
-	/* Save EBX in the first scratch register slot.  */
-	movl	%ebx, oSCRATCH1(%edx)
-
-	/* Get the base address and size of the default shadow stack
-	   which must be the current shadow stack since nothing has
-	   been recorded yet.  */
-	sub	$24, %esp
-	mov	%esp, %ecx
-	movl	$ARCH_CET_STATUS, %ebx
-	movl	$__NR_arch_prctl, %eax
-	ENTER_KERNEL
-	testl	%eax, %eax
-	jz	L(continue_no_err)
-
-	/* This should never happen.  */
-	hlt
-
-L(continue_no_err):
-	/* Restore EBX from the first scratch register slot.  */
-	movl	oSCRATCH1(%edx), %ebx
-
-	/* Record the base of the current shadow stack.  */
-	movl	8(%esp), %eax
+	/* When the shadow stack base is unset, the default shadow
+	   stack is in use.  Use the current shadow stack pointer
+	   as the marker for the default shadow stack.  */
+	rdsspd	%eax
 	movl	%eax, %gs:SSP_BASE_OFFSET
-	add	$24, %esp
 
 L(shadow_stack_bound_recorded):
 	/* Load address of the context data structure.  */
diff --git a/sysdeps/unix/sysv/linux/i386/makecontext.S b/sysdeps/unix/sysv/linux/i386/makecontext.S
index 346cdd0e0a..bf481420dc 100644
--- a/sysdeps/unix/sysv/linux/i386/makecontext.S
+++ b/sysdeps/unix/sysv/linux/i386/makecontext.S
@@ -73,28 +73,37 @@ ENTRY(__makecontext)
 	testl	$X86_FEATURE_1_SHSTK, %gs:FEATURE_1_OFFSET
 	jz	L(skip_ssp)
 
-	/* Reload the pointer to ucontext.  */
-	movl	4(%esp), %eax
+	/* Shadow stack is enabled.  Load the pointer to ucontext in
+	   ECX.  */
+	movl	4(%esp), %ecx
+
+	/* Pass the address of __ssp[1] in EDX.  */
+	leal	(oSSP + 4)(%ecx), %edx
+	/* Pass stack size in EAX.  */
+	movl	oSS_SIZE(%ecx), %eax
+
+	/* Call __allocate_shadow_stack to allocate a new shadow stack.  */
+	call	__allocate_shadow_stack
+	/* Check for error return.  */
+	testl	%eax, %eax
+	jne	L(hlt)		/* This should never happen.  */
 
-	/* Shadow stack is enabled.  We need to allocate a new shadow
-	   stack.  */
-	subl	oSS_SP(%eax), %edx
-	shrl	$STACK_SIZE_TO_SHADOW_STACK_SIZE_SHIFT, %edx
+	/* Reload the pointer to ucontext into ECX.  */
+	movl	4(%esp), %ecx
 
-	/* Align shadow stack size to 8 bytes.  */
-	addl	$7, %edx
-	andl	$-8, %edx
+	/* Load the base address of the new shadow stack into EAX.  */
+	movl	(oSSP + 4)(%ecx), %eax
 
-	/* Store shadow stack size in __ssp[2].  */
-	movl	%edx, (oSSP + 8)(%eax)
+	/* Store the new shadow stack pointer, the shadow stack base +
+	   the shadow stack size - 4, in __ssp[0].  */
+	addl	(oSSP + 8)(%ecx), %eax
+	subl	$4, %eax
+	movl	%eax, oSSP(%ecx)
 
 	/* Save ESI in the second scratch register slot.  */
-	movl	%esi, oSCRATCH2(%eax)
+	movl	%esi, oSCRATCH2(%ecx)
 	/* Save EDI in the third scratch register slot.  */
-	movl	%edi, oSCRATCH3(%eax)
-
-	/* Save the pointer to ucontext.  */
-	movl	%eax, %edi
+	movl	%edi, oSCRATCH3(%ecx)
 
 	/* Get the original shadow stack pointer.  */
 	rdsspd	%esi
@@ -104,7 +113,7 @@ ENTRY(__makecontext)
 	andl	$-8, %esi
 
 	/* Load the top of the new stack into EDX.  */
-	movl	oESP(%eax), %edx
+	movl	oESP(%ecx), %edx
 
 	/* We need to terminate the FDE here because the unwinder looks
 	   at ra-1 for unwind information.  */
@@ -118,39 +127,14 @@ ENTRY(__makecontext)
 	   onto stack.  */
 	addl	$4, %esp
 
-	/* Allocate the new shadow stack.  Save EBX in the first scratch
-	   register slot.  */
-	movl	%ebx, oSCRATCH1(%eax)
-
-	/* CET syscall takes 64-bit sizes.  */
-	subl	$16, %esp
-	movl	(oSSP + 8)(%eax), %ecx
-	movl	%ecx, (%esp)
-	movl	$0, 4(%esp)
-	movl	%ecx, 8(%esp)
-	movl	$0, 12(%esp)
-	movl	%esp, %ecx
-
-	movl	$ARCH_CET_ALLOC_SHSTK, %ebx
-	movl	$__NR_arch_prctl, %eax
-	ENTER_KERNEL
-	testl	%eax, %eax
-	jne	L(hlt)		/* This should never happen.  */
-
-	/* Copy the base address of the new shadow stack to __ssp[1].  */
-	movl	(%esp), %eax
-	movl	%eax, (oSSP + 4)(%edi)
-
-	addl	$16, %esp
+	/* Load the new shadow stack base in __ssp[1] into EAX.  */
+	movl	(oSSP + 4)(%ecx), %eax
 
-	/* Restore EBX from the first scratch register slot.  */
-	movl	oSCRATCH1(%edi), %ebx
-
-	/* Get the size of the new shadow stack.  */
-	movl	(oSSP + 8)(%edi), %ecx
+	/* Load the new shadow stack size in __ssp[2] into EDI.  */
+	movl	(oSSP + 8)(%ecx), %edi
 
 	/* Use the restore stoken to restore the new shadow stack.  */
-	rstorssp -8(%eax, %ecx)
+	rstorssp -8(%eax, %edi)
 
 	/* Save the restore token at the next 8 byte aligned boundary
 	   on the original shadow stack.  */
@@ -162,27 +146,21 @@ ENTRY(__makecontext)
 	jmp	L(exitcode)
 1:
 
-	/* Get the new shadow stack pointer.  */
-	rdsspd	%eax
-
 	/* Use the restore stoken to restore the original shadow stack.  */
 	rstorssp -8(%esi)
 
 	/* Save the restore token on the new shadow stack.  */
 	saveprevssp
 
-	/* Store the new shadow stack pointer in __ssp[0].  */
-	movl	%eax, oSSP(%edi)
-
 	/* Restore the original stack.  */
 	mov	%edx, %esp
 
 	cfi_startproc
 
 	/* Restore ESI from the second scratch register slot.  */
-	movl	oSCRATCH2(%edi), %esi
+	movl	oSCRATCH2(%ecx), %esi
 	/* Restore EDI from the third scratch register slot.  */
-	movl	oSCRATCH3(%edi), %edi
+	movl	oSCRATCH3(%ecx), %edi
 
 	ret
 
diff --git a/sysdeps/unix/sysv/linux/i386/swapcontext.S b/sysdeps/unix/sysv/linux/i386/swapcontext.S
index 551df10c91..d4f31fa5d5 100644
--- a/sysdeps/unix/sysv/linux/i386/swapcontext.S
+++ b/sysdeps/unix/sysv/linux/i386/swapcontext.S
@@ -85,25 +85,11 @@ ENTRY(__swapcontext)
 	cmpl	%gs:SSP_BASE_OFFSET, %eax
 	jnz	L(shadow_stack_bound_recorded)
 
-	/* Get the base address and size of the default shadow stack
-	   which must be the current shadow stack since nothing has
-	   been recorded yet.  */
-	sub	$24, %esp
-	mov	%esp, %ecx
-	movl	$ARCH_CET_STATUS, %ebx
-	movl	$__NR_arch_prctl, %eax
-	ENTER_KERNEL
-	testl	%eax, %eax
-	jz	L(continue_no_err)
-
-	/* This should never happen.  */
-	hlt
-
-L(continue_no_err):
-	/* Record the base of the current shadow stack.  */
-	movl	8(%esp), %eax
+	/* When the shadow stack base is unset, the default shadow
+	   stack is in use.  Use the current shadow stack pointer
+	   as the marker for the default shadow stack.  */
+	rdsspd	%eax
 	movl	%eax, %gs:SSP_BASE_OFFSET
-	add	$24, %esp
 
 L(shadow_stack_bound_recorded):
 	/* Load address of the context data structure we save in.  */
diff --git a/sysdeps/unix/sysv/linux/x86/Makefile b/sysdeps/unix/sysv/linux/x86/Makefile
index 9dfdd689a9..ed0d6500b9 100644
--- a/sysdeps/unix/sysv/linux/x86/Makefile
+++ b/sysdeps/unix/sysv/linux/x86/Makefile
@@ -44,6 +44,7 @@ CFLAGS-tst-cet-vfork-1.c += -mshstk
 endif
 
 ifeq ($(subdir),stdlib)
+sysdep_routines += allocate-shadow-stack
 tests += tst-cet-setcontext-1
 CFLAGS-tst-cet-setcontext-1.c += -mshstk
 endif
diff --git a/sysdeps/unix/sysv/linux/x86/allocate-shadow-stack.c b/sysdeps/unix/sysv/linux/x86/allocate-shadow-stack.c
new file mode 100644
index 0000000000..3a76db1a60
--- /dev/null
+++ b/sysdeps/unix/sysv/linux/x86/allocate-shadow-stack.c
@@ -0,0 +1,54 @@
+/* Helper function to allocate shadow stack.
+   Copyright (C) 2023 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, see
+   <https://www.gnu.org/licenses/>.  */
+
+#include <sysdep.h>
+#include <stdint.h>
+#include <errno.h>
+#include <sys/mman.h>
+#include <libc-pointer-arith.h>
+#include <allocate-shadow-stack.h>
+
+/* NB: This can be treated as a syscall by caller.  */
+
+#ifndef __x86_64__
+__attribute__ ((regparm (2)))
+#endif
+long int
+__allocate_shadow_stack (size_t stack_size,
+			 shadow_stack_size_t *child_stack)
+{
+#ifdef __NR_map_shadow_stack
+  size_t shadow_stack_size
+    = stack_size >> STACK_SIZE_TO_SHADOW_STACK_SIZE_SHIFT;
+  /* Align shadow stack to 8 bytes.  */
+  shadow_stack_size = ALIGN_UP (shadow_stack_size, 8);
+  void *shadow_stack = (void *)INLINE_SYSCALL_CALL
+    (map_shadow_stack, NULL, shadow_stack_size, SHADOW_STACK_SET_TOKEN);
+  /* Report the map_shadow_stack error.  */
+  if (shadow_stack == MAP_FAILED)
+    return -errno;
+
+  /* Save the shadow stack base and size on child stack.  */
+  child_stack[0] = (uintptr_t) shadow_stack;
+  child_stack[1] = shadow_stack_size;
+
+  return 0;
+#else
+  return -ENOSYS;
+#endif
+}
diff --git a/sysdeps/unix/sysv/linux/x86/allocate-shadow-stack.h b/sysdeps/unix/sysv/linux/x86/allocate-shadow-stack.h
new file mode 100644
index 0000000000..834373e0d3
--- /dev/null
+++ b/sysdeps/unix/sysv/linux/x86/allocate-shadow-stack.h
@@ -0,0 +1,27 @@
+/* Helper function to allocate shadow stack.
+   Copyright (C) 2023 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, see
+   <https://www.gnu.org/licenses/>.  */
+
+#include <ucontext.h>
+
+typedef __typeof (((ucontext_t *) 0)->__ssp[0]) shadow_stack_size_t;
+
+extern long int __allocate_shadow_stack (size_t, shadow_stack_size_t *)
+#ifndef __x86_64__
+  __attribute__ ((regparm (2)))
+#endif
+  attribute_hidden;
diff --git a/sysdeps/unix/sysv/linux/x86/bits/mman.h b/sysdeps/unix/sysv/linux/x86/bits/mman.h
index b335ceff43..17bb078375 100644
--- a/sysdeps/unix/sysv/linux/x86/bits/mman.h
+++ b/sysdeps/unix/sysv/linux/x86/bits/mman.h
@@ -26,6 +26,11 @@
 /* Other flags.  */
 #define MAP_32BIT	0x40		/* Only give out 32-bit addresses.  */
 
+#ifdef __USE_MISC
+/* Set up a restore token in the newly allocatd shadow stack */
+# define SHADOW_STACK_SET_TOKEN 0x1
+#endif
+
 #include <bits/mman-map-flags-generic.h>
 
 /* Include generic Linux declarations.  */
diff --git a/sysdeps/unix/sysv/linux/x86/cpu-features.c b/sysdeps/unix/sysv/linux/x86/cpu-features.c
index 41e7600668..0e6e2bf855 100644
--- a/sysdeps/unix/sysv/linux/x86/cpu-features.c
+++ b/sysdeps/unix/sysv/linux/x86/cpu-features.c
@@ -23,10 +23,15 @@
 static inline int __attribute__ ((always_inline))
 get_cet_status (void)
 {
-  unsigned long long cet_status[3];
-  if (INTERNAL_SYSCALL_CALL (arch_prctl, ARCH_CET_STATUS, cet_status) == 0)
-    return cet_status[0];
-  return 0;
+  unsigned long long kernel_feature;
+  unsigned int status = 0;
+  if (INTERNAL_SYSCALL_CALL (arch_prctl, ARCH_SHSTK_STATUS,
+			     &kernel_feature) == 0)
+    {
+      if ((kernel_feature & ARCH_SHSTK_SHSTK) != 0)
+	status = GNU_PROPERTY_X86_FEATURE_1_SHSTK;
+    }
+  return status;
 }
 
 # ifndef SHARED
diff --git a/sysdeps/unix/sysv/linux/x86/dl-cet.h b/sysdeps/unix/sysv/linux/x86/dl-cet.h
index c885bf1323..da220ac627 100644
--- a/sysdeps/unix/sysv/linux/x86/dl-cet.h
+++ b/sysdeps/unix/sysv/linux/x86/dl-cet.h
@@ -21,12 +21,20 @@
 static inline int __attribute__ ((always_inline))
 dl_cet_disable_cet (unsigned int cet_feature)
 {
-  return (int) INTERNAL_SYSCALL_CALL (arch_prctl, ARCH_CET_DISABLE,
-				      cet_feature);
+  if (cet_feature != GNU_PROPERTY_X86_FEATURE_1_SHSTK)
+    return -1;
+  long long int kernel_feature = ARCH_SHSTK_SHSTK;
+  return (int) INTERNAL_SYSCALL_CALL (arch_prctl, ARCH_SHSTK_DISABLE,
+				      kernel_feature);
 }
 
 static inline int __attribute__ ((always_inline))
-dl_cet_lock_cet (void)
+dl_cet_lock_cet (unsigned int cet_feature)
 {
-  return (int) INTERNAL_SYSCALL_CALL (arch_prctl, ARCH_CET_LOCK, 0);
+  if (cet_feature != GNU_PROPERTY_X86_FEATURE_1_SHSTK)
+    return -1;
+  /* Lock all SHSTK features.  */
+  long long int kernel_feature = -1;
+  return (int) INTERNAL_SYSCALL_CALL (arch_prctl, ARCH_SHSTK_LOCK,
+				      kernel_feature);
 }
diff --git a/sysdeps/unix/sysv/linux/x86/include/asm/prctl.h b/sysdeps/unix/sysv/linux/x86/include/asm/prctl.h
index 45ad0b052f..2f511321ad 100644
--- a/sysdeps/unix/sysv/linux/x86/include/asm/prctl.h
+++ b/sysdeps/unix/sysv/linux/x86/include/asm/prctl.h
@@ -4,24 +4,19 @@
 
 #include_next <asm/prctl.h>
 
-#ifndef ARCH_CET_STATUS
-/* CET features:
-   IBT:   GNU_PROPERTY_X86_FEATURE_1_IBT
-   SHSTK: GNU_PROPERTY_X86_FEATURE_1_SHSTK
- */
-/* Return CET features in unsigned long long *addr:
-     features: addr[0].
-     shadow stack base address: addr[1].
-     shadow stack size: addr[2].
- */
-# define ARCH_CET_STATUS	0x3001
-/* Disable CET features in unsigned int features.  */
-# define ARCH_CET_DISABLE	0x3002
-/* Lock all CET features.  */
-# define ARCH_CET_LOCK		0x3003
-/* Allocate a new shadow stack with unsigned long long *addr:
-     IN: requested shadow stack size: *addr.
-     OUT: allocated shadow stack address: *addr.
- */
-# define ARCH_CET_ALLOC_SHSTK	0x3004
-#endif /* ARCH_CET_STATUS */
+#ifndef ARCH_SHSTK_ENABLE
+/* Enable SHSTK features in unsigned long int features.  */
+# define ARCH_SHSTK_ENABLE		0x5001
+/* Disable SHSTK features in unsigned long int features.  */
+# define ARCH_SHSTK_DISABLE		0x5002
+/* Lock SHSTK features in unsigned long int features.  */
+# define ARCH_SHSTK_LOCK		0x5003
+/* Unlock SHSTK features in unsigned long int features.  */
+# define ARCH_SHSTK_UNLOCK		0x5004
+/* Return SHSTK features in unsigned long int features.  */
+# define ARCH_SHSTK_STATUS		0x5005
+
+/* ARCH_SHSTK_ features bits */
+# define ARCH_SHSTK_SHSTK		0x1
+# define ARCH_SHSTK_WRSS		0x2
+#endif
diff --git a/sysdeps/unix/sysv/linux/x86/tst-cet-setcontext-1.c b/sysdeps/unix/sysv/linux/x86/tst-cet-setcontext-1.c
index 837a9fd0eb..2ea66c803b 100644
--- a/sysdeps/unix/sysv/linux/x86/tst-cet-setcontext-1.c
+++ b/sysdeps/unix/sysv/linux/x86/tst-cet-setcontext-1.c
@@ -87,15 +87,14 @@ do_test (void)
   ctx[4].uc_link = &ctx[0];
   makecontext (&ctx[4], (void (*) (void)) f1, 0);
 
-  /* NB: When shadow stack is enabled, makecontext calls arch_prctl
-     with ARCH_CET_ALLOC_SHSTK to allocate a new shadow stack which
-     can be unmapped.  The base address and size of the new shadow
-     stack are returned in __ssp[1] and __ssp[2].  makecontext is
-     called for CTX1, CTX3 and CTX4.  But only CTX1 is used.  New
-     shadow stacks are allocated in the order of CTX3, CTX1, CTX4.
-     It is very likely that CTX1's shadow stack is placed between
-     CTX3 and CTX4.  We munmap CTX3's and CTX4's shadow stacks to
-     create gaps above and below CTX1's shadow stack.  We check
+  /* NB: When shadow stack is enabled, makecontext calls map_shadow_stack
+     to allocate a new shadow stack which can be unmapped.  The base
+     address and size of the new shadow stack are returned in __ssp[1]
+     and __ssp[2].  makecontext is called for CTX1, CTX3 and CTX4.  But
+     only CTX1 is used.  New shadow stacks are allocated in the order
+     of CTX3, CTX1, CTX4.  It is very likely that CTX1's shadow stack is
+     placed between CTX3 and CTX4.  We munmap CTX3's and CTX4's shadow
+     stacks to create gaps above and below CTX1's shadow stack.  We check
      that setcontext CTX1 works correctly in this case.  */
   if (_get_ssp () != 0)
     {
diff --git a/sysdeps/unix/sysv/linux/x86_64/__start_context.S b/sysdeps/unix/sysv/linux/x86_64/__start_context.S
index f6436dd6bb..ae04203c90 100644
--- a/sysdeps/unix/sysv/linux/x86_64/__start_context.S
+++ b/sysdeps/unix/sysv/linux/x86_64/__start_context.S
@@ -24,20 +24,14 @@
 /* Use CALL to push __start_context onto the new stack as well as the new
    shadow stack.  RDI points to ucontext:
    Incoming:
-     __ssp[0]: The original caller's shadow stack pointer.
-     __ssp[1]: The size of the new shadow stack.
-     __ssp[2]: The size of the new shadow stack.
-   Outgoing:
      __ssp[0]: The new shadow stack pointer.
      __ssp[1]: The base address of the new shadow stack.
      __ssp[2]: The size of the new shadow stack.
  */
 
 ENTRY(__push___start_context)
-	/* Save the pointer to ucontext.  */
-	movq	%rdi, %r9
 	/* Get the original shadow stack pointer.  */
-	rdsspq	%r8
+	rdsspq	%rcx
 	/* Save the original stack pointer.  */
 	movq	%rsp, %rdx
 	/* Load the top of the new stack into RSI.  */
@@ -45,24 +39,12 @@ ENTRY(__push___start_context)
 	/* Add 8 bytes to RSI since CALL will push the 8-byte return
 	   address onto stack.  */
 	leaq	8(%rsi), %rsp
-	/* Allocate the new shadow stack.  The size of the new shadow
-	   stack is passed in __ssp[1].  */
-	lea	(oSSP + 8)(%rdi), %RSI_LP
-	movl	$ARCH_CET_ALLOC_SHSTK, %edi
-	movl	$__NR_arch_prctl, %eax
-	/* The new shadow stack base is returned in __ssp[1].  */
-	syscall
-	testq	%rax, %rax
-	jne	L(hlt)		/* This should never happen.  */
-
-	/* Get the size of the new shadow stack.  */
-	movq	8(%rsi), %rdi
-
-	/* Get the base address of the new shadow stack.  */
-	movq	(%rsi), %rsi
-
+	/* The size of the new shadow stack is stored in __ssp[2].  */
+	mov	(oSSP + 16)(%rdi), %RSI_LP
+	/* The new shadow stack base is stored in __ssp[1].  */
+	mov	(oSSP + 8)(%rdi), %RAX_LP
 	/* Use the restore stoken to restore the new shadow stack.  */
-	rstorssp -8(%rsi, %rdi)
+	rstorssp -8(%rax, %rsi)
 
 	/* Save the restore token on the original shadow stack.  */
 	saveprevssp
@@ -73,18 +55,12 @@ ENTRY(__push___start_context)
 	jmp	__start_context
 1:
 
-	/* Get the new shadow stack pointer.  */
-	rdsspq	%rdi
-
 	/* Use the restore stoken to restore the original shadow stack.  */
-	rstorssp -8(%r8)
+	rstorssp -8(%rcx)
 
 	/* Save the restore token on the new shadow stack.  */
 	saveprevssp
 
-	/* Store the new shadow stack pointer in __ssp[0].  */
-	movq	%rdi, oSSP(%r9)
-
 	/* Restore the original stack.  */
 	mov	%rdx, %rsp
 	ret
diff --git a/sysdeps/unix/sysv/linux/x86_64/getcontext.S b/sysdeps/unix/sysv/linux/x86_64/getcontext.S
index a00e2f6290..71f3802dca 100644
--- a/sysdeps/unix/sysv/linux/x86_64/getcontext.S
+++ b/sysdeps/unix/sysv/linux/x86_64/getcontext.S
@@ -58,35 +58,15 @@ ENTRY(__getcontext)
 	testl	$X86_FEATURE_1_SHSTK, %fs:FEATURE_1_OFFSET
 	jz	L(no_shstk)
 
-	/* Save RDI in RDX which won't be clobbered by syscall.  */
-	movq	%rdi, %rdx
-
 	xorl	%eax, %eax
 	cmpq	%fs:SSP_BASE_OFFSET, %rax
 	jnz	L(shadow_stack_bound_recorded)
 
-	/* Get the base address and size of the default shadow stack
-	   which must be the current shadow stack since nothing has
-	   been recorded yet.  */
-	sub	$24, %RSP_LP
-	mov	%RSP_LP, %RSI_LP
-	movl	$ARCH_CET_STATUS, %edi
-	movl	$__NR_arch_prctl, %eax
-	syscall
-	testq	%rax, %rax
-	jz	L(continue_no_err)
-
-	/* This should never happen.  */
-	hlt
-
-L(continue_no_err):
-	/* Record the base of the current shadow stack.  */
-	movq	8(%rsp), %rax
+	/* When the shadow stack base is unset, the default shadow
+	   stack is in use.  Use the current shadow stack pointer
+	   as the marker for the default shadow stack.  */
+	rdsspq	%rax
 	movq	%rax, %fs:SSP_BASE_OFFSET
-	add	$24, %RSP_LP
-
-	/* Restore RDI.  */
-	movq	%rdx, %rdi
 
 L(shadow_stack_bound_recorded):
 	/* Get the current shadow stack pointer.  */
@@ -94,7 +74,7 @@ L(shadow_stack_bound_recorded):
 	/* NB: Save the caller's shadow stack so that we can jump back
 	   to the caller directly.  */
 	addq	$8, %rax
-	movq	%rax, oSSP(%rdx)
+	movq	%rax, oSSP(%rdi)
 
 	/* Save the current shadow stack base in ucontext.  */
 	movq	%fs:SSP_BASE_OFFSET, %rax
diff --git a/sysdeps/unix/sysv/linux/x86_64/makecontext.c b/sysdeps/unix/sysv/linux/x86_64/makecontext.c
index de9e03eb81..788b730132 100644
--- a/sysdeps/unix/sysv/linux/x86_64/makecontext.c
+++ b/sysdeps/unix/sysv/linux/x86_64/makecontext.c
@@ -24,6 +24,8 @@
 # include <pthread.h>
 # include <libc-pointer-arith.h>
 # include <sys/prctl.h>
+# include <sys/mman.h>
+# include <allocate-shadow-stack.h>
 #endif
 
 #include "ucontext_i.h"
@@ -88,23 +90,24 @@ __makecontext (ucontext_t *ucp, void (*func) (void), int argc, ...)
   if ((feature_1 & X86_FEATURE_1_SHSTK) != 0)
     {
       /* Shadow stack is enabled.  We need to allocate a new shadow
-         stack.  */
-      unsigned long ssp_size = (((uintptr_t) sp
-				 - (uintptr_t) ucp->uc_stack.ss_sp)
-				>> STACK_SIZE_TO_SHADOW_STACK_SIZE_SHIFT);
-      /* Align shadow stack to 8 bytes.  */
-      ssp_size = ALIGN_UP (ssp_size, 8);
-
-      ucp->__ssp[1] = ssp_size;
-      ucp->__ssp[2] = ssp_size;
-
-      /* Call __push___start_context to allocate a new shadow stack,
-	 push __start_context onto the new stack as well as the new
-	 shadow stack.  NB: After __push___start_context returns,
+         stack.  NB:
 	   ucp->__ssp[0]: The new shadow stack pointer.
 	   ucp->__ssp[1]: The base address of the new shadow stack.
 	   ucp->__ssp[2]: The size of the new shadow stack.
        */
+      long int ret
+	= __allocate_shadow_stack (((uintptr_t) sp
+				    - (uintptr_t) ucp->uc_stack.ss_sp),
+				   &ucp->__ssp[1]);
+      if (ret != 0)
+	{
+	  /* FIXME: What should we do?  */
+	  abort ();
+	}
+
+      ucp->__ssp[0] = ucp->__ssp[1] + ucp->__ssp[2] - 8;
+      /* Call __push___start_context to push __start_context onto the new
+	 stack as well as the new shadow stack.  */
       __push___start_context (ucp);
     }
   else
diff --git a/sysdeps/unix/sysv/linux/x86_64/swapcontext.S b/sysdeps/unix/sysv/linux/x86_64/swapcontext.S
index 5925752164..2f2fe9875b 100644
--- a/sysdeps/unix/sysv/linux/x86_64/swapcontext.S
+++ b/sysdeps/unix/sysv/linux/x86_64/swapcontext.S
@@ -109,25 +109,11 @@ ENTRY(__swapcontext)
 	cmpq	%fs:SSP_BASE_OFFSET, %rax
 	jnz	L(shadow_stack_bound_recorded)
 
-	/* Get the base address and size of the default shadow stack
-	   which must be the current shadow stack since nothing has
-	   been recorded yet.  */
-	sub	$24, %RSP_LP
-	mov	%RSP_LP, %RSI_LP
-	movl	$ARCH_CET_STATUS, %edi
-	movl	$__NR_arch_prctl, %eax
-	syscall
-	testq	%rax, %rax
-	jz	L(continue_no_err)
-
-	/* This should never happen.  */
-	hlt
-
-L(continue_no_err):
-	/* Record the base of the current shadow stack.  */
-	movq	8(%rsp), %rax
+	/* When the shadow stack base is unset, the default shadow
+	   stack is in use.  Use the current shadow stack pointer
+	   as the marker for the default shadow stack.  */
+	rdsspq	%rax
 	movq	%rax, %fs:SSP_BASE_OFFSET
-	add	$24, %RSP_LP
 
 L(shadow_stack_bound_recorded):
         /* If we unwind the stack, we can't undo stack unwinding.  Just
diff --git a/sysdeps/x86/cpu-features.c b/sysdeps/x86/cpu-features.c
index 5bff8ec0b4..6c0d6c100d 100644
--- a/sysdeps/x86/cpu-features.c
+++ b/sysdeps/x86/cpu-features.c
@@ -851,8 +851,9 @@ no_cpuid:
 
 # ifndef SHARED
       /* Check if IBT and SHSTK are enabled by kernel.  */
-      if ((cet_status & GNU_PROPERTY_X86_FEATURE_1_IBT)
-	  || (cet_status & GNU_PROPERTY_X86_FEATURE_1_SHSTK))
+      if ((cet_status
+	   & (GNU_PROPERTY_X86_FEATURE_1_IBT
+	      | GNU_PROPERTY_X86_FEATURE_1_SHSTK)))
 	{
 	  /* Disable IBT and/or SHSTK if they are enabled by kernel, but
 	     disabled by environment variable:
@@ -861,9 +862,11 @@ no_cpuid:
 	   */
 	  unsigned int cet_feature = 0;
 	  if (!CPU_FEATURE_USABLE (IBT))
-	    cet_feature |= GNU_PROPERTY_X86_FEATURE_1_IBT;
+	    cet_feature |= (cet_status
+			    & GNU_PROPERTY_X86_FEATURE_1_IBT);
 	  if (!CPU_FEATURE_USABLE (SHSTK))
-	    cet_feature |= GNU_PROPERTY_X86_FEATURE_1_SHSTK;
+	    cet_feature |= (cet_status
+			    & GNU_PROPERTY_X86_FEATURE_1_SHSTK);
 
 	  if (cet_feature)
 	    {
@@ -878,7 +881,9 @@ no_cpuid:
 	     lock CET if IBT or SHSTK is enabled permissively.  */
 	  if (GL(dl_x86_feature_control).ibt != cet_permissive
 	      && GL(dl_x86_feature_control).shstk != cet_permissive)
-	    dl_cet_lock_cet ();
+	    dl_cet_lock_cet (GL(dl_x86_feature_1)
+			     & (GNU_PROPERTY_X86_FEATURE_1_IBT
+				| GNU_PROPERTY_X86_FEATURE_1_SHSTK));
 	}
 # endif
     }
diff --git a/sysdeps/x86/dl-cet.c b/sysdeps/x86/dl-cet.c
index 67c51ee8c2..8b911fd931 100644
--- a/sysdeps/x86/dl-cet.c
+++ b/sysdeps/x86/dl-cet.c
@@ -201,7 +201,7 @@ dl_cet_check_startup (struct link_map *m, struct dl_cet_info *info)
 	feature_1_lock |= GNU_PROPERTY_X86_FEATURE_1_SHSTK;
 
       if (feature_1_lock != 0
-	  && dl_cet_lock_cet () != 0)
+	  && dl_cet_lock_cet (feature_1_lock) != 0)
 	_dl_fatal_printf ("%s: can't lock CET\n", info->program);
     }
 
diff --git a/sysdeps/x86_64/nptl/tls.h b/sysdeps/x86_64/nptl/tls.h
index 1403f939f7..4bcc2552a1 100644
--- a/sysdeps/x86_64/nptl/tls.h
+++ b/sysdeps/x86_64/nptl/tls.h
@@ -60,7 +60,7 @@ typedef struct
   void *__private_tm[4];
   /* GCC split stack support.  */
   void *__private_ss;
-  /* The lowest address of shadow stack,  */
+  /* The marker for the current shadow stack.  */
   unsigned long long int ssp_base;
   /* Must be kept even if it is no longer used by glibc since programs,
      like AddressSanitizer, depend on the size of tcbhead_t.  */
-- 
2.40.1