From 5a35e9b70f16e433731a3957d8933ffe1b88e6d0 Mon Sep 17 00:00:00 2001
From: Arjun Shankar <arjun@redhat.com>
Date: Tue, 12 Mar 2024 17:10:19 +0100
Subject: [PATCH] malloc: Do not use MAP_NORESERVE to allocate heap segments
 (RHEL-21884)

Resolves: RHEL-21884
---
 glibc-RHEL-21884.patch | 73 ++++++++++++++++++++++++++++++++++++++++++
 glibc.spec             |  6 +++-
 2 files changed, 78 insertions(+), 1 deletion(-)
 create mode 100644 glibc-RHEL-21884.patch

diff --git a/glibc-RHEL-21884.patch b/glibc-RHEL-21884.patch
new file mode 100644
index 0000000..39f1500
--- /dev/null
+++ b/glibc-RHEL-21884.patch
@@ -0,0 +1,73 @@
+commit 85860ad6eaf4c9739318f6b2a1ff7c2fa6b12ab5
+Author: Florian Weimer <fweimer@redhat.com>
+Date:   Mon Aug 15 16:45:40 2022 +0200
+
+    malloc: Do not use MAP_NORESERVE to allocate heap segments
+    
+    Address space for heap segments is reserved in a mmap call with
+    MAP_ANONYMOUS | MAP_PRIVATE and protection flags PROT_NONE.  This
+    reservation does not count against the RSS limit of the process or
+    system.  Backing memory is allocated using mprotect in alloc_new_heap
+    and grow_heap, and at this point, the allocator expects the kernel
+    to provide memory (subject to memory overcommit).
+    
+    The SIGSEGV that might generate due to MAP_NORESERVE (according to
+    the mmap manual page) does not seem to occur in practice, it's always
+    SIGKILL from the OOM killer.  Even if there is a way that SIGSEGV
+    could be generated, it is confusing to applications that this only
+    happens for secondary heaps, not for large mmap-based allocations,
+    and not for the main arena.
+    
+    Reviewed-by: Siddhesh Poyarekar <siddhesh@sourceware.org>
+
+Conflicts:
+	malloc/arena.c
+	(huge page support was added upstream)
+
+diff --git a/malloc/arena.c b/malloc/arena.c
+index 667484630ed0afa5..2852783355d3d869 100644
+--- a/malloc/arena.c
++++ b/malloc/arena.c
+@@ -466,8 +466,7 @@ new_heap (size_t size, size_t top_pad)
+   p2 = MAP_FAILED;
+   if (aligned_heap_area)
+     {
+-      p2 = (char *) MMAP (aligned_heap_area, HEAP_MAX_SIZE, PROT_NONE,
+-                          MAP_NORESERVE);
++      p2 = (char *) MMAP (aligned_heap_area, HEAP_MAX_SIZE, PROT_NONE, 0);
+       aligned_heap_area = NULL;
+       if (p2 != MAP_FAILED && ((unsigned long) p2 & (HEAP_MAX_SIZE - 1)))
+         {
+@@ -477,7 +476,7 @@ new_heap (size_t size, size_t top_pad)
+     }
+   if (p2 == MAP_FAILED)
+     {
+-      p1 = (char *) MMAP (0, HEAP_MAX_SIZE << 1, PROT_NONE, MAP_NORESERVE);
++      p1 = (char *) MMAP (0, HEAP_MAX_SIZE << 1, PROT_NONE, 0);
+       if (p1 != MAP_FAILED)
+         {
+           p2 = (char *) (((unsigned long) p1 + (HEAP_MAX_SIZE - 1))
+@@ -493,7 +492,7 @@ new_heap (size_t size, size_t top_pad)
+         {
+           /* Try to take the chance that an allocation of only HEAP_MAX_SIZE
+              is already aligned. */
+-          p2 = (char *) MMAP (0, HEAP_MAX_SIZE, PROT_NONE, MAP_NORESERVE);
++          p2 = (char *) MMAP (0, HEAP_MAX_SIZE, PROT_NONE, 0);
+           if (p2 == MAP_FAILED)
+             return 0;
+ 
+diff --git a/malloc/malloc.c b/malloc/malloc.c
+index 375f50f5db13e234..fe80b8239756a7c9 100644
+--- a/malloc/malloc.c
++++ b/malloc/malloc.c
+@@ -1112,10 +1112,6 @@ static mchunkptr mremap_chunk(mchunkptr p, size_t new_size);
+ # define MAP_ANONYMOUS MAP_ANON
+ #endif
+ 
+-#ifndef MAP_NORESERVE
+-# define MAP_NORESERVE 0
+-#endif
+-
+ #define MMAP(addr, size, prot, flags) \
+  __mmap((addr), (size), (prot), (flags)|MAP_ANONYMOUS|MAP_PRIVATE, -1, 0)
+ 
diff --git a/glibc.spec b/glibc.spec
index ba1c492..3c13fdb 100644
--- a/glibc.spec
+++ b/glibc.spec
@@ -155,7 +155,7 @@ end \
 Summary: The GNU libc libraries
 Name: glibc
 Version: %{glibcversion}
-Release: 102%{?dist}
+Release: 103%{?dist}
 
 # In general, GPLv2+ is used by programs, LGPLv2+ is used for
 # libraries.
@@ -811,6 +811,7 @@ Patch574: glibc-RHEL-21556.patch
 Patch575: glibc-RHEL-23472.patch
 Patch576: glibc-RHEL-20172-1.patch
 Patch577: glibc-RHEL-20172-2.patch
+Patch578: glibc-RHEL-21884.patch
 
 ##############################################################################
 # Continued list of core "glibc" package information:
@@ -2969,6 +2970,9 @@ update_gconv_modules_cache ()
 %endif
 
 %changelog
+* Tue Mar 12 2024 Arjun Shankar <arjun@redhat.com> - 2.34-103
+- malloc: Do not use MAP_NORESERVE to allocate heap segments (RHEL-21884)
+
 * Fri Mar  8 2024 DJ Delorie <dj@redhat.com> - 2.34-102
 - Add glibc.cpu.prefer_map_32bit_exec tunable (RHEL-20172)