Fix mysql's stack overflow problem (or perhaps more accurately, work around

brain damage in RHEL5 kernel)

mysql-stack-guard.patch

@@ -0,0 +1,94 @@
+mysql is not accounting for the "guard page" when setting thread stack size
+requests.  This is fatal on PPC systems, which may use guard pages as large
+as 64K.  I'll bet a good deal that the hacks it uses for IA64 are a result
+of misdiagnosis of a similar problem, so remove them.
+
+It is not at this point entirely clear whether mysql is wrong in ignoring
+the guard page, or whether this is a RHEL bug (see our bz#435337).  So not
+reporting this upstream yet.  But we need the patch now, so we can build mysql
+in rawhide (the build machines are using RHEL5 kernels).
+
+
+diff -Naur mysql-5.0.45.orig/sql/mysqld.cc mysql-5.0.45/sql/mysqld.cc
+--- mysql-5.0.45.orig/sql/mysqld.cc	2007-07-04 09:06:03.000000000 -0400
++++ mysql-5.0.45/sql/mysqld.cc	2008-02-28 15:17:20.000000000 -0500
+@@ -2286,6 +2286,7 @@
+ {
+   int error;
+   pthread_attr_t thr_attr;
++  size_t guard_size = 0;
+   DBUG_ENTER("start_signal_handler");
+ 
+   (void) pthread_attr_init(&thr_attr);
+@@ -2294,15 +2295,9 @@
+   (void) pthread_attr_setdetachstate(&thr_attr,PTHREAD_CREATE_DETACHED);
+   if (!(opt_specialflag & SPECIAL_NO_PRIOR))
+     my_pthread_attr_setprio(&thr_attr,INTERRUPT_PRIOR);
+-#if defined(__ia64__) || defined(__ia64)
+-  /*
+-    Peculiar things with ia64 platforms - it seems we only have half the
+-    stack size in reality, so we have to double it here
+-  */
+-  pthread_attr_setstacksize(&thr_attr,thread_stack*2);
+-#else
+-  pthread_attr_setstacksize(&thr_attr,thread_stack);
+-#endif
++
++  pthread_attr_getguardsize(&thr_attr, &guard_size);
++  pthread_attr_setstacksize(&thr_attr, thread_stack + guard_size);
+ #endif
+ 
+   (void) pthread_mutex_lock(&LOCK_thread_count);
+@@ -3499,37 +3494,29 @@
+   init_signals();
+   if (!(opt_specialflag & SPECIAL_NO_PRIOR))
+     my_pthread_setprio(pthread_self(),CONNECT_PRIOR);
+-#if defined(__ia64__) || defined(__ia64)
+-  /*
+-    Peculiar things with ia64 platforms - it seems we only have half the
+-    stack size in reality, so we have to double it here
+-  */
+-  pthread_attr_setstacksize(&connection_attrib,thread_stack*2);
+-#else
+-  pthread_attr_setstacksize(&connection_attrib,thread_stack);
+-#endif
+-#ifdef HAVE_PTHREAD_ATTR_GETSTACKSIZE
++
+   {
++    size_t guard_size = 0;
++    size_t stack_size = 0;
++
++    pthread_attr_getguardsize(&connection_attrib, &guard_size);
++
++    pthread_attr_setstacksize(&connection_attrib, thread_stack + guard_size);
++
++#ifdef HAVE_PTHREAD_ATTR_GETSTACKSIZE
+     /* Retrieve used stack size;  Needed for checking stack overflows */
+-    size_t stack_size= 0;
+     pthread_attr_getstacksize(&connection_attrib, &stack_size);
+-#if defined(__ia64__) || defined(__ia64)
+-    stack_size/= 2;
+-#endif
+     /* We must check if stack_size = 0 as Solaris 2.9 can return 0 here */
+-    if (stack_size && stack_size < thread_stack)
++    if (stack_size && stack_size < thread_stack + guard_size)
+     {
+       if (global_system_variables.log_warnings)
+-	sql_print_warning("Asked for %lu thread stack, but got %ld",
+-			  thread_stack, (long) stack_size);
+-#if defined(__ia64__) || defined(__ia64)
+-      thread_stack= stack_size*2;
+-#else
+-      thread_stack= stack_size;
+-#endif
++	sql_print_warning("Asked for %lu+%lu thread stack, but got %ld",
++			  thread_stack, (long) guard_size, (long) stack_size);
++      thread_stack= stack_size - guard_size;
+     }
+-  }
+ #endif
++  }
++
+ #ifdef __NETWARE__
+   /* Increasing stacksize of threads on NetWare */
+ 

mysql-stack.patch

@@ -4,28 +4,11 @@ Knock it up to twice the pre-5.0.33 value to have some margin for future
 changes in compiler, glibc, etc.  Suspect this number if
 "execution_constants" regression test fails.
 
-As of 5.0.45 and F9, it seems 16384 isn't enough anymore either (ppc fails,
+Note: STACK_MIN_SIZE has to be enough less than DEFAULT_THREAD_STACK / 8
-and seems to now need something above 24576 --- why?).  STACK_MIN_SIZE has
+to provide some headroom, which means that its value can't be raised too much
-to be enough less than DEFAULT_THREAD_STACK / 8 to provide some headroom,
+further without increasing the latter as well.
-which means we are now also forced to raise DEFAULT_THREAD_STACK for 32-bit
-arches.
 
 
-diff -Naur mysql-5.0.45.orig/include/my_pthread.h mysql-5.0.45/include/my_pthread.h
---- mysql-5.0.45.orig/include/my_pthread.h	2007-07-04 09:06:05.000000000 -0400
-+++ mysql-5.0.45/include/my_pthread.h	2008-01-08 17:32:37.000000000 -0500
-@@ -735,9 +735,9 @@
-   MySQL can survive with 32K, but some glibc libraries require > 128K stack
-   To resolve hostnames. Also recursive stored procedures needs stack.
- */
--#define DEFAULT_THREAD_STACK	(256*1024L)
-+#define DEFAULT_THREAD_STACK	(512*1024L)
- #else
--#define DEFAULT_THREAD_STACK	(192*1024)
-+#define DEFAULT_THREAD_STACK	(512*1024)
- #endif
- #endif
- 
 diff -Naur mysql-5.0.45.orig/sql/mysql_priv.h mysql-5.0.45/sql/mysql_priv.h
 --- mysql-5.0.45.orig/sql/mysql_priv.h	2007-07-04 09:06:41.000000000 -0400
 +++ mysql-5.0.45/sql/mysql_priv.h	2008-01-08 17:32:22.000000000 -0500
@@ -34,7 +17,7 @@ diff -Naur mysql-5.0.45.orig/sql/mysql_priv.h mysql-5.0.45/sql/mysql_priv.h
   "execution_constants" test to pass.
   */
 -#define STACK_MIN_SIZE          12000   // Abort if less stack during eval.
-+#define STACK_MIN_SIZE          49152   // Abort if less stack during eval.
++#define STACK_MIN_SIZE          16384   // Abort if less stack during eval.
  
  #define STACK_MIN_SIZE_FOR_OPEN 1024*80
  #define STACK_BUFF_ALLOC        352     // For stack overrun checks

mysql.spec

@@ -1,6 +1,6 @@
 Name: mysql
 Version: 5.0.45
-Release: 8%{?dist}
+Release: 9%{?dist}
 Summary: MySQL client programs and shared libraries
 Group: Applications/Databases
 URL: http://www.mysql.com
@@ -32,6 +32,7 @@ Patch11: mysql-innodb-crash.patch
 Patch12: mysql-rename-bug.patch
 Patch13: mysql-view-bug.patch
 Patch14: mysql-ss-test.patch
+Patch15: mysql-stack-guard.patch
 
 BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-root
 BuildRequires: gperf, perl, readline-devel, openssl-devel
@@ -141,6 +142,7 @@ the MySQL sources.
 %patch12 -p1
 %patch13 -p1
 %patch14 -p1
+%patch15 -p1
 
 libtoolize --force
 aclocal
@@ -482,6 +484,13 @@ fi
 %{_mandir}/man1/mysql_client_test.1*
 
 %changelog
+* Thu Feb 28 2008 Tom Lane <tgl@redhat.com> 5.0.45-9
+- Fix the stack overflow problem encountered in January.  It seems the real
+issue is that the buildfarm machines were moved to RHEL5, which uses 64K not
+4K pages on PPC, and because RHEL5 takes the guard area out of the requested
+thread stack size we no longer had enough headroom.
+Related: #435337
+
 * Tue Feb 19 2008 Fedora Release Engineering <rel-eng@fedoraproject.org> - 5.0.45-8
 - Autorebuild for GCC 4.3