mysql8.4/community-mysql-openssl3.patch

Make MySQL compile with openssl 3.x without FIPS properly implemented

This change takes some pieces from MariaDB, including compat_ssl.h and
changes in my_md5.cc.

MySQL utilizes FIPS_mode() and FIPS_mode_set() functions that are not
available in OpenSSL 3.x any more. This patch only mocks the call of
those functions, returning 0 every time, which effectively makes usage
of those functions non working. For making the MySQL build with
OpenSSL 3.x this seems to be enough though.

diff -rup mysql-8.0.22-orig/cmake/ssl.cmake mysql-8.0.22/cmake/ssl.cmake
--- mysql-8.0.22-orig/cmake/ssl.cmake	2021-05-19 21:36:33.161996422 +0200
+++ mysql-8.0.22/cmake/ssl.cmake	2021-05-19 23:06:54.211877057 +0200
@@ -227,8 +227,7 @@ MACRO (MYSQL_CHECK_SSL)
     ENDIF()
     IF(OPENSSL_INCLUDE_DIR AND
        OPENSSL_LIBRARY   AND
-       CRYPTO_LIBRARY      AND
-       OPENSSL_MAJOR_VERSION STREQUAL "1"
+       CRYPTO_LIBRARY
       )
       SET(OPENSSL_FOUND TRUE)
       FIND_PROGRAM(OPENSSL_EXECUTABLE openssl
diff -rup mysql-8.0.22-orig/include/ssl_compat.h mysql-8.0.22/include/ssl_compat.h
--- mysql-8.0.22-orig/include/ssl_compat.h	2021-05-19 23:19:36.152956356 +0200
+++ mysql-8.0.22/include/ssl_compat.h	2021-05-19 23:06:55.048885933 +0200
@@ -0,0 +1,105 @@
+/*
+ Copyright (c) 2016, 2021, MariaDB Corporation.
+
+ This program is free software; you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation; version 2 of the License.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with this program; if not, write to the Free Software
+ Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301  USA */
+
+#include <openssl/opensslv.h>
+
+/* OpenSSL version specific definitions */
+#if defined(OPENSSL_VERSION_NUMBER)
+
+#if OPENSSL_VERSION_NUMBER >= 0x10100000L && !defined(LIBRESSL_VERSION_NUMBER)
+#define HAVE_OPENSSL11 1
+#define SSL_LIBRARY OpenSSL_version(OPENSSL_VERSION)
+#define ERR_remove_state(X) ERR_clear_error()
+#define EVP_CIPHER_CTX_SIZE 176
+#define EVP_MD_CTX_SIZE 48
+#undef EVP_MD_CTX_init
+#define EVP_MD_CTX_init(X) do { memset((X), 0, EVP_MD_CTX_SIZE); EVP_MD_CTX_reset(X); } while(0)
+#undef EVP_CIPHER_CTX_init
+#define EVP_CIPHER_CTX_init(X) do { memset((X), 0, EVP_CIPHER_CTX_SIZE); EVP_CIPHER_CTX_reset(X); } while(0)
+
+/*
+  Macros below are deprecated. OpenSSL 1.1 may define them or not,
+  depending on how it was built.
+*/
+#undef ERR_free_strings
+#define ERR_free_strings()
+#undef EVP_cleanup
+#define EVP_cleanup()
+#undef CRYPTO_cleanup_all_ex_data
+#define CRYPTO_cleanup_all_ex_data()
+#undef SSL_load_error_strings
+#define SSL_load_error_strings()
+
+#else
+#define HAVE_OPENSSL10 1
+#ifdef HAVE_WOLFSSL
+#define SSL_LIBRARY "WolfSSL " WOLFSSL_VERSION
+#else
+#define SSL_LIBRARY SSLeay_version(SSLEAY_VERSION)
+#endif
+
+#ifdef HAVE_WOLFSSL
+#undef ERR_remove_state
+#define ERR_remove_state(x) do {} while(0)
+#elif defined (HAVE_ERR_remove_thread_state)
+#define ERR_remove_state(X) ERR_remove_thread_state(NULL)
+#endif /* HAVE_ERR_remove_thread_state */
+
+#endif /* HAVE_OPENSSL11 */
+#endif
+
+#ifdef HAVE_WOLFSSL
+#define EVP_MD_CTX_SIZE                 sizeof(wc_Md5)
+#endif
+
+#ifndef HAVE_OPENSSL11
+#ifndef ASN1_STRING_get0_data
+#define ASN1_STRING_get0_data(X)        ASN1_STRING_data(X)
+#endif
+#ifndef EVP_MD_CTX_SIZE
+#define EVP_MD_CTX_SIZE                 sizeof(EVP_MD_CTX)
+#endif
+
+#define DH_set0_pqg(D,P,Q,G)            ((D)->p= (P), (D)->g= (G))
+#define EVP_CIPHER_CTX_buf_noconst(ctx) ((ctx)->buf)
+#define EVP_CIPHER_CTX_encrypting(ctx)  ((ctx)->encrypt)
+#define EVP_CIPHER_CTX_SIZE             sizeof(EVP_CIPHER_CTX)
+
+#ifndef HAVE_WOLFSSL
+#define OPENSSL_init_ssl(X,Y)           SSL_library_init()
+#define EVP_MD_CTX_reset(X) EVP_MD_CTX_cleanup(X)
+#define EVP_CIPHER_CTX_reset(X) EVP_CIPHER_CTX_cleanup(X)
+#define X509_get0_notBefore(X) X509_get_notBefore(X)
+#define X509_get0_notAfter(X) X509_get_notAfter(X)
+#endif
+#endif
+
+#ifndef TLS1_3_VERSION
+//#define SSL_CTX_set_ciphersuites(X,Y) 0
+#endif
+
+#ifdef	__cplusplus
+extern "C" {
+#endif /* __cplusplus */
+
+int check_openssl_compatibility();
+
+#define FIPS_mode_set(X) 0
+#define FIPS_mode() 0
+
+#ifdef	__cplusplus
+}
+#endif
diff -rup mysql-8.0.22-orig/mysys/my_md5.cc mysql-8.0.22/mysys/my_md5.cc
--- mysql-8.0.22-orig/mysys/my_md5.cc	2021-05-19 21:36:31.738980913 +0200
+++ mysql-8.0.22/mysys/my_md5.cc	2021-05-19 23:13:41.380194493 +0200
@@ -34,13 +34,12 @@
 
 #include <openssl/crypto.h>
 #include <openssl/md5.h>
+#include <openssl/evp.h>
+#include <ssl_compat.h>
 
 static void my_md5_hash(unsigned char *digest, unsigned const char *buf,
                         int len) {
-  MD5_CTX ctx;
-  MD5_Init(&ctx);
-  MD5_Update(&ctx, buf, len);
-  MD5_Final(digest, &ctx);
+  MD5(buf, len, digest);
 }
 
 /**
diff -rup mysql-8.0.22-orig/plugin/group_replication/libmysqlgcs/src/bindings/xcom/xcom/xcom_ssl_transport.cc mysql-8.0.22/plugin/group_replication/libmysqlgcs/src/bindings/xcom/xcom/xcom_ssl_transport.cc
--- mysql-8.0.22-orig/plugin/group_replication/libmysqlgcs/src/bindings/xcom/xcom/xcom_ssl_transport.cc	2021-05-19 21:36:14.531793376 +0200
+++ mysql-8.0.22/plugin/group_replication/libmysqlgcs/src/bindings/xcom/xcom/xcom_ssl_transport.cc	2021-05-19 23:06:55.049885943 +0200
@@ -25,6 +25,7 @@
 #include <assert.h>
 #include <stdlib.h>
 
+#include <openssl/crypto.h>
 #include <openssl/dh.h>
 #include <openssl/opensslv.h>
 #include <openssl/x509v3.h>
@@ -35,6 +36,7 @@
 #endif
 
 #include "openssl/engine.h"
+#include <ssl_compat.h>
 
 #include "xcom/task_debug.h"
 #include "xcom/x_platform.h"
diff -rup mysql-8.0.22-orig/plugin/x/client/xconnection_impl.cc mysql-8.0.22/plugin/x/client/xconnection_impl.cc
--- mysql-8.0.22-orig/plugin/x/client/xconnection_impl.cc	2021-05-19 21:36:14.388791818 +0200
+++ mysql-8.0.22/plugin/x/client/xconnection_impl.cc	2021-05-19 23:06:55.049885943 +0200
@@ -31,6 +31,7 @@
 #ifdef HAVE_NETINET_IN_H
 #include <netinet/in.h>
 #endif  // HAVE_NETINET_IN_H
+#include <openssl/crypto.h>
 #include <openssl/x509v3.h>
 #include <cassert>
 #include <chrono>  // NOLINT(build/c++11)
@@ -38,6 +39,7 @@
 #include <limits>
 #include <sstream>
 #include <string>
+#include <ssl_compat.h>
 
 #include "errmsg.h"       // NOLINT(build/include_subdir)
 #include "my_config.h"    // NOLINT(build/include_subdir)
diff -rup mysql-8.0.22-orig/vio/viosslfactories.cc mysql-8.0.22/vio/viosslfactories.cc
--- mysql-8.0.22-orig/vio/viosslfactories.cc	2021-05-19 21:36:33.310998046 +0200
+++ mysql-8.0.22/vio/viosslfactories.cc	2021-05-19 23:06:55.049885943 +0200
@@ -39,7 +39,9 @@
 #include "mysys_err.h"
 #include "vio/vio_priv.h"
 
+#include <openssl/crypto.h>
 #include <openssl/dh.h>
+#include <ssl_compat.h>
 
 #if OPENSSL_VERSION_NUMBER < 0x10002000L
 #include <openssl/ec.h>