Make MySQL compile with openssl 3.x without FIPS properly implemented This change takes some pieces from MariaDB, including compat_ssl.h and changes in my_md5.cc. MySQL utilizes FIPS_mode() and FIPS_mode_set() functions that are not available in OpenSSL 3.x any more. This patch only mocks the call of those functions, returning 0 every time, which effectively makes usage of those functions non working. For making the MySQL build with OpenSSL 3.x this seems to be enough though. diff -rup mysql-8.0.22-orig/cmake/ssl.cmake mysql-8.0.22/cmake/ssl.cmake --- mysql-8.0.22-orig/cmake/ssl.cmake 2021-05-19 21:36:33.161996422 +0200 +++ mysql-8.0.22/cmake/ssl.cmake 2021-05-19 23:06:54.211877057 +0200 @@ -227,8 +227,7 @@ MACRO (MYSQL_CHECK_SSL) ENDIF() IF(OPENSSL_INCLUDE_DIR AND OPENSSL_LIBRARY AND - CRYPTO_LIBRARY AND - OPENSSL_MAJOR_VERSION STREQUAL "1" + CRYPTO_LIBRARY ) SET(OPENSSL_FOUND TRUE) FIND_PROGRAM(OPENSSL_EXECUTABLE openssl diff -rup mysql-8.0.22-orig/include/ssl_compat.h mysql-8.0.22/include/ssl_compat.h --- mysql-8.0.22-orig/include/ssl_compat.h 2021-05-19 23:19:36.152956356 +0200 +++ mysql-8.0.22/include/ssl_compat.h 2021-05-19 23:06:55.048885933 +0200 @@ -0,0 +1,105 @@ +/* + Copyright (c) 2016, 2021, MariaDB Corporation. + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ + +#include + +/* OpenSSL version specific definitions */ +#if defined(OPENSSL_VERSION_NUMBER) + +#if OPENSSL_VERSION_NUMBER >= 0x10100000L && !defined(LIBRESSL_VERSION_NUMBER) +#define HAVE_OPENSSL11 1 +#define SSL_LIBRARY OpenSSL_version(OPENSSL_VERSION) +#define ERR_remove_state(X) ERR_clear_error() +#define EVP_CIPHER_CTX_SIZE 176 +#define EVP_MD_CTX_SIZE 48 +#undef EVP_MD_CTX_init +#define EVP_MD_CTX_init(X) do { memset((X), 0, EVP_MD_CTX_SIZE); EVP_MD_CTX_reset(X); } while(0) +#undef EVP_CIPHER_CTX_init +#define EVP_CIPHER_CTX_init(X) do { memset((X), 0, EVP_CIPHER_CTX_SIZE); EVP_CIPHER_CTX_reset(X); } while(0) + +/* + Macros below are deprecated. OpenSSL 1.1 may define them or not, + depending on how it was built. +*/ +#undef ERR_free_strings +#define ERR_free_strings() +#undef EVP_cleanup +#define EVP_cleanup() +#undef CRYPTO_cleanup_all_ex_data +#define CRYPTO_cleanup_all_ex_data() +#undef SSL_load_error_strings +#define SSL_load_error_strings() + +#else +#define HAVE_OPENSSL10 1 +#ifdef HAVE_WOLFSSL +#define SSL_LIBRARY "WolfSSL " WOLFSSL_VERSION +#else +#define SSL_LIBRARY SSLeay_version(SSLEAY_VERSION) +#endif + +#ifdef HAVE_WOLFSSL +#undef ERR_remove_state +#define ERR_remove_state(x) do {} while(0) +#elif defined (HAVE_ERR_remove_thread_state) +#define ERR_remove_state(X) ERR_remove_thread_state(NULL) +#endif /* HAVE_ERR_remove_thread_state */ + +#endif /* HAVE_OPENSSL11 */ +#endif + +#ifdef HAVE_WOLFSSL +#define EVP_MD_CTX_SIZE sizeof(wc_Md5) +#endif + +#ifndef HAVE_OPENSSL11 +#ifndef ASN1_STRING_get0_data +#define ASN1_STRING_get0_data(X) ASN1_STRING_data(X) +#endif +#ifndef EVP_MD_CTX_SIZE +#define EVP_MD_CTX_SIZE sizeof(EVP_MD_CTX) +#endif + +#define DH_set0_pqg(D,P,Q,G) ((D)->p= (P), (D)->g= (G)) +#define EVP_CIPHER_CTX_buf_noconst(ctx) ((ctx)->buf) +#define EVP_CIPHER_CTX_encrypting(ctx) ((ctx)->encrypt) +#define EVP_CIPHER_CTX_SIZE sizeof(EVP_CIPHER_CTX) + +#ifndef HAVE_WOLFSSL +#define OPENSSL_init_ssl(X,Y) SSL_library_init() +#define EVP_MD_CTX_reset(X) EVP_MD_CTX_cleanup(X) +#define EVP_CIPHER_CTX_reset(X) EVP_CIPHER_CTX_cleanup(X) +#define X509_get0_notBefore(X) X509_get_notBefore(X) +#define X509_get0_notAfter(X) X509_get_notAfter(X) +#endif +#endif + +#ifndef TLS1_3_VERSION +//#define SSL_CTX_set_ciphersuites(X,Y) 0 +#endif + +#ifdef __cplusplus +extern "C" { +#endif /* __cplusplus */ + +int check_openssl_compatibility(); + +#define FIPS_mode_set(X) 0 +#define FIPS_mode() 0 + +#ifdef __cplusplus +} +#endif diff -rup mysql-8.0.22-orig/mysys/my_md5.cc mysql-8.0.22/mysys/my_md5.cc --- mysql-8.0.22-orig/mysys/my_md5.cc 2021-05-19 21:36:31.738980913 +0200 +++ mysql-8.0.22/mysys/my_md5.cc 2021-05-19 23:13:41.380194493 +0200 @@ -34,13 +34,12 @@ #include #include +#include +#include static void my_md5_hash(unsigned char *digest, unsigned const char *buf, int len) { - MD5_CTX ctx; - MD5_Init(&ctx); - MD5_Update(&ctx, buf, len); - MD5_Final(digest, &ctx); + MD5(buf, len, digest); } /** diff -rup mysql-8.0.22-orig/plugin/group_replication/libmysqlgcs/src/bindings/xcom/xcom/xcom_ssl_transport.cc mysql-8.0.22/plugin/group_replication/libmysqlgcs/src/bindings/xcom/xcom/xcom_ssl_transport.cc --- mysql-8.0.22-orig/plugin/group_replication/libmysqlgcs/src/bindings/xcom/xcom/xcom_ssl_transport.cc 2021-05-19 21:36:14.531793376 +0200 +++ mysql-8.0.22/plugin/group_replication/libmysqlgcs/src/bindings/xcom/xcom/xcom_ssl_transport.cc 2021-05-19 23:06:55.049885943 +0200 @@ -25,6 +25,7 @@ #include #include +#include #include #include #include @@ -35,6 +36,7 @@ #endif #include "openssl/engine.h" +#include #include "xcom/task_debug.h" #include "xcom/x_platform.h" diff -rup mysql-8.0.22-orig/plugin/x/client/xconnection_impl.cc mysql-8.0.22/plugin/x/client/xconnection_impl.cc --- mysql-8.0.22-orig/plugin/x/client/xconnection_impl.cc 2021-05-19 21:36:14.388791818 +0200 +++ mysql-8.0.22/plugin/x/client/xconnection_impl.cc 2021-05-19 23:06:55.049885943 +0200 @@ -31,6 +31,7 @@ #ifdef HAVE_NETINET_IN_H #include #endif // HAVE_NETINET_IN_H +#include #include #include #include // NOLINT(build/c++11) @@ -38,6 +39,7 @@ #include #include #include +#include #include "errmsg.h" // NOLINT(build/include_subdir) #include "my_config.h" // NOLINT(build/include_subdir) diff -rup mysql-8.0.22-orig/vio/viosslfactories.cc mysql-8.0.22/vio/viosslfactories.cc --- mysql-8.0.22-orig/vio/viosslfactories.cc 2021-05-19 21:36:33.310998046 +0200 +++ mysql-8.0.22/vio/viosslfactories.cc 2021-05-19 23:06:55.049885943 +0200 @@ -39,7 +39,9 @@ #include "mysys_err.h" #include "vio/vio_priv.h" +#include #include +#include #if OPENSSL_VERSION_NUMBER < 0x10002000L #include