Fix OpenSSL 3.x compatibility

Resolves: #1962047

mariadb-openssl3.patch

@@ -0,0 +1,163 @@
+diff -rup mariadb-10.5.9-orig/mysql-test/main/tls_version1.opt mariadb-10.5.9/mysql-test/main/tls_version1.opt
+--- mariadb-10.5.9-orig/mysql-test/main/tls_version1.opt	2021-05-19 18:52:49.627469097 +0200
++++ mariadb-10.5.9/mysql-test/main/tls_version1.opt	2021-05-21 22:34:44.131913619 +0200
+@@ -1 +1 @@
+---tls_version=TLSv1.0
++--tls_version=TLSv1.2
+diff -rup mariadb-10.5.9-orig/mysql-test/main/tls_version1.result mariadb-10.5.9/mysql-test/main/tls_version1.result
+--- mariadb-10.5.9-orig/mysql-test/main/tls_version1.result	2021-05-19 18:52:49.592468722 +0200
++++ mariadb-10.5.9/mysql-test/main/tls_version1.result	2021-05-21 22:34:44.131913619 +0200
+@@ -1,6 +1,6 @@
+ Variable_name	Value
+-Ssl_version	TLSv1
++Ssl_version	TLSv1.2
+ Variable_name	Value
+-Ssl_version	TLSv1
++Ssl_version	TLSv1.2
+ @@tls_version
+-TLSv1.0
++TLSv1.2
+diff -rup mariadb-10.5.9-orig/mysql-test/main/tls_version1.test mariadb-10.5.9/mysql-test/main/tls_version1.test
+--- mariadb-10.5.9-orig/mysql-test/main/tls_version1.test	2021-05-19 18:52:49.577468561 +0200
++++ mariadb-10.5.9/mysql-test/main/tls_version1.test	2021-05-21 22:34:44.131913619 +0200
+@@ -3,10 +3,10 @@
+ 
+ -- source include/have_ssl_communication.inc
+ --exec $MYSQL --host=localhost --ssl -e "show status like 'ssl_version';"
+---error 1
+ --exec $MYSQL --host=localhost --ssl --tls_version=TLSv1.2 -e "show status like 'ssl_version';"
+ --error 1
+ --exec $MYSQL --host=localhost --ssl --tls_version=TLSv1.1 -e "show status like 'ssl_version';"
++--error 1
+ --exec $MYSQL --host=localhost --ssl --tls_version=TLSv1.0 -e "show status like 'ssl_version';"
+ --exec $MYSQL --host=localhost --ssl -e "select @@tls_version;"
+ 
+diff -rup mariadb-10.5.9-orig/mysys_ssl/my_crypt.cc mariadb-10.5.9/mysys_ssl/my_crypt.cc
+--- mariadb-10.5.9-orig/mysys_ssl/my_crypt.cc	2021-05-19 18:52:49.167464162 +0200
++++ mariadb-10.5.9/mysys_ssl/my_crypt.cc	2021-05-21 22:34:44.132913630 +0200
+@@ -38,22 +38,14 @@
+ class MyCTX
+ {
+ public:
+-  char ctx_buf[EVP_CIPHER_CTX_SIZE + CTX_ALIGN];
+-  EVP_CIPHER_CTX* ctx;
++  EVP_CIPHER_CTX* ctx= NULL;
+   MyCTX()
+   {
+-#if CTX_ALIGN > 0
+-    uintptr_t p= ((uintptr_t)ctx_buf + (CTX_ALIGN - 1)) & ~(CTX_ALIGN - 1);
+-    ctx = reinterpret_cast<EVP_CIPHER_CTX*>(p);
+-#else
+-    ctx = (EVP_CIPHER_CTX*)ctx_buf;
+-#endif
+-
+-    EVP_CIPHER_CTX_init(ctx);
++    ctx = EVP_CIPHER_CTX_new();
+   }
+   virtual ~MyCTX()
+   {
+-    EVP_CIPHER_CTX_reset(ctx);
++    EVP_CIPHER_CTX_free(ctx);
+     ERR_remove_state(0);
+   }
+ 
+diff -rup mariadb-10.5.9-orig/mysys_ssl/my_md5.cc mariadb-10.5.9/mysys_ssl/my_md5.cc
+--- mariadb-10.5.9-orig/mysys_ssl/my_md5.cc	2021-05-19 18:52:49.167464162 +0200
++++ mariadb-10.5.9/mysys_ssl/my_md5.cc	2021-05-24 15:25:11.365769072 +0200
+@@ -52,12 +52,13 @@ static void md5_result(EVP_MD_CTX *conte
+ 
+ static void md5_init(EVP_MD_CTX *context)
+ {
+-  EVP_MD_CTX_init(context);
++  const EVP_MD *md;
+ #ifdef EVP_MD_CTX_FLAG_NON_FIPS_ALLOW
+   /* Ok to ignore FIPS: MD5 is not used for crypto here */
+   EVP_MD_CTX_set_flags(context, EVP_MD_CTX_FLAG_NON_FIPS_ALLOW);
+ #endif
+-  EVP_DigestInit_ex(context, EVP_md5(), NULL);
++  md = EVP_get_digestbyname("MD5");
++  EVP_DigestInit_ex(context, md, NULL);
+ }
+ 
+ static void md5_input(EVP_MD_CTX *context, const uchar *buf, unsigned len)
+@@ -68,7 +69,6 @@ static void md5_input(EVP_MD_CTX *contex
+ static void md5_result(EVP_MD_CTX *context, uchar digest[MD5_HASH_SIZE])
+ {
+   EVP_DigestFinal_ex(context, digest, NULL);
+-  EVP_MD_CTX_reset(context);
+ }
+ 
+ #endif /* HAVE_WOLFSSL */
+@@ -84,11 +84,13 @@ static void md5_result(EVP_MD_CTX *conte
+ */
+ void my_md5(uchar *digest, const char *buf, size_t len)
+ {
+-  char ctx_buf[EVP_MD_CTX_SIZE];
+-  EVP_MD_CTX * const ctx= (EVP_MD_CTX*)ctx_buf;
++  EVP_MD_CTX * const ctx= EVP_MD_CTX_new();
++
+   md5_init(ctx);
+   md5_input(ctx, (const uchar *)buf, (uint) len);
+   md5_result(ctx, digest);
++
++  EVP_MD_CTX_free(ctx);
+ }
+ 
+ 
+@@ -108,8 +110,7 @@ void my_md5_multi(uchar *digest, ...)
+ {
+   va_list args;
+   const uchar *str;
+-  char ctx_buf[EVP_MD_CTX_SIZE];
+-  EVP_MD_CTX * const ctx= (EVP_MD_CTX*)ctx_buf;
++  EVP_MD_CTX * const ctx= EVP_MD_CTX_new();
+   va_start(args, digest);
+ 
+   md5_init(ctx);
+@@ -118,6 +119,7 @@ void my_md5_multi(uchar *digest, ...)
+ 
+   md5_result(ctx, digest);
+   va_end(args);
++  EVP_MD_CTX_free(ctx);
+ }
+ 
+ size_t my_md5_context_size()
+Only in mariadb-10.5.9-orig/mysys_ssl: my_md5.cc.patchmd5
+diff -rup mariadb-10.5.9-orig/mysys_ssl/my_sha.ic mariadb-10.5.9/mysys_ssl/my_sha.ic
+--- mariadb-10.5.9-orig/mysys_ssl/my_sha.ic	2021-05-19 18:52:49.167464162 +0200
++++ mariadb-10.5.9/mysys_ssl/my_sha.ic	2021-05-21 22:34:44.132913630 +0200
+@@ -146,11 +146,11 @@ static void sha_result(CONTEXT *context,
+ */
+ void my_sha(uchar *digest, const char *buf, size_t len)
+ {
+-  CONTEXT context;
++  CONTEXT *context= (CONTEXT *)alloca(sizeof(CONTEXT));
+ 
+-  sha_init_fast(&context);
+-  sha_input(&context, (const uchar *)buf, (unsigned int)len);
+-  sha_result(&context, digest);
++  sha_init_fast(context);
++  sha_input(context, (const uchar *)buf, (unsigned int)len);
++  sha_result(context, digest);
+ }
+ 
+ 
+@@ -171,14 +171,14 @@ void my_sha_multi(uchar *digest, ...)
+   va_list args;
+   va_start(args, digest);
+ 
+-  CONTEXT context;
++  CONTEXT *context= (CONTEXT *)alloca(sizeof(CONTEXT));
+   const uchar *str;
+ 
+-  sha_init_fast(&context);
++  sha_init_fast(context);
+   for (str= va_arg(args, const uchar*); str; str= va_arg(args, const uchar*))
+-    sha_input(&context, str, (uint) va_arg(args, size_t));
++    sha_input(context, str, (uint) va_arg(args, size_t));
+ 
+-  sha_result(&context, digest);
++  sha_result(context, digest);
+   va_end(args);
+ }
+ 

mariadb.spec

@@ -161,7 +161,7 @@
 
 Name:             mariadb
 Version:          10.5.9
-Release:          8%{?with_debug:.debug}%{?dist}
+Release:          9%{?with_debug:.debug}%{?dist}
 Epoch:            3
 
 Summary:          A very fast and robust SQL database server
@@ -221,6 +221,8 @@ Patch10:          %{pkgnamepatch}-ssl-cipher-tests.patch
 Patch11:          %{pkgnamepatch}-pcdir.patch
 #   Patch15:  Add option to edit groonga's and groonga-normalizer-mysql install path
 Patch15:          %{pkgnamepatch}-groonga.patch
+#   Patch16:  Fix openssl 3.0 compatibility at least partially (some tests still fail)
+Patch16:          %{pkgnamepatch}-openssl3.patch
 
 BuildRequires:    make
 BuildRequires:    cmake gcc-c++
@@ -766,6 +768,7 @@ rm -r storage/rocksdb/
 %patch10 -p1
 %patch11 -p1
 %patch15 -p1
+%patch16 -p1
 
 # generate a list of tests that fail, but are not disabled by upstream
 cat %{SOURCE50} | tee -a mysql-test/unstable-tests
@@ -1680,6 +1683,10 @@ fi
 %endif
 
 %changelog
+* Fri May 21 2021 Honza Horak <hhorak@redhat.com> - 3:10.5.9-9
+- Fix OpenSSL 3.x compatibility
+  Resolves: #1962047
+
 * Mon May 03 2021 Michal Schorm <mschorm@redhat.com> - 3:10.5.9-8
 - Fix package Conflicts