dotnet8.0/runtime-104994-openssl-engines-optional.patch

From 5eb7fd1d9909c3824caba0bc6a3f376557961aef Mon Sep 17 00:00:00 2001
From: Kevin Jones <kevin@vcsjones.com>
Date: Tue, 16 Jul 2024 16:56:59 -0400
Subject: [PATCH 1/2] Light up support for OpenSSL ENGINEs only if they are
 available.

Some Linux distributions are phasing out support for OpenSSL 1.x ENGINEs. They are doing this by moving the header, `engine.h`, to a separate package or removing the header entirely. The actual OpenSSL shared libraries still contain the engine APIs. This makes the change an API, not ABI, break.

We react to this by disabling OpenSSL engine support on non-portable builds that are missing the engine header. Portable builds will continue to probe the loaded OpenSSL library for support, and non-portable builds will only support ENGINEs if the header is present.
---
 .../Interop.EvpPkey.cs                        | 22 +++++--
 .../src/Resources/Strings.resx                |  3 +
 .../configure.cmake                           |  6 ++
 .../opensslshim.h                             | 36 ++++++++--
 .../pal_crypto_config.h.in                    |  1 +
 .../pal_evp_pkey.c                            | 65 ++++++++++++++-----
 .../pal_evp_pkey.h                            |  6 +-
 7 files changed, 109 insertions(+), 30 deletions(-)

diff --git a/src/libraries/Common/src/Interop/Unix/System.Security.Cryptography.Native/Interop.EvpPkey.cs b/src/libraries/Common/src/Interop/Unix/System.Security.Cryptography.Native/Interop.EvpPkey.cs
index 77b80634bd0ce..4e8659b5653b7 100644
--- a/src/runtime/src/libraries/Common/src/Interop/Unix/System.Security.Cryptography.Native/Interop.EvpPkey.cs
+++ b/src/runtime/src/libraries/Common/src/Interop/Unix/System.Security.Cryptography.Native/Interop.EvpPkey.cs
@@ -217,7 +217,8 @@ internal static ArraySegment<byte> RentEncodeSubjectPublicKeyInfo(SafeEvpPKeyHan
         [LibraryImport(Libraries.CryptoNative, StringMarshalling = StringMarshalling.Utf8)]
         private static partial SafeEvpPKeyHandle CryptoNative_LoadPrivateKeyFromEngine(
             string engineName,
-            string keyName);
+            string keyName,
+            [MarshalAs(UnmanagedType.Bool)] out bool haveEngine);
 
         internal static SafeEvpPKeyHandle LoadPrivateKeyFromEngine(
             string engineName,
@@ -226,7 +227,13 @@ internal static ArraySegment<byte> RentEncodeSubjectPublicKeyInfo(SafeEvpPKeyHan
             Debug.Assert(engineName is not null);
             Debug.Assert(keyName is not null);
 
-            SafeEvpPKeyHandle pkey = CryptoNative_LoadPrivateKeyFromEngine(engineName, keyName);
+            SafeEvpPKeyHandle pkey = CryptoNative_LoadPrivateKeyFromEngine(engineName, keyName, out bool haveEngine);
+
+            if (!haveEngine)
+            {
+                pkey.Dispose();
+                throw new CryptographicException(SR.Cryptography_EnginesNotSupported);
+            }
 
             if (pkey.IsInvalid)
             {
@@ -240,7 +247,8 @@ internal static ArraySegment<byte> RentEncodeSubjectPublicKeyInfo(SafeEvpPKeyHan
         [LibraryImport(Libraries.CryptoNative, StringMarshalling = StringMarshalling.Utf8)]
         private static partial SafeEvpPKeyHandle CryptoNative_LoadPublicKeyFromEngine(
             string engineName,
-            string keyName);
+            string keyName,
+            [MarshalAs(UnmanagedType.Bool)] out bool haveEngine);
 
         internal static SafeEvpPKeyHandle LoadPublicKeyFromEngine(
             string engineName,
@@ -249,7 +257,13 @@ internal static ArraySegment<byte> RentEncodeSubjectPublicKeyInfo(SafeEvpPKeyHan
             Debug.Assert(engineName is not null);
             Debug.Assert(keyName is not null);
 
-            SafeEvpPKeyHandle pkey = CryptoNative_LoadPublicKeyFromEngine(engineName, keyName);
+            SafeEvpPKeyHandle pkey = CryptoNative_LoadPublicKeyFromEngine(engineName, keyName, out bool haveEngine);
+
+            if (!haveEngine)
+            {
+                pkey.Dispose();
+                throw new CryptographicException(SR.Cryptography_EnginesNotSupported);
+            }
 
             if (pkey.IsInvalid)
             {
diff --git a/src/libraries/System.Security.Cryptography/src/Resources/Strings.resx b/src/libraries/System.Security.Cryptography/src/Resources/Strings.resx
index 4bcfdcfd3454f..5cc5cce830a35 100644
--- a/src/runtime/src/libraries/System.Security.Cryptography/src/Resources/Strings.resx
+++ b/src/runtime/src/libraries/System.Security.Cryptography/src/Resources/Strings.resx
@@ -327,6 +327,9 @@
   <data name="Cryptography_EncryptedIncorrectLength" xml:space="preserve">
     <value>{0} unexpectedly produced a ciphertext with the incorrect length.</value>
   </data>
+  <data name="Cryptography_EnginesNotSupported" xml:space="preserve">
+    <value>OpenSSL ENGINE is not available on this platform.</value>
+  </data>
   <data name="Cryptography_ExceedKdfExtractLimit" xml:space="preserve">
     <value>The total number of bytes extracted cannot exceed UInt32.MaxValue * hash length.</value>
   </data>
diff --git a/src/native/libs/System.Security.Cryptography.Native/configure.cmake b/src/native/libs/System.Security.Cryptography.Native/configure.cmake
index 74ed49f5d1916..10a6b986268b0 100644
--- a/src/runtime/src/native/libs/System.Security.Cryptography.Native/configure.cmake
+++ b/src/runtime/src/native/libs/System.Security.Cryptography.Native/configure.cmake
@@ -1,5 +1,6 @@
 include(CheckLibraryExists)
 include(CheckFunctionExists)
+include(CheckSourceCompiles)
 
 set(CMAKE_REQUIRED_INCLUDES ${OPENSSL_INCLUDE_DIR})
 set(CMAKE_REQUIRED_LIBRARIES ${OPENSSL_CRYPTO_LIBRARY} ${OPENSSL_SSL_LIBRARY})
@@ -22,6 +23,11 @@ check_function_exists(
     HAVE_OPENSSL_SHA3
 )
 
+check_source_compiles(C "
+#include <openssl/engine.h>
+int main(void) { ENGINE_init(NULL); return 1; }"
+HAVE_OPENSSL_ENGINE)
+
 configure_file(
     ${CMAKE_CURRENT_SOURCE_DIR}/pal_crypto_config.h.in
     ${CMAKE_CURRENT_BINARY_DIR}/pal_crypto_config.h)
diff --git a/src/native/libs/System.Security.Cryptography.Native/opensslshim.h b/src/native/libs/System.Security.Cryptography.Native/opensslshim.h
index 57ba6a6809649..3d6d77895b64f 100644
--- a/src/runtime/src/native/libs/System.Security.Cryptography.Native/opensslshim.h
+++ b/src/runtime/src/native/libs/System.Security.Cryptography.Native/opensslshim.h
@@ -17,7 +17,6 @@
 #include <openssl/dsa.h>
 #include <openssl/ec.h>
 #include <openssl/ecdsa.h>
-#include <openssl/engine.h>
 #include <openssl/err.h>
 #include <openssl/evp.h>
 #include <openssl/hmac.h>
@@ -46,6 +45,11 @@
 #include <openssl/provider.h>
 #endif
 
+#if HAVE_OPENSSL_ENGINE
+// Some Linux distributions build without engine support.
+#include <openssl/engine.h>
+#endif
+
 #if OPENSSL_VERSION_NUMBER >= OPENSSL_VERSION_1_1_1_RTM
 #define HAVE_OPENSSL_SET_CIPHERSUITES 1
 #else
@@ -168,6 +172,24 @@ const EVP_MD *EVP_shake256(void);
 int EVP_DigestFinalXOF(EVP_MD_CTX *ctx, unsigned char *md, size_t len);
 #endif
 
+#if !HAVE_OPENSSL_ENGINE
+#undef HAVE_OPENSSL_ENGINE
+#define HAVE_OPENSSL_ENGINE 1
+
+ENGINE *ENGINE_by_id(const char *id);
+int ENGINE_init(ENGINE *e);
+int ENGINE_finish(ENGINE *e);
+ENGINE *ENGINE_new(void);
+int ENGINE_free(ENGINE *e);
+typedef EVP_PKEY *(*ENGINE_LOAD_KEY_PTR)(ENGINE *, const char *,
+                                         UI_METHOD *ui_method,
+                                         void *callback_data);
+EVP_PKEY *ENGINE_load_private_key(ENGINE *e, const char *key_id,
+                                  UI_METHOD *ui_method, void *callback_data);
+EVP_PKEY *ENGINE_load_public_key(ENGINE *e, const char *key_id,
+                                 UI_METHOD *ui_method, void *callback_data);
+#endif
+
 #define API_EXISTS(fn) (fn != NULL)
 
 // List of all functions from the libssl that are used in the System.Security.Cryptography.Native.
@@ -298,12 +320,12 @@ int EVP_DigestFinalXOF(EVP_MD_CTX *ctx, unsigned char *md, size_t len);
     REQUIRED_FUNCTION(EC_POINT_mul) \
     REQUIRED_FUNCTION(EC_POINT_new) \
     REQUIRED_FUNCTION(EC_POINT_set_affine_coordinates_GFp) \
-    REQUIRED_FUNCTION(ENGINE_by_id) \
-    REQUIRED_FUNCTION(ENGINE_finish) \
-    REQUIRED_FUNCTION(ENGINE_free) \
-    REQUIRED_FUNCTION(ENGINE_init) \
-    REQUIRED_FUNCTION(ENGINE_load_public_key) \
-    REQUIRED_FUNCTION(ENGINE_load_private_key) \
+    LIGHTUP_FUNCTION(ENGINE_by_id) \
+    LIGHTUP_FUNCTION(ENGINE_finish) \
+    LIGHTUP_FUNCTION(ENGINE_free) \
+    LIGHTUP_FUNCTION(ENGINE_init) \
+    LIGHTUP_FUNCTION(ENGINE_load_public_key) \
+    LIGHTUP_FUNCTION(ENGINE_load_private_key) \
     REQUIRED_FUNCTION(ERR_clear_error) \
     REQUIRED_FUNCTION(ERR_error_string_n) \
     REQUIRED_FUNCTION(ERR_get_error) \
diff --git a/src/native/libs/System.Security.Cryptography.Native/pal_crypto_config.h.in b/src/native/libs/System.Security.Cryptography.Native/pal_crypto_config.h.in
index d7aef5a7d1b67..30d1219eb98b0 100644
--- a/src/runtime/src/native/libs/System.Security.Cryptography.Native/pal_crypto_config.h.in
+++ b/src/runtime/src/native/libs/System.Security.Cryptography.Native/pal_crypto_config.h.in
@@ -4,3 +4,4 @@
 #cmakedefine01 HAVE_OPENSSL_ALPN
 #cmakedefine01 HAVE_OPENSSL_CHACHA20POLY1305
 #cmakedefine01 HAVE_OPENSSL_SHA3
+#cmakedefine01 HAVE_OPENSSL_ENGINE
diff --git a/src/native/libs/System.Security.Cryptography.Native/pal_evp_pkey.c b/src/native/libs/System.Security.Cryptography.Native/pal_evp_pkey.c
index 80183b97a77c9..dea4f277b8969 100644
--- a/src/runtime/src/native/libs/System.Security.Cryptography.Native/pal_evp_pkey.c
+++ b/src/runtime/src/native/libs/System.Security.Cryptography.Native/pal_evp_pkey.c
@@ -511,41 +511,72 @@ int32_t CryptoNative_EncodeSubjectPublicKeyInfo(EVP_PKEY* pkey, uint8_t* buf)
     return i2d_PUBKEY(pkey, &buf);
 }
 
+#if HAVE_OPENSSL_ENGINE
 static EVP_PKEY* LoadKeyFromEngine(
     const char* engineName,
     const char* keyName,
-    ENGINE_LOAD_KEY_PTR load_func)
+    ENGINE_LOAD_KEY_PTR load_func,
+    int32_t* haveEngine)
 {
+    assert(haveEngine);
     ERR_clear_error();
 
-    EVP_PKEY* ret = NULL;
-    ENGINE* engine = NULL;
+    if (API_EXISTS(ENGINE_by_id) && API_EXISTS(ENGINE_init) && API_EXISTS(ENGINE_finish) && API_EXISTS(ENGINE_free))
+    {
+        *haveEngine = 1;
+        EVP_PKEY* ret = NULL;
+        ENGINE* engine = NULL;
 
-    // Per https://github.com/openssl/openssl/discussions/21427
-    // using EVP_PKEY after freeing ENGINE is correct.
-    engine = ENGINE_by_id(engineName);
+        // Per https://github.com/openssl/openssl/discussions/21427
+        // using EVP_PKEY after freeing ENGINE is correct.
+        engine = ENGINE_by_id(engineName);
 
-    if (engine != NULL)
-    {
-        if (ENGINE_init(engine))
+        if (engine != NULL)
         {
-            ret = load_func(engine, keyName, NULL, NULL);
+            if (ENGINE_init(engine))
+            {
+                ret = load_func(engine, keyName, NULL, NULL);
+
+                ENGINE_finish(engine);
+            }
 
-            ENGINE_finish(engine);
+            ENGINE_free(engine);
         }
 
-        ENGINE_free(engine);
+        return ret;
     }
 
-    return ret;
+    *haveEngine = 0;
+    return NULL;
 }
+#endif
 
-EVP_PKEY* CryptoNative_LoadPrivateKeyFromEngine(const char* engineName, const char* keyName)
+EVP_PKEY* CryptoNative_LoadPrivateKeyFromEngine(const char* engineName, const char* keyName, int32_t* haveEngine)
 {
-    return LoadKeyFromEngine(engineName, keyName, ENGINE_load_private_key);
+#if HAVE_OPENSSL_ENGINE
+    if (API_EXISTS(ENGINE_load_private_key))
+    {
+        return LoadKeyFromEngine(engineName, keyName, ENGINE_load_private_key, haveEngine);
+    }
+#endif
+    (void)engineName;
+    (void)keyName;
+    (void)haveEngine;
+    *haveEngine = 0;
+    return NULL;
 }
 
-EVP_PKEY* CryptoNative_LoadPublicKeyFromEngine(const char* engineName, const char* keyName)
+EVP_PKEY* CryptoNative_LoadPublicKeyFromEngine(const char* engineName, const char* keyName, int32_t* haveEngine)
 {
-    return LoadKeyFromEngine(engineName, keyName, ENGINE_load_public_key);
+#if HAVE_OPENSSL_ENGINE
+    if (API_EXISTS(ENGINE_load_private_key))
+    {
+        return LoadKeyFromEngine(engineName, keyName, ENGINE_load_public_key, haveEngine);
+    }
+#endif
+    (void)engineName;
+    (void)keyName;
+    (void)haveEngine;
+    *haveEngine = 0;
+    return NULL;
 }
diff --git a/src/native/libs/System.Security.Cryptography.Native/pal_evp_pkey.h b/src/native/libs/System.Security.Cryptography.Native/pal_evp_pkey.h
index 64d289dc6f488..e4d5f85d4b9ec 100644
--- a/src/runtime/src/native/libs/System.Security.Cryptography.Native/pal_evp_pkey.h
+++ b/src/runtime/src/native/libs/System.Security.Cryptography.Native/pal_evp_pkey.h
@@ -93,12 +93,14 @@ PALEXPORT int32_t CryptoNative_EncodeSubjectPublicKeyInfo(EVP_PKEY* pkey, uint8_
 Load a named key, via ENGINE_load_private_key, from the named engine.
 
 Returns a valid EVP_PKEY* on success, NULL on failure.
+haveEngine is 1 if OpenSSL ENGINE's are supported, otherwise 0.
 */
-PALEXPORT EVP_PKEY* CryptoNative_LoadPrivateKeyFromEngine(const char* engineName, const char* keyName);
+PALEXPORT EVP_PKEY* CryptoNative_LoadPrivateKeyFromEngine(const char* engineName, const char* keyName, int32_t* haveEngine);
 
 /*
 Load a named key, via ENGINE_load_public_key, from the named engine.
 
 Returns a valid EVP_PKEY* on success, NULL on failure.
+haveEngine is 1 if OpenSSL ENGINE's are supported, otherwise 0.
 */
-PALEXPORT EVP_PKEY* CryptoNative_LoadPublicKeyFromEngine(const char* engineName, const char* keyName);
+PALEXPORT EVP_PKEY* CryptoNative_LoadPublicKeyFromEngine(const char* engineName, const char* keyName, int32_t* haveEngine);

From c81f7a8f5000151edfac9f9d891121a01a64f1ba Mon Sep 17 00:00:00 2001
From: Kevin Jones <kevin@vcsjones.com>
Date: Wed, 17 Jul 2024 07:47:53 -0400
Subject: [PATCH 2/2] Fix detection of ENGINE APIs with OpenSSL 3

---
 .../libs/System.Security.Cryptography.Native/configure.cmake     | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/native/libs/System.Security.Cryptography.Native/configure.cmake b/src/native/libs/System.Security.Cryptography.Native/configure.cmake
index 10a6b986268b0..4a70e70899c5e 100644
--- a/src/runtime/src/native/libs/System.Security.Cryptography.Native/configure.cmake
+++ b/src/runtime/src/native/libs/System.Security.Cryptography.Native/configure.cmake
@@ -4,6 +4,7 @@ include(CheckSourceCompiles)
 
 set(CMAKE_REQUIRED_INCLUDES ${OPENSSL_INCLUDE_DIR})
 set(CMAKE_REQUIRED_LIBRARIES ${OPENSSL_CRYPTO_LIBRARY} ${OPENSSL_SSL_LIBRARY})
+set(CMAKE_REQUIRED_DEFINITIONS -DOPENSSL_API_COMPAT=0x10100000L)
 
 check_function_exists(
     EC_GF2m_simple_method