dotnet3.1/corefx-openssl-0001-Use-EVP_PKEY-for-RSA-key-generation.patch

From 2b6b45878b1be4d77eec34ab5bc80b626995a8c5 Mon Sep 17 00:00:00 2001
From: Jeremy Barton <jbarton@microsoft.com>
Date: Fri, 19 Mar 2021 15:05:41 -0700
Subject: [PATCH 01/11] Use EVP_PKEY for RSA key generation

---
 .../Interop.EvpPkey.Rsa.cs                    | 16 ++++++++
 .../Interop.Rsa.cs                            |  3 --
 .../Security/Cryptography/RSAOpenSsl.cs       | 37 ++++---------------
 .../apibridge.c                               |  8 ++++
 .../apibridge.h                               |  1 +
 .../opensslshim.h                             | 23 ++++++++++++
 .../pal_evp_pkey_rsa.c                        | 29 +++++++++++++++
 .../pal_evp_pkey_rsa.h                        |  5 +++
 .../pal_rsa.c                                 |  5 ---
 .../pal_rsa.h                                 |  7 ----
 ...em.Security.Cryptography.Algorithms.csproj |  3 ++
 11 files changed, 93 insertions(+), 44 deletions(-)

diff --git a/src/Common/src/Interop/Unix/System.Security.Cryptography.Native/Interop.EvpPkey.Rsa.cs b/src/Common/src/Interop/Unix/System.Security.Cryptography.Native/Interop.EvpPkey.Rsa.cs
index 1f61a826a9..c28522784b 100644
--- a/src/Common/src/Interop/Unix/System.Security.Cryptography.Native/Interop.EvpPkey.Rsa.cs
+++ b/src/Common/src/Interop/Unix/System.Security.Cryptography.Native/Interop.EvpPkey.Rsa.cs
@@ -10,6 +10,22 @@ internal static partial class Interop
 {
     internal static partial class Crypto
     {
+        [DllImport(Libraries.CryptoNative, EntryPoint = "CryptoNative_RsaGenerateKey")]
+        private static extern SafeEvpPKeyHandle CryptoNative_RsaGenerateKey(int keySize);
+
+        internal static SafeEvpPKeyHandle RsaGenerateKey(int keySize)
+        {
+            SafeEvpPKeyHandle pkey = CryptoNative_RsaGenerateKey(keySize);
+
+            if (pkey.IsInvalid)
+            {
+                pkey.Dispose();
+                throw CreateOpenSslCryptographicException();
+            }
+
+            return pkey;
+        }
+
         [DllImport(Libraries.CryptoNative, EntryPoint = "CryptoNative_EvpPkeyGetRsa")]
         internal static extern SafeRsaHandle EvpPkeyGetRsa(SafeEvpPKeyHandle pkey);
 
diff --git a/src/Common/src/Interop/Unix/System.Security.Cryptography.Native/Interop.Rsa.cs b/src/Common/src/Interop/Unix/System.Security.Cryptography.Native/Interop.Rsa.cs
index a7b85ae011..a05f020ada 100644
--- a/src/Common/src/Interop/Unix/System.Security.Cryptography.Native/Interop.Rsa.cs
+++ b/src/Common/src/Interop/Unix/System.Security.Cryptography.Native/Interop.Rsa.cs
@@ -89,9 +89,6 @@ internal static partial class Crypto
         [DllImport(Libraries.CryptoNative, EntryPoint = "CryptoNative_RsaSize")]
         internal static extern int RsaSize(SafeRsaHandle rsa);
 
-        [DllImport(Libraries.CryptoNative, EntryPoint = "CryptoNative_RsaGenerateKeyEx")]
-        internal static extern int RsaGenerateKeyEx(SafeRsaHandle rsa, int bits, SafeBignumHandle e);
-
         internal static bool RsaSign(int type, ReadOnlySpan<byte> m, int m_len, Span<byte> sigret, out int siglen, SafeRsaHandle rsa) =>
             RsaSign(type, ref MemoryMarshal.GetReference(m), m_len, ref MemoryMarshal.GetReference(sigret), out siglen, rsa);
 
diff --git a/src/Common/src/System/Security/Cryptography/RSAOpenSsl.cs b/src/Common/src/System/Security/Cryptography/RSAOpenSsl.cs
index 6741d28bba..4d4a8414b3 100644
--- a/src/Common/src/System/Security/Cryptography/RSAOpenSsl.cs
+++ b/src/Common/src/System/Security/Cryptography/RSAOpenSsl.cs
@@ -25,9 +25,6 @@ public sealed partial class RSAOpenSsl : RSA
     {
         private const int BitsPerByte = 8;
 
-        // 65537 (0x10001) in big-endian form
-        private static readonly byte[] s_defaultExponent = { 0x01, 0x00, 0x01 };
-
         private Lazy<SafeRsaHandle> _key;
 
         public RSAOpenSsl()
@@ -585,36 +582,18 @@ private static void CheckBoolReturn(int returnValue)
 
         private SafeRsaHandle GenerateKey()
         {
-            SafeRsaHandle key = Interop.Crypto.RsaCreate();
-            bool generated = false;
-
-            Interop.Crypto.CheckValidOpenSslHandle(key);
-
-            try
+            using (SafeEvpPKeyHandle pkey = Interop.Crypto.RsaGenerateKey(KeySize))
             {
-                using (SafeBignumHandle exponent = Interop.Crypto.CreateBignum(s_defaultExponent))
-                {
-                    // The documentation for RSA_generate_key_ex does not say that it returns only
-                    // 0 or 1, so the call marshals it back as a full Int32 and checks for a value
-                    // of 1 explicitly.
-                    int response = Interop.Crypto.RsaGenerateKeyEx(
-                        key,
-                        KeySize,
-                        exponent);
-
-                    CheckBoolReturn(response);
-                    generated = true;
-                }
-            }
-            finally
-            {
-                if (!generated)
+                SafeRsaHandle rsa = Interop.Crypto.EvpPkeyGetRsa(pkey);
+
+                if (rsa.IsInvalid)
                 {
-                    key.Dispose();
+                    rsa.Dispose();
+                    throw Interop.Crypto.CreateOpenSslCryptographicException();
                 }
-            }
 
-            return key;
+                return rsa;
+            }
         }
 
         protected override byte[] HashData(byte[] data, int offset, int count, HashAlgorithmName hashAlgorithm) =>
diff --git a/src/Native/Unix/System.Security.Cryptography.Native/apibridge.c b/src/Native/Unix/System.Security.Cryptography.Native/apibridge.c
index 167de7fd8e..def7198deb 100644
--- a/src/Native/Unix/System.Security.Cryptography.Native/apibridge.c
+++ b/src/Native/Unix/System.Security.Cryptography.Native/apibridge.c
@@ -728,4 +728,12 @@ void local_SSL_CTX_set_security_level(SSL_CTX* ctx, int32_t level)
     (void)ctx;
     (void)level;
 }
+
+int32_t local_RSA_pkey_ctx_ctrl(EVP_PKEY_CTX* ctx, int32_t optype, int32_t cmd, int32_t p1, void* p2)
+{
+    // On OpenSSL 1.0.2 there aren't two different identifiers for RSA,
+    // so just pass the request on th EVP_PKEY_CTX_ctrl with the only identifier defined.
+    return EVP_PKEY_CTX_ctrl(ctx, EVP_PKEY_RSA, optype, cmd, p1, p2);
+}
+
 #endif
diff --git a/src/Native/Unix/System.Security.Cryptography.Native/apibridge.h b/src/Native/Unix/System.Security.Cryptography.Native/apibridge.h
index 5f62864b24..b58611ae73 100644
--- a/src/Native/Unix/System.Security.Cryptography.Native/apibridge.h
+++ b/src/Native/Unix/System.Security.Cryptography.Native/apibridge.h
@@ -26,6 +26,7 @@ int32_t local_RSA_meth_get_flags(const RSA_METHOD* meth);
 int32_t local_RSA_set0_crt_params(RSA* rsa, BIGNUM* dmp1, BIGNUM* dmq1, BIGNUM* iqmp);
 int32_t local_RSA_set0_factors(RSA* rsa, BIGNUM* p, BIGNUM* q);
 int32_t local_RSA_set0_key(RSA* rsa, BIGNUM* n, BIGNUM* e, BIGNUM* d);
+int32_t local_RSA_pkey_ctx_ctrl(EVP_PKEY_CTX* ctx, int32_t optype, int32_t cmd, int32_t p1, void* p2);
 int32_t local_SSL_is_init_finished(const SSL* ssl);
 unsigned long local_SSL_CTX_set_options(SSL_CTX* ctx, unsigned long options);
 void local_SSL_CTX_set_security_level(SSL_CTX* ctx, int32_t level);
diff --git a/src/Native/Unix/System.Security.Cryptography.Native/opensslshim.h b/src/Native/Unix/System.Security.Cryptography.Native/opensslshim.h
index ed3994926d..dff6091e9e 100644
--- a/src/Native/Unix/System.Security.Cryptography.Native/opensslshim.h
+++ b/src/Native/Unix/System.Security.Cryptography.Native/opensslshim.h
@@ -145,6 +145,7 @@ void RSA_get0_factors(const RSA* rsa, const BIGNUM** p, const BIGNUM** q);
 void RSA_get0_key(const RSA* rsa, const BIGNUM** n, const BIGNUM** e, const BIGNUM** d);
 int32_t RSA_meth_get_flags(const RSA_METHOD* meth);
 const RSA_METHOD* RSA_PKCS1_OpenSSL(void);
+int32_t RSA_pkey_ctx_ctrl(EVP_PKEY_CTX* ctx, int32_t optype, int32_t cmd, int32_t p1, void* p2);
 int32_t RSA_set0_crt_params(RSA* rsa, BIGNUM* dmp1, BIGNUM* dmq1, BIGNUM* iqmp);
 int32_t RSA_set0_factors(RSA* rsa, BIGNUM* p, BIGNUM* q);
 int32_t RSA_set0_key(RSA* rsa, BIGNUM* n, BIGNUM* e, BIGNUM* d);
@@ -170,6 +171,13 @@ const X509_ALGOR* X509_get0_tbs_sigalg(const X509* x509);
 X509_PUBKEY* X509_get_X509_PUBKEY(const X509* x509);
 int32_t X509_get_version(const X509* x509);
 int32_t X509_up_ref(X509* x509);
+
+// Redefine EVP_PKEY_CTX_set_rsa operations to use (local_)RSA_pkey_ctx_ctrl so the path is the same
+// for 1.0-built on 1.1 as on 1.1-built on 1.1.
+#undef EVP_PKEY_CTX_set_rsa_keygen_bits
+#define EVP_PKEY_CTX_set_rsa_keygen_bits(ctx, bits) \
+    RSA_pkey_ctx_ctrl(ctx, EVP_PKEY_OP_KEYGEN, EVP_PKEY_CTRL_RSA_KEYGEN_BITS, bits, NULL)
+
 #endif
 
 #if OPENSSL_VERSION_NUMBER < OPENSSL_VERSION_1_0_2_RTM
@@ -341,8 +349,12 @@ void SSL_get0_alpn_selected(const SSL* ssl, const unsigned char** protocol, unsi
     RENAMED_FUNCTION(EVP_MD_CTX_free, EVP_MD_CTX_destroy) \
     RENAMED_FUNCTION(EVP_MD_CTX_new, EVP_MD_CTX_create) \
     REQUIRED_FUNCTION(EVP_MD_size) \
+    REQUIRED_FUNCTION(EVP_PKEY_CTX_ctrl) \
     REQUIRED_FUNCTION(EVP_PKEY_CTX_free) \
+    REQUIRED_FUNCTION(EVP_PKEY_CTX_get0_pkey) \
     REQUIRED_FUNCTION(EVP_PKEY_CTX_new) \
+    REQUIRED_FUNCTION(EVP_PKEY_CTX_new_id) \
+    REQUIRED_FUNCTION(EVP_PKEY_base_id) \
     REQUIRED_FUNCTION(EVP_PKEY_derive_set_peer) \
     REQUIRED_FUNCTION(EVP_PKEY_derive_init) \
     REQUIRED_FUNCTION(EVP_PKEY_derive) \
@@ -350,6 +362,8 @@ void SSL_get0_alpn_selected(const SSL* ssl, const unsigned char** protocol, unsi
     REQUIRED_FUNCTION(EVP_PKEY_get1_DSA) \
     REQUIRED_FUNCTION(EVP_PKEY_get1_EC_KEY) \
     REQUIRED_FUNCTION(EVP_PKEY_get1_RSA) \
+    REQUIRED_FUNCTION(EVP_PKEY_keygen) \
+    REQUIRED_FUNCTION(EVP_PKEY_keygen_init) \
     REQUIRED_FUNCTION(EVP_PKEY_new) \
     REQUIRED_FUNCTION(EVP_PKEY_set1_DSA) \
     REQUIRED_FUNCTION(EVP_PKEY_set1_EC_KEY) \
@@ -432,6 +446,7 @@ void SSL_get0_alpn_selected(const SSL* ssl, const unsigned char** protocol, unsi
     FALLBACK_FUNCTION(RSA_get0_key) \
     FALLBACK_FUNCTION(RSA_meth_get_flags) \
     REQUIRED_FUNCTION(RSA_new) \
+    FALLBACK_FUNCTION(RSA_pkey_ctx_ctrl) \
     RENAMED_FUNCTION(RSA_PKCS1_OpenSSL, RSA_PKCS1_SSLeay) \
     REQUIRED_FUNCTION(RSA_private_decrypt) \
     REQUIRED_FUNCTION(RSA_private_encrypt) \
@@ -727,8 +742,12 @@ FOR_ALL_OPENSSL_FUNCTIONS
 #define EVP_MD_CTX_free EVP_MD_CTX_free_ptr
 #define EVP_MD_CTX_new EVP_MD_CTX_new_ptr
 #define EVP_MD_size EVP_MD_size_ptr
+#define EVP_PKEY_CTX_ctrl EVP_PKEY_CTX_ctrl_ptr
 #define EVP_PKEY_CTX_free EVP_PKEY_CTX_free_ptr
+#define EVP_PKEY_CTX_get0_pkey EVP_PKEY_CTX_get0_pkey_ptr
 #define EVP_PKEY_CTX_new EVP_PKEY_CTX_new_ptr
+#define EVP_PKEY_CTX_new_id EVP_PKEY_CTX_new_id_ptr
+#define EVP_PKEY_base_id EVP_PKEY_base_id_ptr
 #define EVP_PKEY_derive_set_peer EVP_PKEY_derive_set_peer_ptr
 #define EVP_PKEY_derive_init EVP_PKEY_derive_init_ptr
 #define EVP_PKEY_derive EVP_PKEY_derive_ptr
@@ -736,6 +755,8 @@ FOR_ALL_OPENSSL_FUNCTIONS
 #define EVP_PKEY_get1_DSA EVP_PKEY_get1_DSA_ptr
 #define EVP_PKEY_get1_EC_KEY EVP_PKEY_get1_EC_KEY_ptr
 #define EVP_PKEY_get1_RSA EVP_PKEY_get1_RSA_ptr
+#define EVP_PKEY_keygen EVP_PKEY_keygen_ptr
+#define EVP_PKEY_keygen_init EVP_PKEY_keygen_init_ptr
 #define EVP_PKEY_new EVP_PKEY_new_ptr
 #define EVP_PKEY_set1_DSA EVP_PKEY_set1_DSA_ptr
 #define EVP_PKEY_set1_EC_KEY EVP_PKEY_set1_EC_KEY_ptr
@@ -818,6 +839,7 @@ FOR_ALL_OPENSSL_FUNCTIONS
 #define RSA_get_method RSA_get_method_ptr
 #define RSA_meth_get_flags RSA_meth_get_flags_ptr
 #define RSA_new RSA_new_ptr
+#define RSA_pkey_ctx_ctrl RSA_pkey_ctx_ctrl_ptr
 #define RSA_PKCS1_OpenSSL RSA_PKCS1_OpenSSL_ptr
 #define RSA_private_decrypt RSA_private_decrypt_ptr
 #define RSA_private_encrypt RSA_private_encrypt_ptr
@@ -1026,6 +1048,7 @@ FOR_ALL_OPENSSL_FUNCTIONS
 #define RSA_set0_crt_params local_RSA_set0_crt_params
 #define RSA_set0_factors local_RSA_set0_factors
 #define RSA_set0_key local_RSA_set0_key
+#define RSA_pkey_ctx_ctrl local_RSA_pkey_ctx_ctrl
 #define SSL_CTX_set_security_level local_SSL_CTX_set_security_level
 #define SSL_is_init_finished local_SSL_is_init_finished
 #define X509_CRL_get0_nextUpdate local_X509_CRL_get0_nextUpdate
diff --git a/src/Native/Unix/System.Security.Cryptography.Native/pal_evp_pkey_rsa.c b/src/Native/Unix/System.Security.Cryptography.Native/pal_evp_pkey_rsa.c
index e8d961dbd2..29f9238ce9 100644
--- a/src/Native/Unix/System.Security.Cryptography.Native/pal_evp_pkey_rsa.c
+++ b/src/Native/Unix/System.Security.Cryptography.Native/pal_evp_pkey_rsa.c
@@ -4,6 +4,35 @@
 
 #include "pal_evp_pkey_rsa.h"
 
+EVP_PKEY* CryptoNative_RsaGenerateKey(int keySize)
+{
+    EVP_PKEY_CTX* ctx = EVP_PKEY_CTX_new_id(EVP_PKEY_RSA, NULL);
+
+    if (ctx == NULL)
+    {
+        return NULL;
+    }
+
+    EVP_PKEY* pkey = NULL;
+    EVP_PKEY* ret = NULL;
+
+    if (EVP_PKEY_keygen_init(ctx) == 1 &&
+        EVP_PKEY_CTX_set_rsa_keygen_bits(ctx, keySize) == 1 &&
+        EVP_PKEY_keygen(ctx, &pkey) == 1)
+    {
+        ret = pkey;
+        pkey = NULL;
+    }
+
+    if (pkey != NULL)
+    {
+        EVP_PKEY_free(pkey);
+    }
+
+    EVP_PKEY_CTX_free(ctx);
+    return ret;
+}
+
 RSA* CryptoNative_EvpPkeyGetRsa(EVP_PKEY* pkey)
 {
     return EVP_PKEY_get1_RSA(pkey);
diff --git a/src/Native/Unix/System.Security.Cryptography.Native/pal_evp_pkey_rsa.h b/src/Native/Unix/System.Security.Cryptography.Native/pal_evp_pkey_rsa.h
index d8ff369670..1fda149414 100644
--- a/src/Native/Unix/System.Security.Cryptography.Native/pal_evp_pkey_rsa.h
+++ b/src/Native/Unix/System.Security.Cryptography.Native/pal_evp_pkey_rsa.h
@@ -6,6 +6,11 @@
 #include "pal_compiler.h"
 #include "opensslshim.h"
 
+/*
+Creates an RSA key of the requested size.
+*/
+DLLEXPORT EVP_PKEY* CryptoNative_RsaGenerateKey(int32_t keySize);
+
 /*
 Shims the EVP_PKEY_get1_RSA method.
 
diff --git a/src/Native/Unix/System.Security.Cryptography.Native/pal_rsa.c b/src/Native/Unix/System.Security.Cryptography.Native/pal_rsa.c
index f764815a04..080027de0e 100644
--- a/src/Native/Unix/System.Security.Cryptography.Native/pal_rsa.c
+++ b/src/Native/Unix/System.Security.Cryptography.Native/pal_rsa.c
@@ -143,11 +143,6 @@ int32_t CryptoNative_RsaSize(RSA* rsa)
     return RSA_size(rsa);
 }
 
-int32_t CryptoNative_RsaGenerateKeyEx(RSA* rsa, int32_t bits, BIGNUM* e)
-{
-    return RSA_generate_key_ex(rsa, bits, e, NULL);
-}
-
 int32_t
 CryptoNative_RsaSign(int32_t type, const uint8_t* m, int32_t mlen, uint8_t* sigret, int32_t* siglen, RSA* rsa)
 {
diff --git a/src/Native/Unix/System.Security.Cryptography.Native/pal_rsa.h b/src/Native/Unix/System.Security.Cryptography.Native/pal_rsa.h
index b85fed627f..1c0bc2df47 100644
--- a/src/Native/Unix/System.Security.Cryptography.Native/pal_rsa.h
+++ b/src/Native/Unix/System.Security.Cryptography.Native/pal_rsa.h
@@ -86,13 +86,6 @@ Returns the RSA modulus size in bytes.
 */
 DLLEXPORT int32_t CryptoNative_RsaSize(RSA* rsa);
 
-/*
-Shims the RSA_generate_key_ex method.
-
-Returns 1 upon success, otherwise 0.
-*/
-DLLEXPORT int32_t CryptoNative_RsaGenerateKeyEx(RSA* rsa, int32_t bits, BIGNUM* e);
-
 /*
 Shims the RSA_sign method.
 
diff --git a/src/System.Security.Cryptography.Algorithms/src/System.Security.Cryptography.Algorithms.csproj b/src/System.Security.Cryptography.Algorithms/src/System.Security.Cryptography.Algorithms.csproj
index fa63b6e2fe..6ad2b78e01 100644
--- a/src/System.Security.Cryptography.Algorithms/src/System.Security.Cryptography.Algorithms.csproj
+++ b/src/System.Security.Cryptography.Algorithms/src/System.Security.Cryptography.Algorithms.csproj
@@ -507,6 +507,9 @@
     <Compile Include="$(CommonPath)\Interop\Unix\System.Security.Cryptography.Native\Interop.EvpPkey.Ecdh.cs">
       <Link>Common\Interop\Unix\System.Security.Cryptography.Native\Interop.EvpPkey.Ecdh.cs</Link>
     </Compile>
+    <Compile Include="$(CommonPath)\Interop\Unix\System.Security.Cryptography.Native\Interop.EvpPkey.Rsa.cs">
+      <Link>Common\Interop\Unix\System.Security.Cryptography.Native\Interop.EvpPkey.Rsa.cs</Link>
+    </Compile>
     <Compile Include="$(CommonPath)\Interop\Unix\System.Security.Cryptography.Native\Interop.EVP.cs">
       <Link>Common\Interop\Unix\System.Security.Cryptography.Native\Interop.EVP.cs</Link>
     </Compile>
-- 
2.31.1