openssh/openssh-9.0p1-evp-pkcs11.patch

diff --color -ru -x regress -x autom4te.cache -x '*.o' -x '*.lo' -x Makefile -x config.status -x configure~ -x configure.ac openssh-9.0p1/ssh-ecdsa.c openssh-9.0p1-patched/ssh-ecdsa.c
--- openssh-9.0p1/ssh-ecdsa.c	2023-05-24 08:54:03.926443958 +0200
+++ openssh-9.0p1-patched/ssh-ecdsa.c	2023-05-24 09:46:19.082925921 +0200
@@ -74,8 +74,18 @@
 	if ((hash_alg = sshkey_ec_nid_to_hash_alg(key->ecdsa_nid)) == -1)
 		return SSH_ERR_INTERNAL_ERROR;
 
-  	if ((ret = ssh_create_evp_ec(key->ecdsa, key->ecdsa_nid, &pkey)) != 0)
-		return ret;
+#ifdef ENABLE_PKCS11
+	if (is_ecdsa_pkcs11(key->ecdsa)) {
+		if ((pkey = EVP_PKEY_new()) == NULL ||
+		    EVP_PKEY_set1_EC_KEY(pkey, key->ecdsa) != 1)
+			return SSH_ERR_ALLOC_FAIL;
+	} else {
+#endif
+		if ((ret = ssh_create_evp_ec(key->ecdsa, key->ecdsa_nid, &pkey)) != 0)
+			return ret;
+#ifdef ENABLE_PKCS11
+	}
+#endif
 	ret = sshkey_calculate_signature(pkey, hash_alg, &sigb, &len, data,
 	    datalen);
 	EVP_PKEY_free(pkey);
diff --color -ru -x regress -x autom4te.cache -x '*.o' -x '*.lo' -x Makefile -x config.status -x configure~ -x configure.ac openssh-9.0p1/sshkey.h openssh-9.0p1-patched/sshkey.h
--- openssh-9.0p1/sshkey.h	2023-05-24 08:54:03.926443958 +0200
+++ openssh-9.0p1-patched/sshkey.h	2023-05-24 08:57:22.930642788 +0200
@@ -340,6 +340,10 @@
     const u_char *data, size_t datalen, u_int compat);
 #endif
 
+#ifdef ENABLE_PKCS11
+int pkcs11_get_ecdsa_idx(void);
+#endif
+
 #if !defined(WITH_OPENSSL)
 # undef RSA
 # undef DSA
diff --color -ru -x regress -x autom4te.cache -x '*.o' -x '*.lo' -x Makefile -x config.status -x configure~ -x configure.ac openssh-9.0p1/ssh-pkcs11.c openssh-9.0p1-patched/ssh-pkcs11.c
--- openssh-9.0p1/ssh-pkcs11.c	2023-05-24 08:54:03.888443542 +0200
+++ openssh-9.0p1-patched/ssh-pkcs11.c	2023-05-24 09:48:13.101168512 +0200
@@ -776,8 +776,24 @@
 
 	return (0);
 }
+
+int
+is_ecdsa_pkcs11(EC_KEY *ecdsa)
+{
+	if (EC_KEY_get_ex_data(ecdsa, ec_key_idx) != NULL)
+		return 1;
+	return 0;
+}
 #endif /* OPENSSL_HAS_ECC && HAVE_EC_KEY_METHOD_NEW */
 
+int
+is_rsa_pkcs11(RSA *rsa)
+{
+	if (RSA_get_ex_data(rsa, rsa_idx) != NULL)
+		return 1;
+	return 0;
+}
+
 /* remove trailing spaces */
 static void
 rmspace(u_char *buf, size_t len)
diff --color -ru -x regress -x autom4te.cache -x '*.o' -x '*.lo' -x Makefile -x config.status -x configure~ -x configure.ac openssh-9.0p1/ssh-pkcs11-client.c openssh-9.0p1-patched/ssh-pkcs11-client.c
--- openssh-9.0p1/ssh-pkcs11-client.c	2023-05-24 08:54:03.887443531 +0200
+++ openssh-9.0p1-patched/ssh-pkcs11-client.c	2023-05-24 09:49:41.741134514 +0200
@@ -225,8 +225,36 @@
 static RSA_METHOD	*helper_rsa;
 #if defined(OPENSSL_HAS_ECC) && defined(HAVE_EC_KEY_METHOD_NEW)
 static EC_KEY_METHOD	*helper_ecdsa;
+
+int
+is_ecdsa_pkcs11(EC_KEY *ecdsa)
+{
+	const EC_KEY_METHOD *meth;
+	ECDSA_SIG *(*sign_sig)(const unsigned char *dgst, int dgstlen,
+		const BIGNUM *kinv, const BIGNUM *rp, EC_KEY *eckey) = NULL;
+
+	meth = EC_KEY_get_method(ecdsa);
+	EC_KEY_METHOD_get_sign(meth, NULL, NULL, &sign_sig);
+	if (sign_sig == ecdsa_do_sign)
+		return 1;
+	return 0;
+}
 #endif /* OPENSSL_HAS_ECC && HAVE_EC_KEY_METHOD_NEW */
 
+int
+is_rsa_pkcs11(RSA *rsa)
+{
+	const RSA_METHOD *meth;
+	int (*priv_enc)(int flen, const unsigned char *from,
+        	unsigned char *to, RSA *rsa, int padding) = NULL;
+
+	meth = RSA_get_method(rsa);
+	priv_enc = RSA_meth_get_priv_enc(meth);
+	if (priv_enc == rsa_encrypt)
+		return 1;
+	return 0;
+}
+
 /* redirect private key crypto operations to the ssh-pkcs11-helper */
 static void
 wrap_key(struct sshkey *k)
diff --color -ru -x regress -x autom4te.cache -x '*.o' -x '*.lo' -x Makefile -x config.status -x configure~ -x configure.ac openssh-9.0p1/ssh-pkcs11.h openssh-9.0p1-patched/ssh-pkcs11.h
--- openssh-9.0p1/ssh-pkcs11.h	2023-05-24 08:54:03.888443542 +0200
+++ openssh-9.0p1-patched/ssh-pkcs11.h	2023-05-24 09:50:03.981376886 +0200
@@ -39,6 +39,11 @@
 	    u_int32_t *);
 #endif
 
+#ifdef HAVE_EC_KEY_METHOD_NEW
+int is_ecdsa_pkcs11(EC_KEY *ecdsa);
+#endif
+int is_rsa_pkcs11(RSA *rsa);
+
 #if !defined(WITH_OPENSSL) && defined(ENABLE_PKCS11)
 #undef ENABLE_PKCS11
 #endif
diff --color -ru -x regress -x autom4te.cache -x '*.o' -x '*.lo' -x Makefile -x config.status -x configure~ -x configure.ac openssh-9.0p1/ssh-rsa.c openssh-9.0p1-patched/ssh-rsa.c
--- openssh-9.0p1/ssh-rsa.c	2023-05-24 08:54:03.927443969 +0200
+++ openssh-9.0p1-patched/ssh-rsa.c	2023-05-24 09:51:50.358536178 +0200
@@ -174,8 +174,18 @@
 	if (RSA_bits(key->rsa) < SSH_RSA_MINIMUM_MODULUS_SIZE)
 		return SSH_ERR_KEY_LENGTH;
 
-  	if ((ret = ssh_create_evp_rsa(key, &pkey)) != 0)
-		return ret;
+#ifdef ENABLE_PKCS11
+	if (is_rsa_pkcs11(key->rsa)) {
+		if ((pkey = EVP_PKEY_new()) == NULL ||
+		    EVP_PKEY_set1_RSA(pkey, key->rsa) != 1)
+			return SSH_ERR_ALLOC_FAIL;
+	} else {
+#endif
+		if ((ret = ssh_create_evp_rsa(key, &pkey)) != 0)
+			return ret;
+#ifdef ENABLE_PKCS11
+	}
+#endif
 	ret = sshkey_calculate_signature(pkey, hash_alg, &sig, &len, data,
 	    datalen);
 	EVP_PKEY_free(pkey);