openssl-pkcs11/openssl-pkcs11-ossl3.patch

From 6efcf3c52db1857aaa18741a509741519b0c5775 Mon Sep 17 00:00:00 2001
From: Doug Engert <deengert@gmail.com>
Date: Fri, 29 Jul 2022 17:54:42 -0500
Subject: [PATCH] Deffer initializing crypto routines in PKCS11 engine until
 needed

Fixes:#456

bind_helper in eng_font.c is split into bind_helper and bind_helper2
The calls to ENGINE_set_RSA, ENGINE_set_EC, ENGINE_set_ECDH and
ENGINE_set_pkey_meths are moved to bind_helper2.

bind_helper2 is called from load_pubkey and load_privkey.

This in effect gets around the problem OpenSSL 3.0.x has when
it loads the pkcs11 engine from openssl.cnf, and then tries to use it
as a default provider even when no engine was specified on
the command line.

 On branch deffer_init_crypto
 Changes to be committed:
	modified:   eng_front.c
---
 src/eng_front.c | 28 ++++++++++++++++++++++++----
 1 file changed, 24 insertions(+), 4 deletions(-)

diff --git a/src/eng_front.c b/src/eng_front.c
index 3a3c891..bfc3502 100644
--- a/src/eng_front.c
+++ b/src/eng_front.c
@@ -82,6 +82,8 @@ static const ENGINE_CMD_DEFN engine_cmd_defns[] = {
 	{0, NULL, NULL, 0}
 };
 
+static int bind_helper2(ENGINE *e);
+
 static ENGINE_CTX *get_ctx(ENGINE *engine)
 {
 	ENGINE_CTX *ctx;
@@ -174,6 +176,7 @@ static EVP_PKEY *load_pubkey(ENGINE *engine, const char *s_key_id,
 	ctx = get_ctx(engine);
 	if (!ctx)
 		return 0;
+	bind_helper2(engine);
 	return ctx_load_pubkey(ctx, s_key_id, ui_method, callback_data);
 }
 
@@ -186,6 +189,7 @@ static EVP_PKEY *load_privkey(ENGINE *engine, const char *s_key_id,
 	ctx = get_ctx(engine);
 	if (!ctx)
 		return 0;
+	bind_helper2(engine);
 	pkey = ctx_load_privkey(ctx, s_key_id, ui_method, callback_data);
 #ifdef EVP_F_EVP_PKEY_SET1_ENGINE
 	/* EVP_PKEY_set1_engine() is required for OpenSSL 1.1.x,
@@ -219,6 +223,25 @@ static int bind_helper(ENGINE *e)
 			!ENGINE_set_ctrl_function(e, engine_ctrl) ||
 			!ENGINE_set_cmd_defns(e, engine_cmd_defns) ||
 			!ENGINE_set_name(e, PKCS11_ENGINE_NAME) ||
+
+			!ENGINE_set_load_pubkey_function(e, load_pubkey) ||
+			!ENGINE_set_load_privkey_function(e, load_privkey)) {
+		return 0;
+	} else {
+		ERR_load_ENG_strings();
+		return 1;
+	}
+}
+
+/*
+ * With OpenSSL 3.x, engines might be used because defined in openssl.cnf
+ * which will cause problems
+ * only add engine routines after a call to load keys
+ */
+
+static int bind_helper2(ENGINE *e)
+{
+	if (
 #ifndef OPENSSL_NO_RSA
 			!ENGINE_set_RSA(e, PKCS11_get_rsa_method()) ||
 #endif
@@ -235,12 +258,9 @@ static int bind_helper(ENGINE *e)
 			!ENGINE_set_ECDH(e, PKCS11_get_ecdh_method()) ||
 #endif
 #endif /* OPENSSL_VERSION_NUMBER */
-			!ENGINE_set_pkey_meths(e, PKCS11_pkey_meths) ||
-			!ENGINE_set_load_pubkey_function(e, load_pubkey) ||
-			!ENGINE_set_load_privkey_function(e, load_privkey)) {
+			!ENGINE_set_pkey_meths(e, PKCS11_pkey_meths)) {
 		return 0;
 	} else {
-		ERR_load_ENG_strings();
 		return 1;
 	}
 }