From 6242770aa25fd6296a9893b6d5d057b58208a274 Mon Sep 17 00:00:00 2001
From: Dmitry Belyavskiy <dbelyavs@redhat.com>
Date: Wed, 1 Nov 2023 10:44:40 +0100
Subject: [PATCH] Using DigestSign/DigestVerify functions for better FIPS
 compatibility

Resolves: RHEL-5217
---
 openssh-7.7p1-fips.patch | 52 ++++++++++++++++++++++++++++++++++++++--
 openssh.spec             |  2 ++
 2 files changed, 52 insertions(+), 2 deletions(-)

diff --git a/openssh-7.7p1-fips.patch b/openssh-7.7p1-fips.patch
index c6db70a..01b9741 100644
--- a/openssh-7.7p1-fips.patch
+++ b/openssh-7.7p1-fips.patch
@@ -471,12 +471,60 @@ diff -up openssh-7.9p1/sshkey.c.fips openssh-7.9p1/sshkey.c
  
  #include "xmss_fast.h"
  
+@@ -392,13 +394,14 @@ sshkey_calculate_signature(EVP_PKEY *pkey
+ {
+ 	EVP_MD_CTX *ctx = NULL;
+ 	u_char *sig = NULL;
+-	int ret, slen, len;
++	int ret, slen;
++	size_t len;
+ 
+ 	if (sigp == NULL || lenp == NULL) {
+ 		return SSH_ERR_INVALID_ARGUMENT;
+ 	}
+ 
+-	slen = EVP_PKEY_size(pkey);
++	slen = EVP_PKEY_get_size(pkey);
+ 	if (slen <= 0 || slen > SSHBUF_MAX_BIGNUM)
+ 		return SSH_ERR_INVALID_ARGUMENT;
+ 
+@@ -411,9 +414,10 @@ sshkey_calculate_signature(EVP_PKEY *pkey
+ 		ret = SSH_ERR_ALLOC_FAIL;
+ 		goto error;
+ 	}
+-	if (EVP_SignInit_ex(ctx, ssh_digest_to_md(hash_alg), NULL) <= 0 ||
+-	    EVP_SignUpdate(ctx, data, datalen) <= 0 ||
+-	    EVP_SignFinal(ctx, sig, &len, pkey) <= 0) {
++	if (EVP_DigestSignInit(ctx, NULL, ssh_digest_to_md(hash_alg),
++	        NULL, pkey) != 1 ||
++	    EVP_DigestSignUpdate(ctx, data, datalen) != 1 ||
++	    EVP_DigestSignFinal(ctx, sig, &len) != 1) {
+ 		ret = SSH_ERR_LIBCRYPTO_ERROR;
+ 		goto error;
+ 	}
+@@ -440,12 +444,13 @@ sshkey_verify_signature(EVP_PKEY *pkey
+ 	if ((ctx = EVP_MD_CTX_new()) == NULL) {
+ 		return SSH_ERR_ALLOC_FAIL;
+ 	}
+-	if (EVP_VerifyInit_ex(ctx, ssh_digest_to_md(hash_alg), NULL) <= 0 ||
+-	    EVP_VerifyUpdate(ctx, data, datalen) <= 0) {
++	if (EVP_DigestVerifyInit(ctx, NULL, ssh_digest_to_md(hash_alg),
++	    NULL, pkey) != 1 ||
++	    EVP_DigestVerifyUpdate(ctx, data, datalen) != 1) {
+ 		ret = SSH_ERR_LIBCRYPTO_ERROR;
+ 		goto done;
+ 	}
+-	ret = EVP_VerifyFinal(ctx, sigbuf, siglen, pkey);
++	ret = EVP_DigestVerifyFinal(ctx, sigbuf, siglen);
+ 	switch (ret) {
+ 	case 1:
+ 		ret = 0;
 @@ -1514,6 +1516,8 @@ rsa_generate_private_key(u_int bits, RSA
  	}
  	if (!BN_set_word(f4, RSA_F4) ||
  	    !RSA_generate_key_ex(private, bits, f4, NULL)) {
-+			if (FIPS_mode())
-+				logit("%s: the key length might be unsupported by FIPS mode approved key generation method", __func__);
++		if (FIPS_mode())
++			logit("%s: the key length might be unsupported by FIPS mode approved key generation method", __func__);
  		ret = SSH_ERR_LIBCRYPTO_ERROR;
  		goto out;
  	}
diff --git a/openssh.spec b/openssh.spec
index d54ec38..4f45bd8 100644
--- a/openssh.spec
+++ b/openssh.spec
@@ -824,6 +824,8 @@ getent passwd sshd >/dev/null || \
   Resolves: RHEL-11548
 - Avoid sshd_config 256K limit
   Resolves: RHEL-5279
+- Using DigestSign/DigestVerify functions for better FIPS compatibility
+  Resolves: RHEL-5217
 
 * Thu Aug 24 2023 Dmitry Belyavskiy <dbelyavs@redhat.com> - 8.0p1-19
 - rebuilt