openssl/0089-signature-Clamp-PSS-salt-len-to-MD-len.patch

From 39a91c33e2b89a0fe42e3791d3dc304519a52182 Mon Sep 17 00:00:00 2001
From: Clemens Lang <cllang@redhat.com>
Date: Fri, 18 Nov 2022 12:35:33 +0100
Subject: [PATCH] signature: Clamp PSS salt len to MD len

Since FIPS 186-4 subsection 5.5 limits the acceptable PSS salt length to
the size of the message digest, change the default automatic behavior
when signing to use at most the digest size as salt length. Shorter
values are still possible when long hashes are used with short keys.

Signed-off-by: Clemens Lang <cllang@redhat.com>
---
 crypto/rsa/rsa_ameth.c                        | 19 +++++++++++++++++--
 crypto/rsa/rsa_pss.c                          | 11 +++++++++++
 doc/man3/EVP_PKEY_CTX_ctrl.pod                |  4 +++-
 providers/implementations/signature/rsa_sig.c | 18 ++++++++++++++++--
 4 files changed, 47 insertions(+), 5 deletions(-)

diff --git a/crypto/rsa/rsa_ameth.c b/crypto/rsa/rsa_ameth.c
index b1580ca..dc81627 100644
--- a/crypto/rsa/rsa_ameth.c
+++ b/crypto/rsa/rsa_ameth.c
@@ -449,6 +449,7 @@ static RSA_PSS_PARAMS *rsa_ctx_to_pss(EVP_PKEY_CTX *pkctx)
     const EVP_MD *sigmd, *mgf1md;
     EVP_PKEY *pk = EVP_PKEY_CTX_get0_pkey(pkctx);
     int saltlen;
+    int saltlenMax = -1;
 
     if (EVP_PKEY_CTX_get_signature_md(pkctx, &sigmd) <= 0)
         return NULL;
@@ -456,14 +457,28 @@ static RSA_PSS_PARAMS *rsa_ctx_to_pss(EVP_PKEY_CTX *pkctx)
         return NULL;
     if (!EVP_PKEY_CTX_get_rsa_pss_saltlen(pkctx, &saltlen))
         return NULL;
-    if (saltlen == -1) {
+    if (saltlen == RSA_PSS_SALTLEN_DIGEST) {
         saltlen = EVP_MD_get_size(sigmd);
-    } else if (saltlen == -2 || saltlen == -3) {
+    } else if (saltlen == RSA_PSS_SALTLEN_AUTO) {
+        /* FIPS 186-4 section 5 "The RSA Digital Signature Algorithm",
+         * subsection 5.5 "PKCS #1" says: "For RSASSA-PSS […] the length (in
+         * bytes) of the salt (sLen) shall satisfy 0 ≤ sLen ≤ hLen, where hLen
+         * is the length of the hash function output block (in bytes)."
+         *
+         * Switch the meaning of RSA_PSS_SALTLEN_AUTO to use at most the digest
+         * length in FIPS mode, so that the default does not violate FIPS
+         * 186-4. */
+        saltlen = RSA_PSS_SALTLEN_MAX;
+        saltlenMax = EVP_MD_get_size(sigmd);
+    }
+    if (saltlen == RSA_PSS_SALTLEN_MAX) {
         saltlen = EVP_PKEY_get_size(pk) - EVP_MD_get_size(sigmd) - 2;
         if ((EVP_PKEY_get_bits(pk) & 0x7) == 1)
             saltlen--;
         if (saltlen < 0)
             return NULL;
+        if (saltlenMax >= 0 && saltlen > saltlenMax)
+            saltlen = saltlenMax;
     }
 
     return ossl_rsa_pss_params_create(sigmd, mgf1md, saltlen);
diff --git a/crypto/rsa/rsa_pss.c b/crypto/rsa/rsa_pss.c
index e8681b0..d8f9207 100644
--- a/crypto/rsa/rsa_pss.c
+++ b/crypto/rsa/rsa_pss.c
@@ -168,6 +168,7 @@ int RSA_padding_add_PKCS1_PSS_mgf1(RSA *rsa, unsigned char *EM,
     int hLen, maskedDBLen, MSBits, emLen;
     unsigned char *H, *salt = NULL, *p;
     EVP_MD_CTX *ctx = NULL;
+    int sLenMax = -1;
 
     if (mgf1Hash == NULL)
         mgf1Hash = Hash;
@@ -190,10 +191,18 @@ int RSA_padding_add_PKCS1_PSS_mgf1(RSA *rsa, unsigned char *EM,
      *      -3      same as above (on signing)
      *      -N      reserved
      */
+    /* FIPS 186-4 section 5 "The RSA Digital Signature Algorithm", subsection
+     * 5.5 "PKCS #1" says: "For RSASSA-PSS […] the length (in bytes) of the
+     * salt (sLen) shall satisfy 0 ≤ sLen ≤ hLen, where hLen is the length of
+     * the hash function output block (in bytes)."
+     *
+     * Switch the meaning of RSA_PSS_SALTLEN_AUTO to use at most the digest
+     * length in FIPS mode, so that the default does not violate FIPS 186-4. */
     if (sLen == RSA_PSS_SALTLEN_DIGEST) {
         sLen = hLen;
     } else if (sLen == RSA_PSS_SALTLEN_MAX_SIGN) {
         sLen = RSA_PSS_SALTLEN_MAX;
+        sLenMax = hLen;
     } else if (sLen < RSA_PSS_SALTLEN_MAX) {
         ERR_raise(ERR_LIB_RSA, RSA_R_SLEN_CHECK_FAILED);
         goto err;
@@ -211,6 +220,8 @@ int RSA_padding_add_PKCS1_PSS_mgf1(RSA *rsa, unsigned char *EM,
     }
     if (sLen == RSA_PSS_SALTLEN_MAX) {
         sLen = emLen - hLen - 2;
+        if (sLenMax >= 0 && sLen > sLenMax)
+            sLen = sLenMax;
     } else if (sLen > emLen - hLen - 2) {
         ERR_raise(ERR_LIB_RSA, RSA_R_DATA_TOO_LARGE_FOR_KEY_SIZE);
         goto err;
diff --git a/doc/man3/EVP_PKEY_CTX_ctrl.pod b/doc/man3/EVP_PKEY_CTX_ctrl.pod
index 3075eaa..5463472 100644
--- a/doc/man3/EVP_PKEY_CTX_ctrl.pod
+++ b/doc/man3/EVP_PKEY_CTX_ctrl.pod
@@ -287,7 +287,9 @@ sets the salt length to the maximum permissible value.
 
 causes the salt length to be automatically determined based on the
 B<PSS> block structure when verifying.  When signing, it has the same
-meaning as B<RSA_PSS_SALTLEN_MAX>.
+meaning as B<RSA_PSS_SALTLEN_MAX> up to a maximum of the digest length to
+comply with FIPS 186-4 section 5.5.  This maximum is specific to Red Hat,
+upstream also uses larger values.
 
 =back
 
diff --git a/providers/implementations/signature/rsa_sig.c b/providers/implementations/signature/rsa_sig.c
index 3ce5efd..519c6a2 100644
--- a/providers/implementations/signature/rsa_sig.c
+++ b/providers/implementations/signature/rsa_sig.c
@@ -200,13 +200,27 @@ static void *rsa_newctx(void *provctx, const char *propq)
 static int rsa_pss_compute_saltlen(PROV_RSA_CTX *ctx)
 {
     int saltlen = ctx->saltlen;
- 
+    int saltlenMax = -1;
+
+    /* FIPS 186-4 section 5 "The RSA Digital Signature Algorithm", subsection
+     * 5.5 "PKCS #1" says: "For RSASSA-PSS […] the length (in bytes) of the
+     * salt (sLen) shall satisfy 0 ≤ sLen ≤ hLen, where hLen is the length of
+     * the hash function output block (in bytes)."
+     *
+     * Switch the meaning of RSA_PSS_SALTLEN_AUTO to use at most the digest
+     * length in FIPS mode, so that the default does not violate FIPS 186-4. */
     if (saltlen == RSA_PSS_SALTLEN_DIGEST) {
         saltlen = EVP_MD_get_size(ctx->md);
-    } else if (saltlen == RSA_PSS_SALTLEN_AUTO || saltlen == RSA_PSS_SALTLEN_MAX) {
+    } else if (saltlen == RSA_PSS_SALTLEN_AUTO) {
+        saltlen = RSA_PSS_SALTLEN_MAX;
+        saltlenMax = EVP_MD_get_size(ctx->md);
+    }
+    if (saltlen == RSA_PSS_SALTLEN_MAX) {
         saltlen = RSA_size(ctx->rsa) - EVP_MD_get_size(ctx->md) - 2;
         if ((RSA_bits(ctx->rsa) & 0x7) == 1)
             saltlen--;
+        if (saltlenMax >= 0 && saltlen > saltlenMax)
+            saltlen = saltlenMax;
     }
     if (saltlen < 0) {
         ERR_raise(ERR_LIB_PROV, ERR_R_INTERNAL_ERROR);
-- 
2.38.1