Fix issues in bundling nettle

This unbreaks FIPS integrity checks against missing Nettle libs, as
well as stop exposing them through gnutls.pc.

Related: RHEL-50011
Signed-off-by: Daiki Ueno <dueno@redhat.com>

gnutls-3.8.7-nettle-static.patch

@@ -0,0 +1,165 @@
+From 558cf23853f6ad0537daff4613d316265857b7fd Mon Sep 17 00:00:00 2001
+From: Daiki Ueno <ueno@gnu.org>
+Date: Wed, 21 Aug 2024 14:50:54 +0900
+Subject: [PATCH] fips: skip HMAC checks of nettle libraries when statically
+ linked
+
+Since commit b6e9b10347ed577a9a37b7b28e1a039c5f6ccb16, it is possible
+to link Nettle libraries statically.  In that case, FIPS integrity
+checks against the Nettle shared libraries should be skipped as they
+are not used by GnuTLS.
+
+Signed-off-by: Daiki Ueno <ueno@gnu.org>
+---
+ lib/fips.c     | 32 ++++++++++++++++++++++++--------
+ lib/fipshmac.c | 12 ++++--------
+ 2 files changed, 28 insertions(+), 16 deletions(-)
+
+diff --git a/lib/fips.c b/lib/fips.c
+index e5fce6b1b9..dc86a44354 100644
+--- a/lib/fips.c
++++ b/lib/fips.c
+@@ -157,14 +157,6 @@ void _gnutls_fips_mode_reset_zombie(void)
+ #define GNUTLS_LIBRARY_SONAME "none"
+ #endif
+ 
+-#ifndef NETTLE_LIBRARY_SONAME
+-#define NETTLE_LIBRARY_SONAME "none"
+-#endif
+-
+-#ifndef HOGWEED_LIBRARY_SONAME
+-#define HOGWEED_LIBRARY_SONAME "none"
+-#endif
+-
+ #define HMAC_SIZE 32
+ #define HMAC_ALGO GNUTLS_MAC_SHA256
+ #define HMAC_FORMAT_VERSION 1
+@@ -177,8 +169,12 @@ struct hmac_entry {
+ struct hmac_file {
+ 	int version;
+ 	struct hmac_entry gnutls;
++#ifdef NETTLE_LIBRARY_SONAME
+ 	struct hmac_entry nettle;
++#endif
++#ifdef HOGWEED_LIBRARY_SONAME
+ 	struct hmac_entry hogweed;
++#endif
+ #ifdef GMP_LIBRARY_SONAME
+ 	struct hmac_entry gmp;
+ #endif
+@@ -186,8 +182,12 @@ struct hmac_file {
+ 
+ struct lib_paths {
+ 	char gnutls[GNUTLS_PATH_MAX];
++#ifdef NETTLE_LIBRARY_SONAME
+ 	char nettle[GNUTLS_PATH_MAX];
++#endif
++#ifdef HOGWEED_LIBRARY_SONAME
+ 	char hogweed[GNUTLS_PATH_MAX];
++#endif
+ #ifdef GMP_LIBRARY_SONAME
+ 	char gmp[GNUTLS_PATH_MAX];
+ #endif
+@@ -250,10 +250,14 @@ static int handler(void *user, const char *section, const char *name,
+ 		}
+ 	} else if (!strcmp(section, GNUTLS_LIBRARY_SONAME)) {
+ 		return lib_handler(&p->gnutls, section, name, value);
++#ifdef NETTLE_LIBRARY_SONAME
+ 	} else if (!strcmp(section, NETTLE_LIBRARY_SONAME)) {
+ 		return lib_handler(&p->nettle, section, name, value);
++#endif
++#ifdef HOGWEED_LIBRARY_SONAME
+ 	} else if (!strcmp(section, HOGWEED_LIBRARY_SONAME)) {
+ 		return lib_handler(&p->hogweed, section, name, value);
++#endif
+ #ifdef GMP_LIBRARY_SONAME
+ 	} else if (!strcmp(section, GMP_LIBRARY_SONAME)) {
+ 		return lib_handler(&p->gmp, section, name, value);
+@@ -403,10 +407,14 @@ static int callback(struct dl_phdr_info *info, size_t size, void *data)
+ 
+ 	if (!strcmp(soname, GNUTLS_LIBRARY_SONAME))
+ 		_gnutls_str_cpy(paths->gnutls, GNUTLS_PATH_MAX, path);
++#ifdef NETTLE_LIBRARY_SONAME
+ 	else if (!strcmp(soname, NETTLE_LIBRARY_SONAME))
+ 		_gnutls_str_cpy(paths->nettle, GNUTLS_PATH_MAX, path);
++#endif
++#ifdef HOGWEED_LIBRARY_SONAME
+ 	else if (!strcmp(soname, HOGWEED_LIBRARY_SONAME))
+ 		_gnutls_str_cpy(paths->hogweed, GNUTLS_PATH_MAX, path);
++#endif
+ #ifdef GMP_LIBRARY_SONAME
+ 	else if (!strcmp(soname, GMP_LIBRARY_SONAME))
+ 		_gnutls_str_cpy(paths->gmp, GNUTLS_PATH_MAX, path);
+@@ -423,14 +431,18 @@ static int load_lib_paths(struct lib_paths *paths)
+ 		_gnutls_debug_log("Gnutls library path was not found\n");
+ 		return gnutls_assert_val(GNUTLS_E_FILE_ERROR);
+ 	}
++#ifdef NETTLE_LIBRARY_SONAME
+ 	if (paths->nettle[0] == '\0') {
+ 		_gnutls_debug_log("Nettle library path was not found\n");
+ 		return gnutls_assert_val(GNUTLS_E_FILE_ERROR);
+ 	}
++#endif
++#ifdef HOGWEED_LIBRARY_SONAME
+ 	if (paths->hogweed[0] == '\0') {
+ 		_gnutls_debug_log("Hogweed library path was not found\n");
+ 		return gnutls_assert_val(GNUTLS_E_FILE_ERROR);
+ 	}
++#endif
+ #ifdef GMP_LIBRARY_SONAME
+ 	if (paths->gmp[0] == '\0') {
+ 		_gnutls_debug_log("Gmp library path was not found\n");
+@@ -483,12 +495,16 @@ static int check_binary_integrity(void)
+ 	ret = check_lib_hmac(&hmac.gnutls, paths.gnutls);
+ 	if (ret < 0)
+ 		return ret;
++#ifdef NETTLE_LIBRARY_SONAME
+ 	ret = check_lib_hmac(&hmac.nettle, paths.nettle);
+ 	if (ret < 0)
+ 		return ret;
++#endif
++#ifdef HOGWEED_LIBRARY_SONAME
+ 	ret = check_lib_hmac(&hmac.hogweed, paths.hogweed);
+ 	if (ret < 0)
+ 		return ret;
++#endif
+ #ifdef GMP_LIBRARY_SONAME
+ 	ret = check_lib_hmac(&hmac.gmp, paths.gmp);
+ 	if (ret < 0)
+diff --git a/lib/fipshmac.c b/lib/fipshmac.c
+index d3561b4c47..5c3202c561 100644
+--- a/lib/fipshmac.c
++++ b/lib/fipshmac.c
+@@ -40,14 +40,6 @@
+ #define GNUTLS_LIBRARY_SONAME "none"
+ #endif
+ 
+-#ifndef NETTLE_LIBRARY_SONAME
+-#define NETTLE_LIBRARY_SONAME "none"
+-#endif
+-
+-#ifndef HOGWEED_LIBRARY_SONAME
+-#define HOGWEED_LIBRARY_SONAME "none"
+-#endif
+-
+ #define HMAC_SIZE 32
+ #define HMAC_ALGO GNUTLS_MAC_SHA256
+ #define HMAC_STR_SIZE (2 * HMAC_SIZE + 1)
+@@ -117,10 +109,14 @@ static int callback(struct dl_phdr_info *info, size_t size, void *data)
+ 
+ 	if (!strcmp(soname, GNUTLS_LIBRARY_SONAME))
+ 		return print_lib(data ? data : path, soname);
++#ifdef NETTLE_LIBRARY_SONAME
+ 	if (!strcmp(soname, NETTLE_LIBRARY_SONAME))
+ 		return print_lib(path, soname);
++#endif
++#ifdef HOGWEED_LIBRARY_SONAME
+ 	if (!strcmp(soname, HOGWEED_LIBRARY_SONAME))
+ 		return print_lib(path, soname);
++#endif
+ #ifdef GMP_LIBRARY_SONAME
+ 	if (!strcmp(soname, GMP_LIBRARY_SONAME))
+ 		return print_lib(path, soname);
+-- 
+2.46.0
+

gnutls.spec

@@ -32,6 +32,8 @@ Patch: gnutls-3.7.6-fips-sha1-sigver.patch
 Patch: gnutls-3.7.8-ktls_skip_tls12_chachapoly_test.patch
 # upstreamed: https://gitlab.com/gnutls/gnutls/-/merge_requests/1867
 Patch: gnutls-3.8.7-pkgconf-dlopen.patch
+# upstreamed: https://gitlab.com/gnutls/gnutls/-/merge_requests/1868
+Patch: gnutls-3.8.7-nettle-static.patch
 
 %bcond_without bootstrap
 %bcond_without dane
@@ -301,6 +303,10 @@ patch -p1 < %{SOURCE201}
 popd
 %endif
 
+%if %{with bundled_gmp}
+sed -i 's/@GMP_LIBS@//' lib/gnutls.pc.in
+%endif
+
 %build
 %define _lto_cflags %{nil}
 
@@ -420,6 +426,11 @@ pushd native_build
            --with-default-priority-string="@SYSTEM"
 
 %make_build
+
+%if %{with bundled_nettle}
+sed -i '/^Requires.private:/s/\(nettle\|hogweed\)[ ,]*//g' lib/gnutls.pc
+%endif
+
 popd
 
 %if %{with mingw}