resource-agents/bz2014415-nfsserver-add-nfs_server_scope-parameter.patch

From 764dacb6195f8940f13b9c322b1bc8189c5619fc Mon Sep 17 00:00:00 2001
From: Lars Ellenberg <lars.ellenberg@linbit.com>
Date: Mon, 6 Sep 2021 12:13:42 +0200
Subject: [PATCH 1/6] Fix NFSv4 lock failover: set NFS Server Scope

Problem: https://github.com/ClusterLabs/resource-agents/issues/1644
RFC8881, 8.4.2.1 State Reclaim:

| If the server scope is different, the client should not attempt to
| reclaim locks. In this situation, no lock reclaim is possible.
| Any attempt to re-obtain the locks with non-reclaim operations is
| problematic since there is no guarantee that the existing
| filehandles will be recognized by the new server, or that if
| recognized, they denote the same objects. It is best to treat the
| locks as having been revoked by the reconfiguration event.

That's why for lock reclaim to even be attempted, we have to define and set
the same server scope for NFSD on all cluster nodes in the NFS failover
cluster. And in linux, that is done by setting the uts nodename for the
command that starts the nfsd kernel threads.

For "init scripts", just set it directly using unshare --uts.
For systemd units, add NFS_SERVER_SCOPE to some environment files
and inject the "unshare --uts" into the ExecStart command lines
using override drop-in files.
---
 heartbeat/nfsserver | 120 +++++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 119 insertions(+), 1 deletion(-)

diff --git a/heartbeat/nfsserver b/heartbeat/nfsserver
index 96b19abe36..0888378645 100755
--- a/heartbeat/nfsserver
+++ b/heartbeat/nfsserver
@@ -5,6 +5,18 @@
 # by hxinwei@gmail.com
 # License: GNU General Public License v2 (GPLv2) and later
 
+
+# I don't know for certain whether all services actuall _need_ this,
+# I know that at least nfs-server needs it.
+# The rgmanager resource agent in rgmanager/src/resources/nfsserver.sh.in
+# did the unshare for gssd and idmapd as well, even though it seems unclear why.
+# Let's start with just the nfs-server, and add others if/when we have clear
+# indication they need it.
+#NFSD_RELATED_SYSTEMD_SERVICE_FOR_UNSHARE_UTS_NAMESPACE="nfs-idmapd.service nfs-mountd.service nfs-server.service nfsdcld.service rpc-gssd.service rpc-statd.service rpcbind.service"
+NFSD_RELATED_SYSTEMD_SERVICE_FOR_UNSHARE_UTS_NAMESPACE="nfs-server.service"
+SYSTEMD_ENVIRONMENT_FILE_NFS_SERVER_SCOPE=/run/sysconfig/nfs-server-scope
+SYSTEMD_UNSHARE_UTS_DROPIN=51-resource-agents-unshare-uts.conf
+
 if [ -n "$OCF_DEBUG_LIBRARY" ]; then
     . $OCF_DEBUG_LIBRARY
 else
@@ -99,6 +111,31 @@ Specifies the length of sm-notify retry time (minutes).
 <content type="integer" default="" />
 </parameter>
 
+<parameter name="nfs_server_scope" unique="0" required="0">
+<longdesc lang="en">
+RFC8881, 8.4.2.1 State Reclaim:
+
+If the server scope is different, the client should not attempt to
+reclaim locks. In this situation, no lock reclaim is possible.
+Any attempt to re-obtain the locks with non-reclaim operations is
+problematic since there is no guarantee that the existing
+filehandles will be recognized by the new server, or that if
+recognized, they denote the same objects. It is best to treat the
+locks as having been revoked by the reconfiguration event.
+
+For lock reclaim to even be attempted, we have to define and set the same
+server scope for NFSD on all cluster nodes in the NFS failover cluster.
+
+This agent won't "guess" a suitable server scope name for you, you need to
+explicitly specify this. But without it, NFSv4 lock reclaim after failover
+won't work properly.  Suggested value: the failover "service IP".
+</longdesc>
+<shortdesc lang="en">
+RFC8881 NFS server scope for (lock) state reclaim after failover.
+</shortdesc>
+<content type="string"/>
+</parameter>
+
 <parameter name="nfs_ip" unique="0" required="0">
 <longdesc lang="en">
 Comma separated list of floating IP addresses used to access the nfs service
@@ -269,7 +306,11 @@ nfs_exec()
 	set_exec_mode
 
 	case $EXEC_MODE in 
-		1) ${OCF_RESKEY_nfs_init_script} $cmd;;
+		1) if [ -z "$OCF_RESKEY_nfs_server_scope" ] ; then
+			${OCF_RESKEY_nfs_init_script} $cmd
+		   else
+			unshare -u /bin/sh -c "hostname ${OCF_RESKEY_nfs_server_scope}; exec ${OCF_RESKEY_nfs_init_script} $cmd"
+		   fi ;;
 		2) if ! echo $svc | grep -q "\."; then
 			svc="${svc}.service"
 		   fi
@@ -623,6 +664,74 @@ notify_locks()
 	fi
 }
 
+# Problem: https://github.com/ClusterLabs/resource-agents/issues/1644
+# RFC8881, 8.4.2.1 State Reclaim:
+#
+# | If the server scope is different, the client should not attempt to
+# | reclaim locks. In this situation, no lock reclaim is possible.
+# | Any attempt to re-obtain the locks with non-reclaim operations is
+# | problematic since there is no guarantee that the existing
+# | filehandles will be recognized by the new server, or that if
+# | recognized, they denote the same objects. It is best to treat the
+# | locks as having been revoked by the reconfiguration event.
+#
+# That's why for lock reclaim to even be attempted, we have to define and set
+# the same server scope for NFSD on all cluster nodes in the NFS failover
+# cluster. And in linux, that is done by setting the uts nodename for the
+# command that starts the nfsd kernel threads.
+#
+inject_unshare_uts_name_into_systemd_units ()
+{
+	local END_TAG="# END OF DROP-IN FOR NFS SERVER SCOPE"
+	local services
+	services=$(systemctl list-unit-files --no-legend $NFSD_RELATED_SYSTEMD_SERVICE_FOR_UNSHARE_UTS_NAMESPACE | cut -d ' ' -f1)
+
+	local svc dir dropin edited_exec_start do_reload=false
+	for svc in $services ; do
+		dir=/run/systemd/system/$svc.d
+		dropin=$dir/$SYSTEMD_UNSHARE_UTS_DROPIN
+		grep -sqF "$END_TAG" "$dropin" && continue
+
+		test -d "$dir" || mkdir -p "$dir"
+		test -e "$dropin" && rm -f "$dropin"
+
+		edited_exec_start=$(systemctl cat $svc | sed -ne "s#^ExecStart=\\(.*\\)#ExecStart=/usr/bin/unshare --uts /bin/sh -ec 'hostname \${NFS_SERVER_SCOPE}; exec \"\$@\"' -- \\1#p")
+		cat > "$dropin" <<___
+[Service]
+EnvironmentFile=$SYSTEMD_ENVIRONMENT_FILE_NFS_SERVER_SCOPE
+# reset list of exec start, then re-populate with unshared uts namespace
+ExecStart=
+$edited_exec_start
+$END_TAG
+___
+		do_reload=true
+		ocf_log debug "injected unshare --uts into $dropin"
+	done
+
+	mkdir -p "${SYSTEMD_ENVIRONMENT_FILE_NFS_SERVER_SCOPE%/*}"
+	echo "NFS_SERVER_SCOPE=$OCF_RESKEY_nfs_server_scope" > "$SYSTEMD_ENVIRONMENT_FILE_NFS_SERVER_SCOPE"
+
+	$do_reload && systemctl daemon-reload
+}
+
+remove_unshare_uts_dropins ()
+{
+	local services
+	services=$(systemctl list-unit-files --no-legend $NFSD_RELATED_SYSTEMD_SERVICE_FOR_UNSHARE_UTS_NAMESPACE)
+
+	local svc dir dropin do_reload=false
+	for svc in $services ; do
+		dir=/run/systemd/system/$svc.d
+		dropin=$dir/$SYSTEMD_UNSHARE_UTS_DROPIN
+		test -e "$dropin" || continue
+		rm -f "$dropin"
+		do_reload=true
+		ocf_log debug "removed unshare --uts from $svc"
+	done
+	rm -f "${SYSTEMD_ENVIRONMENT_FILE_NFS_SERVER_SCOPE}"
+	$do_reload && systemctl daemon-reload
+}
+
 nfsserver_start ()
 {
 	local rc;
@@ -636,6 +745,13 @@ nfsserver_start ()
 	is_redhat_based && set_env_args
 	bind_tree
 	prepare_directory
+	case $EXEC_MODE in [23])
+		if [ -z "$OCF_RESKEY_nfs_server_scope" ] ; then
+			remove_unshare_uts_dropins
+		else
+			inject_unshare_uts_name_into_systemd_units
+		fi ;;
+	esac
 
 	if ! `mount | grep -q " on $OCF_RESKEY_rpcpipefs_dir "`; then
 		mount -t rpc_pipefs sunrpc $OCF_RESKEY_rpcpipefs_dir
@@ -854,6 +970,8 @@ nfsserver_stop ()
 		ocf_log info "NFS server stopped"
 	fi
 
+	case $EXEC_MODE in [23]) remove_unshare_uts_dropins;; esac
+
 	return $rc
 }
 

From 515697b53c1614d05d39491c9af83e8d8b844b17 Mon Sep 17 00:00:00 2001
From: Lars Ellenberg <lars.ellenberg@linbit.com>
Date: Fri, 8 Oct 2021 12:01:41 +0200
Subject: [PATCH 2/6] Fix NFSv4 lock failover: set NFS Server Scope, regardless
 of EXEC_MODE

Debian (and other systems) may provide "init scripts",
which will only redirect back to systemd.

If we just unshare --uts the init script invocation,
the uts namespace is useless in that case.

If systemd is running, mangle the nfs-server.service unit,
independent of the "EXEC_MODE".
---
 heartbeat/nfsserver | 18 ++++++++++++++----
 1 file changed, 14 insertions(+), 4 deletions(-)

diff --git a/heartbeat/nfsserver b/heartbeat/nfsserver
index 0888378645..054aabbaf6 100755
--- a/heartbeat/nfsserver
+++ b/heartbeat/nfsserver
@@ -745,13 +745,20 @@ nfsserver_start ()
 	is_redhat_based && set_env_args
 	bind_tree
 	prepare_directory
-	case $EXEC_MODE in [23])
+
+	# Debian (and other systems) may provide "init scripts",
+	# which will only redirect back to systemd.
+	# If we just unshare --uts the init script invocation,
+	# the uts namespace is useless in that case.
+	# If systemd is running, mangle the nfs-server.service unit,
+	# independent of the "EXEC_MODE" we detected.
+	if $systemd_is_running ; then
 		if [ -z "$OCF_RESKEY_nfs_server_scope" ] ; then
 			remove_unshare_uts_dropins
 		else
 			inject_unshare_uts_name_into_systemd_units
-		fi ;;
-	esac
+		fi
+	fi
 
 	if ! `mount | grep -q " on $OCF_RESKEY_rpcpipefs_dir "`; then
 		mount -t rpc_pipefs sunrpc $OCF_RESKEY_rpcpipefs_dir
@@ -970,7 +977,9 @@ nfsserver_stop ()
 		ocf_log info "NFS server stopped"
 	fi
 
-	case $EXEC_MODE in [23]) remove_unshare_uts_dropins;; esac
+	if $systemd_is_running; then
+		remove_unshare_uts_dropins
+	fi
 
 	return $rc
 }
@@ -1008,6 +1017,7 @@ nfsserver_validate ()
 }
 
 nfsserver_validate
+systemd_is_running && systemd_is_running=true || systemd_is_running=false
 
 case $__OCF_ACTION in
 	start)      nfsserver_start

From e83c20d88f404f9f9d829c654883d60eb6cc9ff3 Mon Sep 17 00:00:00 2001
From: Lars Ellenberg <lars.ellenberg@linbit.com>
Date: Fri, 8 Oct 2021 17:06:18 +0200
Subject: [PATCH 3/6] Fix NFSv4 lock failover: add missing "|cut -f1" in
 remove_unshare_uts_dropins

---
 heartbeat/nfsserver | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/heartbeat/nfsserver b/heartbeat/nfsserver
index 054aabbaf6..d3db89a537 100755
--- a/heartbeat/nfsserver
+++ b/heartbeat/nfsserver
@@ -717,7 +717,7 @@ ___
 remove_unshare_uts_dropins ()
 {
 	local services
-	services=$(systemctl list-unit-files --no-legend $NFSD_RELATED_SYSTEMD_SERVICE_FOR_UNSHARE_UTS_NAMESPACE)
+	services=$(systemctl list-unit-files --no-legend $NFSD_RELATED_SYSTEMD_SERVICE_FOR_UNSHARE_UTS_NAMESPACE | cut -d ' ' -f1)
 
 	local svc dir dropin do_reload=false
 	for svc in $services ; do

From b5b0e4a0b60d285af576b2d8ecfbe95e5a177a87 Mon Sep 17 00:00:00 2001
From: Lars Ellenberg <lars.ellenberg@linbit.com>
Date: Fri, 8 Oct 2021 17:07:13 +0200
Subject: [PATCH 4/6] Fix NFSv4 lock failover: get rid of "world-inaccessible"
 warning

by temporarily changing the umask before generating the dropins
---
 heartbeat/nfsserver | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/heartbeat/nfsserver b/heartbeat/nfsserver
index d3db89a537..447e0302b2 100755
--- a/heartbeat/nfsserver
+++ b/heartbeat/nfsserver
@@ -687,6 +687,8 @@ inject_unshare_uts_name_into_systemd_units ()
 	services=$(systemctl list-unit-files --no-legend $NFSD_RELATED_SYSTEMD_SERVICE_FOR_UNSHARE_UTS_NAMESPACE | cut -d ' ' -f1)
 
 	local svc dir dropin edited_exec_start do_reload=false
+	local old_umask=$(umask)
+	umask 0022
 	for svc in $services ; do
 		dir=/run/systemd/system/$svc.d
 		dropin=$dir/$SYSTEMD_UNSHARE_UTS_DROPIN
@@ -710,6 +712,7 @@ ___
 
 	mkdir -p "${SYSTEMD_ENVIRONMENT_FILE_NFS_SERVER_SCOPE%/*}"
 	echo "NFS_SERVER_SCOPE=$OCF_RESKEY_nfs_server_scope" > "$SYSTEMD_ENVIRONMENT_FILE_NFS_SERVER_SCOPE"
+	umask $old_umask
 
 	$do_reload && systemctl daemon-reload
 }

From 3c6c91ce5a00eeef9cd766389d73a0b42580a1e6 Mon Sep 17 00:00:00 2001
From: Lars Ellenberg <lars.ellenberg@linbit.com>
Date: Fri, 8 Oct 2021 17:08:09 +0200
Subject: [PATCH 5/6] Fix NFSv4 lock failover: deal with "special executable
 prefix" chars in ExecStart

---
 heartbeat/nfsserver | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/heartbeat/nfsserver b/heartbeat/nfsserver
index 447e0302b2..5326bd2c6e 100755
--- a/heartbeat/nfsserver
+++ b/heartbeat/nfsserver
@@ -697,7 +697,7 @@ inject_unshare_uts_name_into_systemd_units ()
 		test -d "$dir" || mkdir -p "$dir"
 		test -e "$dropin" && rm -f "$dropin"
 
-		edited_exec_start=$(systemctl cat $svc | sed -ne "s#^ExecStart=\\(.*\\)#ExecStart=/usr/bin/unshare --uts /bin/sh -ec 'hostname \${NFS_SERVER_SCOPE}; exec \"\$@\"' -- \\1#p")
+		edited_exec_start=$(systemctl cat $svc | sed -ne "s#^ExecStart=\\([-+:!@]*\\)\\(.*\\)#ExecStart=\\1/usr/bin/unshare --uts /bin/sh -c 'hostname \${NFS_SERVER_SCOPE}; exec \"\$@\"' -- \\2#p")
 		cat > "$dropin" <<___
 [Service]
 EnvironmentFile=$SYSTEMD_ENVIRONMENT_FILE_NFS_SERVER_SCOPE

From 512fbaf61e6d24a1236ef50e323ea17a62485c36 Mon Sep 17 00:00:00 2001
From: Lars Ellenberg <lars.ellenberg@linbit.com>
Date: Fri, 8 Oct 2021 17:08:59 +0200
Subject: [PATCH 6/6] Fix NFSv4 lock failover: add rpc-statd-notify to the
 comment list of potentially interesting services

---
 heartbeat/nfsserver | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/heartbeat/nfsserver b/heartbeat/nfsserver
index 5326bd2c6e..240dd1a76c 100755
--- a/heartbeat/nfsserver
+++ b/heartbeat/nfsserver
@@ -12,7 +12,7 @@
 # did the unshare for gssd and idmapd as well, even though it seems unclear why.
 # Let's start with just the nfs-server, and add others if/when we have clear
 # indication they need it.
-#NFSD_RELATED_SYSTEMD_SERVICE_FOR_UNSHARE_UTS_NAMESPACE="nfs-idmapd.service nfs-mountd.service nfs-server.service nfsdcld.service rpc-gssd.service rpc-statd.service rpcbind.service"
+#NFSD_RELATED_SYSTEMD_SERVICE_FOR_UNSHARE_UTS_NAMESPACE="nfs-idmapd.service nfs-mountd.service nfs-server.service nfsdcld.service rpc-gssd.service rpc-statd.service rpc-statd-notify.service rpcbind.service"
 NFSD_RELATED_SYSTEMD_SERVICE_FOR_UNSHARE_UTS_NAMESPACE="nfs-server.service"
 SYSTEMD_ENVIRONMENT_FILE_NFS_SERVER_SCOPE=/run/sysconfig/nfs-server-scope
 SYSTEMD_UNSHARE_UTS_DROPIN=51-resource-agents-unshare-uts.conf