import device-mapper-multipath-0.8.4-37.el8

2023-03-28 08:31:37 +00:00 · 2023-03-28 08:31:37 +00:00 · 9e9d41a1be
commit 9e9d41a1be
parent 0c566e9b36
23 changed files with 3052 additions and 2 deletions
--- a/SOURCES/0110-multipath-fix-systemd-timers-in-the-initramfs.patch
+++ b/SOURCES/0110-multipath-fix-systemd-timers-in-the-initramfs.patch
@ -0,0 +1,35 @@
 From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
 From: Benjamin Marzinski <bmarzins@redhat.com>
 Date: Fri, 5 Aug 2022 18:16:03 -0500
 Subject: [PATCH] multipath: fix systemd timers in the initramfs
 The systemd timers created for "find_multipaths smart" conflict with
 shutdown.target, but not with initrd-cleanup.service.  This can make
 these timers trigger after the inirtd has started shutting down,
 restarting multipathd (which then stops initrd-cleanup.service, since it
 conflicts).  To avoid this, make sure the timers and the unit they
 trigger conflict with inird-cleanup.service. Also don't make them start
 multipathd.  "multipath -u" will not return "maybe" if multipathd isn't
 running or set to run, and since we no longer wait for udev-settle,
 multipathd starts up pretty quickly, so it shouldn't be a problem to
 not trigger it here.
 Signed-off-by: Benjamin Marzinski <bmarzins@redhat.com>
 Reviewed-by: Martin Wilck <mwilck@suse.com>
 ---
 multipath/multipath.rules | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
 diff --git a/multipath/multipath.rules b/multipath/multipath.rules
 index 0486bf70..68c30644 100644
 --- a/multipath/multipath.rules
 +++ b/multipath/multipath.rules
@@ -72,7 +72,7 @@ ENV{.SAVED_FM_WAIT_UNTIL}=="?*", GOTO="pretend_mpath"
 #
 # We must trigger an "add" event because LVM2 will only act on those.
 -RUN+="/usr/bin/systemd-run --unit=cancel-multipath-wait-$kernel --description 'cancel waiting for multipath siblings of $kernel' --no-block --timer-property DefaultDependencies=no --timer-property Conflicts=shutdown.target --timer-property Before=shutdown.target --timer-property AccuracySec=500ms --property DefaultDependencies=no --property Conflicts=shutdown.target --property Before=shutdown.target --property Wants=multipathd.service --property After=multipathd.service --on-active=$env{FIND_MULTIPATHS_WAIT_UNTIL} /usr/bin/udevadm trigger --action=add $sys$devpath"
 +RUN+="/usr/bin/systemd-run --unit=cancel-multipath-wait-$kernel --description 'cancel waiting for multipath siblings of $kernel' --no-block --timer-property DefaultDependencies=no --timer-property Conflicts=shutdown.target --timer-property Before=shutdown.target --timer-property Conflicts=initrd-cleanup.service --timer-property Before=initrd-cleanup.service --timer-property AccuracySec=500ms --property DefaultDependencies=no --property Conflicts=shutdown.target --property Before=shutdown.target --property Conflicts=initrd-cleanup.service --property Before=initrd-cleanup.service --on-active=$env{FIND_MULTIPATHS_WAIT_UNTIL} /usr/bin/udevadm trigger --action=add $sys$devpath"
 LABEL="pretend_mpath"
 ENV{DM_MULTIPATH_DEVICE_PATH}="1"
--- a/SOURCES/0111-multipathd-factor-out-the-code-to-flush-a-map-with-n.patch
+++ b/SOURCES/0111-multipathd-factor-out-the-code-to-flush-a-map-with-n.patch
@ -0,0 +1,90 @@
 From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
 From: Benjamin Marzinski <bmarzins@redhat.com>
 Date: Tue, 9 Aug 2022 16:46:26 -0500
 Subject: [PATCH] multipathd: factor out the code to flush a map with no paths
 The code to flush a multipath device because all of its paths have
 been removed will be used by another caller, so factor it out of
 ev_remove_path().
 Signed-off-by: Benjamin Marzinski <bmarzins@redhat.com>
 Reviewed-by: Martin Wilck <mwilck@suse.com>
 ---
 multipathd/main.c | 56 ++++++++++++++++++++++++-----------------------
 1 file changed, 29 insertions(+), 27 deletions(-)
 diff --git a/multipathd/main.c b/multipathd/main.c
 index a6ffbe32..9b1098f6 100644
 --- a/multipathd/main.c
 +++ b/multipathd/main.c
@@ -487,6 +487,30 @@ int update_multipath (struct vectors *vecs, char *mapname, int reset)
 	return 0;
 }
 +static bool
 +flush_map_nopaths(struct multipath *mpp, struct vectors *vecs) {
 +	char alias[WWID_SIZE];
 +
 +	/*
 +	 * flush_map will fail if the device is open
 +	 */
 +	strlcpy(alias, mpp->alias, WWID_SIZE);
 +	if (mpp->flush_on_last_del == FLUSH_ENABLED) {
 +		condlog(2, "%s Last path deleted, disabling queueing",
 +			mpp->alias);
 +		mpp->retry_tick = 0;
 +		mpp->no_path_retry = NO_PATH_RETRY_FAIL;
 +		mpp->disable_queueing = 1;
 +		mpp->stat_map_failures++;
 +		dm_queue_if_no_path(mpp->alias, 0);
 +	}
 +	if (!flush_map(mpp, vecs, 1)) {
 +		condlog(2, "%s: removed map after removing all paths", alias);
 +		return true;
 +	}
 +	return false;
 +}
 +
 static int
 update_map (struct multipath *mpp, struct vectors *vecs, int new_map)
 {
@@ -1185,34 +1209,12 @@ ev_remove_path (struct path *pp, struct vectors * vecs, int need_do_map)
 			vector_del_slot(mpp->paths, i);
 		/*
 -		 * remove the map IF removing the last path
 +		 * remove the map IF removing the last path. If
 +		 * flush_map_nopaths succeeds, the path has been removed.
 		 */
 -		if (VECTOR_SIZE(mpp->paths) == 0) {
 -			char alias[WWID_SIZE];
 -
 -			/*
 -			 * flush_map will fail if the device is open
 -			 */
 -			strlcpy(alias, mpp->alias, WWID_SIZE);
 -			if (mpp->flush_on_last_del == FLUSH_ENABLED) {
 -				condlog(2, "%s Last path deleted, disabling queueing", mpp->alias);
 -				mpp->retry_tick = 0;
 -				mpp->no_path_retry = NO_PATH_RETRY_FAIL;
 -				mpp->disable_queueing = 1;
 -				mpp->stat_map_failures++;
 -				dm_queue_if_no_path(mpp->alias, 0);
 -			}
 -			if (!flush_map(mpp, vecs, 1)) {
 -				condlog(2, "%s: removed map after"
 -					" removing all paths",
 -					alias);
 -				retval = 0;
 -				goto out;
 -			}
 -			/*
 -			 * Not an error, continue
 -			 */
 -		}
 +		if (VECTOR_SIZE(mpp->paths) == 0 &&
 +		    flush_map_nopaths(mpp, vecs))
 +			goto out;
 		if (mpp->hwe == NULL)
 			extract_hwe_from_path(mpp);
--- a/SOURCES/0112-libmultipath-return-success-if-we-raced-to-remove-a-.patch
+++ b/SOURCES/0112-libmultipath-return-success-if-we-raced-to-remove-a-.patch
@ -0,0 +1,47 @@
 From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
 From: Benjamin Marzinski <bmarzins@redhat.com>
 Date: Tue, 9 Aug 2022 16:46:27 -0500
 Subject: [PATCH] libmultipath: return success if we raced to remove a map and
 lost
 _dm_flush_map() was returning failure if it failed to remove a map,
 even if that was because the map had already been removed. Return
 success in this case.
 Signed-off-by: Benjamin Marzinski <bmarzins@redhat.com>
 Reviewed-by: Martin Wilck <mwilck@suse.com>
 ---
 libmultipath/devmapper.c | 4 ++++
 multipathd/main.c        | 4 ----
 2 files changed, 4 insertions(+), 4 deletions(-)
 diff --git a/libmultipath/devmapper.c b/libmultipath/devmapper.c
 index e3c3c119..ee81acc3 100644
 --- a/libmultipath/devmapper.c
 +++ b/libmultipath/devmapper.c
@@ -916,6 +916,10 @@ int _dm_flush_map (const char * mapname, int need_sync, int deferred_remove,
 			}
 			condlog(4, "multipath map %s removed", mapname);
 			return 0;
 +		} else if (dm_is_mpath(mapname) != 1) {
 +			condlog(4, "multipath map %s removed externally",
 +				mapname);
 +			return 0; /*we raced with someone else removing it */
 		} else {
 			condlog(2, "failed to remove multipath map %s",
 				mapname);
 diff --git a/multipathd/main.c b/multipathd/main.c
 index 9b1098f6..9eafd5b7 100644
 --- a/multipathd/main.c
 +++ b/multipathd/main.c
@@ -673,10 +673,6 @@ flush_map(struct multipath * mpp, struct vectors * vecs, int nopaths)
 	 * the spurious uevent we may generate with the dm_flush_map call below
 	 */
 	if (r) {
 -		/*
 -		 * May not really be an error -- if the map was already flushed
 -		 * from the device mapper by dmsetup(8) for instance.
 -		 */
 		if (r == 1)
 			condlog(0, "%s: can't flush", mpp->alias);
 		else {
--- a/SOURCES/0113-multipathd-Handle-losing-all-path-in-update_map.patch
+++ b/SOURCES/0113-multipathd-Handle-losing-all-path-in-update_map.patch
@ -0,0 +1,36 @@
 From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
 From: Benjamin Marzinski <bmarzins@redhat.com>
 Date: Tue, 9 Aug 2022 16:46:28 -0500
 Subject: [PATCH] multipathd: Handle losing all path in update_map
 Its possible that when a multipath device is being updated, it will end
 up that all the paths for it are gone. This can happen if paths are
 added and then removed again before multipathd processes the uevent for
 the newly created multipath device. In this case multipathd wasn't
 taking the proper action for the case where all the paths had been
 removed. If flush_on_last_del was set, multipathd wasn't disabling
 flushing and if deferred_remove was set, it wasn't doing a deferred
 remove. Multipathd should call flush_map_nopaths(), just like
 ev_remove_path() does when the last path is removed.
 Signed-off-by: Benjamin Marzinski <bmarzins@redhat.com>
 Reviewed-by: Martin Wilck <mwilck@suse.com>
 ---
 multipathd/main.c | 4 ++++
 1 file changed, 4 insertions(+)
 diff --git a/multipathd/main.c b/multipathd/main.c
 index 9eafd5b7..870ae7d8 100644
 --- a/multipathd/main.c
 +++ b/multipathd/main.c
@@ -526,6 +526,10 @@ retry:
 		goto fail;
 	}
 	verify_paths(mpp, vecs);
 +	if (VECTOR_SIZE(mpp->paths) == 0 &&
 +	    flush_map_nopaths(mpp, vecs))
 +		return 1;
 +
 	mpp->action = ACT_RELOAD;
 	if (setup_map(mpp, params, PARAMS_SIZE, vecs)) {
--- a/SOURCES/0114-multipathd-ignore-duplicated-multipathd-command-keys.patch
+++ b/SOURCES/0114-multipathd-ignore-duplicated-multipathd-command-keys.patch
@ -0,0 +1,153 @@
 From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
 From: Benjamin Marzinski <bmarzins@redhat.com>
 Date: Thu, 8 Sep 2022 11:54:49 -0500
 Subject: [PATCH] multipathd: ignore duplicated multipathd command keys
 multipath adds rather than or-s the values of command keys. Fix this.
 Also, return an invalid fingerprint if a key is used more than once.
 Signed-off-by: Benjamin Marzinski <bmarzins@redhat.com>
 ---
 multipathd/cli.c  |   8 ++--
 multipathd/main.c | 106 +++++++++++++++++++++++-----------------------
 2 files changed, 58 insertions(+), 56 deletions(-)
 diff --git a/multipathd/cli.c b/multipathd/cli.c
 index 85d73dfb..45e80197 100644
 --- a/multipathd/cli.c
 +++ b/multipathd/cli.c
@@ -341,9 +341,11 @@ fingerprint(vector vec)
 	if (!vec)
 		return 0;
 -	vector_foreach_slot(vec, kw, i)
 -		fp += kw->code;
 -
 +	vector_foreach_slot(vec, kw, i) {
 +		if (fp & kw->code)
 +			return (uint64_t)-1;
 +		fp |= kw->code;
 +	}
 	return fp;
 }
 diff --git a/multipathd/main.c b/multipathd/main.c
 index 870ae7d8..cd68a9d2 100644
 --- a/multipathd/main.c
 +++ b/multipathd/main.c
@@ -1628,62 +1628,62 @@ uxlsnrloop (void * ap)
 	/* Tell main thread that thread has started */
 	post_config_state(DAEMON_CONFIGURE);
 -	set_handler_callback(LIST+PATHS, cli_list_paths);
 -	set_handler_callback(LIST+PATHS+FMT, cli_list_paths_fmt);
 -	set_handler_callback(LIST+PATHS+RAW+FMT, cli_list_paths_raw);
 -	set_handler_callback(LIST+PATH, cli_list_path);
 -	set_handler_callback(LIST+MAPS, cli_list_maps);
 -	set_handler_callback(LIST+STATUS, cli_list_status);
 -	set_unlocked_handler_callback(LIST+DAEMON, cli_list_daemon);
 -	set_handler_callback(LIST+MAPS+STATUS, cli_list_maps_status);
 -	set_handler_callback(LIST+MAPS+STATS, cli_list_maps_stats);
 -	set_handler_callback(LIST+MAPS+FMT, cli_list_maps_fmt);
 -	set_handler_callback(LIST+MAPS+RAW+FMT, cli_list_maps_raw);
 -	set_handler_callback(LIST+MAPS+TOPOLOGY, cli_list_maps_topology);
 -	set_handler_callback(LIST+TOPOLOGY, cli_list_maps_topology);
 -	set_handler_callback(LIST+MAPS+JSON, cli_list_maps_json);
 -	set_handler_callback(LIST+MAP+TOPOLOGY, cli_list_map_topology);
 -	set_handler_callback(LIST+MAP+FMT, cli_list_map_fmt);
 -	set_handler_callback(LIST+MAP+RAW+FMT, cli_list_map_fmt);
 -	set_handler_callback(LIST+MAP+JSON, cli_list_map_json);
 -	set_handler_callback(LIST+CONFIG+LOCAL, cli_list_config_local);
 -	set_handler_callback(LIST+CONFIG, cli_list_config);
 -	set_handler_callback(LIST+BLACKLIST, cli_list_blacklist);
 -	set_handler_callback(LIST+DEVICES, cli_list_devices);
 -	set_handler_callback(LIST+WILDCARDS, cli_list_wildcards);
 -	set_handler_callback(RESET+MAPS+STATS, cli_reset_maps_stats);
 -	set_handler_callback(RESET+MAP+STATS, cli_reset_map_stats);
 -	set_handler_callback(ADD+PATH, cli_add_path);
 -	set_handler_callback(DEL+PATH, cli_del_path);
 -	set_handler_callback(ADD+MAP, cli_add_map);
 -	set_handler_callback(DEL+MAP, cli_del_map);
 -	set_handler_callback(DEL+MAPS, cli_del_maps);
 -	set_handler_callback(SWITCH+MAP+GROUP, cli_switch_group);
 +	set_handler_callback(LIST|PATHS, cli_list_paths);
 +	set_handler_callback(LIST|PATHS|FMT, cli_list_paths_fmt);
 +	set_handler_callback(LIST|PATHS|RAW|FMT, cli_list_paths_raw);
 +	set_handler_callback(LIST|PATH, cli_list_path);
 +	set_handler_callback(LIST|MAPS, cli_list_maps);
 +	set_handler_callback(LIST|STATUS, cli_list_status);
 +	set_unlocked_handler_callback(LIST|DAEMON, cli_list_daemon);
 +	set_handler_callback(LIST|MAPS|STATUS, cli_list_maps_status);
 +	set_handler_callback(LIST|MAPS|STATS, cli_list_maps_stats);
 +	set_handler_callback(LIST|MAPS|FMT, cli_list_maps_fmt);
 +	set_handler_callback(LIST|MAPS|RAW|FMT, cli_list_maps_raw);
 +	set_handler_callback(LIST|MAPS|TOPOLOGY, cli_list_maps_topology);
 +	set_handler_callback(LIST|TOPOLOGY, cli_list_maps_topology);
 +	set_handler_callback(LIST|MAPS|JSON, cli_list_maps_json);
 +	set_handler_callback(LIST|MAP|TOPOLOGY, cli_list_map_topology);
 +	set_handler_callback(LIST|MAP|FMT, cli_list_map_fmt);
 +	set_handler_callback(LIST|MAP|RAW|FMT, cli_list_map_fmt);
 +	set_handler_callback(LIST|MAP|JSON, cli_list_map_json);
 +	set_handler_callback(LIST|CONFIG|LOCAL, cli_list_config_local);
 +	set_handler_callback(LIST|CONFIG, cli_list_config);
 +	set_handler_callback(LIST|BLACKLIST, cli_list_blacklist);
 +	set_handler_callback(LIST|DEVICES, cli_list_devices);
 +	set_handler_callback(LIST|WILDCARDS, cli_list_wildcards);
 +	set_handler_callback(RESET|MAPS|STATS, cli_reset_maps_stats);
 +	set_handler_callback(RESET|MAP|STATS, cli_reset_map_stats);
 +	set_handler_callback(ADD|PATH, cli_add_path);
 +	set_handler_callback(DEL|PATH, cli_del_path);
 +	set_handler_callback(ADD|MAP, cli_add_map);
 +	set_handler_callback(DEL|MAP, cli_del_map);
 +	set_handler_callback(DEL|MAPS, cli_del_maps);
 +	set_handler_callback(SWITCH|MAP|GROUP, cli_switch_group);
 	set_unlocked_handler_callback(RECONFIGURE, cli_reconfigure);
 -	set_handler_callback(SUSPEND+MAP, cli_suspend);
 -	set_handler_callback(RESUME+MAP, cli_resume);
 -	set_handler_callback(RESIZE+MAP, cli_resize);
 -	set_handler_callback(RELOAD+MAP, cli_reload);
 -	set_handler_callback(RESET+MAP, cli_reassign);
 -	set_handler_callback(REINSTATE+PATH, cli_reinstate);
 -	set_handler_callback(FAIL+PATH, cli_fail);
 -	set_handler_callback(DISABLEQ+MAP, cli_disable_queueing);
 -	set_handler_callback(RESTOREQ+MAP, cli_restore_queueing);
 -	set_handler_callback(DISABLEQ+MAPS, cli_disable_all_queueing);
 -	set_handler_callback(RESTOREQ+MAPS, cli_restore_all_queueing);
 +	set_handler_callback(SUSPEND|MAP, cli_suspend);
 +	set_handler_callback(RESUME|MAP, cli_resume);
 +	set_handler_callback(RESIZE|MAP, cli_resize);
 +	set_handler_callback(RELOAD|MAP, cli_reload);
 +	set_handler_callback(RESET|MAP, cli_reassign);
 +	set_handler_callback(REINSTATE|PATH, cli_reinstate);
 +	set_handler_callback(FAIL|PATH, cli_fail);
 +	set_handler_callback(DISABLEQ|MAP, cli_disable_queueing);
 +	set_handler_callback(RESTOREQ|MAP, cli_restore_queueing);
 +	set_handler_callback(DISABLEQ|MAPS, cli_disable_all_queueing);
 +	set_handler_callback(RESTOREQ|MAPS, cli_restore_all_queueing);
 	set_unlocked_handler_callback(QUIT, cli_quit);
 	set_unlocked_handler_callback(SHUTDOWN, cli_shutdown);
 -	set_handler_callback(GETPRSTATUS+MAP, cli_getprstatus);
 -	set_handler_callback(SETPRSTATUS+MAP, cli_setprstatus);
 -	set_handler_callback(UNSETPRSTATUS+MAP, cli_unsetprstatus);
 -	set_handler_callback(FORCEQ+DAEMON, cli_force_no_daemon_q);
 -	set_handler_callback(RESTOREQ+DAEMON, cli_restore_no_daemon_q);
 -	set_handler_callback(GETPRKEY+MAP, cli_getprkey);
 -	set_handler_callback(SETPRKEY+MAP+KEY, cli_setprkey);
 -	set_handler_callback(UNSETPRKEY+MAP, cli_unsetprkey);
 -	set_handler_callback(SETMARGINAL+PATH, cli_set_marginal);
 -	set_handler_callback(UNSETMARGINAL+PATH, cli_unset_marginal);
 -	set_handler_callback(UNSETMARGINAL+MAP, cli_unset_all_marginal);
 +	set_handler_callback(GETPRSTATUS|MAP, cli_getprstatus);
 +	set_handler_callback(SETPRSTATUS|MAP, cli_setprstatus);
 +	set_handler_callback(UNSETPRSTATUS|MAP, cli_unsetprstatus);
 +	set_handler_callback(FORCEQ|DAEMON, cli_force_no_daemon_q);
 +	set_handler_callback(RESTOREQ|DAEMON, cli_restore_no_daemon_q);
 +	set_handler_callback(GETPRKEY|MAP, cli_getprkey);
 +	set_handler_callback(SETPRKEY|MAP|KEY, cli_setprkey);
 +	set_handler_callback(UNSETPRKEY|MAP, cli_unsetprkey);
 +	set_handler_callback(SETMARGINAL|PATH, cli_set_marginal);
 +	set_handler_callback(UNSETMARGINAL|PATH, cli_unset_marginal);
 +	set_handler_callback(UNSETMARGINAL|MAP, cli_unset_all_marginal);
 	umask(077);
 	uxsock_listen(&uxsock_trigger, ux_sock, ap);
--- a/SOURCES/0115-multipath-tools-use-run-instead-of-dev-shm.patch
+++ b/SOURCES/0115-multipath-tools-use-run-instead-of-dev-shm.patch
@ -0,0 +1,141 @@
 From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
 From: Martin Wilck <mwilck@suse.com>
 Date: Thu, 1 Sep 2022 19:21:30 +0200
 Subject: [PATCH] multipath-tools: use /run instead of /dev/shm
 /dev/shm may have unsafe permissions. Use /run instead.
 Use systemd's tmpfiles.d mechanism to create /run/multipath
 early during boot.
 For backward compatibilty, make the runtime directory configurable
 via the "runtimedir" make variable.
 Signed-off-by: Martin Wilck <mwilck@suse.com>
 Signed-off-by: Benjamin Marzinski <bmarzins@redhat.com>
 ---
 .gitignore                                        | 2 ++
 Makefile.inc                                      | 4 +++-
 libmultipath/defaults.h                           | 2 +-
 multipath/Makefile                                | 9 +++++++--
 multipath/{multipath.rules => multipath.rules.in} | 4 ++--
 multipath/tmpfiles.conf.in                        | 1 +
 6 files changed, 16 insertions(+), 6 deletions(-)
 rename multipath/{multipath.rules => multipath.rules.in} (96%)
 create mode 100644 multipath/tmpfiles.conf.in
 diff --git a/.gitignore b/.gitignore
 index 9926756b..f90b0350 100644
 --- a/.gitignore
 +++ b/.gitignore
@@ -8,6 +8,8 @@
 *.d
 kpartx/kpartx
 multipath/multipath
 +multipath/multipath.rules
 +multipath/tmpfiles.conf
 multipathd/multipathd
 mpathpersist/mpathpersist
 .nfs*
 diff --git a/Makefile.inc b/Makefile.inc
 index d471f045..287f0e7b 100644
 --- a/Makefile.inc
 +++ b/Makefile.inc
@@ -60,6 +60,7 @@ exec_prefix	= $(prefix)
 usr_prefix	= $(prefix)
 bindir		= $(exec_prefix)/usr/sbin
 libudevdir	= $(prefix)/$(SYSTEMDPATH)/udev
 +tmpfilesdir	= $(prefix)/$(SYSTEMDPATH)/tmpfiles.d
 udevrulesdir	= $(libudevdir)/rules.d
 multipathdir	= $(TOPDIR)/libmultipath
 man8dir		= $(prefix)/usr/share/man/man8
@@ -76,6 +77,7 @@ libdmmpdir	= $(TOPDIR)/libdmmp
 nvmedir		= $(TOPDIR)/libmultipath/nvme
 includedir	= $(prefix)/usr/include
 pkgconfdir	= $(usrlibdir)/pkgconfig
 +runtimedir	= /$(RUN)
 GZIP		= gzip -9 -c
 RM		= rm -f
@@ -117,7 +119,7 @@ OPTFLAGS	+= -Werror -Wextra -Wstrict-prototypes -Wformat=2 \
 		   -Werror=cast-qual $(ERROR_DISCARDED_QUALIFIERS) \
 		   --param=ssp-buffer-size=4
 -CFLAGS		:= $(OPTFLAGS) -DBIN_DIR=\"$(bindir)\" -DLIB_STRING=\"${LIB}\" -DRUN_DIR=\"${RUN}\" \
 +CFLAGS		:= $(OPTFLAGS) -DBIN_DIR=\"$(bindir)\" -DLIB_STRING=\"${LIB}\" -DRUN_DIR=\"${RUN}\" -DRUNTIME_DIR=\"$(runtimedir)\" \
 		   -MMD -MP $(CFLAGS)
 BIN_CFLAGS	= -fPIE -DPIE
 LIB_CFLAGS	= -fPIC
 diff --git a/libmultipath/defaults.h b/libmultipath/defaults.h
 index f1cb000d..dcd9f5a7 100644
 --- a/libmultipath/defaults.h
 +++ b/libmultipath/defaults.h
@@ -65,7 +65,7 @@
 #define DEFAULT_WWIDS_FILE	"/etc/multipath/wwids"
 #define DEFAULT_PRKEYS_FILE    "/etc/multipath/prkeys"
 #define DEFAULT_CONFIG_DIR	"/etc/multipath/conf.d"
 -#define MULTIPATH_SHM_BASE	"/dev/shm/multipath/"
 +#define MULTIPATH_SHM_BASE	RUNTIME_DIR "/multipath/"
 static inline char *set_default(char *str)
 diff --git a/multipath/Makefile b/multipath/Makefile
 index e720c7f6..f3d98012 100644
 --- a/multipath/Makefile
 +++ b/multipath/Makefile
@@ -12,7 +12,7 @@ EXEC = multipath
 OBJS = main.o
 -all: $(EXEC)
 +all: $(EXEC) multipath.rules tmpfiles.conf
 $(EXEC): $(OBJS) $(multipathdir)/libmultipath.so $(mpathcmddir)/libmpathcmd.so
 	$(CC) $(CFLAGS) $(OBJS) -o $(EXEC) $(LDFLAGS) $(LIBDEPS)
@@ -27,6 +27,8 @@ install:
 	$(INSTALL_PROGRAM) -d $(DESTDIR)$(udevrulesdir)
 	$(INSTALL_PROGRAM) -m 644 11-dm-mpath.rules $(DESTDIR)$(udevrulesdir)
 	$(INSTALL_PROGRAM) -m 644 $(EXEC).rules $(DESTDIR)$(libudevdir)/rules.d/62-multipath.rules
 +	$(INSTALL_PROGRAM) -d $(DESTDIR)$(tmpfilesdir)
 +	$(INSTALL_PROGRAM) -m 644 tmpfiles.conf $(DESTDIR)$(tmpfilesdir)/multipath.conf
 	$(INSTALL_PROGRAM) -d $(DESTDIR)$(man8dir)
 	$(INSTALL_PROGRAM) -m 644 $(EXEC).8.gz $(DESTDIR)$(man8dir)
 	$(INSTALL_PROGRAM) -d $(DESTDIR)$(man5dir)
@@ -43,9 +45,12 @@ uninstall:
 	$(RM) $(DESTDIR)$(man8dir)/mpathconf.8.gz
 clean: dep_clean
 -	$(RM) core *.o $(EXEC) *.gz
 +	$(RM) core *.o $(EXEC) *.gz multipath.rules tmpfiles.conf
 include $(wildcard $(OBJS:.o=.d))
 dep_clean:
 	$(RM) $(OBJS:.o=.d)
 +
 +%: %.in
 +	sed 's,@RUNTIME_DIR@,$(runtimedir),' $< >$@
 diff --git a/multipath/multipath.rules b/multipath/multipath.rules.in
 similarity index 96%
 rename from multipath/multipath.rules
 rename to multipath/multipath.rules.in
 index 68c30644..5c4447a2 100644
 --- a/multipath/multipath.rules
 +++ b/multipath/multipath.rules.in
@@ -1,8 +1,8 @@
 # Set DM_MULTIPATH_DEVICE_PATH if the device should be handled by multipath
 SUBSYSTEM!="block", GOTO="end_mpath"
 KERNEL!="sd*|dasd*|nvme*", GOTO="end_mpath"
 -ACTION=="remove", TEST=="/dev/shm/multipath/find_multipaths/$major:$minor", \
 -	RUN+="/usr/bin/rm -f /dev/shm/multipath/find_multipaths/$major:$minor"
 +ACTION=="remove", TEST=="@RUNTIME_DIR@/multipath/find_multipaths/$major:$minor", \
 +	RUN+="/usr/bin/rm -f @RUNTIME_DIR@/multipath/find_multipaths/$major:$minor"
 ACTION!="add|change", GOTO="end_mpath"
 IMPORT{cmdline}="nompath"
 diff --git a/multipath/tmpfiles.conf.in b/multipath/tmpfiles.conf.in
 new file mode 100644
 index 00000000..21be438a
 --- /dev/null
 +++ b/multipath/tmpfiles.conf.in
@@ -0,0 +1 @@
 +d @RUNTIME_DIR@/multipath 0700 root root -
--- a/SOURCES/0116-kpartx-hold-device-open-until-partitions-have-been-c.patch
+++ b/SOURCES/0116-kpartx-hold-device-open-until-partitions-have-been-c.patch
@ -0,0 +1,47 @@
 From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
 From: Benjamin Marzinski <bmarzins@redhat.com>
 Date: Wed, 19 Oct 2022 12:57:10 -0500
 Subject: [PATCH] kpartx: hold device open until partitions have been created
 kpartx was closing the whole device after it read the partition
 information off it.  This allowed a race, where the device could be
 removed and another one created with the same major:minor, after kpartx
 read the partition information but before it created the partition
 devices.
 Signed-off-by: Benjamin Marzinski <bmarzins@redhat.com>
 Reviewed-by: Martin Wilck <mwilck@suse.com>
 ---
 kpartx/kpartx.c | 11 +++--------
 1 file changed, 3 insertions(+), 8 deletions(-)
 diff --git a/kpartx/kpartx.c b/kpartx/kpartx.c
 index a337a07b..e62b764f 100644
 --- a/kpartx/kpartx.c
 +++ b/kpartx/kpartx.c
@@ -426,12 +426,7 @@ main(int argc, char **argv){
 		if (n >= 0)
 			printf("%s: %d slices\n", ptp->type, n);
 #endif
 -
 -		if (n > 0) {
 -			close(fd);
 -			fd = -1;
 -		}
 -		else
 +		if (n <= 0)
 			continue;
 		switch(what) {
@@ -649,9 +644,9 @@ main(int argc, char **argv){
 		if (n > 0)
 			break;
 	}
 +	if (fd != -1)
 +		close(fd);
 	if (what == LIST && loopcreated) {
 -		if (fd != -1)
 -			close(fd);
 		if (del_loop(device)) {
 			if (verbose)
 				fprintf(stderr, "can't del loop : %s\n",
--- a/SOURCES/0117-libmultipath-cleanup-remove_feature.patch
+++ b/SOURCES/0117-libmultipath-cleanup-remove_feature.patch
@ -0,0 +1,153 @@
 From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
 From: Benjamin Marzinski <bmarzins@redhat.com>
 Date: Fri, 7 Oct 2022 12:35:37 -0500
 Subject: [PATCH] libmultipath: cleanup remove_feature
 remove_feature() didn't correctly handle feature strings that used
 whitespace other than spaces, which the kernel allows. It also didn't
 check if the feature string to be removed was part of a larger feature
 token. Finally, it did a lot of unnecessary work. By failing if the
 feature string to be removed contains leading or trailing whitespace,
 the function can be significanly simplified.
 Signed-off-by: Benjamin Marzinski <bmarzins@redhat.com>
 Reviewed-by: Martin Wilck <mwilck@suse.com>
 ---
 libmultipath/structs.c | 82 +++++++++++++++---------------------------
 1 file changed, 29 insertions(+), 53 deletions(-)
 diff --git a/libmultipath/structs.c b/libmultipath/structs.c
 index 9f86eb69..471087e2 100644
 --- a/libmultipath/structs.c
 +++ b/libmultipath/structs.c
@@ -6,6 +6,7 @@
 #include <unistd.h>
 #include <libdevmapper.h>
 #include <libudev.h>
 +#include <ctype.h>
 #include "checkers.h"
 #include "memory.h"
@@ -633,7 +634,7 @@ int add_feature(char **f, const char *n)
 int remove_feature(char **f, const char *o)
 {
 -	int c = 0, d, l;
 +	int c = 0, d;
 	char *e, *p, *n;
 	const char *q;
@@ -644,33 +645,35 @@ int remove_feature(char **f, const char *o)
 	if (!o || *o == '\0')
 		return 0;
 -	/* Check if not present */
 -	if (!strstr(*f, o))
 +	d = strlen(o);
 +	if (isspace(*o) || isspace(*(o + d - 1))) {
 +		condlog(0, "internal error: feature \"%s\" has leading or trailing spaces", o);
 +		return 1;
 +	}
 +
 +	/* Check if present and not part of a larger feature token*/
 +	p = *f + 1; /* the size must be at the start of the features string */
 +	while ((p = strstr(p, o)) != NULL) {
 +		if (isspace(*(p - 1)) &&
 +		    (isspace(*(p + d)) || *(p + d) == '\0'))
 +			break;
 +		p += d;
 +	}
 +	if (!p)
 		return 0;
 	/* Get feature count */
 	c = strtoul(*f, &e, 10);
 -	if (*f == e)
 -		/* parse error */
 +	if (*f == e || !isspace(*e)) {
 +		condlog(0, "parse error in feature string \"%s\"", *f);
 		return 1;
 -
 -	/* Normalize features */
 -	while (*o == ' ') {
 -		o++;
 	}
 -	/* Just spaces, return */
 -	if (*o == '\0')
 -		return 0;
 -	q = o + strlen(o);
 -	while (*q == ' ')
 -		q--;
 -	d = (int)(q - o);
 	/* Update feature count */
 	c--;
 	q = o;
 -	while (q[0] != '\0') {
 -		if (q[0] == ' ' && q[1] != ' ' && q[1] != '\0')
 +	while (*q != '\0') {
 +		if (isspace(*q) && !isspace(*(q + 1)) && *(q + 1) != '\0')
 			c--;
 		q++;
 	}
@@ -684,15 +687,8 @@ int remove_feature(char **f, const char *o)
 		goto out;
 	}
 -	/* Search feature to be removed */
 -	e = strstr(*f, o);
 -	if (!e)
 -		/* Not found, return */
 -		return 0;
 -
 	/* Update feature count space */
 -	l = strlen(*f) - d;
 -	n =  MALLOC(l + 1);
 +	n =  MALLOC(strlen(*f) - d + 1);
 	if (!n)
 		return 1;
@@ -702,36 +698,16 @@ int remove_feature(char **f, const char *o)
 	 * Copy existing features up to the feature
 	 * about to be removed
 	 */
 -	p = strchr(*f, ' ');
 -	if (!p) {
 -		/* Internal error, feature string inconsistent */
 -		FREE(n);
 -		return 1;
 -	}
 -	while (*p == ' ')
 -		p++;
 -	p--;
 -	if (e != p) {
 -		do {
 -			e--;
 -			d++;
 -		} while (*e == ' ');
 -		e++; d--;
 -		strncat(n, p, (size_t)(e - p));
 -		p += (size_t)(e - p);
 -	}
 +	strncat(n, e, (size_t)(p - e));
 	/* Skip feature to be removed */
 	p += d;
 -
 	/* Copy remaining features */
 -	if (strlen(p)) {
 -		while (*p == ' ')
 -			p++;
 -		if (strlen(p)) {
 -			p--;
 -			strcat(n, p);
 -		}
 -	}
 +	while (isspace(*p))
 +		p++;
 +	if (*p != '\0')
 +		strcat(n, p);
 +	else
 +		strchop(n);
 out:
 	FREE(*f);
--- a/SOURCES/0118-libmultipath-cleanup-add_feature.patch
+++ b/SOURCES/0118-libmultipath-cleanup-add_feature.patch
@ -0,0 +1,107 @@
 From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
 From: Benjamin Marzinski <bmarzins@redhat.com>
 Date: Fri, 7 Oct 2022 12:35:38 -0500
 Subject: [PATCH] libmultipath: cleanup add_feature
 add_feature() didn't correctly handle feature strings that used
 whitespace other than spaces, which the kernel allows. It also didn't
 allow adding features with multiple tokens. When it looked to see if the
 feature string to be added already existed, it didn't check if the match
 was part of a larger token. Finally, it did unnecessary work.  By using
 asprintf() to create the string, the function can be signifcantly
 simplified.
 Signed-off-by: Benjamin Marzinski <bmarzins@redhat.com>
 Reviewed-by: Martin Wilck <mwilck@suse.com>
 ---
 libmultipath/structs.c | 49 +++++++++++++++++++++---------------------
 1 file changed, 24 insertions(+), 25 deletions(-)
 diff --git a/libmultipath/structs.c b/libmultipath/structs.c
 index 471087e2..84f9c959 100644
 --- a/libmultipath/structs.c
 +++ b/libmultipath/structs.c
@@ -572,23 +572,33 @@ int add_feature(char **f, const char *n)
 {
 	int c = 0, d, l;
 	char *e, *t;
 +	const char *p;
 	if (!f)
 		return 1;
 	/* Nothing to do */
 -	if (!n || *n == '0')
 +	if (!n || *n == '\0')
 		return 0;
 -	if (strchr(n, ' ') != NULL) {
 -		condlog(0, "internal error: feature \"%s\" contains spaces", n);
 +	l = strlen(n);
 +	if (isspace(*n) || isspace(*(n + l - 1))) {
 +		condlog(0, "internal error: feature \"%s\" has leading or trailing spaces", n);
 		return 1;
 	}
 +	p = n;
 +	d = 1;
 +	while (*p != '\0') {
 +		if (isspace(*p) && !isspace(*(p + 1)) && *(p + 1) != '\0')
 +			d++;
 +		p++;
 +	}
 +
 	/* default feature is null */
 	if(!*f)
 	{
 -		l = asprintf(&t, "1 %s", n);
 +		l = asprintf(&t, "%0d %s", d, n);
 		if(l == -1)
 			return 1;
@@ -597,35 +607,24 @@ int add_feature(char **f, const char *n)
 	}
 	/* Check if feature is already present */
 -	if (strstr(*f, n))
 -		return 0;
 +	e = *f;
 +	while ((e = strstr(e, n)) != NULL) {
 +		if (isspace(*(e - 1)) &&
 +		    (isspace(*(e + l)) || *(e + l) == '\0'))
 +			return 0;
 +		e += l;
 +	}
 	/* Get feature count */
 	c = strtoul(*f, &e, 10);
 -	if (*f == e || (*e != ' ' && *e != '\0')) {
 +	if (*f == e || (!isspace(*e) && *e != '\0')) {
 		condlog(0, "parse error in feature string \"%s\"", *f);
 		return 1;
 	}
 -
 -	/* Add 1 digit and 1 space */
 -	l = strlen(e) + strlen(n) + 2;
 -
 -	c++;
 -	/* Check if we need more digits for feature count */
 -	for (d = c; d >= 10; d /= 10)
 -		l++;
 -
 -	t = MALLOC(l + 1);
 -	if (!t)
 +	c += d;
 +	if (asprintf(&t, "%0d%s %s", c, e, n) < 0)
 		return 1;
 -	/* e: old feature string with leading space, or "" */
 -	if (*e == ' ')
 -		while (*(e + 1) == ' ')
 -			e++;
 -
 -	snprintf(t, l + 1, "%0d%s %s", c, e, n);
 -
 	FREE(*f);
 	*f = t;
--- a/SOURCES/0119-multipath-tests-tests-for-adding-and-removing-featur.patch
+++ b/SOURCES/0119-multipath-tests-tests-for-adding-and-removing-featur.patch
@ -0,0 +1,350 @@
 From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
 From: Benjamin Marzinski <bmarzins@redhat.com>
 Date: Fri, 7 Oct 2022 12:35:39 -0500
 Subject: [PATCH] multipath tests: tests for adding and removing features
 Signed-off-by: Benjamin Marzinski <bmarzins@redhat.com>
 Reviewed-by: Martin Wilck <mwilck@suse.com>
 ---
 tests/Makefile   |   2 +-
 tests/features.c | 318 +++++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 319 insertions(+), 1 deletion(-)
 create mode 100644 tests/features.c
 diff --git a/tests/Makefile b/tests/Makefile
 index 77ff3249..914413b8 100644
 --- a/tests/Makefile
 +++ b/tests/Makefile
@@ -13,7 +13,7 @@ CFLAGS += $(BIN_CFLAGS) -I$(multipathdir) -I$(mpathcmddir) \
 LIBDEPS += -L$(multipathdir) -lmultipath -lcmocka
 TESTS := uevent parser util dmevents hwtable blacklist unaligned vpd pgpolicy \
 -	 alias directio
 +	 alias directio features
 .SILENT: $(TESTS:%=%.o)
 .PRECIOUS: $(TESTS:%=%-test)
 diff --git a/tests/features.c b/tests/features.c
 new file mode 100644
 index 00000000..1e2e6bff
 --- /dev/null
 +++ b/tests/features.c
@@ -0,0 +1,318 @@
 +#include <stddef.h>
 +#include <stdarg.h>
 +#include <setjmp.h>
 +#include <cmocka.h>
 +
 +#include "structs.h"
 +#include "globals.c"
 +
 +static void test_af_null_features_ptr(void **state)
 +{
 +	assert_int_equal(add_feature(NULL, "test"), 1);
 +}
 +
 +static void af_helper(const char *features_start, const char *addition,
 +		      const char *features_end, int result)
 +{
 +	char *f = NULL, *orig = NULL;
 +
 +	if (features_start) {
 +		f = strdup(features_start);
 +		assert_non_null(f);
 +		orig = f;
 +	}
 +	assert_int_equal(add_feature(&f, addition), result);
 +	if (result != 0 || features_end == NULL)
 +		assert_ptr_equal(orig, f);
 +	else
 +		assert_string_equal(f, features_end);
 +	free(f);
 +}
 +
 +static void test_af_null_addition1(void **state)
 +{
 +	af_helper("0", NULL, NULL, 0);
 +}
 +
 +static void test_af_null_addition2(void **state)
 +{
 +	af_helper("1 queue_if_no_path", NULL, NULL, 0);
 +}
 +
 +static void test_af_empty_addition(void **state)
 +{
 +	af_helper("2 pg_init_retries 5", "", NULL, 0);
 +}
 +
 +static void test_af_invalid_addition1(void **state)
 +{
 +	af_helper("2 pg_init_retries 5", " ", NULL, 1);
 +}
 +
 +static void test_af_invalid_addition2(void **state)
 +{
 +	af_helper("2 pg_init_retries 5", "\tbad", NULL, 1);
 +}
 +
 +static void test_af_invalid_addition3(void **state)
 +{
 +	af_helper("2 pg_init_retries 5", "bad ", NULL, 1);
 +}
 + 
 +static void test_af_invalid_addition4(void **state)
 +{
 +	af_helper("2 pg_init_retries 5", " bad ", NULL, 1);
 +}
 + 
 +static void test_af_null_features1(void **state)
 +{
 +	af_helper(NULL, "test", "1 test", 0);
 +}
 +
 +static void test_af_null_features2(void **state)
 +{
 +	af_helper(NULL, "test\t  more", "2 test\t  more", 0);
 +}
 +
 +static void test_af_null_features3(void **state)
 +{
 +	af_helper(NULL, "test\neven\tmore", "3 test\neven\tmore", 0);
 +}
 +
 +static void test_af_already_exists1(void **state)
 +{
 +	af_helper("4 this is a test", "test", NULL, 0);
 +}
 +
 +static void test_af_already_exists2(void **state)
 +{
 +	af_helper("5 contest testy intestine test retest", "test", NULL, 0);
 +}
 +
 +static void test_af_almost_exists(void **state)
 +{
 +	af_helper("3 contest testy intestine", "test",
 +		  "4 contest testy intestine test", 0);
 +}
 +
 +static void test_af_bad_features1(void **state)
 +{
 +	af_helper("bad", "test", NULL, 1);
 +}
 +
 +static void test_af_bad_features2(void **state)
 +{
 +	af_helper("1bad", "test", NULL, 1);
 +}
 +
 +static void test_af_add1(void **state)
 +{
 +	af_helper("0", "test", "1 test", 0);
 +}
 +
 +static void test_af_add2(void **state)
 +{
 +	af_helper("0", "this is a test", "4 this is a test", 0);
 +}
 +
 +static void test_af_add3(void **state)
 +{
 +	af_helper("1 features", "more values", "3 features more values", 0);
 +}
 +
 +static void test_af_add4(void **state)
 +{
 +	af_helper("2 one\ttwo", "three\t four", "4 one\ttwo three\t four", 0);
 +}
 +
 +static int test_add_features(void)
 +{
 +	const struct CMUnitTest tests[] = {
 +		cmocka_unit_test(test_af_null_features_ptr),
 +		cmocka_unit_test(test_af_null_addition1),
 +		cmocka_unit_test(test_af_null_addition2),
 +		cmocka_unit_test(test_af_empty_addition),
 +		cmocka_unit_test(test_af_invalid_addition1),
 +		cmocka_unit_test(test_af_invalid_addition2),
 +		cmocka_unit_test(test_af_invalid_addition3),
 +		cmocka_unit_test(test_af_invalid_addition4),
 +		cmocka_unit_test(test_af_null_features1),
 +		cmocka_unit_test(test_af_null_features2),
 +		cmocka_unit_test(test_af_null_features3),
 +		cmocka_unit_test(test_af_already_exists1),
 +		cmocka_unit_test(test_af_already_exists2),
 +		cmocka_unit_test(test_af_almost_exists),
 +		cmocka_unit_test(test_af_bad_features1),
 +		cmocka_unit_test(test_af_bad_features2),
 +		cmocka_unit_test(test_af_add1),
 +		cmocka_unit_test(test_af_add2),
 +		cmocka_unit_test(test_af_add3),
 +		cmocka_unit_test(test_af_add4),
 +	};
 +	return cmocka_run_group_tests(tests, NULL, NULL);
 +}
 +
 +static void test_rf_null_features_ptr(void **state)
 +{
 +	assert_int_equal(remove_feature(NULL, "test"), 1);
 +}
 +
 +static void test_rf_null_features(void **state)
 +{
 +	char *f = NULL;
 +	assert_int_equal(remove_feature(&f, "test"), 1);
 +}
 +
 +static void rf_helper(const char *features_start, const char *removal,
 +		      const char *features_end, int result)
 +{
 +	char *f = strdup(features_start);
 +	char *orig = f;
 +
 +	assert_non_null(f);
 +	assert_int_equal(remove_feature(&f, removal), result);
 +	if (result != 0 || features_end == NULL)
 +		assert_ptr_equal(orig, f);
 +	else
 +		assert_string_equal(f, features_end);
 +	free(f);
 +}
 +
 +static void test_rf_null_removal(void **state)
 +{
 +	rf_helper("1 feature", NULL, NULL, 0);
 +}
 +
 +static void test_rf_empty_removal(void **state)
 +{
 +	rf_helper("1 feature", "", NULL, 0);
 +}
 +
 +static void test_rf_invalid_removal1(void **state)
 +{
 +	rf_helper("1 feature", " ", NULL, 1);
 +}
 +
 +static void test_rf_invalid_removal2(void **state)
 +{
 +	rf_helper("1 feature", " bad", NULL, 1);
 +}
 +
 +static void test_rf_invalid_removal3(void **state)
 +{
 +	rf_helper("1 feature", "bad\n", NULL, 1);
 +}
 +
 +static void test_rf_invalid_removal4(void **state)
 +{
 +	rf_helper("1 feature", "\tbad  \n", NULL, 1);
 +}
 +
 +static void test_rf_bad_features1(void **state)
 +{
 +	rf_helper("invalid feature test string", "test", NULL, 1);
 +}
 +
 +static void test_rf_bad_features2(void **state)
 +{
 +	rf_helper("2no space test", "test", NULL, 1);
 +}
 +
 +static void test_rf_missing_removal1(void **state)
 +{
 +	rf_helper("0", "test", NULL, 0);
 +}
 +
 +static void test_rf_missing_removal2(void **state)
 +{
 +	rf_helper("1 detest", "test", NULL, 0);
 +}
 +
 +static void test_rf_missing_removal3(void **state)
 +{
 +	rf_helper("4 testing one two three", "test", NULL, 0);
 +}
 +
 +static void test_rf_missing_removal4(void **state)
 +{
 +	rf_helper("1 contestant", "test", NULL, 0);
 +}
 +
 +static void test_rf_missing_removal5(void **state)
 +{
 +	rf_helper("3 testament protest detestable", "test", NULL, 0);
 +}
 +
 +static void test_rf_remove_all_features1(void **state)
 +{
 +	rf_helper("1 test", "test", "0", 0);
 +}
 +
 +static void test_rf_remove_all_features2(void **state)
 +{
 +	rf_helper("2 another\t test", "another\t test", "0", 0);
 +}
 +
 +static void test_rf_remove1(void **state)
 +{
 +	rf_helper("2 feature1 feature2", "feature2", "1 feature1", 0);
 +}
 +
 +static void test_rf_remove2(void **state)
 +{
 +	rf_helper("2 feature1 feature2", "feature1", "1 feature2", 0);
 +}
 +
 +static void test_rf_remove3(void **state)
 +{
 +	rf_helper("3 test1 test\ttest2", "test", "2 test1 test2", 0);
 +}
 +
 +static void test_rf_remove4(void **state)
 +{
 +	rf_helper("4 this\t is a test", "is a", "2 this\t test", 0);
 +}
 +
 +static void test_rf_remove5(void **state)
 +{
 +	rf_helper("3 one more test", "more test", "1 one", 0);
 +}
 +
 +static int test_remove_features(void)
 +{
 +	const struct CMUnitTest tests[] = {
 +		cmocka_unit_test(test_rf_null_features_ptr),
 +		cmocka_unit_test(test_rf_null_features),
 +		cmocka_unit_test(test_rf_null_removal),
 +		cmocka_unit_test(test_rf_empty_removal),
 +		cmocka_unit_test(test_rf_invalid_removal1),
 +		cmocka_unit_test(test_rf_invalid_removal2),
 +		cmocka_unit_test(test_rf_invalid_removal3),
 +		cmocka_unit_test(test_rf_invalid_removal4),
 +		cmocka_unit_test(test_rf_bad_features1),
 +		cmocka_unit_test(test_rf_bad_features2),
 +		cmocka_unit_test(test_rf_missing_removal1),
 +		cmocka_unit_test(test_rf_missing_removal2),
 +		cmocka_unit_test(test_rf_missing_removal3),
 +		cmocka_unit_test(test_rf_missing_removal4),
 +		cmocka_unit_test(test_rf_missing_removal5),
 +		cmocka_unit_test(test_rf_remove_all_features1),
 +		cmocka_unit_test(test_rf_remove_all_features2),
 +		cmocka_unit_test(test_rf_remove1),
 +		cmocka_unit_test(test_rf_remove2),
 +		cmocka_unit_test(test_rf_remove3),
 +		cmocka_unit_test(test_rf_remove4),
 +		cmocka_unit_test(test_rf_remove5),
 +	};
 +	return cmocka_run_group_tests(tests, NULL, NULL);
 +}
 +
 +int main(void)
 +{
 +	int ret = 0;
 +
 +	ret += test_add_features();
 +	ret += test_remove_features();
 +
 +	return ret;
 +}
--- a/SOURCES/0120-libmultipath-fix-queue_mode-feature-handling.patch
+++ b/SOURCES/0120-libmultipath-fix-queue_mode-feature-handling.patch
@ -0,0 +1,212 @@
 From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
 From: Benjamin Marzinski <bmarzins@redhat.com>
 Date: Fri, 7 Oct 2022 12:35:40 -0500
 Subject: [PATCH] libmultipath: fix queue_mode feature handling
 device-mapper is not able to change the queue_mode on a table reload.
 Make sure that when multipath sets up the map, both on regular reloads
 and reconfigures, it keeps the queue_mode the same.
 Signed-off-by: Benjamin Marzinski <bmarzins@redhat.com>
 Reviewed-by: Martin Wilck <mwilck@suse.com>
 ---
 libmultipath/configure.c   |  4 +++
 libmultipath/dmparser.c    |  2 ++
 libmultipath/propsel.c     | 55 ++++++++++++++++++++++++++++++++++++++
 libmultipath/structs.h     |  7 +++++
 libmultipath/util.c        | 10 +++++++
 libmultipath/util.h        |  1 +
 multipath/multipath.conf.5 |  7 +++--
 7 files changed, 84 insertions(+), 2 deletions(-)
 diff --git a/libmultipath/configure.c b/libmultipath/configure.c
 index 6cad0468..287289f7 100644
 --- a/libmultipath/configure.c
 +++ b/libmultipath/configure.c
@@ -1102,6 +1102,7 @@ int coalesce_paths (struct vectors * vecs, vector newmp, char * refwwid,
 	uint64_t *size_mismatch_seen;
 	bool map_processed = false;
 	bool no_daemon = false;
 +	struct multipath * cmpp;
 	/* ignore refwwid if it's empty */
 	if (refwwid && !strlen(refwwid))
@@ -1197,6 +1198,9 @@ int coalesce_paths (struct vectors * vecs, vector newmp, char * refwwid,
 		}
 		verify_paths(mpp, vecs);
 +		cmpp = find_mp_by_wwid(curmp, mpp->wwid);
 +		if (cmpp)
 +			mpp->queue_mode = cmpp->queue_mode;
 		params[0] = '\0';
 		if (setup_map(mpp, params, PARAMS_SIZE, vecs)) {
 			remove_map(mpp, vecs, 0);
 diff --git a/libmultipath/dmparser.c b/libmultipath/dmparser.c
 index b856a07f..b9c4dabc 100644
 --- a/libmultipath/dmparser.c
 +++ b/libmultipath/dmparser.c
@@ -164,6 +164,8 @@ int disassemble_map(vector pathvec, char *params, struct multipath *mpp,
 		FREE(word);
 	}
 +	mpp->queue_mode = strstr(mpp->features, "queue_mode bio") ?
 +			  QUEUE_MODE_BIO : QUEUE_MODE_RQ;
 	/*
 	 * hwhandler
 diff --git a/libmultipath/propsel.c b/libmultipath/propsel.c
 index be79902f..3f119dd9 100644
 --- a/libmultipath/propsel.c
 +++ b/libmultipath/propsel.c
@@ -26,6 +26,7 @@
 #include "propsel.h"
 #include <inttypes.h>
 #include <libudev.h>
 +#include <ctype.h>
 pgpolicyfn *pgpolicies[] = {
 	NULL,
@@ -413,6 +414,59 @@ void reconcile_features_with_options(const char *id, char **features, int* no_pa
 	}
 }
 +static void reconcile_features_with_queue_mode(struct multipath *mp)
 +{
 +	char *space = NULL, *val = NULL, *mode_str = NULL, *feat;
 +	int features_mode = QUEUE_MODE_UNDEF;
 +
 +	if (!mp->features)
 +		return;
 +
 +	pthread_cleanup_push(cleanup_free_ptr, &space);
 +	pthread_cleanup_push(cleanup_free_ptr, &val);
 +	pthread_cleanup_push(cleanup_free_ptr, &mode_str);
 +
 +	if (!(feat = strstr(mp->features, "queue_mode")) ||
 +	    feat == mp->features || !isspace(*(feat - 1)) ||
 +	    sscanf(feat, "queue_mode%m[ \f\n\r\t\v]%ms", &space, &val) != 2)
 +		goto sync_mode;
 +	if (asprintf(&mode_str, "queue_mode%s%s", space, val) < 0) {
 +		condlog(1, "failed to allocate space for queue_mode feature string");
 +		mode_str = NULL; /* value undefined on failure */
 +		goto exit;
 +	}
 +
 +	if (!strcmp(val, "rq") || !strcmp(val, "mq"))
 +		features_mode = QUEUE_MODE_RQ;
 +	else if (!strcmp(val, "bio"))
 +		features_mode = QUEUE_MODE_BIO;
 +	if (features_mode == QUEUE_MODE_UNDEF) {
 +		condlog(2, "%s: ignoring invalid feature '%s'",
 +			mp->alias, mode_str);
 +		goto sync_mode;
 +	}
 +
 +	if (mp->queue_mode == QUEUE_MODE_UNDEF)
 +		mp->queue_mode = features_mode;
 +	if (mp->queue_mode == features_mode)
 +		goto exit;
 +
 +	condlog(2,
 +		"%s: ignoring feature '%s' because queue_mode is set to '%s'",
 +		mp->alias, mode_str,
 +		(mp->queue_mode == QUEUE_MODE_RQ)? "rq" : "bio");
 +
 +sync_mode:
 +	if (mode_str)
 +		remove_feature(&mp->features, mode_str);
 +	if (mp->queue_mode == QUEUE_MODE_BIO)
 +		add_feature(&mp->features, "queue_mode bio");
 +exit:
 +	pthread_cleanup_pop(1);
 +	pthread_cleanup_pop(1);
 +	pthread_cleanup_pop(1);
 +}
 +
 int select_features(struct config *conf, struct multipath *mp)
 {
 	const char *origin;
@@ -428,6 +482,7 @@ out:
 	reconcile_features_with_options(mp->alias, &mp->features,
 					&mp->no_path_retry,
 					&mp->retain_hwhandler);
 +	reconcile_features_with_queue_mode(mp);
 	condlog(3, "%s: features = \"%s\" %s", mp->alias, mp->features, origin);
 	return 0;
 }
 diff --git a/libmultipath/structs.h b/libmultipath/structs.h
 index 3ed5cfc1..9a404da7 100644
 --- a/libmultipath/structs.h
 +++ b/libmultipath/structs.h
@@ -187,6 +187,12 @@ enum max_sectors_kb_states {
 	MAX_SECTORS_KB_MIN = 4,  /* can't be smaller than page size */
 };
 +enum queue_mode_states {
 +	QUEUE_MODE_UNDEF = 0,
 +	QUEUE_MODE_BIO,
 +	QUEUE_MODE_RQ,
 +};
 +
 enum scsi_protocol {
 	SCSI_PROTOCOL_FCP = 0,	/* Fibre Channel */
 	SCSI_PROTOCOL_SPI = 1,	/* parallel SCSI */
@@ -397,6 +403,7 @@ struct multipath {
 	int needs_paths_uevent;
 	int ghost_delay;
 	int ghost_delay_tick;
 +	int queue_mode;
 	uid_t uid;
 	gid_t gid;
 	mode_t mode;
 diff --git a/libmultipath/util.c b/libmultipath/util.c
 index dd30a46e..e04d20ab 100644
 --- a/libmultipath/util.c
 +++ b/libmultipath/util.c
@@ -465,6 +465,16 @@ void free_scandir_result(struct scandir_result *res)
 	FREE(res->di);
 }
 +void cleanup_free_ptr(void *arg)
 +{
 +	void **p = arg;
 +
 +	if (p && *p) {
 +		free(*p);
 +		*p = NULL;
 +	}
 +}
 +
 void close_fd(void *arg)
 {
 	close((long)arg);
 diff --git a/libmultipath/util.h b/libmultipath/util.h
 index ce277680..f898c829 100644
 --- a/libmultipath/util.h
 +++ b/libmultipath/util.h
@@ -44,6 +44,7 @@ void set_max_fds(rlim_t max_fds);
 	pthread_cleanup_push(((void (*)(void *))&f), (arg))
 void close_fd(void *arg);
 +void cleanup_free_ptr(void *arg);
 void cleanup_mutex(void *arg);
 struct scandir_result {
 diff --git a/multipath/multipath.conf.5 b/multipath/multipath.conf.5
 index 8e418372..61d2712b 100644
 --- a/multipath/multipath.conf.5
 +++ b/multipath/multipath.conf.5
@@ -458,8 +458,11 @@ precedence. See KNOWN ISSUES.
 <mode> can be \fIbio\fR, \fIrq\fR or \fImq\fR, which corresponds to
 bio-based, request-based, and block-multiqueue (blk-mq) request-based,
 respectively.
 -The default depends on the kernel parameter \fBdm_mod.use_blk_mq\fR. It is
 -\fImq\fR if the latter is set, and \fIrq\fR otherwise.
 +Before kernel 4.20 The default depends on the kernel parameter
 +\fBdm_mod.use_blk_mq\fR. It is \fImq\fR if the latter is set, and \fIrq\fR
 +otherwise. Since kernel 4.20, \fIrq\fR and \fImq\fR both correspond to
 +block-multiqueue. Once a multipath device has been created, its queue_mode
 +cannot be changed.
 .TP
 The default is: \fB<unset>\fR
 .RE
--- a/SOURCES/0121-multipath-tests-tests-for-reconcile_features_with_qu.patch
+++ b/SOURCES/0121-multipath-tests-tests-for-reconcile_features_with_qu.patch
@ -0,0 +1,290 @@
 From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
 From: Benjamin Marzinski <bmarzins@redhat.com>
 Date: Fri, 7 Oct 2022 12:35:41 -0500
 Subject: [PATCH] multipath tests: tests for reconcile_features_with_queue_mode
 Signed-off-by: Benjamin Marzinski <bmarzins@redhat.com>
 Reviewed-by: Martin Wilck <mwilck@suse.com>
 ---
 tests/Makefile   |   2 +
 tests/features.c | 232 ++++++++++++++++++++++++++++++++++++++++++++++-
 2 files changed, 233 insertions(+), 1 deletion(-)
 diff --git a/tests/Makefile b/tests/Makefile
 index 914413b8..f3e49487 100644
 --- a/tests/Makefile
 +++ b/tests/Makefile
@@ -29,6 +29,7 @@ endif
 ifneq ($(DIO_TEST_DEV),)
 directio-test_FLAGS := -DDIO_TEST_DEV=\"$(DIO_TEST_DEV)\"
 endif
 +features-test_FLAGS := -I$(multipathdir)/nvme
 # test-specific linker flags
 # XYZ-test_TESTDEPS: test libraries containing __wrap_xyz functions
@@ -53,6 +54,7 @@ alias-test_LIBDEPS := -lpthread -ldl
 ifneq ($(DIO_TEST_DEV),)
 directio-test_LIBDEPS := -laio
 endif
 +features-test_LIBDEPS := -ludev -lpthread
 %.o: %.c
 	$(CC) $(CFLAGS) $($*-test_FLAGS) -c -o $@ $<
 diff --git a/tests/features.c b/tests/features.c
 index 1e2e6bff..01fbccb7 100644
 --- a/tests/features.c
 +++ b/tests/features.c
@@ -1,9 +1,10 @@
 +#define _GNU_SOURCE
 #include <stddef.h>
 #include <stdarg.h>
 #include <setjmp.h>
 #include <cmocka.h>
 -#include "structs.h"
 +#include "../libmultipath/propsel.c"
 #include "globals.c"
 static void test_af_null_features_ptr(void **state)
@@ -307,12 +308,241 @@ static int test_remove_features(void)
 	return cmocka_run_group_tests(tests, NULL, NULL);
 }
 +static void test_cf_null_features(void **state)
 +{
 +	struct multipath mp = {
 +			.alias = "test",
 +	};
 +	reconcile_features_with_queue_mode(&mp);
 +	assert_null(mp.features);
 +}
 +
 +static void cf_helper(const char *features_start, const char *features_end,
 +		      int queue_mode_start, int queue_mode_end)
 +{
 +	struct multipath mp = {
 +			.alias = "test",
 +			.features = strdup(features_start),
 +			.queue_mode = queue_mode_start,
 +	};
 +	char *orig = mp.features;
 +
 +	assert_non_null(orig);
 +	reconcile_features_with_queue_mode(&mp);
 +	if (!features_end)
 +		assert_ptr_equal(orig, mp.features);
 +	else
 +		assert_string_equal(mp.features, features_end);
 +	free(mp.features);
 +	assert_int_equal(mp.queue_mode, queue_mode_end);
 +}
 +
 +static void test_cf_unset_unset1(void **state)
 +{
 +	cf_helper("0", NULL, QUEUE_MODE_UNDEF, QUEUE_MODE_UNDEF);
 +}
 +
 +static void test_cf_unset_unset2(void **state)
 +{
 +	cf_helper("1 queue_mode", NULL, QUEUE_MODE_UNDEF, QUEUE_MODE_UNDEF);
 +}
 +
 +static void test_cf_unset_unset3(void **state)
 +{
 +	cf_helper("queue_mode", NULL, QUEUE_MODE_UNDEF, QUEUE_MODE_UNDEF);
 +}
 +
 +static void test_cf_unset_unset4(void **state)
 +{
 +	cf_helper("2 queue_model bio", NULL, QUEUE_MODE_UNDEF,
 +		  QUEUE_MODE_UNDEF);
 +}
 +
 +static void test_cf_unset_unset5(void **state)
 +{
 +	cf_helper("1 queue_if_no_path", NULL, QUEUE_MODE_UNDEF,
 +		  QUEUE_MODE_UNDEF);
 +}
 +
 +static void test_cf_invalid_unset1(void **state)
 +{
 +	cf_helper("2 queue_mode biop", "0", QUEUE_MODE_UNDEF, QUEUE_MODE_UNDEF);
 +}
 +
 +static void test_cf_invalid_unset2(void **state)
 +{
 +	cf_helper("3 queue_mode rqs queue_if_no_path", "1 queue_if_no_path",
 +		  QUEUE_MODE_UNDEF, QUEUE_MODE_UNDEF);
 +}
 +
 +static void test_cf_rq_unset1(void **state)
 +{
 +	cf_helper("2 queue_mode rq", NULL, QUEUE_MODE_UNDEF, QUEUE_MODE_RQ);
 +}
 +
 +static void test_cf_rq_unset2(void **state)
 +{
 +	cf_helper("2 queue_mode mq", NULL, QUEUE_MODE_UNDEF, QUEUE_MODE_RQ);
 +}
 +
 +static void test_cf_bio_unset(void **state)
 +{
 +	cf_helper("2 queue_mode bio", NULL, QUEUE_MODE_UNDEF, QUEUE_MODE_BIO);
 +}
 +
 +static void test_cf_unset_bio1(void **state)
 +{
 +	cf_helper("1 queue_if_no_path", "3 queue_if_no_path queue_mode bio",
 +		  QUEUE_MODE_BIO, QUEUE_MODE_BIO);
 +}
 +
 +static void test_cf_unset_bio2(void **state)
 +{
 +	cf_helper("0", "2 queue_mode bio", QUEUE_MODE_BIO, QUEUE_MODE_BIO);
 +}
 +
 +static void test_cf_unset_bio3(void **state)
 +{
 +	cf_helper("2 pg_init_retries 50", "4 pg_init_retries 50 queue_mode bio",
 +		  QUEUE_MODE_BIO, QUEUE_MODE_BIO);
 +}
 +
 +static void test_cf_invalid_bio1(void **state)
 +{
 +	cf_helper("2 queue_mode bad", "2 queue_mode bio",
 +		  QUEUE_MODE_BIO, QUEUE_MODE_BIO);
 +}
 +
 +static void test_cf_invalid_bio2(void **state)
 +{
 +	cf_helper("3 queue_if_no_path queue_mode\tbad", "3 queue_if_no_path queue_mode bio",
 +		  QUEUE_MODE_BIO, QUEUE_MODE_BIO);
 +}
 +
 +static void test_cf_bio_bio1(void **state)
 +{
 +	cf_helper("2 queue_mode bio", NULL, QUEUE_MODE_BIO, QUEUE_MODE_BIO);
 +}
 +
 +static void test_cf_bio_bio2(void **state)
 +{
 +	cf_helper("3 queue_if_no_path queue_mode bio", NULL, QUEUE_MODE_BIO, QUEUE_MODE_BIO);
 +}
 +
 +static void test_cf_bio_bio3(void **state)
 +{
 +	cf_helper("3 queue_mode\nbio queue_if_no_path", NULL, QUEUE_MODE_BIO, QUEUE_MODE_BIO);
 +}
 +
 +static void test_cf_bio_rq1(void **state)
 +{
 +	cf_helper("2\nqueue_mode\tbio", "0", QUEUE_MODE_RQ, QUEUE_MODE_RQ);
 +}
 +
 +static void test_cf_bio_rq2(void **state)
 +{
 +	cf_helper("3 queue_if_no_path\nqueue_mode bio", "1 queue_if_no_path",
 +		  QUEUE_MODE_RQ, QUEUE_MODE_RQ);
 +}
 +
 +static void test_cf_bio_rq3(void **state)
 +{
 +	cf_helper("4 queue_mode bio pg_init_retries 20", "2 pg_init_retries 20",
 +		  QUEUE_MODE_RQ, QUEUE_MODE_RQ);
 +}
 +
 +static void test_cf_unset_rq1(void **state)
 +{
 +	cf_helper("0", NULL, QUEUE_MODE_RQ, QUEUE_MODE_RQ);
 +}
 +
 +static void test_cf_unset_rq2(void **state)
 +{
 +	cf_helper("2 pg_init_retries 15", NULL, QUEUE_MODE_RQ, QUEUE_MODE_RQ);
 +}
 +
 +static void test_cf_invalid_rq1(void **state)
 +{
 +	cf_helper("2 queue_mode bionic", "0", QUEUE_MODE_RQ, QUEUE_MODE_RQ);
 +}
 +
 +static void test_cf_invalid_rq2(void **state)
 +{
 +	cf_helper("3 queue_mode b\nqueue_if_no_path", "1 queue_if_no_path",
 +		  QUEUE_MODE_RQ, QUEUE_MODE_RQ);
 +}
 +
 +static void test_cf_rq_rq1(void **state)
 +{
 +	cf_helper("2 queue_mode rq", NULL, QUEUE_MODE_RQ, QUEUE_MODE_RQ);
 +}
 +
 +static void test_cf_rq_rq2(void **state)
 +{
 +	cf_helper("3 queue_mode\t \trq\nqueue_if_no_path", NULL, QUEUE_MODE_RQ, QUEUE_MODE_RQ);
 +}
 +
 +static void test_cf_rq_bio1(void **state)
 +{
 +	cf_helper("2 queue_mode rq", "2 queue_mode bio", QUEUE_MODE_BIO,
 +		  QUEUE_MODE_BIO);
 +}
 +
 +static void test_cf_rq_bio2(void **state)
 +{
 +	cf_helper("3 queue_if_no_path\nqueue_mode rq", "3 queue_if_no_path queue_mode bio", QUEUE_MODE_BIO, QUEUE_MODE_BIO);
 +}
 +
 +static void test_cf_rq_bio3(void **state)
 +{
 +	cf_helper("3 queue_mode rq\nqueue_if_no_path", "3 queue_if_no_path queue_mode bio", QUEUE_MODE_BIO, QUEUE_MODE_BIO);
 +}
 +
 +static int test_reconcile_features(void)
 +{
 +	const struct CMUnitTest tests[] = {
 +		cmocka_unit_test(test_cf_null_features),
 +		cmocka_unit_test(test_cf_unset_unset1),
 +		cmocka_unit_test(test_cf_unset_unset2),
 +		cmocka_unit_test(test_cf_unset_unset3),
 +		cmocka_unit_test(test_cf_unset_unset4),
 +		cmocka_unit_test(test_cf_unset_unset5),
 +		cmocka_unit_test(test_cf_invalid_unset1),
 +		cmocka_unit_test(test_cf_invalid_unset2),
 +		cmocka_unit_test(test_cf_rq_unset1),
 +		cmocka_unit_test(test_cf_rq_unset2),
 +		cmocka_unit_test(test_cf_bio_unset),
 +		cmocka_unit_test(test_cf_unset_bio1),
 +		cmocka_unit_test(test_cf_unset_bio2),
 +		cmocka_unit_test(test_cf_unset_bio3),
 +		cmocka_unit_test(test_cf_invalid_bio1),
 +		cmocka_unit_test(test_cf_invalid_bio2),
 +		cmocka_unit_test(test_cf_bio_bio1),
 +		cmocka_unit_test(test_cf_bio_bio2),
 +		cmocka_unit_test(test_cf_bio_bio3),
 +		cmocka_unit_test(test_cf_bio_rq1),
 +		cmocka_unit_test(test_cf_bio_rq2),
 +		cmocka_unit_test(test_cf_bio_rq3),
 +		cmocka_unit_test(test_cf_unset_rq1),
 +		cmocka_unit_test(test_cf_unset_rq2),
 +		cmocka_unit_test(test_cf_invalid_rq1),
 +		cmocka_unit_test(test_cf_invalid_rq2),
 +		cmocka_unit_test(test_cf_rq_rq1),
 +		cmocka_unit_test(test_cf_rq_rq2),
 +		cmocka_unit_test(test_cf_rq_bio1),
 +		cmocka_unit_test(test_cf_rq_bio2),
 +		cmocka_unit_test(test_cf_rq_bio3),
 +	};
 +	return cmocka_run_group_tests(tests, NULL, NULL);
 +}
 +
 int main(void)
 {
 	int ret = 0;
 	ret += test_add_features();
 	ret += test_remove_features();
 +	ret += test_reconcile_features();
 	return ret;
 }
--- a/SOURCES/0122-libmultipath-prepare-proto_id-for-use-by-non-scsi-de.patch
+++ b/SOURCES/0122-libmultipath-prepare-proto_id-for-use-by-non-scsi-de.patch
@ -0,0 +1,143 @@
 From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
 From: Benjamin Marzinski <bmarzins@redhat.com>
 Date: Fri, 7 Oct 2022 12:35:42 -0500
 Subject: [PATCH] libmultipath: prepare proto_id for use by non-scsi devivces
 Make sure that when we are checking for a scsi protocol, we are first
 checking that we are working with a scsi path.
 Signed-off-by: Benjamin Marzinski <bmarzins@redhat.com>
 Reviewed-by: Martin Wilck <mwilck@suse.com>
 ---
 libmultipath/configure.c   |  9 +++++----
 libmultipath/discovery.c   | 13 ++++++++-----
 libmultipath/print.c       |  6 ++++--
 libmultipath/structs.c     |  2 +-
 libmultipath/structs.h     |  4 +++-
 multipathd/fpin_handlers.c |  2 +-
 6 files changed, 22 insertions(+), 14 deletions(-)
 diff --git a/libmultipath/configure.c b/libmultipath/configure.c
 index 287289f7..8e1bc488 100644
 --- a/libmultipath/configure.c
 +++ b/libmultipath/configure.c
@@ -223,10 +223,11 @@ int rr_optimize_path_order(struct pathgroup *pgp)
 	total_paths = VECTOR_SIZE(pgp->paths);
 	vector_foreach_slot(pgp->paths, pp, i) {
 -		if (pp->sg_id.proto_id != SCSI_PROTOCOL_FCP &&
 -			pp->sg_id.proto_id != SCSI_PROTOCOL_SAS &&
 -			pp->sg_id.proto_id != SCSI_PROTOCOL_ISCSI &&
 -			pp->sg_id.proto_id != SCSI_PROTOCOL_SRP) {
 +		if (pp->bus != SYSFS_BUS_SCSI ||
 +		    (pp->sg_id.proto_id != SCSI_PROTOCOL_FCP &&
 +		     pp->sg_id.proto_id != SCSI_PROTOCOL_SAS &&
 +		     pp->sg_id.proto_id != SCSI_PROTOCOL_ISCSI &&
 +		     pp->sg_id.proto_id != SCSI_PROTOCOL_SRP)) {
 			/* return success as default path order
 			 * is maintained in path group
 			 */
 diff --git a/libmultipath/discovery.c b/libmultipath/discovery.c
 index 36cc389e..5f4e0794 100644
 --- a/libmultipath/discovery.c
 +++ b/libmultipath/discovery.c
@@ -468,10 +468,11 @@ int sysfs_get_host_adapter_name(const struct path *pp, char *adapter_name)
 	proto_id = pp->sg_id.proto_id;
 -	if (proto_id != SCSI_PROTOCOL_FCP &&
 -	    proto_id != SCSI_PROTOCOL_SAS &&
 -	    proto_id != SCSI_PROTOCOL_ISCSI &&
 -	    proto_id != SCSI_PROTOCOL_SRP) {
 +	if (pp->bus != SYSFS_BUS_SCSI ||
 +	    (proto_id != SCSI_PROTOCOL_FCP &&
 +	     proto_id != SCSI_PROTOCOL_SAS &&
 +	     proto_id != SCSI_PROTOCOL_ISCSI &&
 +	     proto_id != SCSI_PROTOCOL_SRP)) {
 		return 1;
 	}
 	/* iscsi doesn't have adapter info in sysfs
@@ -1722,8 +1723,10 @@ sysfs_pathinfo(struct path * pp, vector hwtable)
 		pp->bus = SYSFS_BUS_CCISS;
 	if (!strncmp(pp->dev,"dasd", 4))
 		pp->bus = SYSFS_BUS_CCW;
 -	if (!strncmp(pp->dev,"sd", 2))
 +	if (!strncmp(pp->dev,"sd", 2)) {
 		pp->bus = SYSFS_BUS_SCSI;
 +		pp->sg_id.proto_id = SCSI_PROTOCOL_UNSPEC;
 +	}
 	if (!strncmp(pp->dev,"nvme", 4))
 		pp->bus = SYSFS_BUS_NVME;
 diff --git a/libmultipath/print.c b/libmultipath/print.c
 index 8a6fbe83..8a85df66 100644
 --- a/libmultipath/print.c
 +++ b/libmultipath/print.c
@@ -558,7 +558,8 @@ snprint_host_attr (char * buff, size_t len, const struct path * pp, char *attr)
 	const char *value = NULL;
 	int ret;
 -	if (pp->sg_id.proto_id != SCSI_PROTOCOL_FCP)
 +	if (pp->bus != SYSFS_BUS_SCSI ||
 +	    pp->sg_id.proto_id != SCSI_PROTOCOL_FCP)
 		return snprintf(buff, len, "[undef]");
 	sprintf(host_id, "host%d", pp->sg_id.host_no);
 	host_dev = udev_device_new_from_subsystem_sysname(udev, "fc_host",
@@ -597,7 +598,8 @@ snprint_tgt_wwpn (char * buff, size_t len, const struct path * pp)
 	const char *value = NULL;
 	int ret;
 -	if (pp->sg_id.proto_id != SCSI_PROTOCOL_FCP)
 +	if (pp->bus != SYSFS_BUS_SCSI ||
 +	    pp->sg_id.proto_id != SCSI_PROTOCOL_FCP)
 		return snprintf(buff, len, "[undef]");
 	sprintf(rport_id, "rport-%d:%d-%d",
 		pp->sg_id.host_no, pp->sg_id.channel, pp->sg_id.transport_id);
 diff --git a/libmultipath/structs.c b/libmultipath/structs.c
 index 84f9c959..1122cfae 100644
 --- a/libmultipath/structs.c
 +++ b/libmultipath/structs.c
@@ -115,7 +115,7 @@ alloc_path (void)
 		pp->sg_id.channel = -1;
 		pp->sg_id.scsi_id = -1;
 		pp->sg_id.lun = -1;
 -		pp->sg_id.proto_id = SCSI_PROTOCOL_UNSPEC;
 +		pp->sg_id.proto_id = PROTOCOL_UNSET;
 		pp->fd = -1;
 		pp->tpgs = TPGS_UNDEF;
 		pp->priority = PRIO_UNDEF;
 diff --git a/libmultipath/structs.h b/libmultipath/structs.h
 index 9a404da7..960ea024 100644
 --- a/libmultipath/structs.h
 +++ b/libmultipath/structs.h
@@ -193,6 +193,8 @@ enum queue_mode_states {
 	QUEUE_MODE_RQ,
 };
 +#define PROTOCOL_UNSET -1
 +
 enum scsi_protocol {
 	SCSI_PROTOCOL_FCP = 0,	/* Fibre Channel */
 	SCSI_PROTOCOL_SPI = 1,	/* parallel SCSI */
@@ -294,7 +296,7 @@ struct sg_id {
 	int lun;
 	short h_cmd_per_lun;
 	short d_queue_depth;
 -	enum scsi_protocol proto_id;
 +	int proto_id;
 	int transport_id;
 };
 diff --git a/multipathd/fpin_handlers.c b/multipathd/fpin_handlers.c
 index b14366d7..599f2893 100644
 --- a/multipathd/fpin_handlers.c
 +++ b/multipathd/fpin_handlers.c
@@ -220,7 +220,7 @@ static int  fpin_chk_wwn_setpath_marginal(uint16_t host_num,  struct vectors *ve
 	vector_foreach_slot(vecs->pathvec, pp, k) {
 		/* Checks the host number and also for the SCSI FCP */
 -		if (pp->sg_id.proto_id != SCSI_PROTOCOL_FCP || host_num !=  pp->sg_id.host_no)
 +		if (pp->bus != SYSFS_BUS_SCSI || pp->sg_id.proto_id != SCSI_PROTOCOL_FCP || host_num !=  pp->sg_id.host_no)
 			continue;
 		sprintf(rport_id, "rport-%d:%d-%d",
 				pp->sg_id.host_no, pp->sg_id.channel, pp->sg_id.transport_id);
--- a/SOURCES/0123-libmultipath-get-nvme-path-transport-protocol.patch
+++ b/SOURCES/0123-libmultipath-get-nvme-path-transport-protocol.patch
@ -0,0 +1,194 @@
 From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
 From: Benjamin Marzinski <bmarzins@redhat.com>
 Date: Fri, 7 Oct 2022 12:35:43 -0500
 Subject: [PATCH] libmultipath: get nvme path transport protocol
 Read the transport protocol from /sys/block/nvmeXnY/device/transport.
 Update protocol_name[] and bus_protocol_id() to store the nvme protocol
 names after the scsi protocol names.
 Signed-off-by: Benjamin Marzinski <bmarzins@redhat.com>
 Reviewed-by: Martin Wilck <mwilck@suse.com>
 ---
 libmultipath/discovery.c   | 18 ++++++++++++++++--
 libmultipath/structs.c     | 22 +++++++++++++++++-----
 libmultipath/structs.h     | 33 +++++++++++++++++++++------------
 multipath/multipath.conf.5 | 10 +++++++---
 4 files changed, 61 insertions(+), 22 deletions(-)
 diff --git a/libmultipath/discovery.c b/libmultipath/discovery.c
 index 5f4e0794..eb7a634b 100644
 --- a/libmultipath/discovery.c
 +++ b/libmultipath/discovery.c
@@ -1455,6 +1455,7 @@ nvme_sysfs_pathinfo (struct path * pp, vector hwtable)
 	struct udev_device *parent;
 	const char *attr_path = NULL;
 	const char *attr;
 +	int i;
 	attr_path = udev_device_get_sysname(pp->udev);
 	if (!attr_path)
@@ -1476,6 +1477,18 @@ nvme_sysfs_pathinfo (struct path * pp, vector hwtable)
 	attr = udev_device_get_sysattr_value(parent, "cntlid");
 	pp->sg_id.channel = attr ? atoi(attr) : 0;
 +	attr = udev_device_get_sysattr_value(parent, "transport");
 +	if (attr) {
 +		for (i = 0; i < NVME_PROTOCOL_UNSPEC; i++){
 +			if (protocol_name[SYSFS_BUS_NVME + i] &&
 +			    !strcmp(attr,
 +				    protocol_name[SYSFS_BUS_NVME + i] + 5)) {
 +				pp->sg_id.proto_id = i;
 +				break;
 +			}
 +		}
 +	}
 +
 	snprintf(pp->vendor_id, SCSI_VENDOR_SIZE, "NVME");
 	snprintf(pp->product_id, PATH_PRODUCT_SIZE, "%s",
 		 udev_device_get_sysattr_value(parent, "model"));
@@ -1727,9 +1740,10 @@ sysfs_pathinfo(struct path * pp, vector hwtable)
 		pp->bus = SYSFS_BUS_SCSI;
 		pp->sg_id.proto_id = SCSI_PROTOCOL_UNSPEC;
 	}
 -	if (!strncmp(pp->dev,"nvme", 4))
 +	if (!strncmp(pp->dev,"nvme", 4)) {
 		pp->bus = SYSFS_BUS_NVME;
 -
 +		pp->sg_id.proto_id = NVME_PROTOCOL_UNSPEC;
 +	}
 	switch (pp->bus) {
 	case SYSFS_BUS_SCSI:
 		return scsi_sysfs_pathinfo(pp, hwtable);
 diff --git a/libmultipath/structs.c b/libmultipath/structs.c
 index 1122cfae..7bdf9152 100644
 --- a/libmultipath/structs.c
 +++ b/libmultipath/structs.c
@@ -25,7 +25,6 @@ const char * const protocol_name[LAST_BUS_PROTOCOL_ID + 1] = {
 	[SYSFS_BUS_UNDEF] = "undef",
 	[SYSFS_BUS_CCW] = "ccw",
 	[SYSFS_BUS_CCISS] = "cciss",
 -	[SYSFS_BUS_NVME] = "nvme",
 	[SYSFS_BUS_SCSI + SCSI_PROTOCOL_FCP] = "scsi:fcp",
 	[SYSFS_BUS_SCSI + SCSI_PROTOCOL_SPI] = "scsi:spi",
 	[SYSFS_BUS_SCSI + SCSI_PROTOCOL_SSA] = "scsi:ssa",
@@ -37,6 +36,13 @@ const char * const protocol_name[LAST_BUS_PROTOCOL_ID + 1] = {
 	[SYSFS_BUS_SCSI + SCSI_PROTOCOL_ATA] = "scsi:ata",
 	[SYSFS_BUS_SCSI + SCSI_PROTOCOL_USB] = "scsi:usb",
 	[SYSFS_BUS_SCSI + SCSI_PROTOCOL_UNSPEC] = "scsi:unspec",
 +	[SYSFS_BUS_NVME + NVME_PROTOCOL_PCIE] = "nvme:pcie",
 +	[SYSFS_BUS_NVME + NVME_PROTOCOL_RDMA] = "nvme:rdma",
 +	[SYSFS_BUS_NVME + NVME_PROTOCOL_FC] = "nvme:fc",
 +	[SYSFS_BUS_NVME + NVME_PROTOCOL_TCP] = "nvme:tcp",
 +	[SYSFS_BUS_NVME + NVME_PROTOCOL_LOOP] = "nvme:loop",
 +	[SYSFS_BUS_NVME + NVME_PROTOCOL_APPLE_NVME] = "nvme:apple-nvme",
 +	[SYSFS_BUS_NVME + NVME_PROTOCOL_UNSPEC] = "nvme:unspec",
 };
 struct adapter_group *
@@ -716,11 +722,17 @@ out:
 }
 unsigned int bus_protocol_id(const struct path *pp) {
 -	if (!pp || pp->bus < 0 || pp->bus > SYSFS_BUS_SCSI)
 +	if (!pp || pp->bus < 0 || pp->bus > SYSFS_BUS_NVME)
 		return SYSFS_BUS_UNDEF;
 -	if (pp->bus != SYSFS_BUS_SCSI)
 +	if (pp->bus != SYSFS_BUS_SCSI && pp->bus != SYSFS_BUS_NVME)
 		return pp->bus;
 -	if ((int)pp->sg_id.proto_id < 0 || pp->sg_id.proto_id > SCSI_PROTOCOL_UNSPEC)
 +	if (pp->sg_id.proto_id < 0)
 		return SYSFS_BUS_UNDEF;
 -	return SYSFS_BUS_SCSI + pp->sg_id.proto_id;
 +	if (pp->bus == SYSFS_BUS_SCSI &&
 +	    pp->sg_id.proto_id > SCSI_PROTOCOL_UNSPEC)
 +		return SYSFS_BUS_UNDEF;
 +	if (pp->bus == SYSFS_BUS_NVME &&
 +	    pp->sg_id.proto_id > NVME_PROTOCOL_UNSPEC)
 +		return SYSFS_BUS_UNDEF;
 +	return pp->bus + pp->sg_id.proto_id;
 }
 diff --git a/libmultipath/structs.h b/libmultipath/structs.h
 index 960ea024..9130efb5 100644
 --- a/libmultipath/structs.h
 +++ b/libmultipath/structs.h
@@ -57,15 +57,6 @@ enum failback_mode {
 	FAILBACK_FOLLOWOVER
 };
 -/* SYSFS_BUS_SCSI should be last, see bus_protocol_id() */
 -enum sysfs_buses {
 -	SYSFS_BUS_UNDEF,
 -	SYSFS_BUS_CCW,
 -	SYSFS_BUS_CCISS,
 -	SYSFS_BUS_NVME,
 -	SYSFS_BUS_SCSI,
 -};
 -
 enum pathstates {
 	PSTATE_UNDEF,
 	PSTATE_FAILED,
@@ -207,14 +198,32 @@ enum scsi_protocol {
 	SCSI_PROTOCOL_ATA = 8,
 	SCSI_PROTOCOL_USB = 9,  /* USB Attached SCSI (UAS), and others */
 	SCSI_PROTOCOL_UNSPEC = 0xa, /* No specific protocol */
 +	SCSI_PROTOCOL_END = 0xb, /* offset of the next sysfs_buses entry */
 +};
 +
 +/* values from /sys/class/nvme/nvmeX */
 +enum nvme_protocol {
 +	NVME_PROTOCOL_PCIE = 0,
 +	NVME_PROTOCOL_RDMA = 1,
 +	NVME_PROTOCOL_FC = 2,
 +	NVME_PROTOCOL_TCP = 3,
 +	NVME_PROTOCOL_LOOP = 4,
 +	NVME_PROTOCOL_APPLE_NVME = 5,
 +	NVME_PROTOCOL_UNSPEC = 6, /* unknown protocol */
 +};
 +
 +enum sysfs_buses {
 +	SYSFS_BUS_UNDEF,
 +	SYSFS_BUS_CCW,
 +	SYSFS_BUS_CCISS,
 +	SYSFS_BUS_SCSI,
 +	SYSFS_BUS_NVME = SYSFS_BUS_SCSI + SCSI_PROTOCOL_END,
 };
 /*
  * Linear ordering of bus/protocol
 - * This assumes that SYSFS_BUS_SCSI is last in enum sysfs_buses
 - * SCSI is the only bus type for which we distinguish protocols.
  */
 -#define LAST_BUS_PROTOCOL_ID (SYSFS_BUS_SCSI + SCSI_PROTOCOL_UNSPEC)
 +#define LAST_BUS_PROTOCOL_ID (SYSFS_BUS_NVME + NVME_PROTOCOL_UNSPEC)
 unsigned int bus_protocol_id(const struct path *pp);
 extern const char * const protocol_name[];
 diff --git a/multipath/multipath.conf.5 b/multipath/multipath.conf.5
 index 61d2712b..1f5a40b6 100644
 --- a/multipath/multipath.conf.5
 +++ b/multipath/multipath.conf.5
@@ -1369,7 +1369,9 @@ Regular expression for the protocol of a device to be excluded/included.
 The protocol strings that multipath recognizes are \fIscsi:fcp\fR,
 \fIscsi:spi\fR, \fIscsi:ssa\fR, \fIscsi:sbp\fR, \fIscsi:srp\fR,
 \fIscsi:iscsi\fR, \fIscsi:sas\fR, \fIscsi:adt\fR, \fIscsi:ata\fR,
 -\fIscsi:unspec\fR, \fIccw\fR, \fIcciss\fR, \fInvme\fR, and \fIundef\fR.
 +\fIscsi:unspec\fR, \fInvme:pcie\fR, \fInvme:rdma\fR, \fInvme:fc\fR,
 +\fInvme:tcp\fR, \fInvme:loop\fR, \fInvme:apple-nvme\fR, \fInvme:unspec\fR,
 +\fIccw\fR, \fIcciss\fR, and \fIundef\fR.
 The protocol that a path is using can be viewed by running
 \fBmultipathd show paths format "%d %P"\fR
 .RE
@@ -1757,8 +1759,10 @@ The protocol subsection recognizes the following mandatory attribute:
 The protocol string of the path device. The possible values are \fIscsi:fcp\fR,
 \fIscsi:spi\fR, \fIscsi:ssa\fR, \fIscsi:sbp\fR, \fIscsi:srp\fR,
 \fIscsi:iscsi\fR, \fIscsi:sas\fR, \fIscsi:adt\fR, \fIscsi:ata\fR,
 -\fIscsi:unspec\fR, \fIccw\fR, \fIcciss\fR, \fInvme\fR, and \fIundef\fR. This is
 -\fBnot\fR a regular expression. the path device protcol string must match
 +\fIscsi:unspec\fR, \fInvme:pcie\fR, \fInvme:rdma\fR, \fInvme:fc\fR,
 +\fInvme:tcp\fR, \fInvme:loop\fR, \fInvme:apple-nvme\fR, \fInvme:unspec\fR,
 +\fIccw\fR, \fIcciss\fR, and \fIundef\fR. This is
 +\fBnot\fR a regular expression. the path device protocol string must match
 exactly. The protocol that a path is using can be viewed by running
 \fBmultipathd show paths format "%d %P"\fR
 .LP
--- a/SOURCES/0124-libmultipath-enforce-queue_mode-bio-for-nmve-tcp-pat.patch
+++ b/SOURCES/0124-libmultipath-enforce-queue_mode-bio-for-nmve-tcp-pat.patch
@ -0,0 +1,111 @@
 From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
 From: Benjamin Marzinski <bmarzins@redhat.com>
 Date: Fri, 7 Oct 2022 12:35:44 -0500
 Subject: [PATCH] libmultipath: enforce queue_mode bio for nmve:tcp paths
 nvme:tcp devices set BLK_MQ_F_BLOCKING (they are the only block devices
 which multipath supports that do so), meaning that block_mq expects that
 they can block at certain points while servicing a request.  However,
 due to the way device-mapper sets up its queue, it is not able to set
 BLK_MQ_F_BLOCKING when it includes paths that set this flag.  Patches
 were written to address this issue but they were rejected upstream
 https://lore.kernel.org/linux-block/YcH%2FE4JNag0QYYAa@infradead.org/T/#t
 The proposed solution was to have multipath use the bio queue_mode for
 multipath devices that include nvme:tcp paths.
 Multipath devices now automatically add the "queue_mode bio" feature if
 they include nvme:tcp paths.  If a multipath devices was created with
 "queue_mode rq", it will disallow the addition of nvme:tcp paths.
 Signed-off-by: Benjamin Marzinski <bmarzins@redhat.com>
 Reviewed-by: Martin Wilck <mwilck@suse.com>
 ---
 libmultipath/configure.c   | 17 ++++++++++++++++-
 libmultipath/structs_vec.c |  7 +++++++
 multipath/multipath.conf.5 |  4 +++-
 3 files changed, 26 insertions(+), 2 deletions(-)
 diff --git a/libmultipath/configure.c b/libmultipath/configure.c
 index 8e1bc488..c341793c 100644
 --- a/libmultipath/configure.c
 +++ b/libmultipath/configure.c
@@ -297,6 +297,7 @@ int setup_map(struct multipath *mpp, char *params, int params_size,
 	      struct vectors *vecs)
 {
 	struct pathgroup * pgp;
 +	struct path *pp;
 	struct config *conf;
 	int i, n_paths, marginal_pathgroups;
@@ -308,6 +309,14 @@ int setup_map(struct multipath *mpp, char *params, int params_size,
 		return 1;
 	}
 +	/* Force QUEUE_MODE_BIO for maps with nvme:tcp paths */
 +	vector_foreach_slot(mpp->paths, pp, i) {
 +		if (pp->bus == SYSFS_BUS_NVME &&
 +		    pp->sg_id.proto_id == NVME_PROTOCOL_TCP) {
 +			mpp->queue_mode = QUEUE_MODE_BIO;
 +			break;
 +		}
 +	}
 	/*
 	 * free features, selector, and hwhandler properties if they are being reused
 	 */
@@ -1161,6 +1170,13 @@ int coalesce_paths (struct vectors * vecs, vector newmp, char * refwwid,
 			continue;
 		}
 +		cmpp = find_mp_by_wwid(curmp, pp1->wwid);
 +		if (cmpp && cmpp->queue_mode == QUEUE_MODE_RQ &&
 +		    pp1->bus == SYSFS_BUS_NVME && pp1->sg_id.proto_id ==
 +		    NVME_PROTOCOL_TCP) {
 +			orphan_path(pp1, "nvme:tcp path not allowed with request queue_mode multipath device");
 +			continue;
 +		}
 		/*
 		 * at this point, we know we really got a new mp
 		 */
@@ -1199,7 +1215,6 @@ int coalesce_paths (struct vectors * vecs, vector newmp, char * refwwid,
 		}
 		verify_paths(mpp, vecs);
 -		cmpp = find_mp_by_wwid(curmp, mpp->wwid);
 		if (cmpp)
 			mpp->queue_mode = cmpp->queue_mode;
 		params[0] = '\0';
 diff --git a/libmultipath/structs_vec.c b/libmultipath/structs_vec.c
 index 8137ea21..24ac022e 100644
 --- a/libmultipath/structs_vec.c
 +++ b/libmultipath/structs_vec.c
@@ -68,6 +68,13 @@ int adopt_paths(vector pathvec, struct multipath *mpp)
 					pp->dev, mpp->alias);
 				continue;
 			}
 +			if (mpp->queue_mode == QUEUE_MODE_RQ &&
 +			    pp->bus == SYSFS_BUS_NVME &&
 +			    pp->sg_id.proto_id == NVME_PROTOCOL_TCP) {
 +				condlog(2, "%s: mulitpath device %s created with request queue_mode. Unable to add nvme:tcp paths",
 +					pp->dev, mpp->alias);
 +				continue;
 +			}
 			condlog(3, "%s: ownership set to %s",
 				pp->dev, mpp->alias);
 			pp->mpp = mpp;
 diff --git a/multipath/multipath.conf.5 b/multipath/multipath.conf.5
 index 1f5a40b6..cb07a62c 100644
 --- a/multipath/multipath.conf.5
 +++ b/multipath/multipath.conf.5
@@ -462,7 +462,9 @@ Before kernel 4.20 The default depends on the kernel parameter
 \fBdm_mod.use_blk_mq\fR. It is \fImq\fR if the latter is set, and \fIrq\fR
 otherwise. Since kernel 4.20, \fIrq\fR and \fImq\fR both correspond to
 block-multiqueue. Once a multipath device has been created, its queue_mode
 -cannot be changed.
 +cannot be changed. \fInvme:tcp\fR paths are only supported in multipath
 +devices with queue_mode set to \fIbio\fR. multipath will automatically
 +set this when creating a device with \fInvme:tcp\fR paths.
 .TP
 The default is: \fB<unset>\fR
 .RE
--- a/SOURCES/0125-multipath-add-historical-service-time-to-the-man-pag.patch
+++ b/SOURCES/0125-multipath-add-historical-service-time-to-the-man-pag.patch
@ -0,0 +1,26 @@
 From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
 From: Benjamin Marzinski <bmarzins@redhat.com>
 Date: Tue, 15 Nov 2022 09:01:36 -0600
 Subject: [PATCH] multipath: add historical-service-time to the man page
 Signed-off-by: Benjamin Marzinski <bmarzins@redhat.com>
 ---
 multipath/multipath.conf.5 | 5 +++++
 1 file changed, 5 insertions(+)
 diff --git a/multipath/multipath.conf.5 b/multipath/multipath.conf.5
 index cb07a62c..d8a98435 100644
 --- a/multipath/multipath.conf.5
 +++ b/multipath/multipath.conf.5
@@ -205,6 +205,11 @@ of outstanding I/O to the path.
 (Since 2.6.31 kernel) Choose the path for the next bunch of I/O based on the amount
 of outstanding I/O to the path and its relative throughput.
 .TP
 +.I "historical-service-time 0"
 +(Since 4.18.0-305.3.el8 kernel) Choose the path for the next bunch of I/O based
 +on the estimation of future service time based on the history of previous I/O
 +submitted to each path.
 +.TP
 The default is: \fBservice-time 0\fR
 .RE
 .
--- a/SOURCES/0126-libmultipath-copy-mpp-hwe-from-pp-hwe.patch
+++ b/SOURCES/0126-libmultipath-copy-mpp-hwe-from-pp-hwe.patch
@ -0,0 +1,215 @@
 From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
 From: Martin Wilck <mwilck@suse.com>
 Date: Wed, 16 Sep 2020 22:22:36 +0200
 Subject: [PATCH] libmultipath: copy mpp->hwe from pp->hwe
 Since f0462f0 ("libmultipath: use vector for for pp->hwe and mp->hwe"),
 we've been trying to fix issues caused by paths getting freed and mpp->hwe
 dangling. This approach couldn't work because we need mpp->hwe to persist,
 even if all paths are removed from the map. Before f0462f0, a simple
 assignment worked, because the lifetime of the hwe wasn't bound to the
 path. But now, we need to copy the vector. It turns out that we need to set
 mpp->hwe only in two places, add_map_with_path() and setup_map(), and
 that the code is simplified overall.
 Even now, it can happen that a map is added with add_map_without_paths(),
 and has no paths. In that case, calling do_set_from_hwe() with a NULL
 pointer is not a bug, so remove the message.
 Fixes: f0462f0 ("libmultipath: use vector for for pp->hwe and mp->hwe")
 Reviewed-by: Benjamin Marzinski <bmarzins@redhat.com>
 Signed-off-by: Benjamin Marzinski <bmarzins@redhat.com>
 ---
 libmultipath/configure.c   |  8 ++++++++
 libmultipath/propsel.c     |  2 +-
 libmultipath/structs.c     | 15 ++++++++++++++
 libmultipath/structs.h     |  1 +
 libmultipath/structs_vec.c | 41 +++++++++++++++++++-------------------
 multipathd/main.c          | 10 ----------
 6 files changed, 45 insertions(+), 32 deletions(-)
 diff --git a/libmultipath/configure.c b/libmultipath/configure.c
 index c341793c..6e06fea2 100644
 --- a/libmultipath/configure.c
 +++ b/libmultipath/configure.c
@@ -324,6 +324,14 @@ int setup_map(struct multipath *mpp, char *params, int params_size,
 	if (mpp->disable_queueing && VECTOR_SIZE(mpp->paths) != 0)
 		mpp->disable_queueing = 0;
 +	/*
 +	 * If this map was created with add_map_without_path(),
 +	 * mpp->hwe might not be set yet.
 +	 */
 +	if (!mpp->hwe)
 +		extract_hwe_from_path(mpp);
 +
 +
 	/*
 	 * properties selectors
 	 *
 diff --git a/libmultipath/propsel.c b/libmultipath/propsel.c
 index 3f119dd9..bc5c27ab 100644
 --- a/libmultipath/propsel.c
 +++ b/libmultipath/propsel.c
@@ -66,7 +66,7 @@ do {									\
 	__do_set_from_vec(struct hwentry, var, (src)->hwe, dest)
 #define do_set_from_hwe(var, src, dest, msg)				\
 -	if (__do_set_from_hwe(var, src, dest)) {			\
 +	if (src->hwe && __do_set_from_hwe(var, src, dest)) {		\
 		origin = msg;						\
 		goto out;						\
 	}
 diff --git a/libmultipath/structs.c b/libmultipath/structs.c
 index 7bdf9152..5bb6caf8 100644
 --- a/libmultipath/structs.c
 +++ b/libmultipath/structs.c
@@ -242,6 +242,17 @@ alloc_multipath (void)
 	return mpp;
 }
 +void *set_mpp_hwe(struct multipath *mpp, const struct path *pp)
 +{
 +	if (!mpp || !pp || !pp->hwe)
 +		return NULL;
 +	if (mpp->hwe)
 +		return mpp->hwe;
 +	mpp->hwe = vector_convert(NULL, pp->hwe,
 +				  struct hwentry, identity);
 +	return mpp->hwe;
 +}
 +
 void free_multipath_attributes(struct multipath *mpp)
 {
 	if (!mpp)
@@ -283,6 +294,10 @@ free_multipath (struct multipath * mpp, enum free_path_mode free_paths)
 	free_pathvec(mpp->paths, free_paths);
 	free_pgvec(mpp->pg, free_paths);
 +	if (mpp->hwe) {
 +		vector_free(mpp->hwe);
 +		mpp->hwe = NULL;
 +	}
 	FREE_PTR(mpp->mpcontext);
 	FREE(mpp);
 }
 diff --git a/libmultipath/structs.h b/libmultipath/structs.h
 index 9130efb5..44980b4e 100644
 --- a/libmultipath/structs.h
 +++ b/libmultipath/structs.h
@@ -499,6 +499,7 @@ struct host_group {
 struct path * alloc_path (void);
 struct pathgroup * alloc_pathgroup (void);
 struct multipath * alloc_multipath (void);
 +void *set_mpp_hwe(struct multipath *mpp, const struct path *pp);
 void free_path (struct path *);
 void free_pathvec (vector vec, enum free_path_mode free_paths);
 void free_pathgroup (struct pathgroup * pgp, enum free_path_mode free_paths);
 diff --git a/libmultipath/structs_vec.c b/libmultipath/structs_vec.c
 index 24ac022e..9613252f 100644
 --- a/libmultipath/structs_vec.c
 +++ b/libmultipath/structs_vec.c
@@ -180,24 +180,24 @@ extract_hwe_from_path(struct multipath * mpp)
 	if (mpp->hwe || !mpp->paths)
 		return;
 -	condlog(3, "%s: searching paths for valid hwe", mpp->alias);
 +	condlog(4, "%s: searching paths for valid hwe", mpp->alias);
 	/* doing this in two passes seems like paranoia to me */
 	vector_foreach_slot(mpp->paths, pp, i) {
 -		if (pp->state != PATH_UP)
 -			continue;
 -		if (pp->hwe) {
 -			mpp->hwe = pp->hwe;
 -			return;
 -		}
 +		if (pp->state == PATH_UP && pp->hwe)
 +			goto done;
 	}
 	vector_foreach_slot(mpp->paths, pp, i) {
 -		if (pp->state == PATH_UP)
 -			continue;
 -		if (pp->hwe) {
 -			mpp->hwe = pp->hwe;
 -			return;
 -		}
 +		if (pp->state != PATH_UP && pp->hwe)
 +			goto done;
 	}
 +done:
 +	if (i < VECTOR_SIZE(mpp->paths))
 +		(void)set_mpp_hwe(mpp, pp);
 +
 +	if (mpp->hwe)
 +		condlog(3, "%s: got hwe from path %s", mpp->alias, pp->dev);
 +	else
 +		condlog(2, "%s: no hwe found", mpp->alias);
 }
 int
@@ -445,9 +445,15 @@ struct multipath *add_map_with_path(struct vectors *vecs, struct path *pp,
 	conf = get_multipath_config();
 	mpp->mpe = find_mpe(conf->mptable, pp->wwid);
 -	mpp->hwe = pp->hwe;
 	put_multipath_config(conf);
 +	/*
 +	 * We need to call this before select_alias(),
 +	 * because that accesses hwe properties.
 +	 */
 +	if (pp->hwe && !set_mpp_hwe(mpp, pp))
 +		goto out;
 +
 	strcpy(mpp->wwid, pp->wwid);
 	find_existing_alias(mpp, vecs);
 	if (select_alias(conf, mpp))
@@ -497,12 +503,6 @@ int verify_paths(struct multipath *mpp, struct vectors *vecs)
 			vector_del_slot(mpp->paths, i);
 			i--;
 -			/* Make sure mpp->hwe doesn't point to freed memory.
 -			 * We call extract_hwe_from_path() below to restore
 -			 * mpp->hwe
 -			 */
 -			if (mpp->hwe == pp->hwe)
 -				mpp->hwe = NULL;
 			if ((j = find_slot(vecs->pathvec,
 					   (void *)pp)) != -1)
 				vector_del_slot(vecs->pathvec, j);
@@ -512,7 +512,6 @@ int verify_paths(struct multipath *mpp, struct vectors *vecs)
 				mpp->alias, pp->dev, pp->dev_t);
 		}
 	}
 -	extract_hwe_from_path(mpp);
 	return count;
 }
 diff --git a/multipathd/main.c b/multipathd/main.c
 index cd68a9d2..769dcaee 100644
 --- a/multipathd/main.c
 +++ b/multipathd/main.c
@@ -1198,13 +1198,6 @@ ev_remove_path (struct path *pp, struct vectors * vecs, int need_do_map)
 			goto fail;
 		}
 -		/*
 -		 * Make sure mpp->hwe doesn't point to freed memory
 -		 * We call extract_hwe_from_path() below to restore mpp->hwe
 -		 */
 -		if (mpp->hwe == pp->hwe)
 -			mpp->hwe = NULL;
 -
 		if ((i = find_slot(mpp->paths, (void *)pp)) != -1)
 			vector_del_slot(mpp->paths, i);
@@ -1216,9 +1209,6 @@ ev_remove_path (struct path *pp, struct vectors * vecs, int need_do_map)
 		    flush_map_nopaths(mpp, vecs))
 			goto out;
 -		if (mpp->hwe == NULL)
 -			extract_hwe_from_path(mpp);
 -
 		if (setup_map(mpp, params, PARAMS_SIZE, vecs)) {
 			condlog(0, "%s: failed to setup map for"
 				" removal of path %s", mpp->alias, pp->dev);
--- a/SOURCES/0127-libmultipath-don-t-leak-memory-on-invalid-strings.patch
+++ b/SOURCES/0127-libmultipath-don-t-leak-memory-on-invalid-strings.patch
@ -0,0 +1,35 @@
 From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
 From: Benjamin Marzinski <bmarzins@redhat.com>
 Date: Wed, 14 Dec 2022 15:38:19 -0600
 Subject: [PATCH] libmultipath: don't leak memory on invalid strings
 If set_path() or set_str_noslash() are called with a bad value, they
 ignore it and continue to use the old value. But they weren't freeing
 the bad value, causing a memory leak.
 Signed-off-by: Benjamin Marzinski <bmarzins@redhat.com>
 Reviewed-by: Martin Wilck <mwilck@suse.com>
 ---
 libmultipath/dict.c | 2 ++
 1 file changed, 2 insertions(+)
 diff --git a/libmultipath/dict.c b/libmultipath/dict.c
 index d7cd94a5..a8c9e989 100644
 --- a/libmultipath/dict.c
 +++ b/libmultipath/dict.c
@@ -169,6 +169,7 @@ set_path(vector strvec, void *ptr, const char *file, int line_nr)
 	if ((*str_ptr)[0] != '/'){
 		condlog(1, "%s line %d, %s is not an absolute path. Ignoring",
 			file, line_nr, *str_ptr);
 +		free(*str_ptr);
 		*str_ptr = old_str;
 	} else
 		free(old_str);
@@ -189,6 +190,7 @@ set_str_noslash(vector strvec, void *ptr, const char *file, int line_nr)
 	if (strchr(*str_ptr, '/')) {
 		condlog(1, "%s line %d, %s cannot contain a slash. Ignoring",
 			file, line_nr, *str_ptr);
 +		free(*str_ptr);
 		*str_ptr = old_str;
 	} else
 		free(old_str);
--- a/SOURCES/0128-libmutipath-validate-the-argument-count-of-config-st.patch
+++ b/SOURCES/0128-libmutipath-validate-the-argument-count-of-config-st.patch
@ -0,0 +1,195 @@
 From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
 From: Benjamin Marzinski <bmarzins@redhat.com>
 Date: Wed, 14 Dec 2022 15:38:20 -0600
 Subject: [PATCH] libmutipath: validate the argument count of config strings
 The features, path_selector, and hardware_handler config options pass
 their strings directly into the kernel.  If users omit the argument
 counts from these strings, or use the wrong value, the kernel's table
 parsing gets completely messed up, and the error messages it prints
 don't reflect what actully went wrong. To avoid messing up the
 kernel table parsing, verify that these strings correctly set the
 argument count to the number of arguments they have.
 Signed-off-by: Benjamin Marzinski <bmarzins@redhat.com>
 Reviewed-by: Martin Wilck <mwilck@suse.com>
 ---
 libmultipath/dict.c | 110 ++++++++++++++++++++++++++++++++++++++++----
 1 file changed, 101 insertions(+), 9 deletions(-)
 diff --git a/libmultipath/dict.c b/libmultipath/dict.c
 index a8c9e989..952b33d0 100644
 --- a/libmultipath/dict.c
 +++ b/libmultipath/dict.c
@@ -155,6 +155,58 @@ set_dir(vector strvec, void *ptr, const char *file, int line_nr)
 	return 0;
 }
 +static int
 +set_arg_str(vector strvec, void *ptr, int count_idx, const char *file,
 +	    int line_nr)
 +{
 +	char **str_ptr = (char **)ptr;
 +	char *old_str = *str_ptr;
 +	const char * const spaces = " \f\r\t\v";
 +	char *p, *end;
 +	int idx = -1;
 +	long int count = -1;
 +
 +	*str_ptr = set_value(strvec);
 +	if (!*str_ptr) {
 +		free(old_str);
 +		return 1;
 +	}
 +	p = *str_ptr;
 +	while (*p != '\0') {
 +		p += strspn(p, spaces);
 +		if (*p == '\0')
 +			break;
 +		idx += 1;
 +		if (idx == count_idx) {
 +			errno = 0;
 +			count = strtol(p, &end, 10);
 +			if (errno == ERANGE || end == p ||
 +			    !(isspace(*end) || *end == '\0')) {
 +				count = -1;
 +				break;
 +			}
 +		}
 +		p += strcspn(p, spaces);
 +	}
 +	if (count < 0) {
 +		condlog(1, "%s line %d, missing argument count for %s",
 +			file, line_nr, (char*)VECTOR_SLOT(strvec, 0));
 +		goto fail;
 +	}
 +	if (count != idx - count_idx) {
 +		condlog(1, "%s line %d, invalid argument count for %s:, got '%ld' expected '%d'",
 +			file, line_nr, (char*)VECTOR_SLOT(strvec, 0), count,
 +			idx - count_idx);
 +		goto fail;
 +	}
 +	free(old_str);
 +	return 0;
 +fail:
 +	free(*str_ptr);
 +	*str_ptr = old_str;
 +	return 0;
 +}
 +
 static int
 set_path(vector strvec, void *ptr, const char *file, int line_nr)
 {
@@ -337,6 +389,14 @@ def_ ## option ## _handler (struct config *conf, vector strvec,         \
 	return set_int(strvec, &conf->option, minval, maxval, file, line_nr); \
 }
 +#define declare_def_arg_str_handler(option, count_idx)			\
 +static int								\
 +def_ ## option ## _handler (struct config *conf, vector strvec,		\
 +			    const char *file, int line_nr)		\
 +{									\
 +	return set_arg_str(strvec, &conf->option, count_idx, file, line_nr); \
 +}
 +
 #define declare_def_snprint(option, function)				\
 static int								\
 snprint_def_ ## option (struct config *conf, char * buff, int len,	\
@@ -389,6 +449,17 @@ hw_ ## option ## _handler (struct config *conf, vector strvec,		\
 	return set_int(strvec, &hwe->option, minval, maxval, file, line_nr); \
 }
 +#define declare_hw_arg_str_handler(option, count_idx)			\
 +static int								\
 +hw_ ## option ## _handler (struct config *conf, vector strvec,		\
 +			    const char *file, int line_nr)		\
 +{									\
 +	struct hwentry * hwe = VECTOR_LAST_SLOT(conf->hwtable);		\
 +	if (!hwe)							\
 +		return 1;						\
 +	return set_arg_str(strvec, &hwe->option, count_idx, file, line_nr); \
 +}
 +
 #define declare_hw_snprint(option, function)				\
 static int								\
@@ -420,6 +491,16 @@ ovr_ ## option ## _handler (struct config *conf, vector strvec,		\
 		       file, line_nr); \
 }
 +#define declare_ovr_arg_str_handler(option, count_idx)			\
 +static int								\
 +ovr_ ## option ## _handler (struct config *conf, vector strvec,		\
 +			    const char *file, int line_nr)		\
 +{									\
 +	if (!conf->overrides)						\
 +		return 1;						\
 +	return set_arg_str(strvec, &conf->overrides->option, count_idx, file, line_nr); \
 +}
 +
 #define declare_ovr_snprint(option, function)				\
 static int								\
 snprint_ovr_ ## option (struct config *conf, char * buff, int len,	\
@@ -450,6 +531,17 @@ mp_ ## option ## _handler (struct config *conf, vector strvec,		\
 	return set_int(strvec, &mpe->option, minval, maxval, file, line_nr); \
 }
 +#define declare_mp_arg_str_handler(option, count_idx)			\
 +static int								\
 +mp_ ## option ## _handler (struct config *conf, vector strvec,		\
 +			    const char *file, int line_nr)		\
 +{									\
 +	struct mpentry * mpe = VECTOR_LAST_SLOT(conf->mptable);		\
 +	if (!mpe)							\
 +		return 1;						\
 +	return set_arg_str(strvec, &mpe->option, count_idx, file, line_nr); \
 +}
 +
 #define declare_mp_snprint(option, function)				\
 static int								\
 snprint_mp_ ## option (struct config *conf, char * buff, int len,	\
@@ -634,13 +726,13 @@ snprint_def_marginal_pathgroups(struct config *conf, char *buff, int len,
 }
 -declare_def_handler(selector, set_str)
 +declare_def_arg_str_handler(selector, 1)
 declare_def_snprint_defstr(selector, print_str, DEFAULT_SELECTOR)
 -declare_hw_handler(selector, set_str)
 +declare_hw_arg_str_handler(selector, 1)
 declare_hw_snprint(selector, print_str)
 -declare_ovr_handler(selector, set_str)
 +declare_ovr_arg_str_handler(selector, 1)
 declare_ovr_snprint(selector, print_str)
 -declare_mp_handler(selector, set_str)
 +declare_mp_arg_str_handler(selector, 1)
 declare_mp_snprint(selector, print_str)
 static int snprint_uid_attrs(struct config *conf, char *buff, int len,
@@ -717,13 +809,13 @@ declare_hw_snprint(prio_args, print_str)
 declare_mp_handler(prio_args, set_str)
 declare_mp_snprint(prio_args, print_str)
 -declare_def_handler(features, set_str)
 +declare_def_arg_str_handler(features, 0)
 declare_def_snprint_defstr(features, print_str, DEFAULT_FEATURES)
 -declare_ovr_handler(features, set_str)
 +declare_ovr_arg_str_handler(features, 0)
 declare_ovr_snprint(features, print_str)
 -declare_hw_handler(features, set_str)
 +declare_hw_arg_str_handler(features, 0)
 declare_hw_snprint(features, print_str)
 -declare_mp_handler(features, set_str)
 +declare_mp_arg_str_handler(features, 0)
 declare_mp_snprint(features, print_str)
 declare_def_handler(checker_name, set_str)
@@ -1894,7 +1986,7 @@ declare_hw_snprint(revision, print_str)
 declare_hw_handler(bl_product, set_regex)
 declare_hw_snprint(bl_product, print_str)
 -declare_hw_handler(hwhandler, set_str)
 +declare_hw_arg_str_handler(hwhandler, 0)
 declare_hw_snprint(hwhandler, print_str)
 /*
--- a/SOURCES/0129-libmultipath-select-resize-action-even-if-reload-is-.patch
+++ b/SOURCES/0129-libmultipath-select-resize-action-even-if-reload-is-.patch
@ -0,0 +1,45 @@
 From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
 From: Benjamin Marzinski <bmarzins@redhat.com>
 Date: Tue, 31 Jan 2023 09:58:55 -0600
 Subject: [PATCH] libmultipath: select resize action even if reload is forced
 The ACT_RESIZE action is the same as the ACT_RELOAD action, except that
 it flushes outstanding IO because the device size is changing and
 the new size might be too small for some of the outstanding IO. If we've
 detected a size change, and a forced reload is requested, we still need
 to flush the IO because the reload will change the device size.
 Signed-off-by: Benjamin Marzinski <bmarzins@redhat.com>
 ---
 libmultipath/configure.c | 13 +++++++------
 1 file changed, 7 insertions(+), 6 deletions(-)
 diff --git a/libmultipath/configure.c b/libmultipath/configure.c
 index 6e06fea2..ecf24f95 100644
 --- a/libmultipath/configure.c
 +++ b/libmultipath/configure.c
@@ -734,17 +734,18 @@ select_action (struct multipath * mpp, vector curmp, int force_reload)
 		return;
 	}
 -	if (force_reload) {
 +	if (cmpp->size != mpp->size) {
 		mpp->force_udev_reload = 1;
 -		mpp->action = ACT_RELOAD;
 -		condlog(3, "%s: set ACT_RELOAD (forced by user)",
 +		mpp->action = ACT_RESIZE;
 +		condlog(3, "%s: set ACT_RESIZE (size change)",
 			mpp->alias);
 		return;
 	}
 -	if (cmpp->size != mpp->size) {
 +
 +	if (force_reload) {
 		mpp->force_udev_reload = 1;
 -		mpp->action = ACT_RESIZE;
 -		condlog(3, "%s: set ACT_RESIZE (size change)",
 +		mpp->action = ACT_RELOAD;
 +		condlog(3, "%s: set ACT_RELOAD (forced by user)",
 			mpp->alias);
 		return;
 	}
--- a/SOURCES/0130-libmultipath-cleanup-ACT_CREATE-code-in-select_actio.patch
+++ b/SOURCES/0130-libmultipath-cleanup-ACT_CREATE-code-in-select_actio.patch
@ -0,0 +1,67 @@
 From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
 From: Benjamin Marzinski <bmarzins@redhat.com>
 Date: Tue, 31 Jan 2023 10:35:10 -0600
 Subject: [PATCH] libmultipath: cleanup ACT_CREATE code in select_action
 Combine the two separate blocks that set ACT_CREATE into one.
 Signed-off-by: Benjamin Marzinski <bmarzins@redhat.com>
 ---
 libmultipath/configure.c | 38 +++++++++++++++++---------------------
 1 file changed, 17 insertions(+), 21 deletions(-)
 diff --git a/libmultipath/configure.c b/libmultipath/configure.c
 index ecf24f95..303d2380 100644
 --- a/libmultipath/configure.c
 +++ b/libmultipath/configure.c
@@ -693,33 +693,29 @@ select_action (struct multipath * mpp, vector curmp, int force_reload)
 	cmpp = find_mp_by_wwid(curmp, mpp->wwid);
 	cmpp_by_name = find_mp_by_alias(curmp, mpp->alias);
 -	if (!cmpp_by_name) {
 -		if (cmpp) {
 -			condlog(2, "%s: rename %s to %s", mpp->wwid,
 -				cmpp->alias, mpp->alias);
 -			strlcpy(mpp->alias_old, cmpp->alias, WWID_SIZE);
 -			mpp->action = ACT_RENAME;
 -			if (force_reload) {
 -				mpp->force_udev_reload = 1;
 -				mpp->action = ACT_FORCERENAME;
 -			}
 -			return;
 +	if (!cmpp) {
 +		if (cmpp_by_name) {
 +			condlog(1, "%s: can't use alias \"%s\" used by %s, falling back to WWID",
 +				mpp->wwid, mpp->alias, cmpp_by_name->wwid);
 +			/* We can do this because wwid wasn't found */
 +			free(mpp->alias);
 +			mpp->alias = strdup(mpp->wwid);
 		}
 		mpp->action = ACT_CREATE;
 -		condlog(3, "%s: set ACT_CREATE (map does not exist)",
 -			mpp->alias);
 +		condlog(3, "%s: set ACT_CREATE (map does not exist%s)",
 +			mpp->alias, cmpp_by_name ? ", name changed" : "");
 		return;
 	}
 -	if (!cmpp) {
 -		condlog(1, "%s: can't use alias \"%s\" used by %s, falling back to WWID",
 -			mpp->wwid, mpp->alias, cmpp_by_name->wwid);
 -		/* We can do this because wwid wasn't found */
 -		free(mpp->alias);
 -		mpp->alias = strdup(mpp->wwid);
 -		mpp->action = ACT_CREATE;
 -		condlog(3, "%s: set ACT_CREATE (map does not exist, name changed)",
 +	if (!cmpp_by_name) {
 +		condlog(2, "%s: rename %s to %s", mpp->wwid, cmpp->alias,
 			mpp->alias);
 +		strlcpy(mpp->alias_old, cmpp->alias, WWID_SIZE);
 +		mpp->action = ACT_RENAME;
 +		if (force_reload) {
 +			mpp->force_udev_reload = 1;
 +			mpp->action = ACT_FORCERENAME;
 +		}
 		return;
 	}
--- a/SOURCES/0131-libmultipath-keep-renames-from-stopping-other-multip.patch
+++ b/SOURCES/0131-libmultipath-keep-renames-from-stopping-other-multip.patch
@ -0,0 +1,270 @@
 From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
 From: Benjamin Marzinski <bmarzins@redhat.com>
 Date: Tue, 31 Jan 2023 12:00:31 -0600
 Subject: [PATCH] libmultipath: keep renames from stopping other multipath
 actions
 If select_action() is called and a multipath device needs to be renamed,
 the code currently checks if force_reload is set, and if so, does the
 reload after the rename.  But if force_reload isn't set, only the rename
 happens, regardless of what other actions are needed. This can happen if
 multipathd starts up and a device needs both a reload and a rename.
 Make multipath check for resize, reload, and switch pathgroup along with
 rename, and do both if necessary.
 Signed-off-by: Benjamin Marzinski <bmarzins@redhat.com>
 ---
 libmultipath/configure.c | 92 +++++++++++++++++++++-------------------
 libmultipath/configure.h |  4 +-
 2 files changed, 51 insertions(+), 45 deletions(-)
 diff --git a/libmultipath/configure.c b/libmultipath/configure.c
 index 303d2380..65a0b208 100644
 --- a/libmultipath/configure.c
 +++ b/libmultipath/configure.c
@@ -690,6 +690,7 @@ select_action (struct multipath * mpp, vector curmp, int force_reload)
 	struct multipath * cmpp_by_name;
 	char * mpp_feat, * cmpp_feat;
 +	mpp->action = ACT_NOTHING;
 	cmpp = find_mp_by_wwid(curmp, mpp->wwid);
 	cmpp_by_name = find_mp_by_alias(curmp, mpp->alias);
@@ -712,14 +713,8 @@ select_action (struct multipath * mpp, vector curmp, int force_reload)
 			mpp->alias);
 		strlcpy(mpp->alias_old, cmpp->alias, WWID_SIZE);
 		mpp->action = ACT_RENAME;
 -		if (force_reload) {
 -			mpp->force_udev_reload = 1;
 -			mpp->action = ACT_FORCERENAME;
 -		}
 -		return;
 -	}
 -
 -	if (cmpp != cmpp_by_name) {
 +		/* don't return here. Check for other needed actions */
 +	} else if (cmpp != cmpp_by_name) {
 		condlog(2, "%s: unable to rename %s to %s (%s is used by %s)",
 			mpp->wwid, cmpp->alias, mpp->alias,
 			mpp->alias, cmpp_by_name->wwid);
@@ -727,12 +722,13 @@ select_action (struct multipath * mpp, vector curmp, int force_reload)
 		FREE(mpp->alias);
 		mpp->alias = STRDUP(cmpp->alias);
 		mpp->action = ACT_IMPOSSIBLE;
 -		return;
 +		/* don't return here. Check for other needed actions */
 	}
 	if (cmpp->size != mpp->size) {
 		mpp->force_udev_reload = 1;
 -		mpp->action = ACT_RESIZE;
 +		mpp->action = mpp->action == ACT_RENAME ? ACT_RESIZE_RENAME :
 +			      ACT_RESIZE;
 		condlog(3, "%s: set ACT_RESIZE (size change)",
 			mpp->alias);
 		return;
@@ -740,7 +736,8 @@ select_action (struct multipath * mpp, vector curmp, int force_reload)
 	if (force_reload) {
 		mpp->force_udev_reload = 1;
 -		mpp->action = ACT_RELOAD;
 +		mpp->action = mpp->action == ACT_RENAME ? ACT_RELOAD_RENAME :
 +			      ACT_RELOAD;
 		condlog(3, "%s: set ACT_RELOAD (forced by user)",
 			mpp->alias);
 		return;
@@ -749,7 +746,8 @@ select_action (struct multipath * mpp, vector curmp, int force_reload)
 	if (mpp->no_path_retry != NO_PATH_RETRY_UNDEF &&
 	    !!strstr(mpp->features, "queue_if_no_path") !=
 	    !!strstr(cmpp->features, "queue_if_no_path")) {
 -		mpp->action =  ACT_RELOAD;
 +		mpp->action = mpp->action == ACT_RENAME ? ACT_RELOAD_RENAME :
 +			      ACT_RELOAD;
 		condlog(3, "%s: set ACT_RELOAD (no_path_retry change)",
 			mpp->alias);
 		return;
@@ -759,7 +757,8 @@ select_action (struct multipath * mpp, vector curmp, int force_reload)
 	    (strlen(cmpp->hwhandler) != strlen(mpp->hwhandler) ||
 	     strncmp(cmpp->hwhandler, mpp->hwhandler,
 		    strlen(mpp->hwhandler)))) {
 -		mpp->action = ACT_RELOAD;
 +		mpp->action = mpp->action == ACT_RENAME ? ACT_RELOAD_RENAME :
 +			      ACT_RELOAD;
 		condlog(3, "%s: set ACT_RELOAD (hwhandler change)",
 			mpp->alias);
 		return;
@@ -769,7 +768,8 @@ select_action (struct multipath * mpp, vector curmp, int force_reload)
 	    !!strstr(mpp->features, "retain_attached_hw_handler") !=
 	    !!strstr(cmpp->features, "retain_attached_hw_handler") &&
 	    get_linux_version_code() < KERNEL_VERSION(4, 3, 0)) {
 -		mpp->action = ACT_RELOAD;
 +		mpp->action = mpp->action == ACT_RENAME ? ACT_RELOAD_RENAME :
 +			      ACT_RELOAD;
 		condlog(3, "%s: set ACT_RELOAD (retain_hwhandler change)",
 			mpp->alias);
 		return;
@@ -783,9 +783,13 @@ select_action (struct multipath * mpp, vector curmp, int force_reload)
 		remove_feature(&cmpp_feat, "queue_if_no_path");
 		remove_feature(&cmpp_feat, "retain_attached_hw_handler");
 		if (strncmp(mpp_feat, cmpp_feat, PARAMS_SIZE)) {
 -			mpp->action =  ACT_RELOAD;
 +			mpp->action = mpp->action == ACT_RENAME ?
 +				      ACT_RELOAD_RENAME : ACT_RELOAD;
 			condlog(3, "%s: set ACT_RELOAD (features change)",
 				mpp->alias);
 +			FREE(cmpp_feat);
 +			FREE(mpp_feat);
 +			return;
 		}
 	}
 	FREE(cmpp_feat);
@@ -793,44 +797,49 @@ select_action (struct multipath * mpp, vector curmp, int force_reload)
 	if (!cmpp->selector || strncmp(cmpp->selector, mpp->selector,
 		    strlen(mpp->selector))) {
 -		mpp->action = ACT_RELOAD;
 +		mpp->action = mpp->action == ACT_RENAME ? ACT_RELOAD_RENAME :
 +			      ACT_RELOAD;
 		condlog(3, "%s: set ACT_RELOAD (selector change)",
 			mpp->alias);
 		return;
 	}
 	if (cmpp->minio != mpp->minio) {
 -		mpp->action = ACT_RELOAD;
 +		mpp->action = mpp->action == ACT_RENAME ? ACT_RELOAD_RENAME :
 +			      ACT_RELOAD;
 		condlog(3, "%s: set ACT_RELOAD (minio change, %u->%u)",
 			mpp->alias, cmpp->minio, mpp->minio);
 		return;
 	}
 	if (!cmpp->pg || VECTOR_SIZE(cmpp->pg) != VECTOR_SIZE(mpp->pg)) {
 -		mpp->action = ACT_RELOAD;
 +		mpp->action = mpp->action == ACT_RENAME ? ACT_RELOAD_RENAME :
 +			      ACT_RELOAD;
 		condlog(3, "%s: set ACT_RELOAD (path group number change)",
 			mpp->alias);
 		return;
 	}
 	if (pgcmp(mpp, cmpp)) {
 -		mpp->action = ACT_RELOAD;
 +		mpp->action = mpp->action == ACT_RENAME ? ACT_RELOAD_RENAME :
 +			      ACT_RELOAD;
 		condlog(3, "%s: set ACT_RELOAD (path group topology change)",
 			mpp->alias);
 		return;
 	}
 	if (cmpp->nextpg != mpp->bestpg) {
 -		mpp->action = ACT_SWITCHPG;
 +		mpp->action = mpp->action == ACT_RENAME ? ACT_SWITCHPG_RENAME :
 +			      ACT_SWITCHPG;
 		condlog(3, "%s: set ACT_SWITCHPG (next path group change)",
 			mpp->alias);
 		return;
 	}
 	if (!is_mpp_known_to_udev(cmpp)) {
 -		mpp->action = ACT_RELOAD;
 +		mpp->action = mpp->action == ACT_RENAME ? ACT_SWITCHPG_RENAME :
 +			      ACT_SWITCHPG;
 		condlog(3, "%s: set ACT_RELOAD (udev device not initialized)",
 			mpp->alias);
 		return;
 	}
 -	mpp->action = ACT_NOTHING;
 -	condlog(3, "%s: set ACT_NOTHING (map unchanged)",
 -		mpp->alias);
 +	if (mpp->action == ACT_NOTHING)
 +		condlog(3, "%s: set ACT_NOTHING (map unchanged)", mpp->alias);
 	return;
 }
@@ -924,6 +933,17 @@ int domap(struct multipath *mpp, char *params, int is_daemon)
 		mpp->action = ACT_RELOAD;
 	}
 +	if (mpp->action == ACT_RENAME || mpp->action == ACT_SWITCHPG_RENAME ||
 +	    mpp->action == ACT_RELOAD_RENAME ||
 +	    mpp->action == ACT_RESIZE_RENAME) {
 +		conf = get_multipath_config();
 +		pthread_cleanup_push(put_multipath_config, conf);
 +		r = dm_rename(mpp->alias_old, mpp->alias,
 +			      conf->partition_delim, mpp->skip_kpartx);
 +		pthread_cleanup_pop(1);
 +		if (r == DOMAP_FAIL)
 +			return r;
 +	}
 	switch (mpp->action) {
 	case ACT_REJECT:
 	case ACT_NOTHING:
@@ -931,6 +951,7 @@ int domap(struct multipath *mpp, char *params, int is_daemon)
 		return DOMAP_EXIST;
 	case ACT_SWITCHPG:
 +	case ACT_SWITCHPG_RENAME:
 		dm_switchgroup(mpp->alias, mpp->bestpg);
 		/*
 		 * we may have avoided reinstating paths because there where in
@@ -957,6 +978,7 @@ int domap(struct multipath *mpp, char *params, int is_daemon)
 		break;
 	case ACT_RELOAD:
 +	case ACT_RELOAD_RENAME:
 		sysfs_set_max_sectors_kb(mpp, 1);
 		if (mpp->ghost_delay_tick > 0 && pathcount(mpp, PATH_UP))
 			mpp->ghost_delay_tick = 0;
@@ -964,6 +986,7 @@ int domap(struct multipath *mpp, char *params, int is_daemon)
 		break;
 	case ACT_RESIZE:
 +	case ACT_RESIZE_RENAME:
 		sysfs_set_max_sectors_kb(mpp, 1);
 		if (mpp->ghost_delay_tick > 0 && pathcount(mpp, PATH_UP))
 			mpp->ghost_delay_tick = 0;
@@ -971,29 +994,10 @@ int domap(struct multipath *mpp, char *params, int is_daemon)
 		break;
 	case ACT_RENAME:
 -		conf = get_multipath_config();
 -		pthread_cleanup_push(put_multipath_config, conf);
 -		r = dm_rename(mpp->alias_old, mpp->alias,
 -			      conf->partition_delim, mpp->skip_kpartx);
 -		pthread_cleanup_pop(1);
 -		break;
 -
 -	case ACT_FORCERENAME:
 -		conf = get_multipath_config();
 -		pthread_cleanup_push(put_multipath_config, conf);
 -		r = dm_rename(mpp->alias_old, mpp->alias,
 -			      conf->partition_delim, mpp->skip_kpartx);
 -		pthread_cleanup_pop(1);
 -		if (r) {
 -			sysfs_set_max_sectors_kb(mpp, 1);
 -			if (mpp->ghost_delay_tick > 0 &&
 -			    pathcount(mpp, PATH_UP))
 -				mpp->ghost_delay_tick = 0;
 -			r = dm_addmap_reload(mpp, params, 0);
 -		}
 		break;
 	default:
 +		r = DOMAP_FAIL;
 		break;
 	}
 diff --git a/libmultipath/configure.h b/libmultipath/configure.h
 index 5cf08d45..1a93f49d 100644
 --- a/libmultipath/configure.h
 +++ b/libmultipath/configure.h
@@ -18,9 +18,11 @@ enum actions {
 	ACT_RENAME,
 	ACT_CREATE,
 	ACT_RESIZE,
 -	ACT_FORCERENAME,
 +	ACT_RELOAD_RENAME,
 	ACT_DRY_RUN,
 	ACT_IMPOSSIBLE,
 +	ACT_RESIZE_RENAME,
 +	ACT_SWITCHPG_RENAME,
 };
 /*
--- a/SPECS/device-mapper-multipath.spec
+++ b/SPECS/device-mapper-multipath.spec
@ -1,7 +1,7 @@
 Summary: Tools to manage multipath devices using device-mapper
 Name: device-mapper-multipath
 Version: 0.8.4
-Release: 27%{?dist}
+Release: 37%{?dist}
 License: GPLv2
 Group: System Environment/Base
 URL: http://christophe.varoqui.free.fr/
@ -120,6 +120,28 @@ Patch00106: 0106-multipathd-disallow-changing-to-from-fpin-marginal-p.patch
 Patch00107: 0107-libmultipath-unset-detect_checker-for-clariion-Unity.patch
 Patch00108: 0108-multipathd-Add-missing-ctype-include.patch
 Patch00109: 0109-multipathd-replace-libreadline-with-libedit.patch
 Patch00110: 0110-multipath-fix-systemd-timers-in-the-initramfs.patch
 Patch00111: 0111-multipathd-factor-out-the-code-to-flush-a-map-with-n.patch
 Patch00112: 0112-libmultipath-return-success-if-we-raced-to-remove-a-.patch
 Patch00113: 0113-multipathd-Handle-losing-all-path-in-update_map.patch
 Patch00114: 0114-multipathd-ignore-duplicated-multipathd-command-keys.patch
 Patch00115: 0115-multipath-tools-use-run-instead-of-dev-shm.patch
 Patch00116: 0116-kpartx-hold-device-open-until-partitions-have-been-c.patch
 Patch00117: 0117-libmultipath-cleanup-remove_feature.patch
 Patch00118: 0118-libmultipath-cleanup-add_feature.patch
 Patch00119: 0119-multipath-tests-tests-for-adding-and-removing-featur.patch
 Patch00120: 0120-libmultipath-fix-queue_mode-feature-handling.patch
 Patch00121: 0121-multipath-tests-tests-for-reconcile_features_with_qu.patch
 Patch00122: 0122-libmultipath-prepare-proto_id-for-use-by-non-scsi-de.patch
 Patch00123: 0123-libmultipath-get-nvme-path-transport-protocol.patch
 Patch00124: 0124-libmultipath-enforce-queue_mode-bio-for-nmve-tcp-pat.patch
 Patch00125: 0125-multipath-add-historical-service-time-to-the-man-pag.patch
 Patch00126: 0126-libmultipath-copy-mpp-hwe-from-pp-hwe.patch
 Patch00127: 0127-libmultipath-don-t-leak-memory-on-invalid-strings.patch
 Patch00128: 0128-libmutipath-validate-the-argument-count-of-config-st.patch
 Patch00129: 0129-libmultipath-select-resize-action-even-if-reload-is-.patch
 Patch00130: 0130-libmultipath-cleanup-ACT_CREATE-code-in-select_actio.patch
 Patch00131: 0131-libmultipath-keep-renames-from-stopping-other-multip.patch
 # runtime
 Requires: %{name}-libs = %{version}-%{release}
@ -218,7 +240,8 @@ make install \
 	rcdir=%{_initrddir} \
 	unitdir=%{_unitdir} \
 	includedir=%{_includedir} \
-	pkgconfdir=%{_pkgconfdir}
+	pkgconfdir=%{_pkgconfdir} \
 	tmpfilesdir=%{_tmpfilesdir}
 # tree fix up
 install -d %{buildroot}/etc/multipath
@ -257,6 +280,7 @@ fi
 %{_mandir}/man8/mpathpersist.8.gz
 %config %{_udevrulesdir}/62-multipath.rules
 %config %{_udevrulesdir}/11-dm-mpath.rules
 %{_tmpfilesdir}/multipath.conf
 %{!?_licensedir:%global license %%doc}
 %license LICENSES/GPL-2.0 LICENSES/LGPL-2.0
 %doc README
@ -322,6 +346,70 @@ fi
 %{_pkgconfdir}/libdmmp.pc
 %changelog
 * Fri Feb  3 2023 Benjamin Marzinski <bmarzins@redhat.com> 0.8.4-37
 - Fix bugzilla linked to the changes (was previously linked to
  the wrong bug, 2162537)
 - Resolves: bz #2166468
 * Wed Feb  1 2023 Benjamin Marzinski <bmarzins@redhat.com> 0.8.4-36
 - Add 0129-libmultipath-select-resize-action-even-if-reload-is-.patch
 - Add 0130-libmultipath-cleanup-ACT_CREATE-code-in-select_actio.patch
 - Add 0131-libmultipath-keep-renames-from-stopping-other-multip.patch
 - Resolves: bz #2166468
 * Mon Jan 16 2023 Benjamin Marzinski <bmarzins@redhat.com> 0.8.4-35
 - Add 0127-libmultipath-don-t-leak-memory-on-invalid-strings.patch
 - Add 0128-libmutipath-validate-the-argument-count-of-config-st.patch
 - Resolves: bz #2155560
 * Tue Nov 29 2022 Benjamin Marzinski <bmarzins@redhat.com> 0.8.4-34
 - Add 0126-libmultipath-copy-mpp-hwe-from-pp-hwe.patch
  * Fixes bz #2126714
 - Cleanup multiple CI tests
 - Resolves: bz #2126714
 * Wed Nov 23 2022 Benjamin Marzinski <bmarzins@redhat.com> 0.8.4-33
 - Add 0125-multipath-add-historical-service-time-to-the-man-pag.patch
  * Fixes bz #2141996
 - Modify tests/multipath_conf_syntax/main.sh
  * fix unrelated test error
 - Resolves: bz #2141996
 * Thu Nov 10 2022 Benjamin Marzinski <bmarzins@redhat.com> 0.8.4-32
 - Add 0116-kpartx-hold-device-open-until-partitions-have-been-c.patch
  * Fixes bz #2128885
 - Add 0117-libmultipath-cleanup-remove_feature.patch
 - Add 0118-libmultipath-cleanup-add_feature.patch
 - Add 0119-multipath-tests-tests-for-adding-and-removing-featur.patch
 - Add 0120-libmultipath-fix-queue_mode-feature-handling.patch
 - Add 0121-multipath-tests-tests-for-reconcile_features_with_qu.patch
 - Add 0122-libmultipath-prepare-proto_id-for-use-by-non-scsi-de.patch
 - Add 0123-libmultipath-get-nvme-path-transport-protocol.patch
 - Add 0124-libmultipath-enforce-queue_mode-bio-for-nmve-tcp-pat.patch
  * Fixes bz #2022359
 - Resolves: bz #2022359, #2128885
 * Thu Oct 13 2022 Benjamin Marzinski <bmarzins@redhat.com> 0.8.4-31
 - Add 0114-multipathd-ignore-duplicated-multipathd-command-keys.patch
  * Fixes bz #2133996
 - Add 0115-multipath-tools-use-run-instead-of-dev-shm.patch
  * Fixes bz #2133990
 - Resolves: bz #2133990, #2133996
 * Fri Sep  9 2022 Benjamin Marzinski <bmarzins@redhat.com> 0.8.4-30
 - Add 0111-multipathd-factor-out-the-code-to-flush-a-map-with-n.patch
 - Add 0112-libmultipath-return-success-if-we-raced-to-remove-a-.patch
 - Add 0113-multipathd-Handle-losing-all-path-in-update_map.patch
 - Resolves: bz #2110485
 * Fri Sep  2 2022 Benjamin Marzinski <bmarzins@redhat.com> 0.8.4-29
 - Rebuild for rhel-8.8.0
 - Resolves: bz #2123446
 * Wed Aug 24 2022 Benjamin Marzinski <bmarzins@redhat.com> 0.8.4-28
 - Add 0110-multipath-fix-systemd-timers-in-the-initramfs.patch
 - Resolves: bz #1916168
 * Fri Aug 19 2022 Benjamin Marzinski <bmarzins@redhat.com> 0.8.4-27
 - Add 0108-multipathd-Add-missing-ctype-include.patch
 - Add 0109-multipathd-replace-libreadline-with-libedit.patch