Build nvml engines for ppc64le; fix segfault with external engines

0001-fio-fix-dlopen-refcounting-of-dynamic-engines.patch

@@ -0,0 +1,84 @@
+From 48ff7df9daea86c82a572b0a840bb8371b6b1a29 Mon Sep 17 00:00:00 2001
+From: Eric Sandeen <sandeen@redhat.com>
+Date: Mon, 25 Jan 2021 13:23:48 -0600
+Subject: [PATCH] fio: fix dlopen refcounting of dynamic engines
+
+ioengine_load() will dlclose the dynamic library if it matches one
+that we've already got open, but this defeats the built-in refcounting
+done by dlopen/dlclose.  As each thread exits, it calls free_ioengine(),
+and this may do a final dlclose on a dynamic ioengine that is still
+in use if we don't have the proper reference count.
+
+Fix this by dropping the explicit dlclose of a "matching" dlopened
+dynamic engine library, and let each dlclose decrement the refcount
+on the engine library as is normal.
+
+This also adds/modifies a couple of debug messages to help track this.
+
+Signed-off-by: Eric Sandeen <sandeen@redhat.com>
+Signed-off-by: Jens Axboe <axboe@kernel.dk>
+---
+ init.c      | 5 +----
+ ioengines.c | 8 ++++++--
+ 2 files changed, 7 insertions(+), 6 deletions(-)
+
+diff --git a/init.c b/init.c
+index ab38b334..d6dbaf7c 100644
+--- a/init.c
++++ b/init.c
+@@ -1109,11 +1109,8 @@ int ioengine_load(struct thread_data *td)
+ 		if (!ops)
+ 			goto fail;
+ 
+-		if (ops == td->io_ops && dlhandle == td->io_ops->dlhandle) {
+-			if (dlhandle)
+-				dlclose(dlhandle);
++		if (ops == td->io_ops && dlhandle == td->io_ops->dlhandle)
+ 			return 0;
+-		}
+ 
+ 		if (dlhandle && dlhandle != td->io_ops->dlhandle)
+ 			dlclose(dlhandle);
+diff --git a/ioengines.c b/ioengines.c
+index dcc9496d..f88b0537 100644
+--- a/ioengines.c
++++ b/ioengines.c
+@@ -95,6 +95,7 @@ static void *dlopen_external(struct thread_data *td, const char *engine)
+ 
+ 	sprintf(engine_path, "%s/fio-%s.so", FIO_EXT_ENG_DIR, engine);
+ 
++	dprint(FD_IO, "dlopen external %s\n", engine_path);
+ 	dlhandle = dlopen(engine_path, RTLD_LAZY);
+ 	if (!dlhandle)
+ 		log_info("Engine %s not found; Either name is invalid, was not built, or fio-engine-%s package is missing.\n",
+@@ -116,7 +117,7 @@ static struct ioengine_ops *dlopen_ioengine(struct thread_data *td,
+ 	    !strncmp(engine_lib, "aio", 3))
+ 		engine_lib = "libaio";
+ 
+-	dprint(FD_IO, "dload engine %s\n", engine_lib);
++	dprint(FD_IO, "dlopen engine %s\n", engine_lib);
+ 
+ 	dlerror();
+ 	dlhandle = dlopen(engine_lib, RTLD_LAZY);
+@@ -194,7 +195,9 @@ struct ioengine_ops *load_ioengine(struct thread_data *td)
+ 	 * so as not to break job files not using the prefix.
+ 	 */
+ 	ops = __load_ioengine(td->o.ioengine);
+-	if (!ops)
++
++	/* We do re-dlopen existing handles, for reference counting */
++	if (!ops || ops->dlhandle)
+ 		ops = dlopen_ioengine(td, name);
+ 
+ 	/*
+@@ -229,6 +232,7 @@ void free_ioengine(struct thread_data *td)
+ 	}
+ 
+ 	if (td->io_ops->dlhandle) {
++		dprint(FD_IO, "dlclose ioengine %s\n", td->io_ops->name);
+ 		dlclose(td->io_ops->dlhandle);
+ 		td->io_ops->dlhandle = NULL;
+ 	}
+-- 
+2.26.2
+

0001-fio-move-dynamic-library-handle-to-io_ops-structure.patch

@@ -0,0 +1,110 @@
+From f6931a1dd35896433c8cc2e10de51372a2c496c4 Mon Sep 17 00:00:00 2001
+From: Eric Sandeen <sandeen@redhat.com>
+Date: Mon, 25 Jan 2021 13:18:31 -0600
+Subject: [PATCH] fio: move dynamic library handle to io_ops structure
+
+Keeping a dynamic engine's dlopen'd dlhandle on a thread structure doesn't
+make sense; that thread may exit while others are still using the engine.
+
+Move the dlhandle onto the ops structure itself.
+
+We still only call dlopen for the first thead, which leaves a refcounting
+issue which will be fixed in the next patch.
+
+Signed-off-by: Eric Sandeen <sandeen@redhat.com>
+Signed-off-by: Jens Axboe <axboe@kernel.dk>
+---
+ fio.h       | 1 -
+ init.c      | 6 +++---
+ ioengines.c | 8 ++++----
+ ioengines.h | 3 ++-
+ 4 files changed, 9 insertions(+), 9 deletions(-)
+
+diff --git a/fio.h b/fio.h
+index ee582a72..062abfa7 100644
+--- a/fio.h
++++ b/fio.h
+@@ -281,7 +281,6 @@ struct thread_data {
+ 	 * IO engine private data and dlhandle.
+ 	 */
+ 	void *io_ops_data;
+-	void *io_ops_dlhandle;
+ 
+ 	/*
+ 	 * Queue depth of io_u's that fio MIGHT do
+diff --git a/init.c b/init.c
+index 1d14df16..ab38b334 100644
+--- a/init.c
++++ b/init.c
+@@ -1104,18 +1104,18 @@ int ioengine_load(struct thread_data *td)
+ 		 * for this name and see if they match. If they do, then
+ 		 * the engine is unchanged.
+ 		 */
+-		dlhandle = td->io_ops_dlhandle;
++		dlhandle = td->io_ops->dlhandle;
+ 		ops = load_ioengine(td);
+ 		if (!ops)
+ 			goto fail;
+ 
+-		if (ops == td->io_ops && dlhandle == td->io_ops_dlhandle) {
++		if (ops == td->io_ops && dlhandle == td->io_ops->dlhandle) {
+ 			if (dlhandle)
+ 				dlclose(dlhandle);
+ 			return 0;
+ 		}
+ 
+-		if (dlhandle && dlhandle != td->io_ops_dlhandle)
++		if (dlhandle && dlhandle != td->io_ops->dlhandle)
+ 			dlclose(dlhandle);
+ 
+ 		/* Unload the old engine. */
+diff --git a/ioengines.c b/ioengines.c
+index 5ac512ae..dcc9496d 100644
+--- a/ioengines.c
++++ b/ioengines.c
+@@ -155,7 +155,7 @@ static struct ioengine_ops *dlopen_ioengine(struct thread_data *td,
+ 		return NULL;
+ 	}
+ 
+-	td->io_ops_dlhandle = dlhandle;
++	ops->dlhandle = dlhandle;
+ 	return ops;
+ }
+ 
+@@ -228,9 +228,9 @@ void free_ioengine(struct thread_data *td)
+ 		td->eo = NULL;
+ 	}
+ 
+-	if (td->io_ops_dlhandle) {
+-		dlclose(td->io_ops_dlhandle);
+-		td->io_ops_dlhandle = NULL;
++	if (td->io_ops->dlhandle) {
++		dlclose(td->io_ops->dlhandle);
++		td->io_ops->dlhandle = NULL;
+ 	}
+ 
+ 	td->io_ops = NULL;
+diff --git a/ioengines.h b/ioengines.h
+index a928b211..839b318d 100644
+--- a/ioengines.h
++++ b/ioengines.h
+@@ -8,7 +8,7 @@
+ #include "io_u.h"
+ #include "zbd_types.h"
+ 
+-#define FIO_IOOPS_VERSION	27
++#define FIO_IOOPS_VERSION	28
+ 
+ #ifndef CONFIG_DYNAMIC_ENGINES
+ #define FIO_STATIC	static
+@@ -30,6 +30,7 @@ struct ioengine_ops {
+ 	const char *name;
+ 	int version;
+ 	int flags;
++	void *dlhandle;
+ 	int (*setup)(struct thread_data *);
+ 	int (*init)(struct thread_data *);
+ 	int (*post_init)(struct thread_data *);
+-- 
+2.26.2
+

fio.spec

@@ -1,12 +1,14 @@
 Name:		fio
 Version:	3.25
-Release:	2%{?dist}
+Release:	3%{?dist}
 Summary:	Multithreaded IO generation tool
 
 License:	GPLv2
 URL:		http://git.kernel.dk/?p=fio.git;a=summary
 Source:		http://brick.kernel.dk/snaps/%{name}-%{version}.tar.bz2
 
+Patch0:		0001-fio-move-dynamic-library-handle-to-io_ops-structure.patch
+Patch1:		0001-fio-fix-dlopen-refcounting-of-dynamic-engines.patch
 
 BuildRequires:	gcc
 BuildRequires:	libaio-devel
@@ -15,7 +17,7 @@ BuildRequires:	python3-devel
 BuildRequires:	libnbd-devel
 BuildRequires:	libcurl-devel
 BuildRequires:	openssl-devel
-%ifarch x86_64
+%ifarch x86_64 ppc64le
 BuildRequires:	libpmem-devel
 BuildRequires:	libpmemblk-devel
 %endif
@@ -40,7 +42,7 @@ BuildRequires: make
 Recommends:     %{name}-engine-libaio
 Recommends:     %{name}-engine-http
 Recommends:     %{name}-engine-nbd
-%ifarch x86-64
+%ifarch x86-64 ppc64le
 Recommends:     %{name}-engine-dev-dax
 Recommends:     %{name}-engine-pmemblk
 Recommends:     %{name}-engine-libpmem
@@ -82,7 +84,7 @@ Requires:       %{name}%{?_isa} = %{version}-%{release}
 %description engine-nbd
 Network Block Device (NBD) engine for %{name}.
 
-%ifarch x86_64
+%ifarch x86_64 ppc64le
 %package engine-dev-dax
 Summary:        PMDK dev-dax engine for %{name}.
 Requires:       %{name}%{?_isa} = %{version}-%{release}
@@ -93,7 +95,7 @@ Read and write using device DAX to a persistent memory device
 (e.g., /dev/dax0.0) through the PMDK libpmem library.
 %endif
 
-%ifarch x86_64
+%ifarch x86_64 ppc64le
 %package engine-pmemblk
 Summary:        PMDK pmemblk engine for %{name}.
 Requires:       %{name}%{?_isa} = %{version}-%{release}
@@ -104,7 +106,7 @@ Read and write using filesystem DAX to a file on a filesystem mounted with
 DAX on a persistent memory device through the PMDK libpmemblk library.
 %endif
 
-%ifarch x86_64
+%ifarch x86_64 ppc64le
 %package engine-libpmem
 Summary:        PMDK pmemblk engine for %{name}.
 Requires:       %{name}%{?_isa} = %{version}-%{release}
@@ -170,7 +172,7 @@ make install prefix=%{_prefix} mandir=%{_mandir} libdir=%{_libdir}/fio DESTDIR=$
 %{_mandir}/man1/*
 %{_datadir}/%{name}/*
 
-%ifarch x86_64
+%ifarch x86_64 ppc64le
 %files engine-dev-dax
 %{_libdir}/fio/fio-dev-dax.so
 %endif
@@ -181,7 +183,7 @@ make install prefix=%{_prefix} mandir=%{_mandir} libdir=%{_libdir}/fio DESTDIR=$
 %files engine-libaio
 %{_libdir}/fio/fio-libaio.so
 
-%ifarch x86_64
+%ifarch x86_64 ppc64le
 %files engine-libpmem
 %{_libdir}/fio/fio-libpmem.so
 %endif
@@ -189,7 +191,7 @@ make install prefix=%{_prefix} mandir=%{_mandir} libdir=%{_libdir}/fio DESTDIR=$
 %files engine-nbd
 %{_libdir}/fio/fio-nbd.so
 
-%ifarch x86_64
+%ifarch x86_64 ppc64le
 %files engine-pmemblk
 %{_libdir}/fio/fio-pmemblk.so
 %endif
@@ -208,6 +210,10 @@ make install prefix=%{_prefix} mandir=%{_mandir} libdir=%{_libdir}/fio DESTDIR=$
 %endif
 
 %changelog
+* Mon Feb 08 2021 Eric Sandeen <sandeen@redhat.com> 3.25-3
+- Fix segfault with external IO engines and multiple threads
+- Enable dev-dax, pmemblk, libpmem engines for ppc64le
+
 * Tue Jan 26 2021 Fedora Release Engineering <releng@fedoraproject.org> - 3.25-2
 - Rebuilt for https://fedoraproject.org/wiki/Fedora_34_Mass_Rebuild