fio/0001-Revert-pmemblk-remove-pmemblk-engine.patch

From 2383a1ec2ad9090259f6d200b92676acff37de3b Mon Sep 17 00:00:00 2001
From: Pavel Reichl <preichl@redhat.com>
Date: Fri, 9 Jun 2023 18:18:48 +0200
Subject: [PATCH] Revert "pmemblk: remove pmemblk engine"

This reverts commit 04c1cdc4c108c6537681ab7c50daaed6d2fb4c93.

Signed-off-by: Pavel Reichl <preichl@redhat.com>
---
 HOWTO.rst               |   5 +
 Makefile                |   5 +
 ci/actions-install.sh   |   1 +
 configure               |  41 ++++
 engines/pmemblk.c       | 449 ++++++++++++++++++++++++++++++++++++++++
 examples/pmemblk.fio    |  71 +++++++
 fio.1                   |   5 +
 options.c               |   6 +
 os/windows/examples.wxs |   4 +
 10 files changed, 587 insertions(+)
 create mode 100644 engines/pmemblk.c
 create mode 100644 examples/pmemblk.fio

diff --git a/HOWTO.rst b/HOWTO.rst
index 32fff5ecbde42cf894214f766f38130dba079760..4f003524f69e5cced1195fb0f7efcc2590648122 100644
--- a/HOWTO.rst
+++ b/HOWTO.rst
@@ -2147,6 +2147,11 @@ I/O engine
 			before overwriting. The `trimwrite` mode works well for this
 			constraint.
 
+		**pmemblk**
+			Read and write using filesystem DAX to a file on a filesystem
+			mounted with DAX on a persistent memory device through the PMDK
+			libpmemblk library.
+
 		**dev-dax**
 			Read and write using device DAX to a persistent memory device (e.g.,
 			/dev/dax0.0) through the PMDK libpmem library.
diff --git a/Makefile b/Makefile
index 6d7fd4e2bbbdeb196d22299d379bebb29172d538..89205ebf498f957ceedefdf1b0e565f08c85060f 100644
--- a/Makefile
+++ b/Makefile
@@ -208,6 +208,11 @@ ifdef CONFIG_MTD
   SOURCE += oslib/libmtd.c
   SOURCE += oslib/libmtd_legacy.c
 endif
+ifdef CONFIG_PMEMBLK
+  pmemblk_SRCS = engines/pmemblk.c
+  pmemblk_LIBS = -lpmemblk
+  ENGINES += pmemblk
+endif
 ifdef CONFIG_LINUX_DEVDAX
   dev-dax_SRCS = engines/dev-dax.c
   dev-dax_LIBS = -lpmem
diff --git a/ci/actions-install.sh b/ci/actions-install.sh
index 95241e78825a9939814a747daf486f866949e392..2f1a0cbaeef4b528a93813d18b94f9bc041bfa04 100755
--- a/ci/actions-install.sh
+++ b/ci/actions-install.sh
@@ -47,6 +47,7 @@ DPKGCFG
                 libnbd-dev
                 libpmem-dev
                 libpmem2-dev
+                libpmemblk-dev
                 libprotobuf-c-dev
                 librbd-dev
                 libtcmalloc-minimal4
diff --git a/configure b/configure
index 74416fd48bc73e35cd8fd5440b9733efd1d0adbb..f6b160c99d374034ba308b74a306e3eb1570be1e 100755
--- a/configure
+++ b/configure
@@ -163,6 +163,7 @@ show_help="no"
 exit_val=0
 gfio_check="no"
 libhdfs="no"
+pmemblk="no"
 devdax="no"
 pmem="no"
 cuda="no"
@@ -2260,6 +2261,43 @@ if test "$libpmem" = "yes" && test "$disable_pmem" = "no"; then
   fi
 fi
 
+##########################################
+# Check whether we have libpmemblk
+# libpmem is a prerequisite
+if test "$libpmemblk" != "yes" ; then
+  libpmemblk="no"
+fi
+if test "$libpmem" = "yes"; then
+  cat > $TMPC << EOF
+#include <libpmemblk.h>
+int main(int argc, char **argv)
+{
+  PMEMblkpool *pbp;
+  pbp = pmemblk_open("", 0);
+  return 0;
+}
+EOF
+  if compile_prog "" "-lpmemblk" "libpmemblk"; then
+    libpmemblk="yes"
+  fi
+fi
+print_config "libpmemblk" "$libpmemblk"
+
+# Choose libpmem-based ioengines
+if test "$libpmem" = "yes" && test "$disable_pmem" = "no"; then
+  devdax="yes"
+  if test "$libpmem1_5" = "yes"; then
+    pmem="yes"
+  fi
+  if test "$libpmemblk" = "yes"; then
+    pmemblk="yes"
+  fi
+fi
+
+##########################################
+# Report whether pmemblk engine is enabled
+print_config "PMDK pmemblk engine" "$pmemblk"
+
 ##########################################
 # Report whether dev-dax engine is enabled
 print_config "PMDK dev-dax engine" "$devdax"
@@ -3188,6 +3226,9 @@ fi
 if test "$mtd" = "yes" ; then
   output_sym "CONFIG_MTD"
 fi
+if test "$pmemblk" = "yes" ; then
+  output_sym "CONFIG_PMEMBLK"
+fi
 if test "$devdax" = "yes" ; then
   output_sym "CONFIG_LINUX_DEVDAX"
 fi
diff --git a/engines/pmemblk.c b/engines/pmemblk.c
new file mode 100644
index 0000000000000000000000000000000000000000..849d8a15a0da59d07209c2475b78a1e4098c143a
--- /dev/null
+++ b/engines/pmemblk.c
@@ -0,0 +1,449 @@
+/*
+ * pmemblk: IO engine that uses PMDK libpmemblk to read and write data
+ *
+ * Copyright (C) 2016 Hewlett Packard Enterprise Development LP
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License,
+ * version 2 as published by the Free Software Foundation..
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public
+ * License along with this program; if not, write to the Free
+ * Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ */
+
+/*
+ * pmemblk engine
+ *
+ * IO engine that uses libpmemblk to read and write data
+ *
+ * To use:
+ *   ioengine=pmemblk
+ *
+ * Other relevant settings:
+ *   thread=1   REQUIRED
+ *   iodepth=1
+ *   direct=1
+ *   unlink=1
+ *   filename=/mnt/pmem0/fiotestfile,BSIZE,FSIZEMiB
+ *
+ *   thread must be set to 1 for pmemblk as multiple processes cannot
+ *     open the same block pool file.
+ *
+ *   iodepth should be set to 1 as pmemblk is always synchronous.
+ *   Use numjobs to scale up.
+ *
+ *   direct=1 is implied as pmemblk is always direct. A warning message
+ *   is printed if this is not specified.
+ *
+ *   unlink=1 removes the block pool file after testing, and is optional.
+ *
+ *   The pmem device must have a DAX-capable filesystem and be mounted
+ *   with DAX enabled.  filename must point to a file on that filesystem.
+ *
+ *   Example:
+ *     mkfs.xfs /dev/pmem0
+ *     mkdir /mnt/pmem0
+ *     mount -o dax /dev/pmem0 /mnt/pmem0
+ *
+ *   When specifying the filename, if the block pool file does not already
+ *   exist, then the pmemblk engine creates the pool file if you specify
+ *   the block and file sizes.  BSIZE is the block size in bytes.
+ *   FSIZEMB is the pool file size in MiB.
+ *
+ *   See examples/pmemblk.fio for more.
+ *
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <sys/uio.h>
+#include <errno.h>
+#include <assert.h>
+#include <string.h>
+#include <libpmem.h>
+#include <libpmemblk.h>
+
+#include "../fio.h"
+
+/*
+ * libpmemblk
+ */
+typedef struct fio_pmemblk_file *fio_pmemblk_file_t;
+
+struct fio_pmemblk_file {
+	fio_pmemblk_file_t pmb_next;
+	char *pmb_filename;
+	uint64_t pmb_refcnt;
+	PMEMblkpool *pmb_pool;
+	size_t pmb_bsize;
+	size_t pmb_nblocks;
+};
+
+static fio_pmemblk_file_t Cache;
+
+static pthread_mutex_t CacheLock = PTHREAD_MUTEX_INITIALIZER;
+
+#define PMB_CREATE   (0x0001)	/* should create file */
+
+fio_pmemblk_file_t fio_pmemblk_cache_lookup(const char *filename)
+{
+	fio_pmemblk_file_t i;
+
+	for (i = Cache; i != NULL; i = i->pmb_next)
+		if (!strcmp(filename, i->pmb_filename))
+			return i;
+
+	return NULL;
+}
+
+static void fio_pmemblk_cache_insert(fio_pmemblk_file_t pmb)
+{
+	pmb->pmb_next = Cache;
+	Cache = pmb;
+}
+
+static void fio_pmemblk_cache_remove(fio_pmemblk_file_t pmb)
+{
+	fio_pmemblk_file_t i;
+
+	if (pmb == Cache) {
+		Cache = Cache->pmb_next;
+		pmb->pmb_next = NULL;
+		return;
+	}
+
+	for (i = Cache; i != NULL; i = i->pmb_next)
+		if (pmb == i->pmb_next) {
+			i->pmb_next = i->pmb_next->pmb_next;
+			pmb->pmb_next = NULL;
+			return;
+		}
+}
+
+/*
+ * to control block size and gross file size at the libpmemblk
+ * level, we allow the block size and file size to be appended
+ * to the file name:
+ *
+ *   path[,bsize,fsizemib]
+ *
+ * note that we do not use the fio option "filesize" to dictate
+ * the file size because we can only give libpmemblk the gross
+ * file size, which is different from the net or usable file
+ * size (which is probably what fio wants).
+ *
+ * the final path without the parameters is returned in ppath.
+ * the block size and file size are returned in pbsize and fsize.
+ *
+ * note that the user specifies the file size in MiB, but
+ * we return bytes from here.
+ */
+static void pmb_parse_path(const char *pathspec, char **ppath, uint64_t *pbsize,
+			   uint64_t *pfsize)
+{
+	char *path;
+	char *s;
+	uint64_t bsize;
+	uint64_t fsizemib;
+
+	path = strdup(pathspec);
+	if (!path) {
+		*ppath = NULL;
+		return;
+	}
+
+	/* extract sizes, if given */
+	s = strrchr(path, ',');
+	if (s && (fsizemib = strtoull(s + 1, NULL, 10))) {
+		*s = 0;
+		s = strrchr(path, ',');
+		if (s && (bsize = strtoull(s + 1, NULL, 10))) {
+			*s = 0;
+			*ppath = path;
+			*pbsize = bsize;
+			*pfsize = fsizemib << 20;
+			return;
+		}
+	}
+
+	/* size specs not found */
+	strcpy(path, pathspec);
+	*ppath = path;
+	*pbsize = 0;
+	*pfsize = 0;
+}
+
+static fio_pmemblk_file_t pmb_open(const char *pathspec, int flags)
+{
+	fio_pmemblk_file_t pmb;
+	char *path = NULL;
+	uint64_t bsize = 0;
+	uint64_t fsize = 0;
+
+	pmb_parse_path(pathspec, &path, &bsize, &fsize);
+	if (!path)
+		return NULL;
+
+	pthread_mutex_lock(&CacheLock);
+
+	pmb = fio_pmemblk_cache_lookup(path);
+	if (!pmb) {
+		pmb = malloc(sizeof(*pmb));
+		if (!pmb)
+			goto error;
+
+		/* try opening existing first, create it if needed */
+		pmb->pmb_pool = pmemblk_open(path, bsize);
+		if (!pmb->pmb_pool && (errno == ENOENT) &&
+		    (flags & PMB_CREATE) && (0 < fsize) && (0 < bsize)) {
+			pmb->pmb_pool =
+			    pmemblk_create(path, bsize, fsize, 0644);
+		}
+		if (!pmb->pmb_pool) {
+			log_err("pmemblk: unable to open pmemblk pool file %s (%s)\n",
+			     path, strerror(errno));
+			goto error;
+		}
+
+		pmb->pmb_filename = path;
+		pmb->pmb_next = NULL;
+		pmb->pmb_refcnt = 0;
+		pmb->pmb_bsize = pmemblk_bsize(pmb->pmb_pool);
+		pmb->pmb_nblocks = pmemblk_nblock(pmb->pmb_pool);
+
+		fio_pmemblk_cache_insert(pmb);
+	} else {
+		free(path);
+	}
+
+	pmb->pmb_refcnt += 1;
+
+	pthread_mutex_unlock(&CacheLock);
+
+	return pmb;
+
+error:
+	if (pmb) {
+		if (pmb->pmb_pool)
+			pmemblk_close(pmb->pmb_pool);
+		pmb->pmb_pool = NULL;
+		pmb->pmb_filename = NULL;
+		free(pmb);
+	}
+	if (path)
+		free(path);
+
+	pthread_mutex_unlock(&CacheLock);
+	return NULL;
+}
+
+static void pmb_close(fio_pmemblk_file_t pmb, const bool keep)
+{
+	pthread_mutex_lock(&CacheLock);
+
+	pmb->pmb_refcnt--;
+
+	if (!keep && !pmb->pmb_refcnt) {
+		pmemblk_close(pmb->pmb_pool);
+		pmb->pmb_pool = NULL;
+		free(pmb->pmb_filename);
+		pmb->pmb_filename = NULL;
+		fio_pmemblk_cache_remove(pmb);
+		free(pmb);
+	}
+
+	pthread_mutex_unlock(&CacheLock);
+}
+
+static int pmb_get_flags(struct thread_data *td, uint64_t *pflags)
+{
+	static int thread_warned = 0;
+	static int odirect_warned = 0;
+
+	uint64_t flags = 0;
+
+	if (!td->o.use_thread) {
+		if (!thread_warned) {
+			thread_warned = 1;
+			log_err("pmemblk: must set thread=1 for pmemblk engine\n");
+		}
+		return 1;
+	}
+
+	if (!td->o.odirect && !odirect_warned) {
+		odirect_warned = 1;
+		log_info("pmemblk: direct == 0, but pmemblk is always direct\n");
+	}
+
+	if (td->o.allow_create)
+		flags |= PMB_CREATE;
+
+	(*pflags) = flags;
+	return 0;
+}
+
+static int fio_pmemblk_open_file(struct thread_data *td, struct fio_file *f)
+{
+	uint64_t flags = 0;
+	fio_pmemblk_file_t pmb;
+
+	if (pmb_get_flags(td, &flags))
+		return 1;
+
+	pmb = pmb_open(f->file_name, flags);
+	if (!pmb)
+		return 1;
+
+	FILE_SET_ENG_DATA(f, pmb);
+	return 0;
+}
+
+static int fio_pmemblk_close_file(struct thread_data fio_unused *td,
+				  struct fio_file *f)
+{
+	fio_pmemblk_file_t pmb = FILE_ENG_DATA(f);
+
+	if (pmb)
+		pmb_close(pmb, false);
+
+	FILE_SET_ENG_DATA(f, NULL);
+	return 0;
+}
+
+static int fio_pmemblk_get_file_size(struct thread_data *td, struct fio_file *f)
+{
+	uint64_t flags = 0;
+	fio_pmemblk_file_t pmb = FILE_ENG_DATA(f);
+
+	if (fio_file_size_known(f))
+		return 0;
+
+	if (!pmb) {
+		if (pmb_get_flags(td, &flags))
+			return 1;
+		pmb = pmb_open(f->file_name, flags);
+		if (!pmb)
+			return 1;
+	}
+
+	f->real_file_size = pmb->pmb_bsize * pmb->pmb_nblocks;
+
+	fio_file_set_size_known(f);
+
+	if (!FILE_ENG_DATA(f))
+		pmb_close(pmb, true);
+
+	return 0;
+}
+
+static enum fio_q_status fio_pmemblk_queue(struct thread_data *td,
+					   struct io_u *io_u)
+{
+	struct fio_file *f = io_u->file;
+	fio_pmemblk_file_t pmb = FILE_ENG_DATA(f);
+
+	unsigned long long off;
+	unsigned long len;
+	void *buf;
+
+	fio_ro_check(td, io_u);
+
+	switch (io_u->ddir) {
+	case DDIR_READ:
+	case DDIR_WRITE:
+		off = io_u->offset;
+		len = io_u->xfer_buflen;
+
+		io_u->error = EINVAL;
+		if (off % pmb->pmb_bsize)
+			break;
+		if (len % pmb->pmb_bsize)
+			break;
+		if ((off + len) / pmb->pmb_bsize > pmb->pmb_nblocks)
+			break;
+
+		io_u->error = 0;
+		buf = io_u->xfer_buf;
+		off /= pmb->pmb_bsize;
+		len /= pmb->pmb_bsize;
+		while (0 < len) {
+			if (io_u->ddir == DDIR_READ) {
+				if (0 != pmemblk_read(pmb->pmb_pool, buf, off)) {
+					io_u->error = errno;
+					break;
+				}
+			} else if (0 != pmemblk_write(pmb->pmb_pool, buf, off)) {
+				io_u->error = errno;
+				break;
+			}
+			buf += pmb->pmb_bsize;
+			off++;
+			len--;
+		}
+		off *= pmb->pmb_bsize;
+		len *= pmb->pmb_bsize;
+		io_u->resid = io_u->xfer_buflen - (off - io_u->offset);
+		break;
+	case DDIR_SYNC:
+	case DDIR_DATASYNC:
+	case DDIR_SYNC_FILE_RANGE:
+		/* we're always sync'd */
+		io_u->error = 0;
+		break;
+	default:
+		io_u->error = EINVAL;
+		break;
+	}
+
+	return FIO_Q_COMPLETED;
+}
+
+static int fio_pmemblk_unlink_file(struct thread_data *td, struct fio_file *f)
+{
+	char *path = NULL;
+	uint64_t bsize = 0;
+	uint64_t fsize = 0;
+
+	/*
+	 * we need our own unlink in case the user has specified
+	 * the block and file sizes in the path name.  we parse
+	 * the file_name to determine the file name we actually used.
+	 */
+
+	pmb_parse_path(f->file_name, &path, &bsize, &fsize);
+	if (!path)
+		return ENOENT;
+
+	unlink(path);
+	free(path);
+	return 0;
+}
+
+FIO_STATIC struct ioengine_ops ioengine = {
+	.name = "pmemblk",
+	.version = FIO_IOOPS_VERSION,
+	.queue = fio_pmemblk_queue,
+	.open_file = fio_pmemblk_open_file,
+	.close_file = fio_pmemblk_close_file,
+	.get_file_size = fio_pmemblk_get_file_size,
+	.unlink_file = fio_pmemblk_unlink_file,
+	.flags = FIO_SYNCIO | FIO_DISKLESSIO | FIO_NOEXTEND | FIO_NODISKUTIL,
+};
+
+static void fio_init fio_pmemblk_register(void)
+{
+	register_ioengine(&ioengine);
+}
+
+static void fio_exit fio_pmemblk_unregister(void)
+{
+	unregister_ioengine(&ioengine);
+}
diff --git a/examples/pmemblk.fio b/examples/pmemblk.fio
new file mode 100644
index 0000000000000000000000000000000000000000..59bb2a8a5acbf0e03d16a988f6ae0b9eb84575d2
--- /dev/null
+++ b/examples/pmemblk.fio
@@ -0,0 +1,71 @@
+[global]
+bs=1m
+ioengine=pmemblk
+norandommap
+time_based
+runtime=30
+group_reporting
+disable_lat=1
+disable_slat=1
+disable_clat=1
+clat_percentiles=0
+cpus_allowed_policy=split
+
+# For the pmemblk engine:
+#
+#   IOs always complete immediately
+#   IOs are always direct
+#   Must use threads
+#
+iodepth=1
+direct=1
+thread
+numjobs=16
+#
+# Unlink can be used to remove the files when done, but if you are
+# using serial runs with stonewall, and you want the files to be created
+# only once and unlinked only at the very end, then put the unlink=1
+# in the last group.  This is the method demonstrated here.
+#
+# Note that if you have a read-only group and if the files will be
+# newly created, then all of the data will read back as zero and the
+# read will be optimized, yielding performance that is different from
+# that of reading non-zero blocks (or unoptimized zero blocks).
+#
+unlink=0
+#
+# The pmemblk engine does IO to files in a DAX-mounted filesystem.
+# The filesystem should be created on an NVDIMM (e.g /dev/pmem0)
+# and then mounted with the '-o dax' option.  Note that the engine
+# accesses the underlying NVDIMM directly, bypassing the kernel block
+# layer, so the usual filesystem/disk performance monitoring tools such
+# as iostat will not provide useful data.
+#
+# Here we specify a test file on each of two NVDIMMs.  The first
+# number after the file name is the block size in bytes (4096 bytes
+# in this example).  The second number is the size of the file to
+# create in MiB (1 GiB in this example); note that the actual usable
+# space available to fio will be less than this as libpmemblk requires
+# some space for metadata.
+#
+# Currently, the minimum block size is 512 bytes and the minimum file
+# size is about 17 MiB (these are libpmemblk requirements).
+#
+# While both files in this example have the same block size and file
+# size, this is not required.
+#
+filename=/pmem0/fio-test,4096,1024
+#filename=/pmem1/fio-test,4096,1024
+
+[pmemblk-write]
+rw=randwrite
+stonewall
+
+[pmemblk-read]
+rw=randread
+stonewall
+#
+# We're done, so unlink the file:
+#
+unlink=1
+
diff --git a/fio.1 b/fio.1
index 80bf3371a3556406cbcb29323bb21f55e769b9f9..3dc7e062a063d54bcd79120a7d6dc1ffd934b80c 100644
--- a/fio.1
+++ b/fio.1
@@ -1960,6 +1960,11 @@ e.g., on NAND, writing sequentially to erase blocks and discarding
 before overwriting. The \fBtrimwrite\fR mode works well for this
 constraint.
 .TP
+.B pmemblk
+Read and write using filesystem DAX to a file on a filesystem
+mounted with DAX on a persistent memory device through the PMDK
+libpmemblk library.
+.TP
 .B dev\-dax
 Read and write using device DAX to a persistent memory device (e.g.,
 /dev/dax0.0) through the PMDK libpmem library.
diff --git a/options.c b/options.c
index 8193fb29fe2b1cdfab8e745b9522aeb507f5361e..6c58577d8dbfd0a8dfd2b77e11c9be57e4aaaf10 100644
--- a/options.c
+++ b/options.c
@@ -2125,6 +2125,12 @@ struct fio_option fio_options[FIO_MAX_OPTS] = {
 			    .help = "Hadoop Distributed Filesystem (HDFS) engine"
 			  },
 #endif
+#ifdef CONFIG_PMEMBLK
+			  { .ival = "pmemblk",
+			    .help = "PMDK libpmemblk based IO engine",
+			  },
+
+#endif
 #ifdef CONFIG_IME
 			  { .ival = "ime_psync",
 			    .help = "DDN's IME synchronous IO engine",
diff --git a/os/windows/examples.wxs b/os/windows/examples.wxs
index d70c77133f5a9f24908fffde9ce5ad20dbad2562..9308ba8be829c62b88cb06470a068cc2aef3f7dc 100755
--- a/os/windows/examples.wxs
+++ b/os/windows/examples.wxs
@@ -125,6 +125,9 @@
                 <Component>
                   <File Source="..\..\examples\numa.fio" />
                 </Component>
+                <Component>
+                  <File Source="..\..\examples\pmemblk.fio" />
+                </Component>
                 <Component>
                   <File Source="..\..\examples\poisson-rate-submission.fio" />
                 </Component>
@@ -209,6 +212,7 @@
             <ComponentRef Id="netio_multicast.fio" />
             <ComponentRef Id="null.fio" />
             <ComponentRef Id="numa.fio" />
+            <ComponentRef Id="pmemblk.fio" />
             <ComponentRef Id="poisson_rate_submission.fio" />
             <ComponentRef Id="rados.fio"/>
             <ComponentRef Id="rand_zones.fio" />
-- 
2.41.0