diff --git a/0001-common-Add-ONCE-macro-to-run-code-only-once.patch b/0001-common-Add-ONCE-macro-to-run-code-only-once.patch new file mode 100644 index 0000000..4b34524 --- /dev/null +++ b/0001-common-Add-ONCE-macro-to-run-code-only-once.patch @@ -0,0 +1,272 @@ +From de37da4184c55c6811dd02707fdd3b1773a7ce66 Mon Sep 17 00:00:00 2001 +From: "Richard W.M. Jones" +Date: Fri, 4 Jul 2025 08:13:48 +0100 +Subject: [PATCH] common: Add ONCE macro to run code only once + +This macro can be used to run code once, especially for debug messages +and similar. eg: + + /* Print this once in the log. */ + ONCE (nbdkit_debug ("falling back to less efficient method")); + +(cherry picked from commit ad8630deab4639e636212f11a5a47d2c34ef2949) +--- + .gitignore | 1 + + common/include/Makefile.am | 6 ++ + common/include/once.h | 67 ++++++++++++++++++++ + common/include/test-once.c | 126 +++++++++++++++++++++++++++++++++++++ + 4 files changed, 200 insertions(+) + create mode 100644 common/include/once.h + create mode 100644 common/include/test-once.c + +diff --git a/.gitignore b/.gitignore +index 3629ef39..827fd53c 100644 +--- a/.gitignore ++++ b/.gitignore +@@ -68,6 +68,7 @@ plugins/*/*.3 + /common/include/test-iszero + /common/include/test-minmax + /common/include/test-nextnonzero ++/common/include/test-once + /common/include/test-random + /common/include/test-tvdiff + /common/protocol/generate-protostrings.sh +diff --git a/common/include/Makefile.am b/common/include/Makefile.am +index ca488e68..3a3757e2 100644 +--- a/common/include/Makefile.am ++++ b/common/include/Makefile.am +@@ -49,6 +49,7 @@ EXTRA_DIST = \ + iszero.h \ + minmax.h \ + nextnonzero.h \ ++ once.h \ + random.h \ + rounding.h \ + static-assert.h \ +@@ -71,6 +72,7 @@ TESTS = \ + test-iszero \ + test-minmax \ + test-nextnonzero \ ++ test-once \ + test-random \ + test-tvdiff \ + $(NULL) +@@ -120,6 +122,10 @@ test_nextnonzero_SOURCES = test-nextnonzero.c nextnonzero.h + test_nextnonzero_CPPFLAGS = -I$(srcdir) + test_nextnonzero_CFLAGS = $(WARNINGS_CFLAGS) + ++test_once_SOURCES = test-once.c once.h ++test_once_CPPFLAGS = -I$(srcdir) ++test_once_CFLAGS = $(WARNINGS_CFLAGS) ++ + test_random_SOURCES = test-random.c random.h + test_random_CPPFLAGS = -I$(srcdir) + test_random_CFLAGS = $(WARNINGS_CFLAGS) +diff --git a/common/include/once.h b/common/include/once.h +new file mode 100644 +index 00000000..bb93e767 +--- /dev/null ++++ b/common/include/once.h +@@ -0,0 +1,67 @@ ++/* nbdkit ++ * Copyright Red Hat ++ * ++ * Redistribution and use in source and binary forms, with or without ++ * modification, are permitted provided that the following conditions are ++ * met: ++ * ++ * * Redistributions of source code must retain the above copyright ++ * notice, this list of conditions and the following disclaimer. ++ * ++ * * Redistributions in binary form must reproduce the above copyright ++ * notice, this list of conditions and the following disclaimer in the ++ * documentation and/or other materials provided with the distribution. ++ * ++ * * Neither the name of Red Hat nor the names of its contributors may be ++ * used to endorse or promote products derived from this software without ++ * specific prior written permission. ++ * ++ * THIS SOFTWARE IS PROVIDED BY RED HAT AND CONTRIBUTORS ''AS IS'' AND ++ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, ++ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A ++ * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL RED HAT OR ++ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, ++ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT ++ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF ++ * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ++ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, ++ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT ++ * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF ++ * SUCH DAMAGE. ++ */ ++ ++#ifndef NBDKIT_ONCE_H ++#define NBDKIT_ONCE_H ++ ++#ifdef HAVE_STDATOMIC_H ++#include ++#else ++/* This is best effort on platforms that don't support atomic. ++ * 32 bit ints are generally fine in reality. ++ */ ++#define _Atomic /**/ ++#endif ++ ++#include "unique-name.h" ++ ++/* Run the statement once (per nbdkit run). */ ++#define ONCE(stmt) ONCE_1(NBDKIT_UNIQUE_NAME(_once), (stmt)) ++ ++/* The actual implementation: ++ * ++ * The comparison with 0 avoids var wrapping around. Mostly var will ++ * only be 0 or 1, or in rare cases other small integers. ++ * ++ * The atomic increment & comparison with 1 is what only allows a ++ * single thread to run the statement. ++ * ++ * To avoid optimisations: Use 'volatile' so reads and writes are not ++ * removed, and use 'unsigned' to avoid any with signed overflow. ++ */ ++#define ONCE_1(var, stmt) \ ++ do { \ ++ static volatile _Atomic unsigned var = 0; \ ++ if (var == 0 && ++var == 1) { stmt; } \ ++ } while (0) ++ ++#endif /* NBDKIT_ONCE_H */ +diff --git a/common/include/test-once.c b/common/include/test-once.c +new file mode 100644 +index 00000000..d7dd5c42 +--- /dev/null ++++ b/common/include/test-once.c +@@ -0,0 +1,126 @@ ++/* nbdkit ++ * Copyright Red Hat ++ * ++ * Redistribution and use in source and binary forms, with or without ++ * modification, are permitted provided that the following conditions are ++ * met: ++ * ++ * * Redistributions of source code must retain the above copyright ++ * notice, this list of conditions and the following disclaimer. ++ * ++ * * Redistributions in binary form must reproduce the above copyright ++ * notice, this list of conditions and the following disclaimer in the ++ * documentation and/or other materials provided with the distribution. ++ * ++ * * Neither the name of Red Hat nor the names of its contributors may be ++ * used to endorse or promote products derived from this software without ++ * specific prior written permission. ++ * ++ * THIS SOFTWARE IS PROVIDED BY RED HAT AND CONTRIBUTORS ''AS IS'' AND ++ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, ++ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A ++ * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL RED HAT OR ++ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, ++ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT ++ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF ++ * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ++ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, ++ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT ++ * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF ++ * SUCH DAMAGE. ++ */ ++ ++#include ++ ++#include ++#include ++ ++#ifndef HAVE_STDATOMIC_H ++ ++/* Skip the test if no */ ++ ++int ++main (void) ++{ ++ printf ("SKIP: no on this platform\n"); ++ exit (77); ++} ++ ++#else /* HAVE_STDATOMIC_H */ ++ ++#include ++#include ++#include ++ ++#undef NDEBUG /* Keep test strong even for nbdkit built without assertions */ ++#include ++ ++#include ++ ++#include "once.h" ++ ++#define NR_THREADS 8 ++ ++static volatile _Atomic unsigned count1 = 0, count2 = 0, ++ count3 = 0, count4 = 0; ++static pthread_barrier_t barrier; ++ ++static void * __attribute__((noreturn)) ++start_thread (void *idxp) ++{ ++ //int idx = * (int*) idxp; ++ ++ pthread_barrier_wait (&barrier); ++ ++ for (;;) { ++ ONCE (count1++); ++ ONCE (count2++); ++ ONCE (count3++); ++ ONCE (count4++); ++ } ++} ++ ++int ++main (void) ++{ ++ int i, err; ++ pthread_t th[NR_THREADS]; ++ int idx[NR_THREADS]; ++ ++ err = pthread_barrier_init (&barrier, NULL, NR_THREADS); ++ if (err != 0) { ++ errno = err; ++ perror ("pthread_barrier_init"); ++ exit (EXIT_FAILURE); ++ } ++ ++ for (i = 0; i < NR_THREADS; ++i) { ++ idx[i] = i; ++ err = pthread_create (&th[i], NULL, start_thread, &idx[i]); ++ if (err != 0) { ++ errno = err; ++ perror ("pthread_create"); ++ exit (EXIT_FAILURE); ++ } ++ } ++ ++ do { ++ sleep (1); ++ } while (count1 + count2 + count3 + count4 < 4); ++ ++ for (i = 0; i < NR_THREADS; ++i) { ++ pthread_cancel (th[i]); ++ } ++ ++ pthread_barrier_destroy (&barrier); ++ ++ if (count1 != 1 || count2 != 1 || count3 != 1 || count4 != 1) { ++ fprintf (stderr, "FAIL: counts incremented to %u %u %u %u " ++ "(expected 1 1 1 1)\n", count1, count2, count3, count4); ++ exit (EXIT_FAILURE); ++ } ++ ++ exit (EXIT_SUCCESS); ++} ++ ++#endif /* HAVE_STDATOMIC_H */ +-- +2.47.1 + diff --git a/0002-file-zero-Print-the-debug-message-on-the-fallback-pa.patch b/0002-file-zero-Print-the-debug-message-on-the-fallback-pa.patch new file mode 100644 index 0000000..2c0ef9a --- /dev/null +++ b/0002-file-zero-Print-the-debug-message-on-the-fallback-pa.patch @@ -0,0 +1,38 @@ +From 56dba3f1fe87f119e05b74787197ec776ef2692d Mon Sep 17 00:00:00 2001 +From: "Richard W.M. Jones" +Date: Fri, 4 Jul 2025 08:20:43 +0100 +Subject: [PATCH] file: zero: Print the debug message on the fallback path once + +Use the new ONCE() macro to print the debug message when we fall back +to emulating zero only once. (Actually the core server code contains +a similar message so we probably don't need this at all.) + +(cherry picked from commit fbb5d8211bf4c30144d01be80720e1a63ecd6e81) +--- + plugins/file/file.c | 3 ++- + 1 file changed, 2 insertions(+), 1 deletion(-) + +diff --git a/plugins/file/file.c b/plugins/file/file.c +index 9c43ff24..b881da37 100644 +--- a/plugins/file/file.c ++++ b/plugins/file/file.c +@@ -71,6 +71,7 @@ + #include "isaligned.h" + #include "ispowerof2.h" + #include "minmax.h" ++#include "once.h" + #include "utils.h" + + static enum { +@@ -1165,7 +1166,7 @@ file_zero (void *handle, uint32_t count, uint64_t offset, uint32_t flags) + + /* Trigger a fall back to writing */ + if (file_debug_zero) +- nbdkit_debug ("zero falling back to writing"); ++ ONCE (nbdkit_debug ("%s: zero falling back to writing", h->name)); + errno = EOPNOTSUPP; + return -1; + +-- +2.47.1 + diff --git a/0003-file-trim-Don-t-try-BLKDISCARD-if-earlier-FALLOC_FL_.patch b/0003-file-trim-Don-t-try-BLKDISCARD-if-earlier-FALLOC_FL_.patch new file mode 100644 index 0000000..67a288c --- /dev/null +++ b/0003-file-trim-Don-t-try-BLKDISCARD-if-earlier-FALLOC_FL_.patch @@ -0,0 +1,65 @@ +From c9132973f88015586aa847ffcaa96e86bb23776f Mon Sep 17 00:00:00 2001 +From: "Richard W.M. Jones" +Date: Fri, 4 Jul 2025 08:14:55 +0100 +Subject: [PATCH] file: trim: Don't try BLKDISCARD if earlier + FALLOC_FL_PUNCH_HOLE worked + +In file_zero, we test if the operations we are trying succeed and if +so jump to a single 'out:' label where we deal with the success path +out of that function. + +We did not do the same thing in file_trim. Thus in the case where +FALLOC_FL_PUNCH_HOLE succeeds, we might fall through to trying +BLKDISCARD as well. As it happens we probably don't do this (at +least, in Linux) because we only try BLKDISCARD for block devices, and +FALLOC_FL_PUNCH_HOLE does not work on those. But it's a good thing to +clean up this code anyway, especially if we were to add more cases in +future. + +This also adds a debug message if none of the trim methods worked, +which is also analogous to what happens in the same part of file_zero. + +(cherry picked from commit 909e483c121c69e6b2759ef9d5401eb3d5acc998) +--- + plugins/file/file.c | 11 +++++++++++ + 1 file changed, 11 insertions(+) + +diff --git a/plugins/file/file.c b/plugins/file/file.c +index b881da37..66d03d4f 100644 +--- a/plugins/file/file.c ++++ b/plugins/file/file.c +@@ -1191,6 +1191,7 @@ file_trim (void *handle, uint32_t count, uint64_t offset, uint32_t flags) + + r = do_fallocate (h->fd, FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE, + offset, count); ++ if (r == 0) goto out; + if (r == -1) { + if (!is_enotsup (errno)) { + nbdkit_error ("fallocate: %s: offset=%" PRIu64 ", count=%" PRIu32 ":" +@@ -1214,6 +1215,7 @@ file_trim (void *handle, uint32_t count, uint64_t offset, uint32_t flags) + uint64_t range[2] = {offset, count}; + + r = ioctl (h->fd, BLKDISCARD, &range); ++ if (r == 0) goto out; + if (r == -1) { + if (!is_enotsup (errno)) { + nbdkit_error ("ioctl: %s: offset=%" PRIu64 ", count=%" PRIu32 ":" +@@ -1227,6 +1229,15 @@ file_trim (void *handle, uint32_t count, uint64_t offset, uint32_t flags) + } + #endif + ++ /* Trim is advisory. If we got here, we were unable to trim. */ ++ ONCE (nbdkit_debug ("%s: could not trim, no trim methods worked", ++ h->name)); ++ return 0; ++ ++#ifdef __clang__ ++ __attribute__ ((unused)) ++#endif ++ out: + if ((flags & NBDKIT_FLAG_FUA) && file_flush (handle, 0) == -1) + return -1; + +-- +2.47.1 + diff --git a/0004-common-include-test-once.c-Skip-test-on-macOS-which-.patch b/0004-common-include-test-once.c-Skip-test-on-macOS-which-.patch new file mode 100644 index 0000000..0eae34c --- /dev/null +++ b/0004-common-include-test-once.c-Skip-test-on-macOS-which-.patch @@ -0,0 +1,63 @@ +From 48869f1c0b6e4c318b680f6f672a9f90dfe31bff Mon Sep 17 00:00:00 2001 +From: "Richard W.M. Jones" +Date: Tue, 8 Jul 2025 21:39:04 +0100 +Subject: [PATCH] common/include/test-once.c: Skip test on macOS which lacks + pthread_barrier_t + +See: +https://pubs.opengroup.org/onlinepubs/009695399/basedefs/xbd_chap02.html +(cherry picked from commit 8271f9244f1521c716460820d8162e7641018674) +--- + common/include/test-once.c | 16 ++++++++-------- + 1 file changed, 8 insertions(+), 8 deletions(-) + +diff --git a/common/include/test-once.c b/common/include/test-once.c +index d7dd5c42..512b1a20 100644 +--- a/common/include/test-once.c ++++ b/common/include/test-once.c +@@ -34,29 +34,29 @@ + + #include + #include ++#include ++#include + +-#ifndef HAVE_STDATOMIC_H ++#if !defined(HAVE_STDATOMIC_H) || !defined(_POSIX_BARRIERS) + +-/* Skip the test if no */ ++/* Skip the test if no or pthread_barrier_t */ + + int + main (void) + { +- printf ("SKIP: no on this platform\n"); ++ fprintf (stderr, ++ "SKIP: no or pthread_barrier_t on this platform\n"); + exit (77); + } + +-#else /* HAVE_STDATOMIC_H */ ++#else + + #include + #include +-#include + + #undef NDEBUG /* Keep test strong even for nbdkit built without assertions */ + #include + +-#include +- + #include "once.h" + + #define NR_THREADS 8 +@@ -123,4 +123,4 @@ main (void) + exit (EXIT_SUCCESS); + } + +-#endif /* HAVE_STDATOMIC_H */ ++#endif +-- +2.47.1 + diff --git a/0005-common-include-test-once.c-Further-fixes-for-pthread.patch b/0005-common-include-test-once.c-Further-fixes-for-pthread.patch new file mode 100644 index 0000000..cb62f98 --- /dev/null +++ b/0005-common-include-test-once.c-Further-fixes-for-pthread.patch @@ -0,0 +1,57 @@ +From f694d06d432d10699e26b2234f7a285fc018e94c Mon Sep 17 00:00:00 2001 +From: "Richard W.M. Jones" +Date: Wed, 9 Jul 2025 12:20:34 +0100 +Subject: [PATCH] common/include/test-once.c: Further fixes for + pthread_barrier_t + +macOS defines _POSIX_BARRIERS but turns out to lack any implementation +of pthread_barrier_t. WTF. + +FreeBSD requires linking with pthread else we get: + + ld: error: undefined symbol: pthread_barrier_init + +Fixes: commit ad8630deab4639e636212f11a5a47d2c34ef2949 +Fixes: commit 8271f9244f1521c716460820d8162e7641018674 +(cherry picked from commit 0d0e2b3d49cf8c9aa8cd37bb36b7002eb4624a2c) +--- + common/include/Makefile.am | 3 ++- + common/include/test-once.c | 7 +++++-- + 2 files changed, 7 insertions(+), 3 deletions(-) + +diff --git a/common/include/Makefile.am b/common/include/Makefile.am +index 3a3757e2..00a01091 100644 +--- a/common/include/Makefile.am ++++ b/common/include/Makefile.am +@@ -124,7 +124,8 @@ test_nextnonzero_CFLAGS = $(WARNINGS_CFLAGS) + + test_once_SOURCES = test-once.c once.h + test_once_CPPFLAGS = -I$(srcdir) +-test_once_CFLAGS = $(WARNINGS_CFLAGS) ++test_once_CFLAGS = $(WARNINGS_CFLAGS) $(PTHREAD_CFLAGS) ++test_once_LDFLAGS = $(PTHREAD_LIBS) + + test_random_SOURCES = test-random.c random.h + test_random_CPPFLAGS = -I$(srcdir) +diff --git a/common/include/test-once.c b/common/include/test-once.c +index 512b1a20..304d512a 100644 +--- a/common/include/test-once.c ++++ b/common/include/test-once.c +@@ -37,9 +37,12 @@ + #include + #include + +-#if !defined(HAVE_STDATOMIC_H) || !defined(_POSIX_BARRIERS) ++#if !defined(HAVE_STDATOMIC_H) || !defined(_POSIX_BARRIERS) || \ ++ defined(__APPLE__) + +-/* Skip the test if no or pthread_barrier_t */ ++/* Skip the test if no or pthread_barrier_t or on macOS ++ * which defines _POSIX_BARRIERS but doesn't actually have them. ++ */ + + int + main (void) +-- +2.47.1 + diff --git a/0006-Remove-deprecated-cacheextents-filter.patch b/0006-Remove-deprecated-cacheextents-filter.patch new file mode 100644 index 0000000..4953728 --- /dev/null +++ b/0006-Remove-deprecated-cacheextents-filter.patch @@ -0,0 +1,692 @@ +From dc8c6aae6aa1c62083421e2b2ce2988e970f2579 Mon Sep 17 00:00:00 2001 +From: "Richard W.M. Jones" +Date: Tue, 1 Jul 2025 13:26:08 +0100 +Subject: [PATCH] Remove deprecated cacheextents filter + +This is removed along the development branch (1.45) and in the next +stable version of nbdkit (1.46). + +(cherry picked from commit 886050f6de9179b0e267e15b20376313090b2d3c) +--- + configure.ac | 2 - + docs/nbdkit-protocol.pod | 9 +- + docs/nbdkit-release-notes-1.14.pod | 2 +- + docs/nbdkit-release-notes-1.44.pod | 2 +- + filters/cache/nbdkit-cache-filter.pod | 8 +- + filters/cacheextents/Makefile.am | 74 ------ + filters/cacheextents/cacheextents.c | 212 ------------------ + .../nbdkit-cacheextents-filter.pod | 76 ------- + filters/cow/nbdkit-cow-filter.pod | 1 - + .../extentlist/nbdkit-extentlist-filter.pod | 1 - + tests/Makefile.am | 4 - + tests/test-cacheextents.sh | 114 ---------- + 12 files changed, 9 insertions(+), 496 deletions(-) + delete mode 100644 filters/cacheextents/Makefile.am + delete mode 100644 filters/cacheextents/cacheextents.c + delete mode 100644 filters/cacheextents/nbdkit-cacheextents-filter.pod + delete mode 100755 tests/test-cacheextents.sh + +diff --git a/configure.ac b/configure.ac +index 0dca333f..9b057e6f 100644 +--- a/configure.ac ++++ b/configure.ac +@@ -143,7 +143,6 @@ filters="\ + blocksize-policy \ + bzip2 \ + cache \ +- cacheextents \ + checkwrite \ + cow \ + ddrescue \ +@@ -1787,7 +1786,6 @@ AC_CONFIG_FILES([Makefile + filters/blocksize-policy/Makefile + filters/bzip2/Makefile + filters/cache/Makefile +- filters/cacheextents/Makefile + filters/checkwrite/Makefile + filters/cow/Makefile + filters/ddrescue/Makefile +diff --git a/docs/nbdkit-protocol.pod b/docs/nbdkit-protocol.pod +index ef1934fd..93f5c5fa 100644 +--- a/docs/nbdkit-protocol.pod ++++ b/docs/nbdkit-protocol.pod +@@ -275,14 +275,13 @@ filters do not work properly in this case. + blocksize-policy Yes + bzip2 No + cache No +- cacheextents No + checkwrite Yes + cow Yes, since 1.44 + delay Yes + error Yes + evil Yes +- + exitlast Yes ++ + exitwhen Yes + exportname Yes + ext2 No +@@ -292,8 +291,8 @@ filters do not work properly in this case. + ip Yes + limit Yes + log Yes +- + luks No ++ + lzip No + multi-conn Yes + nocache Yes +@@ -303,8 +302,8 @@ filters do not work properly in this case. + nozero Yes + offset Yes, but unlikely to be useful + openonce Yes +- + partition No ++ + pause Yes + protect Yes, but unlikely to be useful + qcow2dec No +@@ -314,8 +313,8 @@ filters do not work properly in this case. + retry Yes + retry-request Yes + rotational Yes +- + scan Yes ++ + spinning Yes + stats Yes + swab Yes +diff --git a/docs/nbdkit-release-notes-1.14.pod b/docs/nbdkit-release-notes-1.14.pod +index 627e7e88..3c8c5d53 100644 +--- a/docs/nbdkit-release-notes-1.14.pod ++++ b/docs/nbdkit-release-notes-1.14.pod +@@ -50,7 +50,7 @@ plugin’s own choice of thread model. Used to determine how the thread + model affects performance, or to serialize plugins if required (Eric + Blake). + +-New L to cache extents requests, ++New nbdkit-cacheextents-filter to cache extents requests, + especially useful with VDDK which has a slow implementation of extents + (Martin Kletzander). + +diff --git a/docs/nbdkit-release-notes-1.44.pod b/docs/nbdkit-release-notes-1.44.pod +index 62d69aa5..e5872763 100644 +--- a/docs/nbdkit-release-notes-1.44.pod ++++ b/docs/nbdkit-release-notes-1.44.pod +@@ -59,7 +59,7 @@ eg. C<@4M> to move the offset to 4194304 (Eric Blake). + New L which can be used to open the + underlying plugin once, sharing the plugin across connections. + +-L I, and is ++I, and is + expected to be removed in S. + + L now understands that the NBD protocol export +diff --git a/filters/cache/nbdkit-cache-filter.pod b/filters/cache/nbdkit-cache-filter.pod +index ffa86919..7a64ca66 100644 +--- a/filters/cache/nbdkit-cache-filter.pod ++++ b/filters/cache/nbdkit-cache-filter.pod +@@ -29,10 +29,9 @@ does not have effective caching, or (with C) to defeat + flush requests from the client (which is unsafe and can cause data + loss, as the name suggests). + +-This filter only caches image contents. To cache image metadata, use +-L between this filter and the plugin. +-To accelerate sequential reads, use L or +-L on top of this filter. ++This filter only caches image contents. To accelerate sequential ++reads, use L or L ++on top of this filter. + + =head1 PARAMETERS + +@@ -181,7 +180,6 @@ C first appeared in nbdkit 1.2. + + L, + L, +-L, + L, + L, + L, +diff --git a/filters/cacheextents/Makefile.am b/filters/cacheextents/Makefile.am +deleted file mode 100644 +index 26ac6642..00000000 +--- a/filters/cacheextents/Makefile.am ++++ /dev/null +@@ -1,74 +0,0 @@ +-# nbdkit +-# Copyright Red Hat +-# +-# Redistribution and use in source and binary forms, with or without +-# modification, are permitted provided that the following conditions are +-# met: +-# +-# * Redistributions of source code must retain the above copyright +-# notice, this list of conditions and the following disclaimer. +-# +-# * Redistributions in binary form must reproduce the above copyright +-# notice, this list of conditions and the following disclaimer in the +-# documentation and/or other materials provided with the distribution. +-# +-# * Neither the name of Red Hat nor the names of its contributors may be +-# used to endorse or promote products derived from this software without +-# specific prior written permission. +-# +-# THIS SOFTWARE IS PROVIDED BY RED HAT AND CONTRIBUTORS ''AS IS'' AND +-# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +-# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A +-# PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL RED HAT OR +-# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +-# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +-# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF +-# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +-# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +-# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT +-# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +-# SUCH DAMAGE. +- +-include $(top_srcdir)/common-rules.mk +- +-EXTRA_DIST = nbdkit-cacheextents-filter.pod +- +-filter_LTLIBRARIES = nbdkit-cacheextents-filter.la +- +-nbdkit_cacheextents_filter_la_SOURCES = \ +- cacheextents.c \ +- $(top_srcdir)/include/nbdkit-filter.h \ +- $(NULL) +- +-nbdkit_cacheextents_filter_la_CPPFLAGS = \ +- -I$(top_srcdir)/include \ +- -I$(top_builddir)/include \ +- -I$(top_srcdir)/common/include \ +- -I$(top_srcdir)/common/utils \ +- $(NULL) +-nbdkit_cacheextents_filter_la_CFLAGS = $(WARNINGS_CFLAGS) +-nbdkit_cacheextents_filter_la_LDFLAGS = \ +- -module -avoid-version -shared $(NO_UNDEFINED_ON_WINDOWS) \ +- $(NULL) +-if USE_LINKER_SCRIPT +-nbdkit_cacheextents_filter_la_LDFLAGS += \ +- -Wl,--version-script=$(top_srcdir)/filters/filters.syms +-endif +-nbdkit_cacheextents_filter_la_LIBADD = \ +- $(top_builddir)/common/utils/libutils.la \ +- $(top_builddir)/common/replacements/libcompat.la \ +- $(IMPORT_LIBRARY_ON_WINDOWS) \ +- $(NULL) +- +-if HAVE_POD +- +-man_MANS = nbdkit-cacheextents-filter.1 +-CLEANFILES += $(man_MANS) +- +-nbdkit-cacheextents-filter.1: nbdkit-cacheextents-filter.pod \ +- $(top_builddir)/podwrapper.pl +- $(PODWRAPPER) --section=1 --man $@ \ +- --html $(top_builddir)/html/$@.html \ +- $< +- +-endif HAVE_POD +diff --git a/filters/cacheextents/cacheextents.c b/filters/cacheextents/cacheextents.c +deleted file mode 100644 +index 71f73c41..00000000 +--- a/filters/cacheextents/cacheextents.c ++++ /dev/null +@@ -1,212 +0,0 @@ +-/* nbdkit +- * Copyright Red Hat +- * +- * Redistribution and use in source and binary forms, with or without +- * modification, are permitted provided that the following conditions are +- * met: +- * +- * * Redistributions of source code must retain the above copyright +- * notice, this list of conditions and the following disclaimer. +- * +- * * Redistributions in binary form must reproduce the above copyright +- * notice, this list of conditions and the following disclaimer in the +- * documentation and/or other materials provided with the distribution. +- * +- * * Neither the name of Red Hat nor the names of its contributors may be +- * used to endorse or promote products derived from this software without +- * specific prior written permission. +- * +- * THIS SOFTWARE IS PROVIDED BY RED HAT AND CONTRIBUTORS ''AS IS'' AND +- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +- * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A +- * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL RED HAT OR +- * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF +- * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +- * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +- * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT +- * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +- * SUCH DAMAGE. +- */ +- +-#include +- +-#include +-#include +-#include +-#include +-#include +-#include +- +-#include +- +-#include +- +-#include "cleanup.h" +- +-/* -D cacheextents.cache=1: Debug cache operations. */ +-NBDKIT_DLL_PUBLIC int cacheextents_debug_cache = 0; +- +-/* This lock protects the global state. */ +-static pthread_mutex_t lock = PTHREAD_MUTEX_INITIALIZER; +- +-/* Cached extents from the last extents () call and its start and end +- * for the sake of simplicity. +- */ +-struct nbdkit_extents *cache_extents; +-static uint64_t cache_start; +-static uint64_t cache_end; +- +-static void +-cacheextents_unload (void) +-{ +- nbdkit_extents_free (cache_extents); +-} +- +-static int +-cacheextents_add (struct nbdkit_extents *extents, int *err) +-{ +- size_t i = 0; +- +- for (i = 0; i < nbdkit_extents_count (cache_extents); i++) { +- struct nbdkit_extent ex = nbdkit_get_extent (cache_extents, i); +- if (nbdkit_add_extent (extents, ex.offset, ex.length, ex.type) == -1) { +- *err = errno; +- return -1; +- } +- } +- +- return 0; +-} +- +-static int +-fill (struct nbdkit_extents *extents, int *err) +-{ +- size_t i = 0; +- size_t count = nbdkit_extents_count (extents); +- struct nbdkit_extent first = nbdkit_get_extent (extents, 0); +- struct nbdkit_extent last = nbdkit_get_extent (extents, count - 1); +- +- nbdkit_extents_free (cache_extents); +- cache_start = first.offset; +- cache_end = last.offset + last.length; +- cache_extents = nbdkit_extents_new (cache_start, cache_end); +- +- if (!cache_extents) +- return -1; +- +- for (i = 0; i < count; i++) { +- struct nbdkit_extent ex = nbdkit_get_extent (extents, i); +- +- if (cacheextents_debug_cache) +- nbdkit_debug ("cacheextents: updating cache with:" +- " offset=%" PRIu64 +- " length=%" PRIu64 +- " type=%x", +- ex.offset, ex.length, ex.type); +- +- if (nbdkit_add_extent (cache_extents, ex.offset, ex.length, +- ex.type) == -1) { +- *err = errno; +- nbdkit_extents_free (cache_extents); +- cache_extents = NULL; +- return -1; +- } +- } +- +- return 0; +-} +- +-static int +-cacheextents_extents (nbdkit_next *next, +- void *handle, uint32_t count, uint64_t offset, +- uint32_t flags, +- struct nbdkit_extents *extents, +- int *err) +-{ +- ACQUIRE_LOCK_FOR_CURRENT_SCOPE (&lock); +- +- if (cacheextents_debug_cache) +- nbdkit_debug ("cacheextents:" +- " cache_start=%" PRIu64 +- " cache_end=%" PRIu64 +- " cache_extents=%p", +- cache_start, cache_end, cache_extents); +- +- if (cache_extents && +- offset >= cache_start && offset < cache_end) { +- if (cacheextents_debug_cache) +- nbdkit_debug ("cacheextents: returning from cache"); +- return cacheextents_add (extents, err); +- } +- +- if (cacheextents_debug_cache) +- nbdkit_debug ("cacheextents: cache miss"); +- +- /* Clear REQ_ONE to ask the plugin for as much information as it is +- * willing to return (the plugin may still truncate if it is too +- * costly to provide everything). +- */ +- flags &= ~(NBDKIT_FLAG_REQ_ONE); +- if (next->extents (next, count, offset, flags, extents, err) == -1) +- return -1; +- +- return fill (extents, err); +-} +- +-/* Any changes to the data needs to clean the cache. +- * +- * Similar to the readahead filter this could be more intelligent, but +- * there would be very little benefit. +- */ +- +-static void +-kill_cacheextents (void) +-{ +- ACQUIRE_LOCK_FOR_CURRENT_SCOPE (&lock); +- nbdkit_extents_free (cache_extents); +- cache_extents = NULL; +-} +- +-static int +-cacheextents_pwrite (nbdkit_next *next, +- void *handle, +- const void *buf, uint32_t count, uint64_t offset, +- uint32_t flags, int *err) +-{ +- kill_cacheextents (); +- return next->pwrite (next, buf, count, offset, flags, err); +-} +- +-static int +-cacheextents_trim (nbdkit_next *next, +- void *handle, +- uint32_t count, uint64_t offset, uint32_t flags, +- int *err) +-{ +- kill_cacheextents (); +- return next->trim (next, count, offset, flags, err); +-} +- +-static int +-cacheextents_zero (nbdkit_next *next, +- void *handle, +- uint32_t count, uint64_t offset, uint32_t flags, +- int *err) +-{ +- kill_cacheextents (); +- return next->zero (next, count, offset, flags, err); +-} +- +-static struct nbdkit_filter filter = { +- .name = "cacheextents", +- .longname = "nbdkit cacheextents filter", +- .unload = cacheextents_unload, +- .pwrite = cacheextents_pwrite, +- .trim = cacheextents_trim, +- .zero = cacheextents_zero, +- .extents = cacheextents_extents, +-}; +- +-NBDKIT_REGISTER_FILTER (filter) +diff --git a/filters/cacheextents/nbdkit-cacheextents-filter.pod b/filters/cacheextents/nbdkit-cacheextents-filter.pod +deleted file mode 100644 +index 0693ca80..00000000 +--- a/filters/cacheextents/nbdkit-cacheextents-filter.pod ++++ /dev/null +@@ -1,76 +0,0 @@ +-=head1 NAME +- +-nbdkit-cacheextents-filter - cache extents +- +-=head1 SYNOPSIS +- +- nbdkit --filter=cacheextents plugin +- +-=head1 DEPRECATED +- +-B 1.43.10> and +-will be removed in S>. There is no direct replacement, +-but as the filter only worked for a narrow and unusual range of access +-patterns it is likely that it has no effect and you can just stop +-using it. +- +-=head1 DESCRIPTION +- +-C is a filter that caches the result of last +-extents() call. +- +-A common use for this filter is to improve performance when using a +-client performing a linear pass over the entire image while asking for +-only one extent at a time (such as S>), but where +-the plugin can provide multiple extents for the same high latency as a +-single extent (such as L). For example: +- +- nbdkit --filter=cacheextents --run 'qemu-img map "$uri"' vddk ... +- +-For files with big extents (when it is unlikely for one extents() call +-to return multiple different extents) this does not slow down the +-access. +- +-This filter only caches image metadata; to also cache image contents, +-place this filter between L and the plugin. +- +-=head1 PARAMETERS +- +-There are no parameters specific to nbdkit-cacheextents-filter. Any +-parameters are passed through to and processed by the underlying +-plugin in the normal way. +- +-=head1 FILES +- +-=over 4 +- +-=item F<$filterdir/nbdkit-cacheextents-filter.so> +- +-The filter. +- +-Use C to find the location of C<$filterdir>. +- +-=back +- +-=head1 VERSION +- +-C first appeared in nbdkit 1.14. +- +-=head1 SEE ALSO +- +-L, +-L, +-L, +-L, +-L, +-L, +-L, +-L. +- +-=head1 AUTHORS +- +-Martin Kletzander +- +-=head1 COPYRIGHT +- +-Copyright Red Hat +diff --git a/filters/cow/nbdkit-cow-filter.pod b/filters/cow/nbdkit-cow-filter.pod +index fd551d93..9462a28d 100644 +--- a/filters/cow/nbdkit-cow-filter.pod ++++ b/filters/cow/nbdkit-cow-filter.pod +@@ -169,7 +169,6 @@ C first appeared in nbdkit 1.2. + L, + L, + L, +-L, + L, + L, + L, +diff --git a/filters/extentlist/nbdkit-extentlist-filter.pod b/filters/extentlist/nbdkit-extentlist-filter.pod +index d5ac81eb..44c81635 100644 +--- a/filters/extentlist/nbdkit-extentlist-filter.pod ++++ b/filters/extentlist/nbdkit-extentlist-filter.pod +@@ -85,7 +85,6 @@ C first appeared in nbdkit 1.18. + =head1 SEE ALSO + + L, +-L, + L, + L, + L. +diff --git a/tests/Makefile.am b/tests/Makefile.am +index 6d94c327..c16b5912 100644 +--- a/tests/Makefile.am ++++ b/tests/Makefile.am +@@ -1694,10 +1694,6 @@ EXTRA_DIST += \ + test-cache-unaligned.sh \ + $(NULL) + +-# cacheextents filter test. +-TESTS += test-cacheextents.sh +-EXTRA_DIST += test-cacheextents.sh +- + # checkwrite filter test. + TESTS += \ + test-checkwrite.sh \ +diff --git a/tests/test-cacheextents.sh b/tests/test-cacheextents.sh +deleted file mode 100755 +index 34d66217..00000000 +--- a/tests/test-cacheextents.sh ++++ /dev/null +@@ -1,114 +0,0 @@ +-#!/usr/bin/env bash +-# nbdkit +-# Copyright Red Hat +-# +-# Redistribution and use in source and binary forms, with or without +-# modification, are permitted provided that the following conditions are +-# met: +-# +-# * Redistributions of source code must retain the above copyright +-# notice, this list of conditions and the following disclaimer. +-# +-# * Redistributions in binary form must reproduce the above copyright +-# notice, this list of conditions and the following disclaimer in the +-# documentation and/or other materials provided with the distribution. +-# +-# * Neither the name of Red Hat nor the names of its contributors may be +-# used to endorse or promote products derived from this software without +-# specific prior written permission. +-# +-# THIS SOFTWARE IS PROVIDED BY RED HAT AND CONTRIBUTORS ''AS IS'' AND +-# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +-# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A +-# PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL RED HAT OR +-# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +-# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +-# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF +-# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +-# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +-# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT +-# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +-# SUCH DAMAGE. +- +-source ./functions.sh +-set -x +-set -u +-set -e +- +-requires_plugin sh +-requires_filter cacheextents +-requires grep --version +-requires qemu-io --version +-requires dd iflag=count_bytes >$accessfile_full +-size=4M +-block_size=$((1024*1024)) +-case "$1" in +- thread_model) echo parallel ;; +- get_size) echo $size ;; +- can_extents) ;; +- extents) +- echo "extents request: $@" >>$accessfile_full +- offset=$(($4 / $block_size)) +- count=$(($3 / $block_size)) +- length=$(($offset + $count)) +- for i in $(seq $offset $length); do +- echo ${i}M $block_size $((i%4)) >>$accessfile_full +- echo ${i}M $block_size $((i%4)) +- done +- ;; +- pread) dd if=/dev/zero count=$3 iflag=count_bytes ;; +- can_write) ;; +- pwrite) dd of=/dev/null ;; +- can_trim) ;; +- trim) ;; +- can_zero) ;; +- zero) ;; +- *) exit 2 ;; +-esac +-EOF +- +-export accessfile_full +-start_nbdkit \ +- -P $pidfile \ +- -U $sock \ +- --filter=cacheextents \ +- sh - <<<"$plugin" +- +-test_me() { +- num_accesses=$1 +- shift +- +- qemu-io -f raw "$@" "$sockurl" +- test "$(grep -c "^extents request: " $accessfile)" -eq "$num_accesses" +- ret=$? +- rm -f "$accessfile" +- return $ret +-} +- +-# First one causes caching, the rest should be returned from cache. +-test_me 1 -c 'map' -c 'map' -c 'map' +-# First one is still cached from last time, discard should kill the cache, then +-# one request should go through. +-test_me 1 -c 'map' -c 'discard 0 1' -c 'map' +-# Same as above, only this time the cache is killed before all the operations as +-# well. This is used from now on to clear the cache as it seems nicer and +-# faster than running new nbdkit for each test. +-test_me 2 -c 'discard 0 1' -c 'map' -c 'discard 0 1' -c 'map' +-# Write should kill the cache as well. +-test_me 2 -c 'discard 0 1' -c 'map' -c 'write 0 1' -c 'map' +-# Alloc should use cached data from map +-test_me 1 -c 'discard 0 1' -c 'map' -c 'alloc 0' +-# Read should not kill the cache +-test_me 1 -c 'discard 0 1' -c 'map' -c 'read 0 1' -c 'map' -c 'alloc 0' +-- +2.47.1 + diff --git a/0007-New-filter-nbdkit-count-filter-count-bytes-read-writ.patch b/0007-New-filter-nbdkit-count-filter-count-bytes-read-writ.patch new file mode 100644 index 0000000..832fb98 --- /dev/null +++ b/0007-New-filter-nbdkit-count-filter-count-bytes-read-writ.patch @@ -0,0 +1,500 @@ +From 9c28df70cbde94e58e448c8953965510e5d952c2 Mon Sep 17 00:00:00 2001 +From: "Richard W.M. Jones" +Date: Tue, 8 Jul 2025 17:49:00 +0100 +Subject: [PATCH] New filter: nbdkit-count-filter: count bytes read, written + etc. + +This produces a summary of the number of bytes read, written, etc +through the filter. + +This is mainly of use to virt-v2v where it's commonly asked how much +data was transferred over the wire or written to disk, and we don't +currently have an easy way to answer that. By simply adding this +filter, the numbers will be known from the virt-v2v conversion log. + +(cherry picked from commit 3512c3ce9308b4d940119ac6cc87f1baa9afb655) +--- + configure.ac | 2 + + docs/nbdkit-protocol.pod | 9 +- + filters/count/Makefile.am | 70 ++++++++++++++ + filters/count/count.c | 132 ++++++++++++++++++++++++++ + filters/count/nbdkit-count-filter.pod | 55 +++++++++++ + filters/log/nbdkit-log-filter.pod | 1 + + filters/stats/nbdkit-stats-filter.pod | 3 + + plugins/file/nbdkit-file-plugin.pod | 1 + + tests/Makefile.am | 4 + + tests/test-count.sh | 55 +++++++++++ + 10 files changed, 328 insertions(+), 4 deletions(-) + create mode 100644 filters/count/Makefile.am + create mode 100644 filters/count/count.c + create mode 100644 filters/count/nbdkit-count-filter.pod + create mode 100755 tests/test-count.sh + +diff --git a/configure.ac b/configure.ac +index 9b057e6f..26e59462 100644 +--- a/configure.ac ++++ b/configure.ac +@@ -144,6 +144,7 @@ filters="\ + bzip2 \ + cache \ + checkwrite \ ++ count \ + cow \ + ddrescue \ + delay \ +@@ -1787,6 +1788,7 @@ AC_CONFIG_FILES([Makefile + filters/bzip2/Makefile + filters/cache/Makefile + filters/checkwrite/Makefile ++ filters/count/Makefile + filters/cow/Makefile + filters/ddrescue/Makefile + filters/delay/Makefile +diff --git a/docs/nbdkit-protocol.pod b/docs/nbdkit-protocol.pod +index 93f5c5fa..edf0efb0 100644 +--- a/docs/nbdkit-protocol.pod ++++ b/docs/nbdkit-protocol.pod +@@ -276,12 +276,13 @@ filters do not work properly in this case. + bzip2 No + cache No + checkwrite Yes ++ count Yes + cow Yes, since 1.44 + delay Yes + error Yes + evil Yes ++ + exitlast Yes +- + exitwhen Yes + exportname Yes + ext2 No +@@ -291,8 +292,8 @@ filters do not work properly in this case. + ip Yes + limit Yes + log Yes ++ + luks No +- + lzip No + multi-conn Yes + nocache Yes +@@ -302,8 +303,8 @@ filters do not work properly in this case. + nozero Yes + offset Yes, but unlikely to be useful + openonce Yes ++ + partition No +- + pause Yes + protect Yes, but unlikely to be useful + qcow2dec No +@@ -313,8 +314,8 @@ filters do not work properly in this case. + retry Yes + retry-request Yes + rotational Yes ++ + scan Yes +- + spinning Yes + stats Yes + swab Yes +diff --git a/filters/count/Makefile.am b/filters/count/Makefile.am +new file mode 100644 +index 00000000..20456e17 +--- /dev/null ++++ b/filters/count/Makefile.am +@@ -0,0 +1,70 @@ ++# nbdkit ++# Copyright Red Hat ++# ++# Redistribution and use in source and binary forms, with or without ++# modification, are permitted provided that the following conditions are ++# met: ++# ++# * Redistributions of source code must retain the above copyright ++# notice, this list of conditions and the following disclaimer. ++# ++# * Redistributions in binary form must reproduce the above copyright ++# notice, this list of conditions and the following disclaimer in the ++# documentation and/or other materials provided with the distribution. ++# ++# * Neither the name of Red Hat nor the names of its contributors may be ++# used to endorse or promote products derived from this software without ++# specific prior written permission. ++# ++# THIS SOFTWARE IS PROVIDED BY RED HAT AND CONTRIBUTORS ''AS IS'' AND ++# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, ++# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A ++# PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL RED HAT OR ++# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, ++# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT ++# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF ++# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ++# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, ++# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT ++# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF ++# SUCH DAMAGE. ++ ++include $(top_srcdir)/common-rules.mk ++ ++EXTRA_DIST = nbdkit-count-filter.pod ++ ++filter_LTLIBRARIES = nbdkit-count-filter.la ++ ++nbdkit_count_filter_la_SOURCES = \ ++ count.c \ ++ $(top_srcdir)/include/nbdkit-filter.h \ ++ $(NULL) ++ ++nbdkit_count_filter_la_CPPFLAGS = \ ++ -I$(top_srcdir)/include \ ++ -I$(top_builddir)/include \ ++ $(NULL) ++nbdkit_count_filter_la_CFLAGS = $(WARNINGS_CFLAGS) ++nbdkit_count_filter_la_LDFLAGS = \ ++ -module -avoid-version -shared $(NO_UNDEFINED_ON_WINDOWS) \ ++ $(NULL) ++if USE_LINKER_SCRIPT ++nbdkit_count_filter_la_LDFLAGS += \ ++ -Wl,--version-script=$(top_srcdir)/filters/filters.syms ++endif ++nbdkit_count_filter_la_LIBADD = \ ++ $(IMPORT_LIBRARY_ON_WINDOWS) \ ++ $(NULL) ++ ++if HAVE_POD ++ ++man_MANS = nbdkit-count-filter.1 ++CLEANFILES += $(man_MANS) ++ ++nbdkit-count-filter.1: nbdkit-count-filter.pod \ ++ $(top_builddir)/podwrapper.pl ++ $(PODWRAPPER) --section=1 --man $@ \ ++ --html $(top_builddir)/html/$@.html \ ++ $< ++ ++endif HAVE_POD +diff --git a/filters/count/count.c b/filters/count/count.c +new file mode 100644 +index 00000000..8af7f5a0 +--- /dev/null ++++ b/filters/count/count.c +@@ -0,0 +1,132 @@ ++/* nbdkit ++ * Copyright Red Hat ++ * ++ * Redistribution and use in source and binary forms, with or without ++ * modification, are permitted provided that the following conditions are ++ * met: ++ * ++ * * Redistributions of source code must retain the above copyright ++ * notice, this list of conditions and the following disclaimer. ++ * ++ * * Redistributions in binary form must reproduce the above copyright ++ * notice, this list of conditions and the following disclaimer in the ++ * documentation and/or other materials provided with the distribution. ++ * ++ * * Neither the name of Red Hat nor the names of its contributors may be ++ * used to endorse or promote products derived from this software without ++ * specific prior written permission. ++ * ++ * THIS SOFTWARE IS PROVIDED BY RED HAT AND CONTRIBUTORS ''AS IS'' AND ++ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, ++ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A ++ * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL RED HAT OR ++ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, ++ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT ++ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF ++ * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ++ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, ++ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT ++ * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF ++ * SUCH DAMAGE. ++ */ ++ ++#include ++ ++#include ++#include ++#include ++#include ++ ++#include ++ ++#ifdef HAVE_STDATOMIC_H ++#include ++#else ++/* Only used for counting statistics. */ ++#define _Atomic /**/ ++#endif ++ ++static _Atomic uint64_t bytes_read, bytes_written, bytes_zeroed, bytes_trimmed; ++ ++static void ++count_unload (void) ++{ ++ nbdkit_debug ("count bytes: " ++ "read %" PRIu64 ", " ++ "written %" PRIu64 ", " ++ "zeroed %" PRIu64 ", " ++ "trimmed %" PRIu64, ++ bytes_read, bytes_written, bytes_zeroed, bytes_trimmed); ++} ++ ++/* Read data. */ ++static int ++count_pread (nbdkit_next *next, ++ void *handle, ++ void *buf, ++ uint32_t count, uint64_t offset, uint32_t flags, ++ int *err) ++{ ++ int r; ++ ++ r = next->pread (next, buf, count, offset, flags, err); ++ if (r >= 0) ++ bytes_read += count; ++ return r; ++} ++ ++/* Write data. */ ++static int ++count_pwrite (nbdkit_next *next, ++ void *handle, ++ const void *buf, ++ uint32_t count, uint64_t offset, uint32_t flags, ++ int *err) ++{ ++ int r; ++ ++ r = next->pwrite (next, buf, count, offset, flags, err); ++ if (r >= 0) ++ bytes_written += count; ++ return r; ++} ++ ++/* Trim data. */ ++static int ++count_trim (nbdkit_next *next, ++ void *handle, uint32_t count, uint64_t offset, uint32_t flags, ++ int *err) ++{ ++ int r; ++ ++ r = next->trim (next, count, offset, flags, err); ++ if (r >= 0) ++ bytes_trimmed += count; ++ return r; ++} ++ ++/* Zero data. */ ++static int ++count_zero (nbdkit_next *next, ++ void *handle, uint32_t count, uint64_t offset, uint32_t flags, ++ int *err) ++{ ++ int r; ++ ++ r = next->zero (next, count, offset, flags, err); ++ if (r >= 0) ++ bytes_zeroed += count; ++ return r; ++} ++ ++static struct nbdkit_filter filter = { ++ .name = "count", ++ .longname = "nbdkit count filter", ++ .unload = count_unload, ++ .pread = count_pread, ++ .pwrite = count_pwrite, ++ .trim = count_trim, ++ .zero = count_zero, ++}; ++ ++NBDKIT_REGISTER_FILTER (filter) +diff --git a/filters/count/nbdkit-count-filter.pod b/filters/count/nbdkit-count-filter.pod +new file mode 100644 +index 00000000..f0437000 +--- /dev/null ++++ b/filters/count/nbdkit-count-filter.pod +@@ -0,0 +1,55 @@ ++=head1 NAME ++ ++nbdkit-count-filter - count bytes read, written, zeroed and trimmed ++ ++=head1 SYNOPSIS ++ ++ nbdkit --filter=count plugin ++ ++=head1 DESCRIPTION ++ ++C is a filter for L that simply counts ++the number of bytes that are read, written, zeroed and trimmed, and ++reports this number in debugging output when the filter is unloaded ++(usually when nbdkit exits). ++ ++This is a very simple and lightweight filter. For much more ++comprehensive stats about and logging of operations, use ++L or L instead. ++ ++=head1 PARAMETERS ++ ++There are no parameters specific to this filter. All parameters are ++passed through to the underlying plugin. ++ ++=head1 FILES ++ ++=over 4 ++ ++=item F<$filterdir/nbdkit-count-filter.so> ++ ++The filter. ++ ++Use C to find the location of C<$filterdir>. ++ ++=back ++ ++=head1 VERSION ++ ++C first appeared in nbdkit 1.46. ++ ++=head1 SEE ALSO ++ ++L, ++L, ++L, ++L, ++L. ++ ++=head1 AUTHORS ++ ++Richard W.M. Jones ++ ++=head1 COPYRIGHT ++ ++Copyright Red Hat +diff --git a/filters/log/nbdkit-log-filter.pod b/filters/log/nbdkit-log-filter.pod +index b91b60c4..256701a1 100644 +--- a/filters/log/nbdkit-log-filter.pod ++++ b/filters/log/nbdkit-log-filter.pod +@@ -208,6 +208,7 @@ L, + L, + L, + L, ++L, + L. + + =head1 AUTHORS +diff --git a/filters/stats/nbdkit-stats-filter.pod b/filters/stats/nbdkit-stats-filter.pod +index c0d2b45c..10074b4a 100644 +--- a/filters/stats/nbdkit-stats-filter.pod ++++ b/filters/stats/nbdkit-stats-filter.pod +@@ -13,6 +13,8 @@ C is a filter that displays statistics about NBD + operations, such as the number of bytes read and written. Statistics + are written to a file once when nbdkit exits. + ++A lighter weight version of this is L. ++ + =head1 EXAMPLE OUTPUT + + # nbdkit --filter=exitlast --filter=stats memory 25G statsfile=example.txt +@@ -113,6 +115,7 @@ C first appeared in nbdkit 1.14. + + L, + L, ++L, + L. + + =head1 AUTHORS +diff --git a/plugins/file/nbdkit-file-plugin.pod b/plugins/file/nbdkit-file-plugin.pod +index 63d07617..626827b2 100644 +--- a/plugins/file/nbdkit-file-plugin.pod ++++ b/plugins/file/nbdkit-file-plugin.pod +@@ -316,6 +316,7 @@ L, + L, + L, + L, ++L, + L, + L, + L, +diff --git a/tests/Makefile.am b/tests/Makefile.am +index c16b5912..d7053ba2 100644 +--- a/tests/Makefile.am ++++ b/tests/Makefile.am +@@ -1706,6 +1706,10 @@ EXTRA_DIST += \ + test-checkwrite-fail.sh \ + $(NULL) + ++# count filter test. ++TESTS += test-count.sh ++EXTRA_DIST += test-count.sh ++ + # cow filter test. + if HAVE_MKE2FS_WITH_D + TESTS += \ +diff --git a/tests/test-count.sh b/tests/test-count.sh +new file mode 100755 +index 00000000..e2e10704 +--- /dev/null ++++ b/tests/test-count.sh +@@ -0,0 +1,55 @@ ++#!/usr/bin/env bash ++# nbdkit ++# Copyright Red Hat ++# ++# Redistribution and use in source and binary forms, with or without ++# modification, are permitted provided that the following conditions are ++# met: ++# ++# * Redistributions of source code must retain the above copyright ++# notice, this list of conditions and the following disclaimer. ++# ++# * Redistributions in binary form must reproduce the above copyright ++# notice, this list of conditions and the following disclaimer in the ++# documentation and/or other materials provided with the distribution. ++# ++# * Neither the name of Red Hat nor the names of its contributors may be ++# used to endorse or promote products derived from this software without ++# specific prior written permission. ++# ++# THIS SOFTWARE IS PROVIDED BY RED HAT AND CONTRIBUTORS ''AS IS'' AND ++# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, ++# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A ++# PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL RED HAT OR ++# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, ++# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT ++# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF ++# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ++# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, ++# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT ++# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF ++# SUCH DAMAGE. ++ ++# Test nbdkit-count-filter. ++ ++source ./functions.sh ++set -e ++set -x ++set -u ++ ++requires_run ++requires_filter count ++requires_plugin sparse-random ++requires_nbdcopy ++ ++log=test-count.out ++rm -f $log ++cleanup_fn rm -f $log ++ ++# We use sparse-random plugin because it both provides some data for ++# nbdcopy to copy, and allows writes. ++nbdkit -v --filter=count sparse-random 1G \ ++ --run 'nbdcopy "$uri" "$uri"' 2>$log ++ ++# Check that something got logged when the filter was unloaded. ++grep "count bytes:" $log +-- +2.47.1 + diff --git a/0008-count-Clarify-documentation.patch b/0008-count-Clarify-documentation.patch new file mode 100644 index 0000000..cac2436 --- /dev/null +++ b/0008-count-Clarify-documentation.patch @@ -0,0 +1,41 @@ +From 745b959b44a67cd3def7c60b873701ad79137bda Mon Sep 17 00:00:00 2001 +From: "Richard W.M. Jones" +Date: Tue, 8 Jul 2025 21:20:52 +0100 +Subject: [PATCH] count: Clarify documentation + +Updates: commit 3512c3ce9308b4d940119ac6cc87f1baa9afb655 +(cherry picked from commit 7c84314ec7411fc1090bf1ca417c453d08fcf364) +--- + filters/count/nbdkit-count-filter.pod | 8 +++++--- + 1 file changed, 5 insertions(+), 3 deletions(-) + +diff --git a/filters/count/nbdkit-count-filter.pod b/filters/count/nbdkit-count-filter.pod +index f0437000..c20b4737 100644 +--- a/filters/count/nbdkit-count-filter.pod ++++ b/filters/count/nbdkit-count-filter.pod +@@ -4,17 +4,19 @@ nbdkit-count-filter - count bytes read, written, zeroed and trimmed + + =head1 SYNOPSIS + +- nbdkit --filter=count plugin ++ nbdkit -fv --filter=count plugin + + =head1 DESCRIPTION + + C is a filter for L that simply counts + the number of bytes that are read, written, zeroed and trimmed, and + reports this number in debugging output when the filter is unloaded +-(usually when nbdkit exits). ++(usually when nbdkit exits). The filter output can only be seen when ++debugging is enabled (I<-v>) which usually implies using the ++foreground (I<-f>) option as well. + + This is a very simple and lightweight filter. For much more +-comprehensive stats about and logging of operations, use ++comprehensive stats and logging of operations, use + L or L instead. + + =head1 PARAMETERS +-- +2.47.1 + diff --git a/nbdkit.spec b/nbdkit.spec index 6e152ac..156c329 100644 --- a/nbdkit.spec +++ b/nbdkit.spec @@ -55,7 +55,7 @@ Name: nbdkit Version: 1.44.1 -Release: 1%{?dist} +Release: 2%{?dist} Summary: NBD server License: BSD-3-Clause @@ -80,7 +80,14 @@ Source3: copy-patches.sh # https://gitlab.com/nbdkit/nbdkit/-/commits/rhel-10.1/ # Patches. -#(none) +Patch0001: 0001-common-Add-ONCE-macro-to-run-code-only-once.patch +Patch0002: 0002-file-zero-Print-the-debug-message-on-the-fallback-pa.patch +Patch0003: 0003-file-trim-Don-t-try-BLKDISCARD-if-earlier-FALLOC_FL_.patch +Patch0004: 0004-common-include-test-once.c-Skip-test-on-macOS-which-.patch +Patch0005: 0005-common-include-test-once.c-Further-fixes-for-pthread.patch +Patch0006: 0006-Remove-deprecated-cacheextents-filter.patch +Patch0007: 0007-New-filter-nbdkit-count-filter-count-bytes-read-writ.patch +Patch0008: 0008-count-Clarify-documentation.patch # For automatic RPM Provides generation. # See: https://rpm-software-management.github.io/rpm/manual/dependency_generators.html @@ -599,10 +606,10 @@ nbdkit-blocksize-policy-filter Set block size constraints and policy. nbdkit-cache-filter Server-side cache. -nbdkit-cacheextents-filter Cache extents. - nbdkit-checkwrite-filter Check writes match contents of plugin. +nbdkit-count-filter Count bytes read, written, zeroed and trimmed. + nbdkit-cow-filter Copy-on-write overlay for read-only plugins. nbdkit-ddrescue-filter Filter for serving from ddrescue dump. @@ -1347,8 +1354,8 @@ fi %{_libdir}/%{name}/filters/nbdkit-blocksize-filter.so %{_libdir}/%{name}/filters/nbdkit-blocksize-policy-filter.so %{_libdir}/%{name}/filters/nbdkit-cache-filter.so -%{_libdir}/%{name}/filters/nbdkit-cacheextents-filter.so %{_libdir}/%{name}/filters/nbdkit-checkwrite-filter.so +%{_libdir}/%{name}/filters/nbdkit-count-filter.so %{_libdir}/%{name}/filters/nbdkit-cow-filter.so %{_libdir}/%{name}/filters/nbdkit-ddrescue-filter.so %{_libdir}/%{name}/filters/nbdkit-delay-filter.so @@ -1393,8 +1400,8 @@ fi %{_mandir}/man1/nbdkit-blocksize-filter.1* %{_mandir}/man1/nbdkit-blocksize-policy-filter.1* %{_mandir}/man1/nbdkit-cache-filter.1* -%{_mandir}/man1/nbdkit-cacheextents-filter.1* %{_mandir}/man1/nbdkit-checkwrite-filter.1* +%{_mandir}/man1/nbdkit-count-filter.1* %{_mandir}/man1/nbdkit-cow-filter.1* %{_mandir}/man1/nbdkit-ddrescue-filter.1* %{_mandir}/man1/nbdkit-delay-filter.1* @@ -1551,12 +1558,13 @@ fi %changelog -* Sat Jul 05 2025 Richard W.M. Jones - 1.44.1-1 +* Wed Jul 09 2025 Richard W.M. Jones - 1.44.1-2 - Rebase to nbdkit 1.44.1 resolves: RHEL-78830, RHEL-101180 - Synch the spec file with Fedora Rawhide. - nbdkit-ondemand-plugin moves into a new subpackage. -- New nbdkit-time-limit-filter. +- New nbdkit-count-filter & nbdkit-time-limit-filter. +- Remove nbdkit-cacheextents-filter. - Add extra system call checking and debugging to nbdkit-file-plugin resolves: RHEL-85515 - Allow nbdkit-file-plugin to zero and trim block devices