rpms/nbdkit
6
0
Fork 0
Code Issues Pull Requests Releases Activity
ca4b55ce0a
nbdkit/0007-New-filter-nbdkit-count-filter-count-bytes-read-writ.patch
Richard W.M. Jones ca4b55ce0a Further rebase along 1.44 branch. 
- New nbdkit-count-filter.
- Remove nbdkit-cacheextents-filter.
- file: trim: Don't try BLKDISCARD if earlier FALLOC_FL_PUNCH_HOLE worked

related: RHEL-101180
2025-07-09 16:32:21 +01:00

501 lines
15 KiB
Diff
Raw Blame History

From 9c28df70cbde94e58e448c8953965510e5d952c2 Mon Sep 17 00:00:00 2001
From: "Richard W.M. Jones" <rjones@redhat.com>
Date: Tue, 8 Jul 2025 17:49:00 +0100
Subject: [PATCH] New filter: nbdkit-count-filter: count bytes read, written
etc.
This produces a summary of the number of bytes read, written, etc
through the filter.
This is mainly of use to virt-v2v where it's commonly asked how much
data was transferred over the wire or written to disk, and we don't
currently have an easy way to answer that. By simply adding this
filter, the numbers will be known from the virt-v2v conversion log.
(cherry picked from commit 3512c3ce9308b4d940119ac6cc87f1baa9afb655)
---
configure.ac | 2 +
docs/nbdkit-protocol.pod | 9 +-
filters/count/Makefile.am | 70 ++++++++++++++
filters/count/count.c | 132 ++++++++++++++++++++++++++
filters/count/nbdkit-count-filter.pod | 55 +++++++++++
filters/log/nbdkit-log-filter.pod | 1 +
filters/stats/nbdkit-stats-filter.pod | 3 +
plugins/file/nbdkit-file-plugin.pod | 1 +
tests/Makefile.am | 4 +
tests/test-count.sh | 55 +++++++++++
10 files changed, 328 insertions(+), 4 deletions(-)
create mode 100644 filters/count/Makefile.am
create mode 100644 filters/count/count.c
create mode 100644 filters/count/nbdkit-count-filter.pod
create mode 100755 tests/test-count.sh
diff --git a/configure.ac b/configure.ac
index 9b057e6f..26e59462 100644
--- a/configure.ac
+++ b/configure.ac
@@ -144,6 +144,7 @@ filters="\
bzip2 \
cache \
checkwrite \
+ count \
cow \
ddrescue \
delay \
@@ -1787,6 +1788,7 @@ AC_CONFIG_FILES([Makefile
filters/bzip2/Makefile
filters/cache/Makefile
filters/checkwrite/Makefile
+ filters/count/Makefile
filters/cow/Makefile
filters/ddrescue/Makefile
filters/delay/Makefile
diff --git a/docs/nbdkit-protocol.pod b/docs/nbdkit-protocol.pod
index 93f5c5fa..edf0efb0 100644
--- a/docs/nbdkit-protocol.pod
+++ b/docs/nbdkit-protocol.pod
@@ -276,12 +276,13 @@ filters do not work properly in this case.
bzip2 No
cache No
checkwrite Yes
+ count Yes
cow Yes, since 1.44
delay Yes
error Yes
evil Yes
+
exitlast Yes
-
exitwhen Yes
exportname Yes
ext2 No
@@ -291,8 +292,8 @@ filters do not work properly in this case.
ip Yes
limit Yes
log Yes
+
luks No
-
lzip No
multi-conn Yes
nocache Yes
@@ -302,8 +303,8 @@ filters do not work properly in this case.
nozero Yes
offset Yes, but unlikely to be useful
openonce Yes
+
partition No
-
pause Yes
protect Yes, but unlikely to be useful
qcow2dec No
@@ -313,8 +314,8 @@ filters do not work properly in this case.
retry Yes
retry-request Yes
rotational Yes
+
scan Yes
-
spinning Yes
stats Yes
swab Yes
diff --git a/filters/count/Makefile.am b/filters/count/Makefile.am
new file mode 100644
index 00000000..20456e17
--- /dev/null
+++ b/filters/count/Makefile.am
@@ -0,0 +1,70 @@
+# nbdkit
+# Copyright Red Hat
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#
+# * Redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in the
+# documentation and/or other materials provided with the distribution.
+#
+# * Neither the name of Red Hat nor the names of its contributors may be
+# used to endorse or promote products derived from this software without
+# specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY RED HAT AND CONTRIBUTORS ''AS IS'' AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+# PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL RED HAT OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
+# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+# SUCH DAMAGE.
+
+include $(top_srcdir)/common-rules.mk
+
+EXTRA_DIST = nbdkit-count-filter.pod
+
+filter_LTLIBRARIES = nbdkit-count-filter.la
+
+nbdkit_count_filter_la_SOURCES = \
+ count.c \
+ $(top_srcdir)/include/nbdkit-filter.h \
+ $(NULL)
+
+nbdkit_count_filter_la_CPPFLAGS = \
+ -I$(top_srcdir)/include \
+ -I$(top_builddir)/include \
+ $(NULL)
+nbdkit_count_filter_la_CFLAGS = $(WARNINGS_CFLAGS)
+nbdkit_count_filter_la_LDFLAGS = \
+ -module -avoid-version -shared $(NO_UNDEFINED_ON_WINDOWS) \
+ $(NULL)
+if USE_LINKER_SCRIPT
+nbdkit_count_filter_la_LDFLAGS += \
+ -Wl,--version-script=$(top_srcdir)/filters/filters.syms
+endif
+nbdkit_count_filter_la_LIBADD = \
+ $(IMPORT_LIBRARY_ON_WINDOWS) \
+ $(NULL)
+
+if HAVE_POD
+
+man_MANS = nbdkit-count-filter.1
+CLEANFILES += $(man_MANS)
+
+nbdkit-count-filter.1: nbdkit-count-filter.pod \
+ $(top_builddir)/podwrapper.pl
+ $(PODWRAPPER) --section=1 --man $@ \
+ --html $(top_builddir)/html/$@.html \
+ $<
+
+endif HAVE_POD
diff --git a/filters/count/count.c b/filters/count/count.c
new file mode 100644
index 00000000..8af7f5a0
--- /dev/null
+++ b/filters/count/count.c
@@ -0,0 +1,132 @@
+/* nbdkit
+ * Copyright Red Hat
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * * Neither the name of Red Hat nor the names of its contributors may be
+ * used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY RED HAT AND CONTRIBUTORS ''AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+ * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL RED HAT OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
+ * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+ * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <config.h>
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <stdint.h>
+#include <inttypes.h>
+
+#include <nbdkit-filter.h>
+
+#ifdef HAVE_STDATOMIC_H
+#include <stdatomic.h>
+#else
+/* Only used for counting statistics. */
+#define _Atomic /**/
+#endif
+
+static _Atomic uint64_t bytes_read, bytes_written, bytes_zeroed, bytes_trimmed;
+
+static void
+count_unload (void)
+{
+ nbdkit_debug ("count bytes: "
+ "read %" PRIu64 ", "
+ "written %" PRIu64 ", "
+ "zeroed %" PRIu64 ", "
+ "trimmed %" PRIu64,
+ bytes_read, bytes_written, bytes_zeroed, bytes_trimmed);
+}
+
+/* Read data. */
+static int
+count_pread (nbdkit_next *next,
+ void *handle,
+ void *buf,
+ uint32_t count, uint64_t offset, uint32_t flags,
+ int *err)
+{
+ int r;
+
+ r = next->pread (next, buf, count, offset, flags, err);
+ if (r >= 0)
+ bytes_read += count;
+ return r;
+}
+
+/* Write data. */
+static int
+count_pwrite (nbdkit_next *next,
+ void *handle,
+ const void *buf,
+ uint32_t count, uint64_t offset, uint32_t flags,
+ int *err)
+{
+ int r;
+
+ r = next->pwrite (next, buf, count, offset, flags, err);
+ if (r >= 0)
+ bytes_written += count;
+ return r;
+}
+
+/* Trim data. */
+static int
+count_trim (nbdkit_next *next,
+ void *handle, uint32_t count, uint64_t offset, uint32_t flags,
+ int *err)
+{
+ int r;
+
+ r = next->trim (next, count, offset, flags, err);
+ if (r >= 0)
+ bytes_trimmed += count;
+ return r;
+}
+
+/* Zero data. */
+static int
+count_zero (nbdkit_next *next,
+ void *handle, uint32_t count, uint64_t offset, uint32_t flags,
+ int *err)
+{
+ int r;
+
+ r = next->zero (next, count, offset, flags, err);
+ if (r >= 0)
+ bytes_zeroed += count;
+ return r;
+}
+
+static struct nbdkit_filter filter = {
+ .name = "count",
+ .longname = "nbdkit count filter",
+ .unload = count_unload,
+ .pread = count_pread,
+ .pwrite = count_pwrite,
+ .trim = count_trim,
+ .zero = count_zero,
+};
+
+NBDKIT_REGISTER_FILTER (filter)
diff --git a/filters/count/nbdkit-count-filter.pod b/filters/count/nbdkit-count-filter.pod
new file mode 100644
index 00000000..f0437000
--- /dev/null
+++ b/filters/count/nbdkit-count-filter.pod
@@ -0,0 +1,55 @@
+=head1 NAME
+
+nbdkit-count-filter - count bytes read, written, zeroed and trimmed
+
+=head1 SYNOPSIS
+
+ nbdkit --filter=count plugin
+
+=head1 DESCRIPTION
+
+C<nbdkit-count-filter> is a filter for L<nbdkit(1)> that simply counts
+the number of bytes that are read, written, zeroed and trimmed, and
+reports this number in debugging output when the filter is unloaded
+(usually when nbdkit exits).
+
+This is a very simple and lightweight filter. For much more
+comprehensive stats about and logging of operations, use
+L<nbdkit-stats-filter(1)> or L<nbdkit-log-filter(1)> instead.
+
+=head1 PARAMETERS
+
+There are no parameters specific to this filter. All parameters are
+passed through to the underlying plugin.
+
+=head1 FILES
+
+=over 4
+
+=item F<$filterdir/nbdkit-count-filter.so>
+
+The filter.
+
+Use C<nbdkit --dump-config> to find the location of C<$filterdir>.
+
+=back
+
+=head1 VERSION
+
+C<nbdkit-count-filter> first appeared in nbdkit 1.46.
+
+=head1 SEE ALSO
+
+L<nbdkit(1)>,
+L<nbdkit-file-plugin(1)>,
+L<nbdkit-log-filter(1)>,
+L<nbdkit-stats-filter(1)>,
+L<nbdkit-filter(3)>.
+
+=head1 AUTHORS
+
+Richard W.M. Jones
+
+=head1 COPYRIGHT
+
+Copyright Red Hat
diff --git a/filters/log/nbdkit-log-filter.pod b/filters/log/nbdkit-log-filter.pod
index b91b60c4..256701a1 100644
--- a/filters/log/nbdkit-log-filter.pod
+++ b/filters/log/nbdkit-log-filter.pod
@@ -208,6 +208,7 @@ L<nbdkit(1)>,
L<nbdkit-file-plugin(1)>,
L<nbdkit-cow-filter(1)>,
L<nbdkit-filter(3)>,
+L<nbdkit-count-filter(1)>,
L<nbdkit-stats-filter(1)>.
=head1 AUTHORS
diff --git a/filters/stats/nbdkit-stats-filter.pod b/filters/stats/nbdkit-stats-filter.pod
index c0d2b45c..10074b4a 100644
--- a/filters/stats/nbdkit-stats-filter.pod
+++ b/filters/stats/nbdkit-stats-filter.pod
@@ -13,6 +13,8 @@ C<nbdkit-stats-filter> is a filter that displays statistics about NBD
operations, such as the number of bytes read and written. Statistics
are written to a file once when nbdkit exits.
+A lighter weight version of this is L<nbdkit-count-filter(1)>.
+
=head1 EXAMPLE OUTPUT
# nbdkit --filter=exitlast --filter=stats memory 25G statsfile=example.txt
@@ -113,6 +115,7 @@ C<nbdkit-stats-filter> first appeared in nbdkit 1.14.
L<nbdkit(1)>,
L<nbdkit-filter(3)>,
+L<nbdkit-count-filter(1)>,
L<nbdkit-log-filter(1)>.
=head1 AUTHORS
diff --git a/plugins/file/nbdkit-file-plugin.pod b/plugins/file/nbdkit-file-plugin.pod
index 63d07617..626827b2 100644
--- a/plugins/file/nbdkit-file-plugin.pod
+++ b/plugins/file/nbdkit-file-plugin.pod
@@ -316,6 +316,7 @@ L<nbdkit-split-plugin(1)>,
L<nbdkit-partitioning-plugin(1)>,
L<nbdkit-tmpdisk-plugin(1)>,
L<nbdkit-nfs-plugin(1)>,
+L<nbdkit-count-filter(1)>,
L<nbdkit-exportname-filter(1)>,
L<nbdkit-fua-filter(1)>,
L<nbdkit-luks-filter(1)>,
diff --git a/tests/Makefile.am b/tests/Makefile.am
index c16b5912..d7053ba2 100644
--- a/tests/Makefile.am
+++ b/tests/Makefile.am
@@ -1706,6 +1706,10 @@ EXTRA_DIST += \
test-checkwrite-fail.sh \
$(NULL)
+# count filter test.
+TESTS += test-count.sh
+EXTRA_DIST += test-count.sh
+
# cow filter test.
if HAVE_MKE2FS_WITH_D
TESTS += \
diff --git a/tests/test-count.sh b/tests/test-count.sh
new file mode 100755
index 00000000..e2e10704
--- /dev/null
+++ b/tests/test-count.sh
@@ -0,0 +1,55 @@
+#!/usr/bin/env bash
+# nbdkit
+# Copyright Red Hat
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#
+# * Redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in the
+# documentation and/or other materials provided with the distribution.
+#
+# * Neither the name of Red Hat nor the names of its contributors may be
+# used to endorse or promote products derived from this software without
+# specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY RED HAT AND CONTRIBUTORS ''AS IS'' AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+# PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL RED HAT OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
+# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+# SUCH DAMAGE.
+
+# Test nbdkit-count-filter.
+
+source ./functions.sh
+set -e
+set -x
+set -u
+
+requires_run
+requires_filter count
+requires_plugin sparse-random
+requires_nbdcopy
+
+log=test-count.out
+rm -f $log
+cleanup_fn rm -f $log
+
+# We use sparse-random plugin because it both provides some data for
+# nbdcopy to copy, and allows writes.
+nbdkit -v --filter=count sparse-random 1G \
+ --run 'nbdcopy "$uri" "$uri"' 2>$log
+
+# Check that something got logged when the filter was unloaded.
+grep "count bytes:" $log
--
2.47.1
Reference in New Issue View Git Blame Copy Permalink