From 9c28df70cbde94e58e448c8953965510e5d952c2 Mon Sep 17 00:00:00 2001 From: "Richard W.M. Jones" Date: Tue, 8 Jul 2025 17:49:00 +0100 Subject: [PATCH] New filter: nbdkit-count-filter: count bytes read, written etc. This produces a summary of the number of bytes read, written, etc through the filter. This is mainly of use to virt-v2v where it's commonly asked how much data was transferred over the wire or written to disk, and we don't currently have an easy way to answer that. By simply adding this filter, the numbers will be known from the virt-v2v conversion log. (cherry picked from commit 3512c3ce9308b4d940119ac6cc87f1baa9afb655) --- configure.ac | 2 + docs/nbdkit-protocol.pod | 9 +- filters/count/Makefile.am | 70 ++++++++++++++ filters/count/count.c | 132 ++++++++++++++++++++++++++ filters/count/nbdkit-count-filter.pod | 55 +++++++++++ filters/log/nbdkit-log-filter.pod | 1 + filters/stats/nbdkit-stats-filter.pod | 3 + plugins/file/nbdkit-file-plugin.pod | 1 + tests/Makefile.am | 4 + tests/test-count.sh | 55 +++++++++++ 10 files changed, 328 insertions(+), 4 deletions(-) create mode 100644 filters/count/Makefile.am create mode 100644 filters/count/count.c create mode 100644 filters/count/nbdkit-count-filter.pod create mode 100755 tests/test-count.sh diff --git a/configure.ac b/configure.ac index 9b057e6f..26e59462 100644 --- a/configure.ac +++ b/configure.ac @@ -144,6 +144,7 @@ filters="\ bzip2 \ cache \ checkwrite \ + count \ cow \ ddrescue \ delay \ @@ -1787,6 +1788,7 @@ AC_CONFIG_FILES([Makefile filters/bzip2/Makefile filters/cache/Makefile filters/checkwrite/Makefile + filters/count/Makefile filters/cow/Makefile filters/ddrescue/Makefile filters/delay/Makefile diff --git a/docs/nbdkit-protocol.pod b/docs/nbdkit-protocol.pod index 93f5c5fa..edf0efb0 100644 --- a/docs/nbdkit-protocol.pod +++ b/docs/nbdkit-protocol.pod @@ -276,12 +276,13 @@ filters do not work properly in this case. bzip2 No cache No checkwrite Yes + count Yes cow Yes, since 1.44 delay Yes error Yes evil Yes + exitlast Yes - exitwhen Yes exportname Yes ext2 No @@ -291,8 +292,8 @@ filters do not work properly in this case. ip Yes limit Yes log Yes + luks No - lzip No multi-conn Yes nocache Yes @@ -302,8 +303,8 @@ filters do not work properly in this case. nozero Yes offset Yes, but unlikely to be useful openonce Yes + partition No - pause Yes protect Yes, but unlikely to be useful qcow2dec No @@ -313,8 +314,8 @@ filters do not work properly in this case. retry Yes retry-request Yes rotational Yes + scan Yes - spinning Yes stats Yes swab Yes diff --git a/filters/count/Makefile.am b/filters/count/Makefile.am new file mode 100644 index 00000000..20456e17 --- /dev/null +++ b/filters/count/Makefile.am @@ -0,0 +1,70 @@ +# nbdkit +# Copyright Red Hat +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# * Neither the name of Red Hat nor the names of its contributors may be +# used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY RED HAT AND CONTRIBUTORS ''AS IS'' AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A +# PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL RED HAT OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF +# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT +# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +# SUCH DAMAGE. + +include $(top_srcdir)/common-rules.mk + +EXTRA_DIST = nbdkit-count-filter.pod + +filter_LTLIBRARIES = nbdkit-count-filter.la + +nbdkit_count_filter_la_SOURCES = \ + count.c \ + $(top_srcdir)/include/nbdkit-filter.h \ + $(NULL) + +nbdkit_count_filter_la_CPPFLAGS = \ + -I$(top_srcdir)/include \ + -I$(top_builddir)/include \ + $(NULL) +nbdkit_count_filter_la_CFLAGS = $(WARNINGS_CFLAGS) +nbdkit_count_filter_la_LDFLAGS = \ + -module -avoid-version -shared $(NO_UNDEFINED_ON_WINDOWS) \ + $(NULL) +if USE_LINKER_SCRIPT +nbdkit_count_filter_la_LDFLAGS += \ + -Wl,--version-script=$(top_srcdir)/filters/filters.syms +endif +nbdkit_count_filter_la_LIBADD = \ + $(IMPORT_LIBRARY_ON_WINDOWS) \ + $(NULL) + +if HAVE_POD + +man_MANS = nbdkit-count-filter.1 +CLEANFILES += $(man_MANS) + +nbdkit-count-filter.1: nbdkit-count-filter.pod \ + $(top_builddir)/podwrapper.pl + $(PODWRAPPER) --section=1 --man $@ \ + --html $(top_builddir)/html/$@.html \ + $< + +endif HAVE_POD diff --git a/filters/count/count.c b/filters/count/count.c new file mode 100644 index 00000000..8af7f5a0 --- /dev/null +++ b/filters/count/count.c @@ -0,0 +1,132 @@ +/* nbdkit + * Copyright Red Hat + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * * Neither the name of Red Hat nor the names of its contributors may be + * used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY RED HAT AND CONTRIBUTORS ''AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A + * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL RED HAT OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF + * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT + * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include + +#include +#include +#include +#include + +#include + +#ifdef HAVE_STDATOMIC_H +#include +#else +/* Only used for counting statistics. */ +#define _Atomic /**/ +#endif + +static _Atomic uint64_t bytes_read, bytes_written, bytes_zeroed, bytes_trimmed; + +static void +count_unload (void) +{ + nbdkit_debug ("count bytes: " + "read %" PRIu64 ", " + "written %" PRIu64 ", " + "zeroed %" PRIu64 ", " + "trimmed %" PRIu64, + bytes_read, bytes_written, bytes_zeroed, bytes_trimmed); +} + +/* Read data. */ +static int +count_pread (nbdkit_next *next, + void *handle, + void *buf, + uint32_t count, uint64_t offset, uint32_t flags, + int *err) +{ + int r; + + r = next->pread (next, buf, count, offset, flags, err); + if (r >= 0) + bytes_read += count; + return r; +} + +/* Write data. */ +static int +count_pwrite (nbdkit_next *next, + void *handle, + const void *buf, + uint32_t count, uint64_t offset, uint32_t flags, + int *err) +{ + int r; + + r = next->pwrite (next, buf, count, offset, flags, err); + if (r >= 0) + bytes_written += count; + return r; +} + +/* Trim data. */ +static int +count_trim (nbdkit_next *next, + void *handle, uint32_t count, uint64_t offset, uint32_t flags, + int *err) +{ + int r; + + r = next->trim (next, count, offset, flags, err); + if (r >= 0) + bytes_trimmed += count; + return r; +} + +/* Zero data. */ +static int +count_zero (nbdkit_next *next, + void *handle, uint32_t count, uint64_t offset, uint32_t flags, + int *err) +{ + int r; + + r = next->zero (next, count, offset, flags, err); + if (r >= 0) + bytes_zeroed += count; + return r; +} + +static struct nbdkit_filter filter = { + .name = "count", + .longname = "nbdkit count filter", + .unload = count_unload, + .pread = count_pread, + .pwrite = count_pwrite, + .trim = count_trim, + .zero = count_zero, +}; + +NBDKIT_REGISTER_FILTER (filter) diff --git a/filters/count/nbdkit-count-filter.pod b/filters/count/nbdkit-count-filter.pod new file mode 100644 index 00000000..f0437000 --- /dev/null +++ b/filters/count/nbdkit-count-filter.pod @@ -0,0 +1,55 @@ +=head1 NAME + +nbdkit-count-filter - count bytes read, written, zeroed and trimmed + +=head1 SYNOPSIS + + nbdkit --filter=count plugin + +=head1 DESCRIPTION + +C is a filter for L that simply counts +the number of bytes that are read, written, zeroed and trimmed, and +reports this number in debugging output when the filter is unloaded +(usually when nbdkit exits). + +This is a very simple and lightweight filter. For much more +comprehensive stats about and logging of operations, use +L or L instead. + +=head1 PARAMETERS + +There are no parameters specific to this filter. All parameters are +passed through to the underlying plugin. + +=head1 FILES + +=over 4 + +=item F<$filterdir/nbdkit-count-filter.so> + +The filter. + +Use C to find the location of C<$filterdir>. + +=back + +=head1 VERSION + +C first appeared in nbdkit 1.46. + +=head1 SEE ALSO + +L, +L, +L, +L, +L. + +=head1 AUTHORS + +Richard W.M. Jones + +=head1 COPYRIGHT + +Copyright Red Hat diff --git a/filters/log/nbdkit-log-filter.pod b/filters/log/nbdkit-log-filter.pod index b91b60c4..256701a1 100644 --- a/filters/log/nbdkit-log-filter.pod +++ b/filters/log/nbdkit-log-filter.pod @@ -208,6 +208,7 @@ L, L, L, L, +L, L. =head1 AUTHORS diff --git a/filters/stats/nbdkit-stats-filter.pod b/filters/stats/nbdkit-stats-filter.pod index c0d2b45c..10074b4a 100644 --- a/filters/stats/nbdkit-stats-filter.pod +++ b/filters/stats/nbdkit-stats-filter.pod @@ -13,6 +13,8 @@ C is a filter that displays statistics about NBD operations, such as the number of bytes read and written. Statistics are written to a file once when nbdkit exits. +A lighter weight version of this is L. + =head1 EXAMPLE OUTPUT # nbdkit --filter=exitlast --filter=stats memory 25G statsfile=example.txt @@ -113,6 +115,7 @@ C first appeared in nbdkit 1.14. L, L, +L, L. =head1 AUTHORS diff --git a/plugins/file/nbdkit-file-plugin.pod b/plugins/file/nbdkit-file-plugin.pod index 63d07617..626827b2 100644 --- a/plugins/file/nbdkit-file-plugin.pod +++ b/plugins/file/nbdkit-file-plugin.pod @@ -316,6 +316,7 @@ L, L, L, L, +L, L, L, L, diff --git a/tests/Makefile.am b/tests/Makefile.am index c16b5912..d7053ba2 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -1706,6 +1706,10 @@ EXTRA_DIST += \ test-checkwrite-fail.sh \ $(NULL) +# count filter test. +TESTS += test-count.sh +EXTRA_DIST += test-count.sh + # cow filter test. if HAVE_MKE2FS_WITH_D TESTS += \ diff --git a/tests/test-count.sh b/tests/test-count.sh new file mode 100755 index 00000000..e2e10704 --- /dev/null +++ b/tests/test-count.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env bash +# nbdkit +# Copyright Red Hat +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# * Neither the name of Red Hat nor the names of its contributors may be +# used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY RED HAT AND CONTRIBUTORS ''AS IS'' AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A +# PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL RED HAT OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF +# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT +# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +# SUCH DAMAGE. + +# Test nbdkit-count-filter. + +source ./functions.sh +set -e +set -x +set -u + +requires_run +requires_filter count +requires_plugin sparse-random +requires_nbdcopy + +log=test-count.out +rm -f $log +cleanup_fn rm -f $log + +# We use sparse-random plugin because it both provides some data for +# nbdcopy to copy, and allows writes. +nbdkit -v --filter=count sparse-random 1G \ + --run 'nbdcopy "$uri" "$uri"' 2>$log + +# Check that something got logged when the filter was unloaded. +grep "count bytes:" $log -- 2.47.1