From 323edc9d0378286f006d8d46f6d9ca5d6b2c2fba Mon Sep 17 00:00:00 2001 From: "Richard W.M. Jones" Date: Fri, 30 Jul 2021 18:34:42 +0100 Subject: [PATCH] Fix nbdcopy progress bar. Add nbdinfo --map --totals and --can/--is options. resolves: rhbz#1950630 --- ...storage-daemon-5.2.0-is-still-broken.patch | 29 + ...ve-check-valgrind-out-from-condition.patch | 28 + 0003-m4-Remove-on-make-clean.patch | 26 + 0004-One-more-VSOCK-include-fix.patch | 29 + 0005-macOS-Do-not-use-version_script.patch | 51 ++ 0006-macOS-Simple-cloexec-nonblock-fix.patch | 215 ++++++ 0007-copy-Fix-progress-bar.patch | 34 + ...tial-divide-by-zero-when-source-size.patch | 30 + ...als-sub-mode-to-display-summary-of-m.patch | 413 +++++++++++ ...can-is-options-to-test-for-NBD-flags.patch | 679 ++++++++++++++++++ ...percentage-after-field-in-map-totals.patch | 66 ++ ...fo-Require-can_cache-for-info-can.sh.patch | 27 + copy-patches.sh | 2 +- libnbd.spec | 24 +- 14 files changed, 1651 insertions(+), 2 deletions(-) create mode 100644 0001-qemu-storage-daemon-5.2.0-is-still-broken.patch create mode 100644 0002-fuse-move-check-valgrind-out-from-condition.patch create mode 100644 0003-m4-Remove-on-make-clean.patch create mode 100644 0004-One-more-VSOCK-include-fix.patch create mode 100644 0005-macOS-Do-not-use-version_script.patch create mode 100644 0006-macOS-Simple-cloexec-nonblock-fix.patch create mode 100644 0007-copy-Fix-progress-bar.patch create mode 100644 0008-copy-Avoid-potential-divide-by-zero-when-source-size.patch create mode 100644 0009-info-Add-map-totals-sub-mode-to-display-summary-of-m.patch create mode 100644 0010-info-Add-can-is-options-to-test-for-NBD-flags.patch create mode 100644 0011-info-Add-percentage-after-field-in-map-totals.patch create mode 100644 0012-info-Require-can_cache-for-info-can.sh.patch diff --git a/0001-qemu-storage-daemon-5.2.0-is-still-broken.patch b/0001-qemu-storage-daemon-5.2.0-is-still-broken.patch new file mode 100644 index 0000000..ad75e21 --- /dev/null +++ b/0001-qemu-storage-daemon-5.2.0-is-still-broken.patch @@ -0,0 +1,29 @@ +From c6cbc7082f5d6832bec0770a7de99e81d2e2c944 Mon Sep 17 00:00:00 2001 +From: Martin Kletzander +Date: Tue, 22 Jun 2021 14:08:20 +0200 +Subject: [PATCH] qemu-storage-daemon 5.2.0 is still broken + +Signed-off-by: Martin Kletzander +(cherry picked from commit cc80a6b8248f5c82b67fd024a76db362c2783e25) +--- + interop/interop-qemu-storage-daemon.sh | 3 ++- + 1 file changed, 2 insertions(+), 1 deletion(-) + +diff --git a/interop/interop-qemu-storage-daemon.sh b/interop/interop-qemu-storage-daemon.sh +index 16aafab..d3ae032 100755 +--- a/interop/interop-qemu-storage-daemon.sh ++++ b/interop/interop-qemu-storage-daemon.sh +@@ -26,8 +26,9 @@ requires test "x$QEMU_STORAGE_DAEMON" != "x" + requires sed --version + qsd_version="$($QEMU_STORAGE_DAEMON --version | \ + sed -n '1s/qemu-storage-daemon version \([0-9.]*\).*/\1/p')" +-requires_not test "$qsd_version" = "6.0.0" + requires_not test "$qsd_version" = "5.1.0" ++requires_not test "$qsd_version" = "5.2.0" ++requires_not test "$qsd_version" = "6.0.0" + requires nbdsh --version + requires qemu-img --version + +-- +2.31.1 + diff --git a/0002-fuse-move-check-valgrind-out-from-condition.patch b/0002-fuse-move-check-valgrind-out-from-condition.patch new file mode 100644 index 0000000..79f1f4a --- /dev/null +++ b/0002-fuse-move-check-valgrind-out-from-condition.patch @@ -0,0 +1,28 @@ +From 2fb27319faf810342fd5c5e55a567e9a4df4a357 Mon Sep 17 00:00:00 2001 +From: Martin Kletzander +Date: Tue, 22 Jun 2021 16:18:13 +0200 +Subject: [PATCH] fuse: move check-valgrind out from condition + +Signed-off-by: Martin Kletzander +(cherry picked from commit 386685490d669f0940865e23f133013289e73b06) +--- + fuse/Makefile.am | 4 ++-- + 1 file changed, 2 insertions(+), 2 deletions(-) + +diff --git a/fuse/Makefile.am b/fuse/Makefile.am +index 8c66193..d869186 100644 +--- a/fuse/Makefile.am ++++ b/fuse/Makefile.am +@@ -95,7 +95,7 @@ TESTS += \ + test-errors.sh \ + $(NULL) + ++endif HAVE_FUSE ++ + check-valgrind: + LIBNBD_VALGRIND=1 $(MAKE) check +- +-endif HAVE_FUSE +-- +2.31.1 + diff --git a/0003-m4-Remove-on-make-clean.patch b/0003-m4-Remove-on-make-clean.patch new file mode 100644 index 0000000..a78593a --- /dev/null +++ b/0003-m4-Remove-on-make-clean.patch @@ -0,0 +1,26 @@ +From 556fb69b1329905a988effb812639eae419053f0 Mon Sep 17 00:00:00 2001 +From: "Richard W.M. Jones" +Date: Sat, 17 Jul 2021 11:24:57 +0100 +Subject: [PATCH] m4: Remove *~ on make clean + +(cherry picked from commit c55c5d9960809efd27cd044d007a33ea1636f4b0) +--- + Makefile.am | 2 ++ + 1 file changed, 2 insertions(+) + +diff --git a/Makefile.am b/Makefile.am +index 53681a5..0f0427a 100644 +--- a/Makefile.am ++++ b/Makefile.am +@@ -27,6 +27,8 @@ EXTRA_DIST = \ + SECURITY \ + $(NULL) + ++CLEANFILES += m4/*~ ++ + SUBDIRS = \ + generator \ + include \ +-- +2.31.1 + diff --git a/0004-One-more-VSOCK-include-fix.patch b/0004-One-more-VSOCK-include-fix.patch new file mode 100644 index 0000000..2ae2a13 --- /dev/null +++ b/0004-One-more-VSOCK-include-fix.patch @@ -0,0 +1,29 @@ +From 899283bd796f55e8f7c3bee64246768612c641f3 Mon Sep 17 00:00:00 2001 +From: Martin Kletzander +Date: Fri, 9 Jul 2021 12:53:31 +0200 +Subject: [PATCH] One more VSOCK include fix + +This file was forgotten in commit e8ed016c34e1. + +Signed-off-by: Martin Kletzander +(cherry picked from commit 0fb702d931be4e12bfca1a31378728053236f97c) +--- + lib/uri.c | 2 ++ + 1 file changed, 2 insertions(+) + +diff --git a/lib/uri.c b/lib/uri.c +index bcecbad..56f4737 100644 +--- a/lib/uri.c ++++ b/lib/uri.c +@@ -32,6 +32,8 @@ + + #ifdef HAVE_LINUX_VM_SOCKETS_H + #include ++#elif HAVE_SYS_VSOCK_H ++#include + #endif + + #include "internal.h" +-- +2.31.1 + diff --git a/0005-macOS-Do-not-use-version_script.patch b/0005-macOS-Do-not-use-version_script.patch new file mode 100644 index 0000000..21f39eb --- /dev/null +++ b/0005-macOS-Do-not-use-version_script.patch @@ -0,0 +1,51 @@ +From 02d25dfb0809facb8c4038fb3be64fa6d55efdf8 Mon Sep 17 00:00:00 2001 +From: Martin Kletzander +Date: Tue, 13 Jul 2021 13:13:42 +0200 +Subject: [PATCH] macOS: Do not use --version_script + +The linker does not support this option. + +Signed-off-by: Martin Kletzander +(cherry picked from commit 013ec1280911fd2e9ad578a92497e3c23989f037) +--- + configure.ac | 10 ++++++++++ + lib/Makefile.am | 2 +- + 2 files changed, 11 insertions(+), 1 deletion(-) + +diff --git a/configure.ac b/configure.ac +index 1641e16..dd971e6 100644 +--- a/configure.ac ++++ b/configure.ac +@@ -507,6 +507,16 @@ AS_IF([test "x$enable_golang" != "xno"],[ + ],[GOLANG=no]) + AM_CONDITIONAL([HAVE_GOLANG],[test "x$GOLANG" != "xno"]) + ++case $host_os in ++ darwin*) ++ VERSION_SCRIPT= ++ ;; ++ *) ++ VERSION_SCRIPT="-Wl,--version-script=${srcdir}/libnbd.syms" ++ ;; ++esac ++AC_SUBST([VERSION_SCRIPT]) ++ + dnl Produce output files. + AC_CONFIG_HEADERS([config.h]) + +diff --git a/lib/Makefile.am b/lib/Makefile.am +index 968e41a..ece5077 100644 +--- a/lib/Makefile.am ++++ b/lib/Makefile.am +@@ -78,7 +78,7 @@ libnbd_la_LIBADD = \ + $(NULL) + libnbd_la_LDFLAGS = \ + $(PTHREAD_LIBS) \ +- -Wl,--version-script=$(srcdir)/libnbd.syms \ ++ $(VERSION_SCRIPT) \ + -version-info 0:0:0 \ + $(NULL) + +-- +2.31.1 + diff --git a/0006-macOS-Simple-cloexec-nonblock-fix.patch b/0006-macOS-Simple-cloexec-nonblock-fix.patch new file mode 100644 index 0000000..ab84a5a --- /dev/null +++ b/0006-macOS-Simple-cloexec-nonblock-fix.patch @@ -0,0 +1,215 @@ +From d57e9e705b1257e5e241922dc29b333e090302c4 Mon Sep 17 00:00:00 2001 +From: Martin Kletzander +Date: Wed, 7 Jul 2021 13:15:40 +0200 +Subject: [PATCH] macOS: Simple cloexec/nonblock fix + +This is the most trivial way to fix the issue with macOS not having SOCK_CLOEXEC +and SOCK_NONBLOCK. This is the only way to make it work on such platform(s) +unless they are fixed. + +Signed-off-by: Martin Kletzander +(cherry picked from commit 61132c8e9804787e66974020ebb96e3995e0f6dc) +--- + fuzzing/libnbd-fuzz-wrapper.c | 4 + + fuzzing/libnbd-libfuzzer-test.c | 4 + + generator/states-connect-socket-activation.c | 2 +- + generator/states-connect.c | 11 +-- + lib/internal.h | 9 +++ + lib/utils.c | 80 ++++++++++++++++++++ + 6 files changed, 104 insertions(+), 6 deletions(-) + +diff --git a/fuzzing/libnbd-fuzz-wrapper.c b/fuzzing/libnbd-fuzz-wrapper.c +index 99a6d80..eb18809 100644 +--- a/fuzzing/libnbd-fuzz-wrapper.c ++++ b/fuzzing/libnbd-fuzz-wrapper.c +@@ -38,6 +38,10 @@ + + #include + ++#ifndef SOCK_CLOEXEC ++#define SOCK_CLOEXEC 0 /* This file doesn't use exec */ ++#endif ++ + static void client (int s); + static void server (int fd, int s); + +diff --git a/fuzzing/libnbd-libfuzzer-test.c b/fuzzing/libnbd-libfuzzer-test.c +index 5ee29b8..c8d6423 100644 +--- a/fuzzing/libnbd-libfuzzer-test.c ++++ b/fuzzing/libnbd-libfuzzer-test.c +@@ -38,6 +38,10 @@ + + #include + ++#ifndef SOCK_CLOEXEC ++#define SOCK_CLOEXEC 0 /* This file doesn't use exec */ ++#endif ++ + static void client (int sock); + static void server (const uint8_t *data, size_t size, int sock); + +diff --git a/generator/states-connect-socket-activation.c b/generator/states-connect-socket-activation.c +index e601c9b..8a2add3 100644 +--- a/generator/states-connect-socket-activation.c ++++ b/generator/states-connect-socket-activation.c +@@ -131,7 +131,7 @@ STATE_MACHINE { + return 0; + } + +- s = socket (AF_UNIX, SOCK_STREAM|SOCK_CLOEXEC, 0); ++ s = nbd_internal_socket (AF_UNIX, SOCK_STREAM, 0, false); + if (s == -1) { + SET_NEXT_STATE (%.DEAD); + set_error (errno, "socket"); +diff --git a/generator/states-connect.c b/generator/states-connect.c +index fcac86f..8de1218 100644 +--- a/generator/states-connect.c ++++ b/generator/states-connect.c +@@ -52,7 +52,7 @@ STATE_MACHINE { + + assert (!h->sock); + family = h->connaddr.ss_family; +- fd = socket (family, SOCK_STREAM|SOCK_NONBLOCK|SOCK_CLOEXEC, 0); ++ fd = nbd_internal_socket (family, SOCK_STREAM, 0, true); + if (fd == -1) { + SET_NEXT_STATE (%.DEAD); + set_error (errno, "socket"); +@@ -162,9 +162,10 @@ STATE_MACHINE { + return -1; + } + +- fd = socket (h->rp->ai_family, +- h->rp->ai_socktype|SOCK_NONBLOCK|SOCK_CLOEXEC, +- h->rp->ai_protocol); ++ fd = nbd_internal_socket (h->rp->ai_family, ++ h->rp->ai_socktype, ++ h->rp->ai_protocol, ++ true); + if (fd == -1) { + SET_NEXT_STATE (%NEXT_ADDRESS); + return 0; +@@ -227,7 +228,7 @@ STATE_MACHINE { + assert (!h->sock); + assert (h->argv.ptr); + assert (h->argv.ptr[0]); +- if (socketpair (AF_UNIX, SOCK_STREAM|SOCK_CLOEXEC, 0, sv) == -1) { ++ if (nbd_internal_socketpair (AF_UNIX, SOCK_STREAM, 0, sv) == -1) { + SET_NEXT_STATE (%.DEAD); + set_error (errno, "socketpair"); + return 0; +diff --git a/lib/internal.h b/lib/internal.h +index 01f9d8a..0e205ab 100644 +--- a/lib/internal.h ++++ b/lib/internal.h +@@ -467,4 +467,13 @@ extern char *nbd_internal_printable_buffer (const void *buf, size_t count); + extern char *nbd_internal_printable_string (const char *str); + extern char *nbd_internal_printable_string_list (char **list); + ++/* These are wrappers around socket(2) and socketpair(2). They ++ * always set SOCK_CLOEXEC. nbd_internal_socket can set SOCK_NONBLOCK ++ * according to the nonblock parameter. ++ */ ++extern int nbd_internal_socket (int domain, int type, int protocol, ++ bool nonblock); ++extern int nbd_internal_socketpair (int domain, int type, int protocol, ++ int *fds); ++ + #endif /* LIBNBD_INTERNAL_H */ +diff --git a/lib/utils.c b/lib/utils.c +index 260fd6a..3d3b7f4 100644 +--- a/lib/utils.c ++++ b/lib/utils.c +@@ -24,6 +24,7 @@ + #include + #include + #include ++#include + + #include "minmax.h" + +@@ -258,3 +259,82 @@ nbd_internal_printable_string_list (char **list) + return s; + + } ++ ++int nbd_internal_socket(int domain, ++ int type, ++ int protocol, ++ bool nonblock) ++{ ++ int fd; ++ ++ /* So far we do not know about any platform that has SOCK_CLOEXEC and ++ * lacks SOCK_NONBLOCK at the same time. ++ * ++ * The workaround for missing SOCK_CLOEXEC introduces a race which ++ * cannot be fixed until support for SOCK_CLOEXEC is added (or other ++ * fix is implemented). ++ */ ++#ifndef SOCK_CLOEXEC ++ int flags; ++#else ++ type |= SOCK_CLOEXEC; ++ if (nonblock) ++ type |= SOCK_NONBLOCK; ++#endif ++ ++ fd = socket (domain, type, protocol); ++ ++#ifndef SOCK_CLOEXEC ++ if (fd == -1) ++ return -1; ++ ++ if (fcntl (fd, F_SETFD, FD_CLOEXEC) == -1) { ++ close(fd); ++ return -1; ++ } ++ ++ if (nonblock) { ++ flags = fcntl (fd, F_GETFL, 0); ++ if (flags == -1 || ++ fcntl (fd, F_SETFL, flags|O_NONBLOCK) == -1) { ++ close(fd); ++ return -1; ++ } ++ } ++#endif ++ ++ return fd; ++} ++ ++int ++nbd_internal_socketpair (int domain, int type, int protocol, int *fds) ++{ ++ int ret; ++ ++ /* ++ * Same as with nbd_internal_socket() this workaround for missing ++ * SOCK_CLOEXEC introduces a race which cannot be fixed until support ++ * for SOCK_CLOEXEC is added (or other fix is implemented). ++ */ ++#ifndef SOCK_CLOEXEC ++ size_t i; ++#else ++ type |= SOCK_CLOEXEC; ++#endif ++ ++ ret = socketpair (domain, type, protocol, fds); ++ ++#ifndef SOCK_CLOEXEC ++ if (ret == 0) { ++ for (i = 0; i < 2; i++) { ++ if (fcntl (fds[i], F_SETFD, FD_CLOEXEC) == -1) { ++ close(fds[0]); ++ close(fds[1]); ++ return -1; ++ } ++ } ++ } ++#endif ++ ++ return ret; ++} +-- +2.31.1 + diff --git a/0007-copy-Fix-progress-bar.patch b/0007-copy-Fix-progress-bar.patch new file mode 100644 index 0000000..90e36e6 --- /dev/null +++ b/0007-copy-Fix-progress-bar.patch @@ -0,0 +1,34 @@ +From 10ca280bce2cbc5b6e4300c2a64bc86be4757f5a Mon Sep 17 00:00:00 2001 +From: "Richard W.M. Jones" +Date: Fri, 30 Jul 2021 10:47:29 +0100 +Subject: [PATCH] copy: Fix progress bar + +In multi-thread mode we were using the wrong size (the destination, +not the source) to calculate the percent copied. When copying from a +small disk to a large disk, even though we only copy the length of the +small disk, the percentage would stay at 0%. + +Fixes virt-v2v -o null (which uses a max-sized nbdkit null instance as +the destination). + +(cherry picked from commit 76bf210cea8a2bc6d9d463c1589b81dcaa049923) +--- + copy/multi-thread-copying.c | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/copy/multi-thread-copying.c b/copy/multi-thread-copying.c +index c649d2b..4603d8f 100644 +--- a/copy/multi-thread-copying.c ++++ b/copy/multi-thread-copying.c +@@ -70,7 +70,7 @@ get_next_offset (uint64_t *offset, uint64_t *count) + * are called from threads and not necessarily in monotonic order + * so the progress bar would move erratically. + */ +- progress_bar (*offset, dst->size); ++ progress_bar (*offset, src->size); + } + pthread_mutex_unlock (&lock); + return r; +-- +2.31.1 + diff --git a/0008-copy-Avoid-potential-divide-by-zero-when-source-size.patch b/0008-copy-Avoid-potential-divide-by-zero-when-source-size.patch new file mode 100644 index 0000000..1d545b7 --- /dev/null +++ b/0008-copy-Avoid-potential-divide-by-zero-when-source-size.patch @@ -0,0 +1,30 @@ +From 8d9c9a8141ccacbde2f1ccd350745de60945900c Mon Sep 17 00:00:00 2001 +From: "Richard W.M. Jones" +Date: Fri, 30 Jul 2021 10:57:39 +0100 +Subject: [PATCH] copy: Avoid potential divide by zero when source size is zero + +Actually this does not happen because of the way the progress bar is +called from other code, but it's possible that a future change might +introduce a bug, so avoid any problems. + +(cherry picked from commit 266c6e7083b1e0e8373ce166cf0be388604ca805) +--- + copy/progress.c | 2 ++ + 1 file changed, 2 insertions(+) + +diff --git a/copy/progress.c b/copy/progress.c +index 7383299..59ec817 100644 +--- a/copy/progress.c ++++ b/copy/progress.c +@@ -103,6 +103,8 @@ progress_bar (off_t pos, int64_t size) + + if (!progress) + return; ++ if (size == 0) ++ return; + + pthread_mutex_lock (&lock); + if (progress_fd == -1) +-- +2.31.1 + diff --git a/0009-info-Add-map-totals-sub-mode-to-display-summary-of-m.patch b/0009-info-Add-map-totals-sub-mode-to-display-summary-of-m.patch new file mode 100644 index 0000000..7514545 --- /dev/null +++ b/0009-info-Add-map-totals-sub-mode-to-display-summary-of-m.patch @@ -0,0 +1,413 @@ +From 17c24027cea812d4a3517c48bdf8f79bbb0045c2 Mon Sep 17 00:00:00 2001 +From: "Richard W.M. Jones" +Date: Sat, 26 Jun 2021 16:10:40 +0100 +Subject: [PATCH] info: Add --map --totals sub-mode to display summary of map + +This is similar to "qemu-img measure". Some examples: + +$ nbdkit -r file fedora-33.img --run 'nbdinfo --map --totals $uri' +1226113024 19.0 0 data +5216337920 81.0 3 hole,zero + +$ nbdkit -r file fedora-33.img --run 'nbdinfo --map --totals --json $uri | jq' +[ + { + "size": 1226113024, + "percent": 19.0318, + "type": 0, + "description": "data" + }, + { + "size": 5216337920, + "percent": 80.9682, + "type": 3, + "description": "hole,zero" + } +] + +$ nbdkit sparse-random 6G --run 'nbdinfo --map --totals $uri' + 941551616 14.6 0 data +5500899328 85.4 3 hole,zero + +(cherry picked from commit 7968bfe328e86574276283b159a594eeebeaf32a) +--- + info/Makefile.am | 2 + + info/info-map-totals-json.sh | 48 +++++++++++++++++++++ + info/info-map-totals.sh | 43 +++++++++++++++++++ + info/main.c | 15 ++++++- + info/map.c | 82 +++++++++++++++++++++++++++++++++++- + info/nbdinfo.h | 1 + + info/nbdinfo.pod | 38 ++++++++++++++++- + 7 files changed, 225 insertions(+), 4 deletions(-) + create mode 100755 info/info-map-totals-json.sh + create mode 100755 info/info-map-totals.sh + +diff --git a/info/Makefile.am b/info/Makefile.am +index 5c717c7..75c6a75 100644 +--- a/info/Makefile.am ++++ b/info/Makefile.am +@@ -38,6 +38,8 @@ info_sh_files = \ + info-map-base-allocation-zero.sh \ + info-map-qemu-dirty-bitmap.sh \ + info-map-qemu-allocation-depth.sh \ ++ info-map-totals.sh \ ++ info-map-totals-json.sh \ + info-atomic-output.sh \ + $(NULL) + +diff --git a/info/info-map-totals-json.sh b/info/info-map-totals-json.sh +new file mode 100755 +index 0000000..dc386ef +--- /dev/null ++++ b/info/info-map-totals-json.sh +@@ -0,0 +1,48 @@ ++#!/usr/bin/env bash ++# nbd client library in userspace ++# Copyright (C) 2020-2021 Red Hat Inc. ++# ++# This library is free software; you can redistribute it and/or ++# modify it under the terms of the GNU Lesser General Public ++# License as published by the Free Software Foundation; either ++# version 2 of the License, or (at your option) any later version. ++# ++# This library is distributed in the hope that it will be useful, ++# but WITHOUT ANY WARRANTY; without even the implied warranty of ++# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ++# Lesser General Public License for more details. ++# ++# You should have received a copy of the GNU Lesser General Public ++# License along with this library; if not, write to the Free Software ++# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA ++ ++. ../tests/functions.sh ++ ++set -e ++set -x ++ ++requires nbdkit --version ++requires nbdkit -U - null --run 'test "$uri" != ""' ++requires jq --version ++ ++out=info-map-totals-json.out ++cleanup_fn rm -f $out ++rm -f $out ++ ++# The sparse allocator used by nbdkit-data-plugin uses a 32K page ++# size, and extents are always aligned with this. ++nbdkit -U - data data='1 @131072 2' size=1M \ ++ --run '$VG nbdinfo --map --totals --json "$uri"' > $out ++ ++cat $out ++jq . < $out ++ ++test $( jq -r '.[0].size' < $out ) -eq 65536 ++test $( jq -r '.[0].percent' < $out ) = "6.25" ++test $( jq -r '.[0].type' < $out ) -eq 0 ++test $( jq -r '.[0].description' < $out ) = "data" ++ ++test $( jq -r '.[1].size' < $out ) -eq 983040 ++test $( jq -r '.[1].percent' < $out ) = "93.75" ++test $( jq -r '.[1].type' < $out ) -eq 3 ++test $( jq -r '.[1].description' < $out ) = "hole,zero" +diff --git a/info/info-map-totals.sh b/info/info-map-totals.sh +new file mode 100755 +index 0000000..12c1263 +--- /dev/null ++++ b/info/info-map-totals.sh +@@ -0,0 +1,43 @@ ++#!/usr/bin/env bash ++# nbd client library in userspace ++# Copyright (C) 2020-2021 Red Hat Inc. ++# ++# This library is free software; you can redistribute it and/or ++# modify it under the terms of the GNU Lesser General Public ++# License as published by the Free Software Foundation; either ++# version 2 of the License, or (at your option) any later version. ++# ++# This library is distributed in the hope that it will be useful, ++# but WITHOUT ANY WARRANTY; without even the implied warranty of ++# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ++# Lesser General Public License for more details. ++# ++# You should have received a copy of the GNU Lesser General Public ++# License along with this library; if not, write to the Free Software ++# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA ++ ++. ../tests/functions.sh ++ ++set -e ++set -x ++ ++requires nbdkit --version ++requires nbdkit -U - null --run 'test "$uri" != ""' ++requires tr --version ++ ++out=info-map-totals.out ++cleanup_fn rm -f $out ++rm -f $out ++ ++# The sparse allocator used by nbdkit-data-plugin uses a 32K page ++# size, and extents are always aligned with this. ++nbdkit -U - data data='1 @131072 2' size=1M \ ++ --run '$VG nbdinfo --map --totals "$uri"' > $out ++ ++cat $out ++ ++if [ "$(tr -s ' ' < $out)" != " 65536 6.2 0 data ++ 983040 93.8 3 hole,zero" ]; then ++ echo "$0: unexpected output from nbdinfo --map" ++ exit 1 ++fi +diff --git a/info/main.c b/info/main.c +index 8be52b7..fcda25c 100644 +--- a/info/main.c ++++ b/info/main.c +@@ -42,6 +42,7 @@ bool probe_content = false; /* --content / --no-content option */ + bool json_output = false; /* --json option */ + const char *map = NULL; /* --map option */ + bool size_only = false; /* --size option */ ++bool totals = false; /* --totals option */ + + static void __attribute__((noreturn)) + usage (FILE *fp, int exitcode) +@@ -52,7 +53,7 @@ usage (FILE *fp, int exitcode) + "\n" + " nbdinfo [--json] NBD-URI\n" + " nbdinfo --size [--json] NBD-URI\n" +-" nbdinfo --map [--json] NBD-URI\n" ++" nbdinfo --map [--totals] [--json] NBD-URI\n" + " nbdinfo -L|--list [--json] NBD-URI\n" + "\n" + "Other options:\n" +@@ -87,6 +88,7 @@ main (int argc, char *argv[]) + JSON_OPTION, + MAP_OPTION, + SIZE_OPTION, ++ TOTALS_OPTION, + }; + const char *short_options = "LV"; + const struct option long_options[] = { +@@ -99,6 +101,8 @@ main (int argc, char *argv[]) + { "map", optional_argument, NULL, MAP_OPTION }, + { "short-options", no_argument, NULL, SHORT_OPTIONS }, + { "size", no_argument, NULL, SIZE_OPTION }, ++ { "total", no_argument, NULL, TOTALS_OPTION }, ++ { "totals", no_argument, NULL, TOTALS_OPTION }, + { "version", no_argument, NULL, 'V' }, + { NULL } + }; +@@ -155,6 +159,10 @@ main (int argc, char *argv[]) + size_only = true; + break; + ++ case TOTALS_OPTION: ++ totals = true; ++ break; ++ + case 'L': + list_all = true; + break; +@@ -184,6 +192,11 @@ main (int argc, char *argv[]) + progname, "--content", "--no-content"); + exit (EXIT_FAILURE); + } ++ if (totals && !map) { ++ fprintf (stderr, "%s: you must use --totals only with --map option.\n", ++ progname); ++ exit (EXIT_FAILURE); ++ } + + /* Work out if we should probe content. */ + probe_content = !list_all; +diff --git a/info/map.c b/info/map.c +index 82c9507..628033c 100644 +--- a/info/map.c ++++ b/info/map.c +@@ -38,6 +38,7 @@ + DEFINE_VECTOR_TYPE (uint32_vector, uint32_t) + + static void print_extents (uint32_vector *entries); ++static void print_totals (uint32_vector *entries, int64_t size); + static int extent_callback (void *user_data, const char *metacontext, + uint64_t offset, + uint32_t *entries, size_t nr_entries, +@@ -89,7 +90,10 @@ do_map (void) + offset += entries.ptr[i]; + } + +- print_extents (&entries); ++ if (!totals) ++ print_extents (&entries); ++ else ++ print_totals (&entries, size); + free (entries.ptr); + } + +@@ -195,6 +199,82 @@ print_one_extent (uint64_t offset, uint64_t len, uint32_t type) + free (descr); + } + ++/* --map --totals suboption */ ++static void ++print_totals (uint32_vector *entries, int64_t size) ++{ ++ uint32_t type; ++ bool comma = false; ++ ++ /* This is necessary to avoid a divide by zero below, but if the ++ * size of the export is zero then we know we will not print any ++ * information below so return quickly. ++ */ ++ if (size == 0) { ++ if (json_output) fprintf (fp, "[]\n"); ++ return; ++ } ++ ++ if (json_output) fprintf (fp, "[\n"); ++ ++ /* In the outer loop assume we have already printed all entries with ++ * entry type < type. Count all instances of type and at the same ++ * time find the next type that exists > type. ++ */ ++ type = 0; ++ for (;;) { ++ uint64_t next_type = (uint64_t)UINT32_MAX + 1; ++ uint64_t c = 0; ++ size_t i; ++ ++ for (i = 0; i < entries->size; i += 2) { ++ uint32_t t = entries->ptr[i+1]; ++ ++ if (t == type) ++ c += entries->ptr[i]; ++ else if (type < t && t < next_type) ++ next_type = t; ++ } ++ ++ if (c > 0) { ++ char *descr = extent_description (map, type); ++ double percent = 100.0 * c / size; ++ ++ if (!json_output) { ++ fprintf (fp, "%10" PRIu64 " %5.1f %3" PRIu32, ++ c, percent, type); ++ if (descr) ++ fprintf (fp, " %s", descr); ++ fprintf (fp, "\n"); ++ } ++ else { ++ if (comma) ++ fprintf (fp, ",\n"); ++ ++ fprintf (fp, ++ "{ \"size\": %" PRIu64 ", " ++ "\"percent\": %g, " ++ "\"type\": %" PRIu32, ++ c, percent, type); ++ if (descr) { ++ fprintf (fp, ", \"description\": "); ++ print_json_string (descr); ++ } ++ fprintf (fp, " }"); ++ comma = true; ++ } ++ ++ free (descr); ++ } ++ ++ if (next_type == (uint64_t)UINT32_MAX + 1) ++ break; ++ type = next_type; ++ } ++ ++ if (json_output) fprintf (fp, "\n]\n"); ++} ++ + static char * + extent_description (const char *metacontext, uint32_t type) + { +diff --git a/info/nbdinfo.h b/info/nbdinfo.h +index ff13e37..6ff73af 100644 +--- a/info/nbdinfo.h ++++ b/info/nbdinfo.h +@@ -32,6 +32,7 @@ extern bool probe_content; + extern bool json_output; + extern const char *map; + extern bool size_only; ++extern bool totals; + + /* list.c */ + extern void collect_exports (void); +diff --git a/info/nbdinfo.pod b/info/nbdinfo.pod +index f1344d4..0c03bcd 100644 +--- a/info/nbdinfo.pod ++++ b/info/nbdinfo.pod +@@ -8,7 +8,7 @@ nbdinfo - display information and metadata about NBD servers and exports + + nbdinfo --size [--json] NBD-URI + +- nbdinfo --map [--json] NBD-URI ++ nbdinfo --map [--totals] [--json] NBD-URI + + nbdinfo -L|--list [--json] NBD-URI + +@@ -119,6 +119,35 @@ other maps too: + For more information on NBD maps, see I in the NBD + protocol. + ++=head2 Map totals ++ ++Using S> performs the same operation as I<--map> but ++displays a summary of the total size of each type of allocation, in ++bytes and as a percentage (of the virtual size of the export). This ++is useful for estimating how much real storage is used on the server, ++or might be required when copying a sparse image with L. ++ ++In the example below, half (50.0%) of the disk is allocated data and ++half is unallocated: ++ ++ $ nbdinfo --map --totals nbd://localhost/ ++ 1048576 50.0 0 data ++ 1048576 50.0 3 hole,zero ++ ++The fields are: total size in bytes, percentage of the virtual size, ++type, description (optional). ++ ++You can also get the same information in parseable form using I<--json>: ++ ++ $ nbdinfo --map --totals --json nbd://localhost/ ++ [{ "size": 1048576, "percent": 50, ++ "type": 0, "description": "data" }, ++ { "size": 1048576, "percent": 50, ++ "type": 3, "description": "hole,zero" }] ++ ++As with the I<--map> option, by default this shows the ++C<"base:allocation"> map, but you can show the summary for other maps. ++ + =head2 List all exports + + To list all the exports available on an NBD server use the I<--list> +@@ -179,7 +208,7 @@ The output is displayed in JSON format. + Display the map (usually whether parts of the disk are allocated or + sparse) of the given export. This displays the C<"base:allocation"> + map by default, you can choose a different map with the optional +-parameter. ++parameter. Using S> displays a summary. + + =item B<-L> + +@@ -188,6 +217,11 @@ parameter. + List all the exports on an NBD server. The export name in the NBD URI + is ignored. + ++=item B<--totals> ++ ++Use S> to display a summary. See L ++above. ++ + =item B<--size> + + Display only the size in bytes of the export. +-- +2.31.1 + diff --git a/0010-info-Add-can-is-options-to-test-for-NBD-flags.patch b/0010-info-Add-can-is-options-to-test-for-NBD-flags.patch new file mode 100644 index 0000000..cc23091 --- /dev/null +++ b/0010-info-Add-can-is-options-to-test-for-NBD-flags.patch @@ -0,0 +1,679 @@ +From 5a4dd174e893aec775c3f93c5c94c0a633437245 Mon Sep 17 00:00:00 2001 +From: "Richard W.M. Jones" +Date: Sat, 26 Jun 2021 20:15:01 +0100 +Subject: [PATCH] info: Add --can|--is options to test for NBD flags + +You can use them to test export properties from shell scripts, eg: + + if nbdinfo --is read-only nbd://localhost ; then + error "this NBD server does not support writing" + fi + +Note in this implementation --can and --is are really synonyms. + +(cherry picked from commit b39ab287a94f5743775dc13a33b814653d11bdd1) +--- + info/Makefile.am | 6 ++ + info/can.c | 94 ++++++++++++++++++++++++++++++++ + info/info-can-connect.sh | 29 ++++++++++ + info/info-can-read.sh | 29 ++++++++++ + info/info-can-zero.sh | 35 ++++++++++++ + info/info-can.sh | 93 +++++++++++++++++++++++++++++++ + info/info-is-read-only.sh | 36 ++++++++++++ + info/main.c | 21 ++++++- + info/nbdinfo.h | 5 ++ + info/nbdinfo.pod | 112 ++++++++++++++++++++++++++++++++++++++ + 10 files changed, 458 insertions(+), 2 deletions(-) + create mode 100644 info/can.c + create mode 100755 info/info-can-connect.sh + create mode 100755 info/info-can-read.sh + create mode 100755 info/info-can-zero.sh + create mode 100755 info/info-can.sh + create mode 100755 info/info-is-read-only.sh + +diff --git a/info/Makefile.am b/info/Makefile.am +index 75c6a75..a5708ec 100644 +--- a/info/Makefile.am ++++ b/info/Makefile.am +@@ -18,6 +18,11 @@ + include $(top_srcdir)/subdir-rules.mk + + info_sh_files = \ ++ info-can.sh \ ++ info-can-connect.sh \ ++ info-can-read.sh \ ++ info-can-zero.sh \ ++ info-is-read-only.sh \ + info-list.sh \ + info-list-json.sh \ + info-list-qemu.sh \ +@@ -63,6 +68,7 @@ TESTS = + + nbdinfo_SOURCES = \ + nbdinfo.h \ ++ can.c \ + list.c \ + main.c \ + map.c \ +diff --git a/info/can.c b/info/can.c +new file mode 100644 +index 0000000..ee8bbb7 +--- /dev/null ++++ b/info/can.c +@@ -0,0 +1,94 @@ ++/* NBD client library in userspace ++ * Copyright (C) 2020-2021 Red Hat Inc. ++ * ++ * This library is free software; you can redistribute it and/or ++ * modify it under the terms of the GNU Lesser General Public ++ * License as published by the Free Software Foundation; either ++ * version 2 of the License, or (at your option) any later version. ++ * ++ * This library is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ++ * Lesser General Public License for more details. ++ * ++ * You should have received a copy of the GNU Lesser General Public ++ * License along with this library; if not, write to the Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA ++ */ ++ ++#include ++ ++#include ++#include ++#include ++ ++#include ++ ++#include "nbdinfo.h" ++ ++int can_exit_code; ++ ++void ++do_can (void) ++{ ++ int feature; ++ ++ if (strcasecmp (can, "connect") == 0 || ++ strcasecmp (can, "read") == 0) ++ feature = 1; ++ ++ else if (strcasecmp (can, "readonly") == 0 || ++ strcasecmp (can, "read-only") == 0 || ++ strcasecmp (can, "read_only") == 0) ++ feature = nbd_is_read_only (nbd); ++ ++ else if (strcasecmp (can, "write") == 0) { ++ feature = nbd_is_read_only (nbd); ++ if (feature >= 0) feature = !feature; ++ } ++ ++ else if (strcasecmp (can, "rotational") == 0) ++ feature = nbd_is_rotational (nbd); ++ ++ else if (strcasecmp (can, "cache") == 0) ++ feature = nbd_can_cache (nbd); ++ ++ else if (strcasecmp (can, "df") == 0) ++ feature = nbd_can_df (nbd); ++ ++ else if (strcasecmp (can, "fastzero") == 0 || ++ strcasecmp (can, "fast-zero") == 0 || ++ strcasecmp (can, "fast_zero") == 0) ++ feature = nbd_can_fast_zero (nbd); ++ ++ else if (strcasecmp (can, "flush") == 0) ++ feature = nbd_can_flush (nbd); ++ ++ else if (strcasecmp (can, "fua") == 0) ++ feature = nbd_can_fua (nbd); ++ ++ else if (strcasecmp (can, "multiconn") == 0 || ++ strcasecmp (can, "multi-conn") == 0 || ++ strcasecmp (can, "multi_conn") == 0) ++ feature = nbd_can_multi_conn (nbd); ++ ++ else if (strcasecmp (can, "trim") == 0) ++ feature = nbd_can_trim (nbd); ++ ++ else if (strcasecmp (can, "zero") == 0) ++ feature = nbd_can_zero (nbd); ++ ++ else { ++ fprintf (stderr, "%s: unknown --can or --is option: %s\n", ++ progname, can); ++ exit (EXIT_FAILURE); ++ } ++ ++ if (feature == -1) { ++ fprintf (stderr, "%s: %s\n", progname, nbd_get_error ()); ++ exit (EXIT_FAILURE); ++ } ++ ++ /* Translate the feature bool into an exit code. This is used in main(). */ ++ can_exit_code = feature ? EXIT_SUCCESS : 2; ++} +diff --git a/info/info-can-connect.sh b/info/info-can-connect.sh +new file mode 100755 +index 0000000..eecc290 +--- /dev/null ++++ b/info/info-can-connect.sh +@@ -0,0 +1,29 @@ ++#!/usr/bin/env bash ++# nbd client library in userspace ++# Copyright (C) 2020-2021 Red Hat Inc. ++# ++# This library is free software; you can redistribute it and/or ++# modify it under the terms of the GNU Lesser General Public ++# License as published by the Free Software Foundation; either ++# version 2 of the License, or (at your option) any later version. ++# ++# This library is distributed in the hope that it will be useful, ++# but WITHOUT ANY WARRANTY; without even the implied warranty of ++# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ++# Lesser General Public License for more details. ++# ++# You should have received a copy of the GNU Lesser General Public ++# License along with this library; if not, write to the Free Software ++# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA ++ ++. ../tests/functions.sh ++ ++set -e ++set -x ++ ++# --can connect always returns true. ++ ++requires nbdkit null --version ++ ++nbdkit -v -U - null \ ++ --run '$VG nbdinfo --can connect "nbd+unix:///?socket=$unixsocket"' +diff --git a/info/info-can-read.sh b/info/info-can-read.sh +new file mode 100755 +index 0000000..b1edeab +--- /dev/null ++++ b/info/info-can-read.sh +@@ -0,0 +1,29 @@ ++#!/usr/bin/env bash ++# nbd client library in userspace ++# Copyright (C) 2020-2021 Red Hat Inc. ++# ++# This library is free software; you can redistribute it and/or ++# modify it under the terms of the GNU Lesser General Public ++# License as published by the Free Software Foundation; either ++# version 2 of the License, or (at your option) any later version. ++# ++# This library is distributed in the hope that it will be useful, ++# but WITHOUT ANY WARRANTY; without even the implied warranty of ++# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ++# Lesser General Public License for more details. ++# ++# You should have received a copy of the GNU Lesser General Public ++# License along with this library; if not, write to the Free Software ++# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA ++ ++. ../tests/functions.sh ++ ++set -e ++set -x ++ ++# --can read always returns true. ++ ++requires nbdkit null --version ++ ++nbdkit -v -U - null \ ++ --run '$VG nbdinfo --can read "nbd+unix:///?socket=$unixsocket"' +diff --git a/info/info-can-zero.sh b/info/info-can-zero.sh +new file mode 100755 +index 0000000..e388243 +--- /dev/null ++++ b/info/info-can-zero.sh +@@ -0,0 +1,35 @@ ++#!/usr/bin/env bash ++# nbd client library in userspace ++# Copyright (C) 2020-2021 Red Hat Inc. ++# ++# This library is free software; you can redistribute it and/or ++# modify it under the terms of the GNU Lesser General Public ++# License as published by the Free Software Foundation; either ++# version 2 of the License, or (at your option) any later version. ++# ++# This library is distributed in the hope that it will be useful, ++# but WITHOUT ANY WARRANTY; without even the implied warranty of ++# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ++# Lesser General Public License for more details. ++# ++# You should have received a copy of the GNU Lesser General Public ++# License along with this library; if not, write to the Free Software ++# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA ++ ++. ../tests/functions.sh ++ ++set -e ++set -x ++ ++# nbdkit emulates zeroing so we have to use the nozero filter to test ++# the negative case below. ++ ++requires nbdkit null --version ++requires nbdkit null --filter=nozero --version ++ ++nbdkit -v -U - null \ ++ --run '$VG nbdinfo --can zero "nbd+unix:///?socket=$unixsocket"' ++ ++nbdkit -v -U - null \ ++ --filter=nozero \ ++ --run '! $VG nbdinfo --can zero "nbd+unix:///?socket=$unixsocket"' +diff --git a/info/info-can.sh b/info/info-can.sh +new file mode 100755 +index 0000000..4154e38 +--- /dev/null ++++ b/info/info-can.sh +@@ -0,0 +1,93 @@ ++#!/usr/bin/env bash ++# nbd client library in userspace ++# Copyright (C) 2020-2021 Red Hat Inc. ++# ++# This library is free software; you can redistribute it and/or ++# modify it under the terms of the GNU Lesser General Public ++# License as published by the Free Software Foundation; either ++# version 2 of the License, or (at your option) any later version. ++# ++# This library is distributed in the hope that it will be useful, ++# but WITHOUT ANY WARRANTY; without even the implied warranty of ++# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ++# Lesser General Public License for more details. ++# ++# You should have received a copy of the GNU Lesser General Public ++# License along with this library; if not, write to the Free Software ++# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA ++ ++. ../tests/functions.sh ++ ++set -e ++set -x ++ ++requires nbdkit --version ++requires nbdkit sh --version ++ ++# --is read-only and --can write are tested in info-is-read-only.sh ++ ++# --can connect is tested in info-can-connect.sh ++ ++# --can read is tested in info-can-read.sh ++ ++# --can zero is tested in info-can-zero.sh ++ ++# --can df is hard to test. nbdkit newstyle probably always sets this ++# and oldstyle never, but that feels like depending a bit too much on ++# the implementation. ++ ++# --can cache and --can fua require special handling because in ++# nbdkit-sh-plugin we must print "native" or "none". Also the can_fua ++# flag is only sent if the export is writable (hence can_write below). ++ ++for flag in cache fua; do ++ export flag ++ nbdkit -v -U - sh - \ ++ --run '$VG nbdinfo --can $flag "nbd+unix:///?socket=$unixsocket"' <<'EOF' ++case "$1" in ++ get_size) echo 1024 ;; ++ pread) ;; ++ can_write) ;; ++ can_$flag) echo native ;; ++ *) exit 2 ;; ++esac ++EOF ++ ++ nbdkit -v -U - sh - \ ++ --run '! $VG nbdinfo --can $flag "nbd+unix:///?socket=$unixsocket"' <<'EOF' ++case "$1" in ++ get_size) echo 1024 ;; ++ pread) ;; ++ can_write) ;; ++ can_$flag) echo none ;; ++ *) exit 2 ;; ++esac ++EOF ++done ++ ++# These ones are normal booleans. ++ ++for flag in fast_zero flush multi_conn trim ; do ++ export flag ++ nbdkit -v -U - sh - \ ++ --run '$VG nbdinfo --can $flag "nbd+unix:///?socket=$unixsocket"' <<'EOF' ++case "$1" in ++ get_size) echo 1024 ;; ++ pread) ;; ++ can_write) ;; ++ can_$flag) exit 0 ;; ++ *) exit 2 ;; ++esac ++EOF ++ ++ nbdkit -v -U - sh - \ ++ --run '! $VG nbdinfo --can $flag "nbd+unix:///?socket=$unixsocket"' <<'EOF' ++case "$1" in ++ get_size) echo 1024 ;; ++ pread) ;; ++ can_write) ;; ++ can_$flag) exit 3 ;; ++ *) exit 2 ;; ++esac ++EOF ++done +diff --git a/info/info-is-read-only.sh b/info/info-is-read-only.sh +new file mode 100755 +index 0000000..db024b4 +--- /dev/null ++++ b/info/info-is-read-only.sh +@@ -0,0 +1,36 @@ ++#!/usr/bin/env bash ++# nbd client library in userspace ++# Copyright (C) 2020-2021 Red Hat Inc. ++# ++# This library is free software; you can redistribute it and/or ++# modify it under the terms of the GNU Lesser General Public ++# License as published by the Free Software Foundation; either ++# version 2 of the License, or (at your option) any later version. ++# ++# This library is distributed in the hope that it will be useful, ++# but WITHOUT ANY WARRANTY; without even the implied warranty of ++# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ++# Lesser General Public License for more details. ++# ++# You should have received a copy of the GNU Lesser General Public ++# License along with this library; if not, write to the Free Software ++# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA ++ ++. ../tests/functions.sh ++ ++set -e ++set -x ++ ++# Test --is read-only and --can write. ++ ++requires nbdkit --version ++requires nbdkit null --version ++ ++nbdkit -U - -r null \ ++ --run '$VG nbdinfo --is read-only "nbd+unix:///?socket=$unixsocket"' ++nbdkit -U - -r null \ ++ --run '! $VG nbdinfo --can write "nbd+unix:///?socket=$unixsocket"' ++nbdkit -U - null \ ++ --run '$VG nbdinfo --can write "nbd+unix:///?socket=$unixsocket"' ++nbdkit -U - null \ ++ --run '! $VG nbdinfo --is read-only "nbd+unix:///?socket=$unixsocket"' +diff --git a/info/main.c b/info/main.c +index fcda25c..82946ee 100644 +--- a/info/main.c ++++ b/info/main.c +@@ -40,6 +40,7 @@ FILE *fp; /* output file descriptor */ + bool list_all = false; /* --list option */ + bool probe_content = false; /* --content / --no-content option */ + bool json_output = false; /* --json option */ ++const char *can = NULL; /* --is/--can option */ + const char *map = NULL; /* --map option */ + bool size_only = false; /* --size option */ + bool totals = false; /* --totals option */ +@@ -53,6 +54,8 @@ usage (FILE *fp, int exitcode) + "\n" + " nbdinfo [--json] NBD-URI\n" + " nbdinfo --size [--json] NBD-URI\n" ++" nbdinfo --is read-only|rotational NBD-URI\n" ++" nbdinfo --can cache|connect|... NBD-URI\n" + " nbdinfo --map [--totals] [--json] NBD-URI\n" + " nbdinfo -L|--list [--json] NBD-URI\n" + "\n" +@@ -66,6 +69,8 @@ usage (FILE *fp, int exitcode) + " nbdinfo nbd://localhost\n" + " nbdinfo \"nbd+unix:///?socket=/tmp/unixsock\"\n" + " nbdinfo --size nbd://example.com\n" ++" nbdinfo --can connect nbd://example.com\n" ++" nbdinfo --is read-only nbd://example.com\n" + " nbdinfo --map nbd://example.com\n" + " nbdinfo --json nbd://example.com\n" + " nbdinfo --list nbd://example.com\n" +@@ -86,6 +91,7 @@ main (int argc, char *argv[]) + CONTENT_OPTION, + NO_CONTENT_OPTION, + JSON_OPTION, ++ CAN_OPTION, + MAP_OPTION, + SIZE_OPTION, + TOTALS_OPTION, +@@ -93,8 +99,10 @@ main (int argc, char *argv[]) + const char *short_options = "LV"; + const struct option long_options[] = { + { "help", no_argument, NULL, HELP_OPTION }, ++ { "can", required_argument, NULL, CAN_OPTION }, + { "content", no_argument, NULL, CONTENT_OPTION }, + { "no-content", no_argument, NULL, NO_CONTENT_OPTION }, ++ { "is", required_argument, NULL, CAN_OPTION }, + { "json", no_argument, NULL, JSON_OPTION }, + { "list", no_argument, NULL, 'L' }, + { "long-options", no_argument, NULL, LONG_OPTIONS }, +@@ -151,6 +159,10 @@ main (int argc, char *argv[]) + no_content_flag = true; + break; + ++ case CAN_OPTION: ++ can = optarg; ++ break; ++ + case MAP_OPTION: + map = optarg ? optarg : "base:allocation"; + break; +@@ -181,7 +193,7 @@ main (int argc, char *argv[]) + usage (stderr, EXIT_FAILURE); + + /* You cannot combine certain options. */ +- if (!!list_all + !!map + !!size_only > 1) { ++ if (!!list_all + !!can + !!map + !!size_only > 1) { + fprintf (stderr, + "%s: you cannot use --list, --map and --size together.\n", + progname); +@@ -204,6 +216,8 @@ main (int argc, char *argv[]) + probe_content = true; + if (no_content_flag) + probe_content = false; ++ if (can) ++ probe_content = false; + if (map) + probe_content = false; + +@@ -227,7 +241,7 @@ main (int argc, char *argv[]) + nbd_set_uri_allow_local_file (nbd, true); /* Allow ?tls-psk-file. */ + + /* Set optional modes in the handle. */ +- if (!map && !size_only) { ++ if (!can && !map && !size_only) { + nbd_set_opt_mode (nbd, true); + nbd_set_full_info (nbd, true); + } +@@ -246,6 +260,8 @@ main (int argc, char *argv[]) + + if (size_only) /* --size (!list_all) */ + do_size (); ++ else if (can) /* --is/--can (!list_all) */ ++ do_can (); + else if (map) /* --map (!list_all) */ + do_map (); + else { /* not --size or --map */ +@@ -304,5 +320,6 @@ main (int argc, char *argv[]) + + free (output); + ++ if (can) exit (can_exit_code); + exit (list_okay ? EXIT_SUCCESS : EXIT_FAILURE); + } +diff --git a/info/nbdinfo.h b/info/nbdinfo.h +index 6ff73af..566a2cb 100644 +--- a/info/nbdinfo.h ++++ b/info/nbdinfo.h +@@ -30,10 +30,15 @@ extern FILE *fp; + extern bool list_all; + extern bool probe_content; + extern bool json_output; ++extern const char *can; + extern const char *map; + extern bool size_only; + extern bool totals; + ++/* can.c */ ++extern int can_exit_code; ++extern void do_can (void); ++ + /* list.c */ + extern void collect_exports (void); + extern bool list_all_exports (const char *uri); +diff --git a/info/nbdinfo.pod b/info/nbdinfo.pod +index 0c03bcd..fb20f0e 100644 +--- a/info/nbdinfo.pod ++++ b/info/nbdinfo.pod +@@ -8,6 +8,10 @@ nbdinfo - display information and metadata about NBD servers and exports + + nbdinfo --size [--json] NBD-URI + ++ nbdinfo --is read-only|rotational NBD-URI ++ ++ nbdinfo --can cache|connect|... NBD-URI ++ + nbdinfo --map [--totals] [--json] NBD-URI + + nbdinfo -L|--list [--json] NBD-URI +@@ -87,6 +91,66 @@ scripting) use the I<--size> parameter: + $ nbdinfo --size nbd://localhost + 1048576 + ++=head2 Test for flags ++ ++Use one of the options below to test NBD flags. The command does not ++print anything. Instead it exits with success (S) if ++true, or failure (S) if false. (Other exit codes ++indicate an error querying the flag). You can use it in shell scripts ++like this: ++ ++ if nbdinfo --is read-only nbd://localhost || ++ ! nbdinfo --can trim nbd://localhost ++ then ++ error "the device must support writing and trimming" ++ fi ++ ++=over 4 ++ ++=item nbdinfo --is read-only URI ++ ++Test if the server export is read-only. ++ ++=item nbdinfo --can write URI ++ ++For convenience this is the opposite of I<--is read-only>. ++ ++=item nbdinfo --can read URI ++ ++All NBD servers must support read, so this always exits with success ++(unless there is a failure connecting to the URI). ++ ++=item nbdinfo --can connect URI ++ ++Test if we can connect to the NBD URI. ++ ++=item nbdinfo --is rotational URI ++ ++Test if the server export is backed by something which behaves like a ++rotating disk: accessing nearby blocks may be faster than random ++access and requests should be sorted to improve performance. Many ++servers do not or cannot report this accurately. ++ ++=item nbdinfo --can cache URI ++ ++=item nbdinfo --can df URI ++ ++=item nbdinfo --can fast-zero URI ++ ++=item nbdinfo --can flush URI ++ ++=item nbdinfo --can fua URI ++ ++=item nbdinfo --can multi-conn URI ++ ++=item nbdinfo --can trim URI ++ ++=item nbdinfo --can zero URI ++ ++Test other properties of the NBD server export. ++ ++=back ++ + =head2 Map + + To show a map of which areas of the disk are allocated and sparse, use +@@ -181,6 +245,40 @@ API can be used for more complex queries. + + Display brief command line help and exit. + ++=item B<--can cache> ++ ++=item B<--can connect> ++ ++=item B<--can df> ++ ++=item B<--can fast-zero> ++ ++=item B<--can flush> ++ ++=item B<--can fua> ++ ++=item B<--can multi-conn> ++ ++=item B<--can read> ++ ++=item B<--can trim> ++ ++=item B<--can write> ++ ++=item B<--can zero> ++ ++Test properties of the NBD server export. The command does not print ++anything. Instead it exits with success (S) if true, or ++failure (S) if false. (Other exit codes indicate an ++error querying the flag). ++ ++For further information see the L ++and the following libnbd functions: L, ++L, L, L, ++L, L, L, ++L, L. ++ + =item B<--content> + + =item B<--no-content> +@@ -197,6 +295,20 @@ When using I<--list>, the default is I<--no-content> (since + downloading from each export is expensive). To enable content probing + use I<--list --content>. + ++=item B<--is read-only> ++ ++=item B<--is rotational> ++ ++Test if the NBD server export is read-only and rotational. The ++command does not print anything. Instead it exits with success ++(S) if true, or failure (S) if false. ++(Other exit codes indicate an error querying the flag). ++ ++For further information see the L ++and the following libnbd functions: L, ++L. ++ + =item B<--json> + + The output is displayed in JSON format. +-- +2.31.1 + diff --git a/0011-info-Add-percentage-after-field-in-map-totals.patch b/0011-info-Add-percentage-after-field-in-map-totals.patch new file mode 100644 index 0000000..acc07a4 --- /dev/null +++ b/0011-info-Add-percentage-after-field-in-map-totals.patch @@ -0,0 +1,66 @@ +From f4e3a9ac3d80c0e73e3b92fdb1faa704595fe61e Mon Sep 17 00:00:00 2001 +From: "Richard W.M. Jones" +Date: Mon, 28 Jun 2021 18:54:21 +0100 +Subject: [PATCH] info: Add percentage after field in --map --totals + +Add % sign after the percentage field in plain output (not JSON). + +$ nbdkit -r file fedora-33.img --run 'nbdinfo --map --totals $uri' +1226113024 19.0% 0 data +5216337920 81.0% 3 hole,zero + +Thanks: Eric Blake +Updates: commit 7968bfe328e86574276283b159a594eeebeaf32a +(cherry picked from commit 8d39d388cc18b2e16a01ba2b64f51672b5b91389) +--- + info/info-map-totals.sh | 4 ++-- + info/map.c | 2 +- + info/nbdinfo.pod | 4 ++-- + 3 files changed, 5 insertions(+), 5 deletions(-) + +diff --git a/info/info-map-totals.sh b/info/info-map-totals.sh +index 12c1263..c637ff5 100755 +--- a/info/info-map-totals.sh ++++ b/info/info-map-totals.sh +@@ -36,8 +36,8 @@ nbdkit -U - data data='1 @131072 2' size=1M \ + + cat $out + +-if [ "$(tr -s ' ' < $out)" != " 65536 6.2 0 data +- 983040 93.8 3 hole,zero" ]; then ++if [ "$(tr -s ' ' < $out)" != " 65536 6.2% 0 data ++ 983040 93.8% 3 hole,zero" ]; then + echo "$0: unexpected output from nbdinfo --map" + exit 1 + fi +diff --git a/info/map.c b/info/map.c +index 628033c..de7b7ab 100644 +--- a/info/map.c ++++ b/info/map.c +@@ -241,7 +241,7 @@ print_totals (uint32_vector *entries, int64_t size) + double percent = 100.0 * c / size; + + if (!json_output) { +- fprintf (fp, "%10" PRIu64 " %5.1f %3" PRIu32, ++ fprintf (fp, "%10" PRIu64 " %5.1f%% %3" PRIu32, + c, percent, type); + if (descr) + fprintf (fp, " %s", descr); +diff --git a/info/nbdinfo.pod b/info/nbdinfo.pod +index fb20f0e..1699712 100644 +--- a/info/nbdinfo.pod ++++ b/info/nbdinfo.pod +@@ -195,8 +195,8 @@ In the example below, half (50.0%) of the disk is allocated data and + half is unallocated: + + $ nbdinfo --map --totals nbd://localhost/ +- 1048576 50.0 0 data +- 1048576 50.0 3 hole,zero ++ 1048576 50.0% 0 data ++ 1048576 50.0% 3 hole,zero + + The fields are: total size in bytes, percentage of the virtual size, + type, description (optional). +-- +2.31.1 + diff --git a/0012-info-Require-can_cache-for-info-can.sh.patch b/0012-info-Require-can_cache-for-info-can.sh.patch new file mode 100644 index 0000000..4c7037e --- /dev/null +++ b/0012-info-Require-can_cache-for-info-can.sh.patch @@ -0,0 +1,27 @@ +From 7f5693e0b7bfa64129f9218de863757075f2fff7 Mon Sep 17 00:00:00 2001 +From: Martin Kletzander +Date: Wed, 7 Jul 2021 12:06:38 +0200 +Subject: [PATCH] info: Require can_cache for info-can.sh + +Signed-off-by: Martin Kletzander +(cherry picked from commit 65cdf7354b8e4dd6db7dd31fa4321bea1d43650d) +--- + info/info-can.sh | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/info/info-can.sh b/info/info-can.sh +index 4154e38..00ba981 100755 +--- a/info/info-can.sh ++++ b/info/info-can.sh +@@ -22,7 +22,7 @@ set -e + set -x + + requires nbdkit --version +-requires nbdkit sh --version ++requires bash -c "nbdkit sh --dump-config | grep has_can_cache=1" + + # --is read-only and --can write are tested in info-is-read-only.sh + +-- +2.31.1 + diff --git a/copy-patches.sh b/copy-patches.sh index 00041ac..83867ef 100755 --- a/copy-patches.sh +++ b/copy-patches.sh @@ -6,7 +6,7 @@ set -e # directory. Use it like this: # ./copy-patches.sh -rhel_version=8.3 +rhel_version=9.0 # Check we're in the right directory. if [ ! -f libnbd.spec ]; then diff --git a/libnbd.spec b/libnbd.spec index 9ad7a1f..69b56ae 100644 --- a/libnbd.spec +++ b/libnbd.spec @@ -2,7 +2,7 @@ %global verify_tarball_signature 1 # If there are patches which touch autotools files, set this to 1. -%global patches_touch_autotools %{nil} +%global patches_touch_autotools 1 # The source directory. %global source_directory 1.8-stable @@ -25,6 +25,23 @@ Source2: libguestfs.keyring # Maintainer script which helps with handling patches. Source3: copy-patches.sh +# Patches are stored in the upstream repository: +# https://gitlab.com/nbdkit/libnbd/-/commits/rhel-9.0/ + +# Patches. +Patch0001: 0001-qemu-storage-daemon-5.2.0-is-still-broken.patch +Patch0002: 0002-fuse-move-check-valgrind-out-from-condition.patch +Patch0003: 0003-m4-Remove-on-make-clean.patch +Patch0004: 0004-One-more-VSOCK-include-fix.patch +Patch0005: 0005-macOS-Do-not-use-version_script.patch +Patch0006: 0006-macOS-Simple-cloexec-nonblock-fix.patch +Patch0007: 0007-copy-Fix-progress-bar.patch +Patch0008: 0008-copy-Avoid-potential-divide-by-zero-when-source-size.patch +Patch0009: 0009-info-Add-map-totals-sub-mode-to-display-summary-of-m.patch +Patch0010: 0010-info-Add-can-is-options-to-test-for-NBD-flags.patch +Patch0011: 0011-info-Add-percentage-after-field-in-map-totals.patch +Patch0012: 0012-info-Require-can_cache-for-info-can.sh.patch + %if 0%{patches_touch_autotools} BuildRequires: autoconf, automake, libtool %endif @@ -317,6 +334,11 @@ make %{?_smp_mflags} check || { %changelog +* Fri Jul 30 2021 Richard W.M. Jones - 1.8.2-2 +- Fix nbdcopy progress bar. +- Add nbdinfo --map --totals and --can/--is options. + resolves: rhbz#1950630 + * Sat Jul 03 2021 Richard W.M. Jones - 1.8.2-1 - New upstream stable version 1.8.2.