From 95f059319c59aae40e25681f239fe8e00a41461d Mon Sep 17 00:00:00 2001 From: CentOS Sources Date: Tue, 15 Nov 2022 01:24:58 -0500 Subject: [PATCH] import libnbd-1.12.6-1.el9 --- .gitignore | 2 +- .libnbd.metadata | 2 +- SOURCES/0001-Add-nbddump-tool.patch | 1223 +++++++++++++++++ ...Add-new-API-nbd_set_pread_initialize.patch | 469 ------- ...sually-separate-columns-0-7-and-8-15.patch | 153 +++ SOURCES/0003-dump-Fix-build-on-i686.patch | 38 + .../0004-dump-Fix-tests-on-Debian-10.patch | 41 + ...mp-data.sh-Test-requires-nbdkit-1.22.patch | 31 + ...referred-block-size-in-the-operation.patch | 163 +++ ...Use-preferred-block-size-for-copying.patch | 457 ++++++ ...mp-Add-another-example-to-the-manual.patch | 29 + ...to-Use-GNUTLS_NO_SIGNAL-if-available.patch | 93 ++ ...ore-TLS-premature-termination-after-.patch | 100 ++ SOURCES/copy-patches.sh | 2 +- SOURCES/libnbd-1.10.5.tar.gz.sig | 17 - SOURCES/libnbd-1.12.6.tar.gz.sig | 17 + SPECS/libnbd.spec | 33 +- 17 files changed, 2377 insertions(+), 493 deletions(-) create mode 100644 SOURCES/0001-Add-nbddump-tool.patch delete mode 100644 SOURCES/0001-api-Add-new-API-nbd_set_pread_initialize.patch create mode 100644 SOURCES/0002-dump-Visually-separate-columns-0-7-and-8-15.patch create mode 100644 SOURCES/0003-dump-Fix-build-on-i686.patch create mode 100644 SOURCES/0004-dump-Fix-tests-on-Debian-10.patch create mode 100644 SOURCES/0005-dump-dump-data.sh-Test-requires-nbdkit-1.22.patch create mode 100644 SOURCES/0006-copy-Store-the-preferred-block-size-in-the-operation.patch create mode 100644 SOURCES/0007-copy-Use-preferred-block-size-for-copying.patch create mode 100644 SOURCES/0008-dump-Add-another-example-to-the-manual.patch create mode 100644 SOURCES/0009-lib-crypto-Use-GNUTLS_NO_SIGNAL-if-available.patch create mode 100644 SOURCES/0010-lib-crypto.c-Ignore-TLS-premature-termination-after-.patch delete mode 100644 SOURCES/libnbd-1.10.5.tar.gz.sig create mode 100644 SOURCES/libnbd-1.12.6.tar.gz.sig diff --git a/.gitignore b/.gitignore index 2d07496..ea01122 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,2 @@ SOURCES/libguestfs.keyring -SOURCES/libnbd-1.10.5.tar.gz +SOURCES/libnbd-1.12.6.tar.gz diff --git a/.libnbd.metadata b/.libnbd.metadata index c70b7df..f20759f 100644 --- a/.libnbd.metadata +++ b/.libnbd.metadata @@ -1,2 +1,2 @@ cc1b37b9cfafa515aab3eefd345ecc59aac2ce7b SOURCES/libguestfs.keyring -ae15a534a451d34bfc13397b6ca7a7287cf2371a SOURCES/libnbd-1.10.5.tar.gz +2d4eb0846d51c25fa7d04295972cbb5a617984ef SOURCES/libnbd-1.12.6.tar.gz diff --git a/SOURCES/0001-Add-nbddump-tool.patch b/SOURCES/0001-Add-nbddump-tool.patch new file mode 100644 index 0000000..5600ba9 --- /dev/null +++ b/SOURCES/0001-Add-nbddump-tool.patch @@ -0,0 +1,1223 @@ +From 90fd39da16256407b9229cd17a830739b03629d6 Mon Sep 17 00:00:00 2001 +From: "Richard W.M. Jones" +Date: Thu, 30 Jun 2022 09:07:27 +0100 +Subject: [PATCH] Add nbddump tool + +You can already do this operation using: + + nbdcopy -- $uri - | hexdump -C + +but that is slow, especially for large, sparse disks. This tool uses +sparseness information to skip zero sections of the disk, and it has +nice colourized output. + +(cherry picked from commit c4107b9a40d6451630dcccf1bf6596c8e56420be) +--- + .gitignore | 3 + + Makefile.am | 1 + + README | 2 + + bash-completion/Makefile.am | 9 +- + bash-completion/nbdsh | 6 + + configure.ac | 1 + + copy/nbdcopy.pod | 3 +- + docs/libnbd.pod | 1 + + dump/Makefile.am | 79 ++++++ + dump/dump-data.sh | 57 +++++ + dump/dump-empty-qcow2.sh | 46 ++++ + dump/dump-pattern.sh | 56 +++++ + dump/dump.c | 464 ++++++++++++++++++++++++++++++++++++ + dump/nbddump.pod | 116 +++++++++ + dump/test-long-options.sh | 35 +++ + dump/test-short-options.sh | 35 +++ + dump/test-version.sh | 33 +++ + fuse/nbdfuse.pod | 1 + + info/nbdinfo.pod | 1 + + run.in | 1 + + sh/nbdsh.pod | 1 + + 21 files changed, 947 insertions(+), 4 deletions(-) + create mode 100644 dump/Makefile.am + create mode 100755 dump/dump-data.sh + create mode 100755 dump/dump-empty-qcow2.sh + create mode 100755 dump/dump-pattern.sh + create mode 100644 dump/dump.c + create mode 100644 dump/nbddump.pod + create mode 100755 dump/test-long-options.sh + create mode 100755 dump/test-short-options.sh + create mode 100755 dump/test-version.sh + +diff --git a/.gitignore b/.gitignore +index 498eabc..3771655 100644 +--- a/.gitignore ++++ b/.gitignore +@@ -29,6 +29,7 @@ Makefile.in + /aclocal.m4 + /autom4te.cache + /bash-completion/nbdcopy ++/bash-completion/nbddump + /bash-completion/nbdfuse + /bash-completion/nbdinfo + /common/include/test-array-size +@@ -57,6 +58,8 @@ Makefile.in + !/docs/nbd_close.3 + !/docs/nbd_create.pod + !/docs/nbd_get_err??.3 ++/dump/nbddump ++/dump/nbddump.1 + /examples/aio-connect-read + /examples/batched-read-write + /examples/connect-command +diff --git a/Makefile.am b/Makefile.am +index 303b95c..9e7a281 100644 +--- a/Makefile.am ++++ b/Makefile.am +@@ -46,6 +46,7 @@ SUBDIRS = \ + sh \ + info \ + copy \ ++ dump \ + fuse \ + ocaml \ + ocaml/examples \ +diff --git a/README b/README +index e50c19d..4f9298e 100644 +--- a/README ++++ b/README +@@ -23,6 +23,8 @@ The key features are: + + * Copying tool (nbdcopy) for high performance copying and streaming. + ++ * Hexdump tool (nbddump) to print NBD content. ++ + * Query tool (nbdinfo) to query NBD servers. + + * FUSE support (nbdfuse) to mount NBD in the local filesystem. +diff --git a/bash-completion/Makefile.am b/bash-completion/Makefile.am +index 41d7b13..cab8ffb 100644 +--- a/bash-completion/Makefile.am ++++ b/bash-completion/Makefile.am +@@ -24,17 +24,20 @@ EXTRA_DIST = \ + + if HAVE_BASH_COMPLETION + +-bashcomp_DATA = nbdfuse nbdsh ++bashcomp_DATA = nbddump nbdfuse nbdsh + + if HAVE_LIBXML2 + bashcomp_DATA += nbdcopy nbdinfo + endif HAVE_LIBXML2 + +- + nbdcopy: nbdsh + rm -f $@ + $(LN_S) $(srcdir)/nbdsh $@ + ++nbddump: nbdsh ++ rm -f $@ ++ $(LN_S) $(srcdir)/nbdsh $@ ++ + nbdfuse: nbdsh + rm -f $@ + $(LN_S) $(srcdir)/nbdsh $@ +@@ -43,6 +46,6 @@ nbdinfo: nbdsh + rm -f $@ + $(LN_S) $(srcdir)/nbdsh $@ + +-CLEANFILES += nbdcopy nbdfuse nbdinfo ++CLEANFILES += nbdcopy nbddump nbdfuse nbdinfo + + endif +diff --git a/bash-completion/nbdsh b/bash-completion/nbdsh +index a740be9..a342003 100644 +--- a/bash-completion/nbdsh ++++ b/bash-completion/nbdsh +@@ -47,6 +47,11 @@ _nbdcopy () + _libnbd_command nbdcopy + } + ++_nbddump () ++{ ++ _libnbd_command nbddump ++} ++ + _nbdfuse () + { + _libnbd_command nbdfuse +@@ -64,6 +69,7 @@ _nbdsh () + + # Install the handler function. + complete -o default -F _nbdcopy nbdcopy ++complete -o default -F _nbddump nbddump + complete -o default -F _nbdfuse nbdfuse + complete -o default -F _nbdinfo nbdinfo + complete -o default -F _nbdsh nbdsh +diff --git a/configure.ac b/configure.ac +index b1bfaac..49ca8ab 100644 +--- a/configure.ac ++++ b/configure.ac +@@ -574,6 +574,7 @@ AC_CONFIG_FILES([Makefile + common/utils/Makefile + copy/Makefile + docs/Makefile ++ dump/Makefile + examples/Makefile + fuse/Makefile + fuzzing/Makefile +diff --git a/copy/nbdcopy.pod b/copy/nbdcopy.pod +index 7fe3fd1..fd10f7c 100644 +--- a/copy/nbdcopy.pod ++++ b/copy/nbdcopy.pod +@@ -285,7 +285,7 @@ Some examples follow. + In this example, L is run as a subprocess. The + subprocess opens F and exposes it as NBD to nbdcopy. + nbdcopy streams this to stdout (C<->) into the pipe which is read by +-L. ++L. (See also L) + + =head2 nbdcopy -- [ qemu-nbd -f qcow2 disk.qcow2 ] [ nbdkit memory 1G ] + +@@ -299,6 +299,7 @@ so this command has no overall effect, but is useful for testing. + =head1 SEE ALSO + + L, ++L, + L, + L, + L, +diff --git a/docs/libnbd.pod b/docs/libnbd.pod +index 13facc6..076cafb 100644 +--- a/docs/libnbd.pod ++++ b/docs/libnbd.pod +@@ -1044,6 +1044,7 @@ L. + + L, + L, ++L, + L, + L, + L, +diff --git a/dump/Makefile.am b/dump/Makefile.am +new file mode 100644 +index 0000000..9fd4fed +--- /dev/null ++++ b/dump/Makefile.am +@@ -0,0 +1,79 @@ ++# nbd client library in userspace ++# Copyright (C) 2020-2022 Red Hat Inc. ++# ++# This library is free software; you can redistribute it and/or ++# modify it under the terms of the GNU Lesser General Public ++# License as published by the Free Software Foundation; either ++# version 2 of the License, or (at your option) any later version. ++# ++# This library is distributed in the hope that it will be useful, ++# but WITHOUT ANY WARRANTY; without even the implied warranty of ++# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ++# Lesser General Public License for more details. ++# ++# You should have received a copy of the GNU Lesser General Public ++# License along with this library; if not, write to the Free Software ++# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA ++ ++include $(top_srcdir)/subdir-rules.mk ++ ++EXTRA_DIST = \ ++ dump-data.sh \ ++ dump-empty-qcow2.sh \ ++ dump-pattern.sh \ ++ nbddump.pod \ ++ test-long-options.sh \ ++ test-short-options.sh \ ++ test-version.sh \ ++ $(NULL) ++ ++bin_PROGRAMS = nbddump ++ ++nbddump_SOURCES = \ ++ dump.c \ ++ $(NULL) ++nbddump_CPPFLAGS = \ ++ -I$(top_srcdir)/include \ ++ -I$(top_srcdir)/common/include \ ++ -I$(top_srcdir)/common/utils \ ++ $(NULL) ++nbddump_CFLAGS = \ ++ $(WARNINGS_CFLAGS) \ ++ $(NULL) ++nbddump_LDADD = \ ++ $(top_builddir)/common/utils/libutils.la \ ++ $(top_builddir)/lib/libnbd.la \ ++ $(NULL) ++ ++if HAVE_POD ++ ++man_MANS = \ ++ nbddump.1 \ ++ $(NULL) ++ ++nbddump.1: nbddump.pod $(top_builddir)/podwrapper.pl ++ $(PODWRAPPER) --section=1 --man $@ \ ++ --html $(top_builddir)/html/$@.html \ ++ $< ++ ++endif HAVE_POD ++ ++TESTS_ENVIRONMENT = \ ++ LIBNBD_DEBUG=1 \ ++ $(MALLOC_CHECKS) \ ++ EXPECTED_VERSION=$(VERSION) \ ++ QEMU_NBD=$(QEMU_NBD) \ ++ $(NULL) ++LOG_COMPILER = $(top_builddir)/run ++ ++TESTS = \ ++ dump-data.sh \ ++ dump-empty-qcow2.sh \ ++ dump-pattern.sh \ ++ test-long-options.sh \ ++ test-short-options.sh \ ++ test-version.sh \ ++ $(NULL) ++ ++check-valgrind: ++ LIBNBD_VALGRIND=1 $(MAKE) check +diff --git a/dump/dump-data.sh b/dump/dump-data.sh +new file mode 100755 +index 0000000..23d09da +--- /dev/null ++++ b/dump/dump-data.sh +@@ -0,0 +1,57 @@ ++#!/usr/bin/env bash ++# nbd client library in userspace ++# Copyright (C) 2020-2022 Red Hat Inc. ++# ++# This library is free software; you can redistribute it and/or ++# modify it under the terms of the GNU Lesser General Public ++# License as published by the Free Software Foundation; either ++# version 2 of the License, or (at your option) any later version. ++# ++# This library is distributed in the hope that it will be useful, ++# but WITHOUT ANY WARRANTY; without even the implied warranty of ++# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ++# Lesser General Public License for more details. ++# ++# You should have received a copy of the GNU Lesser General Public ++# License along with this library; if not, write to the Free Software ++# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA ++ ++. ../tests/functions.sh ++ ++set -e ++set -x ++ ++requires nbdkit --version ++requires nbdkit data --dump-plugin ++ ++output=dump-data.out ++rm -f $output ++cleanup_fn rm -f $output ++ ++nbdkit -U - data data=' ++ @32768 1 ++ @65535 "hello, world!" ++ @17825790 "spanning buffer boundary" ++ @20000000 0 ++' --run 'nbddump "$uri"' > $output ++ ++cat $output ++ ++if [ "$(cat $output)" != '0000000000: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| ++* ++0000008000: 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| ++0000008010: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| ++* ++000000fff0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 68 |...............h| ++0000010000: 65 6c 6c 6f 2c 20 77 6f 72 6c 64 21 00 00 00 00 |ello, world!....| ++0000010010: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| ++* ++00010ffff0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 73 70 |..............sp| ++0001100000: 61 6e 6e 69 6e 67 20 62 75 66 66 65 72 20 62 6f |anning buffer bo| ++0001100010: 75 6e 64 61 72 79 00 00 00 00 00 00 00 00 00 00 |undary..........| ++0001100020: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| ++* ++0001312d00: 00 |. |' ]; then ++ echo "$0: unexpected output from nbddump command" ++ exit 1 ++fi +diff --git a/dump/dump-empty-qcow2.sh b/dump/dump-empty-qcow2.sh +new file mode 100755 +index 0000000..c9e583b +--- /dev/null ++++ b/dump/dump-empty-qcow2.sh +@@ -0,0 +1,46 @@ ++#!/usr/bin/env bash ++# nbd client library in userspace ++# Copyright (C) 2020-2022 Red Hat Inc. ++# ++# This library is free software; you can redistribute it and/or ++# modify it under the terms of the GNU Lesser General Public ++# License as published by the Free Software Foundation; either ++# version 2 of the License, or (at your option) any later version. ++# ++# This library is distributed in the hope that it will be useful, ++# but WITHOUT ANY WARRANTY; without even the implied warranty of ++# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ++# Lesser General Public License for more details. ++# ++# You should have received a copy of the GNU Lesser General Public ++# License along with this library; if not, write to the Free Software ++# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA ++ ++. ../tests/functions.sh ++ ++set -e ++set -x ++ ++requires $QEMU_NBD --version ++requires qemu-img --version ++ ++file=dump-empty-qcow2.qcow2 ++output=dump-empty-qcow2.out ++rm -f $file $output ++cleanup_fn rm -f $file $output ++ ++size=1G ++ ++# Create a large, empty qcow2 file. ++qemu-img create -f qcow2 $file $size ++ ++# Dump it and check the output. ++nbddump -- [ $QEMU_NBD -r -f qcow2 $file ] > $output ++cat $output ++ ++if [ "$(cat $output)" != '0000000000: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| ++* ++003ffffff0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................|' ]; then ++ echo "$0: unexpected output from nbddump command" ++ exit 1 ++fi +diff --git a/dump/dump-pattern.sh b/dump/dump-pattern.sh +new file mode 100755 +index 0000000..e4016a8 +--- /dev/null ++++ b/dump/dump-pattern.sh +@@ -0,0 +1,56 @@ ++#!/usr/bin/env bash ++# nbd client library in userspace ++# Copyright (C) 2020-2022 Red Hat Inc. ++# ++# This library is free software; you can redistribute it and/or ++# modify it under the terms of the GNU Lesser General Public ++# License as published by the Free Software Foundation; either ++# version 2 of the License, or (at your option) any later version. ++# ++# This library is distributed in the hope that it will be useful, ++# but WITHOUT ANY WARRANTY; without even the implied warranty of ++# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ++# Lesser General Public License for more details. ++# ++# You should have received a copy of the GNU Lesser General Public ++# License along with this library; if not, write to the Free Software ++# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA ++ ++. ../tests/functions.sh ++ ++set -e ++set -x ++ ++requires nbdkit --version ++requires nbdkit pattern --dump-plugin ++ ++output=dump-pattern.out ++rm -f $output ++cleanup_fn rm -f $output ++ ++nbdkit -U - pattern size=299 --run 'nbddump "$uri"' > $output ++ ++cat $output ++ ++if [ "$(cat $output)" != '0000000000: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 08 |................| ++0000000010: 00 00 00 00 00 00 00 10 00 00 00 00 00 00 00 18 |................| ++0000000020: 00 00 00 00 00 00 00 20 00 00 00 00 00 00 00 28 |....... .......(| ++0000000030: 00 00 00 00 00 00 00 30 00 00 00 00 00 00 00 38 |.......0.......8| ++0000000040: 00 00 00 00 00 00 00 40 00 00 00 00 00 00 00 48 |.......@.......H| ++0000000050: 00 00 00 00 00 00 00 50 00 00 00 00 00 00 00 58 |.......P.......X| ++0000000060: 00 00 00 00 00 00 00 60 00 00 00 00 00 00 00 68 |.......`.......h| ++0000000070: 00 00 00 00 00 00 00 70 00 00 00 00 00 00 00 78 |.......p.......x| ++0000000080: 00 00 00 00 00 00 00 80 00 00 00 00 00 00 00 88 |................| ++0000000090: 00 00 00 00 00 00 00 90 00 00 00 00 00 00 00 98 |................| ++00000000a0: 00 00 00 00 00 00 00 a0 00 00 00 00 00 00 00 a8 |................| ++00000000b0: 00 00 00 00 00 00 00 b0 00 00 00 00 00 00 00 b8 |................| ++00000000c0: 00 00 00 00 00 00 00 c0 00 00 00 00 00 00 00 c8 |................| ++00000000d0: 00 00 00 00 00 00 00 d0 00 00 00 00 00 00 00 d8 |................| ++00000000e0: 00 00 00 00 00 00 00 e0 00 00 00 00 00 00 00 e8 |................| ++00000000f0: 00 00 00 00 00 00 00 f0 00 00 00 00 00 00 00 f8 |................| ++0000000100: 00 00 00 00 00 00 01 00 00 00 00 00 00 00 01 08 |................| ++0000000110: 00 00 00 00 00 00 01 10 00 00 00 00 00 00 01 18 |................| ++0000000120: 00 00 00 00 00 00 01 20 00 00 00 |....... ... |' ]; then ++ echo "$0: unexpected output from nbddump command" ++ exit 1 ++fi +diff --git a/dump/dump.c b/dump/dump.c +new file mode 100644 +index 0000000..76af04c +--- /dev/null ++++ b/dump/dump.c +@@ -0,0 +1,464 @@ ++/* NBD client library in userspace ++ * Copyright (C) 2020-2022 Red Hat Inc. ++ * ++ * This library is free software; you can redistribute it and/or ++ * modify it under the terms of the GNU Lesser General Public ++ * License as published by the Free Software Foundation; either ++ * version 2 of the License, or (at your option) any later version. ++ * ++ * This library is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ++ * Lesser General Public License for more details. ++ * ++ * You should have received a copy of the GNU Lesser General Public ++ * License along with this library; if not, write to the Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA ++ */ ++ ++#include ++ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#include ++ ++#include "minmax.h" ++#include "rounding.h" ++#include "version.h" ++#include "vector.h" ++ ++DEFINE_VECTOR_TYPE (uint32_vector, uint32_t) ++ ++static const char *progname; ++static struct nbd_handle *nbd; ++static bool colour; ++static uint64_t limit = UINT64_MAX; /* --length (unlimited by default) */ ++static int64_t size; /* actual size */ ++static bool can_meta_context; /* did we get extent data? */ ++ ++/* See do_connect () */ ++static enum { MODE_URI = 1, MODE_SQUARE_BRACKET } mode; ++static char **args; ++ ++/* Read buffer. */ ++static unsigned char buffer[16*1024*1024]; ++ ++static void do_connect (void); ++static void do_dump (void); ++static void catch_signal (int); ++ ++static void __attribute__((noreturn)) ++usage (FILE *fp, int exitcode) ++{ ++ fprintf (fp, ++"\n" ++"Hexdump the content of a disk over NBD:\n" ++"\n" ++" nbddump NBD-URI | [ CMD ARGS ... ]\n" ++"\n" ++"Other options:\n" ++"\n" ++" nbddump --help\n" ++" nbddump --version\n" ++"\n" ++"Examples:\n" ++"\n" ++" nbddump nbd://localhost\n" ++" nbddump -- [ qemu-nbd -r -f qcow2 file.qcow2 ]\n" ++"\n" ++"Please read the nbddump(1) manual page for full usage.\n" ++"\n" ++); ++ exit (exitcode); ++} ++ ++int ++main (int argc, char *argv[]) ++{ ++ enum { ++ HELP_OPTION = CHAR_MAX + 1, ++ LONG_OPTIONS, ++ SHORT_OPTIONS, ++ COLOUR_OPTION, ++ NO_COLOUR_OPTION, ++ }; ++ const char *short_options = "n:V"; ++ const struct option long_options[] = { ++ { "help", no_argument, NULL, HELP_OPTION }, ++ { "long-options", no_argument, NULL, LONG_OPTIONS }, ++ { "short-options", no_argument, NULL, SHORT_OPTIONS }, ++ { "version", no_argument, NULL, 'V' }, ++ ++ { "color", no_argument, NULL, COLOUR_OPTION }, ++ { "colors", no_argument, NULL, COLOUR_OPTION }, ++ { "colour", no_argument, NULL, COLOUR_OPTION }, ++ { "colours", no_argument, NULL, COLOUR_OPTION }, ++ { "no-color", no_argument, NULL, NO_COLOUR_OPTION }, ++ { "no-colors", no_argument, NULL, NO_COLOUR_OPTION }, ++ { "no-colour", no_argument, NULL, NO_COLOUR_OPTION }, ++ { "no-colours", no_argument, NULL, NO_COLOUR_OPTION }, ++ { "length", required_argument, NULL, 'n' }, ++ { "limit", required_argument, NULL, 'n' }, ++ { NULL } ++ }; ++ int c; ++ size_t i; ++ ++ progname = argv[0]; ++ colour = isatty (STDOUT_FILENO); ++ ++ for (;;) { ++ c = getopt_long (argc, argv, short_options, long_options, NULL); ++ if (c == -1) ++ break; ++ ++ switch (c) { ++ case HELP_OPTION: ++ usage (stdout, EXIT_SUCCESS); ++ ++ case LONG_OPTIONS: ++ for (i = 0; long_options[i].name != NULL; ++i) { ++ if (strcmp (long_options[i].name, "long-options") != 0 && ++ strcmp (long_options[i].name, "short-options") != 0) ++ printf ("--%s\n", long_options[i].name); ++ } ++ exit (EXIT_SUCCESS); ++ ++ case SHORT_OPTIONS: ++ for (i = 0; short_options[i]; ++i) { ++ if (short_options[i] != ':' && short_options[i] != '+') ++ printf ("-%c\n", short_options[i]); ++ } ++ exit (EXIT_SUCCESS); ++ ++ case COLOUR_OPTION: ++ colour = true; ++ break; ++ ++ case NO_COLOUR_OPTION: ++ colour = false; ++ break; ++ ++ case 'n': ++ /* XXX Allow human sizes here. */ ++ if (sscanf (optarg, "%" SCNu64, &limit) != 1) { ++ fprintf (stderr, "%s: could not parse --length option: %s\n", ++ progname, optarg); ++ exit (EXIT_FAILURE); ++ } ++ break; ++ ++ case 'V': ++ display_version ("nbddump"); ++ exit (EXIT_SUCCESS); ++ ++ default: ++ usage (stderr, EXIT_FAILURE); ++ } ++ } ++ ++ /* Is it a URI or subprocess? */ ++ if (argc - optind >= 3 && ++ strcmp (argv[optind], "[") == 0 && ++ strcmp (argv[argc-1], "]") == 0) { ++ mode = MODE_SQUARE_BRACKET; ++ argv[argc-1] = NULL; ++ args = &argv[optind+1]; ++ } ++ else if (argc - optind == 1) { ++ mode = MODE_URI; ++ args = &argv[optind]; ++ } ++ else { ++ usage (stderr, EXIT_FAILURE); ++ } ++ ++ /* Open the NBD side. */ ++ nbd = nbd_create (); ++ if (nbd == NULL) { ++ fprintf (stderr, "%s: %s\n", progname, nbd_get_error ()); ++ exit (EXIT_FAILURE); ++ } ++ nbd_set_uri_allow_local_file (nbd, true); /* Allow ?tls-psk-file. */ ++ nbd_add_meta_context (nbd, LIBNBD_CONTEXT_BASE_ALLOCATION); ++ ++ /* Connect to the server. */ ++ do_connect (); ++ can_meta_context = ++ nbd_can_meta_context (nbd, LIBNBD_CONTEXT_BASE_ALLOCATION) > 0; ++ ++ /* Get the size. */ ++ size = nbd_get_size (nbd); ++ if (size == -1) { ++ fprintf (stderr, "%s: %s\n", progname, nbd_get_error ()); ++ exit (EXIT_FAILURE); ++ } ++ ++ /* Before dumping, make sure we restore the terminal on ^C etc. */ ++ signal (SIGINT, catch_signal); ++ signal (SIGQUIT, catch_signal); ++ signal (SIGTERM, catch_signal); ++ signal (SIGHUP, catch_signal); ++ ++ /* Dump the content. */ ++ do_dump (); ++ ++ nbd_shutdown (nbd, 0); ++ nbd_close (nbd); ++ ++ exit (EXIT_SUCCESS); ++} ++ ++/* Connect the handle to the server. */ ++static void ++do_connect (void) ++{ ++ int r; ++ ++ switch (mode) { ++ case MODE_URI: /* NBD-URI */ ++ r = nbd_connect_uri (nbd, args[0]); ++ break; ++ ++ case MODE_SQUARE_BRACKET: /* [ CMD ARGS ... ] */ ++ r = nbd_connect_systemd_socket_activation (nbd, args); ++ break; ++ ++ default: ++ abort (); ++ } ++ ++ if (r == -1) { ++ fprintf (stderr, "%s: %s\n", progname, nbd_get_error ()); ++ exit (EXIT_FAILURE); ++ } ++} ++ ++/* Various ANSI colours, suppressed if --no-colour / not tty output. */ ++static void ++ansi_restore (void) ++{ ++ if (colour) ++ fputs ("\033[0m", stdout); ++} ++ ++static void ++ansi_blue (void) ++{ ++ if (colour) ++ fputs ("\033[1;34m", stdout); ++} ++ ++static void ++ansi_green (void) ++{ ++ if (colour) ++ fputs ("\033[0;32m", stdout); ++} ++ ++static void ++ansi_magenta (void) ++{ ++ if (colour) ++ fputs ("\033[1;35m", stdout); ++} ++ ++static void ++ansi_red (void) ++{ ++ if (colour) ++ fputs ("\033[1;31m", stdout); ++} ++ ++static void ++ansi_grey (void) ++{ ++ if (colour) ++ fputs ("\033[0;90m", stdout); ++} ++ ++static void ++catch_signal (int sig) ++{ ++ printf ("\n"); ++ ansi_restore (); ++ fflush (stdout); ++ _exit (EXIT_FAILURE); ++} ++ ++/* Read the extent map for the next block and return true if it is all ++ * zeroes. This is conservative and returns false if we did not get ++ * the full extent map from the server, or if the server doesn't ++ * support base:allocation at all. ++ */ ++static int ++extent_callback (void *user_data, const char *metacontext, ++ uint64_t offset, ++ uint32_t *entries, size_t nr_entries, ++ int *error) ++{ ++ uint32_vector *list = user_data; ++ size_t i; ++ ++ if (strcmp (metacontext, LIBNBD_CONTEXT_BASE_ALLOCATION) != 0) ++ return 0; ++ ++ /* Just append the entries we got to the list. */ ++ for (i = 0; i < nr_entries; ++i) { ++ if (uint32_vector_append (list, entries[i]) == -1) { ++ perror ("realloc"); ++ exit (EXIT_FAILURE); ++ } ++ } ++ return 0; ++} ++ ++static bool ++test_all_zeroes (uint64_t offset, size_t count) ++{ ++ uint32_vector entries = empty_vector; ++ size_t i; ++ uint64_t count_read; ++ ++ if (!can_meta_context) ++ return false; ++ ++ /* Get the extent map for the block. Note the server doesn't need ++ * to return all requested data here. If it does not then we return ++ * false, causing the main code to do a full read. We could be ++ * smarter and keep asking the server (XXX). ++ */ ++ if (nbd_block_status (nbd, count, offset, ++ (nbd_extent_callback) { ++ .callback = extent_callback, ++ .user_data = &entries }, ++ 0) == -1) { ++ fprintf (stderr, "%s: %s\n", progname, nbd_get_error ()); ++ exit (EXIT_FAILURE); ++ } ++ ++ count_read = 0; ++ for (i = 0; i < entries.len; i += 2) { ++ uint32_t len = entries.ptr[i]; ++ uint32_t type = entries.ptr[i+1]; ++ ++ count_read += len; ++ if (!(type & 2)) /* not zero */ ++ return false; ++ } ++ ++ /* Did we read at least the whole range wanted? */ ++ if (count_read < count) ++ return false; ++ ++ /* If we got here, we read the whole range and it was all zeroes. */ ++ return true; ++} ++ ++/* Hexdump the NBD data. ++ * ++ * XXX In future we could do this all asynch (including writing to ++ * stdout) which could make it very efficient. ++ */ ++static void ++do_dump (void) ++{ ++ /* If --no-colour, don't use unicode in the output. */ ++ const char *splat = colour ? "☆" : "*"; ++ const char *pipe = colour ? "│" : "|"; ++ const char *dot = colour ? "·" : "."; ++ uint64_t offset = 0; ++ uint64_t count = size > limit ? limit : size; ++ size_t i, j, n; ++ char last[16]; ++ bool printed_splat = false, same; ++ ++ while (count) { ++ n = MIN (count, sizeof buffer); ++ ++ if (! test_all_zeroes (offset, n)) { ++ if (nbd_pread (nbd, buffer, n, offset, 0) == -1) { ++ fprintf (stderr, "%s: %s\n", progname, nbd_get_error ()); ++ exit (EXIT_FAILURE); ++ } ++ } ++ else { ++ memset (buffer, 0, n); ++ } ++ ++ /* Make sure a multiple of 16 bytes gets written to the buffer. */ ++ if (n & 15) ++ memset (&buffer[n], 0, 16 - (n & 15)); ++ ++ for (i = 0; i < n; i += 16) { ++ /* Is this line the same as the last line? (Squashing) */ ++ same = ++ offset + i > 0 && /* first line is never squashed */ ++ offset + i + 16 < size && /* last line is never squashed */ ++ memcmp (&buffer[i], last, 16) == 0; ++ if (same) { ++ if (!printed_splat) { ++ printf ("%s\n", splat); ++ printed_splat = true; ++ } ++ continue; ++ } ++ printed_splat = false; ++ memcpy (last, &buffer[i], 16); /* Save the current line. */ ++ ++ /* Print the offset. */ ++ ansi_green (); ++ printf ("%010zx", offset + i); ++ ansi_grey (); ++ printf (": "); ++ ++ /* Print the hex codes. */ ++ for (j = i; j < MIN (i+16, n); ++j) { ++ if (buffer[j]) ++ ansi_blue (); ++ else ++ ansi_grey (); ++ printf ("%02x ", buffer[j]); ++ } ++ ansi_grey (); ++ for (; j < i+16; ++j) ++ printf (" "); ++ ++ /* Print the ASCII codes. */ ++ printf ("%s", pipe); ++ for (j = i; j < MIN (i+16, n); ++j) { ++ char c = (char) buffer[j]; ++ if (isalnum (c)) { ++ ansi_red (); ++ printf ("%c", c); ++ } ++ else if (isprint (c)) { ++ ansi_magenta (); ++ printf ("%c", c); ++ } ++ else { ++ ansi_grey (); ++ printf ("%s", dot); ++ } ++ } ++ ansi_grey (); ++ for (; j < i+16; ++j) ++ printf (" "); ++ printf ("%s\n", pipe); ++ ansi_restore (); ++ } ++ ++ offset += n; ++ count -= n; ++ } ++} +diff --git a/dump/nbddump.pod b/dump/nbddump.pod +new file mode 100644 +index 0000000..5d7864d +--- /dev/null ++++ b/dump/nbddump.pod +@@ -0,0 +1,116 @@ ++=head1 NAME ++ ++nbddump - hexdump the content of a disk over NBD ++ ++=head1 SYNOPSIS ++ ++ nbddump NBD ++ ++C is an NBD URI or subprocess: ++ ++ NBD := nbd://... | nbd+unix:// (or other URI formats) ++ | [ CMD ARGS ... ] ++ ++=for paragraph ++ ++ nbddump --help ++ ++=for paragraph ++ ++ nbddump --version ++ ++=head1 DESCRIPTION ++ ++nbddump prints the content of a disk from an NBD server using the ++usual hexdump format: ++ ++ $ nbddump nbd://localhost ++ 0000: 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 │················│ ++ 0010: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 │················│ ++ ☆ ++ 0100: 68 65 6c 6c 6f 2c 20 77 6f 72 6c 64 21 00 00 00 │hello, world!···│ ++ 0110: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 │················│ ++ ☆ ++ 1000: 00 00 00 21 │···! │ ++ ++=head2 Output format ++ ++The first field (before the C<:>) is the offset within the file, in ++hexadecimal. ++ ++The second field shows the hex codes of bytes read from the file. ++ ++The third field shows the ASCII equivalent characters (if printable). ++ ++A splat character (C<☆>) indicates lines of repeated output which have ++been squashed. (Note this is not just for lines of zero bytes, but ++any case where the next line shown would be the same as the previous ++line.) ++ ++=head2 Subprocess ++ ++nbddump can also run an NBD server as a subprocess. This requires an ++NBD server which understands systemd socket activation, such as ++L or L. ++ ++For example, to dump out a qcow2 file as raw data: ++ ++ nbddump -- [ qemu-nbd -r -f qcow2 file.qcow2 ] ++ ++Note that S> are separate parameters, and must be ++surrounded by spaces. C<--> separates nbddump parameters from ++subprocess parameters. ++ ++=head1 OPTIONS ++ ++=over 4 ++ ++=item B<--help> ++ ++Display brief command line help and exit. ++ ++=item B<--color> ++ ++=item B<--colour> ++ ++=item B<--no-color> ++ ++=item B<--no-colour> ++ ++Enable or disable ANSI colours in output. By default we use colours ++if the output seems to be a terminal, and disable them if not. ++ ++=item B<--length=>N ++ ++=item B<-n> N ++ ++Dump up to I bytes and then stop. ++ ++=item B<-V> ++ ++=item B<--version> ++ ++Display the package name and version and exit. ++ ++=back ++ ++=head1 SEE ALSO ++ ++L, ++L, ++L, ++L, ++L, ++L, ++L, ++L, ++L, ++L. ++ ++=head1 AUTHORS ++ ++Richard W.M. Jones ++ ++=head1 COPYRIGHT ++ ++Copyright (C) 2022 Red Hat Inc. +diff --git a/dump/test-long-options.sh b/dump/test-long-options.sh +new file mode 100755 +index 0000000..924c8f5 +--- /dev/null ++++ b/dump/test-long-options.sh +@@ -0,0 +1,35 @@ ++#!/usr/bin/env bash ++# nbd client library in userspace ++# Copyright (C) 2019-2022 Red Hat Inc. ++# ++# This library is free software; you can redistribute it and/or ++# modify it under the terms of the GNU Lesser General Public ++# License as published by the Free Software Foundation; either ++# version 2 of the License, or (at your option) any later version. ++# ++# This library is distributed in the hope that it will be useful, ++# but WITHOUT ANY WARRANTY; without even the implied warranty of ++# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ++# Lesser General Public License for more details. ++# ++# You should have received a copy of the GNU Lesser General Public ++# License along with this library; if not, write to the Free Software ++# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA ++ ++# Test that nbddump --long-options looks sane. ++ ++. ../tests/functions.sh ++set -e ++set -x ++ ++output=test-long-options.out ++cleanup_fn rm -f $output ++ ++$VG nbddump --long-options > $output ++if [ $? != 0 ]; then ++ echo "$0: unexpected exit status" ++ fail=1 ++fi ++cat $output ++grep -- --length $output ++grep -- --version $output +diff --git a/dump/test-short-options.sh b/dump/test-short-options.sh +new file mode 100755 +index 0000000..325f7df +--- /dev/null ++++ b/dump/test-short-options.sh +@@ -0,0 +1,35 @@ ++#!/usr/bin/env bash ++# nbd client library in userspace ++# Copyright (C) 2019-2022 Red Hat Inc. ++# ++# This library is free software; you can redistribute it and/or ++# modify it under the terms of the GNU Lesser General Public ++# License as published by the Free Software Foundation; either ++# version 2 of the License, or (at your option) any later version. ++# ++# This library is distributed in the hope that it will be useful, ++# but WITHOUT ANY WARRANTY; without even the implied warranty of ++# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ++# Lesser General Public License for more details. ++# ++# You should have received a copy of the GNU Lesser General Public ++# License along with this library; if not, write to the Free Software ++# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA ++ ++# Test that nbddump --short-options looks sane. ++ ++. ../tests/functions.sh ++set -e ++set -x ++ ++output=test-short-options.out ++cleanup_fn rm -f $output ++ ++$VG nbddump --short-options > $output ++if [ $? != 0 ]; then ++ echo "$0: unexpected exit status" ++ fail=1 ++fi ++cat $output ++grep -- -n $output ++grep -- -V $output +diff --git a/dump/test-version.sh b/dump/test-version.sh +new file mode 100755 +index 0000000..fce4ed1 +--- /dev/null ++++ b/dump/test-version.sh +@@ -0,0 +1,33 @@ ++#!/usr/bin/env bash ++# nbd client library in userspace ++# Copyright (C) 2019 Red Hat Inc. ++# ++# This library is free software; you can redistribute it and/or ++# modify it under the terms of the GNU Lesser General Public ++# License as published by the Free Software Foundation; either ++# version 2 of the License, or (at your option) any later version. ++# ++# This library is distributed in the hope that it will be useful, ++# but WITHOUT ANY WARRANTY; without even the implied warranty of ++# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ++# Lesser General Public License for more details. ++# ++# You should have received a copy of the GNU Lesser General Public ++# License along with this library; if not, write to the Free Software ++# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA ++ ++# Test that nbddump --version looks sane. ++ ++fail=0 ++output=$($VG nbddump --version) ++if [ $? != 0 ]; then ++ echo "$0: unexpected exit status" ++ fail=1 ++fi ++if [ "$output" != "nbddump $EXPECTED_VERSION ++libnbd $EXPECTED_VERSION" ]; then ++ echo "$0: unexpected output" ++ fail=1 ++fi ++echo "$output" ++exit $fail +diff --git a/fuse/nbdfuse.pod b/fuse/nbdfuse.pod +index 7c1c817..daa79c1 100644 +--- a/fuse/nbdfuse.pod ++++ b/fuse/nbdfuse.pod +@@ -412,6 +412,7 @@ The differences from nbdfuse are similar to the list above. + + L, + L, ++L, + L, + L, + L, +diff --git a/info/nbdinfo.pod b/info/nbdinfo.pod +index 649cf1c..4733ecd 100644 +--- a/info/nbdinfo.pod ++++ b/info/nbdinfo.pod +@@ -407,6 +407,7 @@ Display the package name and version and exit. + + L, + L, ++L, + L, + L, + L, +diff --git a/run.in b/run.in +index 8a21906..2a171e5 100755 +--- a/run.in ++++ b/run.in +@@ -58,6 +58,7 @@ b="$(cd @abs_builddir@ && pwd)" + + # Set the PATH to contain all libnbd binaries. + prepend PATH "$b/copy" ++prepend PATH "$b/dump" + prepend PATH "$b/fuse" + prepend PATH "$b/info" + prepend PATH "$b/sh" +diff --git a/sh/nbdsh.pod b/sh/nbdsh.pod +index ca5d6af..c9dac4a 100644 +--- a/sh/nbdsh.pod ++++ b/sh/nbdsh.pod +@@ -147,6 +147,7 @@ L. + L, + L, + L, ++L, + L, + L, + L. +-- +2.31.1 + diff --git a/SOURCES/0001-api-Add-new-API-nbd_set_pread_initialize.patch b/SOURCES/0001-api-Add-new-API-nbd_set_pread_initialize.patch deleted file mode 100644 index 4befdbd..0000000 --- a/SOURCES/0001-api-Add-new-API-nbd_set_pread_initialize.patch +++ /dev/null @@ -1,469 +0,0 @@ -From c79706af4e7475bf58861a143b77b77a54e7a1cd Mon Sep 17 00:00:00 2001 -From: Eric Blake -Date: Wed, 9 Feb 2022 15:39:49 -0600 -Subject: [PATCH] api: Add new API nbd_set_pread_initialize() - -The recent patch series for CVE-2022-0485 demonstrated that when -applications using libnbd are not careful about error checking, the -difference on whether a data leak is at least sanitized (all zeroes, -partial reads, or data leftover from a prior read) vs. a dangerous -information leak (uninitialized data from the heap) was partly under -libnbd's control. The previous two patches changed libnbd to always -sanitize, as a security hardening technique that prevents heap leaks -no matter how buggy the client app is. But a blind memset() also adds -an execution delay, even if it doesn't show up as the hot spot in our -profiling when compared to the time spent with network traffic. - -At any rate, if client apps choose to pre-initialize their buffers, or -otherwise audit their code to take on their own risk about not -dereferencing a buffer on failure paths, then the time spent by libnbd -doing memset() is wasted; so it is worth adding a knob to let a user -opt in to faster execution at the expense of giving up our memset() -hardening on their behalf. - -In addition to adding two new APIs, this patch also causes changes to -the four existing APIs nbd_{aio_,}pread{,_structured}, with those -generated lib/api.c changes looking like: - -| --- lib/api.c.bak 2022-02-10 08:17:09.973381979 -0600 -| +++ lib/api.c 2022-02-10 08:22:27.503428024 -0600 -| @@ -2871,7 +2914,8 @@ nbd_pread (struct nbd_handle *h, void *b -| debug (h, "enter: buf= count=%zu offset=%" PRIu64 " flags=0x%x", count, offset, flags); -| } -| -| - memset (buf, 0, count); -| + if (h->pread_initialize) -| + memset (buf, 0, count); -| if (unlikely (!pread_in_permitted_state (h))) { -| ret = -1; -| goto out; - -Message-Id: <20220209220726.1902761-4-eblake@redhat.com> -Acked-by: Laszlo Ersek -[eblake: enhance commit message to show generated file diff, mention CVE -in doc text] -Reviewed-by: Richard W.M. Jones - -(cherry picked from commit e0953cb71250947bb97b25e34ff1ea34bd504bf3) ---- - generator/API.ml | 90 ++++++++++++++++--- - generator/C.ml | 3 +- - .../libnbd/libnbd_110_defaults_test.go | 10 ++- - .../libnbd_120_set_non_defaults_test.go | 12 +++ - lib/handle.c | 17 +++- - lib/internal.h | 5 +- - ocaml/tests/test_110_defaults.ml | 4 +- - ocaml/tests/test_120_set_non_defaults.ml | 5 +- - python/t/110-defaults.py | 3 +- - python/t/120-set-non-defaults.py | 4 +- - tests/errors.c | 25 +++++- - 11 files changed, 156 insertions(+), 22 deletions(-) - -diff --git a/generator/API.ml b/generator/API.ml -index d8df7c8..00ab34f 100644 ---- a/generator/API.ml -+++ b/generator/API.ml -@@ -778,6 +778,49 @@ the time of compilation."; - Link "aio_is_created"; Link "aio_is_ready"]; - }; - -+ "set_pread_initialize", { -+ default_call with -+ args = [Bool "request"]; ret = RErr; -+ shortdesc = "control whether libnbd pre-initializes read buffers"; -+ longdesc = "\ -+By default, libnbd will pre-initialize the contents of a buffer -+passed to calls such as L to all zeroes prior to -+checking for any other errors, so that even if a client application -+passed in an uninitialized buffer but fails to check for errors, it -+will not result in a potential security risk caused by an accidental -+leak of prior heap contents (see CVE-2022-0485 in -+L for an example of a security hole in an -+application built against an earlier version of libnbd that lacked -+consistent pre-initialization). However, for a client application -+that has audited that an uninitialized buffer is never dereferenced, -+or which performs its own pre-initialization, libnbd's sanitization -+efforts merely pessimize performance (although the time spent in -+pre-initialization may pale in comparison to time spent waiting on -+network packets). -+ -+Calling this function with C set to false tells libnbd to -+skip the buffer initialization step in read commands."; -+ see_also = [Link "get_pread_initialize"; -+ Link "set_strict_mode"; -+ Link "pread"; Link "pread_structured"; Link "aio_pread"; -+ Link "aio_pread_structured"]; -+ }; -+ -+ "get_pread_initialize", { -+ default_call with -+ args = []; ret = RBool; -+ may_set_error = false; -+ shortdesc = "see whether libnbd pre-initializes read buffers"; -+ longdesc = "\ -+Return whether libnbd performs a pre-initialization of a buffer passed -+to L and similar to all zeroes, as set by -+L."; -+ see_also = [Link "set_pread_initialize"; -+ Link "set_strict_mode"; -+ Link "pread"; Link "pread_structured"; Link "aio_pread"; -+ Link "aio_pread_structured"]; -+ }; -+ - "set_strict_mode", { - default_call with - args = [ Flags ("flags", strict_flags) ]; ret = RErr; -@@ -1825,11 +1868,16 @@ C. - The C parameter must be C<0> for now (it exists for future NBD - protocol extensions). - --Note that if this command fails, it is unspecified whether the contents --of C will read as zero or as partial results from the server." -+Note that if this command fails, and L -+returns true, then libnbd sanitized C, but it is unspecified -+whether the contents of C will read as zero or as partial results -+from the server. If L returns false, -+then libnbd did not sanitize C, and the contents are undefined -+on failure." - ^ strict_call_description; - see_also = [Link "aio_pread"; Link "pread_structured"; -- Link "get_block_size"; Link "set_strict_mode"]; -+ Link "get_block_size"; Link "set_strict_mode"; -+ Link "set_pread_initialize"]; - example = Some "examples/fetch-first-sector.c"; - }; - -@@ -1907,12 +1955,16 @@ more than one fragment (if that is supported - some servers cannot do - this, see L). Libnbd does not validate that the server - actually obeys the flag. - --Note that if this command fails, it is unspecified whether the contents --of C will read as zero or as partial results from the server." -+Note that if this command fails, and L -+returns true, then libnbd sanitized C, but it is unspecified -+whether the contents of C will read as zero or as partial results -+from the server. If L returns false, -+then libnbd did not sanitize C, and the contents are undefined -+on failure." - ^ strict_call_description; - see_also = [Link "can_df"; Link "pread"; - Link "aio_pread_structured"; Link "get_block_size"; -- Link "set_strict_mode"]; -+ Link "set_strict_mode"; Link "set_pread_initialize"]; - }; - - "pwrite", { -@@ -2420,14 +2472,19 @@ as described in L. - Note that you must ensure C is valid until the command has - completed. Furthermore, if the C parameter to - C is set or if L --reports failure, it is unspecified whether the contents --of C will read as zero or as partial results from the server. -+reports failure, and if L returns true, -+then libnbd sanitized C, but it is unspecified whether the -+contents of C will read as zero or as partial results from the -+server. If L returns false, then -+libnbd did not sanitize C, and the contents are undefined -+on failure. -+ - Other parameters behave as documented in L." - ^ strict_call_description; - example = Some "examples/aio-connect-read.c"; - see_also = [SectionLink "Issuing asynchronous commands"; - Link "aio_pread_structured"; Link "pread"; -- Link "set_strict_mode"]; -+ Link "set_strict_mode"; Link "set_pread_initialize"]; - }; - - "aio_pread_structured", { -@@ -2449,13 +2506,18 @@ as described in L. - Note that you must ensure C is valid until the command has - completed. Furthermore, if the C parameter to - C is set or if L --reports failure, it is unspecified whether the contents --of C will read as zero or as partial results from the server. -+reports failure, and if L returns true, -+then libnbd sanitized C, but it is unspecified whether the -+contents of C will read as zero or as partial results from the -+server. If L returns false, then -+libnbd did not sanitize C, and the contents are undefined -+on failure. -+ - Other parameters behave as documented in L." - ^ strict_call_description; - see_also = [SectionLink "Issuing asynchronous commands"; - Link "aio_pread"; Link "pread_structured"; -- Link "set_strict_mode"]; -+ Link "set_strict_mode"; Link "set_pread_initialize"]; - }; - - "aio_pwrite", { -@@ -3093,6 +3155,10 @@ let first_version = [ - "get_private_data", (1, 8); - "get_uri", (1, 8); - -+ (* Added in 1.11.x development cycle, will be stable and supported in 1.12. *) -+ "set_pread_initialize", (1, 12); -+ "get_pread_initialize", (1, 12); -+ - (* These calls are proposed for a future version of libnbd, but - * have not been added to any released version so far. - "get_tls_certificates", (1, ??); -diff --git a/generator/C.ml b/generator/C.ml -index 4a5bb58..2b6198c 100644 ---- a/generator/C.ml -+++ b/generator/C.ml -@@ -496,7 +496,8 @@ let generate_lib_api_c () = - function - | BytesOut (n, count) - | BytesPersistOut (n, count) -> -- pr " memset (%s, 0, %s);\n" n count -+ pr " if (h->pread_initialize)\n"; -+ pr " memset (%s, 0, %s);\n" n count - | _ -> () - ) args; - -diff --git a/golang/src/libguestfs.org/libnbd/libnbd_110_defaults_test.go b/golang/src/libguestfs.org/libnbd/libnbd_110_defaults_test.go -index b3ceb45..ca7c1c4 100644 ---- a/golang/src/libguestfs.org/libnbd/libnbd_110_defaults_test.go -+++ b/golang/src/libguestfs.org/libnbd/libnbd_110_defaults_test.go -@@ -1,5 +1,5 @@ - /* libnbd golang tests -- * Copyright (C) 2013-2021 Red Hat Inc. -+ * Copyright (C) 2013-2022 Red Hat Inc. - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public -@@ -59,6 +59,14 @@ func Test110Defaults(t *testing.T) { - t.Fatalf("unexpected structured replies state") - } - -+ init, err := h.GetPreadInitialize() -+ if err != nil { -+ t.Fatalf("could not get pread initialize state: %s", err) -+ } -+ if init != true { -+ t.Fatalf("unexpected pread initialize state") -+ } -+ - flags, err := h.GetHandshakeFlags() - if err != nil { - t.Fatalf("could not get handshake flags: %s", err) -diff --git a/golang/src/libguestfs.org/libnbd/libnbd_120_set_non_defaults_test.go b/golang/src/libguestfs.org/libnbd/libnbd_120_set_non_defaults_test.go -index f112456..029f0db 100644 ---- a/golang/src/libguestfs.org/libnbd/libnbd_120_set_non_defaults_test.go -+++ b/golang/src/libguestfs.org/libnbd/libnbd_120_set_non_defaults_test.go -@@ -93,6 +93,18 @@ func Test120SetNonDefaults(t *testing.T) { - t.Fatalf("unexpected structured replies state") - } - -+ err = h.SetPreadInitialize(false) -+ if err != nil { -+ t.Fatalf("could not set pread initialize state: %s", err) -+ } -+ init, err := h.GetPreadInitialize() -+ if err != nil { -+ t.Fatalf("could not get pread initialize state: %s", err) -+ } -+ if init != false { -+ t.Fatalf("unexpected pread initialize state") -+ } -+ - err = h.SetHandshakeFlags(HANDSHAKE_FLAG_MASK + 1) - if err == nil { - t.Fatalf("expect failure for out-of-range flags") -diff --git a/lib/handle.c b/lib/handle.c -index 67aa875..ac6c16e 100644 ---- a/lib/handle.c -+++ b/lib/handle.c -@@ -1,5 +1,5 @@ - /* NBD client library in userspace -- * Copyright (C) 2013-2020 Red Hat Inc. -+ * Copyright (C) 2013-2022 Red Hat Inc. - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public -@@ -64,6 +64,7 @@ nbd_create (void) - h->unique = 1; - h->tls_verify_peer = true; - h->request_sr = true; -+ h->pread_initialize = true; - - h->uri_allow_transports = LIBNBD_ALLOW_TRANSPORT_MASK; - h->uri_allow_tls = LIBNBD_TLS_ALLOW; -@@ -393,6 +394,20 @@ nbd_unlocked_get_handshake_flags (struct nbd_handle *h) - return h->gflags; - } - -+int -+nbd_unlocked_set_pread_initialize (struct nbd_handle *h, bool request) -+{ -+ h->pread_initialize = request; -+ return 0; -+} -+ -+/* NB: may_set_error = false. */ -+int -+nbd_unlocked_get_pread_initialize (struct nbd_handle *h) -+{ -+ return h->pread_initialize; -+} -+ - int - nbd_unlocked_set_strict_mode (struct nbd_handle *h, uint32_t flags) - { -diff --git a/lib/internal.h b/lib/internal.h -index 0e205ab..525499a 100644 ---- a/lib/internal.h -+++ b/lib/internal.h -@@ -1,5 +1,5 @@ - /* nbd client library in userspace: internal definitions -- * Copyright (C) 2013-2020 Red Hat Inc. -+ * Copyright (C) 2013-2022 Red Hat Inc. - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public -@@ -123,6 +123,9 @@ struct nbd_handle { - /* Full info mode. */ - bool full_info; - -+ /* Sanitization for pread. */ -+ bool pread_initialize; -+ - /* Global flags from the server. */ - uint16_t gflags; - -diff --git a/ocaml/tests/test_110_defaults.ml b/ocaml/tests/test_110_defaults.ml -index b36949f..04aa744 100644 ---- a/ocaml/tests/test_110_defaults.ml -+++ b/ocaml/tests/test_110_defaults.ml -@@ -1,6 +1,6 @@ - (* hey emacs, this is OCaml code: -*- tuareg -*- *) - (* libnbd OCaml test case -- * Copyright (C) 2013-2020 Red Hat Inc. -+ * Copyright (C) 2013-2022 Red Hat Inc. - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public -@@ -28,6 +28,8 @@ let () = - assert (tls = NBD.TLS.DISABLE); - let sr = NBD.get_request_structured_replies nbd in - assert (sr = true); -+ let init = NBD.get_pread_initialize nbd in -+ assert (init = true); - let flags = NBD.get_handshake_flags nbd in - assert (flags = NBD.HANDSHAKE_FLAG.mask); - let opt = NBD.get_opt_mode nbd in -diff --git a/ocaml/tests/test_120_set_non_defaults.ml b/ocaml/tests/test_120_set_non_defaults.ml -index 67928bb..f949807 100644 ---- a/ocaml/tests/test_120_set_non_defaults.ml -+++ b/ocaml/tests/test_120_set_non_defaults.ml -@@ -1,6 +1,6 @@ - (* hey emacs, this is OCaml code: -*- tuareg -*- *) - (* libnbd OCaml test case -- * Copyright (C) 2013-2020 Red Hat Inc. -+ * Copyright (C) 2013-2022 Red Hat Inc. - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public -@@ -42,6 +42,9 @@ let () = - NBD.set_request_structured_replies nbd false; - let sr = NBD.get_request_structured_replies nbd in - assert (sr = false); -+ NBD.set_pread_initialize nbd false; -+ let init = NBD.get_pread_initialize nbd in -+ assert (init = false); - (try - NBD.set_handshake_flags nbd [ NBD.HANDSHAKE_FLAG.UNKNOWN 2 ]; - assert false -diff --git a/python/t/110-defaults.py b/python/t/110-defaults.py -index fb961cf..a4262da 100644 ---- a/python/t/110-defaults.py -+++ b/python/t/110-defaults.py -@@ -1,5 +1,5 @@ - # libnbd Python bindings --# Copyright (C) 2010-2020 Red Hat Inc. -+# Copyright (C) 2010-2022 Red Hat Inc. - # - # This program is free software; you can redistribute it and/or modify - # it under the terms of the GNU General Public License as published by -@@ -22,5 +22,6 @@ assert h.get_export_name() == "" - assert h.get_full_info() is False - assert h.get_tls() == nbd.TLS_DISABLE - assert h.get_request_structured_replies() is True -+assert h.get_pread_initialize() is True - assert h.get_handshake_flags() == nbd.HANDSHAKE_FLAG_MASK - assert h.get_opt_mode() is False -diff --git a/python/t/120-set-non-defaults.py b/python/t/120-set-non-defaults.py -index 3da0c23..e71c6ad 100644 ---- a/python/t/120-set-non-defaults.py -+++ b/python/t/120-set-non-defaults.py -@@ -1,5 +1,5 @@ - # libnbd Python bindings --# Copyright (C) 2010-2020 Red Hat Inc. -+# Copyright (C) 2010-2022 Red Hat Inc. - # - # This program is free software; you can redistribute it and/or modify - # it under the terms of the GNU General Public License as published by -@@ -33,6 +33,8 @@ if h.supports_tls(): - assert h.get_tls() == nbd.TLS_ALLOW - h.set_request_structured_replies(False) - assert h.get_request_structured_replies() is False -+h.set_pread_initialize(False) -+assert h.get_pread_initialize() is False - try: - h.set_handshake_flags(nbd.HANDSHAKE_FLAG_MASK + 1) - assert False -diff --git a/tests/errors.c b/tests/errors.c -index f597b7e..0298da8 100644 ---- a/tests/errors.c -+++ b/tests/errors.c -@@ -213,7 +213,15 @@ main (int argc, char *argv[]) - } - - -- /* Issue a connected command when not connected. */ -+ /* Issue a connected command when not connected. pread_initialize defaults -+ * to set. -+ */ -+ if (nbd_get_pread_initialize (nbd) != 1) { -+ fprintf (stderr, "%s: test failed: " -+ "nbd_get_pread_initialize gave unexpected result\n", -+ argv[0]); -+ exit (EXIT_FAILURE); -+ } - buf[0] = '1'; - if (nbd_pread (nbd, buf, 512, 0, 0) != -1) { - fprintf (stderr, "%s: test failed: " -@@ -294,7 +302,14 @@ main (int argc, char *argv[]) - } - check (EINVAL, "nbd_aio_command_completed: "); - -- /* Read from an invalid offset, client-side */ -+ /* Read from an invalid offset, client-side. When pread_initialize is off, -+ * libnbd should not have touched our buffer. -+ */ -+ if (nbd_set_pread_initialize (nbd, false) == -1) { -+ fprintf (stderr, "%s\n", nbd_get_error ()); -+ exit (EXIT_FAILURE); -+ } -+ buf[0] = '1'; - strict = nbd_get_strict_mode (nbd) | LIBNBD_STRICT_BOUNDS; - if (nbd_set_strict_mode (nbd, strict) == -1) { - fprintf (stderr, "%s\n", nbd_get_error ()); -@@ -307,6 +322,12 @@ main (int argc, char *argv[]) - exit (EXIT_FAILURE); - } - check (EINVAL, "nbd_aio_pread: "); -+ if (buf[0] != '1') { -+ fprintf (stderr, "%s: test failed: " -+ "nbd_pread incorrectly sanitized buffer on client-side error\n", -+ argv[0]); -+ exit (EXIT_FAILURE); -+ } - - /* We guarantee callbacks will be freed even on all error paths. */ - if (nbd_aio_pread_structured (nbd, buf, 512, -1, --- -2.31.1 - diff --git a/SOURCES/0002-dump-Visually-separate-columns-0-7-and-8-15.patch b/SOURCES/0002-dump-Visually-separate-columns-0-7-and-8-15.patch new file mode 100644 index 0000000..67fc1af --- /dev/null +++ b/SOURCES/0002-dump-Visually-separate-columns-0-7-and-8-15.patch @@ -0,0 +1,153 @@ +From ec947323528725fcf12b5b9ba32b02d36dbd9621 Mon Sep 17 00:00:00 2001 +From: "Richard W.M. Jones" +Date: Thu, 30 Jun 2022 21:09:39 +0100 +Subject: [PATCH] dump: Visually separate columns 0-7 and 8-15 +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +Before: + +0000090000: 68 65 72 65 20 77 65 20 61 72 65 00 68 65 72 65 │... +0000090010: 20 77 65 20 61 72 65 00 68 65 72 65 20 77 65 20 │... +0000090020: 61 72 65 00 68 65 72 65 20 77 65 20 61 72 65 00 │... + +After: + +0000090000: 68 65 72 65 20 77 65 20 61 72 65 00 68 65 72 65 │... +0000090010: 20 77 65 20 61 72 65 00 68 65 72 65 20 77 65 20 │... +0000090020: 61 72 65 00 68 65 72 65 20 77 65 20 61 72 65 00 │... + +Updates: commit c4107b9a40d6451630dcccf1bf6596c8e56420be +(cherry picked from commit 315a637d3eae003c1d84eb1b88a7b47b534f1e80) +--- + dump/dump-data.sh | 22 +++++++++++----------- + dump/dump-empty-qcow2.sh | 4 ++-- + dump/dump-pattern.sh | 38 +++++++++++++++++++------------------- + dump/dump.c | 5 ++++- + 4 files changed, 36 insertions(+), 33 deletions(-) + +diff --git a/dump/dump-data.sh b/dump/dump-data.sh +index 23d09da..955cd3b 100755 +--- a/dump/dump-data.sh ++++ b/dump/dump-data.sh +@@ -37,21 +37,21 @@ nbdkit -U - data data=' + + cat $output + +-if [ "$(cat $output)" != '0000000000: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| ++if [ "$(cat $output)" != '0000000000: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| + * +-0000008000: 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| +-0000008010: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| ++0000008000: 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| ++0000008010: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| + * +-000000fff0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 68 |...............h| +-0000010000: 65 6c 6c 6f 2c 20 77 6f 72 6c 64 21 00 00 00 00 |ello, world!....| +-0000010010: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| ++000000fff0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 68 |...............h| ++0000010000: 65 6c 6c 6f 2c 20 77 6f 72 6c 64 21 00 00 00 00 |ello, world!....| ++0000010010: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| + * +-00010ffff0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 73 70 |..............sp| +-0001100000: 61 6e 6e 69 6e 67 20 62 75 66 66 65 72 20 62 6f |anning buffer bo| +-0001100010: 75 6e 64 61 72 79 00 00 00 00 00 00 00 00 00 00 |undary..........| +-0001100020: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| ++00010ffff0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 73 70 |..............sp| ++0001100000: 61 6e 6e 69 6e 67 20 62 75 66 66 65 72 20 62 6f |anning buffer bo| ++0001100010: 75 6e 64 61 72 79 00 00 00 00 00 00 00 00 00 00 |undary..........| ++0001100020: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| + * +-0001312d00: 00 |. |' ]; then ++0001312d00: 00 |. |' ]; then + echo "$0: unexpected output from nbddump command" + exit 1 + fi +diff --git a/dump/dump-empty-qcow2.sh b/dump/dump-empty-qcow2.sh +index c9e583b..472b6eb 100755 +--- a/dump/dump-empty-qcow2.sh ++++ b/dump/dump-empty-qcow2.sh +@@ -38,9 +38,9 @@ qemu-img create -f qcow2 $file $size + nbddump -- [ $QEMU_NBD -r -f qcow2 $file ] > $output + cat $output + +-if [ "$(cat $output)" != '0000000000: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| ++if [ "$(cat $output)" != '0000000000: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| + * +-003ffffff0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................|' ]; then ++003ffffff0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................|' ]; then + echo "$0: unexpected output from nbddump command" + exit 1 + fi +diff --git a/dump/dump-pattern.sh b/dump/dump-pattern.sh +index e4016a8..d512b77 100755 +--- a/dump/dump-pattern.sh ++++ b/dump/dump-pattern.sh +@@ -32,25 +32,25 @@ nbdkit -U - pattern size=299 --run 'nbddump "$uri"' > $output + + cat $output + +-if [ "$(cat $output)" != '0000000000: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 08 |................| +-0000000010: 00 00 00 00 00 00 00 10 00 00 00 00 00 00 00 18 |................| +-0000000020: 00 00 00 00 00 00 00 20 00 00 00 00 00 00 00 28 |....... .......(| +-0000000030: 00 00 00 00 00 00 00 30 00 00 00 00 00 00 00 38 |.......0.......8| +-0000000040: 00 00 00 00 00 00 00 40 00 00 00 00 00 00 00 48 |.......@.......H| +-0000000050: 00 00 00 00 00 00 00 50 00 00 00 00 00 00 00 58 |.......P.......X| +-0000000060: 00 00 00 00 00 00 00 60 00 00 00 00 00 00 00 68 |.......`.......h| +-0000000070: 00 00 00 00 00 00 00 70 00 00 00 00 00 00 00 78 |.......p.......x| +-0000000080: 00 00 00 00 00 00 00 80 00 00 00 00 00 00 00 88 |................| +-0000000090: 00 00 00 00 00 00 00 90 00 00 00 00 00 00 00 98 |................| +-00000000a0: 00 00 00 00 00 00 00 a0 00 00 00 00 00 00 00 a8 |................| +-00000000b0: 00 00 00 00 00 00 00 b0 00 00 00 00 00 00 00 b8 |................| +-00000000c0: 00 00 00 00 00 00 00 c0 00 00 00 00 00 00 00 c8 |................| +-00000000d0: 00 00 00 00 00 00 00 d0 00 00 00 00 00 00 00 d8 |................| +-00000000e0: 00 00 00 00 00 00 00 e0 00 00 00 00 00 00 00 e8 |................| +-00000000f0: 00 00 00 00 00 00 00 f0 00 00 00 00 00 00 00 f8 |................| +-0000000100: 00 00 00 00 00 00 01 00 00 00 00 00 00 00 01 08 |................| +-0000000110: 00 00 00 00 00 00 01 10 00 00 00 00 00 00 01 18 |................| +-0000000120: 00 00 00 00 00 00 01 20 00 00 00 |....... ... |' ]; then ++if [ "$(cat $output)" != '0000000000: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 08 |................| ++0000000010: 00 00 00 00 00 00 00 10 00 00 00 00 00 00 00 18 |................| ++0000000020: 00 00 00 00 00 00 00 20 00 00 00 00 00 00 00 28 |....... .......(| ++0000000030: 00 00 00 00 00 00 00 30 00 00 00 00 00 00 00 38 |.......0.......8| ++0000000040: 00 00 00 00 00 00 00 40 00 00 00 00 00 00 00 48 |.......@.......H| ++0000000050: 00 00 00 00 00 00 00 50 00 00 00 00 00 00 00 58 |.......P.......X| ++0000000060: 00 00 00 00 00 00 00 60 00 00 00 00 00 00 00 68 |.......`.......h| ++0000000070: 00 00 00 00 00 00 00 70 00 00 00 00 00 00 00 78 |.......p.......x| ++0000000080: 00 00 00 00 00 00 00 80 00 00 00 00 00 00 00 88 |................| ++0000000090: 00 00 00 00 00 00 00 90 00 00 00 00 00 00 00 98 |................| ++00000000a0: 00 00 00 00 00 00 00 a0 00 00 00 00 00 00 00 a8 |................| ++00000000b0: 00 00 00 00 00 00 00 b0 00 00 00 00 00 00 00 b8 |................| ++00000000c0: 00 00 00 00 00 00 00 c0 00 00 00 00 00 00 00 c8 |................| ++00000000d0: 00 00 00 00 00 00 00 d0 00 00 00 00 00 00 00 d8 |................| ++00000000e0: 00 00 00 00 00 00 00 e0 00 00 00 00 00 00 00 e8 |................| ++00000000f0: 00 00 00 00 00 00 00 f0 00 00 00 00 00 00 00 f8 |................| ++0000000100: 00 00 00 00 00 00 01 00 00 00 00 00 00 00 01 08 |................| ++0000000110: 00 00 00 00 00 00 01 10 00 00 00 00 00 00 01 18 |................| ++0000000120: 00 00 00 00 00 00 01 20 00 00 00 |....... ... |' ]; then + echo "$0: unexpected output from nbddump command" + exit 1 + fi +diff --git a/dump/dump.c b/dump/dump.c +index 76af04c..7818f1f 100644 +--- a/dump/dump.c ++++ b/dump/dump.c +@@ -429,10 +429,13 @@ do_dump (void) + else + ansi_grey (); + printf ("%02x ", buffer[j]); ++ if ((j - i) == 7) printf (" "); + } + ansi_grey (); +- for (; j < i+16; ++j) ++ for (; j < i+16; ++j) { + printf (" "); ++ if ((j - i) == 7) printf (" "); ++ } + + /* Print the ASCII codes. */ + printf ("%s", pipe); +-- +2.31.1 + diff --git a/SOURCES/0003-dump-Fix-build-on-i686.patch b/SOURCES/0003-dump-Fix-build-on-i686.patch new file mode 100644 index 0000000..7ec277e --- /dev/null +++ b/SOURCES/0003-dump-Fix-build-on-i686.patch @@ -0,0 +1,38 @@ +From 590e3a010d2c840314702883e44ec9841e3383c6 Mon Sep 17 00:00:00 2001 +From: "Richard W.M. Jones" +Date: Thu, 30 Jun 2022 22:27:43 +0100 +Subject: [PATCH] dump: Fix build on i686 +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +Because we used the wrong printf format, the build would fail on +32 bit architectures but succeed on 64 bit: + +dump.c: In function ‘do_dump’: +dump.c:421:21: error: format ‘%zx’ expects argument of type ‘size_t’, but argument 2 has type ‘uint64_t’ {aka ‘long long unsigned int’} [-Werror=format=] + printf ("%010zx", offset + i); + ~~~~~^ ~~~~~~~~~~ + %010llx + +(cherry picked from commit ce004c329c7fcd6c60d11673b7a5c5ce3414413b) +--- + dump/dump.c | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/dump/dump.c b/dump/dump.c +index 7818f1f..8bf62f9 100644 +--- a/dump/dump.c ++++ b/dump/dump.c +@@ -418,7 +418,7 @@ do_dump (void) + + /* Print the offset. */ + ansi_green (); +- printf ("%010zx", offset + i); ++ printf ("%010" PRIx64, offset + i); + ansi_grey (); + printf (": "); + +-- +2.31.1 + diff --git a/SOURCES/0004-dump-Fix-tests-on-Debian-10.patch b/SOURCES/0004-dump-Fix-tests-on-Debian-10.patch new file mode 100644 index 0000000..cdb908a --- /dev/null +++ b/SOURCES/0004-dump-Fix-tests-on-Debian-10.patch @@ -0,0 +1,41 @@ +From e7a2815412891d5c13b5b5f0e9aa61882880c87f Mon Sep 17 00:00:00 2001 +From: "Richard W.M. Jones" +Date: Thu, 30 Jun 2022 22:31:00 +0100 +Subject: [PATCH] dump: Fix tests on Debian 10 + +The version of nbdkit on Debian 10 does not set $uri. Check for this +or skip the test. + +(cherry picked from commit 083b1ca30fb5e6e0dc0e4b0eea9ebe8474d3f864) +--- + dump/dump-data.sh | 1 + + dump/dump-pattern.sh | 1 + + 2 files changed, 2 insertions(+) + +diff --git a/dump/dump-data.sh b/dump/dump-data.sh +index 955cd3b..46e4d1e 100755 +--- a/dump/dump-data.sh ++++ b/dump/dump-data.sh +@@ -23,6 +23,7 @@ set -x + + requires nbdkit --version + requires nbdkit data --dump-plugin ++requires nbdkit -U - null --run 'test "$uri" != ""' + + output=dump-data.out + rm -f $output +diff --git a/dump/dump-pattern.sh b/dump/dump-pattern.sh +index d512b77..e2188ac 100755 +--- a/dump/dump-pattern.sh ++++ b/dump/dump-pattern.sh +@@ -23,6 +23,7 @@ set -x + + requires nbdkit --version + requires nbdkit pattern --dump-plugin ++requires nbdkit -U - null --run 'test "$uri" != ""' + + output=dump-pattern.out + rm -f $output +-- +2.31.1 + diff --git a/SOURCES/0005-dump-dump-data.sh-Test-requires-nbdkit-1.22.patch b/SOURCES/0005-dump-dump-data.sh-Test-requires-nbdkit-1.22.patch new file mode 100644 index 0000000..d868281 --- /dev/null +++ b/SOURCES/0005-dump-dump-data.sh-Test-requires-nbdkit-1.22.patch @@ -0,0 +1,31 @@ +From 7c669783b1b3fab902ce34d7914b62617ed8b263 Mon Sep 17 00:00:00 2001 +From: "Richard W.M. Jones" +Date: Thu, 30 Jun 2022 22:35:05 +0100 +Subject: [PATCH] dump/dump-data.sh: Test requires nbdkit 1.22 + +Ubuntu 20.04 has nbdkit 1.16 which lacks support for strings. These +were added in nbdkit 1.22. + +(cherry picked from commit a8fa05ffb8b85f41276ffb52498e4528c08e5f21) +--- + dump/dump-data.sh | 4 ++++ + 1 file changed, 4 insertions(+) + +diff --git a/dump/dump-data.sh b/dump/dump-data.sh +index 46e4d1e..11145b0 100755 +--- a/dump/dump-data.sh ++++ b/dump/dump-data.sh +@@ -25,6 +25,10 @@ requires nbdkit --version + requires nbdkit data --dump-plugin + requires nbdkit -U - null --run 'test "$uri" != ""' + ++# This test requires nbdkit >= 1.22. ++minor=$( nbdkit --dump-config | grep ^version_minor | cut -d= -f2 ) ++requires test $minor -ge 22 ++ + output=dump-data.out + rm -f $output + cleanup_fn rm -f $output +-- +2.31.1 + diff --git a/SOURCES/0006-copy-Store-the-preferred-block-size-in-the-operation.patch b/SOURCES/0006-copy-Store-the-preferred-block-size-in-the-operation.patch new file mode 100644 index 0000000..893a026 --- /dev/null +++ b/SOURCES/0006-copy-Store-the-preferred-block-size-in-the-operation.patch @@ -0,0 +1,163 @@ +From 8dce43a3ea7a529bc37cbe5607a8d52186cc8169 Mon Sep 17 00:00:00 2001 +From: "Richard W.M. Jones" +Date: Tue, 28 Jun 2022 18:27:58 +0100 +Subject: [PATCH] copy: Store the preferred block size in the operations struct + +This will be used in a subsequent commit. At the moment the preferred +block size for all sources / destinations is simply calculated and +stored. + +(cherry picked from commit e6c42f8b2d447bbcc659d6dd33be67335834b2e5) +--- + copy/file-ops.c | 4 +++- + copy/main.c | 29 +++++++++++++++++++++++------ + copy/nbd-ops.c | 10 ++++++++++ + copy/nbdcopy.h | 4 +++- + copy/null-ops.c | 1 + + copy/pipe-ops.c | 1 + + 6 files changed, 41 insertions(+), 8 deletions(-) + +diff --git a/copy/file-ops.c b/copy/file-ops.c +index ab37875..34f08e5 100644 +--- a/copy/file-ops.c ++++ b/copy/file-ops.c +@@ -241,13 +241,15 @@ seek_hole_supported (int fd) + + struct rw * + file_create (const char *name, int fd, +- off_t st_size, bool is_block, direction d) ++ off_t st_size, uint64_t preferred, ++ bool is_block, direction d) + { + struct rw_file *rwf = calloc (1, sizeof *rwf); + if (rwf == NULL) { perror ("calloc"); exit (EXIT_FAILURE); } + + rwf->rw.ops = &file_ops; + rwf->rw.name = name; ++ rwf->rw.preferred = preferred; + rwf->fd = fd; + rwf->is_block = is_block; + +diff --git a/copy/main.c b/copy/main.c +index cc379e9..19ec384 100644 +--- a/copy/main.c ++++ b/copy/main.c +@@ -512,10 +512,26 @@ open_local (const char *filename, direction d) + fprintf (stderr, "%s: %s: %m\n", prog, filename); + exit (EXIT_FAILURE); + } +- if (S_ISBLK (stat.st_mode) || S_ISREG (stat.st_mode)) +- return file_create (filename, fd, stat.st_size, S_ISBLK (stat.st_mode), d); +- else { +- /* Probably stdin/stdout, a pipe or a socket. */ ++ if (S_ISREG (stat.st_mode)) /* Regular file. */ ++ return file_create (filename, fd, ++ stat.st_size, (uint64_t) stat.st_blksize, false, d); ++ else if (S_ISBLK (stat.st_mode)) { /* Block device. */ ++ unsigned int blkioopt; ++ ++#ifdef BLKIOOPT ++ if (ioctl (fd, BLKIOOPT, &blkioopt) == -1) { ++ fprintf (stderr, "warning: cannot get optimal I/O size: %s: %m", ++ filename); ++ blkioopt = 4096; ++ } ++#else ++ blkioopt = 4096; ++#endif ++ ++ return file_create (filename, fd, ++ stat.st_size, (uint64_t) blkioopt, true, d); ++ } ++ else { /* Probably stdin/stdout, a pipe or a socket. */ + synchronous = true; /* Force synchronous mode for pipes. */ + return pipe_create (filename, fd); + } +@@ -528,8 +544,9 @@ print_rw (struct rw *rw, const char *prefix, FILE *fp) + char buf[HUMAN_SIZE_LONGEST]; + + fprintf (fp, "%s: %s \"%s\"\n", prefix, rw->ops->ops_name, rw->name); +- fprintf (fp, "%s: size=%" PRIi64 " (%s)\n", +- prefix, rw->size, human_size (buf, rw->size, NULL)); ++ fprintf (fp, "%s: size=%" PRIi64 " (%s), preferred block size=%" PRIu64 "\n", ++ prefix, rw->size, human_size (buf, rw->size, NULL), ++ rw->preferred); + } + + /* Default implementation of rw->ops->get_extents for backends which +diff --git a/copy/nbd-ops.c b/copy/nbd-ops.c +index 3bc26ba..0988634 100644 +--- a/copy/nbd-ops.c ++++ b/copy/nbd-ops.c +@@ -112,12 +112,22 @@ open_one_nbd_handle (struct rw_nbd *rwn) + * the same way. + */ + if (rwn->handles.len == 0) { ++ int64_t block_size; ++ + rwn->can_zero = nbd_can_zero (nbd) > 0; ++ + rwn->rw.size = nbd_get_size (nbd); + if (rwn->rw.size == -1) { + fprintf (stderr, "%s: %s: %s\n", prog, rwn->rw.name, nbd_get_error ()); + exit (EXIT_FAILURE); + } ++ ++ block_size = nbd_get_block_size (nbd, LIBNBD_SIZE_PREFERRED); ++ if (block_size == -1) { ++ fprintf (stderr, "%s: %s: %s\n", prog, rwn->rw.name, nbd_get_error ()); ++ exit (EXIT_FAILURE); ++ } ++ rwn->rw.preferred = block_size == 0 ? 4096 : block_size; + } + + if (handles_append (&rwn->handles, nbd) == -1) { +diff --git a/copy/nbdcopy.h b/copy/nbdcopy.h +index 19797df..9438cce 100644 +--- a/copy/nbdcopy.h ++++ b/copy/nbdcopy.h +@@ -43,6 +43,7 @@ struct rw { + struct rw_ops *ops; /* Operations. */ + const char *name; /* Printable name, for error messages etc. */ + int64_t size; /* May be -1 for streams. */ ++ uint64_t preferred; /* Preferred block size. */ + /* Followed by private data for the particular subtype. */ + }; + +@@ -53,7 +54,8 @@ typedef enum { READING, WRITING } direction; + + /* Create subtypes. */ + extern struct rw *file_create (const char *name, int fd, +- off_t st_size, bool is_block, direction d); ++ off_t st_size, uint64_t preferred, ++ bool is_block, direction d); + extern struct rw *nbd_rw_create_uri (const char *name, + const char *uri, direction d); + extern struct rw *nbd_rw_create_subprocess (const char **argv, size_t argc, +diff --git a/copy/null-ops.c b/copy/null-ops.c +index 1218a62..99cc9a7 100644 +--- a/copy/null-ops.c ++++ b/copy/null-ops.c +@@ -45,6 +45,7 @@ null_create (const char *name) + rw->rw.ops = &null_ops; + rw->rw.name = name; + rw->rw.size = INT64_MAX; ++ rw->rw.preferred = 4096; + return &rw->rw; + } + +diff --git a/copy/pipe-ops.c b/copy/pipe-ops.c +index 3c8b6c2..3815f82 100644 +--- a/copy/pipe-ops.c ++++ b/copy/pipe-ops.c +@@ -43,6 +43,7 @@ pipe_create (const char *name, int fd) + rwp->rw.ops = &pipe_ops; + rwp->rw.name = name; + rwp->rw.size = -1; ++ rwp->rw.preferred = 4096; + rwp->fd = fd; + return &rwp->rw; + } +-- +2.31.1 + diff --git a/SOURCES/0007-copy-Use-preferred-block-size-for-copying.patch b/SOURCES/0007-copy-Use-preferred-block-size-for-copying.patch new file mode 100644 index 0000000..577f8f1 --- /dev/null +++ b/SOURCES/0007-copy-Use-preferred-block-size-for-copying.patch @@ -0,0 +1,457 @@ +From c8626acc63c4ae1c6cf5d1505e0209ac10f44e81 Mon Sep 17 00:00:00 2001 +From: "Richard W.M. Jones" +Date: Tue, 28 Jun 2022 21:58:55 +0100 +Subject: [PATCH] copy: Use preferred block size for copying + +You're not supposed to read or write NBD servers at a granularity less +than the advertised minimum block size. nbdcopy has ignored this +requirement, and this is usually fine because the NBD servers we care +about support 512-byte sector granularity, and never advertise sizes / +extents less granular than sectors (even if it's a bit suboptimal in a +few cases). + +However there is one new case where we do care: When writing to a +compressed qcow2 file, qemu advertises a minimum and preferred block +size of 64K, and it really means it. You cannot write blocks smaller +than this because of the way qcow2 compression is implemented. + +This commit attempts to do the least work possible to fix this. + +The previous multi-thread-copying loop was driven by the extent map +received from the source. I have modified the loop so that it +iterates over request_size blocks. request_size is set from the +command line (--request-size) but will be adjusted upwards if either +the source or destination preferred block size is larger. So this +will always copy blocks which are at least the preferred block size +(except for the very last block of the disk). + +While copying these blocks we consult the source extent map. If it +contains only zero regions covering the whole block (only_zeroes +function) then we can skip straight to zeroing the target +(fill_dst_range_with_zeroes), else we do read + write as before. + +I only modified the multi-thread-copying loop, not the synchronous +loop. That should be updated in the same way later. + +One side effect of this change is it always makes larger requests, +even for regions we know are sparse. This is clear in the +copy-sparse.sh and copy-sparse-allocated.sh tests which were +previously driven by the 32K sparse map granularity of the source. +Without changing these tests, they would make make 256K reads & writes +(and also read from areas of the disk even though we know they are +sparse). I adjusted these tests to use --request-size=32768 to force +the existing behaviour. + +Note this doesn't attempt to limit the maximum block size when reading +or writing. That is for future work. + +This is a partial fix for https://bugzilla.redhat.com/2047660. +Further changes will be required in virt-v2v. + +Link: https://lists.gnu.org/archive/html/qemu-block/2022-01/threads.html#00729 +Link: https://bugzilla.redhat.com/show_bug.cgi?id=2047660 +(cherry picked from commit 4058fe1ff03fb41156b67302ba1006b9d06b0218) +--- + TODO | 4 +- + copy/Makefile.am | 6 +- + copy/copy-file-to-qcow2-compressed.sh | 64 +++++++++++ + copy/copy-sparse-allocated.sh | 4 +- + copy/copy-sparse.sh | 7 +- + copy/main.c | 13 +++ + copy/multi-thread-copying.c | 149 +++++++++++++++++++------- + copy/nbdcopy.pod | 5 +- + 8 files changed, 202 insertions(+), 50 deletions(-) + create mode 100755 copy/copy-file-to-qcow2-compressed.sh + +diff --git a/TODO b/TODO +index 7c9c15e..bc38d70 100644 +--- a/TODO ++++ b/TODO +@@ -28,7 +28,9 @@ Performance: Chart it over various buffer sizes and threads, as that + Examine other fuzzers: https://gitlab.com/akihe/radamsa + + nbdcopy: +- - Minimum/preferred/maximum block size. ++ - Enforce maximum block size. ++ - Synchronous loop should be adjusted to take into account ++ the NBD preferred block size, as was done for multi-thread loop. + - Benchmark. + - Better page cache usage, see nbdkit-file-plugin options + fadvise=sequential cache=none. +diff --git a/copy/Makefile.am b/copy/Makefile.am +index e729f86..25f75c5 100644 +--- a/copy/Makefile.am ++++ b/copy/Makefile.am +@@ -23,6 +23,7 @@ EXTRA_DIST = \ + copy-file-to-nbd.sh \ + copy-file-to-null.sh \ + copy-file-to-qcow2.sh \ ++ copy-file-to-qcow2-compressed.sh \ + copy-nbd-to-block.sh \ + copy-nbd-to-file.sh \ + copy-nbd-to-hexdump.sh \ +@@ -142,7 +143,10 @@ TESTS += \ + $(NULL) + + if HAVE_QEMU_NBD +-TESTS += copy-file-to-qcow2.sh ++TESTS += \ ++ copy-file-to-qcow2.sh \ ++ copy-file-to-qcow2-compressed.sh \ ++ $(NULL) + endif + + if HAVE_GNUTLS +diff --git a/copy/copy-file-to-qcow2-compressed.sh b/copy/copy-file-to-qcow2-compressed.sh +new file mode 100755 +index 0000000..dfe4fa5 +--- /dev/null ++++ b/copy/copy-file-to-qcow2-compressed.sh +@@ -0,0 +1,64 @@ ++#!/usr/bin/env bash ++# nbd client library in userspace ++# Copyright (C) 2020-2022 Red Hat Inc. ++# ++# This library is free software; you can redistribute it and/or ++# modify it under the terms of the GNU Lesser General Public ++# License as published by the Free Software Foundation; either ++# version 2 of the License, or (at your option) any later version. ++# ++# This library is distributed in the hope that it will be useful, ++# but WITHOUT ANY WARRANTY; without even the implied warranty of ++# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ++# Lesser General Public License for more details. ++# ++# You should have received a copy of the GNU Lesser General Public ++# License along with this library; if not, write to the Free Software ++# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA ++ ++. ../tests/functions.sh ++ ++set -e ++set -x ++ ++requires $QEMU_NBD --version ++requires nbdkit --exit-with-parent --version ++requires nbdkit sparse-random --dump-plugin ++requires qemu-img --version ++requires stat --version ++ ++file1=copy-file-to-qcow2-compressed.file1 ++file2=copy-file-to-qcow2-compressed.file2 ++rm -f $file1 $file2 ++cleanup_fn rm -f $file1 $file2 ++ ++size=1G ++seed=$RANDOM ++ ++# Create a compressed qcow2 file1. ++# ++# sparse-random files should compress easily because by default each ++# block uses repeated bytes. ++qemu-img create -f qcow2 $file1 $size ++nbdcopy -- [ nbdkit --exit-with-parent sparse-random $size seed=$seed ] \ ++ [ $QEMU_NBD --image-opts driver=compress,file.driver=qcow2,file.file.driver=file,file.file.filename=$file1 ] ++ ++ls -l $file1 ++ ++# Create an uncompressed qcow2 file2 with the same data. ++qemu-img create -f qcow2 $file2 $size ++nbdcopy -- [ nbdkit --exit-with-parent sparse-random $size seed=$seed ] \ ++ [ $QEMU_NBD --image-opts driver=qcow2,file.driver=file,file.filename=$file2 ] ++ ++ls -l $file2 ++ ++# file1 < file2 (shows the compression is having some effect). ++size1="$( stat -c %s $file1 )" ++size2="$( stat -c %s $file2 )" ++if [ $size1 -ge $size2 ]; then ++ echo "$0: qcow2 compression did not make the file smaller" ++ exit 1 ++fi ++ ++# Logical content of the files should be identical. ++qemu-img compare -f qcow2 $file1 -F qcow2 $file2 +diff --git a/copy/copy-sparse-allocated.sh b/copy/copy-sparse-allocated.sh +index 203c3b9..465e347 100755 +--- a/copy/copy-sparse-allocated.sh ++++ b/copy/copy-sparse-allocated.sh +@@ -17,8 +17,6 @@ + # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + + # Adapted from copy-sparse.sh. +-# +-# This test depends on the nbdkit default sparse block size (32K). + + . ../tests/functions.sh + +@@ -33,7 +31,7 @@ requires nbdkit eval --version + out=copy-sparse-allocated.out + cleanup_fn rm -f $out + +-$VG nbdcopy --allocated -- \ ++$VG nbdcopy --allocated --request-size=32768 -- \ + [ nbdkit --exit-with-parent data data=' + 1 + @1073741823 1 +diff --git a/copy/copy-sparse.sh b/copy/copy-sparse.sh +index 1a6da86..7912a21 100755 +--- a/copy/copy-sparse.sh ++++ b/copy/copy-sparse.sh +@@ -16,8 +16,6 @@ + # License along with this library; if not, write to the Free Software + # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +-# This test depends on the nbdkit default sparse block size (32K). +- + . ../tests/functions.sh + + set -e +@@ -34,8 +32,9 @@ cleanup_fn rm -f $out + # Copy from a sparse data disk to an nbdkit-eval-plugin instance which + # is logging everything. This allows us to see exactly what nbdcopy + # is writing, to ensure it is writing and zeroing the target as +-# expected. +-$VG nbdcopy -S 0 -- \ ++# expected. Force request size to match nbdkit default sparse ++# allocator block size (32K). ++$VG nbdcopy -S 0 --request-size=32768 -- \ + [ nbdkit --exit-with-parent data data=' + 1 + @1073741823 1 +diff --git a/copy/main.c b/copy/main.c +index 19ec384..0e27db8 100644 +--- a/copy/main.c ++++ b/copy/main.c +@@ -40,6 +40,7 @@ + + #include "ispowerof2.h" + #include "human-size.h" ++#include "minmax.h" + #include "version.h" + #include "nbdcopy.h" + +@@ -379,10 +380,22 @@ main (int argc, char *argv[]) + if (threads < connections) + connections = threads; + ++ /* request_size must always be at least as large as the preferred ++ * size of source & destination. ++ */ ++ request_size = MAX (request_size, src->preferred); ++ request_size = MAX (request_size, dst->preferred); ++ + /* Adapt queue to size to request size if needed. */ + if (request_size > queue_size) + queue_size = request_size; + ++ /* Sparse size (if using) must not be smaller than the destination ++ * preferred size, otherwise we end up creating too small requests. ++ */ ++ if (sparse_size > 0 && sparse_size < dst->preferred) ++ sparse_size = dst->preferred; ++ + /* Truncate the destination to the same size as the source. Only + * has an effect on regular files. + */ +diff --git a/copy/multi-thread-copying.c b/copy/multi-thread-copying.c +index 06cdb8e..9267545 100644 +--- a/copy/multi-thread-copying.c ++++ b/copy/multi-thread-copying.c +@@ -166,6 +166,62 @@ decrease_queue_size (struct worker *worker, size_t len) + worker->queue_size -= len; + } + ++/* Using the extents map 'exts', check if the region ++ * [offset..offset+len-1] intersects only with zero extents. ++ * ++ * The invariant for '*i' is always an extent which starts before or ++ * equal to the current offset. ++ */ ++static bool ++only_zeroes (const extent_list exts, size_t *i, ++ uint64_t offset, unsigned len) ++{ ++ size_t j; ++ ++ /* Invariant. */ ++ assert (*i < exts.len); ++ assert (exts.ptr[*i].offset <= offset); ++ ++ /* Update the invariant. Search for the last possible extent in the ++ * list which is <= offset. ++ */ ++ for (j = *i + 1; j < exts.len; ++j) { ++ if (exts.ptr[j].offset <= offset) ++ *i = j; ++ else ++ break; ++ } ++ ++ /* Check invariant again. */ ++ assert (*i < exts.len); ++ assert (exts.ptr[*i].offset <= offset); ++ ++ /* If *i is not the last extent, then the next extent starts ++ * strictly beyond our current offset. ++ */ ++ assert (*i == exts.len - 1 || exts.ptr[*i + 1].offset > offset); ++ ++ /* Search forward, look for any non-zero extents overlapping the region. */ ++ for (j = *i; j < exts.len; ++j) { ++ uint64_t start, end; ++ ++ /* [start..end-1] is the current extent. */ ++ start = exts.ptr[j].offset; ++ end = exts.ptr[j].offset + exts.ptr[j].length; ++ ++ assert (end > offset); ++ ++ if (start >= offset + len) ++ break; ++ ++ /* Non-zero extent covering this region => test failed. */ ++ if (!exts.ptr[j].zero) ++ return false; ++ } ++ ++ return true; ++} ++ + /* There are 'threads' worker threads, each copying work ranges from + * src to dst until there are no more work ranges. + */ +@@ -177,7 +233,10 @@ worker_thread (void *wp) + extent_list exts = empty_vector; + + while (get_next_offset (&offset, &count)) { +- size_t i; ++ struct command *command; ++ size_t extent_index; ++ bool is_zeroing = false; ++ uint64_t zeroing_start = 0; /* initialized to avoid bogus GCC warning */ + + assert (0 < count && count <= THREAD_WORK_SIZE); + if (extents) +@@ -185,52 +244,64 @@ worker_thread (void *wp) + else + default_get_extents (src, w->index, offset, count, &exts); + +- for (i = 0; i < exts.len; ++i) { +- struct command *command; +- size_t len; ++ extent_index = 0; // index into extents array used to optimize only_zeroes ++ while (count) { ++ const size_t len = MIN (count, request_size); + +- if (exts.ptr[i].zero) { ++ if (only_zeroes (exts, &extent_index, offset, len)) { + /* The source is zero so we can proceed directly to skipping, +- * fast zeroing, or writing zeroes at the destination. ++ * fast zeroing, or writing zeroes at the destination. Defer ++ * zeroing so we can send it as a single large command. + */ +- command = create_command (exts.ptr[i].offset, exts.ptr[i].length, +- true, w); +- fill_dst_range_with_zeroes (command); ++ if (!is_zeroing) { ++ is_zeroing = true; ++ zeroing_start = offset; ++ } + } +- + else /* data */ { +- /* As the extent might be larger than permitted for a single +- * command, we may have to split this into multiple read +- * requests. +- */ +- while (exts.ptr[i].length > 0) { +- len = exts.ptr[i].length; +- if (len > request_size) +- len = request_size; +- +- command = create_command (exts.ptr[i].offset, len, +- false, w); +- +- wait_for_request_slots (w); +- +- /* NOTE: Must increase the queue size after waiting. */ +- increase_queue_size (w, len); +- +- /* Begin the asynch read operation. */ +- src->ops->asynch_read (src, command, +- (nbd_completion_callback) { +- .callback = finished_read, +- .user_data = command, +- }); +- +- exts.ptr[i].offset += len; +- exts.ptr[i].length -= len; ++ /* If we were in the middle of deferred zeroing, do it now. */ ++ if (is_zeroing) { ++ /* Note that offset-zeroing_start can never exceed ++ * THREAD_WORK_SIZE, so there is no danger of overflowing ++ * size_t. ++ */ ++ command = create_command (zeroing_start, offset-zeroing_start, ++ true, w); ++ fill_dst_range_with_zeroes (command); ++ is_zeroing = false; + } ++ ++ /* Issue the asynchronous read command. */ ++ command = create_command (offset, len, false, w); ++ ++ wait_for_request_slots (w); ++ ++ /* NOTE: Must increase the queue size after waiting. */ ++ increase_queue_size (w, len); ++ ++ /* Begin the asynch read operation. */ ++ src->ops->asynch_read (src, command, ++ (nbd_completion_callback) { ++ .callback = finished_read, ++ .user_data = command, ++ }); + } + +- offset += count; +- count = 0; +- } /* for extents */ ++ offset += len; ++ count -= len; ++ } /* while (count) */ ++ ++ /* If we were in the middle of deferred zeroing, do it now. */ ++ if (is_zeroing) { ++ /* Note that offset-zeroing_start can never exceed ++ * THREAD_WORK_SIZE, so there is no danger of overflowing ++ * size_t. ++ */ ++ command = create_command (zeroing_start, offset - zeroing_start, ++ true, w); ++ fill_dst_range_with_zeroes (command); ++ is_zeroing = false; ++ } + } + + /* Wait for in flight NBD requests to finish. */ +diff --git a/copy/nbdcopy.pod b/copy/nbdcopy.pod +index fd10f7c..f06d112 100644 +--- a/copy/nbdcopy.pod ++++ b/copy/nbdcopy.pod +@@ -182,8 +182,9 @@ Set the maximum number of requests in flight per NBD connection. + =item B<--sparse=>N + + Detect all zero blocks of size N (bytes) and make them sparse on the +-output. You can also turn off sparse detection using S>. +-The default is 4096 bytes. ++output. You can also turn off sparse detection using S>. The ++default is 4096 bytes, or the destination preferred block size, ++whichever is larger. + + =item B<--synchronous> + +-- +2.31.1 + diff --git a/SOURCES/0008-dump-Add-another-example-to-the-manual.patch b/SOURCES/0008-dump-Add-another-example-to-the-manual.patch new file mode 100644 index 0000000..1a4eb5b --- /dev/null +++ b/SOURCES/0008-dump-Add-another-example-to-the-manual.patch @@ -0,0 +1,29 @@ +From 5d21b00dbdd1e1a04317bf16afb8f4d2ceaa470f Mon Sep 17 00:00:00 2001 +From: "Richard W.M. Jones" +Date: Sat, 2 Jul 2022 17:12:46 +0100 +Subject: [PATCH] dump: Add another example to the manual + +(cherry picked from commit be3768b077c9542aba34eb821016c36f31d234af) +--- + dump/nbddump.pod | 5 +++++ + 1 file changed, 5 insertions(+) + +diff --git a/dump/nbddump.pod b/dump/nbddump.pod +index 5d7864d..656a965 100644 +--- a/dump/nbddump.pod ++++ b/dump/nbddump.pod +@@ -57,6 +57,11 @@ For example, to dump out a qcow2 file as raw data: + + nbddump -- [ qemu-nbd -r -f qcow2 file.qcow2 ] + ++To dump out an empty floppy disk created by L: ++ ++ mkdir /var/tmp/empty ++ nbddump -- [ nbdkit floppy /var/tmp/empty ] ++ + Note that S> are separate parameters, and must be + surrounded by spaces. C<--> separates nbddump parameters from + subprocess parameters. +-- +2.31.1 + diff --git a/SOURCES/0009-lib-crypto-Use-GNUTLS_NO_SIGNAL-if-available.patch b/SOURCES/0009-lib-crypto-Use-GNUTLS_NO_SIGNAL-if-available.patch new file mode 100644 index 0000000..e33ee16 --- /dev/null +++ b/SOURCES/0009-lib-crypto-Use-GNUTLS_NO_SIGNAL-if-available.patch @@ -0,0 +1,93 @@ +From a432e773e0cdc24cb27ccdda4111744ea2c3b819 Mon Sep 17 00:00:00 2001 +From: "Richard W.M. Jones" +Date: Wed, 27 Jul 2022 17:08:14 +0100 +Subject: [PATCH] lib/crypto: Use GNUTLS_NO_SIGNAL if available + +libnbd has long used MSG_NOSIGNAL to avoid receiving SIGPIPE if we +accidentally write on a closed socket, which is a nice alternative to +using a SIGPIPE signal handler. However with TLS connections, gnutls +did not use this flag and so programs using libnbd + TLS would receive +SIGPIPE in some situations, notably if the server closed the +connection abruptly while we were trying to write something. + +GnuTLS 3.4.2 introduces GNUTLS_NO_SIGNAL which does the same thing. +Use this flag if available. + +RHEL 7 has an older gnutls which lacks this flag. To avoid qemu-nbd +interop tests failing (rarely, but more often with a forthcoming +change to TLS shutdown behaviour), register a SIGPIPE signal handler +in the test if the flag is missing. +--- + configure.ac | 15 +++++++++++++++ + interop/interop.c | 10 ++++++++++ + lib/crypto.c | 7 ++++++- + 3 files changed, 31 insertions(+), 1 deletion(-) + +diff --git a/configure.ac b/configure.ac +index 49ca8ab..6bd9e1b 100644 +--- a/configure.ac ++++ b/configure.ac +@@ -179,6 +179,21 @@ AS_IF([test "$GNUTLS_LIBS" != ""],[ + gnutls_session_set_verify_cert \ + gnutls_transport_is_ktls_enabled \ + ]) ++ AC_MSG_CHECKING([if gnutls has GNUTLS_NO_SIGNAL]) ++ AC_COMPILE_IFELSE( ++ [AC_LANG_PROGRAM([ ++ #include ++ gnutls_session_t session; ++ ], [ ++ gnutls_init(&session, GNUTLS_CLIENT|GNUTLS_NO_SIGNAL); ++ ]) ++ ], [ ++ AC_MSG_RESULT([yes]) ++ AC_DEFINE([HAVE_GNUTLS_NO_SIGNAL], [1], ++ [GNUTLS_NO_SIGNAL found at compile time]) ++ ], [ ++ AC_MSG_RESULT([no]) ++ ]) + LIBS="$old_LIBS" + ]) + +diff --git a/interop/interop.c b/interop/interop.c +index b41f3ca..036545b 100644 +--- a/interop/interop.c ++++ b/interop/interop.c +@@ -84,6 +84,16 @@ main (int argc, char *argv[]) + REQUIRES + #endif + ++ /* Ignore SIGPIPE. We only need this for GnuTLS < 3.4.2, since ++ * newer GnuTLS has the GNUTLS_NO_SIGNAL flag which adds ++ * MSG_NOSIGNAL to each write call. ++ */ ++#if !HAVE_GNUTLS_NO_SIGNAL ++#if TLS ++ signal (SIGPIPE, SIG_IGN); ++#endif ++#endif ++ + /* Create a large sparse temporary file. */ + #ifdef NEEDS_TMPFILE + int fd = mkstemp (TMPFILE); +diff --git a/lib/crypto.c b/lib/crypto.c +index 1272888..ca9520e 100644 +--- a/lib/crypto.c ++++ b/lib/crypto.c +@@ -588,7 +588,12 @@ nbd_internal_crypto_create_session (struct nbd_handle *h, + gnutls_psk_client_credentials_t pskcreds = NULL; + gnutls_certificate_credentials_t xcreds = NULL; + +- err = gnutls_init (&session, GNUTLS_CLIENT|GNUTLS_NONBLOCK); ++ err = gnutls_init (&session, ++ GNUTLS_CLIENT | GNUTLS_NONBLOCK ++#if HAVE_GNUTLS_NO_SIGNAL ++ | GNUTLS_NO_SIGNAL ++#endif ++ ); + if (err < 0) { + set_error (errno, "gnutls_init: %s", gnutls_strerror (err)); + return NULL; +-- +2.31.1 + diff --git a/SOURCES/0010-lib-crypto.c-Ignore-TLS-premature-termination-after-.patch b/SOURCES/0010-lib-crypto.c-Ignore-TLS-premature-termination-after-.patch new file mode 100644 index 0000000..cc91332 --- /dev/null +++ b/SOURCES/0010-lib-crypto.c-Ignore-TLS-premature-termination-after-.patch @@ -0,0 +1,100 @@ +From 8bbee9c0ff052cf8ab5ba81fd1b67e3c45e7012a Mon Sep 17 00:00:00 2001 +From: "Richard W.M. Jones" +Date: Wed, 27 Jul 2022 16:07:37 +0100 +Subject: [PATCH] lib/crypto.c: Ignore TLS premature termination after write + shutdown + +qemu-nbd doesn't call gnutls_bye to cleanly shut down the connection +after we send NBD_CMD_DISC. When copying from a qemu-nbd server (or +any operation which calls nbd_shutdown) you will see errors like this: + + $ nbdcopy nbds://foo?tls-certificates=/var/tmp/pki null: + nbds://foo?tls-certificates=/var/tmp/pki: nbd_shutdown: gnutls_record_recv: The TLS connection was non-properly terminated. + +Relatedly you may also see: + + nbd_shutdown: gnutls_record_recv: Error in the pull function. + +This commit suppresses the error in the case where we know that we +have shut down writes (which happens after NBD_CMD_DISC has been sent +on the wire). +--- + interop/interop.c | 9 --------- + lib/crypto.c | 17 +++++++++++++++++ + lib/internal.h | 1 + + 3 files changed, 18 insertions(+), 9 deletions(-) + +diff --git a/interop/interop.c b/interop/interop.c +index 036545b..cce9407 100644 +--- a/interop/interop.c ++++ b/interop/interop.c +@@ -226,19 +226,10 @@ main (int argc, char *argv[]) + + /* XXX In future test more operations here. */ + +-#if !TLS +- /* XXX qemu doesn't shut down the connection nicely (using +- * gnutls_bye) and because of this the following call will fail +- * with: +- * +- * nbd_shutdown: gnutls_record_recv: The TLS connection was +- * non-properly terminated. +- */ + if (nbd_shutdown (nbd, 0) == -1) { + fprintf (stderr, "%s\n", nbd_get_error ()); + exit (EXIT_FAILURE); + } +-#endif + + nbd_close (nbd); + +diff --git a/lib/crypto.c b/lib/crypto.c +index ca9520e..aa5d820 100644 +--- a/lib/crypto.c ++++ b/lib/crypto.c +@@ -187,6 +187,22 @@ tls_recv (struct nbd_handle *h, struct socket *sock, void *buf, size_t len) + errno = EAGAIN; + return -1; + } ++ if (h->tls_shut_writes && ++ (r == GNUTLS_E_PULL_ERROR || r == GNUTLS_E_PREMATURE_TERMINATION)) { ++ /* qemu-nbd doesn't call gnutls_bye to cleanly shut down the ++ * connection after we send NBD_CMD_DISC, instead it simply ++ * closes the connection. On the client side we see ++ * "gnutls_record_recv: The TLS connection was non-properly ++ * terminated" or "gnutls_record_recv: Error in the pull ++ * function.". ++ * ++ * If we see these errors after we shut down the write side ++ * (h->tls_shut_writes), which happens after we have sent ++ * NBD_CMD_DISC on the wire, downgrade them to a debug message. ++ */ ++ debug (h, "gnutls_record_recv: %s", gnutls_strerror (r)); ++ return 0; /* EOF */ ++ } + set_error (0, "gnutls_record_recv: %s", gnutls_strerror (r)); + errno = EIO; + return -1; +@@ -234,6 +250,7 @@ tls_shut_writes (struct nbd_handle *h, struct socket *sock) + return false; + if (r != 0) + debug (h, "ignoring gnutls_bye failure: %s", gnutls_strerror (r)); ++ h->tls_shut_writes = true; + return sock->u.tls.oldsock->ops->shut_writes (h, sock->u.tls.oldsock); + } + +diff --git a/lib/internal.h b/lib/internal.h +index 6aaced3..f1b4c63 100644 +--- a/lib/internal.h ++++ b/lib/internal.h +@@ -307,6 +307,7 @@ struct nbd_handle { + struct command *reply_cmd; + + bool disconnect_request; /* True if we've queued NBD_CMD_DISC */ ++ bool tls_shut_writes; /* Used by lib/crypto.c to track disconnect. */ + }; + + struct meta_context { +-- +2.31.1 + diff --git a/SOURCES/copy-patches.sh b/SOURCES/copy-patches.sh index 83867ef..65d5933 100755 --- a/SOURCES/copy-patches.sh +++ b/SOURCES/copy-patches.sh @@ -6,7 +6,7 @@ set -e # directory. Use it like this: # ./copy-patches.sh -rhel_version=9.0 +rhel_version=9.1 # Check we're in the right directory. if [ ! -f libnbd.spec ]; then diff --git a/SOURCES/libnbd-1.10.5.tar.gz.sig b/SOURCES/libnbd-1.10.5.tar.gz.sig deleted file mode 100644 index 39755f0..0000000 --- a/SOURCES/libnbd-1.10.5.tar.gz.sig +++ /dev/null @@ -1,17 +0,0 @@ ------BEGIN PGP SIGNATURE----- - -iQJFBAABCAAvFiEE93dPsa0HSn6Mh2fqkXOPc+G3aKAFAmIFM4YRHHJpY2hAYW5u -ZXhpYS5vcmcACgkQkXOPc+G3aKDrPhAAjwTeg6XxTtPAbBFqwgYeux742i4ufnrL -xdCQgtmyF4jFhW6E0q/dYKr32iUxL6BabswcbmHHhjU4XEa16mSPbkbYAMNvsPJa -FpzW4KMJh+vHCarqm3UFiBwVKEEu+VXbOVh9bVjpCdJRx+jE6hbr5tlR9CHQuNQY -3LoZ7YU+QvQyIVQDOzDSD/8swyfOeX1c4D4Wok0w9qgHsT54GQmT4VTQ93Z/8RXK -D3vCNOt2J+bnJx9WNkboNIfS/FI4L6j4TpuB7tucJlk8Wtfj7LI526LBZwaLYO8E -MH63xr0dvbSA19kNY8M3Sff+YqOEKcufZREt5pX6LJLM/ARXKc0KwmaXR+U2zoNy -gFomzICsvhGusP6mgLc60VYSUO+od9qZDYsmGZ0mtNL18ISKqIjRRxEXWG6z6mT7 -kkbifoZC4tOQqKzvswzlWb0upaC5IOju8tGSwpuZosoVcAs63CmU71/CIA7CT4s0 -qUw+g0ISJYXMFGp3gNzkie9d/a8fAhaCYQKuSxtb5bCNaFegc/6djLy5/MAa65x0 -difLH3sHYEtB4CzPQnQ8wM5khZ/D789CtqGFtI4fz1aMShW0AJfnsUU8DtAi9pAg -3wcjR0N8wQLq68Evj0+VO2rzq/2FSKaR9P7r2cobHC+ucAdvZRAzWOMcHIzfszkN -vMYr5xX3G8o= -=1qAE ------END PGP SIGNATURE----- diff --git a/SOURCES/libnbd-1.12.6.tar.gz.sig b/SOURCES/libnbd-1.12.6.tar.gz.sig new file mode 100644 index 0000000..4726053 --- /dev/null +++ b/SOURCES/libnbd-1.12.6.tar.gz.sig @@ -0,0 +1,17 @@ +-----BEGIN PGP SIGNATURE----- + +iQJFBAABCAAvFiEE93dPsa0HSn6Mh2fqkXOPc+G3aKAFAmLhNK4RHHJpY2hAYW5u +ZXhpYS5vcmcACgkQkXOPc+G3aKDzmxAArFrR/cOyqyGZXuYORFRVi7AobCjum4dP +A93R43shnSXXB1CTww5O+LjIghSLs4TEQAOcmcsjsE98X2cz0BuW6gIfGxTpN3WP +fGPDlvezLXGo5zX5WGFkP6oQY97TuGHXKNxStZtWRtDNfrWPWJQuwlm5GSIHdYYr +dFssmDNtIoh/zQz2www9JKspMfehFbTGZswtRjfDwa2Pl69cMy3pH/k4EZZnDx9n +tguzQHOapJJx8RkIwUwFirCBOwdVNbLX+KrGroLcB6MjO6Uhh2C/iYUQDM/xX2r6 +1SugssAmXwZ4/RIDtwBLdQdEoNjSAV7OW1yizTl5P9qFkiPr+Lpnpt4gci9bTZKK +oIy0RtgJOgW5R2tuRlMXkx/7kcGjhUb3Zbux7d7dgrYq16FUPC1dFgub7WSPSqWe ++17iUD+n3NO2MHtR215nKDjuPR59wnvATO6QS+InOb4imyf47Ic7TeEvVuDhb+M3 ++DvIor2WyXWX9kO165Fx9jAicgZJt2L1UvKM1GzGjwBdL0GYbCFltzzhyi4Pd/7x +ijV2QYbyOLXYEpsgmKYrT6MwwXUAye2PkXSv8MaOs3IiFz0bVy3Bfgx4UYNbKo1x +zwVGtIBz39tXpgyS7+F9rvQILVmENGmyTx+GXrv/lE1mFmTt/EEyf+iHMSN1lIkt +59o9LBpOajI= +=idnt +-----END PGP SIGNATURE----- diff --git a/SPECS/libnbd.spec b/SPECS/libnbd.spec index b1800b5..06155a2 100644 --- a/SPECS/libnbd.spec +++ b/SPECS/libnbd.spec @@ -1,3 +1,6 @@ +# Do this until the feature is fixed in Fedora. +%undefine _package_note_flags + # If we should verify tarball signature with GPGv2. %global verify_tarball_signature 1 @@ -5,10 +8,10 @@ %global patches_touch_autotools 1 # The source directory. -%global source_directory 1.10-stable +%global source_directory 1.12-stable Name: libnbd -Version: 1.10.5 +Version: 1.12.6 Release: 1%{?dist} Summary: NBD client library in userspace @@ -26,10 +29,19 @@ Source2: libguestfs.keyring Source3: copy-patches.sh # Patches are stored in the upstream repository: -# https://gitlab.com/nbdkit/libnbd/-/commits/rhel-9.0/ +# https://gitlab.com/nbdkit/libnbd/-/commits/rhel-9.1/ # Patches. -Patch0001: 0001-api-Add-new-API-nbd_set_pread_initialize.patch +Patch0001: 0001-Add-nbddump-tool.patch +Patch0002: 0002-dump-Visually-separate-columns-0-7-and-8-15.patch +Patch0003: 0003-dump-Fix-build-on-i686.patch +Patch0004: 0004-dump-Fix-tests-on-Debian-10.patch +Patch0005: 0005-dump-dump-data.sh-Test-requires-nbdkit-1.22.patch +Patch0006: 0006-copy-Store-the-preferred-block-size-in-the-operation.patch +Patch0007: 0007-copy-Use-preferred-block-size-for-copying.patch +Patch0008: 0008-dump-Add-another-example-to-the-manual.patch +Patch0009: 0009-lib-crypto-Use-GNUTLS_NO_SIGNAL-if-available.patch +Patch0010: 0010-lib-crypto.c-Ignore-TLS-premature-termination-after-.patch %if 0%{patches_touch_autotools} BuildRequires: autoconf, automake, libtool @@ -260,9 +272,11 @@ make %{?_smp_mflags} check || { %doc README %license COPYING.LIB %{_bindir}/nbdcopy +%{_bindir}/nbddump %{_bindir}/nbdinfo %{_libdir}/libnbd.so.* %{_mandir}/man1/nbdcopy.1* +%{_mandir}/man1/nbddump.1* %{_mandir}/man1/nbdinfo.1* @@ -317,12 +331,23 @@ make %{?_smp_mflags} check || { %files bash-completion %dir %{_datadir}/bash-completion/completions %{_datadir}/bash-completion/completions/nbdcopy +%{_datadir}/bash-completion/completions/nbddump %{_datadir}/bash-completion/completions/nbdfuse %{_datadir}/bash-completion/completions/nbdinfo %{_datadir}/bash-completion/completions/nbdsh %changelog +* Thu Jul 28 2022 Richard W.M. Jones - 1.12.6-1 +- Rebase to new stable branch version 1.12.6 + resolves: rhbz#2059288 +- New tool: nbddump +- nbdcopy: Use preferred block size for copying + related: rhbz#2047660 +- Fix remote TLS failures + resolves: rhbz#2111524 + (and 2111813) + * Thu Feb 10 2022 Richard W.M. Jones - 1.10.5-1 - Rebase to new stable branch version 1.10.5 resolves: rhbz#2011708