diff --git a/.gitignore b/.gitignore index f5789db..040f5ab 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,3 @@ -SOURCES/libguestfs.keyring -SOURCES/libnbd-1.6.0.tar.gz -/libnbd-1.6.0.tar.gz +*~ +/libnbd-*.tar.gz +/libnbd-*.tar.gz.sig diff --git a/0001-Add-nbddump-tool.patch b/0001-Add-nbddump-tool.patch deleted file mode 100644 index 5600ba9..0000000 --- a/0001-Add-nbddump-tool.patch +++ /dev/null @@ -1,1223 +0,0 @@ -From 90fd39da16256407b9229cd17a830739b03629d6 Mon Sep 17 00:00:00 2001 -From: "Richard W.M. Jones" -Date: Thu, 30 Jun 2022 09:07:27 +0100 -Subject: [PATCH] Add nbddump tool - -You can already do this operation using: - - nbdcopy -- $uri - | hexdump -C - -but that is slow, especially for large, sparse disks. This tool uses -sparseness information to skip zero sections of the disk, and it has -nice colourized output. - -(cherry picked from commit c4107b9a40d6451630dcccf1bf6596c8e56420be) ---- - .gitignore | 3 + - Makefile.am | 1 + - README | 2 + - bash-completion/Makefile.am | 9 +- - bash-completion/nbdsh | 6 + - configure.ac | 1 + - copy/nbdcopy.pod | 3 +- - docs/libnbd.pod | 1 + - dump/Makefile.am | 79 ++++++ - dump/dump-data.sh | 57 +++++ - dump/dump-empty-qcow2.sh | 46 ++++ - dump/dump-pattern.sh | 56 +++++ - dump/dump.c | 464 ++++++++++++++++++++++++++++++++++++ - dump/nbddump.pod | 116 +++++++++ - dump/test-long-options.sh | 35 +++ - dump/test-short-options.sh | 35 +++ - dump/test-version.sh | 33 +++ - fuse/nbdfuse.pod | 1 + - info/nbdinfo.pod | 1 + - run.in | 1 + - sh/nbdsh.pod | 1 + - 21 files changed, 947 insertions(+), 4 deletions(-) - create mode 100644 dump/Makefile.am - create mode 100755 dump/dump-data.sh - create mode 100755 dump/dump-empty-qcow2.sh - create mode 100755 dump/dump-pattern.sh - create mode 100644 dump/dump.c - create mode 100644 dump/nbddump.pod - create mode 100755 dump/test-long-options.sh - create mode 100755 dump/test-short-options.sh - create mode 100755 dump/test-version.sh - -diff --git a/.gitignore b/.gitignore -index 498eabc..3771655 100644 ---- a/.gitignore -+++ b/.gitignore -@@ -29,6 +29,7 @@ Makefile.in - /aclocal.m4 - /autom4te.cache - /bash-completion/nbdcopy -+/bash-completion/nbddump - /bash-completion/nbdfuse - /bash-completion/nbdinfo - /common/include/test-array-size -@@ -57,6 +58,8 @@ Makefile.in - !/docs/nbd_close.3 - !/docs/nbd_create.pod - !/docs/nbd_get_err??.3 -+/dump/nbddump -+/dump/nbddump.1 - /examples/aio-connect-read - /examples/batched-read-write - /examples/connect-command -diff --git a/Makefile.am b/Makefile.am -index 303b95c..9e7a281 100644 ---- a/Makefile.am -+++ b/Makefile.am -@@ -46,6 +46,7 @@ SUBDIRS = \ - sh \ - info \ - copy \ -+ dump \ - fuse \ - ocaml \ - ocaml/examples \ -diff --git a/README b/README -index e50c19d..4f9298e 100644 ---- a/README -+++ b/README -@@ -23,6 +23,8 @@ The key features are: - - * Copying tool (nbdcopy) for high performance copying and streaming. - -+ * Hexdump tool (nbddump) to print NBD content. -+ - * Query tool (nbdinfo) to query NBD servers. - - * FUSE support (nbdfuse) to mount NBD in the local filesystem. -diff --git a/bash-completion/Makefile.am b/bash-completion/Makefile.am -index 41d7b13..cab8ffb 100644 ---- a/bash-completion/Makefile.am -+++ b/bash-completion/Makefile.am -@@ -24,17 +24,20 @@ EXTRA_DIST = \ - - if HAVE_BASH_COMPLETION - --bashcomp_DATA = nbdfuse nbdsh -+bashcomp_DATA = nbddump nbdfuse nbdsh - - if HAVE_LIBXML2 - bashcomp_DATA += nbdcopy nbdinfo - endif HAVE_LIBXML2 - -- - nbdcopy: nbdsh - rm -f $@ - $(LN_S) $(srcdir)/nbdsh $@ - -+nbddump: nbdsh -+ rm -f $@ -+ $(LN_S) $(srcdir)/nbdsh $@ -+ - nbdfuse: nbdsh - rm -f $@ - $(LN_S) $(srcdir)/nbdsh $@ -@@ -43,6 +46,6 @@ nbdinfo: nbdsh - rm -f $@ - $(LN_S) $(srcdir)/nbdsh $@ - --CLEANFILES += nbdcopy nbdfuse nbdinfo -+CLEANFILES += nbdcopy nbddump nbdfuse nbdinfo - - endif -diff --git a/bash-completion/nbdsh b/bash-completion/nbdsh -index a740be9..a342003 100644 ---- a/bash-completion/nbdsh -+++ b/bash-completion/nbdsh -@@ -47,6 +47,11 @@ _nbdcopy () - _libnbd_command nbdcopy - } - -+_nbddump () -+{ -+ _libnbd_command nbddump -+} -+ - _nbdfuse () - { - _libnbd_command nbdfuse -@@ -64,6 +69,7 @@ _nbdsh () - - # Install the handler function. - complete -o default -F _nbdcopy nbdcopy -+complete -o default -F _nbddump nbddump - complete -o default -F _nbdfuse nbdfuse - complete -o default -F _nbdinfo nbdinfo - complete -o default -F _nbdsh nbdsh -diff --git a/configure.ac b/configure.ac -index b1bfaac..49ca8ab 100644 ---- a/configure.ac -+++ b/configure.ac -@@ -574,6 +574,7 @@ AC_CONFIG_FILES([Makefile - common/utils/Makefile - copy/Makefile - docs/Makefile -+ dump/Makefile - examples/Makefile - fuse/Makefile - fuzzing/Makefile -diff --git a/copy/nbdcopy.pod b/copy/nbdcopy.pod -index 7fe3fd1..fd10f7c 100644 ---- a/copy/nbdcopy.pod -+++ b/copy/nbdcopy.pod -@@ -285,7 +285,7 @@ Some examples follow. - In this example, L is run as a subprocess. The - subprocess opens F and exposes it as NBD to nbdcopy. - nbdcopy streams this to stdout (C<->) into the pipe which is read by --L. -+L. (See also L) - - =head2 nbdcopy -- [ qemu-nbd -f qcow2 disk.qcow2 ] [ nbdkit memory 1G ] - -@@ -299,6 +299,7 @@ so this command has no overall effect, but is useful for testing. - =head1 SEE ALSO - - L, -+L, - L, - L, - L, -diff --git a/docs/libnbd.pod b/docs/libnbd.pod -index 13facc6..076cafb 100644 ---- a/docs/libnbd.pod -+++ b/docs/libnbd.pod -@@ -1044,6 +1044,7 @@ L. - - L, - L, -+L, - L, - L, - L, -diff --git a/dump/Makefile.am b/dump/Makefile.am -new file mode 100644 -index 0000000..9fd4fed ---- /dev/null -+++ b/dump/Makefile.am -@@ -0,0 +1,79 @@ -+# nbd client library in userspace -+# Copyright (C) 2020-2022 Red Hat Inc. -+# -+# This library is free software; you can redistribute it and/or -+# modify it under the terms of the GNU Lesser General Public -+# License as published by the Free Software Foundation; either -+# version 2 of the License, or (at your option) any later version. -+# -+# This library is distributed in the hope that it will be useful, -+# but WITHOUT ANY WARRANTY; without even the implied warranty of -+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -+# Lesser General Public License for more details. -+# -+# You should have received a copy of the GNU Lesser General Public -+# License along with this library; if not, write to the Free Software -+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA -+ -+include $(top_srcdir)/subdir-rules.mk -+ -+EXTRA_DIST = \ -+ dump-data.sh \ -+ dump-empty-qcow2.sh \ -+ dump-pattern.sh \ -+ nbddump.pod \ -+ test-long-options.sh \ -+ test-short-options.sh \ -+ test-version.sh \ -+ $(NULL) -+ -+bin_PROGRAMS = nbddump -+ -+nbddump_SOURCES = \ -+ dump.c \ -+ $(NULL) -+nbddump_CPPFLAGS = \ -+ -I$(top_srcdir)/include \ -+ -I$(top_srcdir)/common/include \ -+ -I$(top_srcdir)/common/utils \ -+ $(NULL) -+nbddump_CFLAGS = \ -+ $(WARNINGS_CFLAGS) \ -+ $(NULL) -+nbddump_LDADD = \ -+ $(top_builddir)/common/utils/libutils.la \ -+ $(top_builddir)/lib/libnbd.la \ -+ $(NULL) -+ -+if HAVE_POD -+ -+man_MANS = \ -+ nbddump.1 \ -+ $(NULL) -+ -+nbddump.1: nbddump.pod $(top_builddir)/podwrapper.pl -+ $(PODWRAPPER) --section=1 --man $@ \ -+ --html $(top_builddir)/html/$@.html \ -+ $< -+ -+endif HAVE_POD -+ -+TESTS_ENVIRONMENT = \ -+ LIBNBD_DEBUG=1 \ -+ $(MALLOC_CHECKS) \ -+ EXPECTED_VERSION=$(VERSION) \ -+ QEMU_NBD=$(QEMU_NBD) \ -+ $(NULL) -+LOG_COMPILER = $(top_builddir)/run -+ -+TESTS = \ -+ dump-data.sh \ -+ dump-empty-qcow2.sh \ -+ dump-pattern.sh \ -+ test-long-options.sh \ -+ test-short-options.sh \ -+ test-version.sh \ -+ $(NULL) -+ -+check-valgrind: -+ LIBNBD_VALGRIND=1 $(MAKE) check -diff --git a/dump/dump-data.sh b/dump/dump-data.sh -new file mode 100755 -index 0000000..23d09da ---- /dev/null -+++ b/dump/dump-data.sh -@@ -0,0 +1,57 @@ -+#!/usr/bin/env bash -+# nbd client library in userspace -+# Copyright (C) 2020-2022 Red Hat Inc. -+# -+# This library is free software; you can redistribute it and/or -+# modify it under the terms of the GNU Lesser General Public -+# License as published by the Free Software Foundation; either -+# version 2 of the License, or (at your option) any later version. -+# -+# This library is distributed in the hope that it will be useful, -+# but WITHOUT ANY WARRANTY; without even the implied warranty of -+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -+# Lesser General Public License for more details. -+# -+# You should have received a copy of the GNU Lesser General Public -+# License along with this library; if not, write to the Free Software -+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA -+ -+. ../tests/functions.sh -+ -+set -e -+set -x -+ -+requires nbdkit --version -+requires nbdkit data --dump-plugin -+ -+output=dump-data.out -+rm -f $output -+cleanup_fn rm -f $output -+ -+nbdkit -U - data data=' -+ @32768 1 -+ @65535 "hello, world!" -+ @17825790 "spanning buffer boundary" -+ @20000000 0 -+' --run 'nbddump "$uri"' > $output -+ -+cat $output -+ -+if [ "$(cat $output)" != '0000000000: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| -+* -+0000008000: 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| -+0000008010: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| -+* -+000000fff0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 68 |...............h| -+0000010000: 65 6c 6c 6f 2c 20 77 6f 72 6c 64 21 00 00 00 00 |ello, world!....| -+0000010010: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| -+* -+00010ffff0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 73 70 |..............sp| -+0001100000: 61 6e 6e 69 6e 67 20 62 75 66 66 65 72 20 62 6f |anning buffer bo| -+0001100010: 75 6e 64 61 72 79 00 00 00 00 00 00 00 00 00 00 |undary..........| -+0001100020: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| -+* -+0001312d00: 00 |. |' ]; then -+ echo "$0: unexpected output from nbddump command" -+ exit 1 -+fi -diff --git a/dump/dump-empty-qcow2.sh b/dump/dump-empty-qcow2.sh -new file mode 100755 -index 0000000..c9e583b ---- /dev/null -+++ b/dump/dump-empty-qcow2.sh -@@ -0,0 +1,46 @@ -+#!/usr/bin/env bash -+# nbd client library in userspace -+# Copyright (C) 2020-2022 Red Hat Inc. -+# -+# This library is free software; you can redistribute it and/or -+# modify it under the terms of the GNU Lesser General Public -+# License as published by the Free Software Foundation; either -+# version 2 of the License, or (at your option) any later version. -+# -+# This library is distributed in the hope that it will be useful, -+# but WITHOUT ANY WARRANTY; without even the implied warranty of -+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -+# Lesser General Public License for more details. -+# -+# You should have received a copy of the GNU Lesser General Public -+# License along with this library; if not, write to the Free Software -+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA -+ -+. ../tests/functions.sh -+ -+set -e -+set -x -+ -+requires $QEMU_NBD --version -+requires qemu-img --version -+ -+file=dump-empty-qcow2.qcow2 -+output=dump-empty-qcow2.out -+rm -f $file $output -+cleanup_fn rm -f $file $output -+ -+size=1G -+ -+# Create a large, empty qcow2 file. -+qemu-img create -f qcow2 $file $size -+ -+# Dump it and check the output. -+nbddump -- [ $QEMU_NBD -r -f qcow2 $file ] > $output -+cat $output -+ -+if [ "$(cat $output)" != '0000000000: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| -+* -+003ffffff0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................|' ]; then -+ echo "$0: unexpected output from nbddump command" -+ exit 1 -+fi -diff --git a/dump/dump-pattern.sh b/dump/dump-pattern.sh -new file mode 100755 -index 0000000..e4016a8 ---- /dev/null -+++ b/dump/dump-pattern.sh -@@ -0,0 +1,56 @@ -+#!/usr/bin/env bash -+# nbd client library in userspace -+# Copyright (C) 2020-2022 Red Hat Inc. -+# -+# This library is free software; you can redistribute it and/or -+# modify it under the terms of the GNU Lesser General Public -+# License as published by the Free Software Foundation; either -+# version 2 of the License, or (at your option) any later version. -+# -+# This library is distributed in the hope that it will be useful, -+# but WITHOUT ANY WARRANTY; without even the implied warranty of -+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -+# Lesser General Public License for more details. -+# -+# You should have received a copy of the GNU Lesser General Public -+# License along with this library; if not, write to the Free Software -+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA -+ -+. ../tests/functions.sh -+ -+set -e -+set -x -+ -+requires nbdkit --version -+requires nbdkit pattern --dump-plugin -+ -+output=dump-pattern.out -+rm -f $output -+cleanup_fn rm -f $output -+ -+nbdkit -U - pattern size=299 --run 'nbddump "$uri"' > $output -+ -+cat $output -+ -+if [ "$(cat $output)" != '0000000000: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 08 |................| -+0000000010: 00 00 00 00 00 00 00 10 00 00 00 00 00 00 00 18 |................| -+0000000020: 00 00 00 00 00 00 00 20 00 00 00 00 00 00 00 28 |....... .......(| -+0000000030: 00 00 00 00 00 00 00 30 00 00 00 00 00 00 00 38 |.......0.......8| -+0000000040: 00 00 00 00 00 00 00 40 00 00 00 00 00 00 00 48 |.......@.......H| -+0000000050: 00 00 00 00 00 00 00 50 00 00 00 00 00 00 00 58 |.......P.......X| -+0000000060: 00 00 00 00 00 00 00 60 00 00 00 00 00 00 00 68 |.......`.......h| -+0000000070: 00 00 00 00 00 00 00 70 00 00 00 00 00 00 00 78 |.......p.......x| -+0000000080: 00 00 00 00 00 00 00 80 00 00 00 00 00 00 00 88 |................| -+0000000090: 00 00 00 00 00 00 00 90 00 00 00 00 00 00 00 98 |................| -+00000000a0: 00 00 00 00 00 00 00 a0 00 00 00 00 00 00 00 a8 |................| -+00000000b0: 00 00 00 00 00 00 00 b0 00 00 00 00 00 00 00 b8 |................| -+00000000c0: 00 00 00 00 00 00 00 c0 00 00 00 00 00 00 00 c8 |................| -+00000000d0: 00 00 00 00 00 00 00 d0 00 00 00 00 00 00 00 d8 |................| -+00000000e0: 00 00 00 00 00 00 00 e0 00 00 00 00 00 00 00 e8 |................| -+00000000f0: 00 00 00 00 00 00 00 f0 00 00 00 00 00 00 00 f8 |................| -+0000000100: 00 00 00 00 00 00 01 00 00 00 00 00 00 00 01 08 |................| -+0000000110: 00 00 00 00 00 00 01 10 00 00 00 00 00 00 01 18 |................| -+0000000120: 00 00 00 00 00 00 01 20 00 00 00 |....... ... |' ]; then -+ echo "$0: unexpected output from nbddump command" -+ exit 1 -+fi -diff --git a/dump/dump.c b/dump/dump.c -new file mode 100644 -index 0000000..76af04c ---- /dev/null -+++ b/dump/dump.c -@@ -0,0 +1,464 @@ -+/* NBD client library in userspace -+ * Copyright (C) 2020-2022 Red Hat Inc. -+ * -+ * This library is free software; you can redistribute it and/or -+ * modify it under the terms of the GNU Lesser General Public -+ * License as published by the Free Software Foundation; either -+ * version 2 of the License, or (at your option) any later version. -+ * -+ * This library is distributed in the hope that it will be useful, -+ * but WITHOUT ANY WARRANTY; without even the implied warranty of -+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -+ * Lesser General Public License for more details. -+ * -+ * You should have received a copy of the GNU Lesser General Public -+ * License along with this library; if not, write to the Free Software -+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA -+ */ -+ -+#include -+ -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+ -+#include -+ -+#include "minmax.h" -+#include "rounding.h" -+#include "version.h" -+#include "vector.h" -+ -+DEFINE_VECTOR_TYPE (uint32_vector, uint32_t) -+ -+static const char *progname; -+static struct nbd_handle *nbd; -+static bool colour; -+static uint64_t limit = UINT64_MAX; /* --length (unlimited by default) */ -+static int64_t size; /* actual size */ -+static bool can_meta_context; /* did we get extent data? */ -+ -+/* See do_connect () */ -+static enum { MODE_URI = 1, MODE_SQUARE_BRACKET } mode; -+static char **args; -+ -+/* Read buffer. */ -+static unsigned char buffer[16*1024*1024]; -+ -+static void do_connect (void); -+static void do_dump (void); -+static void catch_signal (int); -+ -+static void __attribute__((noreturn)) -+usage (FILE *fp, int exitcode) -+{ -+ fprintf (fp, -+"\n" -+"Hexdump the content of a disk over NBD:\n" -+"\n" -+" nbddump NBD-URI | [ CMD ARGS ... ]\n" -+"\n" -+"Other options:\n" -+"\n" -+" nbddump --help\n" -+" nbddump --version\n" -+"\n" -+"Examples:\n" -+"\n" -+" nbddump nbd://localhost\n" -+" nbddump -- [ qemu-nbd -r -f qcow2 file.qcow2 ]\n" -+"\n" -+"Please read the nbddump(1) manual page for full usage.\n" -+"\n" -+); -+ exit (exitcode); -+} -+ -+int -+main (int argc, char *argv[]) -+{ -+ enum { -+ HELP_OPTION = CHAR_MAX + 1, -+ LONG_OPTIONS, -+ SHORT_OPTIONS, -+ COLOUR_OPTION, -+ NO_COLOUR_OPTION, -+ }; -+ const char *short_options = "n:V"; -+ const struct option long_options[] = { -+ { "help", no_argument, NULL, HELP_OPTION }, -+ { "long-options", no_argument, NULL, LONG_OPTIONS }, -+ { "short-options", no_argument, NULL, SHORT_OPTIONS }, -+ { "version", no_argument, NULL, 'V' }, -+ -+ { "color", no_argument, NULL, COLOUR_OPTION }, -+ { "colors", no_argument, NULL, COLOUR_OPTION }, -+ { "colour", no_argument, NULL, COLOUR_OPTION }, -+ { "colours", no_argument, NULL, COLOUR_OPTION }, -+ { "no-color", no_argument, NULL, NO_COLOUR_OPTION }, -+ { "no-colors", no_argument, NULL, NO_COLOUR_OPTION }, -+ { "no-colour", no_argument, NULL, NO_COLOUR_OPTION }, -+ { "no-colours", no_argument, NULL, NO_COLOUR_OPTION }, -+ { "length", required_argument, NULL, 'n' }, -+ { "limit", required_argument, NULL, 'n' }, -+ { NULL } -+ }; -+ int c; -+ size_t i; -+ -+ progname = argv[0]; -+ colour = isatty (STDOUT_FILENO); -+ -+ for (;;) { -+ c = getopt_long (argc, argv, short_options, long_options, NULL); -+ if (c == -1) -+ break; -+ -+ switch (c) { -+ case HELP_OPTION: -+ usage (stdout, EXIT_SUCCESS); -+ -+ case LONG_OPTIONS: -+ for (i = 0; long_options[i].name != NULL; ++i) { -+ if (strcmp (long_options[i].name, "long-options") != 0 && -+ strcmp (long_options[i].name, "short-options") != 0) -+ printf ("--%s\n", long_options[i].name); -+ } -+ exit (EXIT_SUCCESS); -+ -+ case SHORT_OPTIONS: -+ for (i = 0; short_options[i]; ++i) { -+ if (short_options[i] != ':' && short_options[i] != '+') -+ printf ("-%c\n", short_options[i]); -+ } -+ exit (EXIT_SUCCESS); -+ -+ case COLOUR_OPTION: -+ colour = true; -+ break; -+ -+ case NO_COLOUR_OPTION: -+ colour = false; -+ break; -+ -+ case 'n': -+ /* XXX Allow human sizes here. */ -+ if (sscanf (optarg, "%" SCNu64, &limit) != 1) { -+ fprintf (stderr, "%s: could not parse --length option: %s\n", -+ progname, optarg); -+ exit (EXIT_FAILURE); -+ } -+ break; -+ -+ case 'V': -+ display_version ("nbddump"); -+ exit (EXIT_SUCCESS); -+ -+ default: -+ usage (stderr, EXIT_FAILURE); -+ } -+ } -+ -+ /* Is it a URI or subprocess? */ -+ if (argc - optind >= 3 && -+ strcmp (argv[optind], "[") == 0 && -+ strcmp (argv[argc-1], "]") == 0) { -+ mode = MODE_SQUARE_BRACKET; -+ argv[argc-1] = NULL; -+ args = &argv[optind+1]; -+ } -+ else if (argc - optind == 1) { -+ mode = MODE_URI; -+ args = &argv[optind]; -+ } -+ else { -+ usage (stderr, EXIT_FAILURE); -+ } -+ -+ /* Open the NBD side. */ -+ nbd = nbd_create (); -+ if (nbd == NULL) { -+ fprintf (stderr, "%s: %s\n", progname, nbd_get_error ()); -+ exit (EXIT_FAILURE); -+ } -+ nbd_set_uri_allow_local_file (nbd, true); /* Allow ?tls-psk-file. */ -+ nbd_add_meta_context (nbd, LIBNBD_CONTEXT_BASE_ALLOCATION); -+ -+ /* Connect to the server. */ -+ do_connect (); -+ can_meta_context = -+ nbd_can_meta_context (nbd, LIBNBD_CONTEXT_BASE_ALLOCATION) > 0; -+ -+ /* Get the size. */ -+ size = nbd_get_size (nbd); -+ if (size == -1) { -+ fprintf (stderr, "%s: %s\n", progname, nbd_get_error ()); -+ exit (EXIT_FAILURE); -+ } -+ -+ /* Before dumping, make sure we restore the terminal on ^C etc. */ -+ signal (SIGINT, catch_signal); -+ signal (SIGQUIT, catch_signal); -+ signal (SIGTERM, catch_signal); -+ signal (SIGHUP, catch_signal); -+ -+ /* Dump the content. */ -+ do_dump (); -+ -+ nbd_shutdown (nbd, 0); -+ nbd_close (nbd); -+ -+ exit (EXIT_SUCCESS); -+} -+ -+/* Connect the handle to the server. */ -+static void -+do_connect (void) -+{ -+ int r; -+ -+ switch (mode) { -+ case MODE_URI: /* NBD-URI */ -+ r = nbd_connect_uri (nbd, args[0]); -+ break; -+ -+ case MODE_SQUARE_BRACKET: /* [ CMD ARGS ... ] */ -+ r = nbd_connect_systemd_socket_activation (nbd, args); -+ break; -+ -+ default: -+ abort (); -+ } -+ -+ if (r == -1) { -+ fprintf (stderr, "%s: %s\n", progname, nbd_get_error ()); -+ exit (EXIT_FAILURE); -+ } -+} -+ -+/* Various ANSI colours, suppressed if --no-colour / not tty output. */ -+static void -+ansi_restore (void) -+{ -+ if (colour) -+ fputs ("\033[0m", stdout); -+} -+ -+static void -+ansi_blue (void) -+{ -+ if (colour) -+ fputs ("\033[1;34m", stdout); -+} -+ -+static void -+ansi_green (void) -+{ -+ if (colour) -+ fputs ("\033[0;32m", stdout); -+} -+ -+static void -+ansi_magenta (void) -+{ -+ if (colour) -+ fputs ("\033[1;35m", stdout); -+} -+ -+static void -+ansi_red (void) -+{ -+ if (colour) -+ fputs ("\033[1;31m", stdout); -+} -+ -+static void -+ansi_grey (void) -+{ -+ if (colour) -+ fputs ("\033[0;90m", stdout); -+} -+ -+static void -+catch_signal (int sig) -+{ -+ printf ("\n"); -+ ansi_restore (); -+ fflush (stdout); -+ _exit (EXIT_FAILURE); -+} -+ -+/* Read the extent map for the next block and return true if it is all -+ * zeroes. This is conservative and returns false if we did not get -+ * the full extent map from the server, or if the server doesn't -+ * support base:allocation at all. -+ */ -+static int -+extent_callback (void *user_data, const char *metacontext, -+ uint64_t offset, -+ uint32_t *entries, size_t nr_entries, -+ int *error) -+{ -+ uint32_vector *list = user_data; -+ size_t i; -+ -+ if (strcmp (metacontext, LIBNBD_CONTEXT_BASE_ALLOCATION) != 0) -+ return 0; -+ -+ /* Just append the entries we got to the list. */ -+ for (i = 0; i < nr_entries; ++i) { -+ if (uint32_vector_append (list, entries[i]) == -1) { -+ perror ("realloc"); -+ exit (EXIT_FAILURE); -+ } -+ } -+ return 0; -+} -+ -+static bool -+test_all_zeroes (uint64_t offset, size_t count) -+{ -+ uint32_vector entries = empty_vector; -+ size_t i; -+ uint64_t count_read; -+ -+ if (!can_meta_context) -+ return false; -+ -+ /* Get the extent map for the block. Note the server doesn't need -+ * to return all requested data here. If it does not then we return -+ * false, causing the main code to do a full read. We could be -+ * smarter and keep asking the server (XXX). -+ */ -+ if (nbd_block_status (nbd, count, offset, -+ (nbd_extent_callback) { -+ .callback = extent_callback, -+ .user_data = &entries }, -+ 0) == -1) { -+ fprintf (stderr, "%s: %s\n", progname, nbd_get_error ()); -+ exit (EXIT_FAILURE); -+ } -+ -+ count_read = 0; -+ for (i = 0; i < entries.len; i += 2) { -+ uint32_t len = entries.ptr[i]; -+ uint32_t type = entries.ptr[i+1]; -+ -+ count_read += len; -+ if (!(type & 2)) /* not zero */ -+ return false; -+ } -+ -+ /* Did we read at least the whole range wanted? */ -+ if (count_read < count) -+ return false; -+ -+ /* If we got here, we read the whole range and it was all zeroes. */ -+ return true; -+} -+ -+/* Hexdump the NBD data. -+ * -+ * XXX In future we could do this all asynch (including writing to -+ * stdout) which could make it very efficient. -+ */ -+static void -+do_dump (void) -+{ -+ /* If --no-colour, don't use unicode in the output. */ -+ const char *splat = colour ? "☆" : "*"; -+ const char *pipe = colour ? "│" : "|"; -+ const char *dot = colour ? "·" : "."; -+ uint64_t offset = 0; -+ uint64_t count = size > limit ? limit : size; -+ size_t i, j, n; -+ char last[16]; -+ bool printed_splat = false, same; -+ -+ while (count) { -+ n = MIN (count, sizeof buffer); -+ -+ if (! test_all_zeroes (offset, n)) { -+ if (nbd_pread (nbd, buffer, n, offset, 0) == -1) { -+ fprintf (stderr, "%s: %s\n", progname, nbd_get_error ()); -+ exit (EXIT_FAILURE); -+ } -+ } -+ else { -+ memset (buffer, 0, n); -+ } -+ -+ /* Make sure a multiple of 16 bytes gets written to the buffer. */ -+ if (n & 15) -+ memset (&buffer[n], 0, 16 - (n & 15)); -+ -+ for (i = 0; i < n; i += 16) { -+ /* Is this line the same as the last line? (Squashing) */ -+ same = -+ offset + i > 0 && /* first line is never squashed */ -+ offset + i + 16 < size && /* last line is never squashed */ -+ memcmp (&buffer[i], last, 16) == 0; -+ if (same) { -+ if (!printed_splat) { -+ printf ("%s\n", splat); -+ printed_splat = true; -+ } -+ continue; -+ } -+ printed_splat = false; -+ memcpy (last, &buffer[i], 16); /* Save the current line. */ -+ -+ /* Print the offset. */ -+ ansi_green (); -+ printf ("%010zx", offset + i); -+ ansi_grey (); -+ printf (": "); -+ -+ /* Print the hex codes. */ -+ for (j = i; j < MIN (i+16, n); ++j) { -+ if (buffer[j]) -+ ansi_blue (); -+ else -+ ansi_grey (); -+ printf ("%02x ", buffer[j]); -+ } -+ ansi_grey (); -+ for (; j < i+16; ++j) -+ printf (" "); -+ -+ /* Print the ASCII codes. */ -+ printf ("%s", pipe); -+ for (j = i; j < MIN (i+16, n); ++j) { -+ char c = (char) buffer[j]; -+ if (isalnum (c)) { -+ ansi_red (); -+ printf ("%c", c); -+ } -+ else if (isprint (c)) { -+ ansi_magenta (); -+ printf ("%c", c); -+ } -+ else { -+ ansi_grey (); -+ printf ("%s", dot); -+ } -+ } -+ ansi_grey (); -+ for (; j < i+16; ++j) -+ printf (" "); -+ printf ("%s\n", pipe); -+ ansi_restore (); -+ } -+ -+ offset += n; -+ count -= n; -+ } -+} -diff --git a/dump/nbddump.pod b/dump/nbddump.pod -new file mode 100644 -index 0000000..5d7864d ---- /dev/null -+++ b/dump/nbddump.pod -@@ -0,0 +1,116 @@ -+=head1 NAME -+ -+nbddump - hexdump the content of a disk over NBD -+ -+=head1 SYNOPSIS -+ -+ nbddump NBD -+ -+C is an NBD URI or subprocess: -+ -+ NBD := nbd://... | nbd+unix:// (or other URI formats) -+ | [ CMD ARGS ... ] -+ -+=for paragraph -+ -+ nbddump --help -+ -+=for paragraph -+ -+ nbddump --version -+ -+=head1 DESCRIPTION -+ -+nbddump prints the content of a disk from an NBD server using the -+usual hexdump format: -+ -+ $ nbddump nbd://localhost -+ 0000: 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 │················│ -+ 0010: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 │················│ -+ ☆ -+ 0100: 68 65 6c 6c 6f 2c 20 77 6f 72 6c 64 21 00 00 00 │hello, world!···│ -+ 0110: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 │················│ -+ ☆ -+ 1000: 00 00 00 21 │···! │ -+ -+=head2 Output format -+ -+The first field (before the C<:>) is the offset within the file, in -+hexadecimal. -+ -+The second field shows the hex codes of bytes read from the file. -+ -+The third field shows the ASCII equivalent characters (if printable). -+ -+A splat character (C<☆>) indicates lines of repeated output which have -+been squashed. (Note this is not just for lines of zero bytes, but -+any case where the next line shown would be the same as the previous -+line.) -+ -+=head2 Subprocess -+ -+nbddump can also run an NBD server as a subprocess. This requires an -+NBD server which understands systemd socket activation, such as -+L or L. -+ -+For example, to dump out a qcow2 file as raw data: -+ -+ nbddump -- [ qemu-nbd -r -f qcow2 file.qcow2 ] -+ -+Note that S> are separate parameters, and must be -+surrounded by spaces. C<--> separates nbddump parameters from -+subprocess parameters. -+ -+=head1 OPTIONS -+ -+=over 4 -+ -+=item B<--help> -+ -+Display brief command line help and exit. -+ -+=item B<--color> -+ -+=item B<--colour> -+ -+=item B<--no-color> -+ -+=item B<--no-colour> -+ -+Enable or disable ANSI colours in output. By default we use colours -+if the output seems to be a terminal, and disable them if not. -+ -+=item B<--length=>N -+ -+=item B<-n> N -+ -+Dump up to I bytes and then stop. -+ -+=item B<-V> -+ -+=item B<--version> -+ -+Display the package name and version and exit. -+ -+=back -+ -+=head1 SEE ALSO -+ -+L, -+L, -+L, -+L, -+L, -+L, -+L, -+L, -+L, -+L. -+ -+=head1 AUTHORS -+ -+Richard W.M. Jones -+ -+=head1 COPYRIGHT -+ -+Copyright (C) 2022 Red Hat Inc. -diff --git a/dump/test-long-options.sh b/dump/test-long-options.sh -new file mode 100755 -index 0000000..924c8f5 ---- /dev/null -+++ b/dump/test-long-options.sh -@@ -0,0 +1,35 @@ -+#!/usr/bin/env bash -+# nbd client library in userspace -+# Copyright (C) 2019-2022 Red Hat Inc. -+# -+# This library is free software; you can redistribute it and/or -+# modify it under the terms of the GNU Lesser General Public -+# License as published by the Free Software Foundation; either -+# version 2 of the License, or (at your option) any later version. -+# -+# This library is distributed in the hope that it will be useful, -+# but WITHOUT ANY WARRANTY; without even the implied warranty of -+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -+# Lesser General Public License for more details. -+# -+# You should have received a copy of the GNU Lesser General Public -+# License along with this library; if not, write to the Free Software -+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA -+ -+# Test that nbddump --long-options looks sane. -+ -+. ../tests/functions.sh -+set -e -+set -x -+ -+output=test-long-options.out -+cleanup_fn rm -f $output -+ -+$VG nbddump --long-options > $output -+if [ $? != 0 ]; then -+ echo "$0: unexpected exit status" -+ fail=1 -+fi -+cat $output -+grep -- --length $output -+grep -- --version $output -diff --git a/dump/test-short-options.sh b/dump/test-short-options.sh -new file mode 100755 -index 0000000..325f7df ---- /dev/null -+++ b/dump/test-short-options.sh -@@ -0,0 +1,35 @@ -+#!/usr/bin/env bash -+# nbd client library in userspace -+# Copyright (C) 2019-2022 Red Hat Inc. -+# -+# This library is free software; you can redistribute it and/or -+# modify it under the terms of the GNU Lesser General Public -+# License as published by the Free Software Foundation; either -+# version 2 of the License, or (at your option) any later version. -+# -+# This library is distributed in the hope that it will be useful, -+# but WITHOUT ANY WARRANTY; without even the implied warranty of -+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -+# Lesser General Public License for more details. -+# -+# You should have received a copy of the GNU Lesser General Public -+# License along with this library; if not, write to the Free Software -+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA -+ -+# Test that nbddump --short-options looks sane. -+ -+. ../tests/functions.sh -+set -e -+set -x -+ -+output=test-short-options.out -+cleanup_fn rm -f $output -+ -+$VG nbddump --short-options > $output -+if [ $? != 0 ]; then -+ echo "$0: unexpected exit status" -+ fail=1 -+fi -+cat $output -+grep -- -n $output -+grep -- -V $output -diff --git a/dump/test-version.sh b/dump/test-version.sh -new file mode 100755 -index 0000000..fce4ed1 ---- /dev/null -+++ b/dump/test-version.sh -@@ -0,0 +1,33 @@ -+#!/usr/bin/env bash -+# nbd client library in userspace -+# Copyright (C) 2019 Red Hat Inc. -+# -+# This library is free software; you can redistribute it and/or -+# modify it under the terms of the GNU Lesser General Public -+# License as published by the Free Software Foundation; either -+# version 2 of the License, or (at your option) any later version. -+# -+# This library is distributed in the hope that it will be useful, -+# but WITHOUT ANY WARRANTY; without even the implied warranty of -+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -+# Lesser General Public License for more details. -+# -+# You should have received a copy of the GNU Lesser General Public -+# License along with this library; if not, write to the Free Software -+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA -+ -+# Test that nbddump --version looks sane. -+ -+fail=0 -+output=$($VG nbddump --version) -+if [ $? != 0 ]; then -+ echo "$0: unexpected exit status" -+ fail=1 -+fi -+if [ "$output" != "nbddump $EXPECTED_VERSION -+libnbd $EXPECTED_VERSION" ]; then -+ echo "$0: unexpected output" -+ fail=1 -+fi -+echo "$output" -+exit $fail -diff --git a/fuse/nbdfuse.pod b/fuse/nbdfuse.pod -index 7c1c817..daa79c1 100644 ---- a/fuse/nbdfuse.pod -+++ b/fuse/nbdfuse.pod -@@ -412,6 +412,7 @@ The differences from nbdfuse are similar to the list above. - - L, - L, -+L, - L, - L, - L, -diff --git a/info/nbdinfo.pod b/info/nbdinfo.pod -index 649cf1c..4733ecd 100644 ---- a/info/nbdinfo.pod -+++ b/info/nbdinfo.pod -@@ -407,6 +407,7 @@ Display the package name and version and exit. - - L, - L, -+L, - L, - L, - L, -diff --git a/run.in b/run.in -index 8a21906..2a171e5 100755 ---- a/run.in -+++ b/run.in -@@ -58,6 +58,7 @@ b="$(cd @abs_builddir@ && pwd)" - - # Set the PATH to contain all libnbd binaries. - prepend PATH "$b/copy" -+prepend PATH "$b/dump" - prepend PATH "$b/fuse" - prepend PATH "$b/info" - prepend PATH "$b/sh" -diff --git a/sh/nbdsh.pod b/sh/nbdsh.pod -index ca5d6af..c9dac4a 100644 ---- a/sh/nbdsh.pod -+++ b/sh/nbdsh.pod -@@ -147,6 +147,7 @@ L. - L, - L, - L, -+L, - L, - L, - L. --- -2.31.1 - diff --git a/0002-dump-Visually-separate-columns-0-7-and-8-15.patch b/0002-dump-Visually-separate-columns-0-7-and-8-15.patch deleted file mode 100644 index 67fc1af..0000000 --- a/0002-dump-Visually-separate-columns-0-7-and-8-15.patch +++ /dev/null @@ -1,153 +0,0 @@ -From ec947323528725fcf12b5b9ba32b02d36dbd9621 Mon Sep 17 00:00:00 2001 -From: "Richard W.M. Jones" -Date: Thu, 30 Jun 2022 21:09:39 +0100 -Subject: [PATCH] dump: Visually separate columns 0-7 and 8-15 -MIME-Version: 1.0 -Content-Type: text/plain; charset=UTF-8 -Content-Transfer-Encoding: 8bit - -Before: - -0000090000: 68 65 72 65 20 77 65 20 61 72 65 00 68 65 72 65 │... -0000090010: 20 77 65 20 61 72 65 00 68 65 72 65 20 77 65 20 │... -0000090020: 61 72 65 00 68 65 72 65 20 77 65 20 61 72 65 00 │... - -After: - -0000090000: 68 65 72 65 20 77 65 20 61 72 65 00 68 65 72 65 │... -0000090010: 20 77 65 20 61 72 65 00 68 65 72 65 20 77 65 20 │... -0000090020: 61 72 65 00 68 65 72 65 20 77 65 20 61 72 65 00 │... - -Updates: commit c4107b9a40d6451630dcccf1bf6596c8e56420be -(cherry picked from commit 315a637d3eae003c1d84eb1b88a7b47b534f1e80) ---- - dump/dump-data.sh | 22 +++++++++++----------- - dump/dump-empty-qcow2.sh | 4 ++-- - dump/dump-pattern.sh | 38 +++++++++++++++++++------------------- - dump/dump.c | 5 ++++- - 4 files changed, 36 insertions(+), 33 deletions(-) - -diff --git a/dump/dump-data.sh b/dump/dump-data.sh -index 23d09da..955cd3b 100755 ---- a/dump/dump-data.sh -+++ b/dump/dump-data.sh -@@ -37,21 +37,21 @@ nbdkit -U - data data=' - - cat $output - --if [ "$(cat $output)" != '0000000000: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| -+if [ "$(cat $output)" != '0000000000: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| - * --0000008000: 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| --0000008010: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| -+0000008000: 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| -+0000008010: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| - * --000000fff0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 68 |...............h| --0000010000: 65 6c 6c 6f 2c 20 77 6f 72 6c 64 21 00 00 00 00 |ello, world!....| --0000010010: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| -+000000fff0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 68 |...............h| -+0000010000: 65 6c 6c 6f 2c 20 77 6f 72 6c 64 21 00 00 00 00 |ello, world!....| -+0000010010: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| - * --00010ffff0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 73 70 |..............sp| --0001100000: 61 6e 6e 69 6e 67 20 62 75 66 66 65 72 20 62 6f |anning buffer bo| --0001100010: 75 6e 64 61 72 79 00 00 00 00 00 00 00 00 00 00 |undary..........| --0001100020: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| -+00010ffff0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 73 70 |..............sp| -+0001100000: 61 6e 6e 69 6e 67 20 62 75 66 66 65 72 20 62 6f |anning buffer bo| -+0001100010: 75 6e 64 61 72 79 00 00 00 00 00 00 00 00 00 00 |undary..........| -+0001100020: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| - * --0001312d00: 00 |. |' ]; then -+0001312d00: 00 |. |' ]; then - echo "$0: unexpected output from nbddump command" - exit 1 - fi -diff --git a/dump/dump-empty-qcow2.sh b/dump/dump-empty-qcow2.sh -index c9e583b..472b6eb 100755 ---- a/dump/dump-empty-qcow2.sh -+++ b/dump/dump-empty-qcow2.sh -@@ -38,9 +38,9 @@ qemu-img create -f qcow2 $file $size - nbddump -- [ $QEMU_NBD -r -f qcow2 $file ] > $output - cat $output - --if [ "$(cat $output)" != '0000000000: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| -+if [ "$(cat $output)" != '0000000000: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| - * --003ffffff0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................|' ]; then -+003ffffff0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................|' ]; then - echo "$0: unexpected output from nbddump command" - exit 1 - fi -diff --git a/dump/dump-pattern.sh b/dump/dump-pattern.sh -index e4016a8..d512b77 100755 ---- a/dump/dump-pattern.sh -+++ b/dump/dump-pattern.sh -@@ -32,25 +32,25 @@ nbdkit -U - pattern size=299 --run 'nbddump "$uri"' > $output - - cat $output - --if [ "$(cat $output)" != '0000000000: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 08 |................| --0000000010: 00 00 00 00 00 00 00 10 00 00 00 00 00 00 00 18 |................| --0000000020: 00 00 00 00 00 00 00 20 00 00 00 00 00 00 00 28 |....... .......(| --0000000030: 00 00 00 00 00 00 00 30 00 00 00 00 00 00 00 38 |.......0.......8| --0000000040: 00 00 00 00 00 00 00 40 00 00 00 00 00 00 00 48 |.......@.......H| --0000000050: 00 00 00 00 00 00 00 50 00 00 00 00 00 00 00 58 |.......P.......X| --0000000060: 00 00 00 00 00 00 00 60 00 00 00 00 00 00 00 68 |.......`.......h| --0000000070: 00 00 00 00 00 00 00 70 00 00 00 00 00 00 00 78 |.......p.......x| --0000000080: 00 00 00 00 00 00 00 80 00 00 00 00 00 00 00 88 |................| --0000000090: 00 00 00 00 00 00 00 90 00 00 00 00 00 00 00 98 |................| --00000000a0: 00 00 00 00 00 00 00 a0 00 00 00 00 00 00 00 a8 |................| --00000000b0: 00 00 00 00 00 00 00 b0 00 00 00 00 00 00 00 b8 |................| --00000000c0: 00 00 00 00 00 00 00 c0 00 00 00 00 00 00 00 c8 |................| --00000000d0: 00 00 00 00 00 00 00 d0 00 00 00 00 00 00 00 d8 |................| --00000000e0: 00 00 00 00 00 00 00 e0 00 00 00 00 00 00 00 e8 |................| --00000000f0: 00 00 00 00 00 00 00 f0 00 00 00 00 00 00 00 f8 |................| --0000000100: 00 00 00 00 00 00 01 00 00 00 00 00 00 00 01 08 |................| --0000000110: 00 00 00 00 00 00 01 10 00 00 00 00 00 00 01 18 |................| --0000000120: 00 00 00 00 00 00 01 20 00 00 00 |....... ... |' ]; then -+if [ "$(cat $output)" != '0000000000: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 08 |................| -+0000000010: 00 00 00 00 00 00 00 10 00 00 00 00 00 00 00 18 |................| -+0000000020: 00 00 00 00 00 00 00 20 00 00 00 00 00 00 00 28 |....... .......(| -+0000000030: 00 00 00 00 00 00 00 30 00 00 00 00 00 00 00 38 |.......0.......8| -+0000000040: 00 00 00 00 00 00 00 40 00 00 00 00 00 00 00 48 |.......@.......H| -+0000000050: 00 00 00 00 00 00 00 50 00 00 00 00 00 00 00 58 |.......P.......X| -+0000000060: 00 00 00 00 00 00 00 60 00 00 00 00 00 00 00 68 |.......`.......h| -+0000000070: 00 00 00 00 00 00 00 70 00 00 00 00 00 00 00 78 |.......p.......x| -+0000000080: 00 00 00 00 00 00 00 80 00 00 00 00 00 00 00 88 |................| -+0000000090: 00 00 00 00 00 00 00 90 00 00 00 00 00 00 00 98 |................| -+00000000a0: 00 00 00 00 00 00 00 a0 00 00 00 00 00 00 00 a8 |................| -+00000000b0: 00 00 00 00 00 00 00 b0 00 00 00 00 00 00 00 b8 |................| -+00000000c0: 00 00 00 00 00 00 00 c0 00 00 00 00 00 00 00 c8 |................| -+00000000d0: 00 00 00 00 00 00 00 d0 00 00 00 00 00 00 00 d8 |................| -+00000000e0: 00 00 00 00 00 00 00 e0 00 00 00 00 00 00 00 e8 |................| -+00000000f0: 00 00 00 00 00 00 00 f0 00 00 00 00 00 00 00 f8 |................| -+0000000100: 00 00 00 00 00 00 01 00 00 00 00 00 00 00 01 08 |................| -+0000000110: 00 00 00 00 00 00 01 10 00 00 00 00 00 00 01 18 |................| -+0000000120: 00 00 00 00 00 00 01 20 00 00 00 |....... ... |' ]; then - echo "$0: unexpected output from nbddump command" - exit 1 - fi -diff --git a/dump/dump.c b/dump/dump.c -index 76af04c..7818f1f 100644 ---- a/dump/dump.c -+++ b/dump/dump.c -@@ -429,10 +429,13 @@ do_dump (void) - else - ansi_grey (); - printf ("%02x ", buffer[j]); -+ if ((j - i) == 7) printf (" "); - } - ansi_grey (); -- for (; j < i+16; ++j) -+ for (; j < i+16; ++j) { - printf (" "); -+ if ((j - i) == 7) printf (" "); -+ } - - /* Print the ASCII codes. */ - printf ("%s", pipe); --- -2.31.1 - diff --git a/0003-dump-Fix-build-on-i686.patch b/0003-dump-Fix-build-on-i686.patch deleted file mode 100644 index 7ec277e..0000000 --- a/0003-dump-Fix-build-on-i686.patch +++ /dev/null @@ -1,38 +0,0 @@ -From 590e3a010d2c840314702883e44ec9841e3383c6 Mon Sep 17 00:00:00 2001 -From: "Richard W.M. Jones" -Date: Thu, 30 Jun 2022 22:27:43 +0100 -Subject: [PATCH] dump: Fix build on i686 -MIME-Version: 1.0 -Content-Type: text/plain; charset=UTF-8 -Content-Transfer-Encoding: 8bit - -Because we used the wrong printf format, the build would fail on -32 bit architectures but succeed on 64 bit: - -dump.c: In function ‘do_dump’: -dump.c:421:21: error: format ‘%zx’ expects argument of type ‘size_t’, but argument 2 has type ‘uint64_t’ {aka ‘long long unsigned int’} [-Werror=format=] - printf ("%010zx", offset + i); - ~~~~~^ ~~~~~~~~~~ - %010llx - -(cherry picked from commit ce004c329c7fcd6c60d11673b7a5c5ce3414413b) ---- - dump/dump.c | 2 +- - 1 file changed, 1 insertion(+), 1 deletion(-) - -diff --git a/dump/dump.c b/dump/dump.c -index 7818f1f..8bf62f9 100644 ---- a/dump/dump.c -+++ b/dump/dump.c -@@ -418,7 +418,7 @@ do_dump (void) - - /* Print the offset. */ - ansi_green (); -- printf ("%010zx", offset + i); -+ printf ("%010" PRIx64, offset + i); - ansi_grey (); - printf (": "); - --- -2.31.1 - diff --git a/0004-dump-Fix-tests-on-Debian-10.patch b/0004-dump-Fix-tests-on-Debian-10.patch deleted file mode 100644 index cdb908a..0000000 --- a/0004-dump-Fix-tests-on-Debian-10.patch +++ /dev/null @@ -1,41 +0,0 @@ -From e7a2815412891d5c13b5b5f0e9aa61882880c87f Mon Sep 17 00:00:00 2001 -From: "Richard W.M. Jones" -Date: Thu, 30 Jun 2022 22:31:00 +0100 -Subject: [PATCH] dump: Fix tests on Debian 10 - -The version of nbdkit on Debian 10 does not set $uri. Check for this -or skip the test. - -(cherry picked from commit 083b1ca30fb5e6e0dc0e4b0eea9ebe8474d3f864) ---- - dump/dump-data.sh | 1 + - dump/dump-pattern.sh | 1 + - 2 files changed, 2 insertions(+) - -diff --git a/dump/dump-data.sh b/dump/dump-data.sh -index 955cd3b..46e4d1e 100755 ---- a/dump/dump-data.sh -+++ b/dump/dump-data.sh -@@ -23,6 +23,7 @@ set -x - - requires nbdkit --version - requires nbdkit data --dump-plugin -+requires nbdkit -U - null --run 'test "$uri" != ""' - - output=dump-data.out - rm -f $output -diff --git a/dump/dump-pattern.sh b/dump/dump-pattern.sh -index d512b77..e2188ac 100755 ---- a/dump/dump-pattern.sh -+++ b/dump/dump-pattern.sh -@@ -23,6 +23,7 @@ set -x - - requires nbdkit --version - requires nbdkit pattern --dump-plugin -+requires nbdkit -U - null --run 'test "$uri" != ""' - - output=dump-pattern.out - rm -f $output --- -2.31.1 - diff --git a/0005-dump-dump-data.sh-Test-requires-nbdkit-1.22.patch b/0005-dump-dump-data.sh-Test-requires-nbdkit-1.22.patch deleted file mode 100644 index d868281..0000000 --- a/0005-dump-dump-data.sh-Test-requires-nbdkit-1.22.patch +++ /dev/null @@ -1,31 +0,0 @@ -From 7c669783b1b3fab902ce34d7914b62617ed8b263 Mon Sep 17 00:00:00 2001 -From: "Richard W.M. Jones" -Date: Thu, 30 Jun 2022 22:35:05 +0100 -Subject: [PATCH] dump/dump-data.sh: Test requires nbdkit 1.22 - -Ubuntu 20.04 has nbdkit 1.16 which lacks support for strings. These -were added in nbdkit 1.22. - -(cherry picked from commit a8fa05ffb8b85f41276ffb52498e4528c08e5f21) ---- - dump/dump-data.sh | 4 ++++ - 1 file changed, 4 insertions(+) - -diff --git a/dump/dump-data.sh b/dump/dump-data.sh -index 46e4d1e..11145b0 100755 ---- a/dump/dump-data.sh -+++ b/dump/dump-data.sh -@@ -25,6 +25,10 @@ requires nbdkit --version - requires nbdkit data --dump-plugin - requires nbdkit -U - null --run 'test "$uri" != ""' - -+# This test requires nbdkit >= 1.22. -+minor=$( nbdkit --dump-config | grep ^version_minor | cut -d= -f2 ) -+requires test $minor -ge 22 -+ - output=dump-data.out - rm -f $output - cleanup_fn rm -f $output --- -2.31.1 - diff --git a/0006-copy-Store-the-preferred-block-size-in-the-operation.patch b/0006-copy-Store-the-preferred-block-size-in-the-operation.patch deleted file mode 100644 index 893a026..0000000 --- a/0006-copy-Store-the-preferred-block-size-in-the-operation.patch +++ /dev/null @@ -1,163 +0,0 @@ -From 8dce43a3ea7a529bc37cbe5607a8d52186cc8169 Mon Sep 17 00:00:00 2001 -From: "Richard W.M. Jones" -Date: Tue, 28 Jun 2022 18:27:58 +0100 -Subject: [PATCH] copy: Store the preferred block size in the operations struct - -This will be used in a subsequent commit. At the moment the preferred -block size for all sources / destinations is simply calculated and -stored. - -(cherry picked from commit e6c42f8b2d447bbcc659d6dd33be67335834b2e5) ---- - copy/file-ops.c | 4 +++- - copy/main.c | 29 +++++++++++++++++++++++------ - copy/nbd-ops.c | 10 ++++++++++ - copy/nbdcopy.h | 4 +++- - copy/null-ops.c | 1 + - copy/pipe-ops.c | 1 + - 6 files changed, 41 insertions(+), 8 deletions(-) - -diff --git a/copy/file-ops.c b/copy/file-ops.c -index ab37875..34f08e5 100644 ---- a/copy/file-ops.c -+++ b/copy/file-ops.c -@@ -241,13 +241,15 @@ seek_hole_supported (int fd) - - struct rw * - file_create (const char *name, int fd, -- off_t st_size, bool is_block, direction d) -+ off_t st_size, uint64_t preferred, -+ bool is_block, direction d) - { - struct rw_file *rwf = calloc (1, sizeof *rwf); - if (rwf == NULL) { perror ("calloc"); exit (EXIT_FAILURE); } - - rwf->rw.ops = &file_ops; - rwf->rw.name = name; -+ rwf->rw.preferred = preferred; - rwf->fd = fd; - rwf->is_block = is_block; - -diff --git a/copy/main.c b/copy/main.c -index cc379e9..19ec384 100644 ---- a/copy/main.c -+++ b/copy/main.c -@@ -512,10 +512,26 @@ open_local (const char *filename, direction d) - fprintf (stderr, "%s: %s: %m\n", prog, filename); - exit (EXIT_FAILURE); - } -- if (S_ISBLK (stat.st_mode) || S_ISREG (stat.st_mode)) -- return file_create (filename, fd, stat.st_size, S_ISBLK (stat.st_mode), d); -- else { -- /* Probably stdin/stdout, a pipe or a socket. */ -+ if (S_ISREG (stat.st_mode)) /* Regular file. */ -+ return file_create (filename, fd, -+ stat.st_size, (uint64_t) stat.st_blksize, false, d); -+ else if (S_ISBLK (stat.st_mode)) { /* Block device. */ -+ unsigned int blkioopt; -+ -+#ifdef BLKIOOPT -+ if (ioctl (fd, BLKIOOPT, &blkioopt) == -1) { -+ fprintf (stderr, "warning: cannot get optimal I/O size: %s: %m", -+ filename); -+ blkioopt = 4096; -+ } -+#else -+ blkioopt = 4096; -+#endif -+ -+ return file_create (filename, fd, -+ stat.st_size, (uint64_t) blkioopt, true, d); -+ } -+ else { /* Probably stdin/stdout, a pipe or a socket. */ - synchronous = true; /* Force synchronous mode for pipes. */ - return pipe_create (filename, fd); - } -@@ -528,8 +544,9 @@ print_rw (struct rw *rw, const char *prefix, FILE *fp) - char buf[HUMAN_SIZE_LONGEST]; - - fprintf (fp, "%s: %s \"%s\"\n", prefix, rw->ops->ops_name, rw->name); -- fprintf (fp, "%s: size=%" PRIi64 " (%s)\n", -- prefix, rw->size, human_size (buf, rw->size, NULL)); -+ fprintf (fp, "%s: size=%" PRIi64 " (%s), preferred block size=%" PRIu64 "\n", -+ prefix, rw->size, human_size (buf, rw->size, NULL), -+ rw->preferred); - } - - /* Default implementation of rw->ops->get_extents for backends which -diff --git a/copy/nbd-ops.c b/copy/nbd-ops.c -index 3bc26ba..0988634 100644 ---- a/copy/nbd-ops.c -+++ b/copy/nbd-ops.c -@@ -112,12 +112,22 @@ open_one_nbd_handle (struct rw_nbd *rwn) - * the same way. - */ - if (rwn->handles.len == 0) { -+ int64_t block_size; -+ - rwn->can_zero = nbd_can_zero (nbd) > 0; -+ - rwn->rw.size = nbd_get_size (nbd); - if (rwn->rw.size == -1) { - fprintf (stderr, "%s: %s: %s\n", prog, rwn->rw.name, nbd_get_error ()); - exit (EXIT_FAILURE); - } -+ -+ block_size = nbd_get_block_size (nbd, LIBNBD_SIZE_PREFERRED); -+ if (block_size == -1) { -+ fprintf (stderr, "%s: %s: %s\n", prog, rwn->rw.name, nbd_get_error ()); -+ exit (EXIT_FAILURE); -+ } -+ rwn->rw.preferred = block_size == 0 ? 4096 : block_size; - } - - if (handles_append (&rwn->handles, nbd) == -1) { -diff --git a/copy/nbdcopy.h b/copy/nbdcopy.h -index 19797df..9438cce 100644 ---- a/copy/nbdcopy.h -+++ b/copy/nbdcopy.h -@@ -43,6 +43,7 @@ struct rw { - struct rw_ops *ops; /* Operations. */ - const char *name; /* Printable name, for error messages etc. */ - int64_t size; /* May be -1 for streams. */ -+ uint64_t preferred; /* Preferred block size. */ - /* Followed by private data for the particular subtype. */ - }; - -@@ -53,7 +54,8 @@ typedef enum { READING, WRITING } direction; - - /* Create subtypes. */ - extern struct rw *file_create (const char *name, int fd, -- off_t st_size, bool is_block, direction d); -+ off_t st_size, uint64_t preferred, -+ bool is_block, direction d); - extern struct rw *nbd_rw_create_uri (const char *name, - const char *uri, direction d); - extern struct rw *nbd_rw_create_subprocess (const char **argv, size_t argc, -diff --git a/copy/null-ops.c b/copy/null-ops.c -index 1218a62..99cc9a7 100644 ---- a/copy/null-ops.c -+++ b/copy/null-ops.c -@@ -45,6 +45,7 @@ null_create (const char *name) - rw->rw.ops = &null_ops; - rw->rw.name = name; - rw->rw.size = INT64_MAX; -+ rw->rw.preferred = 4096; - return &rw->rw; - } - -diff --git a/copy/pipe-ops.c b/copy/pipe-ops.c -index 3c8b6c2..3815f82 100644 ---- a/copy/pipe-ops.c -+++ b/copy/pipe-ops.c -@@ -43,6 +43,7 @@ pipe_create (const char *name, int fd) - rwp->rw.ops = &pipe_ops; - rwp->rw.name = name; - rwp->rw.size = -1; -+ rwp->rw.preferred = 4096; - rwp->fd = fd; - return &rwp->rw; - } --- -2.31.1 - diff --git a/0007-copy-Use-preferred-block-size-for-copying.patch b/0007-copy-Use-preferred-block-size-for-copying.patch deleted file mode 100644 index 577f8f1..0000000 --- a/0007-copy-Use-preferred-block-size-for-copying.patch +++ /dev/null @@ -1,457 +0,0 @@ -From c8626acc63c4ae1c6cf5d1505e0209ac10f44e81 Mon Sep 17 00:00:00 2001 -From: "Richard W.M. Jones" -Date: Tue, 28 Jun 2022 21:58:55 +0100 -Subject: [PATCH] copy: Use preferred block size for copying - -You're not supposed to read or write NBD servers at a granularity less -than the advertised minimum block size. nbdcopy has ignored this -requirement, and this is usually fine because the NBD servers we care -about support 512-byte sector granularity, and never advertise sizes / -extents less granular than sectors (even if it's a bit suboptimal in a -few cases). - -However there is one new case where we do care: When writing to a -compressed qcow2 file, qemu advertises a minimum and preferred block -size of 64K, and it really means it. You cannot write blocks smaller -than this because of the way qcow2 compression is implemented. - -This commit attempts to do the least work possible to fix this. - -The previous multi-thread-copying loop was driven by the extent map -received from the source. I have modified the loop so that it -iterates over request_size blocks. request_size is set from the -command line (--request-size) but will be adjusted upwards if either -the source or destination preferred block size is larger. So this -will always copy blocks which are at least the preferred block size -(except for the very last block of the disk). - -While copying these blocks we consult the source extent map. If it -contains only zero regions covering the whole block (only_zeroes -function) then we can skip straight to zeroing the target -(fill_dst_range_with_zeroes), else we do read + write as before. - -I only modified the multi-thread-copying loop, not the synchronous -loop. That should be updated in the same way later. - -One side effect of this change is it always makes larger requests, -even for regions we know are sparse. This is clear in the -copy-sparse.sh and copy-sparse-allocated.sh tests which were -previously driven by the 32K sparse map granularity of the source. -Without changing these tests, they would make make 256K reads & writes -(and also read from areas of the disk even though we know they are -sparse). I adjusted these tests to use --request-size=32768 to force -the existing behaviour. - -Note this doesn't attempt to limit the maximum block size when reading -or writing. That is for future work. - -This is a partial fix for https://bugzilla.redhat.com/2047660. -Further changes will be required in virt-v2v. - -Link: https://lists.gnu.org/archive/html/qemu-block/2022-01/threads.html#00729 -Link: https://bugzilla.redhat.com/show_bug.cgi?id=2047660 -(cherry picked from commit 4058fe1ff03fb41156b67302ba1006b9d06b0218) ---- - TODO | 4 +- - copy/Makefile.am | 6 +- - copy/copy-file-to-qcow2-compressed.sh | 64 +++++++++++ - copy/copy-sparse-allocated.sh | 4 +- - copy/copy-sparse.sh | 7 +- - copy/main.c | 13 +++ - copy/multi-thread-copying.c | 149 +++++++++++++++++++------- - copy/nbdcopy.pod | 5 +- - 8 files changed, 202 insertions(+), 50 deletions(-) - create mode 100755 copy/copy-file-to-qcow2-compressed.sh - -diff --git a/TODO b/TODO -index 7c9c15e..bc38d70 100644 ---- a/TODO -+++ b/TODO -@@ -28,7 +28,9 @@ Performance: Chart it over various buffer sizes and threads, as that - Examine other fuzzers: https://gitlab.com/akihe/radamsa - - nbdcopy: -- - Minimum/preferred/maximum block size. -+ - Enforce maximum block size. -+ - Synchronous loop should be adjusted to take into account -+ the NBD preferred block size, as was done for multi-thread loop. - - Benchmark. - - Better page cache usage, see nbdkit-file-plugin options - fadvise=sequential cache=none. -diff --git a/copy/Makefile.am b/copy/Makefile.am -index e729f86..25f75c5 100644 ---- a/copy/Makefile.am -+++ b/copy/Makefile.am -@@ -23,6 +23,7 @@ EXTRA_DIST = \ - copy-file-to-nbd.sh \ - copy-file-to-null.sh \ - copy-file-to-qcow2.sh \ -+ copy-file-to-qcow2-compressed.sh \ - copy-nbd-to-block.sh \ - copy-nbd-to-file.sh \ - copy-nbd-to-hexdump.sh \ -@@ -142,7 +143,10 @@ TESTS += \ - $(NULL) - - if HAVE_QEMU_NBD --TESTS += copy-file-to-qcow2.sh -+TESTS += \ -+ copy-file-to-qcow2.sh \ -+ copy-file-to-qcow2-compressed.sh \ -+ $(NULL) - endif - - if HAVE_GNUTLS -diff --git a/copy/copy-file-to-qcow2-compressed.sh b/copy/copy-file-to-qcow2-compressed.sh -new file mode 100755 -index 0000000..dfe4fa5 ---- /dev/null -+++ b/copy/copy-file-to-qcow2-compressed.sh -@@ -0,0 +1,64 @@ -+#!/usr/bin/env bash -+# nbd client library in userspace -+# Copyright (C) 2020-2022 Red Hat Inc. -+# -+# This library is free software; you can redistribute it and/or -+# modify it under the terms of the GNU Lesser General Public -+# License as published by the Free Software Foundation; either -+# version 2 of the License, or (at your option) any later version. -+# -+# This library is distributed in the hope that it will be useful, -+# but WITHOUT ANY WARRANTY; without even the implied warranty of -+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -+# Lesser General Public License for more details. -+# -+# You should have received a copy of the GNU Lesser General Public -+# License along with this library; if not, write to the Free Software -+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA -+ -+. ../tests/functions.sh -+ -+set -e -+set -x -+ -+requires $QEMU_NBD --version -+requires nbdkit --exit-with-parent --version -+requires nbdkit sparse-random --dump-plugin -+requires qemu-img --version -+requires stat --version -+ -+file1=copy-file-to-qcow2-compressed.file1 -+file2=copy-file-to-qcow2-compressed.file2 -+rm -f $file1 $file2 -+cleanup_fn rm -f $file1 $file2 -+ -+size=1G -+seed=$RANDOM -+ -+# Create a compressed qcow2 file1. -+# -+# sparse-random files should compress easily because by default each -+# block uses repeated bytes. -+qemu-img create -f qcow2 $file1 $size -+nbdcopy -- [ nbdkit --exit-with-parent sparse-random $size seed=$seed ] \ -+ [ $QEMU_NBD --image-opts driver=compress,file.driver=qcow2,file.file.driver=file,file.file.filename=$file1 ] -+ -+ls -l $file1 -+ -+# Create an uncompressed qcow2 file2 with the same data. -+qemu-img create -f qcow2 $file2 $size -+nbdcopy -- [ nbdkit --exit-with-parent sparse-random $size seed=$seed ] \ -+ [ $QEMU_NBD --image-opts driver=qcow2,file.driver=file,file.filename=$file2 ] -+ -+ls -l $file2 -+ -+# file1 < file2 (shows the compression is having some effect). -+size1="$( stat -c %s $file1 )" -+size2="$( stat -c %s $file2 )" -+if [ $size1 -ge $size2 ]; then -+ echo "$0: qcow2 compression did not make the file smaller" -+ exit 1 -+fi -+ -+# Logical content of the files should be identical. -+qemu-img compare -f qcow2 $file1 -F qcow2 $file2 -diff --git a/copy/copy-sparse-allocated.sh b/copy/copy-sparse-allocated.sh -index 203c3b9..465e347 100755 ---- a/copy/copy-sparse-allocated.sh -+++ b/copy/copy-sparse-allocated.sh -@@ -17,8 +17,6 @@ - # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - - # Adapted from copy-sparse.sh. --# --# This test depends on the nbdkit default sparse block size (32K). - - . ../tests/functions.sh - -@@ -33,7 +31,7 @@ requires nbdkit eval --version - out=copy-sparse-allocated.out - cleanup_fn rm -f $out - --$VG nbdcopy --allocated -- \ -+$VG nbdcopy --allocated --request-size=32768 -- \ - [ nbdkit --exit-with-parent data data=' - 1 - @1073741823 1 -diff --git a/copy/copy-sparse.sh b/copy/copy-sparse.sh -index 1a6da86..7912a21 100755 ---- a/copy/copy-sparse.sh -+++ b/copy/copy-sparse.sh -@@ -16,8 +16,6 @@ - # License along with this library; if not, write to the Free Software - # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - --# This test depends on the nbdkit default sparse block size (32K). -- - . ../tests/functions.sh - - set -e -@@ -34,8 +32,9 @@ cleanup_fn rm -f $out - # Copy from a sparse data disk to an nbdkit-eval-plugin instance which - # is logging everything. This allows us to see exactly what nbdcopy - # is writing, to ensure it is writing and zeroing the target as --# expected. --$VG nbdcopy -S 0 -- \ -+# expected. Force request size to match nbdkit default sparse -+# allocator block size (32K). -+$VG nbdcopy -S 0 --request-size=32768 -- \ - [ nbdkit --exit-with-parent data data=' - 1 - @1073741823 1 -diff --git a/copy/main.c b/copy/main.c -index 19ec384..0e27db8 100644 ---- a/copy/main.c -+++ b/copy/main.c -@@ -40,6 +40,7 @@ - - #include "ispowerof2.h" - #include "human-size.h" -+#include "minmax.h" - #include "version.h" - #include "nbdcopy.h" - -@@ -379,10 +380,22 @@ main (int argc, char *argv[]) - if (threads < connections) - connections = threads; - -+ /* request_size must always be at least as large as the preferred -+ * size of source & destination. -+ */ -+ request_size = MAX (request_size, src->preferred); -+ request_size = MAX (request_size, dst->preferred); -+ - /* Adapt queue to size to request size if needed. */ - if (request_size > queue_size) - queue_size = request_size; - -+ /* Sparse size (if using) must not be smaller than the destination -+ * preferred size, otherwise we end up creating too small requests. -+ */ -+ if (sparse_size > 0 && sparse_size < dst->preferred) -+ sparse_size = dst->preferred; -+ - /* Truncate the destination to the same size as the source. Only - * has an effect on regular files. - */ -diff --git a/copy/multi-thread-copying.c b/copy/multi-thread-copying.c -index 06cdb8e..9267545 100644 ---- a/copy/multi-thread-copying.c -+++ b/copy/multi-thread-copying.c -@@ -166,6 +166,62 @@ decrease_queue_size (struct worker *worker, size_t len) - worker->queue_size -= len; - } - -+/* Using the extents map 'exts', check if the region -+ * [offset..offset+len-1] intersects only with zero extents. -+ * -+ * The invariant for '*i' is always an extent which starts before or -+ * equal to the current offset. -+ */ -+static bool -+only_zeroes (const extent_list exts, size_t *i, -+ uint64_t offset, unsigned len) -+{ -+ size_t j; -+ -+ /* Invariant. */ -+ assert (*i < exts.len); -+ assert (exts.ptr[*i].offset <= offset); -+ -+ /* Update the invariant. Search for the last possible extent in the -+ * list which is <= offset. -+ */ -+ for (j = *i + 1; j < exts.len; ++j) { -+ if (exts.ptr[j].offset <= offset) -+ *i = j; -+ else -+ break; -+ } -+ -+ /* Check invariant again. */ -+ assert (*i < exts.len); -+ assert (exts.ptr[*i].offset <= offset); -+ -+ /* If *i is not the last extent, then the next extent starts -+ * strictly beyond our current offset. -+ */ -+ assert (*i == exts.len - 1 || exts.ptr[*i + 1].offset > offset); -+ -+ /* Search forward, look for any non-zero extents overlapping the region. */ -+ for (j = *i; j < exts.len; ++j) { -+ uint64_t start, end; -+ -+ /* [start..end-1] is the current extent. */ -+ start = exts.ptr[j].offset; -+ end = exts.ptr[j].offset + exts.ptr[j].length; -+ -+ assert (end > offset); -+ -+ if (start >= offset + len) -+ break; -+ -+ /* Non-zero extent covering this region => test failed. */ -+ if (!exts.ptr[j].zero) -+ return false; -+ } -+ -+ return true; -+} -+ - /* There are 'threads' worker threads, each copying work ranges from - * src to dst until there are no more work ranges. - */ -@@ -177,7 +233,10 @@ worker_thread (void *wp) - extent_list exts = empty_vector; - - while (get_next_offset (&offset, &count)) { -- size_t i; -+ struct command *command; -+ size_t extent_index; -+ bool is_zeroing = false; -+ uint64_t zeroing_start = 0; /* initialized to avoid bogus GCC warning */ - - assert (0 < count && count <= THREAD_WORK_SIZE); - if (extents) -@@ -185,52 +244,64 @@ worker_thread (void *wp) - else - default_get_extents (src, w->index, offset, count, &exts); - -- for (i = 0; i < exts.len; ++i) { -- struct command *command; -- size_t len; -+ extent_index = 0; // index into extents array used to optimize only_zeroes -+ while (count) { -+ const size_t len = MIN (count, request_size); - -- if (exts.ptr[i].zero) { -+ if (only_zeroes (exts, &extent_index, offset, len)) { - /* The source is zero so we can proceed directly to skipping, -- * fast zeroing, or writing zeroes at the destination. -+ * fast zeroing, or writing zeroes at the destination. Defer -+ * zeroing so we can send it as a single large command. - */ -- command = create_command (exts.ptr[i].offset, exts.ptr[i].length, -- true, w); -- fill_dst_range_with_zeroes (command); -+ if (!is_zeroing) { -+ is_zeroing = true; -+ zeroing_start = offset; -+ } - } -- - else /* data */ { -- /* As the extent might be larger than permitted for a single -- * command, we may have to split this into multiple read -- * requests. -- */ -- while (exts.ptr[i].length > 0) { -- len = exts.ptr[i].length; -- if (len > request_size) -- len = request_size; -- -- command = create_command (exts.ptr[i].offset, len, -- false, w); -- -- wait_for_request_slots (w); -- -- /* NOTE: Must increase the queue size after waiting. */ -- increase_queue_size (w, len); -- -- /* Begin the asynch read operation. */ -- src->ops->asynch_read (src, command, -- (nbd_completion_callback) { -- .callback = finished_read, -- .user_data = command, -- }); -- -- exts.ptr[i].offset += len; -- exts.ptr[i].length -= len; -+ /* If we were in the middle of deferred zeroing, do it now. */ -+ if (is_zeroing) { -+ /* Note that offset-zeroing_start can never exceed -+ * THREAD_WORK_SIZE, so there is no danger of overflowing -+ * size_t. -+ */ -+ command = create_command (zeroing_start, offset-zeroing_start, -+ true, w); -+ fill_dst_range_with_zeroes (command); -+ is_zeroing = false; - } -+ -+ /* Issue the asynchronous read command. */ -+ command = create_command (offset, len, false, w); -+ -+ wait_for_request_slots (w); -+ -+ /* NOTE: Must increase the queue size after waiting. */ -+ increase_queue_size (w, len); -+ -+ /* Begin the asynch read operation. */ -+ src->ops->asynch_read (src, command, -+ (nbd_completion_callback) { -+ .callback = finished_read, -+ .user_data = command, -+ }); - } - -- offset += count; -- count = 0; -- } /* for extents */ -+ offset += len; -+ count -= len; -+ } /* while (count) */ -+ -+ /* If we were in the middle of deferred zeroing, do it now. */ -+ if (is_zeroing) { -+ /* Note that offset-zeroing_start can never exceed -+ * THREAD_WORK_SIZE, so there is no danger of overflowing -+ * size_t. -+ */ -+ command = create_command (zeroing_start, offset - zeroing_start, -+ true, w); -+ fill_dst_range_with_zeroes (command); -+ is_zeroing = false; -+ } - } - - /* Wait for in flight NBD requests to finish. */ -diff --git a/copy/nbdcopy.pod b/copy/nbdcopy.pod -index fd10f7c..f06d112 100644 ---- a/copy/nbdcopy.pod -+++ b/copy/nbdcopy.pod -@@ -182,8 +182,9 @@ Set the maximum number of requests in flight per NBD connection. - =item B<--sparse=>N - - Detect all zero blocks of size N (bytes) and make them sparse on the --output. You can also turn off sparse detection using S>. --The default is 4096 bytes. -+output. You can also turn off sparse detection using S>. The -+default is 4096 bytes, or the destination preferred block size, -+whichever is larger. - - =item B<--synchronous> - --- -2.31.1 - diff --git a/0008-dump-Add-another-example-to-the-manual.patch b/0008-dump-Add-another-example-to-the-manual.patch deleted file mode 100644 index 1a4eb5b..0000000 --- a/0008-dump-Add-another-example-to-the-manual.patch +++ /dev/null @@ -1,29 +0,0 @@ -From 5d21b00dbdd1e1a04317bf16afb8f4d2ceaa470f Mon Sep 17 00:00:00 2001 -From: "Richard W.M. Jones" -Date: Sat, 2 Jul 2022 17:12:46 +0100 -Subject: [PATCH] dump: Add another example to the manual - -(cherry picked from commit be3768b077c9542aba34eb821016c36f31d234af) ---- - dump/nbddump.pod | 5 +++++ - 1 file changed, 5 insertions(+) - -diff --git a/dump/nbddump.pod b/dump/nbddump.pod -index 5d7864d..656a965 100644 ---- a/dump/nbddump.pod -+++ b/dump/nbddump.pod -@@ -57,6 +57,11 @@ For example, to dump out a qcow2 file as raw data: - - nbddump -- [ qemu-nbd -r -f qcow2 file.qcow2 ] - -+To dump out an empty floppy disk created by L: -+ -+ mkdir /var/tmp/empty -+ nbddump -- [ nbdkit floppy /var/tmp/empty ] -+ - Note that S> are separate parameters, and must be - surrounded by spaces. C<--> separates nbddump parameters from - subprocess parameters. --- -2.31.1 - diff --git a/0009-lib-crypto-Use-GNUTLS_NO_SIGNAL-if-available.patch b/0009-lib-crypto-Use-GNUTLS_NO_SIGNAL-if-available.patch deleted file mode 100644 index e33ee16..0000000 --- a/0009-lib-crypto-Use-GNUTLS_NO_SIGNAL-if-available.patch +++ /dev/null @@ -1,93 +0,0 @@ -From a432e773e0cdc24cb27ccdda4111744ea2c3b819 Mon Sep 17 00:00:00 2001 -From: "Richard W.M. Jones" -Date: Wed, 27 Jul 2022 17:08:14 +0100 -Subject: [PATCH] lib/crypto: Use GNUTLS_NO_SIGNAL if available - -libnbd has long used MSG_NOSIGNAL to avoid receiving SIGPIPE if we -accidentally write on a closed socket, which is a nice alternative to -using a SIGPIPE signal handler. However with TLS connections, gnutls -did not use this flag and so programs using libnbd + TLS would receive -SIGPIPE in some situations, notably if the server closed the -connection abruptly while we were trying to write something. - -GnuTLS 3.4.2 introduces GNUTLS_NO_SIGNAL which does the same thing. -Use this flag if available. - -RHEL 7 has an older gnutls which lacks this flag. To avoid qemu-nbd -interop tests failing (rarely, but more often with a forthcoming -change to TLS shutdown behaviour), register a SIGPIPE signal handler -in the test if the flag is missing. ---- - configure.ac | 15 +++++++++++++++ - interop/interop.c | 10 ++++++++++ - lib/crypto.c | 7 ++++++- - 3 files changed, 31 insertions(+), 1 deletion(-) - -diff --git a/configure.ac b/configure.ac -index 49ca8ab..6bd9e1b 100644 ---- a/configure.ac -+++ b/configure.ac -@@ -179,6 +179,21 @@ AS_IF([test "$GNUTLS_LIBS" != ""],[ - gnutls_session_set_verify_cert \ - gnutls_transport_is_ktls_enabled \ - ]) -+ AC_MSG_CHECKING([if gnutls has GNUTLS_NO_SIGNAL]) -+ AC_COMPILE_IFELSE( -+ [AC_LANG_PROGRAM([ -+ #include -+ gnutls_session_t session; -+ ], [ -+ gnutls_init(&session, GNUTLS_CLIENT|GNUTLS_NO_SIGNAL); -+ ]) -+ ], [ -+ AC_MSG_RESULT([yes]) -+ AC_DEFINE([HAVE_GNUTLS_NO_SIGNAL], [1], -+ [GNUTLS_NO_SIGNAL found at compile time]) -+ ], [ -+ AC_MSG_RESULT([no]) -+ ]) - LIBS="$old_LIBS" - ]) - -diff --git a/interop/interop.c b/interop/interop.c -index b41f3ca..036545b 100644 ---- a/interop/interop.c -+++ b/interop/interop.c -@@ -84,6 +84,16 @@ main (int argc, char *argv[]) - REQUIRES - #endif - -+ /* Ignore SIGPIPE. We only need this for GnuTLS < 3.4.2, since -+ * newer GnuTLS has the GNUTLS_NO_SIGNAL flag which adds -+ * MSG_NOSIGNAL to each write call. -+ */ -+#if !HAVE_GNUTLS_NO_SIGNAL -+#if TLS -+ signal (SIGPIPE, SIG_IGN); -+#endif -+#endif -+ - /* Create a large sparse temporary file. */ - #ifdef NEEDS_TMPFILE - int fd = mkstemp (TMPFILE); -diff --git a/lib/crypto.c b/lib/crypto.c -index 1272888..ca9520e 100644 ---- a/lib/crypto.c -+++ b/lib/crypto.c -@@ -588,7 +588,12 @@ nbd_internal_crypto_create_session (struct nbd_handle *h, - gnutls_psk_client_credentials_t pskcreds = NULL; - gnutls_certificate_credentials_t xcreds = NULL; - -- err = gnutls_init (&session, GNUTLS_CLIENT|GNUTLS_NONBLOCK); -+ err = gnutls_init (&session, -+ GNUTLS_CLIENT | GNUTLS_NONBLOCK -+#if HAVE_GNUTLS_NO_SIGNAL -+ | GNUTLS_NO_SIGNAL -+#endif -+ ); - if (err < 0) { - set_error (errno, "gnutls_init: %s", gnutls_strerror (err)); - return NULL; --- -2.31.1 - diff --git a/0010-lib-crypto.c-Ignore-TLS-premature-termination-after-.patch b/0010-lib-crypto.c-Ignore-TLS-premature-termination-after-.patch deleted file mode 100644 index cc91332..0000000 --- a/0010-lib-crypto.c-Ignore-TLS-premature-termination-after-.patch +++ /dev/null @@ -1,100 +0,0 @@ -From 8bbee9c0ff052cf8ab5ba81fd1b67e3c45e7012a Mon Sep 17 00:00:00 2001 -From: "Richard W.M. Jones" -Date: Wed, 27 Jul 2022 16:07:37 +0100 -Subject: [PATCH] lib/crypto.c: Ignore TLS premature termination after write - shutdown - -qemu-nbd doesn't call gnutls_bye to cleanly shut down the connection -after we send NBD_CMD_DISC. When copying from a qemu-nbd server (or -any operation which calls nbd_shutdown) you will see errors like this: - - $ nbdcopy nbds://foo?tls-certificates=/var/tmp/pki null: - nbds://foo?tls-certificates=/var/tmp/pki: nbd_shutdown: gnutls_record_recv: The TLS connection was non-properly terminated. - -Relatedly you may also see: - - nbd_shutdown: gnutls_record_recv: Error in the pull function. - -This commit suppresses the error in the case where we know that we -have shut down writes (which happens after NBD_CMD_DISC has been sent -on the wire). ---- - interop/interop.c | 9 --------- - lib/crypto.c | 17 +++++++++++++++++ - lib/internal.h | 1 + - 3 files changed, 18 insertions(+), 9 deletions(-) - -diff --git a/interop/interop.c b/interop/interop.c -index 036545b..cce9407 100644 ---- a/interop/interop.c -+++ b/interop/interop.c -@@ -226,19 +226,10 @@ main (int argc, char *argv[]) - - /* XXX In future test more operations here. */ - --#if !TLS -- /* XXX qemu doesn't shut down the connection nicely (using -- * gnutls_bye) and because of this the following call will fail -- * with: -- * -- * nbd_shutdown: gnutls_record_recv: The TLS connection was -- * non-properly terminated. -- */ - if (nbd_shutdown (nbd, 0) == -1) { - fprintf (stderr, "%s\n", nbd_get_error ()); - exit (EXIT_FAILURE); - } --#endif - - nbd_close (nbd); - -diff --git a/lib/crypto.c b/lib/crypto.c -index ca9520e..aa5d820 100644 ---- a/lib/crypto.c -+++ b/lib/crypto.c -@@ -187,6 +187,22 @@ tls_recv (struct nbd_handle *h, struct socket *sock, void *buf, size_t len) - errno = EAGAIN; - return -1; - } -+ if (h->tls_shut_writes && -+ (r == GNUTLS_E_PULL_ERROR || r == GNUTLS_E_PREMATURE_TERMINATION)) { -+ /* qemu-nbd doesn't call gnutls_bye to cleanly shut down the -+ * connection after we send NBD_CMD_DISC, instead it simply -+ * closes the connection. On the client side we see -+ * "gnutls_record_recv: The TLS connection was non-properly -+ * terminated" or "gnutls_record_recv: Error in the pull -+ * function.". -+ * -+ * If we see these errors after we shut down the write side -+ * (h->tls_shut_writes), which happens after we have sent -+ * NBD_CMD_DISC on the wire, downgrade them to a debug message. -+ */ -+ debug (h, "gnutls_record_recv: %s", gnutls_strerror (r)); -+ return 0; /* EOF */ -+ } - set_error (0, "gnutls_record_recv: %s", gnutls_strerror (r)); - errno = EIO; - return -1; -@@ -234,6 +250,7 @@ tls_shut_writes (struct nbd_handle *h, struct socket *sock) - return false; - if (r != 0) - debug (h, "ignoring gnutls_bye failure: %s", gnutls_strerror (r)); -+ h->tls_shut_writes = true; - return sock->u.tls.oldsock->ops->shut_writes (h, sock->u.tls.oldsock); - } - -diff --git a/lib/internal.h b/lib/internal.h -index 6aaced3..f1b4c63 100644 ---- a/lib/internal.h -+++ b/lib/internal.h -@@ -307,6 +307,7 @@ struct nbd_handle { - struct command *reply_cmd; - - bool disconnect_request; /* True if we've queued NBD_CMD_DISC */ -+ bool tls_shut_writes; /* Used by lib/crypto.c to track disconnect. */ - }; - - struct meta_context { --- -2.31.1 - diff --git a/gating.yaml b/gating.yaml old mode 100755 new mode 100644 index 648918d..6985029 --- a/gating.yaml +++ b/gating.yaml @@ -1,6 +1,6 @@ --- !Policy product_versions: - - rhel-9 + - rhel-8 decision_context: osci_compose_gate rules: - !PassingTestCaseRule {test_case_name: osci.brew-build.tier0.functional} diff --git a/libguestfs.keyring b/libguestfs.keyring index f0508c8..bb3eb55 100644 Binary files a/libguestfs.keyring and b/libguestfs.keyring differ diff --git a/libnbd-1.6.0.tar.gz.sig b/libnbd-1.6.0.tar.gz.sig deleted file mode 100644 index fa6006d..0000000 --- a/libnbd-1.6.0.tar.gz.sig +++ /dev/null @@ -1,17 +0,0 @@ ------BEGIN PGP SIGNATURE----- - -iQJFBAABCAAvFiEE93dPsa0HSn6Mh2fqkXOPc+G3aKAFAl/3RFQRHHJpY2hAYW5u -ZXhpYS5vcmcACgkQkXOPc+G3aKD9aw/+Pfg3owjJmhTcCyFvuH2lgiiBb+qL2An+ -hsoax6dM5JxzV6x1Ikgn3C8z2+dLRMowo2FrRgpzTwfaS+ngLDipSC04hKl9MhFN -7OPLCm+L7wcP7KUk4cC0qTSHpHkApo2SP3/bD7vVBYZMYSjgUVFcRoqZlRl3N9RF -7XNsxA2YG9bV4Ln3KbB+k2uxIKNUZIVjmEpretVbb+NTKW9C23ZHicSHYB+Eok1M -iTN6j66rYFn0Xb+L2v7jty19tSdYOMbkdSn0KpniURAWevjjVWGqcojMqW4YuAZ5 -h2MpRfyKFyusbsbtX5bjICTu6+AgFFUALKH7ReDs1RY1cEph9XdBLVulXTggxY05 -E3I1Nns1YmjRlV6ky2Abl2e+Doc44mycINRlwL2q8+Q3TqlVVPFXoVTWxIJ6/Uae -tqnEwWIa2wGv3KU1KLNbWTn1z6I8NM/Nj+7pMKDNnxJzFmHEjL94tmG+iNmHsF34 -vWBZ1q7h9EezxHLOPFYDjlpS+IxeuXakbpuTX2jXvi3zSAbr5WmRR1uO8dAiwu9b -RwOHRmVQOFLAAICYTZDmxl42DpWs5Z2aP7eRwpe8/MOSRiAVepjhUD/bsdaFwmBR -8Z7CGNzyTtt+sy5l7cPBYZ+4RdxWgFEBceBbHs06zdlD/Pui288UQVB/0e9AXYOc -wluyWT1v7sA= -=BaN1 ------END PGP SIGNATURE----- diff --git a/sources b/sources index 24b5998..4925abf 100644 --- a/sources +++ b/sources @@ -1,2 +1,2 @@ -SHA512 (libguestfs.keyring) = 297a15edc7c220222b9f650e0a9361ae132d3f0fed04aeb2237a1d9c3f6dac6f336846434f66480faed72635a33f659e849b052e74b88d1508aeff03f8c9a2ac SHA512 (libnbd-1.6.0.tar.gz) = 4bdd8a52bef6fb136bbb4034cd4521952e1a634334ed193e4883e4bf93baa59b01439135527a4cfeaa3ae9cb92ae70a330d93156666b86c048469c1f5b74ce5f +SHA512 (libnbd-1.6.0.tar.gz.sig) = e487ceece815b4e77a1a5977c6f8173599e12478514f0f014b1ffa7e69467411dbdb77c4b2f16aca5e4fa0f1bac35ab7f57e591db6af085d5e87b21448958c69 diff --git a/tests/tests.yml b/tests/tests.yml old mode 100755 new mode 100644