diff --git a/0001-tests-test-connect.c-Skip-if-exit-with-parent-is-not.patch b/0001-tests-test-connect.c-Skip-if-exit-with-parent-is-not.patch new file mode 100644 index 0000000..8acd9c9 --- /dev/null +++ b/0001-tests-test-connect.c-Skip-if-exit-with-parent-is-not.patch @@ -0,0 +1,33 @@ +From 41775572a5beb8ae49287271af31264684b2bbe3 Mon Sep 17 00:00:00 2001 +From: "Richard W.M. Jones" +Date: Fri, 21 Jul 2023 19:00:31 +0100 +Subject: [PATCH] tests/test-connect.c: Skip if --exit-with-parent is not + supported + +Fixes: commit 933d7401ff623077ba43b4ef1e7f16a7864c5fde +(cherry picked from commit 59664b8b146edcba0353de628528e0a2035b3ba7) +--- + tests/test-connect.c | 7 +++++++ + 1 file changed, 7 insertions(+) + +diff --git a/tests/test-connect.c b/tests/test-connect.c +index f463415b..d7118330 100644 +--- a/tests/test-connect.c ++++ b/tests/test-connect.c +@@ -47,6 +47,13 @@ main (int argc, char *argv[]) + struct nbd_handle *nbd; + int64_t size; + ++ if (system ("nbdkit --exit-with-parent --version") != 0) { ++ printf ("%s: --exit-with-parent is not implemented on this platform, " ++ "skipping\n", ++ argv[0]); ++ exit (77); ++ } ++ + nbd = nbd_create (); + if (nbd == NULL) { + fprintf (stderr, "%s\n", nbd_get_error ()); +-- +2.39.3 + diff --git a/0002-tests-Use-exit-with-parent-in-the-test-framework.patch b/0002-tests-Use-exit-with-parent-in-the-test-framework.patch new file mode 100644 index 0000000..367eddb --- /dev/null +++ b/0002-tests-Use-exit-with-parent-in-the-test-framework.patch @@ -0,0 +1,70 @@ +From 93a2c7321dc5ccd36368d50887cf239cfb883a72 Mon Sep 17 00:00:00 2001 +From: "Richard W.M. Jones" +Date: Fri, 21 Jul 2023 19:01:35 +0100 +Subject: [PATCH] tests: Use --exit-with-parent in the test framework + +Make sure nbdkit always exits, in case some other part of the test is +killed. + +(cherry picked from commit c6299889e80217cfadf488a67961be9eb6d24e38) +--- + tests/Makefile.am | 2 ++ + tests/test.c | 22 +++++++++++++--------- + 2 files changed, 15 insertions(+), 9 deletions(-) + +diff --git a/tests/Makefile.am b/tests/Makefile.am +index e1087689..f6c5ac9a 100644 +--- a/tests/Makefile.am ++++ b/tests/Makefile.am +@@ -594,6 +594,8 @@ EXTRA_DIST += \ + check_LTLIBRARIES += libtest.la + libtest_la_SOURCES = test.c test.h + libtest_la_CFLAGS = $(WARNINGS_CFLAGS) ++libtest_la_CPPFLAGS = -I$(top_srcdir)/common/utils ++libtest_la_LIBADD = $(top_builddir)/common/utils/libutils.la + + # Basic connection test. + LIBNBD_TESTS += test-connect +diff --git a/tests/test.c b/tests/test.c +index 67f69fab..6be10f12 100644 +--- a/tests/test.c ++++ b/tests/test.c +@@ -50,6 +50,8 @@ + #include + #endif + ++#include "exit-with-parent.h" ++ + #include "test.h" + + #ifndef WIN32 +@@ -161,15 +163,17 @@ test_start_nbdkit (const char *arg, ...) + const char *argv[MAX_ARGS+1]; + va_list args; + +- argv[0] = "nbdkit"; +- argv[1] = "-U"; +- argv[2] = kit->sockpath; +- argv[3] = "-P"; +- argv[4] = kit->pidpath; +- argv[5] = "-f"; +- argv[6] = "-v"; +- argv[7] = arg; +- i = 8; ++ i = 0; ++ argv[i++] = "nbdkit"; ++ argv[i++] = "-U"; ++ argv[i++] = kit->sockpath; ++ argv[i++] = "-P"; ++ argv[i++] = kit->pidpath; ++ argv[i++] = "-f"; ++ argv[i++] = "-v"; ++ if (can_exit_with_parent ()) ++ argv[i++] = "--exit-with-parent"; ++ argv[i++] = arg; + + va_start (args, arg); + while ((p = va_arg (args, const char *)) != NULL) { +-- +2.39.3 + diff --git a/0003-protect-Fix-copy-and-paste-error-in-the-documentatio.patch b/0003-protect-Fix-copy-and-paste-error-in-the-documentatio.patch new file mode 100644 index 0000000..41608e4 --- /dev/null +++ b/0003-protect-Fix-copy-and-paste-error-in-the-documentatio.patch @@ -0,0 +1,26 @@ +From e3a62ebad571d6512501941c49f7a8ab66ef5646 Mon Sep 17 00:00:00 2001 +From: "Richard W.M. Jones" +Date: Sat, 22 Jul 2023 09:50:44 +0100 +Subject: [PATCH] protect: Fix copy and paste error in the documentation + +(cherry picked from commit f04607c95c27c4342f41d083ad00ac02c579b391) +--- + filters/protect/nbdkit-protect-filter.pod | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/filters/protect/nbdkit-protect-filter.pod b/filters/protect/nbdkit-protect-filter.pod +index b5ffc7ca..aad72a68 100644 +--- a/filters/protect/nbdkit-protect-filter.pod ++++ b/filters/protect/nbdkit-protect-filter.pod +@@ -102,7 +102,7 @@ Use C to find the location of C<$filterdir>. + + =head1 VERSION + +-C first appeared in nbdkit 1.30. ++C first appeared in nbdkit 1.30. + + =head1 SEE ALSO + +-- +2.39.3 + diff --git a/0004-docs-nbdkit-protocol.pod-Fix-manual-page-name.patch b/0004-docs-nbdkit-protocol.pod-Fix-manual-page-name.patch new file mode 100644 index 0000000..f568fc9 --- /dev/null +++ b/0004-docs-nbdkit-protocol.pod-Fix-manual-page-name.patch @@ -0,0 +1,26 @@ +From c6da52fd3d9a9e5e8ad4648bba686c37b5d3fae7 Mon Sep 17 00:00:00 2001 +From: "Richard W.M. Jones" +Date: Sun, 23 Jul 2023 20:15:23 +0100 +Subject: [PATCH] docs/nbdkit-protocol.pod: Fix manual page name + +Updates: commit dce32051318c3e8bd20825cc0db5df15f95d11fa +(cherry picked from commit 00aa248cec0a64ca293bbedffbf47426ee2b2340) +--- + docs/nbdkit-protocol.pod | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/docs/nbdkit-protocol.pod b/docs/nbdkit-protocol.pod +index 7e89a607..dabdf23c 100644 +--- a/docs/nbdkit-protocol.pod ++++ b/docs/nbdkit-protocol.pod +@@ -1,6 +1,6 @@ + =head1 NAME + +-nbdkit - which parts of the NBD protocol nbdkit supports ++nbdkit-protocol - which parts of the NBD protocol nbdkit supports + + =head1 SYNOPSIS + +-- +2.39.3 + diff --git a/0005-retry-request-Print-operation-we-are-retrying-in-deb.patch b/0005-retry-request-Print-operation-we-are-retrying-in-deb.patch new file mode 100644 index 0000000..0ff2d65 --- /dev/null +++ b/0005-retry-request-Print-operation-we-are-retrying-in-deb.patch @@ -0,0 +1,110 @@ +From e206e3a645ee6d350df210d2ef95614d3f5d97e7 Mon Sep 17 00:00:00 2001 +From: "Richard W.M. Jones" +Date: Fri, 28 Jul 2023 16:43:07 +0100 +Subject: [PATCH] retry-request: Print operation we are retrying in debug + messages + +Reviewed-by: Eric Blake +(cherry picked from commit dbc91fd2eac5eac77293e3deded92e11c0b8d6e3) +--- + filters/retry-request/retry-request.c | 22 +++++++++++----------- + 1 file changed, 11 insertions(+), 11 deletions(-) + +diff --git a/filters/retry-request/retry-request.c b/filters/retry-request/retry-request.c +index ed566080..e5b8344c 100644 +--- a/filters/retry-request/retry-request.c ++++ b/filters/retry-request/retry-request.c +@@ -100,15 +100,15 @@ retry_request_config (nbdkit_next_config *next, nbdkit_backend *nxdata, + * The code between RETRY_START...RETRY_END must set r to 0 or -1 on + * success or failure. *err may also be implicitly assigned. + */ +-#define RETRY_START \ ++#define RETRY_START(what) \ + { \ + unsigned i; \ + \ + r = -1; \ + for (i = 0; r == -1 && i <= retries; ++i) { \ + if (i > 0) { \ +- nbdkit_debug ("retry %u: waiting %u seconds before retrying", \ +- i, delay); \ ++ nbdkit_debug ("retry %u: waiting %u seconds before retrying %s",\ ++ i, delay, what); \ + if (nbdkit_nanosleep (delay, 0) == -1) { \ + if (*err == 0) \ + *err = errno; \ +@@ -130,7 +130,7 @@ retry_request_open (nbdkit_next_open *next, nbdkit_context *nxdata, + if (retry_open_call) { + int *err = &errno; /* used by the RETRY_* macros */ + +- RETRY_START ++ RETRY_START("open") + r = next (nxdata, readonly, exportname); + RETRY_END; + } +@@ -148,7 +148,7 @@ retry_request_pread (nbdkit_next *next, + { + int r; + +- RETRY_START ++ RETRY_START("pread") + r = next->pread (next, buf, count, offset, flags, err); + RETRY_END; + return r; +@@ -162,7 +162,7 @@ retry_request_pwrite (nbdkit_next *next, + { + int r; + +- RETRY_START ++ RETRY_START("pwrite") + r = next->pwrite (next, buf, count, offset, flags, err); + RETRY_END; + return r; +@@ -176,7 +176,7 @@ retry_request_trim (nbdkit_next *next, + { + int r; + +- RETRY_START ++ RETRY_START("trim") + r = next->trim (next, count, offset, flags, err); + RETRY_END; + return r; +@@ -189,7 +189,7 @@ retry_request_flush (nbdkit_next *next, + { + int r; + +- RETRY_START ++ RETRY_START("flush") + r = next->flush (next, flags, err); + RETRY_END; + return r; +@@ -203,7 +203,7 @@ retry_request_zero (nbdkit_next *next, + { + int r; + +- RETRY_START ++ RETRY_START("zero") + r = next->zero (next, count, offset, flags, err); + RETRY_END; + return r; +@@ -218,7 +218,7 @@ retry_request_extents (nbdkit_next *next, + CLEANUP_EXTENTS_FREE struct nbdkit_extents *extents2 = NULL; + int r; + +- RETRY_START { ++ RETRY_START("extents") { + /* Each retry must begin with extents reset to the right beginning. */ + nbdkit_extents_free (extents2); + extents2 = nbdkit_extents_new (offset, next->get_size (next)); +@@ -254,7 +254,7 @@ retry_request_cache (nbdkit_next *next, + { + int r; + +- RETRY_START ++ RETRY_START("cache") + r = next->cache (next, count, offset, flags, err); + RETRY_END; + return r; +-- +2.39.3 + diff --git a/0006-tests-test-ocaml-errorcodes.c-Don-t-use-assert-in-te.patch b/0006-tests-test-ocaml-errorcodes.c-Don-t-use-assert-in-te.patch new file mode 100644 index 0000000..64f4dcd --- /dev/null +++ b/0006-tests-test-ocaml-errorcodes.c-Don-t-use-assert-in-te.patch @@ -0,0 +1,53 @@ +From fdd27622047cd8e25af33b66a57899b75e99db82 Mon Sep 17 00:00:00 2001 +From: "Richard W.M. Jones" +Date: Tue, 1 Aug 2023 11:42:58 +0100 +Subject: [PATCH] tests/test-ocaml-errorcodes.c: Don't use assert in test + +exit with EXIT_FAILURE instead, as the program might be compiled with +-DNDEBUG. + +(cherry picked from commit 678410e23841376d8b742dce3d10c632499367a2) +--- + tests/test-ocaml-errorcodes.c | 16 +++++++++++++--- + 1 file changed, 13 insertions(+), 3 deletions(-) + +diff --git a/tests/test-ocaml-errorcodes.c b/tests/test-ocaml-errorcodes.c +index e59f074b..8ab7e0ae 100644 +--- a/tests/test-ocaml-errorcodes.c ++++ b/tests/test-ocaml-errorcodes.c +@@ -34,8 +34,9 @@ + + #include + #include ++#include ++#include + #include +-#include + + #include + +@@ -84,10 +85,19 @@ main (int argc, char *argv[]) + exit (EXIT_FAILURE); + } + +- assert (nbd_pread (nbd, buf, 512, 0, 0) == 0); ++ if (nbd_pread (nbd, buf, 512, 0, 0) == -1) { ++ fprintf (stderr, "%s: FAIL: did not expect reading sector 0 to fail\n", ++ argv[0]); ++ exit (EXIT_FAILURE); ++ } + + for (i = 0; tests[i].offset != 0; ++i) { +- assert (nbd_pread (nbd, buf, 512, tests[i].offset, 0) == -1); ++ if (nbd_pread (nbd, buf, 512, tests[i].offset, 0) != -1) { ++ fprintf (stderr, ++ "%s: FAIL: reading sector %" PRIu64 "should have failed\n", ++ argv[0], tests[i].offset / 512); ++ exit (EXIT_FAILURE); ++ } + actual_errno = nbd_get_errno (); + if (actual_errno != tests[i].expected_errno) { + fprintf (stderr, "%s: FAIL: actual errno = %d expected errno = %d\n", +-- +2.39.3 + diff --git a/0007-tests-test-ocaml-errorcodes.c-Enable-verbose-message.patch b/0007-tests-test-ocaml-errorcodes.c-Enable-verbose-message.patch new file mode 100644 index 0000000..2f54ddd --- /dev/null +++ b/0007-tests-test-ocaml-errorcodes.c-Enable-verbose-message.patch @@ -0,0 +1,30 @@ +From eca5131cbbc7e1785b266cc92624a2ce9582a7cc Mon Sep 17 00:00:00 2001 +From: "Richard W.M. Jones" +Date: Tue, 1 Aug 2023 12:42:05 +0100 +Subject: [PATCH] tests/test-ocaml-errorcodes.c: Enable verbose messages + +This test crashes when the OCaml plugin is unloaded (specifically when +we call caml_acquire_runtime_system). Enable verbose messages in a +failed attempt to track down what is happening. + +(cherry picked from commit 99567408fdd8ea55b0bb5286b45ce135d91c38b5) +--- + tests/test-ocaml-errorcodes.c | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/tests/test-ocaml-errorcodes.c b/tests/test-ocaml-errorcodes.c +index 8ab7e0ae..f84da3d8 100644 +--- a/tests/test-ocaml-errorcodes.c ++++ b/tests/test-ocaml-errorcodes.c +@@ -78,7 +78,7 @@ main (int argc, char *argv[]) + + if (nbd_connect_command (nbd, + (char *[]) { +- "nbdkit", "-s", "--exit-with-parent", ++ "nbdkit", "-s", "--exit-with-parent", "-v", + "./test-ocaml-errorcodes-plugin.so", + NULL }) == -1) { + fprintf (stderr, "%s\n", nbd_get_error ()); +-- +2.39.3 + diff --git a/0008-curl-Use-the-parallel-thread-model.patch b/0008-curl-Use-the-parallel-thread-model.patch new file mode 100644 index 0000000..9a670be --- /dev/null +++ b/0008-curl-Use-the-parallel-thread-model.patch @@ -0,0 +1,80 @@ +From bfa0b1238f6318d06930ec4d389a27932fc90a16 Mon Sep 17 00:00:00 2001 +From: "Richard W.M. Jones" +Date: Sat, 4 Feb 2023 12:34:57 +0000 +Subject: [PATCH] curl: Use the parallel thread model + +After previous changes, it is now safe to use the parallel thread +model in this plugin. The locking in pool.c protects a single curl +handle from being used from multiple threads. + +An advantage of this is we can now combine the curl plugin with +filters such as readahead and scan. + +This pessimizes some workloads and improves others. See my earlier +comments here: + + https://listman.redhat.com/archives/libguestfs/2023-February/030610.html + https://listman.redhat.com/archives/libguestfs/2023-February/030618.html + https://listman.redhat.com/archives/libguestfs/2023-February/030619.html + https://listman.redhat.com/archives/libguestfs/2023-February/030620.html + +Tests below use this basic command: + + $ time nbdkit -r -U - curl https://cloud.debian.org/images/cloud/bookworm/daily/latest/debian-12-generic-amd64-daily.qcow2 \ + --run '$COPY $uri /var/tmp/out' + +where $COPY is either nbdcopy or qemu-img convert. + +Before this change: + + nbdkit + nbdcopy 0m55.397s + nbdkit + qemu-img convert [over 20 minutes] + +After this change: + + nbdkit + nbdcopy 1m1.235s + nbdkit + qemu-img convert 6m3.262s + nbdkit --filter=readahead --filter=cache + qemu-img convert + 8m16.488s + nbdkit connections=8 + qemu-img convert + 3m48.502s + +Previously [see first link above] we noted that file: URLs are +impacted, and indeed they are, with this command: + + $ time nbdkit -r -U - curl file:/var/tmp/fedora-36.img \ + --run 'nbdcopy --no-extents -p "$uri" null:' + +nearly doubling in run-time (0.78 -> 1.34). However I think this may +be a peculiarity of curl's handling of file. (Use nbdkit-file-plugin +instead). + +Note the fundamental issue here is still lack of multiconn support in +qemu's NBD client. + +(cherry picked from commit f2163754860d041c4cb12dace90591c280eccae8) +--- + plugins/curl/curl.c | 7 +------ + 1 file changed, 1 insertion(+), 6 deletions(-) + +diff --git a/plugins/curl/curl.c b/plugins/curl/curl.c +index 58ffb662..425f1d90 100644 +--- a/plugins/curl/curl.c ++++ b/plugins/curl/curl.c +@@ -537,12 +537,7 @@ curl_close (void *handle) + free (h); + } + +-/* This plugin could support the parallel thread model. It currently +- * uses serialize_requests because parallel has the unfortunate effect +- * of pessimising common workloads. See: +- * https://listman.redhat.com/archives/libguestfs/2023-February/030618.html +- */ +-#define THREAD_MODEL NBDKIT_THREAD_MODEL_SERIALIZE_REQUESTS ++#define THREAD_MODEL NBDKIT_THREAD_MODEL_PARALLEL + + /* Calls get_handle() ... put_handle() to get a handle for the length + * of the current scope. +-- +2.39.3 + diff --git a/0009-curl-Add-ipresolve-option.patch b/0009-curl-Add-ipresolve-option.patch new file mode 100644 index 0000000..ddcebff --- /dev/null +++ b/0009-curl-Add-ipresolve-option.patch @@ -0,0 +1,115 @@ +From a5f1e659d52872a499a5744870cb3e6ff382642f Mon Sep 17 00:00:00 2001 +From: "Richard W.M. Jones" +Date: Tue, 25 Jul 2023 16:57:41 +0100 +Subject: [PATCH] curl: Add ipresolve option + +Allows you to force IPv4 or IPv6. + +(cherry picked from commit d1b27c97f7fac00ee452a1d1b05805ee7145a49c) +--- + plugins/curl/curl.c | 15 +++++++++++++++ + plugins/curl/curldefs.h | 1 + + plugins/curl/nbdkit-curl-plugin.pod | 14 ++++++++++++++ + plugins/curl/pool.c | 2 ++ + 4 files changed, 32 insertions(+) + +diff --git a/plugins/curl/curl.c b/plugins/curl/curl.c +index 425f1d90..91fa65fb 100644 +--- a/plugins/curl/curl.c ++++ b/plugins/curl/curl.c +@@ -68,6 +68,7 @@ struct curl_slist *headers = NULL; + const char *header_script = NULL; + unsigned header_script_renew = 0; + long http_version = CURL_HTTP_VERSION_NONE; ++long ipresolve = CURL_IPRESOLVE_WHATEVER; + char *password = NULL; + #ifndef HAVE_CURLOPT_PROTOCOLS_STR + long protocols = CURLPROTO_ALL; +@@ -314,6 +315,19 @@ curl_config (const char *key, const char *value) + } + } + ++ else if (strcmp (key, "ipresolve") == 0) { ++ if (strcmp (value, "any") == 0 || strcmp (value, "whatever") == 0) ++ ipresolve = CURL_IPRESOLVE_WHATEVER; ++ else if (strcmp (value, "v4") == 0 || strcmp (value, "4") == 0) ++ ipresolve = CURL_IPRESOLVE_V4; ++ else if (strcmp (value, "v6") == 0 || strcmp (value, "6") == 0) ++ ipresolve = CURL_IPRESOLVE_V6; ++ else { ++ nbdkit_error ("unknown ipresolve: %s", value); ++ return -1; ++ } ++ } ++ + else if (strcmp (key, "password") == 0) { + free (password); + if (nbdkit_read_password (value, &password) == -1) +@@ -495,6 +509,7 @@ curl_config_complete (void) + "header-script=