From 23bbe5bb331efcfa608386742defc7802b0384a1 Mon Sep 17 00:00:00 2001 From: "Richard W.M. Jones" Date: Fri, 3 Feb 2023 10:35:40 +0000 Subject: [PATCH] curl: Enable multi-conn for read-only connections resolves: rhbz#2166686 --- ...multi-conn-for-read-only-connections.patch | 95 +++++++++++++++++++ nbdkit.spec | 7 +- 2 files changed, 100 insertions(+), 2 deletions(-) create mode 100644 0004-curl-Enable-multi-conn-for-read-only-connections.patch diff --git a/0004-curl-Enable-multi-conn-for-read-only-connections.patch b/0004-curl-Enable-multi-conn-for-read-only-connections.patch new file mode 100644 index 0000000..f4fe94d --- /dev/null +++ b/0004-curl-Enable-multi-conn-for-read-only-connections.patch @@ -0,0 +1,95 @@ +From 3f74004478d3590840d7eba97a590b7ec954957f Mon Sep 17 00:00:00 2001 +From: "Richard W.M. Jones" +Date: Thu, 2 Feb 2023 13:59:32 +0000 +Subject: [PATCH] curl: Enable multi-conn for read-only connections +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +Comparing before and after this commit shows approximately double the +performance. In other tests this allowed us to download files from +web servers at line speed. + + Benchmark 1: nbdkit -r curl file:/var/tmp/jammy-server-cloudimg-amd64.raw --run "nbdcopy -p \$uri null:" + Time (mean ± σ): 943.8 ms ± 18.8 ms [User: 316.2 ms, System: 1029.7 ms] + Range (min … max): 923.7 ms … 989.2 ms 10 runs + + Benchmark 2: ~/d/nbdkit/nbdkit -r curl file:/var/tmp/jammy-server-cloudimg-amd64.raw --run "nbdcopy -p \$uri null:" + Time (mean ± σ): 455.0 ms ± 6.2 ms [User: 542.2 ms, System: 1824.7 ms] + Range (min … max): 449.1 ms … 471.6 ms 10 runs + + Summary + ' ~/d/nbdkit/nbdkit -r curl file:/var/tmp/jammy-server-cloudimg-amd64.raw --run "nbdcopy -p \$uri null:" ' ran + 2.07 ± 0.05 times faster than ' nbdkit -r curl file:/var/tmp/jammy-server-cloudimg-amd64.raw --run "nbdcopy -p \$uri null:" ' + +Multi-conn is enabled only when we know the connection is read-only: + + $ ./nbdkit -r curl file:/var/tmp/jammy-server-cloudimg-amd64.raw --run ' nbdinfo $uri ' | grep can_multi_conn + can_multi_conn: true + $ ./nbdkit curl file:/var/tmp/jammy-server-cloudimg-amd64.raw --run ' nbdinfo $uri ' | grep can_multi_conn + can_multi_conn: false + +See also: +https://listman.redhat.com/archives/libguestfs/2023-February/030581.html + +Reviewed-by: Eric Blake +(cherry picked from commit bb0f93ad7b9de451874d0c54188bf69cd37c5409) +--- + plugins/curl/curl.c | 14 ++++++++++++++ + plugins/curl/curldefs.h | 1 + + 2 files changed, 15 insertions(+) + +diff --git a/plugins/curl/curl.c b/plugins/curl/curl.c +index e89bea99..eeba5aa4 100644 +--- a/plugins/curl/curl.c ++++ b/plugins/curl/curl.c +@@ -455,6 +455,7 @@ curl_open (int readonly) + nbdkit_error ("calloc: %m"); + return NULL; + } ++ h->readonly = readonly; + + h->c = curl_easy_init (); + if (h->c == NULL) { +@@ -764,6 +765,18 @@ curl_get_size (void *handle) + return h->exportsize; + } + ++/* Multi-conn is safe for read-only connections, but HTTP does not ++ * have any concept of flushing so we cannot use it for read-write ++ * connections. ++ */ ++static int ++curl_can_multi_conn (void *handle) ++{ ++ struct curl_handle *h = handle; ++ ++ return !! h->readonly; ++} ++ + /* NB: The terminology used by libcurl is confusing! + * + * WRITEFUNCTION / write_cb is used when reading from the remote server +@@ -907,6 +920,7 @@ static struct nbdkit_plugin plugin = { + .open = curl_open, + .close = curl_close, + .get_size = curl_get_size, ++ .can_multi_conn = curl_can_multi_conn, + .pread = curl_pread, + .pwrite = curl_pwrite, + }; +diff --git a/plugins/curl/curldefs.h b/plugins/curl/curldefs.h +index f3095f92..9d4949f3 100644 +--- a/plugins/curl/curldefs.h ++++ b/plugins/curl/curldefs.h +@@ -64,6 +64,7 @@ extern const char *user_agent; + /* The per-connection handle. */ + struct curl_handle { + CURL *c; ++ int readonly; + bool accept_range; + int64_t exportsize; + char errbuf[CURL_ERROR_SIZE]; +-- +2.31.1 + diff --git a/nbdkit.spec b/nbdkit.spec index 984376f..46daee2 100644 --- a/nbdkit.spec +++ b/nbdkit.spec @@ -53,7 +53,7 @@ ExclusiveArch: x86_64 Name: nbdkit Version: 1.32.5 -Release: 3%{?dist} +Release: 4%{?dist} Summary: NBD server License: BSD @@ -81,6 +81,7 @@ Source3: copy-patches.sh Patch0001: 0001-ssh-Remove-left-over-comment.patch Patch0002: 0002-ssh-Improve-the-error-message-when-all-authenticatio.patch Patch0003: 0003-luks-Avoid-crash-when-image-does-not-contain-a-LUKS-.patch +Patch0004: 0004-curl-Enable-multi-conn-for-read-only-connections.patch # For automatic RPM Provides generation. # See: https://rpm-software-management.github.io/rpm/manual/dependency_generators.html @@ -1199,7 +1200,7 @@ export LIBGUESTFS_TRACE=1 %changelog -* Tue Jan 10 2023 Richard W.M. Jones - 1.32.5-3 +* Fri Feb 03 2023 Richard W.M. Jones - 1.32.5-4 - Rebase to new stable branch version 1.32.5 resolves: rhbz#2135765 - Move stats filter to new subpackage. @@ -1207,6 +1208,8 @@ export LIBGUESTFS_TRACE=1 resolves: rhbz#2158300 - luks: Avoid crash when image does not contain a LUKS header resolves: rhbz#2159581 +- curl: Enable multi-conn for read-only connections + resolves: rhbz#2166686 * Tue Nov 29 2022 Richard W.M. Jones - 1.30.8-2 - Add support for VDDK 8.0.0