rpms/nbdkit
7
0
Fork 0
Code Issues Pull Requests Releases Activity

import CS nbdkit-1.34.2-1.el9

Browse Source
This commit is contained in:
eabdullin 2023-09-21 19:37:40 +00:00
parent 1d28431c3a
commit af3d139bfe
35 changed files with 5252 additions and 272 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.gitignore vendored
View File

@ -1,2 +1,2 @@
SOURCES/libguestfs.keyring
SOURCES/nbdkit-1.32.5.tar.gz
SOURCES/nbdkit-1.34.2.tar.gz

2
.nbdkit.metadata
View File

@ -1,2 +1,2 @@
cc1b37b9cfafa515aab3eefd345ecc59aac2ce7b SOURCES/libguestfs.keyring
c8260e2f6fb16a16cefe0cf670fc5a0f41dd7110 SOURCES/nbdkit-1.32.5.tar.gz
d1c190dcfe1e601e0eeb4862c49df45c7052955b SOURCES/nbdkit-1.34.2.tar.gz

31
SOURCES/0001-ssh-Remove-left-over-comment.patch
View File

@ -1,31 +0,0 @@
From e0e592775911ebe2178b04b4b20f95fea2f2fe9c Mon Sep 17 00:00:00 2001
From: "Richard W.M. Jones" <rjones@redhat.com>
Date: Thu, 5 Jan 2023 16:05:33 +0000
Subject: [PATCH] ssh: Remove left over comment
This comment was left over from when I copied the libssh example code.
It adds no value so remove it.
(cherry picked from commit c93a8957efcc26652b31f5bc359dfd3c4019b4f8)
---
plugins/ssh/ssh.c | 4 ----
1 file changed, 4 deletions(-)
diff --git a/plugins/ssh/ssh.c b/plugins/ssh/ssh.c
index 6cf40c26..aaa7c2b9 100644
--- a/plugins/ssh/ssh.c
+++ b/plugins/ssh/ssh.c
@@ -356,10 +356,6 @@ authenticate (struct ssh_handle *h)
if (rc == SSH_AUTH_SUCCESS) return 0;
}
- /* Example code tries keyboard-interactive here, but we cannot use
- * that method from a server.
- */
-
if (password != NULL && (method & SSH_AUTH_METHOD_PASSWORD)) {
rc = authenticate_password (h->session, password);
if (rc == SSH_AUTH_SUCCESS) return 0;
--
2.31.1

33
SOURCES/0001-tests-test-connect.c-Skip-if-exit-with-parent-is-not.patch Normal file
View File

@ -0,0 +1,33 @@
From 41775572a5beb8ae49287271af31264684b2bbe3 Mon Sep 17 00:00:00 2001
From: "Richard W.M. Jones" <rjones@redhat.com>
Date: Fri, 21 Jul 2023 19:00:31 +0100
Subject: [PATCH] tests/test-connect.c: Skip if --exit-with-parent is not
supported
Fixes: commit 933d7401ff623077ba43b4ef1e7f16a7864c5fde
(cherry picked from commit 59664b8b146edcba0353de628528e0a2035b3ba7)
---
tests/test-connect.c | 7 +++++++
1 file changed, 7 insertions(+)
diff --git a/tests/test-connect.c b/tests/test-connect.c
index f463415b..d7118330 100644
--- a/tests/test-connect.c
+++ b/tests/test-connect.c
@@ -47,6 +47,13 @@ main (int argc, char *argv[])
struct nbd_handle *nbd;
int64_t size;
+ if (system ("nbdkit --exit-with-parent --version") != 0) {
+ printf ("%s: --exit-with-parent is not implemented on this platform, "
+ "skipping\n",
+ argv[0]);
+ exit (77);
+ }
+
nbd = nbd_create ();
if (nbd == NULL) {
fprintf (stderr, "%s\n", nbd_get_error ());
--
2.39.3

68
SOURCES/0002-ssh-Improve-the-error-message-when-all-authenticatio.patch
View File

@ -1,68 +0,0 @@
From 916f90972af60576591dea4a4f1d07e4dae6d9cf Mon Sep 17 00:00:00 2001
From: "Richard W.M. Jones" <rjones@redhat.com>
Date: Thu, 5 Jan 2023 11:29:32 +0000
Subject: [PATCH] ssh: Improve the error message when all authentication
methods fail
The current error message:
nbdkit: ssh[1]: error: all possible authentication methods failed
is confusing and non-actionable. It's hard even for experts to
understand the relationship between the authentication methods offered
by a server and what we require.
Try to improve the error message in some common situations, especially
where password authentication on the server side is disabled but the
client supplied a password=... parameter. After this change, you will
see an actionable error:
nbdkit: ssh[1]: error: the server does not offer password
authentication but you tried to use a password; if you have root
access to the server, try editing 'sshd_config' and setting
'PasswordAuthentication yes'; otherwise try setting up public key
authentication
Fixes: https://bugzilla.redhat.com/show_bug.cgi?id=2158300
Reviewed-by: Laszlo Ersek <lersek@redhat.com>
(cherry picked from commit bea88cff5ac9c42f1a068ad24d43d5ed0506edaa)
---
plugins/ssh/ssh.c | 22 ++++++++++++++++++++++
1 file changed, 22 insertions(+)
diff --git a/plugins/ssh/ssh.c b/plugins/ssh/ssh.c
index aaa7c2b9..5a132d8f 100644
--- a/plugins/ssh/ssh.c
+++ b/plugins/ssh/ssh.c
@@ -361,6 +361,28 @@ authenticate (struct ssh_handle *h)
if (rc == SSH_AUTH_SUCCESS) return 0;
}
+ /* All compatible methods were tried and none worked. Come up with
+ * an actionable diagnostic message if we recognise the problem.
+ */
+ if (!(method & SSH_AUTH_METHOD_PUBLICKEY) && password == NULL) {
+ nbdkit_error ("the server does not offer public key authentication; "
+ "try using the password=... parameter");
+ return -1;
+ }
+ if ((method & SSH_AUTH_METHOD_PASSWORD) && password != NULL) {
+ nbdkit_error ("password authentication failed, "
+ "is the username and password correct?");
+ return -1;
+ }
+ if (!(method & SSH_AUTH_METHOD_PASSWORD) && password != NULL) {
+ nbdkit_error ("the server does not offer password authentication "
+ "but you tried to use a password; if you have root access "
+ "to the server, try editing 'sshd_config' and setting "
+ "'PasswordAuthentication yes'; otherwise try setting up "
+ "public key authentication");
+ return -1;
+ }
+
nbdkit_error ("all possible authentication methods failed");
return -1;
}
--
2.31.1

70
SOURCES/0002-tests-Use-exit-with-parent-in-the-test-framework.patch Normal file
View File

@ -0,0 +1,70 @@
From 93a2c7321dc5ccd36368d50887cf239cfb883a72 Mon Sep 17 00:00:00 2001
From: "Richard W.M. Jones" <rjones@redhat.com>
Date: Fri, 21 Jul 2023 19:01:35 +0100
Subject: [PATCH] tests: Use --exit-with-parent in the test framework
Make sure nbdkit always exits, in case some other part of the test is
killed.
(cherry picked from commit c6299889e80217cfadf488a67961be9eb6d24e38)
---
tests/Makefile.am | 2 ++
tests/test.c | 22 +++++++++++++---------
2 files changed, 15 insertions(+), 9 deletions(-)
diff --git a/tests/Makefile.am b/tests/Makefile.am
index e1087689..f6c5ac9a 100644
--- a/tests/Makefile.am
+++ b/tests/Makefile.am
@@ -594,6 +594,8 @@ EXTRA_DIST += \
check_LTLIBRARIES += libtest.la
libtest_la_SOURCES = test.c test.h
libtest_la_CFLAGS = $(WARNINGS_CFLAGS)
+libtest_la_CPPFLAGS = -I$(top_srcdir)/common/utils
+libtest_la_LIBADD = $(top_builddir)/common/utils/libutils.la
# Basic connection test.
LIBNBD_TESTS += test-connect
diff --git a/tests/test.c b/tests/test.c
index 67f69fab..6be10f12 100644
--- a/tests/test.c
+++ b/tests/test.c
@@ -50,6 +50,8 @@
#include <sys/wait.h>
#endif
+#include "exit-with-parent.h"
+
#include "test.h"
#ifndef WIN32
@@ -161,15 +163,17 @@ test_start_nbdkit (const char *arg, ...)
const char *argv[MAX_ARGS+1];
va_list args;
- argv[0] = "nbdkit";
- argv[1] = "-U";
- argv[2] = kit->sockpath;
- argv[3] = "-P";
- argv[4] = kit->pidpath;
- argv[5] = "-f";
- argv[6] = "-v";
- argv[7] = arg;
- i = 8;
+ i = 0;
+ argv[i++] = "nbdkit";
+ argv[i++] = "-U";
+ argv[i++] = kit->sockpath;
+ argv[i++] = "-P";
+ argv[i++] = kit->pidpath;
+ argv[i++] = "-f";
+ argv[i++] = "-v";
+ if (can_exit_with_parent ())
+ argv[i++] = "--exit-with-parent";
+ argv[i++] = arg;
va_start (args, arg);
while ((p = va_arg (args, const char *)) != NULL) {
--
2.39.3

44
SOURCES/0003-luks-Avoid-crash-when-image-does-not-contain-a-LUKS-.patch
View File

@ -1,44 +0,0 @@
From dc86950fff020688a17b6ff0dbfea7bdb0d8f1b9 Mon Sep 17 00:00:00 2001
From: "Richard W.M. Jones" <rjones@redhat.com>
Date: Tue, 10 Jan 2023 08:39:11 +0000
Subject: [PATCH] luks: Avoid crash when image does not contain a LUKS header
We attempt to load the LUKS header in the prepare() callback. If this
fails, h->h will be NULL and we'll crash in close() when we attempt to
access and free h->h->masterkey.
This crash could have been triggered another way: if open() followed
by close() was called, without prepare() or other callbacks.
Reported-by: Ming Xie
Fixes: https://bugzilla.redhat.com/show_bug.cgi?id=2159581
(cherry picked from commit cad4b96b17ed4ad7882100efa0d9073ac9d8b11c)
---
filters/luks/luks-encryption.c | 10 ++++++----
1 file changed, 6 insertions(+), 4 deletions(-)
diff --git a/filters/luks/luks-encryption.c b/filters/luks/luks-encryption.c
index 26f81e7b..6f33e76e 100644
--- a/filters/luks/luks-encryption.c
+++ b/filters/luks/luks-encryption.c
@@ -856,11 +856,13 @@ load_header (nbdkit_next *next, const char *passphrase)
void
free_luks_data (struct luks_data *h)
{
- if (h->masterkey) {
- memset (h->masterkey, 0, h->phdr.master_key_len);
- free (h->masterkey);
+ if (h) {
+ if (h->masterkey) {
+ memset (h->masterkey, 0, h->phdr.master_key_len);
+ free (h->masterkey);
+ }
+ free (h);
}
- free (h);
}
uint64_t
--
2.31.1

26
SOURCES/0003-protect-Fix-copy-and-paste-error-in-the-documentatio.patch Normal file
View File

@ -0,0 +1,26 @@
From e3a62ebad571d6512501941c49f7a8ab66ef5646 Mon Sep 17 00:00:00 2001
From: "Richard W.M. Jones" <rjones@redhat.com>
Date: Sat, 22 Jul 2023 09:50:44 +0100
Subject: [PATCH] protect: Fix copy and paste error in the documentation
(cherry picked from commit f04607c95c27c4342f41d083ad00ac02c579b391)
---
filters/protect/nbdkit-protect-filter.pod | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/filters/protect/nbdkit-protect-filter.pod b/filters/protect/nbdkit-protect-filter.pod
index b5ffc7ca..aad72a68 100644
--- a/filters/protect/nbdkit-protect-filter.pod
+++ b/filters/protect/nbdkit-protect-filter.pod
@@ -102,7 +102,7 @@ Use C<nbdkit --dump-config> to find the location of C<$filterdir>.
=head1 VERSION
-C<nbdkit-offset-filter> first appeared in nbdkit 1.30.
+C<nbdkit-protect-filter> first appeared in nbdkit 1.30.
=head1 SEE ALSO
--
2.39.3

95
SOURCES/0004-curl-Enable-multi-conn-for-read-only-connections.patch
View File

@ -1,95 +0,0 @@
From 3f74004478d3590840d7eba97a590b7ec954957f Mon Sep 17 00:00:00 2001
From: "Richard W.M. Jones" <rjones@redhat.com>
Date: Thu, 2 Feb 2023 13:59:32 +0000
Subject: [PATCH] curl: Enable multi-conn for read-only connections
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Comparing before and after this commit shows approximately double the
performance. In other tests this allowed us to download files from
web servers at line speed.
Benchmark 1: nbdkit -r curl file:/var/tmp/jammy-server-cloudimg-amd64.raw --run "nbdcopy -p \$uri null:"
Time (mean ± σ): 943.8 ms ± 18.8 ms [User: 316.2 ms, System: 1029.7 ms]
Range (min … max): 923.7 ms … 989.2 ms 10 runs
Benchmark 2: ~/d/nbdkit/nbdkit -r curl file:/var/tmp/jammy-server-cloudimg-amd64.raw --run "nbdcopy -p \$uri null:"
Time (mean ± σ): 455.0 ms ± 6.2 ms [User: 542.2 ms, System: 1824.7 ms]
Range (min … max): 449.1 ms … 471.6 ms 10 runs
Summary
' ~/d/nbdkit/nbdkit -r curl file:/var/tmp/jammy-server-cloudimg-amd64.raw --run "nbdcopy -p \$uri null:" ' ran
2.07 ± 0.05 times faster than ' nbdkit -r curl file:/var/tmp/jammy-server-cloudimg-amd64.raw --run "nbdcopy -p \$uri null:" '
Multi-conn is enabled only when we know the connection is read-only:
$ ./nbdkit -r curl file:/var/tmp/jammy-server-cloudimg-amd64.raw --run ' nbdinfo $uri ' | grep can_multi_conn
can_multi_conn: true
$ ./nbdkit curl file:/var/tmp/jammy-server-cloudimg-amd64.raw --run ' nbdinfo $uri ' | grep can_multi_conn
can_multi_conn: false
See also:
https://listman.redhat.com/archives/libguestfs/2023-February/030581.html
Reviewed-by: Eric Blake <eblake@redhat.com>
(cherry picked from commit bb0f93ad7b9de451874d0c54188bf69cd37c5409)
---
plugins/curl/curl.c | 14 ++++++++++++++
plugins/curl/curldefs.h | 1 +
2 files changed, 15 insertions(+)
diff --git a/plugins/curl/curl.c b/plugins/curl/curl.c
index e89bea99..eeba5aa4 100644
--- a/plugins/curl/curl.c
+++ b/plugins/curl/curl.c
@@ -455,6 +455,7 @@ curl_open (int readonly)
nbdkit_error ("calloc: %m");
return NULL;
}
+ h->readonly = readonly;
h->c = curl_easy_init ();
if (h->c == NULL) {
@@ -764,6 +765,18 @@ curl_get_size (void *handle)
return h->exportsize;
}
+/* Multi-conn is safe for read-only connections, but HTTP does not
+ * have any concept of flushing so we cannot use it for read-write
+ * connections.
+ */
+static int
+curl_can_multi_conn (void *handle)
+{
+ struct curl_handle *h = handle;
+
+ return !! h->readonly;
+}
+
/* NB: The terminology used by libcurl is confusing!
*
* WRITEFUNCTION / write_cb is used when reading from the remote server
@@ -907,6 +920,7 @@ static struct nbdkit_plugin plugin = {
.open = curl_open,
.close = curl_close,
.get_size = curl_get_size,
+ .can_multi_conn = curl_can_multi_conn,
.pread = curl_pread,
.pwrite = curl_pwrite,
};
diff --git a/plugins/curl/curldefs.h b/plugins/curl/curldefs.h
index f3095f92..9d4949f3 100644
--- a/plugins/curl/curldefs.h
+++ b/plugins/curl/curldefs.h
@@ -64,6 +64,7 @@ extern const char *user_agent;
/* The per-connection handle. */
struct curl_handle {
CURL *c;
+ int readonly;
bool accept_range;
int64_t exportsize;
char errbuf[CURL_ERROR_SIZE];
--
2.31.1

26
SOURCES/0004-docs-nbdkit-protocol.pod-Fix-manual-page-name.patch Normal file
View File

@ -0,0 +1,26 @@
From c6da52fd3d9a9e5e8ad4648bba686c37b5d3fae7 Mon Sep 17 00:00:00 2001
From: "Richard W.M. Jones" <rjones@redhat.com>
Date: Sun, 23 Jul 2023 20:15:23 +0100
Subject: [PATCH] docs/nbdkit-protocol.pod: Fix manual page name
Updates: commit dce32051318c3e8bd20825cc0db5df15f95d11fa
(cherry picked from commit 00aa248cec0a64ca293bbedffbf47426ee2b2340)
---
docs/nbdkit-protocol.pod | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/docs/nbdkit-protocol.pod b/docs/nbdkit-protocol.pod
index 7e89a607..dabdf23c 100644
--- a/docs/nbdkit-protocol.pod
+++ b/docs/nbdkit-protocol.pod
@@ -1,6 +1,6 @@
=head1 NAME
-nbdkit - which parts of the NBD protocol nbdkit supports
+nbdkit-protocol - which parts of the NBD protocol nbdkit supports
=head1 SYNOPSIS
--
2.39.3

110
SOURCES/0005-retry-request-Print-operation-we-are-retrying-in-deb.patch Normal file
View File

@ -0,0 +1,110 @@
From e206e3a645ee6d350df210d2ef95614d3f5d97e7 Mon Sep 17 00:00:00 2001
From: "Richard W.M. Jones" <rjones@redhat.com>
Date: Fri, 28 Jul 2023 16:43:07 +0100
Subject: [PATCH] retry-request: Print operation we are retrying in debug
messages
Reviewed-by: Eric Blake <eblake@redhat.com>
(cherry picked from commit dbc91fd2eac5eac77293e3deded92e11c0b8d6e3)
---
filters/retry-request/retry-request.c | 22 +++++++++++-----------
1 file changed, 11 insertions(+), 11 deletions(-)
diff --git a/filters/retry-request/retry-request.c b/filters/retry-request/retry-request.c
index ed566080..e5b8344c 100644
--- a/filters/retry-request/retry-request.c
+++ b/filters/retry-request/retry-request.c
@@ -100,15 +100,15 @@ retry_request_config (nbdkit_next_config *next, nbdkit_backend *nxdata,
* The code between RETRY_START...RETRY_END must set r to 0 or -1 on
* success or failure. *err may also be implicitly assigned.
*/
-#define RETRY_START \
+#define RETRY_START(what) \
{ \
unsigned i; \
\
r = -1; \
for (i = 0; r == -1 && i <= retries; ++i) { \
if (i > 0) { \
- nbdkit_debug ("retry %u: waiting %u seconds before retrying", \
- i, delay); \
+ nbdkit_debug ("retry %u: waiting %u seconds before retrying %s",\
+ i, delay, what); \
if (nbdkit_nanosleep (delay, 0) == -1) { \
if (*err == 0) \
*err = errno; \
@@ -130,7 +130,7 @@ retry_request_open (nbdkit_next_open *next, nbdkit_context *nxdata,
if (retry_open_call) {
int *err = &errno; /* used by the RETRY_* macros */
- RETRY_START
+ RETRY_START("open")
r = next (nxdata, readonly, exportname);
RETRY_END;
}
@@ -148,7 +148,7 @@ retry_request_pread (nbdkit_next *next,
{
int r;
- RETRY_START
+ RETRY_START("pread")
r = next->pread (next, buf, count, offset, flags, err);
RETRY_END;
return r;
@@ -162,7 +162,7 @@ retry_request_pwrite (nbdkit_next *next,
{
int r;
- RETRY_START
+ RETRY_START("pwrite")
r = next->pwrite (next, buf, count, offset, flags, err);
RETRY_END;
return r;
@@ -176,7 +176,7 @@ retry_request_trim (nbdkit_next *next,
{
int r;
- RETRY_START
+ RETRY_START("trim")
r = next->trim (next, count, offset, flags, err);
RETRY_END;
return r;
@@ -189,7 +189,7 @@ retry_request_flush (nbdkit_next *next,
{
int r;
- RETRY_START
+ RETRY_START("flush")
r = next->flush (next, flags, err);
RETRY_END;
return r;
@@ -203,7 +203,7 @@ retry_request_zero (nbdkit_next *next,
{
int r;
- RETRY_START
+ RETRY_START("zero")
r = next->zero (next, count, offset, flags, err);
RETRY_END;
return r;
@@ -218,7 +218,7 @@ retry_request_extents (nbdkit_next *next,
CLEANUP_EXTENTS_FREE struct nbdkit_extents *extents2 = NULL;
int r;
- RETRY_START {
+ RETRY_START("extents") {
/* Each retry must begin with extents reset to the right beginning. */
nbdkit_extents_free (extents2);
extents2 = nbdkit_extents_new (offset, next->get_size (next));
@@ -254,7 +254,7 @@ retry_request_cache (nbdkit_next *next,
{
int r;
- RETRY_START
+ RETRY_START("cache")
r = next->cache (next, count, offset, flags, err);
RETRY_END;
return r;
--
2.39.3

53
SOURCES/0006-tests-test-ocaml-errorcodes.c-Don-t-use-assert-in-te.patch Normal file
View File

@ -0,0 +1,53 @@
From fdd27622047cd8e25af33b66a57899b75e99db82 Mon Sep 17 00:00:00 2001
From: "Richard W.M. Jones" <rjones@redhat.com>
Date: Tue, 1 Aug 2023 11:42:58 +0100
Subject: [PATCH] tests/test-ocaml-errorcodes.c: Don't use assert in test
exit with EXIT_FAILURE instead, as the program might be compiled with
-DNDEBUG.
(cherry picked from commit 678410e23841376d8b742dce3d10c632499367a2)
---
tests/test-ocaml-errorcodes.c | 16 +++++++++++++---
1 file changed, 13 insertions(+), 3 deletions(-)
diff --git a/tests/test-ocaml-errorcodes.c b/tests/test-ocaml-errorcodes.c
index e59f074b..8ab7e0ae 100644
--- a/tests/test-ocaml-errorcodes.c
+++ b/tests/test-ocaml-errorcodes.c
@@ -34,8 +34,9 @@
#include <stdio.h>
#include <stdlib.h>
+#include <stdint.h>
+#include <inttypes.h>
#include <errno.h>
-#include <assert.h>
#include <libnbd.h>
@@ -84,10 +85,19 @@ main (int argc, char *argv[])
exit (EXIT_FAILURE);
}
- assert (nbd_pread (nbd, buf, 512, 0, 0) == 0);
+ if (nbd_pread (nbd, buf, 512, 0, 0) == -1) {
+ fprintf (stderr, "%s: FAIL: did not expect reading sector 0 to fail\n",
+ argv[0]);
+ exit (EXIT_FAILURE);
+ }
for (i = 0; tests[i].offset != 0; ++i) {
- assert (nbd_pread (nbd, buf, 512, tests[i].offset, 0) == -1);
+ if (nbd_pread (nbd, buf, 512, tests[i].offset, 0) != -1) {
+ fprintf (stderr,
+ "%s: FAIL: reading sector %" PRIu64 "should have failed\n",
+ argv[0], tests[i].offset / 512);
+ exit (EXIT_FAILURE);
+ }
actual_errno = nbd_get_errno ();
if (actual_errno != tests[i].expected_errno) {
fprintf (stderr, "%s: FAIL: actual errno = %d expected errno = %d\n",
--
2.39.3

30
SOURCES/0007-tests-test-ocaml-errorcodes.c-Enable-verbose-message.patch Normal file
View File

@ -0,0 +1,30 @@
From eca5131cbbc7e1785b266cc92624a2ce9582a7cc Mon Sep 17 00:00:00 2001
From: "Richard W.M. Jones" <rjones@redhat.com>
Date: Tue, 1 Aug 2023 12:42:05 +0100
Subject: [PATCH] tests/test-ocaml-errorcodes.c: Enable verbose messages
This test crashes when the OCaml plugin is unloaded (specifically when
we call caml_acquire_runtime_system). Enable verbose messages in a
failed attempt to track down what is happening.
(cherry picked from commit 99567408fdd8ea55b0bb5286b45ce135d91c38b5)
---
tests/test-ocaml-errorcodes.c | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/tests/test-ocaml-errorcodes.c b/tests/test-ocaml-errorcodes.c
index 8ab7e0ae..f84da3d8 100644
--- a/tests/test-ocaml-errorcodes.c
+++ b/tests/test-ocaml-errorcodes.c
@@ -78,7 +78,7 @@ main (int argc, char *argv[])
if (nbd_connect_command (nbd,
(char *[]) {
- "nbdkit", "-s", "--exit-with-parent",
+ "nbdkit", "-s", "--exit-with-parent", "-v",
"./test-ocaml-errorcodes-plugin.so",
NULL }) == -1) {
fprintf (stderr, "%s\n", nbd_get_error ());
--
2.39.3

80
SOURCES/0008-curl-Use-the-parallel-thread-model.patch Normal file
View File

@ -0,0 +1,80 @@
From bfa0b1238f6318d06930ec4d389a27932fc90a16 Mon Sep 17 00:00:00 2001
From: "Richard W.M. Jones" <rjones@redhat.com>
Date: Sat, 4 Feb 2023 12:34:57 +0000
Subject: [PATCH] curl: Use the parallel thread model
After previous changes, it is now safe to use the parallel thread
model in this plugin. The locking in pool.c protects a single curl
handle from being used from multiple threads.
An advantage of this is we can now combine the curl plugin with
filters such as readahead and scan.
This pessimizes some workloads and improves others. See my earlier
comments here:
https://listman.redhat.com/archives/libguestfs/2023-February/030610.html
https://listman.redhat.com/archives/libguestfs/2023-February/030618.html
https://listman.redhat.com/archives/libguestfs/2023-February/030619.html
https://listman.redhat.com/archives/libguestfs/2023-February/030620.html
Tests below use this basic command:
$ time nbdkit -r -U - curl https://cloud.debian.org/images/cloud/bookworm/daily/latest/debian-12-generic-amd64-daily.qcow2 \
--run '$COPY $uri /var/tmp/out'
where $COPY is either nbdcopy or qemu-img convert.
Before this change:
nbdkit + nbdcopy 0m55.397s
nbdkit + qemu-img convert [over 20 minutes]
After this change:
nbdkit + nbdcopy 1m1.235s
nbdkit + qemu-img convert 6m3.262s
nbdkit --filter=readahead --filter=cache + qemu-img convert
8m16.488s
nbdkit connections=8 + qemu-img convert
3m48.502s
Previously [see first link above] we noted that file: URLs are
impacted, and indeed they are, with this command:
$ time nbdkit -r -U - curl file:/var/tmp/fedora-36.img \
--run 'nbdcopy --no-extents -p "$uri" null:'
nearly doubling in run-time (0.78 -> 1.34). However I think this may
be a peculiarity of curl's handling of file. (Use nbdkit-file-plugin
instead).
Note the fundamental issue here is still lack of multiconn support in
qemu's NBD client.
(cherry picked from commit f2163754860d041c4cb12dace90591c280eccae8)
---
plugins/curl/curl.c | 7 +------
1 file changed, 1 insertion(+), 6 deletions(-)
diff --git a/plugins/curl/curl.c b/plugins/curl/curl.c
index 58ffb662..425f1d90 100644
--- a/plugins/curl/curl.c
+++ b/plugins/curl/curl.c
@@ -537,12 +537,7 @@ curl_close (void *handle)
free (h);
}
-/* This plugin could support the parallel thread model. It currently
- * uses serialize_requests because parallel has the unfortunate effect
- * of pessimising common workloads. See:
- * https://listman.redhat.com/archives/libguestfs/2023-February/030618.html
- */
-#define THREAD_MODEL NBDKIT_THREAD_MODEL_SERIALIZE_REQUESTS
+#define THREAD_MODEL NBDKIT_THREAD_MODEL_PARALLEL
/* Calls get_handle() ... put_handle() to get a handle for the length
* of the current scope.
--
2.39.3

115
SOURCES/0009-curl-Add-ipresolve-option.patch Normal file
View File

@ -0,0 +1,115 @@
From a5f1e659d52872a499a5744870cb3e6ff382642f Mon Sep 17 00:00:00 2001
From: "Richard W.M. Jones" <rjones@redhat.com>
Date: Tue, 25 Jul 2023 16:57:41 +0100
Subject: [PATCH] curl: Add ipresolve option
Allows you to force IPv4 or IPv6.
(cherry picked from commit d1b27c97f7fac00ee452a1d1b05805ee7145a49c)
---
plugins/curl/curl.c | 15 +++++++++++++++
plugins/curl/curldefs.h | 1 +
plugins/curl/nbdkit-curl-plugin.pod | 14 ++++++++++++++
plugins/curl/pool.c | 2 ++
4 files changed, 32 insertions(+)
diff --git a/plugins/curl/curl.c b/plugins/curl/curl.c
index 425f1d90..91fa65fb 100644
--- a/plugins/curl/curl.c
+++ b/plugins/curl/curl.c
@@ -68,6 +68,7 @@ struct curl_slist *headers = NULL;
const char *header_script = NULL;
unsigned header_script_renew = 0;
long http_version = CURL_HTTP_VERSION_NONE;
+long ipresolve = CURL_IPRESOLVE_WHATEVER;
char *password = NULL;
#ifndef HAVE_CURLOPT_PROTOCOLS_STR
long protocols = CURLPROTO_ALL;
@@ -314,6 +315,19 @@ curl_config (const char *key, const char *value)
}
}
+ else if (strcmp (key, "ipresolve") == 0) {
+ if (strcmp (value, "any") == 0 || strcmp (value, "whatever") == 0)
+ ipresolve = CURL_IPRESOLVE_WHATEVER;
+ else if (strcmp (value, "v4") == 0 || strcmp (value, "4") == 0)
+ ipresolve = CURL_IPRESOLVE_V4;
+ else if (strcmp (value, "v6") == 0 || strcmp (value, "6") == 0)
+ ipresolve = CURL_IPRESOLVE_V6;
+ else {
+ nbdkit_error ("unknown ipresolve: %s", value);
+ return -1;
+ }
+ }
+
else if (strcmp (key, "password") == 0) {
free (password);
if (nbdkit_read_password (value, &password) == -1)
@@ -495,6 +509,7 @@ curl_config_complete (void)
"header-script=<SCRIPT> Script to set HTTP/HTTPS headers.\n" \
"header-script-renew=<SECS> Time to renew HTTP/HTTPS headers.\n" \
"http-version=none|... Force a particular HTTP protocol.\n" \
+ "ipresolve=any|v4|v6 Force IPv4 or IPv6.\n" \
"password=<PASSWORD> The password for the user account.\n" \
"protocols=PROTO,PROTO,.. Limit protocols allowed.\n" \
"proxy=<PROXY> Set proxy URL.\n" \
diff --git a/plugins/curl/curldefs.h b/plugins/curl/curldefs.h
index 2a74a369..815be2e1 100644
--- a/plugins/curl/curldefs.h
+++ b/plugins/curl/curldefs.h
@@ -61,6 +61,7 @@ extern struct curl_slist *headers;
extern const char *header_script;
extern unsigned header_script_renew;
extern long http_version;
+extern long ipresolve;
extern char *password;
#ifndef HAVE_CURLOPT_PROTOCOLS_STR
extern long protocols;
diff --git a/plugins/curl/nbdkit-curl-plugin.pod b/plugins/curl/nbdkit-curl-plugin.pod
index 070f9a0f..e12ca197 100644
--- a/plugins/curl/nbdkit-curl-plugin.pod
+++ b/plugins/curl/nbdkit-curl-plugin.pod
@@ -208,6 +208,19 @@ meaning curl will negotiate the best protocol with the server. The
other settings are mainly for testing. See L<CURLOPT_HTTP_VERSION(3)>
for details.
+=item B<ipresolve=any>
+
+=item B<ipresolve=v4>
+
+=item B<ipresolve=v6>
+
+(nbdkit E<ge> 1.36)
+
+Force curl to use only IPv4 (C<ipresolve=v4>), only IPv6
+(C<ipresolve=v6>) or any IP version supported by your system
+(C<ipresolve=any>). The default is C<any>. See
+L<CURLOPT_IPRESOLVE(3)>.
+
=item B<password=>PASSWORD
Set the password to use when connecting to the remote server.
@@ -559,6 +572,7 @@ L<CURLOPT_COOKIEFILE(3)>,
L<CURLOPT_COOKIEJAR(3)>,
L<CURLOPT_FOLLOWLOCATION(3)>,
L<CURLOPT_HTTPHEADER(3)>,
+L<CURLOPT_IPRESOLVE(3)>,
L<CURLOPT_PROXY(3)>,
L<CURLOPT_SSL_CIPHER_LIST(3)>,
L<CURLOPT_SSLVERSION(3)>,
diff --git a/plugins/curl/pool.c b/plugins/curl/pool.c
index 731dd367..f0c3cb4f 100644
--- a/plugins/curl/pool.c
+++ b/plugins/curl/pool.c
@@ -257,6 +257,8 @@ allocate_handle (void)
curl_easy_setopt (ch->c, CURLOPT_HTTPHEADER, headers);
if (http_version != CURL_HTTP_VERSION_NONE)
curl_easy_setopt (ch->c, CURLOPT_HTTP_VERSION, (long) http_version);
+ if (ipresolve != CURL_IPRESOLVE_WHATEVER)
+ curl_easy_setopt (ch->c, CURLOPT_IPRESOLVE, (long) ipresolve);
if (password)
curl_easy_setopt (ch->c, CURLOPT_PASSWORD, password);
--
2.39.3

113
SOURCES/0010-curl-Add-resolve-option.patch Normal file
View File

@ -0,0 +1,113 @@
From 55e55ea986ef5fed595bb5a4203e8734d79f1474 Mon Sep 17 00:00:00 2001
From: "Richard W.M. Jones" <rjones@redhat.com>
Date: Tue, 25 Jul 2023 17:36:31 +0100
Subject: [PATCH] curl: Add resolve option
This allows you to force a particular IP address for the URL host
name.
(cherry picked from commit 4f0989cbf0e9eeb959879b1c82b52940f6c5c3cc)
---
plugins/curl/curl.c | 12 ++++++++++++
plugins/curl/curldefs.h | 1 +
plugins/curl/nbdkit-curl-plugin.pod | 7 +++++++
plugins/curl/pool.c | 2 ++
4 files changed, 22 insertions(+)
diff --git a/plugins/curl/curl.c b/plugins/curl/curl.c
index 91fa65fb..381433fd 100644
--- a/plugins/curl/curl.c
+++ b/plugins/curl/curl.c
@@ -78,6 +78,7 @@ const char *protocols = NULL;
const char *proxy = NULL;
char *proxy_password = NULL;
const char *proxy_user = NULL;
+struct curl_slist *resolves = NULL;
bool sslverify = true;
const char *ssl_cipher_list = NULL;
long ssl_version = CURL_SSLVERSION_DEFAULT;
@@ -112,6 +113,8 @@ curl_unload (void)
curl_slist_free_all (headers);
free (password);
free (proxy_password);
+ if (resolves)
+ curl_slist_free_all (resolves);
scripts_unload ();
free_all_handles ();
curl_global_cleanup ();
@@ -356,6 +359,14 @@ curl_config (const char *key, const char *value)
else if (strcmp (key, "proxy-user") == 0)
proxy_user = value;
+ else if (strcmp (key, "resolve") == 0) {
+ resolves = curl_slist_append (headers, value);
+ if (resolves == NULL) {
+ nbdkit_error ("curl_slist_append: %m");
+ return -1;
+ }
+ }
+
else if (strcmp (key, "sslverify") == 0) {
r = nbdkit_parse_bool (value);
if (r == -1)
@@ -515,6 +526,7 @@ curl_config_complete (void)
"proxy=<PROXY> Set proxy URL.\n" \
"proxy-password=<PASSWORD> The proxy password.\n" \
"proxy-user=<USER> The proxy user.\n" \
+ "resolve=<HOST>:<PORT>:<ADDR> Custom host to IP address resolution.\n" \
"sslverify=false Do not verify SSL certificate of remote host.\n" \
"ssl-cipher-list=C1:C2:.. Specify TLS/SSL cipher suites to be used.\n" \
"ssl-version=<VERSION> Specify preferred TLS/SSL version.\n" \
diff --git a/plugins/curl/curldefs.h b/plugins/curl/curldefs.h
index 815be2e1..613cfed7 100644
--- a/plugins/curl/curldefs.h
+++ b/plugins/curl/curldefs.h
@@ -74,6 +74,7 @@ extern const char *proxy_user;
extern bool sslverify;
extern const char *ssl_cipher_list;
extern long ssl_version;
+extern struct curl_slist *resolves;
extern const char *tls13_ciphers;
extern bool tcp_keepalive;
extern bool tcp_nodelay;
diff --git a/plugins/curl/nbdkit-curl-plugin.pod b/plugins/curl/nbdkit-curl-plugin.pod
index e12ca197..a7315047 100644
--- a/plugins/curl/nbdkit-curl-plugin.pod
+++ b/plugins/curl/nbdkit-curl-plugin.pod
@@ -289,6 +289,12 @@ Set the proxy. See L<CURLOPT_PROXY(3)>.
Set the proxy username and password.
+=item B<resolve=>HOSTB<:>PORTB<:>ADDRESS
+
+Provide custom host name to IP address resolution. You can supply
+this option as many times as needed. See L<CURLOPT_RESOLVE(3)> for
+the full details of this option.
+
=item B<sslverify=false>
Don't verify the SSL certificate of the remote host.
@@ -574,6 +580,7 @@ L<CURLOPT_FOLLOWLOCATION(3)>,
L<CURLOPT_HTTPHEADER(3)>,
L<CURLOPT_IPRESOLVE(3)>,
L<CURLOPT_PROXY(3)>,
+L<CURLOPT_RESOLVE(3)>,
L<CURLOPT_SSL_CIPHER_LIST(3)>,
L<CURLOPT_SSLVERSION(3)>,
L<CURLOPT_TCP_KEEPALIVE(3)>,
diff --git a/plugins/curl/pool.c b/plugins/curl/pool.c
index f0c3cb4f..a6e2f9f5 100644
--- a/plugins/curl/pool.c
+++ b/plugins/curl/pool.c
@@ -283,6 +283,8 @@ allocate_handle (void)
curl_easy_setopt (ch->c, CURLOPT_SSL_VERIFYPEER, 0L);
curl_easy_setopt (ch->c, CURLOPT_SSL_VERIFYHOST, 0L);
}
+ if (resolves)
+ curl_easy_setopt (ch->c, CURLOPT_RESOLVE, resolves);
if (ssl_version != CURL_SSLVERSION_DEFAULT)
curl_easy_setopt (ch->c, CURLOPT_SSLVERSION, (long) ssl_version);
if (ssl_cipher_list)
--
2.39.3

82
SOURCES/0011-curl-pool-Add-abstract-load_pool-and-unload_pool-fun.patch Normal file
View File

@ -0,0 +1,82 @@
From f14fbc9498f52661fd8b9c895144eb40bf8bcd64 Mon Sep 17 00:00:00 2001
From: "Richard W.M. Jones" <rjones@redhat.com>
Date: Tue, 25 Jul 2023 18:57:00 +0100
Subject: [PATCH] curl: pool: Add abstract load_pool and unload_pool functions
Simple code refactoring.
(cherry picked from commit 93761b2abf4c5923a410a4f176ee66c690520a2c)
---
plugins/curl/curl.c | 4 +++-
plugins/curl/curldefs.h | 3 ++-
plugins/curl/pool.c | 10 ++++++++--
3 files changed, 13 insertions(+), 4 deletions(-)
diff --git a/plugins/curl/curl.c b/plugins/curl/curl.c
index 381433fd..3557fd53 100644
--- a/plugins/curl/curl.c
+++ b/plugins/curl/curl.c
@@ -103,6 +103,8 @@ curl_load (void)
nbdkit_error ("libcurl initialization failed: %d", (int) r);
exit (EXIT_FAILURE);
}
+
+ load_pool ();
}
static void
@@ -116,7 +118,7 @@ curl_unload (void)
if (resolves)
curl_slist_free_all (resolves);
scripts_unload ();
- free_all_handles ();
+ unload_pool ();
curl_global_cleanup ();
}
diff --git a/plugins/curl/curldefs.h b/plugins/curl/curldefs.h
index 613cfed7..4506f3e1 100644
--- a/plugins/curl/curldefs.h
+++ b/plugins/curl/curldefs.h
@@ -122,9 +122,10 @@ struct curl_handle {
};
/* pool.c */
+extern void load_pool (void);
+extern void unload_pool (void);
extern struct curl_handle *get_handle (void);
extern void put_handle (struct curl_handle *ch);
-extern void free_all_handles (void);
/* scripts.c */
extern int do_scripts (struct curl_handle *ch);
diff --git a/plugins/curl/pool.c b/plugins/curl/pool.c
index a6e2f9f5..f91cdf57 100644
--- a/plugins/curl/pool.c
+++ b/plugins/curl/pool.c
@@ -89,14 +89,20 @@ static curl_handle_list curl_handles = empty_vector;
static pthread_cond_t cond = PTHREAD_COND_INITIALIZER;
static size_t in_use = 0, waiting = 0;
+/* Initialize pool structures. */
+void
+load_pool (void)
+{
+}
+
/* Close and free all handles in the pool. */
void
-free_all_handles (void)
+unload_pool (void)
{
size_t i;
if (curl_debug_pool)
- nbdkit_debug ("free_all_handles: number of curl handles allocated: %zu",
+ nbdkit_debug ("unload_pool: number of curl handles allocated: %zu",
curl_handles.len);
for (i = 0; i < curl_handles.len; ++i)
--
2.39.3

278
SOURCES/0012-curl-Add-D-curl.times-1-to-collect-time-statistics.patch Normal file
View File

@ -0,0 +1,278 @@
From d8bbc2dd00eddd2c36a6a718f7e21e6e7d4f26e5 Mon Sep 17 00:00:00 2001
From: "Richard W.M. Jones" <rjones@redhat.com>
Date: Tue, 25 Jul 2023 21:45:31 +0100
Subject: [PATCH] curl: Add -D curl.times=1 to collect time statistics
(cherry picked from commit 68dddbeb584fb9385915846d259563f74338ffe8)
---
plugins/curl/Makefile.am | 1 +
plugins/curl/curl.c | 3 +
plugins/curl/curldefs.h | 13 +++
plugins/curl/nbdkit-curl-plugin.pod | 21 +++++
plugins/curl/pool.c | 2 +
plugins/curl/times.c | 130 ++++++++++++++++++++++++++++
6 files changed, 170 insertions(+)
create mode 100644 plugins/curl/times.c
diff --git a/plugins/curl/Makefile.am b/plugins/curl/Makefile.am
index 82c09582..f5f1cc1a 100644
--- a/plugins/curl/Makefile.am
+++ b/plugins/curl/Makefile.am
@@ -42,6 +42,7 @@ nbdkit_curl_plugin_la_SOURCES = \
curl.c \
pool.c \
scripts.c \
+ times.c \
$(top_srcdir)/include/nbdkit-plugin.h \
$(NULL)
diff --git a/plugins/curl/curl.c b/plugins/curl/curl.c
index 3557fd53..0d1fcfee 100644
--- a/plugins/curl/curl.c
+++ b/plugins/curl/curl.c
@@ -119,6 +119,7 @@ curl_unload (void)
curl_slist_free_all (resolves);
scripts_unload ();
unload_pool ();
+ display_times ();
curl_global_cleanup ();
}
@@ -639,6 +640,7 @@ curl_pread (void *handle, void *buf, uint32_t count, uint64_t offset)
display_curl_error (ch, r, "pread: curl_easy_perform");
return -1;
}
+ update_times (ch->c);
/* Could use curl_easy_getinfo here to obtain further information
* about the connection.
@@ -683,6 +685,7 @@ curl_pwrite (void *handle, const void *buf, uint32_t count, uint64_t offset)
display_curl_error (ch, r, "pwrite: curl_easy_perform");
return -1;
}
+ update_times (ch->c);
/* Could use curl_easy_getinfo here to obtain further information
* about the connection.
diff --git a/plugins/curl/curldefs.h b/plugins/curl/curldefs.h
index 4506f3e1..dd9791aa 100644
--- a/plugins/curl/curldefs.h
+++ b/plugins/curl/curldefs.h
@@ -44,6 +44,15 @@
#if CURL_AT_LEAST_VERSION (7, 55, 0)
#define HAVE_CURLINFO_CONTENT_LENGTH_DOWNLOAD_T
#endif
+#if CURL_AT_LEAST_VERSION (7, 61, 0)
+#define HAVE_CURLINFO_NAMELOOKUP_TIME_T
+#define HAVE_CURLINFO_CONNECT_TIME_T
+#define HAVE_CURLINFO_APPCONNECT_TIME_T
+#define HAVE_CURLINFO_PRETRANSFER_TIME_T
+#define HAVE_CURLINFO_STARTTRANSFER_TIME_T
+#define HAVE_CURLINFO_TOTAL_TIME_T
+#define HAVE_CURLINFO_REDIRECT_TIME_T
+#endif
#endif
extern const char *url;
@@ -131,6 +140,10 @@ extern void put_handle (struct curl_handle *ch);
extern int do_scripts (struct curl_handle *ch);
extern void scripts_unload (void);
+/* times.c */
+extern void update_times (CURL *c); /* called after every curl_easy_perform */
+extern void display_times (void);
+
/* Translate CURLcode to nbdkit_error. */
#define display_curl_error(ch, r, fs, ...) \
do { \
diff --git a/plugins/curl/nbdkit-curl-plugin.pod b/plugins/curl/nbdkit-curl-plugin.pod
index a7315047..0fd688ed 100644
--- a/plugins/curl/nbdkit-curl-plugin.pod
+++ b/plugins/curl/nbdkit-curl-plugin.pod
@@ -543,6 +543,27 @@ This prints out the headers and cookies generated by the
C<header-script> and C<cookie-script> options, which can be useful
when debugging these scripts.
+=item B<-D curl.times=1>
+
+This prints out additional information about the total time taken to
+do name resolution, connect to the remote server, etc. The
+information is printed in the debug output before nbdkit exits. The
+output will look like:
+
+ nbdkit: debug: times (-D curl.times=1):
+ nbdkit: debug: name resolution : 0.128442 s
+ nbdkit: debug: connection : 4.945213 s
+ nbdkit: debug: SSL negotiation : 4.291362 s
+ nbdkit: debug: pretransfer : 0.104137 s
+ nbdkit: debug: first byte received : 56.115269 s
+ nbdkit: debug: data transfer : 222.633831 s
+ nbdkit: debug: redirection time : 0.000000 s
+
+The cumulative time taken to perform each step is shown (summed across
+all HTTP connections). The redirection time is the total time taken
+doing HTTP redirections. For further information see
+L<curl_easy_getinfo(3)/TIMES>.
+
=item B<-D curl.verbose=1>
This enables very verbose curl debugging. See L<CURLOPT_VERBOSE(3)>.
diff --git a/plugins/curl/pool.c b/plugins/curl/pool.c
index f91cdf57..10f9011d 100644
--- a/plugins/curl/pool.c
+++ b/plugins/curl/pool.c
@@ -475,6 +475,7 @@ get_content_length_accept_range (struct curl_handle *ch)
display_curl_error (ch, r,
"problem doing HEAD request to fetch size of URL [%s]",
url);
+ update_times (ch->c);
/* Get the HTTP status code, if available. */
r = curl_easy_getinfo (ch->c, CURLINFO_RESPONSE_CODE, &code);
@@ -569,6 +570,7 @@ try_fallback_GET_method (struct curl_handle *ch)
curl_easy_setopt (ch->c, CURLOPT_WRITEFUNCTION, error_cb);
curl_easy_setopt (ch->c, CURLOPT_WRITEDATA, ch);
r = curl_easy_perform (ch->c);
+ update_times (ch->c);
/* We expect CURLE_WRITE_ERROR here, but CURLE_OK is possible too
* (eg if the remote has zero length). Other errors might happen
diff --git a/plugins/curl/times.c b/plugins/curl/times.c
new file mode 100644
index 00000000..8cc4cf27
--- /dev/null
+++ b/plugins/curl/times.c
@@ -0,0 +1,130 @@
+/* nbdkit
+ * Copyright Red Hat
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * * Neither the name of Red Hat nor the names of its contributors may be
+ * used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY RED HAT AND CONTRIBUTORS ''AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+ * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL RED HAT OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
+ * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+ * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <config.h>
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <stdbool.h>
+#include <stdint.h>
+#include <inttypes.h>
+
+#include <curl/curl.h>
+
+#include <nbdkit-plugin.h>
+
+#include "array-size.h"
+
+#include "curldefs.h"
+
+/* Use '-D curl.times=1' to set. */
+NBDKIT_DLL_PUBLIC int curl_debug_times = 0;
+
+/* The cumulative times. */
+static struct {
+ bool cumulative;
+ const char *name;
+ CURLINFO info;
+ curl_off_t t;
+} times[] = {
+#ifdef HAVE_CURLINFO_NAMELOOKUP_TIME_T
+ { true, "name resolution", CURLINFO_NAMELOOKUP_TIME_T },
+#endif
+#ifdef HAVE_CURLINFO_CONNECT_TIME_T
+ { true, "connection", CURLINFO_CONNECT_TIME_T },
+#endif
+#ifdef HAVE_CURLINFO_APPCONNECT_TIME_T
+ { true, "SSL negotiation", CURLINFO_APPCONNECT_TIME_T },
+#endif
+#ifdef HAVE_CURLINFO_PRETRANSFER_TIME_T
+ { true, "pretransfer", CURLINFO_PRETRANSFER_TIME_T },
+#endif
+#ifdef HAVE_CURLINFO_STARTTRANSFER_TIME_T
+ { true, "first byte received", CURLINFO_STARTTRANSFER_TIME_T },
+#endif
+#ifdef HAVE_CURLINFO_TOTAL_TIME_T
+ { true, "data transfer", CURLINFO_TOTAL_TIME_T },
+#endif
+#ifdef HAVE_CURLINFO_REDIRECT_TIME_T
+ { false, "redirection time", CURLINFO_REDIRECT_TIME_T },
+#endif
+};
+
+/* This is called after every curl_easy_perform. If -D curl.times=1
+ * then we update the time counters. Refer to curl_easy_getinfo(3)
+ * section "TIMES".
+ */
+void
+update_times (CURL *c)
+{
+ size_t i;
+ CURLcode r;
+ curl_off_t t;
+
+ if (!curl_debug_times) return;
+
+ for (i = 0; i < ARRAY_SIZE (times); ++i) {
+ r = curl_easy_getinfo (c, times[i].info, &t);
+ if (r != CURLE_OK) {
+ nbdkit_debug ("curl_easy_getinfo: error getting time '%s': %s",
+ times[i].name, curl_easy_strerror (r));
+ continue;
+ }
+ if (curl_debug_verbose)
+ nbdkit_debug ("time '%s': %" PRIi64, times[i].name, (int64_t) t);
+ times[i].t += t;
+ }
+}
+
+/* Called when the plugin is unloaded. */
+void
+display_times (void)
+{
+ size_t i;
+ int64_t prev_t = 0, t, v;
+
+ if (!curl_debug_times) return;
+
+ nbdkit_debug ("times (-D curl.times=1):");
+ for (i = 0; i < ARRAY_SIZE (times); ++i) {
+ t = times[i].t; /* in microseconds */
+ if (times[i].cumulative)
+ v = t - prev_t;
+ else
+ v = t;
+ prev_t = t;
+
+ nbdkit_debug ("%-30s: %3" PRIi64 ".%06" PRIi64 " s",
+ times[i].name,
+ v / 1000000, v % 1000000);
+ }
+}
--
2.39.3

34
SOURCES/0013-curl-Fix-call-to-update_times.patch Normal file
View File

@ -0,0 +1,34 @@
From 320af6bbc9d71e3e67cf1d6e59d4a5094324d41c Mon Sep 17 00:00:00 2001
From: "Richard W.M. Jones" <rjones@redhat.com>
Date: Thu, 27 Jul 2023 17:08:50 +0100
Subject: [PATCH] curl: Fix call to update_times
This was called in the wrong place so we didn't count the cost of
making the initial HEAD call on each handle.
Fixes: commit 68dddbeb584fb9385915846d259563f74338ffe8
(cherry picked from commit f2d7041f94af15b158c92a96b9d9fdf4d3b7cdef)
---
plugins/curl/pool.c | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/plugins/curl/pool.c b/plugins/curl/pool.c
index 10f9011d..8db69a71 100644
--- a/plugins/curl/pool.c
+++ b/plugins/curl/pool.c
@@ -471,11 +471,11 @@ get_content_length_accept_range (struct curl_handle *ch)
curl_easy_setopt (ch->c, CURLOPT_HEADERFUNCTION, header_cb);
curl_easy_setopt (ch->c, CURLOPT_HEADERDATA, ch);
r = curl_easy_perform (ch->c);
+ update_times (ch->c);
if (r != CURLE_OK) {
display_curl_error (ch, r,
"problem doing HEAD request to fetch size of URL [%s]",
url);
- update_times (ch->c);
/* Get the HTTP status code, if available. */
r = curl_easy_getinfo (ch->c, CURLINFO_RESPONSE_CODE, &code);
--
2.39.3

2075
SOURCES/0014-curl-Move-configuration-code-to-a-separate-file.patch Normal file
View File

File diff suppressed because it is too large Load Diff

53
SOURCES/0015-curl-Make-times-seconds-field-slightly-wider.patch Normal file
View File

@ -0,0 +1,53 @@
From 110a96d728d54ff31d0d5ff1684427b45e91b832 Mon Sep 17 00:00:00 2001
From: "Richard W.M. Jones" <rjones@redhat.com>
Date: Fri, 28 Jul 2023 11:26:49 +0100
Subject: [PATCH] curl: Make times seconds field slightly wider
Updates: commit 68dddbeb584fb9385915846d259563f74338ffe8
(cherry picked from commit 1622db8c9f6c6e42b679d1d81df019944178b373)
---
plugins/curl/nbdkit-curl-plugin.pod | 14 +++++++-------
plugins/curl/times.c | 2 +-
2 files changed, 8 insertions(+), 8 deletions(-)
diff --git a/plugins/curl/nbdkit-curl-plugin.pod b/plugins/curl/nbdkit-curl-plugin.pod
index 0fd688ed..0774adad 100644
--- a/plugins/curl/nbdkit-curl-plugin.pod
+++ b/plugins/curl/nbdkit-curl-plugin.pod
@@ -551,13 +551,13 @@ information is printed in the debug output before nbdkit exits. The
output will look like:
nbdkit: debug: times (-D curl.times=1):
- nbdkit: debug: name resolution : 0.128442 s
- nbdkit: debug: connection : 4.945213 s
- nbdkit: debug: SSL negotiation : 4.291362 s
- nbdkit: debug: pretransfer : 0.104137 s
- nbdkit: debug: first byte received : 56.115269 s
- nbdkit: debug: data transfer : 222.633831 s
- nbdkit: debug: redirection time : 0.000000 s
+ nbdkit: debug: name resolution : 0.128442 s
+ nbdkit: debug: connection : 4.945213 s
+ nbdkit: debug: SSL negotiation : 4.291362 s
+ nbdkit: debug: pretransfer : 0.104137 s
+ nbdkit: debug: first byte received : 56.115269 s
+ nbdkit: debug: data transfer : 222.633831 s
+ nbdkit: debug: redirection time : 0.000000 s
The cumulative time taken to perform each step is shown (summed across
all HTTP connections). The redirection time is the total time taken
diff --git a/plugins/curl/times.c b/plugins/curl/times.c
index 8cc4cf27..e752a0a9 100644
--- a/plugins/curl/times.c
+++ b/plugins/curl/times.c
@@ -123,7 +123,7 @@ display_times (void)
v = t;
prev_t = t;
- nbdkit_debug ("%-30s: %3" PRIi64 ".%06" PRIi64 " s",
+ nbdkit_debug ("%-30s: %4" PRIi64 ".%06" PRIi64 " s",
times[i].name,
v / 1000000, v % 1000000);
}
--
2.39.3

51
SOURCES/0016-curl-Use-_Atomic-type-to-accumulate-curl-timings.patch Normal file
View File

@ -0,0 +1,51 @@
From 22590e0575e78c4fc754705b46d1458eeadefcf1 Mon Sep 17 00:00:00 2001
From: "Richard W.M. Jones" <rjones@redhat.com>
Date: Sun, 30 Jul 2023 16:45:38 +0100
Subject: [PATCH] curl: Use _Atomic type to accumulate curl timings
Because the global list of times is accessed in parallel by many
threads, they must be accessed atomically.
Fixes: commit 68dddbeb584fb9385915846d259563f74338ffe8
(cherry picked from commit 4c527063336ccf14d286ef7db5766369e1b23845)
---
plugins/curl/curldefs.h | 9 +++++++++
plugins/curl/times.c | 2 +-
2 files changed, 10 insertions(+), 1 deletion(-)
diff --git a/plugins/curl/curldefs.h b/plugins/curl/curldefs.h
index 9169b256..1feba844 100644
--- a/plugins/curl/curldefs.h
+++ b/plugins/curl/curldefs.h
@@ -35,6 +35,15 @@
#include <stdbool.h>
+#ifdef HAVE_STDATOMIC_H
+#include <stdatomic.h>
+#else
+/* Some old platforms lack atomic types, but 32 bit ints are usually
+ * "atomic enough".
+ */
+#define _Atomic /**/
+#endif
+
#include "windows-compat.h"
/* Macro CURL_AT_LEAST_VERSION was added in 2015 (Curl 7.43) so if the
diff --git a/plugins/curl/times.c b/plugins/curl/times.c
index e752a0a9..23e2950b 100644
--- a/plugins/curl/times.c
+++ b/plugins/curl/times.c
@@ -54,7 +54,7 @@ static struct {
bool cumulative;
const char *name;
CURLINFO info;
- curl_off_t t;
+ _Atomic curl_off_t t;
} times[] = {
#ifdef HAVE_CURLINFO_NAMELOOKUP_TIME_T
{ true, "name resolution", CURLINFO_NAMELOOKUP_TIME_T },
--
2.39.3

120
SOURCES/0017-curl-Add-D-curl.verbose.ids-1-to-display-conn-and-xf.patch Normal file
View File

@ -0,0 +1,120 @@
From 07bdd644ae082c8e05afdcda4dcd12816ecc0efc Mon Sep 17 00:00:00 2001
From: "Richard W.M. Jones" <rjones@redhat.com>
Date: Mon, 31 Jul 2023 13:24:39 +0100
Subject: [PATCH] curl: Add -D curl.verbose.ids=1 to display conn and xfer IDs
Enhance debugging output with connection (conn) and transfer (xfer)
IDs. Since there is some overhead to doing this, don't do it by
default. Also it requires libcurl >= 8.2.0.
(cherry picked from commit a3ebf8f1c2cd5d399730ad6887b6b8bed94dc0ce)
---
plugins/curl/config.c | 37 +++++++++++++++++++++++++----
plugins/curl/curldefs.h | 4 ++++
plugins/curl/nbdkit-curl-plugin.pod | 6 +++++
3 files changed, 43 insertions(+), 4 deletions(-)
diff --git a/plugins/curl/config.c b/plugins/curl/config.c
index 742d6080..d68c92ba 100644
--- a/plugins/curl/config.c
+++ b/plugins/curl/config.c
@@ -99,6 +99,9 @@ static size_t error_cb (char *ptr, size_t size, size_t nmemb, void *opaque);
/* Use '-D curl.verbose=1' to set. */
NBDKIT_DLL_PUBLIC int curl_debug_verbose = 0;
+/* Use '-D curl.verbose.ids=1' to set. */
+NBDKIT_DLL_PUBLIC int curl_debug_verbose_ids = 0;
+
void
unload_config (void)
{
@@ -707,6 +710,14 @@ debug_cb (CURL *handle, curl_infotype type,
{
size_t origsize = size;
CLEANUP_FREE char *str;
+ curl_off_t conn_id = -1, xfer_id = -1;
+
+#if defined(HAVE_CURLINFO_CONN_ID) && defined(HAVE_CURLINFO_XFER_ID)
+ if (curl_debug_verbose_ids) {
+ curl_easy_getinfo (handle, CURLINFO_CONN_ID, &conn_id);
+ curl_easy_getinfo (handle, CURLINFO_XFER_ID, &xfer_id);
+ }
+#endif
/* The data parameter passed is NOT \0-terminated, but also it may
* have \n or \r\n line endings. The only sane way to deal with
@@ -726,17 +737,35 @@ debug_cb (CURL *handle, curl_infotype type,
switch (type) {
case CURLINFO_TEXT:
- nbdkit_debug ("%s", str);
+ if (conn_id >= 0 && xfer_id >= 0)
+ nbdkit_debug ("conn %" PRIi64 " xfer %" PRIi64 ": %s",
+ conn_id, xfer_id, str);
+ else
+ nbdkit_debug ("%s",str);
break;
case CURLINFO_HEADER_IN:
- nbdkit_debug ("S: %s", str);
+ if (conn_id >= 0 && xfer_id >= 0)
+ nbdkit_debug ("conn %" PRIi64 " xfer %" PRIi64 ": S: %s",
+ conn_id, xfer_id, str);
+ else
+ nbdkit_debug ("S: %s", str);
break;
case CURLINFO_HEADER_OUT:
- nbdkit_debug ("C: %s", str);
+ if (conn_id >= 0 && xfer_id >= 0)
+ nbdkit_debug ("conn %" PRIi64 " xfer %" PRIi64 ": C: %s",
+ conn_id, xfer_id, str);
+ else
+ nbdkit_debug ("C: %s", str);
break;
default:
/* Assume everything else is binary data that we cannot print. */
- nbdkit_debug ("<data with size=%zu>", origsize);
+ if (conn_id >= 0 && xfer_id >= 0)
+ nbdkit_debug ("conn %" PRIi64 " xfer %" PRIi64 ": "
+ "<data with size=%zu>",
+ conn_id, xfer_id,
+ origsize);
+ else
+ nbdkit_debug ("<data with size=%zu>", origsize);
}
out:
diff --git a/plugins/curl/curldefs.h b/plugins/curl/curldefs.h
index 1feba844..022e8c60 100644
--- a/plugins/curl/curldefs.h
+++ b/plugins/curl/curldefs.h
@@ -62,6 +62,10 @@
#define HAVE_CURLINFO_TOTAL_TIME_T
#define HAVE_CURLINFO_REDIRECT_TIME_T
#endif
+#if CURL_AT_LEAST_VERSION (8, 2, 0)
+#define HAVE_CURLINFO_CONN_ID
+#define HAVE_CURLINFO_XFER_ID
+#endif
#endif
extern const char *url;
diff --git a/plugins/curl/nbdkit-curl-plugin.pod b/plugins/curl/nbdkit-curl-plugin.pod
index 0774adad..7784b553 100644
--- a/plugins/curl/nbdkit-curl-plugin.pod
+++ b/plugins/curl/nbdkit-curl-plugin.pod
@@ -570,6 +570,12 @@ This enables very verbose curl debugging. See L<CURLOPT_VERBOSE(3)>.
This is mainly useful if you suspect there is a bug inside libcurl
itself.
+=item B<-D curl.verbose.ids=1>
+
+This enhances C<-D curl.verbose=1> by printing connection and transfer
+IDs next to each debug message. As this has some overhead it is not
+enabled by default.
+
=back
=head1 FILES
--
2.39.3

85
SOURCES/0018-curl-Rename-unload_config-unload_pool-config_unload-.patch Normal file
View File

@ -0,0 +1,85 @@
From ae6c568fecbef8b2aa72a89d6afd4aabcec10a05 Mon Sep 17 00:00:00 2001
From: "Richard W.M. Jones" <rjones@redhat.com>
Date: Fri, 28 Jul 2023 14:24:04 +0100
Subject: [PATCH] curl: Rename unload_config, unload_pool -> config_unload,
pool_unload
For consistency with scripts_unload.
There is still a function called load_pool, but that inconsistency
will be removed in a follow-on commit.
Reviewed-by: Eric Blake <eblake@redhat.com>
(cherry picked from commit f7bea74f670ba303c7ea60713cbbf301fab865fd)
---
plugins/curl/config.c | 2 +-
plugins/curl/curl.c | 4 ++--
plugins/curl/curldefs.h | 4 ++--
plugins/curl/pool.c | 2 +-
4 files changed, 6 insertions(+), 6 deletions(-)
diff --git a/plugins/curl/config.c b/plugins/curl/config.c
index d68c92ba..276c79d5 100644
--- a/plugins/curl/config.c
+++ b/plugins/curl/config.c
@@ -103,7 +103,7 @@ NBDKIT_DLL_PUBLIC int curl_debug_verbose = 0;
NBDKIT_DLL_PUBLIC int curl_debug_verbose_ids = 0;
void
-unload_config (void)
+config_unload (void)
{
free (cookie);
if (headers)
diff --git a/plugins/curl/curl.c b/plugins/curl/curl.c
index 99a7e00b..72971093 100644
--- a/plugins/curl/curl.c
+++ b/plugins/curl/curl.c
@@ -74,9 +74,9 @@ curl_load (void)
static void
curl_unload (void)
{
- unload_config ();
+ config_unload ();
scripts_unload ();
- unload_pool ();
+ pool_unload ();
display_times ();
curl_global_cleanup ();
}
diff --git a/plugins/curl/curldefs.h b/plugins/curl/curldefs.h
index 022e8c60..cab4a6b1 100644
--- a/plugins/curl/curldefs.h
+++ b/plugins/curl/curldefs.h
@@ -119,13 +119,13 @@ struct curl_handle {
extern int curl_config (const char *key, const char *value);
extern int curl_config_complete (void);
extern const char *curl_config_help;
-extern void unload_config (void);
+extern void config_unload (void);
extern struct curl_handle *allocate_handle (void);
extern void free_handle (struct curl_handle *);
/* pool.c */
extern void load_pool (void);
-extern void unload_pool (void);
+extern void pool_unload (void);
extern struct curl_handle *get_handle (void);
extern void put_handle (struct curl_handle *ch);
diff --git a/plugins/curl/pool.c b/plugins/curl/pool.c
index 91e56f07..50a623a4 100644
--- a/plugins/curl/pool.c
+++ b/plugins/curl/pool.c
@@ -86,7 +86,7 @@ load_pool (void)
/* Close and free all handles in the pool. */
void
-unload_pool (void)
+pool_unload (void)
{
size_t i;
--
2.39.3

96
SOURCES/0019-pool-Add-outline-get_ready-and-after_fork-functions.patch Normal file
View File

@ -0,0 +1,96 @@
From 995b1b62add89b437cb51bb7e15a35c265e188b3 Mon Sep 17 00:00:00 2001
From: "Richard W.M. Jones" <rjones@redhat.com>
Date: Fri, 28 Jul 2023 14:33:03 +0100
Subject: [PATCH] pool: Add outline get_ready and after_fork functions
In a forthcoming commit we will need to create a multi handle and a
background thread, requiring use of the .get_ready (for multi) and
.after_fork (for the thread) plugin methods. This commit removes the
empty load_pool function and adds pool_get_ready and pool_after_fork,
and the associated machinery in curl.c.
This commit on its own does nothing, future commits will fill in these
functions with useful work.
Reviewed-by: Eric Blake <eblake@redhat.com>
(cherry picked from commit 4075a499115a9e12ea74c4838fe00a7d680de2a1)
---
plugins/curl/curl.c | 14 +++++++++++++-
plugins/curl/curldefs.h | 3 ++-
plugins/curl/pool.c | 12 +++++++++---
3 files changed, 24 insertions(+), 5 deletions(-)
diff --git a/plugins/curl/curl.c b/plugins/curl/curl.c
index 72971093..4e727b86 100644
--- a/plugins/curl/curl.c
+++ b/plugins/curl/curl.c
@@ -67,8 +67,18 @@ curl_load (void)
nbdkit_error ("libcurl initialization failed: %d", (int) r);
exit (EXIT_FAILURE);
}
+}
- load_pool ();
+int
+curl_get_ready (void)
+{
+ return pool_get_ready ();
+}
+
+int
+curl_after_fork (void)
+{
+ return pool_after_fork ();
}
static void
@@ -249,6 +259,8 @@ static struct nbdkit_plugin plugin = {
*/
//.config_help = curl_config_help,
.magic_config_key = "url",
+ .get_ready = curl_get_ready,
+ .after_fork = curl_after_fork,
.open = curl_open,
.close = curl_close,
.get_size = curl_get_size,
diff --git a/plugins/curl/curldefs.h b/plugins/curl/curldefs.h
index cab4a6b1..939c8d37 100644
--- a/plugins/curl/curldefs.h
+++ b/plugins/curl/curldefs.h
@@ -124,7 +124,8 @@ extern struct curl_handle *allocate_handle (void);
extern void free_handle (struct curl_handle *);
/* pool.c */
-extern void load_pool (void);
+extern int pool_get_ready (void);
+extern int pool_after_fork (void);
extern void pool_unload (void);
extern struct curl_handle *get_handle (void);
extern void put_handle (struct curl_handle *ch);
diff --git a/plugins/curl/pool.c b/plugins/curl/pool.c
index 50a623a4..eb2d330e 100644
--- a/plugins/curl/pool.c
+++ b/plugins/curl/pool.c
@@ -78,10 +78,16 @@ static curl_handle_list curl_handles = empty_vector;
static pthread_cond_t cond = PTHREAD_COND_INITIALIZER;
static size_t in_use = 0, waiting = 0;
-/* Initialize pool structures. */
-void
-load_pool (void)
+int
+pool_get_ready (void)
{
+ return 0;
+}
+
+int
+pool_after_fork (void)
+{
+ return 0;
}
/* Close and free all handles in the pool. */
--
2.39.3

36
SOURCES/0020-curl-Do-pool_unload-before-config_unload.patch Normal file
View File

@ -0,0 +1,36 @@
From 173530071d11433e26e1be1c11bc0e474f18079b Mon Sep 17 00:00:00 2001
From: "Richard W.M. Jones" <rjones@redhat.com>
Date: Fri, 28 Jul 2023 14:35:42 +0100
Subject: [PATCH] curl: Do pool_unload before config_unload
Since config.c deals with handles (and contains free_handle), and
since pool_unload calls free_handle to free handles, it's better to do
pool_unload first.
I don't believe this is a correctness issue now, but it will be in
subsequent commits.
Reviewed-by: Eric Blake <eblake@redhat.com>
(cherry picked from commit 650bde3bdb85ff2af0ea618fd64e726b7535d686)
---
plugins/curl/curl.c | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/plugins/curl/curl.c b/plugins/curl/curl.c
index 4e727b86..be42de36 100644
--- a/plugins/curl/curl.c
+++ b/plugins/curl/curl.c
@@ -84,9 +84,9 @@ curl_after_fork (void)
static void
curl_unload (void)
{
+ pool_unload ();
config_unload ();
scripts_unload ();
- pool_unload ();
display_times ();
curl_global_cleanup ();
}
--
2.39.3

52
SOURCES/0021-retry-request-Allow-get_size-operation-to-be-retried.patch Normal file
View File

@ -0,0 +1,52 @@
From 0b9807056e8be58b624b67b281accddc630c5d2c Mon Sep 17 00:00:00 2001
From: "Richard W.M. Jones" <rjones@redhat.com>
Date: Fri, 28 Jul 2023 16:47:58 +0100
Subject: [PATCH] retry-request: Allow get_size operation to be retried
This plugin operation might need to do some real work (instead of just
fetching a number from memory), and so it might have to be retried.
In particular, changes to the curl plugin make .get_size into a
heavyweight operation, where previously it was done as a side-effect
of .open. And so we must allow .get_size to be retried independent of
.open.
(cherry picked from commit 0ff9f9e202218e278bf38faf328ec80f25d6b661)
---
filters/retry-request/retry-request.c | 13 +++++++++++++
1 file changed, 13 insertions(+)
diff --git a/filters/retry-request/retry-request.c b/filters/retry-request/retry-request.c
index e5b8344c..8e3dd824 100644
--- a/filters/retry-request/retry-request.c
+++ b/filters/retry-request/retry-request.c
@@ -141,6 +141,18 @@ retry_request_open (nbdkit_next_open *next, nbdkit_context *nxdata,
return r == 0 ? NBDKIT_HANDLE_NOT_NEEDED : NULL;
}
+static int64_t
+retry_request_get_size (nbdkit_next *next, void *handle)
+{
+ int64_t r;
+ int *err = &errno; /* used by the RETRY_* macros */
+
+ RETRY_START("get_size")
+ r = next->get_size (next);
+ RETRY_END;
+ return r;
+}
+
static int
retry_request_pread (nbdkit_next *next,
void *handle, void *buf, uint32_t count, uint64_t offset,
@@ -267,6 +279,7 @@ static struct nbdkit_filter filter = {
.config = retry_request_config,
.config_help = retry_request_config_help,
.open = retry_request_open,
+ .get_size = retry_request_get_size,
.pread = retry_request_pread,
.pwrite = retry_request_pwrite,
.trim = retry_request_trim,
--
2.39.3

42
SOURCES/0022-tests-test-retry-request-mirror.c-Don-t-assume-state.patch Normal file
View File

@ -0,0 +1,42 @@
From 85241affaa52e4d67d0a22329d4cc5e2b0fe5ca3 Mon Sep 17 00:00:00 2001
From: "Richard W.M. Jones" <rjones@redhat.com>
Date: Fri, 28 Jul 2023 16:59:20 +0100
Subject: [PATCH] tests/test-retry-request-mirror.c: Don't assume state after
connect
After forthcoming changes to the curl plugin we cannot assume the
exact mirror we will be connected to after making the NBD connection.
So remove that assumption.
See: commit 38dccd848bd40cccdf012df7a606e13282aaeecb
Reviewed-by: Eric Blake <eblake@redhat.com>
(cherry picked from commit ee03fbd2385dcd8772ad2c838401df0b5d8c5617)
---
tests/test-retry-request-mirror.c | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/tests/test-retry-request-mirror.c b/tests/test-retry-request-mirror.c
index cf42c596..65440d2e 100644
--- a/tests/test-retry-request-mirror.c
+++ b/tests/test-retry-request-mirror.c
@@ -58,7 +58,7 @@ main (int argc, char *argv[])
const char *sockpath;
CLEANUP_FREE char *usp_param = NULL;
int i, j;
- char state = 0;
+ char state;
struct nbd_handle *nbd = NULL;
#ifndef HAVE_CURLOPT_UNIX_SOCKET_PATH
@@ -105,6 +105,8 @@ main (int argc, char *argv[])
if (nbd_connect_unix (nbd, sock /* NBD socket */) == -1)
goto nbd_error;
+ state = 0;
+
for (i = 0; i < 7 /* not divisible by 2 or 3 */; ++i) {
char buf[512];
--
2.39.3

1279
SOURCES/0023-curl-Use-curl-multi-interface.patch Normal file
View File

File diff suppressed because it is too large Load Diff

139
SOURCES/0024-curl-Redefine-connections-N-parameter-as-number-of-H.patch Normal file
View File

@ -0,0 +1,139 @@
From d33cf724dccd014f854a90bb7341fd36f890bb01 Mon Sep 17 00:00:00 2001
From: "Richard W.M. Jones" <rjones@redhat.com>
Date: Fri, 28 Jul 2023 11:49:24 +0100
Subject: [PATCH] curl: Redefine connections=<N> parameter as number of HTTP
connections
Previously (nbdkit 1.34) this was the number of easy handles. However
it turns out that easy handles can open multiple HTTP connections, and
in fact there's no good way to tell how many (and they are not
shared).
Now that we are using a curl multi, curl >= 7.30 provides a way to
limit the total number of actual HTTP connections, so we should just
use it. This is closer to what I intended this parameter to mean.
Also update the documentation to properly describe previous and
current behaviour.
Link: https://curl.se/mail/lib-2019-03/0102.html
Link: https://curl.se/mail/lib-2019-12/0044.html
Reviewed-by: Eric Blake <eblake@redhat.com>
(cherry picked from commit 2bb03f898e49ec892070055b6975914642fa8c5c)
---
plugins/curl/config.c | 2 +-
plugins/curl/curldefs.h | 3 +++
plugins/curl/nbdkit-curl-plugin.pod | 37 +++++++++++++++++++----------
plugins/curl/pool.c | 6 ++++-
4 files changed, 34 insertions(+), 14 deletions(-)
diff --git a/plugins/curl/config.c b/plugins/curl/config.c
index ce82d5f9..a7501707 100644
--- a/plugins/curl/config.c
+++ b/plugins/curl/config.c
@@ -494,7 +494,7 @@ curl_config_complete (void)
const char *curl_config_help =
"cainfo=<CAINFO> Path to Certificate Authority file.\n"
"capath=<CAPATH> Path to directory with CA certificates.\n"
- "connections=<N> Number of libcurl connections to use.\n"
+ "connections=<N> Number of HTTP connections to use.\n"
"cookie=<COOKIE> Set HTTP/HTTPS cookies.\n"
"cookiefile= Enable cookie processing.\n"
"cookiefile=<FILENAME> Read cookies from file.\n"
diff --git a/plugins/curl/curldefs.h b/plugins/curl/curldefs.h
index 6b158d85..73e7a3b4 100644
--- a/plugins/curl/curldefs.h
+++ b/plugins/curl/curldefs.h
@@ -50,6 +50,9 @@
* macro isn't present then Curl is very old.
*/
#ifdef CURL_AT_LEAST_VERSION
+#if CURL_AT_LEAST_VERSION (7, 30, 0)
+#define HAVE_CURLMOPT_MAX_TOTAL_CONNECTIONS
+#endif
#if CURL_AT_LEAST_VERSION (7, 55, 0)
#define HAVE_CURLINFO_CONTENT_LENGTH_DOWNLOAD_T
#endif
diff --git a/plugins/curl/nbdkit-curl-plugin.pod b/plugins/curl/nbdkit-curl-plugin.pod
index 7784b553..9821f111 100644
--- a/plugins/curl/nbdkit-curl-plugin.pod
+++ b/plugins/curl/nbdkit-curl-plugin.pod
@@ -58,10 +58,10 @@ L<CURLOPT_CAPATH(3)> for more information.
(nbdkit E<ge> 1.34)
-Open up to C<N> curl connections to the web server. The default is 4.
-Curl connections are shared between all NBD clients, so you may wish
-to increase this if you expect many simultaneous NBD clients (or a
-single client using many multi-conn connections).
+Open up to C<N> connections to the web server. The default is 16.
+Connections are shared between all NBD clients, so you may wish to
+increase this if you expect many simultaneous NBD clients (or a single
+client using many multi-conn connections).
See L</NBD CONNECTIONS AND CURL HANDLES> below.
@@ -397,15 +397,28 @@ user-agent header.
=head1 NBD CONNECTIONS AND CURL HANDLES
nbdkit E<le> 1.32 used a simple model where a new NBD connection would
-create a new libcurl handle. In practice this meant there was a
-1-to-1 relationship between NBD connections and HTTP connections to
-the remote web server (assuming http: or https: URL).
+create a new libcurl handle. Since a libcurl handle maintains a small
+cache of connections, this meant that the number of HTTP connections
+would be a small multiple of the number of incoming NBD connections
+and the total would not be limited (assuming http: or https: URL).
-nbdkit E<ge> 1.34 changed to using a fixed pool of libcurl handles
-shared across all NBD connections. You can control the maximum number
-of curl handles in the pool with the C<connections> parameter (default
-4). Note that if there are more than 4 NBD connections, they will
-share the 4 web server connections, unless you adjust C<connections>.
+nbdkit 1.34 changed to using a fixed pool of libcurl handles shared
+across all NBD connections. You can control the maximum number of
+curl handles in the pool with the C<connections> parameter (default
+4). Since each curl handle maintains a small cache of connections,
+this meant that the number of HTTP connections would be a small
+multiple of the C<connections> parameter. If there are more than 4
+incoming NBD connections, they will contend for the libcurl handles,
+unless you adjust C<connections>.
+
+nbdkit E<ge> 1.36 changed again to use a curl multi handle
+(L<libcurl-multi(3)>). Now the C<connections> parameter controls the
+maximum number of HTTP connections made to the remote server
+(L<CURLMOPT_MAX_TOTAL_CONNECTIONS(3)>). This is more efficient
+especially with HTTP/2 and HTTP/3, where each HTTP connection can
+contain a very large number of streams (typically up to 100)
+multiplexed over one connection. The default for C<connections> was
+raised to 16.
=head1 HEADER AND COOKIE SCRIPTS
diff --git a/plugins/curl/pool.c b/plugins/curl/pool.c
index 254951d1..7d44dfe5 100644
--- a/plugins/curl/pool.c
+++ b/plugins/curl/pool.c
@@ -79,7 +79,7 @@
/* Use '-D curl.pool=1' to debug handle pool. */
NBDKIT_DLL_PUBLIC int curl_debug_pool = 0;
-unsigned connections = 4;
+unsigned connections = 16;
/* Pipe used to notify background thread that a command is pending in
* the queue. A pointer to the 'struct command' is sent over the
@@ -115,6 +115,10 @@ pool_get_ready (void)
return -1;
}
+#ifdef HAVE_CURLMOPT_MAX_TOTAL_CONNECTIONS
+ curl_multi_setopt(multi, CURLMOPT_MAX_TOTAL_CONNECTIONS, (long) connections);
+#endif
+
return 0;
}
--
2.39.3

57
SOURCES/0025-curl-Disable-this-plugin-on-Windows.patch Normal file
View File

@ -0,0 +1,57 @@
From 852f23db007b13ca8e9e5548c3becbdc78e16601 Mon Sep 17 00:00:00 2001
From: "Richard W.M. Jones" <rjones@redhat.com>
Date: Tue, 1 Aug 2023 11:29:04 +0100
Subject: [PATCH] curl: Disable this plugin on Windows
There is no self pipe trick for Windows (or there is, but it would
require substantial porting work).
(cherry picked from commit 9b0759377f6779bbecc8647c026dcdac7f2ebd89)
---
plugins/curl/Makefile.am | 5 ++++-
tests/Makefile.am | 2 ++
2 files changed, 6 insertions(+), 1 deletion(-)
diff --git a/plugins/curl/Makefile.am b/plugins/curl/Makefile.am
index 7529fb2f..971365f1 100644
--- a/plugins/curl/Makefile.am
+++ b/plugins/curl/Makefile.am
@@ -34,6 +34,8 @@ include $(top_srcdir)/common-rules.mk
EXTRA_DIST = nbdkit-curl-plugin.pod
if HAVE_CURL
+# Disabled on Windows because no self-pipe.
+if !IS_WINDOWS
plugin_LTLIBRARIES = nbdkit-curl-plugin.la
@@ -85,4 +87,5 @@ nbdkit-curl-plugin.1: nbdkit-curl-plugin.pod \
endif HAVE_POD
-endif
+endif !IS_WINDOWS
+endif HAVE_CURL
diff --git a/tests/Makefile.am b/tests/Makefile.am
index f6c5ac9a..08514c7a 100644
--- a/tests/Makefile.am
+++ b/tests/Makefile.am
@@ -643,6 +643,7 @@ EXTRA_DIST += test-cdi.sh
# curl plugin test.
if HAVE_MKE2FS_WITH_D
if HAVE_CURL
+if !IS_WINDOWS
TESTS += \
test-curl-file.sh \
test-curl-header-script-fail.sh \
@@ -749,6 +750,7 @@ test_curl_cookie_script_LDADD = \
$(LIBNBD_LIBS) \
$(NULL)
+endif !IS_WINDOWS
endif HAVE_CURL
endif HAVE_MKE2FS_WITH_D
--
2.39.3

2
SOURCES/copy-patches.sh
View File

@ -6,7 +6,7 @@ set -e
# directory. Use it like this:
# ./copy-patches.sh
rhel_version=9.2
rhel_version=9.3
# Check we're in the right directory.
if [ ! -f nbdkit.spec ]; then

17
SOURCES/nbdkit-1.32.5.tar.gz.sig
View File

@ -1,17 +0,0 @@
-----BEGIN PGP SIGNATURE-----
iQJFBAABCAAvFiEE93dPsa0HSn6Mh2fqkXOPc+G3aKAFAmO0P9URHHJpY2hAYW5u
ZXhpYS5vcmcACgkQkXOPc+G3aKBj/w/+MahWvHpk6oOoif9pvshQ5ZXWWH8+4DCZ
fMPQPRuU3j64tj2kUrp87zChVPkQv27v+RuQcs5OuhfB/nvCIJiK6dSMq6KQmIv2
b3LieGAuIlhr89YIGQRi7j+R8iWiQgm+dT6BNeu3n7kbpEbJPPUHhz2YNlw1x/LJ
mfSEh+0HXKKz7HsCDwUCenq/pCPyD4p9x0UB0xqDT7PLg3qGwpHCMTuslrX3alOu
EYl+NDr9q266IQYGUh1zpSkobLNLvHI+TFyYEvytDnU4MylyslOdDIsA89E/y29r
rSMl9edDjhQ/h51In1Q8rKmlXFrcwDeRUywybn09m1gu++bxls5W1LFAZvp/YBa+
nWYv3o58epJSbhEL6NO5fl88Ea5JJYqhB+I1ezud/nJ3Uu/t9C7m69Mt96U5NhQ5
9irjO1Przz/3ft9+t7hW2u3MFNrEA/u1+e/Jnyr4+g8ZYmM1V7hQWqGwjO09zUZT
5xR41WHxG3ZbuUOv5r5Xt7Xvp1tiiWxiyEWOBydQwsnV9yZR/G2m3eWFE5H315r+
qGQXbr41mnsUAG6G8VXsNwK8cu3YQkZBHm4et4wFmvI6C1n2I7jhQBoXkdi9BKj9
Rh/h3DDmYB2Ud6G/ApWwfRFhSPM/apuUYfuYlXPKteFhtPjfbNSlHkm4hr1lFbK/
+X9eYoD9410=
=qSZX
-----END PGP SIGNATURE-----

17
SOURCES/nbdkit-1.34.2.tar.gz.sig Normal file
View File

@ -0,0 +1,17 @@
-----BEGIN PGP SIGNATURE-----
iQJFBAABCAAvFiEE93dPsa0HSn6Mh2fqkXOPc+G3aKAFAmS2eroRHHJpY2hAYW5u
ZXhpYS5vcmcACgkQkXOPc+G3aKBhcg/+IMU4RsKeuCiU6Y5k+bAYDN1bZWdGowM5
kRiarFOaO+917qj2K1FohXqz8xv+j7OSgQ0AyAfHEYCcrvwUoS3nxld/xf6Vq2NW
myPR4PdhKXx+ed2Y1PRwUykzdHLKIbV2itIQMvZJK6lN8CBEwooFOCbaJuZBUTxX
RA7seFbQeW5sSvqwfAEEPOtvj9Hj4AWOhbYHnxtzIH9NN+skkXATPR8ieqn/64Qx
JVa4Wx23nEHpfR0TzWdIbx08ggcmZstlVu1eiUB8MFhIdPnMnW3hPPWTcScZ45YP
kuxbqk5Q+AG4MsaGIiXaHryRW+bzr/Nn5Wb3AU+As2ZYtLRi4rvcGSPy1dTjblnU
5xVMz2bWjd0g/yU7ErD3NFGhp/RWfCOSOZ3AawbI8FdCMCWcJNIWNtT2sR7Ukqft
vOvim7oulUH6oLWwS/IkH9/SSxRXObMbnZTcqADaCCtif3F+9RkbbHeNfE85UEqm
Qc6VZoea2c2K/GTzfRDxoGuexU8p2zjiGiLltoMQboJqIU/Bz6q5Nqm8agL7Lm4x
/9HXUoF1+OQ8Ga18bQQbwrYIDcsB/xuMq3SlKo5h7az/XMeg3Pxz+YuBIMOcOBP7
N8M/yQZJIs5jkKkJaEt9gX0SzLAK0AiwtMlupew4B9iWrF+uUvM9z2pcxdqDJMCO
EOl+wAKKYqo=
=yvkW
-----END PGP SIGNATURE-----

111
SPECS/nbdkit.spec
View File

@ -46,14 +46,14 @@ ExclusiveArch: x86_64
%global verify_tarball_signature 1
# If there are patches which touch autotools files, set this to 1.
%global patches_touch_autotools %{nil}
%global patches_touch_autotools 1
# The source directory.
%global source_directory 1.32-stable
%global source_directory 1.34-stable
Name: nbdkit
Version: 1.32.5
Release: 4%{?dist}
Version: 1.34.2
Release: 1%{?dist}
Summary: NBD server
License: BSD
@ -75,13 +75,34 @@ Source2: libguestfs.keyring
Source3: copy-patches.sh
# Patches come from the upstream repository:
# https://gitlab.com/nbdkit/nbdkit/-/commits/rhel-9.2/
# https://gitlab.com/nbdkit/nbdkit/-/commits/rhel-9.3/
# Patches.
Patch0001: 0001-ssh-Remove-left-over-comment.patch
Patch0002: 0002-ssh-Improve-the-error-message-when-all-authenticatio.patch
Patch0003: 0003-luks-Avoid-crash-when-image-does-not-contain-a-LUKS-.patch
Patch0004: 0004-curl-Enable-multi-conn-for-read-only-connections.patch
Patch0001: 0001-tests-test-connect.c-Skip-if-exit-with-parent-is-not.patch
Patch0002: 0002-tests-Use-exit-with-parent-in-the-test-framework.patch
Patch0003: 0003-protect-Fix-copy-and-paste-error-in-the-documentatio.patch
Patch0004: 0004-docs-nbdkit-protocol.pod-Fix-manual-page-name.patch
Patch0005: 0005-retry-request-Print-operation-we-are-retrying-in-deb.patch
Patch0006: 0006-tests-test-ocaml-errorcodes.c-Don-t-use-assert-in-te.patch
Patch0007: 0007-tests-test-ocaml-errorcodes.c-Enable-verbose-message.patch
Patch0008: 0008-curl-Use-the-parallel-thread-model.patch
Patch0009: 0009-curl-Add-ipresolve-option.patch
Patch0010: 0010-curl-Add-resolve-option.patch
Patch0011: 0011-curl-pool-Add-abstract-load_pool-and-unload_pool-fun.patch
Patch0012: 0012-curl-Add-D-curl.times-1-to-collect-time-statistics.patch
Patch0013: 0013-curl-Fix-call-to-update_times.patch
Patch0014: 0014-curl-Move-configuration-code-to-a-separate-file.patch
Patch0015: 0015-curl-Make-times-seconds-field-slightly-wider.patch
Patch0016: 0016-curl-Use-_Atomic-type-to-accumulate-curl-timings.patch
Patch0017: 0017-curl-Add-D-curl.verbose.ids-1-to-display-conn-and-xf.patch
Patch0018: 0018-curl-Rename-unload_config-unload_pool-config_unload-.patch
Patch0019: 0019-pool-Add-outline-get_ready-and-after_fork-functions.patch
Patch0020: 0020-curl-Do-pool_unload-before-config_unload.patch
Patch0021: 0021-retry-request-Allow-get_size-operation-to-be-retried.patch
Patch0022: 0022-tests-test-retry-request-mirror.c-Don-t-assume-state.patch
Patch0023: 0023-curl-Use-curl-multi-interface.patch
Patch0024: 0024-curl-Redefine-connections-N-parameter-as-number-of-H.patch
Patch0025: 0025-curl-Disable-this-plugin-on-Windows.patch
# For automatic RPM Provides generation.
# See: https://rpm-software-management.github.io/rpm/manual/dependency_generators.html
@ -111,6 +132,7 @@ BuildRequires: e2fsprogs, e2fsprogs-devel
%if !0%{?rhel}
BuildRequires: xorriso
BuildRequires: rb_libtorrent-devel
BuildRequires: libblkio-devel
%endif
BuildRequires: bash-completion
BuildRequires: perl-devel
@ -269,6 +291,18 @@ This package contains example plugins for %{name}.
# The plugins below have non-trivial dependencies are so are
# packaged separately.
%if !0%{?rhel}
%package blkio-plugin
Summary: libblkio NVMe, vhost-user, vDPA, VFIO plugin for %{name}
License: BSD
Requires: %{name}-server%{?_isa} = %{version}-%{release}
%description blkio-plugin
This package contains libblkio (NVMe, vhost-user, vDPA, VFIO) support
for %{name}.
%endif
%if !0%{?rhel}
%package cc-plugin
Summary: Write small inline C plugins and scripts for %{name}
@ -689,35 +723,65 @@ autoreconf -i
# package into their vendor/ directory.
export PYTHON=%{__python3}
%configure \
--with-extra='%{name}-%{version}-%{release}' \
--disable-static \
--with-extra='%{name}-%{version}-%{release}' \
--with-tls-priority=@NBDKIT,SYSTEM \
--with-bash-completions \
--with-curl \
--with-gnutls \
--with-liblzma \
--with-libnbd \
--with-manpages \
--with-selinux \
--with-ssh \
--with-zlib \
--enable-linuxdisk \
--enable-python \
--disable-golang \
--disable-rust \
--disable-valgrind \
%if !0%{?rhel} && 0%{?have_ocaml}
--enable-ocaml \
%else
--disable-ocaml \
%endif
%if 0%{?rhel}
%if !0%{?rhel}
--enable-lua \
--enable-perl \
--enable-ruby \
--enable-tcl \
--enable-torrent \
--with-libblkio \
--with-ext2 \
--with-iso \
--with-libvirt \
%else
--disable-lua \
--disable-perl \
--disable-ruby \
--disable-tcl \
--disable-torrent \
--without-libblkio \
--without-ext2 \
--without-iso \
--without-libvirt \
%endif
%ifarch x86_64
--enable-vddk \
%else
--disable-vddk \
%endif
%if !0%{?rhel} && 0%{?have_libguestfs}
--with-libguestfs \
%else
--without-libguestfs \
%endif
%ifarch %{complete_test_arches}
%ifarch !0%{?rhel} && 0%{?have_libguestfs} && %{complete_test_arches}
--enable-libguestfs-tests \
%else
--disable-libguestfs-tests \
%endif
--with-tls-priority=@NBDKIT,SYSTEM
%{nil}
# Verify that it picked the correct version of Python
# to avoid RHBZ#1404631 happening again silently.
@ -738,7 +802,7 @@ rm -f $RPM_BUILD_ROOT%{_mandir}/man3/nbdkit-rust-plugin.3*
%if 0%{?rhel}
# In RHEL, remove some plugins we cannot --disable.
for f in cc cdi torrent; do
for f in cc cdi ; do
rm -f $RPM_BUILD_ROOT%{_libdir}/%{name}/plugins/nbdkit-$f-plugin.so
rm -f $RPM_BUILD_ROOT%{_mandir}/man?/nbdkit-$f-plugin.*
done
@ -875,6 +939,15 @@ export LIBGUESTFS_TRACE=1
%{_mandir}/man1/nbdkit-example*-plugin.1*
%if !0%{?rhel}
%files blkio-plugin
%doc README.md
%license LICENSE
%{_libdir}/%{name}/plugins/nbdkit-blkio-plugin.so
%{_mandir}/man1/nbdkit-blkio-plugin.1*
%endif
%if !0%{?rhel}
%files cc-plugin
%doc README.md
@ -1200,6 +1273,16 @@ export LIBGUESTFS_TRACE=1
%changelog
* Tue Aug 01 2023 Richard W.M. Jones <rjones@redhat.com> - 1.34.2-1
- Rebase to 1.34.2
resolves: rhbz#2168629
- Backport nbdkit-curl-plugin "multi" interface support
resolves: rhbz#2228131
* Tue Apr 18 2023 Richard W.M. Jones <rjones@redhat.com> - 1.34.1-1
- Rebase to 1.34.1
resolves: rhbz#2168629
* Fri Feb 03 2023 Richard W.M. Jones <rjones@redhat.com> - 1.32.5-4
- Rebase to new stable branch version 1.32.5
resolves: rhbz#2135765