rpms/nbdkit
7
0
Fork 0
Code Issues Pull Requests Releases Activity

import nbdkit-1.30.8-1.el9

Browse Source
This commit is contained in:
CentOS Sources 2022-11-15 01:24:59 -05:00 committed by Stepan Oksanichenko
parent bf716c3fe9
commit 70a07e8771
66 changed files with 7494 additions and 7541 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.gitignore vendored
View File

@ -1,2 +1,2 @@
SOURCES/libguestfs.keyring
SOURCES/nbdkit-1.28.5.tar.gz
SOURCES/nbdkit-1.30.8.tar.gz

2
.nbdkit.metadata
View File

@ -1,2 +1,2 @@
cc1b37b9cfafa515aab3eefd345ecc59aac2ce7b SOURCES/libguestfs.keyring
60f2c2021658a94d778eb9cde0123d1c092ff15d SOURCES/nbdkit-1.28.5.tar.gz
6c4607ff13e13460cfdf67f47b9fea9ac0a8ebc3 SOURCES/nbdkit-1.30.8.tar.gz

293
SOURCES/0001-ssh-Allow-the-remote-file-to-be-created.patch Normal file
View File

@ -0,0 +1,293 @@
From 6a2b0aac8be655524ea223e32cac0395fcc9f975 Mon Sep 17 00:00:00 2001
From: "Richard W.M. Jones" <rjones@redhat.com>
Date: Fri, 15 Apr 2022 12:08:37 +0100
Subject: [PATCH] ssh: Allow the remote file to be created
This adds new parameters, create=(true|false), create-size=SIZE and
create-mode=MODE to create and truncate the remote file.
Reviewed-by: Laszlo Ersek <lersek@redhat.com>
(cherry picked from commit 0793f30b1071753532362b2ebf9cb8156a88c3c3)
---
plugins/ssh/nbdkit-ssh-plugin.pod | 34 ++++++++-
plugins/ssh/ssh.c | 112 +++++++++++++++++++++++++++---
tests/test-ssh.sh | 13 +++-
3 files changed, 146 insertions(+), 13 deletions(-)
diff --git a/plugins/ssh/nbdkit-ssh-plugin.pod b/plugins/ssh/nbdkit-ssh-plugin.pod
index 3f401c15..2bc2c4a7 100644
--- a/plugins/ssh/nbdkit-ssh-plugin.pod
+++ b/plugins/ssh/nbdkit-ssh-plugin.pod
@@ -5,8 +5,10 @@ nbdkit-ssh-plugin - access disk images over the SSH protocol
=head1 SYNOPSIS
nbdkit ssh host=HOST [path=]PATH
- [compression=true] [config=CONFIG_FILE] [identity=FILENAME]
- [known-hosts=FILENAME] [password=PASSWORD|-|+FILENAME]
+ [compression=true] [config=CONFIG_FILE]
+ [create=true] [create-mode=MODE] [create-size=SIZE]
+ [identity=FILENAME] [known-hosts=FILENAME]
+ [password=PASSWORD|-|+FILENAME]
[port=PORT] [timeout=SECS] [user=USER]
[verify-remote-host=false]
@@ -62,6 +64,34 @@ The C<config> parameter is optional. If it is I<not> specified at all
then F<~/.ssh/config> and F</etc/ssh/ssh_config> are both read.
Missing or unreadable files are ignored.
+=item B<create=true>
+
+(nbdkit E<ge> 1.32)
+
+If set, the remote file will be created. The remote file is created
+on the first NBD connection to nbdkit, not when nbdkit starts up. If
+the file already exists, it will be replaced and any existing content
+lost.
+
+If using this option, you must use C<create-size>. C<create-mode> can
+be used to control the permissions of the new file.
+
+=item B<create-mode=>MODE
+
+(nbdkit E<ge> 1.32)
+
+If using C<create=true> specify the default permissions of the new
+remote file. You can use octal modes like C<create-mode=0777> or
+C<create-mode=0644>. The default is C<0600>, ie. only readable and
+writable by the remote user.
+
+=item B<create-size=>SIZE
+
+(nbdkit E<ge> 1.32)
+
+If using C<create=true>, specify the virtual size of the new disk.
+C<SIZE> can use modifiers like C<100M> etc.
+
=item B<host=>HOST
Specify the name or IP address of the remote host.
diff --git a/plugins/ssh/ssh.c b/plugins/ssh/ssh.c
index 39d77e44..5e314cd7 100644
--- a/plugins/ssh/ssh.c
+++ b/plugins/ssh/ssh.c
@@ -44,6 +44,8 @@
#include <fcntl.h>
#include <sys/stat.h>
+#include <pthread.h>
+
#include <libssh/libssh.h>
#include <libssh/sftp.h>
#include <libssh/callbacks.h>
@@ -51,6 +53,7 @@
#include <nbdkit-plugin.h>
#include "array-size.h"
+#include "cleanup.h"
#include "const-string-vector.h"
#include "minmax.h"
@@ -64,6 +67,9 @@ static const char *known_hosts = NULL;
static const_string_vector identities = empty_vector;
static uint32_t timeout = 0;
static bool compression = false;
+static bool create = false;
+static int64_t create_size = -1;
+static unsigned create_mode = S_IRUSR | S_IWUSR /* 0600 */;
/* config can be:
* NULL => parse options from default file
@@ -167,6 +173,27 @@ ssh_config (const char *key, const char *value)
return -1;
compression = r;
}
+ else if (strcmp (key, "create") == 0) {
+ r = nbdkit_parse_bool (value);
+ if (r == -1)
+ return -1;
+ create = r;
+ }
+ else if (strcmp (key, "create-size") == 0) {
+ create_size = nbdkit_parse_size (value);
+ if (create_size == -1)
+ return -1;
+ }
+ else if (strcmp (key, "create-mode") == 0) {
+ r = nbdkit_parse_unsigned (key, value, &create_mode);
+ if (r == -1)
+ return -1;
+ /* OpenSSH checks this too. */
+ if (create_mode > 0777) {
+ nbdkit_error ("create-mode must be <= 0777");
+ return -1;
+ }
+ }
else {
nbdkit_error ("unknown parameter '%s'", key);
@@ -186,6 +213,13 @@ ssh_config_complete (void)
return -1;
}
+ /* If create=true, create-size must be supplied. */
+ if (create && create_size == -1) {
+ nbdkit_error ("if using create=true, you must specify the size "
+ "of the new remote file using create-size=SIZE");
+ return -1;
+ }
+
return 0;
}
@@ -200,7 +234,10 @@ ssh_config_complete (void)
"identity=<FILENAME> Prepend private key (identity) file.\n" \
"timeout=SECS Set SSH connection timeout.\n" \
"verify-remote-host=false Ignore known_hosts.\n" \
- "compression=true Enable compression."
+ "compression=true Enable compression.\n" \
+ "create=true Create the remote file.\n" \
+ "create-mode=MODE Set the permissions of the remote file.\n" \
+ "create-size=SIZE Set the size of the remote file."
/* Since we must simulate atomic pread and pwrite using seek +
* read/write, calls on each handle must be serialized.
@@ -329,6 +366,65 @@ authenticate (struct ssh_handle *h)
return -1;
}
+/* This function opens or creates the remote file (depending on
+ * create=false|true). Parallel connections might call this function
+ * at the same time, and so we must hold a lock to ensure that the
+ * file is created at most once.
+ */
+static pthread_mutex_t create_lock = PTHREAD_MUTEX_INITIALIZER;
+
+static sftp_file
+open_or_create_path (ssh_session session, sftp_session sftp, int readonly)
+{
+ ACQUIRE_LOCK_FOR_CURRENT_SCOPE (&create_lock);
+ int access_type;
+ int r;
+ sftp_file file;
+
+ access_type = readonly ? O_RDONLY : O_RDWR;
+ if (create) access_type |= O_CREAT | O_TRUNC;
+
+ file = sftp_open (sftp, path, access_type, S_IRWXU);
+ if (!file) {
+ nbdkit_error ("cannot %s file for %s: %s",
+ create ? "create" : "open",
+ readonly ? "reading" : "writing",
+ ssh_get_error (session));
+ return NULL;
+ }
+
+ if (create) {
+ /* There's no sftp_truncate call. However OpenSSH lets you call
+ * SSH_FXP_SETSTAT + SSH_FILEXFER_ATTR_SIZE which invokes
+ * truncate(2) on the server. Libssh doesn't provide a binding
+ * for SSH_FXP_FSETSTAT so we have to pass the session + path.
+ */
+ struct sftp_attributes_struct attrs = {
+ .flags = SSH_FILEXFER_ATTR_SIZE |
+ SSH_FILEXFER_ATTR_PERMISSIONS,
+ .size = create_size,
+ .permissions = create_mode,
+ };
+
+ r = sftp_setstat (sftp, path, &attrs);
+ if (r != SSH_OK) {
+ nbdkit_error ("setstat failed: %s", ssh_get_error (session));
+
+ /* Best-effort attempt to delete the remote file on failure. */
+ r = sftp_unlink (sftp, path);
+ if (r != SSH_OK)
+ nbdkit_debug ("unlink failed: %s", ssh_get_error (session));
+
+ return NULL;
+ }
+ }
+
+ /* On the next connection, don't create or truncate the file. */
+ create = false;
+
+ return file;
+}
+
/* Create the per-connection handle. */
static void *
ssh_open (int readonly)
@@ -337,7 +433,6 @@ ssh_open (int readonly)
const int set = 1;
size_t i;
int r;
- int access_type;
h = calloc (1, sizeof *h);
if (h == NULL) {
@@ -471,7 +566,7 @@ ssh_open (int readonly)
if (authenticate (h) == -1)
goto err;
- /* Open the SFTP connection and file. */
+ /* Open the SFTP connection. */
h->sftp = sftp_new (h->session);
if (!h->sftp) {
nbdkit_error ("failed to allocate sftp session: %s",
@@ -484,14 +579,11 @@ ssh_open (int readonly)
ssh_get_error (h->session));
goto err;
}
- access_type = readonly ? O_RDONLY : O_RDWR;
- h->file = sftp_open (h->sftp, path, access_type, S_IRWXU);
- if (!h->file) {
- nbdkit_error ("cannot open file for %s: %s",
- readonly ? "reading" : "writing",
- ssh_get_error (h->session));
+
+ /* Open or create the remote file. */
+ h->file = open_or_create_path (h->session, h->sftp, readonly);
+ if (!h->file)
goto err;
- }
nbdkit_debug ("opened libssh handle");
diff --git a/tests/test-ssh.sh b/tests/test-ssh.sh
index 6c0ce410..f04b4488 100755
--- a/tests/test-ssh.sh
+++ b/tests/test-ssh.sh
@@ -36,6 +36,7 @@ set -x
requires test -f disk
requires nbdcopy --version
+requires stat --version
# Check that ssh to localhost will work without any passwords or phrases.
#
@@ -48,7 +49,7 @@ then
exit 77
fi
-files="ssh.img"
+files="ssh.img ssh2.img"
rm -f $files
cleanup_fn rm -f $files
@@ -59,3 +60,13 @@ nbdkit -v -D ssh.log=2 -U - \
# The output should be identical.
cmp disk ssh.img
+
+# Copy local file 'ssh.img' to newly created "remote" 'ssh2.img'
+size="$(stat -c %s disk)"
+nbdkit -v -D ssh.log=2 -U - \
+ ssh host=localhost $PWD/ssh2.img \
+ create=true create-size=$size \
+ --run 'nbdcopy ssh.img "$uri"'
+
+# The output should be identical.
+cmp disk ssh2.img
--
2.31.1

117
SOURCES/0001-vddk-Refactor-how-D-vddk.stats-1-is-collected.patch
View File

@ -1,117 +0,0 @@
From 4f2f557b349ad621e502e304c87280835cf13146 Mon Sep 17 00:00:00 2001
From: "Richard W.M. Jones" <rjones@redhat.com>
Date: Thu, 21 Oct 2021 14:49:52 +0100
Subject: [PATCH] vddk: Refactor how -D vddk.stats=1 is collected
In order to allow us to collect more per-API stats, introduce a global
struct per API for storing these stats.
(cherry picked from commit 3d8657f3d9a2c1b59284333566428b4c7ce32a74)
---
plugins/vddk/vddk.c | 36 ++++++++++++++++++------------------
1 file changed, 18 insertions(+), 18 deletions(-)
diff --git a/plugins/vddk/vddk.c b/plugins/vddk/vddk.c
index 80f5870e..3d751544 100644
--- a/plugins/vddk/vddk.c
+++ b/plugins/vddk/vddk.c
@@ -1,5 +1,5 @@
/* nbdkit
- * Copyright (C) 2013-2020 Red Hat Inc.
+ * Copyright (C) 2013-2021 Red Hat Inc.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are
@@ -103,14 +103,23 @@ static bool is_remote;
/* For each VDDK API define a variable to store the time taken (used
* to implement -D vddk.stats=1).
*/
+struct vddk_stat {
+ const char *name; /* function name */
+ int64_t usecs; /* total number of usecs consumed */
+};
static pthread_mutex_t stats_lock = PTHREAD_MUTEX_INITIALIZER;
static void display_stats (void);
-#define STUB(fn,ret,args) static int64_t stats_##fn;
-#define OPTIONAL_STUB(fn,ret,args) static int64_t stats_##fn;
+#define STUB(fn,ret,args) \
+ static struct vddk_stat stats_##fn = { .name = #fn }
+#define OPTIONAL_STUB(fn,ret,args) \
+ static struct vddk_stat stats_##fn = { .name = #fn }
#include "vddk-stubs.h"
#undef STUB
#undef OPTIONAL_STUB
+/* Macros to bracket each VDDK API call, for printing debugging
+ * information and collecting statistics.
+ */
#define VDDK_CALL_START(fn, fs, ...) \
do { \
struct timeval start_t, end_t; \
@@ -131,10 +140,11 @@ static void display_stats (void);
if (vddk_debug_stats) { \
gettimeofday (&end_t, NULL); \
ACQUIRE_LOCK_FOR_CURRENT_SCOPE (&stats_lock); \
- stats_##fn += tvdiff_usec (&start_t, &end_t); \
+ stats_##fn.usecs += tvdiff_usec (&start_t, &end_t); \
} \
} while (0)
+/* Print VDDK errors. */
#define VDDK_ERROR(err, fs, ...) \
do { \
char *vddk_err_msg; \
@@ -167,10 +177,6 @@ vddk_unload (void)
free (password);
}
-struct vddk_stat {
- const char *fn;
- int64_t usecs;
-};
DEFINE_VECTOR_TYPE(statlist, struct vddk_stat)
static int
@@ -179,7 +185,7 @@ stat_compare (const void *vp1, const void *vp2)
const struct vddk_stat *st1 = vp1;
const struct vddk_stat *st2 = vp2;
- /* Note: sorts in reverse order. */
+ /* Note: sorts in reverse order of time spent in each API call. */
if (st1->usecs < st2->usecs) return 1;
else if (st1->usecs > st2->usecs) return -1;
else return 0;
@@ -189,19 +195,13 @@ static void
display_stats (void)
{
statlist stats = empty_vector;
- struct vddk_stat st;
size_t i;
-#define ADD_ONE_STAT(fn_, usecs_) \
- st.fn = fn_; \
- st.usecs = usecs_; \
- statlist_append (&stats, st)
-#define STUB(fn,ret,args) ADD_ONE_STAT (#fn, stats_##fn);
-#define OPTIONAL_STUB(fn,ret,args) ADD_ONE_STAT (#fn, stats_##fn);
+#define STUB(fn,ret,args) statlist_append (&stats, stats_##fn)
+#define OPTIONAL_STUB(fn,ret,args) statlist_append (&stats, stats_##fn)
#include "vddk-stubs.h"
#undef STUB
#undef OPTIONAL_STUB
-#undef ADD_ONE_STAT
qsort (stats.ptr, stats.size, sizeof stats.ptr[0], stat_compare);
@@ -209,7 +209,7 @@ display_stats (void)
nbdkit_debug ("%-40s %9s", "", "µs");
for (i = 0; i < stats.size; ++i) {
if (stats.ptr[i].usecs)
- nbdkit_debug ("%-40s %9" PRIi64, stats.ptr[i].fn, stats.ptr[i].usecs);
+ nbdkit_debug ("%-40s %9" PRIi64, stats.ptr[i].name, stats.ptr[i].usecs);
}
statlist_reset (&stats);
}
--
2.31.1

794
SOURCES/0002-readahead-Rewrite-this-filter-so-it-prefetches-using.patch Normal file
View File

@ -0,0 +1,794 @@
From ac40ae11bc9983e11185749b23e793568cb366cc Mon Sep 17 00:00:00 2001
From: "Richard W.M. Jones" <rjones@redhat.com>
Date: Sat, 16 Apr 2022 18:39:13 +0100
Subject: [PATCH] readahead: Rewrite this filter so it prefetches using .cache
The previous readahead filter did not work well and we have stopped
using it in virt-v2v. However the concept is a good one if we can
make it work. This commit completely rethinks the filter.
Now, in parallel with the ordinary pread command, we issue a prefetch
(ie. .cache) to the underlying plugin for the data immediately
following the pread. For example a simple sequence of operations:
t=1 pread (offset=0, count=65536)
t=2 pread (offset=65536, count=65536)
t=3 pread (offset=131072, count=65536)
would become:
t=1 pread (offset=0, count=65536) <--\ issued
cache (offset=65536, count=65536) <--/ in parallel
t=2 pread (offset=65536, count=65536)
cache (offset=131072, count=65536)
t=3 pread (offset=131072, count=65536)
cache (offset=196608, count=65536)
This requires that the underlying filter(s) and plugin chain can
actually do something with the .cache request. If this is not the
case then the filter does nothing (but it will print a warning). For
plugins which don't have native support for prefetching, it is
sufficient to insert nbdkit-cache-filter after this filter.
(nbdkit-cow-filter can also be used with cow-on-cache=true, but that
is more useful for advanced users who are already using the cow
filter).
The implementation creates a background thread per connection to issue
the parallel .cache requests. This is safer than the alternative (one
background thread in total) since we don't have to deal with the
problem of cache requests being issued with the wrong export name or
stale next pointer. The background thread is controlled by a queue of
commands, with the only possible commands being "cache" or "quit".
Because the background thread issues parallel requests on the same
connection, the underlying plugin must support the parallel thread
model, otherwise we would be violating the plugin's thread model. It
may be possible in future to open a new connection from the background
thread (but with the same exportname), which would lift this
restriction to at least serialize_requests. Because of the current
limitation, nbdkit-curl-plugin cannot use prefetch.
(cherry picked from commit 2ff548d66ad3eae87868402ec5b3319edd12090f)
---
TODO | 22 +-
filters/readahead/Makefile.am | 2 +
filters/readahead/bgthread.c | 76 ++++
filters/readahead/nbdkit-readahead-filter.pod | 70 +++-
filters/readahead/readahead.c | 338 +++++++++---------
filters/readahead/readahead.h | 60 ++++
tests/test-readahead.sh | 2 +-
7 files changed, 367 insertions(+), 203 deletions(-)
create mode 100644 filters/readahead/bgthread.c
create mode 100644 filters/readahead/readahead.h
diff --git a/TODO b/TODO
index 5ae21db5..4d2a9796 100644
--- a/TODO
+++ b/TODO
@@ -62,16 +62,6 @@ General ideas for improvements
continue to keep their non-standard handshake while utilizing nbdkit
to prototype new behaviors in serving the kernel.
-* Background thread for filters. Some filters (readahead, cache and
- proposed scan filter - see below) could be more effective if they
- were able to defer work to a background thread. We finally have
- nbdkit_next_context_open and friends for allowing a background
- thread to have access into the plugin, but still need to worry about
- thread-safety (how much must the filter do vs. nbdkit, to avoid
- calling into the plugin too many times at once) and cleanup
- (spawning the thread during .after_fork is viable, but cleaning it
- up during .unload is too late).
-
* "nbdkit.so": nbdkit as a loadable shared library. The aim of nbdkit
is to make it reusable from other programs (see nbdkit-captive(1)).
If it was a loadable shared library it would be even more reusable.
@@ -228,6 +218,8 @@ Suggestions for filters
* nbdkit-cache-filter should handle ENOSPC errors automatically by
reclaiming blocks from the cache
+* nbdkit-cache-filter could use a background thread for reclaiming.
+
* zstd filter was requested as a way to do what we currently do with
xz but saving many hours on compression (at the cost of hundreds of
MBs of extra data)
@@ -240,6 +232,16 @@ Suggestions for filters
could inject a flush after pausing. However this requires that
filter background threads have access to the plugin (see above).
+nbdkit-readahead-filter:
+
+* The filter should open a new connection to the plugin per background
+ thread so it is able to work with plugins that use the
+ serialize_requests thread model (like curl). At the moment it makes
+ requests on the same connection, so it requires plugins to use the
+ parallel thread model.
+
+* It should combine (or avoid) overlapping cache requests.
+
nbdkit-rate-filter:
* allow other kinds of traffic shaping such as VBR
diff --git a/filters/readahead/Makefile.am b/filters/readahead/Makefile.am
index ee5bb3fb..187993ae 100644
--- a/filters/readahead/Makefile.am
+++ b/filters/readahead/Makefile.am
@@ -37,6 +37,8 @@ filter_LTLIBRARIES = nbdkit-readahead-filter.la
nbdkit_readahead_filter_la_SOURCES = \
readahead.c \
+ readahead.h \
+ bgthread.c \
$(top_srcdir)/include/nbdkit-filter.h \
$(NULL)
diff --git a/filters/readahead/bgthread.c b/filters/readahead/bgthread.c
new file mode 100644
index 00000000..5894bb5f
--- /dev/null
+++ b/filters/readahead/bgthread.c
@@ -0,0 +1,76 @@
+/* nbdkit
+ * Copyright (C) 2019-2022 Red Hat Inc.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * * Neither the name of Red Hat nor the names of its contributors may be
+ * used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY RED HAT AND CONTRIBUTORS ''AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+ * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL RED HAT OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
+ * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+ * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <config.h>
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <stdint.h>
+#include <pthread.h>
+
+#include <nbdkit-filter.h>
+
+#include "readahead.h"
+
+#include "cleanup.h"
+
+void *
+readahead_thread (void *vp)
+{
+ struct bgthread_ctrl *ctrl = vp;
+
+ for (;;) {
+ struct command cmd;
+
+ /* Wait until we are sent at least one command. */
+ {
+ ACQUIRE_LOCK_FOR_CURRENT_SCOPE (&ctrl->lock);
+ while (ctrl->cmds.len == 0)
+ pthread_cond_wait (&ctrl->cond, &ctrl->lock);
+ cmd = ctrl->cmds.ptr[0];
+ command_queue_remove (&ctrl->cmds, 0);
+ }
+
+ switch (cmd.type) {
+ case CMD_QUIT:
+ /* Finish processing and exit the thread. */
+ return NULL;
+
+ case CMD_CACHE:
+ /* Issue .cache (readahead) to underlying plugin. We ignore any
+ * errors because there's no way to communicate that back to the
+ * client, and readahead is only advisory.
+ */
+ cmd.next->cache (cmd.next, cmd.count, cmd.offset, 0, NULL);
+ }
+ }
+}
diff --git a/filters/readahead/nbdkit-readahead-filter.pod b/filters/readahead/nbdkit-readahead-filter.pod
index c220d379..630e5924 100644
--- a/filters/readahead/nbdkit-readahead-filter.pod
+++ b/filters/readahead/nbdkit-readahead-filter.pod
@@ -1,28 +1,66 @@
=head1 NAME
-nbdkit-readahead-filter - prefetch data when reading sequentially
+nbdkit-readahead-filter - prefetch data ahead of sequential reads
=head1 SYNOPSIS
- nbdkit --filter=readahead plugin
+ nbdkit --filter=readahead PLUGIN
+
+ nbdkit --filter=readahead --filter=cache PLUGIN
+
+ nbdkit --filter=readahead --filter=cow PLUGIN cow-on-cache=true
=head1 DESCRIPTION
C<nbdkit-readahead-filter> is a filter that prefetches data when the
-client is reading sequentially.
+client is reading.
-A common use for this filter is to accelerate sequential copy
-operations (like S<C<qemu-img convert>>) when plugin requests have a
-high overhead (like L<nbdkit-curl-plugin(1)>). For example:
-
- nbdkit -U - --filter=readahead curl https://example.com/disk.img \
- --run 'qemu-img convert $nbd disk.img'
+When the client issues a read, this filter issues a parallel prefetch
+(C<.cache>) for subsequent data. Plugins which support this command
+will prefetch the data, making sequential reads faster. For plugins
+which do not support this command, you can inject
+L<nbdkit-cache-filter(1)> below (after) this filter, giving
+approximately the same effect. L<nbdkit-cow-filter(1)> can be used
+instead of nbdkit-cache-filter, if you add the C<cow-on-cache=true>
+option.
The filter uses a simple adaptive algorithm which accelerates
-sequential reads, but has a small penalty if the client does random
-reads. If the client mixes reads with writes or write-like operations
-(trimming, zeroing) then it will work but there can be a large
-performance penalty.
+sequential reads and requires no further configuration.
+
+=head2 Limitations
+
+In a number of significant cases this filter will do nothing. The
+filter will print a warning message if this happens.
+
+=over 4
+
+=item Thread model must be parallel
+
+For example L<nbdkit-curl-plugin(1)> only supports
+C<serialize_requests>, and so this filter cannot perform prefetches in
+parallel with the read requests.
+
+We may be able to lift this restriction in future.
+
+=item Underlying filters or plugin must support C<.cache> (prefetch)
+
+Very many plugins do not have the concept of prefetching and/or
+do not implement the C<.cache> callback, and so there is no
+way for this filter to issue prefetches.
+
+You can usually get around this by adding I<--filter=cache> after this
+filter as explained above. It may be necessary to limit the total
+size of the cache (see L<nbdkit-cache-filter(1)/CACHE MAXIMUM SIZE>).
+
+=item Clients and kernels may do readahead already
+
+It may be the case that NBD clients are already issuing
+C<NBD_CMD_CACHE> (NBD prefetch) commands. It may also be the case
+that your plugin is using local file functions where the kernel is
+doing readahead. In such cases this filter is not necessary and may
+be pessimal.
+
+=back
=head1 PARAMETERS
@@ -50,9 +88,9 @@ C<nbdkit-readahead-filter> first appeared in nbdkit 1.12.
L<nbdkit(1)>,
L<nbdkit-cache-filter(1)>,
-L<nbdkit-curl-plugin(1)>,
+L<nbdkit-cow-filter(1)>,
+L<nbdkit-file-plugin(1)>,
L<nbdkit-retry-filter(1)>,
-L<nbdkit-ssh-plugin(1)>,
L<nbdkit-torrent-plugin(1)>,
L<nbdkit-vddk-plugin(1)>,
L<nbdkit-filter(3)>,
@@ -64,4 +102,4 @@ Richard W.M. Jones
=head1 COPYRIGHT
-Copyright (C) 2019 Red Hat Inc.
+Copyright (C) 2019-2022 Red Hat Inc.
diff --git a/filters/readahead/readahead.c b/filters/readahead/readahead.c
index f5552d4c..1d7ae111 100644
--- a/filters/readahead/readahead.c
+++ b/filters/readahead/readahead.c
@@ -1,5 +1,5 @@
/* nbdkit
- * Copyright (C) 2019-2021 Red Hat Inc.
+ * Copyright (C) 2019-2022 Red Hat Inc.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are
@@ -34,232 +34,218 @@
#include <stdio.h>
#include <stdlib.h>
+#include <stdbool.h>
#include <stdint.h>
#include <string.h>
#include <errno.h>
-
#include <pthread.h>
#include <nbdkit-filter.h>
+#include "readahead.h"
+
#include "cleanup.h"
#include "minmax.h"
-
-/* Copied from server/plugins.c. */
-#define MAX_REQUEST_SIZE (64 * 1024 * 1024)
+#include "vector.h"
/* These could be made configurable in future. */
-#define READAHEAD_MIN 65536
-#define READAHEAD_MAX MAX_REQUEST_SIZE
-
-/* This lock protects the global state. */
-static pthread_mutex_t lock = PTHREAD_MUTEX_INITIALIZER;
-
-/* The real size of the underlying plugin. */
-static uint64_t size;
+#define READAHEAD_MIN 32768
+#define READAHEAD_MAX (4*1024*1024)
/* Size of the readahead window. */
+static pthread_mutex_t window_lock = PTHREAD_MUTEX_INITIALIZER;
static uint64_t window = READAHEAD_MIN;
+static uint64_t last_offset = 0, last_readahead = 0;
-/* The single prefetch buffer shared by all threads, and its virtual
- * location in the virtual disk. The prefetch buffer grows
- * dynamically as required, but never shrinks.
+static int thread_model = -1; /* Thread model of the underlying plugin. */
+
+/* Per-connection data. */
+struct readahead_handle {
+ int can_cache; /* Can the underlying plugin cache? */
+ pthread_t thread; /* The background thread, one per connection. */
+ struct bgthread_ctrl ctrl;
+};
+
+/* We have various requirements of the underlying filter(s) + plugin:
+ * - They must support NBDKIT_CACHE_NATIVE (otherwise our requests
+ * would not do anything useful).
+ * - They must use the PARALLEL thread model (otherwise we could
+ * violate their thread model).
+ */
+static bool
+filter_working (struct readahead_handle *h)
+{
+ return
+ h->can_cache == NBDKIT_CACHE_NATIVE &&
+ thread_model == NBDKIT_THREAD_MODEL_PARALLEL;
+}
+
+static bool
+suggest_cache_filter (struct readahead_handle *h)
+{
+ return
+ h->can_cache != NBDKIT_CACHE_NATIVE &&
+ thread_model == NBDKIT_THREAD_MODEL_PARALLEL;
+}
+
+/* We need to hook into .get_ready() so we can read the final thread
+ * model (of the whole server).
*/
-static char *buffer = NULL;
-static size_t bufsize = 0;
-static uint64_t position;
-static uint32_t length = 0;
+static int
+readahead_get_ready (int final_thread_model)
+{
+ thread_model = final_thread_model;
+ return 0;
+}
+
+static int
+send_command_to_background_thread (struct bgthread_ctrl *ctrl,
+ const struct command cmd)
+{
+ ACQUIRE_LOCK_FOR_CURRENT_SCOPE (&ctrl->lock);
+ if (command_queue_append (&ctrl->cmds, cmd) == -1)
+ return -1;
+ /* Signal the thread if it could be sleeping on an empty queue. */
+ if (ctrl->cmds.len == 1)
+ pthread_cond_signal (&ctrl->cond);
+ return 0;
+}
+
+static void *
+readahead_open (nbdkit_next_open *next, nbdkit_context *nxdata,
+ int readonly, const char *exportname, int is_tls)
+{
+ struct readahead_handle *h;
+ int err;
+
+ if (next (nxdata, readonly, exportname) == -1)
+ return NULL;
+
+ h = malloc (sizeof *h);
+ if (h == NULL) {
+ nbdkit_error ("malloc: %m");
+ return NULL;
+ }
+
+ h->ctrl.cmds = (command_queue) empty_vector;
+ pthread_mutex_init (&h->ctrl.lock, NULL);
+ pthread_cond_init (&h->ctrl.cond, NULL);
+
+ /* Create the background thread. */
+ err = pthread_create (&h->thread, NULL, readahead_thread, &h->ctrl);
+ if (err != 0) {
+ errno = err;
+ nbdkit_error ("pthread_create: %m");
+ pthread_cond_destroy (&h->ctrl.cond);
+ pthread_mutex_destroy (&h->ctrl.lock);
+ free (h);
+ return NULL;
+ }
+
+ return h;
+}
static void
-readahead_unload (void)
+readahead_close (void *handle)
{
- free (buffer);
+ struct readahead_handle *h = handle;
+ const struct command quit_cmd = { .type = CMD_QUIT };
+
+ send_command_to_background_thread (&h->ctrl, quit_cmd);
+ pthread_join (h->thread, NULL);
+ pthread_cond_destroy (&h->ctrl.cond);
+ pthread_mutex_destroy (&h->ctrl.lock);
+ command_queue_reset (&h->ctrl.cmds);
+ free (h);
}
-static int64_t readahead_get_size (nbdkit_next *next, void *handle);
-
-/* In prepare, force a call to get_size which sets the size global. */
static int
-readahead_prepare (nbdkit_next *next, void *handle, int readonly)
+readahead_can_cache (nbdkit_next *next, void *handle)
{
- int64_t r;
+ struct readahead_handle *h = handle;
+ int r;
- r = readahead_get_size (next, handle);
- return r >= 0 ? 0 : -1;
-}
-
-/* Get the size. */
-static int64_t
-readahead_get_size (nbdkit_next *next, void *handle)
-{
- int64_t r;
-
- r = next->get_size (next);
+ /* Call next->can_cache to read the underlying 'can_cache'. */
+ r = next->can_cache (next);
if (r == -1)
return -1;
+ h->can_cache = r;
- ACQUIRE_LOCK_FOR_CURRENT_SCOPE (&lock);
- size = r;
+ if (!filter_working (h)) {
+ nbdkit_error ("readahead: warning: underlying plugin does not support "
+ "NBD_CMD_CACHE or PARALLEL thread model, so the filter "
+ "won't do anything");
+ if (suggest_cache_filter (h))
+ nbdkit_error ("readahead: try adding --filter=cache "
+ "after this filter");
+ /* This is an error, but that's just to ensure that the warning
+ * above is seen. We don't need to return -1 here.
+ */
+ }
return r;
}
-/* Cache */
-static int
-readahead_can_cache (nbdkit_next *next, void *handle)
-{
- /* We are already operating as a cache regardless of the plugin's
- * underlying .can_cache, but it's easiest to just rely on nbdkit's
- * behavior of calling .pread for caching.
- */
- return NBDKIT_CACHE_EMULATE;
-}
-
/* Read data. */
-
-static int
-fill_readahead (nbdkit_next *next,
- uint32_t count, uint64_t offset, uint32_t flags, int *err)
-{
- position = offset;
-
- /* Read at least window bytes, but if count is larger read that.
- * Note that the count cannot be bigger than the buffer size.
- */
- length = MAX (count, window);
-
- /* Don't go beyond the end of the underlying file. */
- length = MIN (length, size - position);
-
- /* Grow the buffer if necessary. */
- if (bufsize < length) {
- char *new_buffer = realloc (buffer, length);
- if (new_buffer == NULL) {
- *err = errno;
- nbdkit_error ("realloc: %m");
- return -1;
- }
- buffer = new_buffer;
- bufsize = length;
- }
-
- if (next->pread (next, buffer, length, offset, flags, err) == -1) {
- length = 0; /* failed to fill the prefetch buffer */
- return -1;
- }
-
- return 0;
-}
-
static int
readahead_pread (nbdkit_next *next,
void *handle, void *buf, uint32_t count, uint64_t offset,
uint32_t flags, int *err)
{
- ACQUIRE_LOCK_FOR_CURRENT_SCOPE (&lock);
+ struct readahead_handle *h = handle;
- while (count > 0) {
- if (length == 0) {
- /* We don't have a prefetch buffer at all. This could be the
- * first request or reset after a miss.
- */
- window = READAHEAD_MIN;
- if (fill_readahead (next, count, offset, flags, err) == -1)
- return -1;
- }
+ /* If the underlying plugin doesn't support caching then skip that
+ * step completely. The filter will do nothing.
+ */
+ if (filter_working (h)) {
+ struct command ra_cmd = { .type = CMD_CACHE, .next = NULL };
+ int64_t size;
- /* Can we satisfy this request partly or entirely from the prefetch
- * buffer?
- */
- else if (position <= offset && offset < position + length) {
- uint32_t n = MIN (position - offset + length, count);
- memcpy (buf, &buffer[offset-position], n);
- buf += n;
- offset += n;
- count -= n;
- }
+ size = next->get_size (next);
+ if (size >= 0) {
+ ACQUIRE_LOCK_FOR_CURRENT_SCOPE (&window_lock);
- /* Does the request start immediately after the prefetch buffer?
- * This is a “hit” allowing us to double the window size.
- */
- else if (offset == position + length) {
- window = MIN (window * 2, READAHEAD_MAX);
- if (fill_readahead (next, count, offset, flags, err) == -1)
- return -1;
+ /* Generate the asynchronous (background) cache command for
+ * the readahead window.
+ */
+ ra_cmd.offset = offset + count;
+ if (ra_cmd.offset < size) {
+ ra_cmd.count = MIN (window, size - ra_cmd.offset);
+ ra_cmd.next = next; /* If .next is non-NULL, we'll send it below. */
+ }
+
+ /* Should we change the window size?
+ * If the last readahead < current offset, double the window.
+ * If not, but we're still making forward progress, keep the window.
+ * If we're not making forward progress, reduce the window to minimum.
+ */
+ if (last_readahead < offset)
+ window = MIN (window * 2, READAHEAD_MAX);
+ else if (last_offset < offset)
+ /* leave window unchanged */ ;
+ else
+ window = READAHEAD_MIN;
+ last_offset = offset;
+ last_readahead = ra_cmd.offset + ra_cmd.count;
}
- /* Else it's a “miss”. Reset everything and start again. */
- else
- length = 0;
+ if (ra_cmd.next &&
+ send_command_to_background_thread (&h->ctrl, ra_cmd) == -1)
+ return -1;
}
- return 0;
-}
-
-/* Any writes or write-like operations kill the prefetch buffer.
- *
- * We could do better here, but for the current use case of this
- * filter it doesn't matter. XXX
- */
-
-static void
-kill_readahead (void)
-{
- ACQUIRE_LOCK_FOR_CURRENT_SCOPE (&lock);
- window = READAHEAD_MIN;
- length = 0;
-}
-
-static int
-readahead_pwrite (nbdkit_next *next,
- void *handle,
- const void *buf, uint32_t count, uint64_t offset,
- uint32_t flags, int *err)
-{
- kill_readahead ();
- return next->pwrite (next, buf, count, offset, flags, err);
-}
-
-static int
-readahead_trim (nbdkit_next *next,
- void *handle,
- uint32_t count, uint64_t offset, uint32_t flags,
- int *err)
-{
- kill_readahead ();
- return next->trim (next, count, offset, flags, err);
-}
-
-static int
-readahead_zero (nbdkit_next *next,
- void *handle,
- uint32_t count, uint64_t offset, uint32_t flags,
- int *err)
-{
- kill_readahead ();
- return next->zero (next, count, offset, flags, err);
-}
-
-static int
-readahead_flush (nbdkit_next *next,
- void *handle, uint32_t flags, int *err)
-{
- kill_readahead ();
- return next->flush (next, flags, err);
+ /* Issue the synchronous read. */
+ return next->pread (next, buf, count, offset, flags, err);
}
static struct nbdkit_filter filter = {
.name = "readahead",
.longname = "nbdkit readahead filter",
- .unload = readahead_unload,
- .prepare = readahead_prepare,
- .get_size = readahead_get_size,
+ .get_ready = readahead_get_ready,
+ .open = readahead_open,
+ .close = readahead_close,
.can_cache = readahead_can_cache,
.pread = readahead_pread,
- .pwrite = readahead_pwrite,
- .trim = readahead_trim,
- .zero = readahead_zero,
- .flush = readahead_flush,
};
NBDKIT_REGISTER_FILTER(filter)
diff --git a/filters/readahead/readahead.h b/filters/readahead/readahead.h
new file mode 100644
index 00000000..a68204d5
--- /dev/null
+++ b/filters/readahead/readahead.h
@@ -0,0 +1,60 @@
+/* nbdkit
+ * Copyright (C) 2019-2022 Red Hat Inc.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * * Neither the name of Red Hat nor the names of its contributors may be
+ * used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY RED HAT AND CONTRIBUTORS ''AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+ * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL RED HAT OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
+ * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+ * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#ifndef NBDKIT_READAHEAD_H
+#define NBDKIT_READAHEAD_H
+
+#include <pthread.h>
+
+#include <nbdkit-filter.h>
+
+#include "vector.h"
+
+/* List of commands issued to the background thread. */
+struct command {
+ enum { CMD_QUIT, CMD_CACHE } type;
+ nbdkit_next *next;
+ uint64_t offset;
+ uint32_t count;
+};
+DEFINE_VECTOR_TYPE(command_queue, struct command);
+
+struct bgthread_ctrl {
+ command_queue cmds; /* Command queue. */
+ pthread_mutex_t lock; /* Lock for queue. */
+ pthread_cond_t cond; /* Condition queue size 0 -> 1. */
+};
+
+/* Start background thread (one per connection). */
+extern void *readahead_thread (void *vp);
+
+#endif /* NBDKIT_READAHEAD_H */
diff --git a/tests/test-readahead.sh b/tests/test-readahead.sh
index 7ec7f8e9..17126e5a 100755
--- a/tests/test-readahead.sh
+++ b/tests/test-readahead.sh
@@ -59,7 +59,7 @@ for i in range(0, 512*10, 512):
echo $((end_t - start_t))
}
-t1=$(test --filter=readahead)
+t1=$(test --filter=readahead --filter=cache)
t2=$(test)
# In the t1 case we should make only 1 request into the plugin,
--
2.31.1

140
SOURCES/0002-vddk-Extend-D-vddk.stats-1-to-show-number-of-calls-a.patch
View File

@ -1,140 +0,0 @@
From edfdfff0dae54a41bbfca30fa60f4fa6438d45b9 Mon Sep 17 00:00:00 2001
From: "Richard W.M. Jones" <rjones@redhat.com>
Date: Thu, 21 Oct 2021 15:10:00 +0100
Subject: [PATCH] vddk: Extend -D vddk.stats=1 to show number of calls and
bytes transferred
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
The new output looks like this:
nbdkit: debug: VDDK function stats (-D vddk.stats=1):
nbdkit: debug: VixDiskLib_... µs calls bytes
nbdkit: debug: Exit 1000854 1
nbdkit: debug: InitEx 79304 1
nbdkit: debug: Flush 13577 1
nbdkit: debug: Write 12534 21 10485760
nbdkit: debug: Open 4753 3
nbdkit: debug: Read 966 20 5242880
nbdkit: debug: Close 574 3
nbdkit: debug: QueryAllocatedBlocks 116 4
nbdkit: debug: ConnectEx 103 3
nbdkit: debug: Disconnect 88 3
nbdkit: debug: GetTransportMode 68 3
nbdkit: debug: GetInfo 46 3
nbdkit: debug: FreeConnectParams 36 3
nbdkit: debug: FreeInfo 36 3
nbdkit: debug: FreeBlockList 22 4
nbdkit: debug: AllocateConnectParams 22 3
(cherry picked from commit 5c80f0d290db45a679d55baf37ff39bacb8ce7ec)
---
plugins/vddk/nbdkit-vddk-plugin.pod | 3 +--
plugins/vddk/vddk.c | 41 +++++++++++++++++++++++++----
2 files changed, 37 insertions(+), 7 deletions(-)
diff --git a/plugins/vddk/nbdkit-vddk-plugin.pod b/plugins/vddk/nbdkit-vddk-plugin.pod
index 078badcc..e53d3286 100644
--- a/plugins/vddk/nbdkit-vddk-plugin.pod
+++ b/plugins/vddk/nbdkit-vddk-plugin.pod
@@ -517,8 +517,7 @@ Suppress debugging of datapath calls (C<Read> and C<Write>).
=item B<-D vddk.stats=1>
-When the plugin exits print some statistics about the amount of time
-spent waiting on each VDDK call.
+When the plugin exits print some statistics about each VDDK call.
=back
diff --git a/plugins/vddk/vddk.c b/plugins/vddk/vddk.c
index 3d751544..5f1d223b 100644
--- a/plugins/vddk/vddk.c
+++ b/plugins/vddk/vddk.c
@@ -106,6 +106,8 @@ static bool is_remote;
struct vddk_stat {
const char *name; /* function name */
int64_t usecs; /* total number of usecs consumed */
+ uint64_t calls; /* number of times called */
+ uint64_t bytes; /* bytes transferred, datapath calls only */
};
static pthread_mutex_t stats_lock = PTHREAD_MUTEX_INITIALIZER;
static void display_stats (void);
@@ -141,6 +143,17 @@ static void display_stats (void);
gettimeofday (&end_t, NULL); \
ACQUIRE_LOCK_FOR_CURRENT_SCOPE (&stats_lock); \
stats_##fn.usecs += tvdiff_usec (&start_t, &end_t); \
+ stats_##fn.calls++; \
+ } \
+ } while (0)
+#define VDDK_CALL_END_DATAPATH(fn, bytes_) \
+ while (0); \
+ if (vddk_debug_stats) { \
+ gettimeofday (&end_t, NULL); \
+ ACQUIRE_LOCK_FOR_CURRENT_SCOPE (&stats_lock); \
+ stats_##fn.usecs += tvdiff_usec (&start_t, &end_t); \
+ stats_##fn.calls++; \
+ stats_##fn.bytes += bytes_; \
} \
} while (0)
@@ -191,6 +204,12 @@ stat_compare (const void *vp1, const void *vp2)
else return 0;
}
+static const char *
+api_name_without_prefix (const char *name)
+{
+ return strncmp (name, "VixDiskLib_", 11) == 0 ? name + 11 : name;
+}
+
static void
display_stats (void)
{
@@ -206,10 +225,22 @@ display_stats (void)
qsort (stats.ptr, stats.size, sizeof stats.ptr[0], stat_compare);
nbdkit_debug ("VDDK function stats (-D vddk.stats=1):");
- nbdkit_debug ("%-40s %9s", "", "µs");
+ nbdkit_debug ("%-24s %15s %5s %15s",
+ "VixDiskLib_...", "µs", "calls", "bytes");
for (i = 0; i < stats.size; ++i) {
- if (stats.ptr[i].usecs)
- nbdkit_debug ("%-40s %9" PRIi64, stats.ptr[i].name, stats.ptr[i].usecs);
+ if (stats.ptr[i].usecs) {
+ if (stats.ptr[i].bytes > 0)
+ nbdkit_debug (" %-22s %15" PRIi64 " %5" PRIu64 " %15" PRIu64,
+ api_name_without_prefix (stats.ptr[i].name),
+ stats.ptr[i].usecs,
+ stats.ptr[i].calls,
+ stats.ptr[i].bytes);
+ else
+ nbdkit_debug (" %-22s %15" PRIi64 " %5" PRIu64,
+ api_name_without_prefix (stats.ptr[i].name),
+ stats.ptr[i].usecs,
+ stats.ptr[i].calls);
+ }
}
statlist_reset (&stats);
}
@@ -831,7 +862,7 @@ vddk_pread (void *handle, void *buf, uint32_t count, uint64_t offset,
"%" PRIu32 " sectors, buffer",
offset, count) {
err = VixDiskLib_Read (h->handle, offset, count, buf);
- } VDDK_CALL_END (VixDiskLib_Read);
+ } VDDK_CALL_END_DATAPATH (VixDiskLib_Read, count * VIXDISKLIB_SECTOR_SIZE);
if (err != VIX_OK) {
VDDK_ERROR (err, "VixDiskLib_Read");
return -1;
@@ -871,7 +902,7 @@ vddk_pwrite (void *handle, const void *buf, uint32_t count, uint64_t offset,
"%" PRIu32 " sectors, buffer",
offset, count) {
err = VixDiskLib_Write (h->handle, offset, count, buf);
- } VDDK_CALL_END (VixDiskLib_Write);
+ } VDDK_CALL_END_DATAPATH (VixDiskLib_Write, count * VIXDISKLIB_SECTOR_SIZE);
if (err != VIX_OK) {
VDDK_ERROR (err, "VixDiskLib_Write");
return -1;
--
2.31.1

120
SOURCES/0003-readahead-Fix-test.patch Normal file
View File

@ -0,0 +1,120 @@
From b41b7d7ddf6d3fba23ac7978c8b272f2ff84265d Mon Sep 17 00:00:00 2001
From: "Richard W.M. Jones" <rjones@redhat.com>
Date: Thu, 21 Apr 2022 16:14:46 +0100
Subject: [PATCH] readahead: Fix test
The previous test turned out to be pretty bad at testing the new
filter. A specific problem is that the filter starts a background
thread which issues .cache requests, while on the main connection
.pread requests are being passed through. The test used
--filter=readahead --filter=cache with the cache filter only caching
on .cache requests (since cache-on-read defaults to false), so only
caching requests made by the background thread.
main thread
client ---- .pread ----- delay-filter -------> plugin
\
\ background thread
.cache --- cache-filter
Under very high load, the background thread could be starved. This
means no requests were being cached at all, and all requests were
passing through the delay filter. It would appear that readahead was
failing (which it was, in a way).
It's not very easy to fix this since readahead is best-effort, but we
can go back to using a simpler plugin that logs reads and caches and
check that they look valid.
Update: commit 2ff548d66ad3eae87868402ec5b3319edd12090f
(cherry picked from commit db1e3311727c6ecab3264a1811d33db1aa45a4d0)
---
tests/test-readahead.sh | 61 +++++++++++++++++++++++------------------
1 file changed, 35 insertions(+), 26 deletions(-)
diff --git a/tests/test-readahead.sh b/tests/test-readahead.sh
index 17126e5a..37f4a06f 100755
--- a/tests/test-readahead.sh
+++ b/tests/test-readahead.sh
@@ -30,43 +30,52 @@
# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
# SUCH DAMAGE.
-# Is the readahead filter faster? Copy a blank disk with a custom
-# plugin that sleeps on every request. Because the readahead filter
-# should result in fewer requests it should run faster.
-
source ./functions.sh
set -e
set -x
-requires_filter delay
+requires_plugin sh
requires nbdsh --version
requires dd iflag=count_bytes </dev/null
-files="readahead.img"
+files="readahead.out"
rm -f $files
cleanup_fn rm -f $files
-test ()
-{
- start_t=$SECONDS
- nbdkit -fv -U - "$@" null size=1M --filter=delay rdelay=5 \
- --run 'nbdsh --uri "$uri" -c "
+nbdkit -fv -U - "$@" sh - \
+ --filter=readahead \
+ --run 'nbdsh --uri "$uri" -c "
for i in range(0, 512*10, 512):
h.pread(512, i)
-"'
+"' <<'EOF'
+case "$1" in
+ thread_model)
+ echo parallel
+ ;;
+ can_cache)
+ echo native
+ ;;
+ get_size)
+ echo 1M
+ ;;
+ cache)
+ echo "$@" >> readahead.out
+ ;;
+ pread)
+ echo "$@" >> readahead.out
+ dd if=/dev/zero count=$3 iflag=count_bytes
+ ;;
+ *)
+ exit 2
+ ;;
+esac
+EOF
- end_t=$SECONDS
- echo $((end_t - start_t))
-}
+cat readahead.out
-t1=$(test --filter=readahead --filter=cache)
-t2=$(test)
-
-# In the t1 case we should make only 1 request into the plugin,
-# resulting in around 1 sleep period (5 seconds). In the t2 case we
-# make 10 requests so sleep for around 50 seconds. t1 should be < t2
-# is every reasonable scenario.
-if [ $t1 -ge $t2 ]; then
- echo "$0: readahead filter took longer, should be shorter"
- exit 1
-fi
+# We should see the pread requests, and additional cache requests for
+# the 32K region following each pread request.
+for i in `seq 0 512 $((512*10 - 512))` ; do
+ grep "pread 512 $i" readahead.out
+ grep "cache 32768 $((i+512))" readahead.out
+done
--
2.31.1

320
SOURCES/0003-vddk-Simplify-and-consolidate-VDDK_CALL_START-END-ma.patch
View File

@ -1,320 +0,0 @@
From cbcf2a2f158a9889bd597b31159ab357dea05cd6 Mon Sep 17 00:00:00 2001
From: "Richard W.M. Jones" <rjones@redhat.com>
Date: Thu, 21 Oct 2021 22:55:17 +0100
Subject: [PATCH] vddk: Simplify and consolidate VDDK_CALL_START/END macros
We don't need the VDDK_CALL_*_DATAPATH versions of these macros
because the compiler is able to optimize some static strcmps.
Furthermore we can remove extra { .. } when the macros are applied.
(cherry picked from commit 3ea0ed6582faa8f800b7a2a15d58032917a21bd5)
---
plugins/vddk/vddk.c | 124 ++++++++++++++++++++------------------------
1 file changed, 56 insertions(+), 68 deletions(-)
diff --git a/plugins/vddk/vddk.c b/plugins/vddk/vddk.c
index 5f1d223b..993f2d76 100644
--- a/plugins/vddk/vddk.c
+++ b/plugins/vddk/vddk.c
@@ -125,28 +125,16 @@ static void display_stats (void);
#define VDDK_CALL_START(fn, fs, ...) \
do { \
struct timeval start_t, end_t; \
+ /* GCC can optimize this away at compile time: */ \
+ const bool datapath = \
+ strcmp (#fn, "VixDiskLib_Read") == 0 || \
+ strcmp (#fn, "VixDiskLib_Write") == 0; \
if (vddk_debug_stats) \
gettimeofday (&start_t, NULL); \
- nbdkit_debug ("VDDK call: %s (" fs ")", #fn, ##__VA_ARGS__); \
- do
-#define VDDK_CALL_START_DATAPATH(fn, fs, ...) \
- do { \
- struct timeval start_t, end_t; \
- if (vddk_debug_stats) \
- gettimeofday (&start_t, NULL); \
- if (vddk_debug_datapath) \
+ if (!datapath || vddk_debug_datapath) \
nbdkit_debug ("VDDK call: %s (" fs ")", #fn, ##__VA_ARGS__); \
do
-#define VDDK_CALL_END(fn) \
- while (0); \
- if (vddk_debug_stats) { \
- gettimeofday (&end_t, NULL); \
- ACQUIRE_LOCK_FOR_CURRENT_SCOPE (&stats_lock); \
- stats_##fn.usecs += tvdiff_usec (&start_t, &end_t); \
- stats_##fn.calls++; \
- } \
- } while (0)
-#define VDDK_CALL_END_DATAPATH(fn, bytes_) \
+#define VDDK_CALL_END(fn, bytes_) \
while (0); \
if (vddk_debug_stats) { \
gettimeofday (&end_t, NULL); \
@@ -161,13 +149,13 @@ static void display_stats (void);
#define VDDK_ERROR(err, fs, ...) \
do { \
char *vddk_err_msg; \
- VDDK_CALL_START (VixDiskLib_GetErrorText, "%lu", err) { \
+ VDDK_CALL_START (VixDiskLib_GetErrorText, "%lu", err) \
vddk_err_msg = VixDiskLib_GetErrorText ((err), NULL); \
- } VDDK_CALL_END (VixDiskLib_GetErrorText); \
+ VDDK_CALL_END (VixDiskLib_GetErrorText, 0); \
nbdkit_error (fs ": %s", ##__VA_ARGS__, vddk_err_msg); \
- VDDK_CALL_START (VixDiskLib_FreeErrorText, "") { \
+ VDDK_CALL_START (VixDiskLib_FreeErrorText, "") \
VixDiskLib_FreeErrorText (vddk_err_msg); \
- } VDDK_CALL_END (VixDiskLib_FreeErrorText); \
+ VDDK_CALL_END (VixDiskLib_FreeErrorText, 0); \
} while (0)
/* Unload the plugin. */
@@ -175,9 +163,9 @@ static void
vddk_unload (void)
{
if (init_called) {
- VDDK_CALL_START (VixDiskLib_Exit, "") {
+ VDDK_CALL_START (VixDiskLib_Exit, "")
VixDiskLib_Exit ();
- } VDDK_CALL_END (VixDiskLib_Exit);
+ VDDK_CALL_END (VixDiskLib_Exit, 0);
}
if (dl)
dlclose (dl);
@@ -572,13 +560,13 @@ vddk_after_fork (void)
VDDK_CALL_START (VixDiskLib_InitEx,
"%d, %d, &debug_fn, &error_fn, &error_fn, %s, %s",
VDDK_MAJOR, VDDK_MINOR,
- libdir, config ? : "NULL") {
+ libdir, config ? : "NULL")
err = VixDiskLib_InitEx (VDDK_MAJOR, VDDK_MINOR,
&debug_function, /* log function */
&error_function, /* warn function */
&error_function, /* panic function */
libdir, config);
- } VDDK_CALL_END (VixDiskLib_InitEx);
+ VDDK_CALL_END (VixDiskLib_InitEx, 0);
if (err != VIX_OK) {
VDDK_ERROR (err, "VixDiskLib_InitEx");
exit (EXIT_FAILURE);
@@ -640,9 +628,9 @@ allocate_connect_params (void)
VixDiskLibConnectParams *ret;
if (VixDiskLib_AllocateConnectParams != NULL) {
- VDDK_CALL_START (VixDiskLib_AllocateConnectParams, "") {
+ VDDK_CALL_START (VixDiskLib_AllocateConnectParams, "")
ret = VixDiskLib_AllocateConnectParams ();
- } VDDK_CALL_END (VixDiskLib_AllocateConnectParams);
+ VDDK_CALL_END (VixDiskLib_AllocateConnectParams, 0);
}
else
ret = calloc (1, sizeof (VixDiskLibConnectParams));
@@ -657,9 +645,9 @@ free_connect_params (VixDiskLibConnectParams *params)
* originally called. Otherwise use free.
*/
if (VixDiskLib_AllocateConnectParams != NULL) {
- VDDK_CALL_START (VixDiskLib_FreeConnectParams, "params") {
+ VDDK_CALL_START (VixDiskLib_FreeConnectParams, "params")
VixDiskLib_FreeConnectParams (params);
- } VDDK_CALL_END (VixDiskLib_FreeConnectParams);
+ VDDK_CALL_END (VixDiskLib_FreeConnectParams, 0);
}
else
free (params);
@@ -716,13 +704,13 @@ vddk_open (int readonly)
"h->params, %d, %s, %s, &connection",
readonly,
snapshot_moref ? : "NULL",
- transport_modes ? : "NULL") {
+ transport_modes ? : "NULL")
err = VixDiskLib_ConnectEx (h->params,
readonly,
snapshot_moref,
transport_modes,
&h->connection);
- } VDDK_CALL_END (VixDiskLib_ConnectEx);
+ VDDK_CALL_END (VixDiskLib_ConnectEx, 0);
if (err != VIX_OK) {
VDDK_ERROR (err, "VixDiskLib_ConnectEx");
goto err1;
@@ -743,25 +731,25 @@ vddk_open (int readonly)
}
VDDK_CALL_START (VixDiskLib_Open,
- "connection, %s, %d, &handle", filename, flags) {
+ "connection, %s, %d, &handle", filename, flags)
err = VixDiskLib_Open (h->connection, filename, flags, &h->handle);
- } VDDK_CALL_END (VixDiskLib_Open);
+ VDDK_CALL_END (VixDiskLib_Open, 0);
if (err != VIX_OK) {
VDDK_ERROR (err, "VixDiskLib_Open: %s", filename);
goto err2;
}
- VDDK_CALL_START (VixDiskLib_GetTransportMode, "handle") {
+ VDDK_CALL_START (VixDiskLib_GetTransportMode, "handle")
transport_mode = VixDiskLib_GetTransportMode (h->handle);
- } VDDK_CALL_END (VixDiskLib_GetTransportMode);
+ VDDK_CALL_END (VixDiskLib_GetTransportMode, 0);
nbdkit_debug ("transport mode: %s", transport_mode);
return h;
err2:
- VDDK_CALL_START (VixDiskLib_Disconnect, "connection") {
+ VDDK_CALL_START (VixDiskLib_Disconnect, "connection")
VixDiskLib_Disconnect (h->connection);
- } VDDK_CALL_END (VixDiskLib_Disconnect);
+ VDDK_CALL_END (VixDiskLib_Disconnect, 0);
err1:
free_connect_params (h->params);
err0:
@@ -776,12 +764,12 @@ vddk_close (void *handle)
ACQUIRE_LOCK_FOR_CURRENT_SCOPE (&open_close_lock);
struct vddk_handle *h = handle;
- VDDK_CALL_START (VixDiskLib_Close, "handle") {
+ VDDK_CALL_START (VixDiskLib_Close, "handle")
VixDiskLib_Close (h->handle);
- } VDDK_CALL_END (VixDiskLib_Close);
- VDDK_CALL_START (VixDiskLib_Disconnect, "connection") {
+ VDDK_CALL_END (VixDiskLib_Close, 0);
+ VDDK_CALL_START (VixDiskLib_Disconnect, "connection")
VixDiskLib_Disconnect (h->connection);
- } VDDK_CALL_END (VixDiskLib_Disconnect);
+ VDDK_CALL_END (VixDiskLib_Disconnect, 0);
free_connect_params (h->params);
free (h);
@@ -796,9 +784,9 @@ vddk_get_size (void *handle)
VixError err;
uint64_t size;
- VDDK_CALL_START (VixDiskLib_GetInfo, "handle, &info") {
+ VDDK_CALL_START (VixDiskLib_GetInfo, "handle, &info")
err = VixDiskLib_GetInfo (h->handle, &info);
- } VDDK_CALL_END (VixDiskLib_GetInfo);
+ VDDK_CALL_END (VixDiskLib_GetInfo, 0);
if (err != VIX_OK) {
VDDK_ERROR (err, "VixDiskLib_GetInfo");
return -1;
@@ -827,9 +815,9 @@ vddk_get_size (void *handle)
info->uuid ? : "NULL");
}
- VDDK_CALL_START (VixDiskLib_FreeInfo, "info") {
+ VDDK_CALL_START (VixDiskLib_FreeInfo, "info")
VixDiskLib_FreeInfo (info);
- } VDDK_CALL_END (VixDiskLib_FreeInfo);
+ VDDK_CALL_END (VixDiskLib_FreeInfo, 0);
return (int64_t) size;
}
@@ -857,12 +845,12 @@ vddk_pread (void *handle, void *buf, uint32_t count, uint64_t offset,
offset /= VIXDISKLIB_SECTOR_SIZE;
count /= VIXDISKLIB_SECTOR_SIZE;
- VDDK_CALL_START_DATAPATH (VixDiskLib_Read,
- "handle, %" PRIu64 " sectors, "
- "%" PRIu32 " sectors, buffer",
- offset, count) {
+ VDDK_CALL_START (VixDiskLib_Read,
+ "handle, %" PRIu64 " sectors, "
+ "%" PRIu32 " sectors, buffer",
+ offset, count)
err = VixDiskLib_Read (h->handle, offset, count, buf);
- } VDDK_CALL_END_DATAPATH (VixDiskLib_Read, count * VIXDISKLIB_SECTOR_SIZE);
+ VDDK_CALL_END (VixDiskLib_Read, count * VIXDISKLIB_SECTOR_SIZE);
if (err != VIX_OK) {
VDDK_ERROR (err, "VixDiskLib_Read");
return -1;
@@ -897,12 +885,12 @@ vddk_pwrite (void *handle, const void *buf, uint32_t count, uint64_t offset,
offset /= VIXDISKLIB_SECTOR_SIZE;
count /= VIXDISKLIB_SECTOR_SIZE;
- VDDK_CALL_START_DATAPATH (VixDiskLib_Write,
- "handle, %" PRIu64 " sectors, "
- "%" PRIu32 " sectors, buffer",
- offset, count) {
+ VDDK_CALL_START (VixDiskLib_Write,
+ "handle, %" PRIu64 " sectors, "
+ "%" PRIu32 " sectors, buffer",
+ offset, count)
err = VixDiskLib_Write (h->handle, offset, count, buf);
- } VDDK_CALL_END_DATAPATH (VixDiskLib_Write, count * VIXDISKLIB_SECTOR_SIZE);
+ VDDK_CALL_END (VixDiskLib_Write, count * VIXDISKLIB_SECTOR_SIZE);
if (err != VIX_OK) {
VDDK_ERROR (err, "VixDiskLib_Write");
return -1;
@@ -945,9 +933,9 @@ vddk_flush (void *handle, uint32_t flags)
* file so it appears to be the correct call to use here.
*/
- VDDK_CALL_START (VixDiskLib_Flush, "handle") {
+ VDDK_CALL_START (VixDiskLib_Flush, "handle")
err = VixDiskLib_Flush (h->handle);
- } VDDK_CALL_END (VixDiskLib_Flush);
+ VDDK_CALL_END (VixDiskLib_Flush, 0);
if (err != VIX_OK) {
VDDK_ERROR (err, "VixDiskLib_Flush");
return -1;
@@ -983,17 +971,17 @@ vddk_can_extents (void *handle)
*/
VDDK_CALL_START (VixDiskLib_QueryAllocatedBlocks,
"handle, 0, %d sectors, %d sectors",
- VIXDISKLIB_MIN_CHUNK_SIZE, VIXDISKLIB_MIN_CHUNK_SIZE) {
+ VIXDISKLIB_MIN_CHUNK_SIZE, VIXDISKLIB_MIN_CHUNK_SIZE)
err = VixDiskLib_QueryAllocatedBlocks (h->handle,
0, VIXDISKLIB_MIN_CHUNK_SIZE,
VIXDISKLIB_MIN_CHUNK_SIZE,
&block_list);
- } VDDK_CALL_END (VixDiskLib_QueryAllocatedBlocks);
+ VDDK_CALL_END (VixDiskLib_QueryAllocatedBlocks, 0);
error_suppression = 0;
if (err == VIX_OK) {
- VDDK_CALL_START (VixDiskLib_FreeBlockList, "block_list") {
+ VDDK_CALL_START (VixDiskLib_FreeBlockList, "block_list")
VixDiskLib_FreeBlockList (block_list);
- } VDDK_CALL_END (VixDiskLib_FreeBlockList);
+ VDDK_CALL_END (VixDiskLib_FreeBlockList, 0);
}
if (err != VIX_OK) {
char *errmsg = VixDiskLib_GetErrorText (err, NULL);
@@ -1073,12 +1061,12 @@ vddk_extents (void *handle, uint32_t count, uint64_t offset, uint32_t flags,
VDDK_CALL_START (VixDiskLib_QueryAllocatedBlocks,
"handle, %" PRIu64 " sectors, %" PRIu64 " sectors, "
"%d sectors",
- start_sector, nr_sectors, VIXDISKLIB_MIN_CHUNK_SIZE) {
+ start_sector, nr_sectors, VIXDISKLIB_MIN_CHUNK_SIZE)
err = VixDiskLib_QueryAllocatedBlocks (h->handle,
start_sector, nr_sectors,
VIXDISKLIB_MIN_CHUNK_SIZE,
&block_list);
- } VDDK_CALL_END (VixDiskLib_QueryAllocatedBlocks);
+ VDDK_CALL_END (VixDiskLib_QueryAllocatedBlocks, 0);
if (err != VIX_OK) {
VDDK_ERROR (err, "VixDiskLib_QueryAllocatedBlocks");
return -1;
@@ -1097,15 +1085,15 @@ vddk_extents (void *handle, uint32_t count, uint64_t offset, uint32_t flags,
add_extent (extents, &position, blk_offset, true) == -1) ||
(add_extent (extents,
&position, blk_offset + blk_length, false) == -1)) {
- VDDK_CALL_START (VixDiskLib_FreeBlockList, "block_list") {
+ VDDK_CALL_START (VixDiskLib_FreeBlockList, "block_list")
VixDiskLib_FreeBlockList (block_list);
- } VDDK_CALL_END (VixDiskLib_FreeBlockList);
+ VDDK_CALL_END (VixDiskLib_FreeBlockList, 0);
return -1;
}
}
- VDDK_CALL_START (VixDiskLib_FreeBlockList, "block_list") {
+ VDDK_CALL_START (VixDiskLib_FreeBlockList, "block_list")
VixDiskLib_FreeBlockList (block_list);
- } VDDK_CALL_END (VixDiskLib_FreeBlockList);
+ VDDK_CALL_END (VixDiskLib_FreeBlockList, 0);
/* There's an implicit hole after the returned list of blocks, up
* to the end of the QueryAllocatedBlocks request.
--
2.31.1

1816
SOURCES/0004-New-filter-luks.patch Normal file
View File

File diff suppressed because it is too large Load Diff

84
SOURCES/0004-vddk-Document-troubleshooting-performance-problems.patch
View File

@ -1,84 +0,0 @@
From 8353ab55b8c6e7f1dc9ea27260fd7ec90b9d75af Mon Sep 17 00:00:00 2001
From: "Richard W.M. Jones" <rjones@redhat.com>
Date: Fri, 22 Oct 2021 18:00:27 +0100
Subject: [PATCH] vddk: Document troubleshooting performance problems
Document how to use -D vddk.stats=1 to diagnose performance problems
with VDDK.
(cherry picked from commit e491978c193f49010cc28ad344d0fb3c1b5ede35)
---
plugins/vddk/nbdkit-vddk-plugin.pod | 57 +++++++++++++++++++++++++++++
1 file changed, 57 insertions(+)
diff --git a/plugins/vddk/nbdkit-vddk-plugin.pod b/plugins/vddk/nbdkit-vddk-plugin.pod
index e53d3286..5a426135 100644
--- a/plugins/vddk/nbdkit-vddk-plugin.pod
+++ b/plugins/vddk/nbdkit-vddk-plugin.pod
@@ -475,6 +475,63 @@ and restarting the C<hostd> service:
For more information see L<https://bugzilla.redhat.com/1614276>.
+=head2 Troubleshooting performance problems
+
+VDDK has very uneven performance with some operations being very slow.
+This plugin has options to allow you to debug performance issues. If
+your application has a debug or diagnostic setting, add the following
+nbdkit command line options:
+
+ -v -D nbdkit.backend.datapath=0 -D vddk.datapath=0 -D vddk.stats=1
+
+C<-v> enables verbose messages and the two datapath options I<disable>
+the very verbose per-read/-write messages. C<-D vddk.stats=1> enables
+a summary when nbdkit exits of the cumulative time taken in each VDDK
+function, the number of times each function was called, and (for read
+and write) the number of bytes transferred. An example of what those
+stats look like can be found here:
+L<https://gitlab.com/nbdkit/nbdkit/-/commit/5c80f0d290db45a679d55baf37ff39bacb8ce7ec>
+
+You can interpret the stats as follows:
+
+=over 4
+
+=item C<Read>
+
+The cumulative time spent waiting for VDDK to return from
+C<VixDiskLib_Read> calls, the number of times this function was
+called, and the total bytes read. You can use this to determine the
+read bandwidth to the VMware server.
+
+=item C<Write>
+
+=item C<Flush>
+
+Same as above, but for writing and flushing writes.
+
+=item C<QueryAllocatedBlocks>
+
+This call is used to query information about the sparseness of the
+remote disk. It is only available in VDDK E<ge> 6.7. The call is
+notably very slow in all versions of VMware we have tested.
+
+=item C<Open>
+
+=item C<Close>
+
+=item C<ConnectEx>
+
+=item C<Disconnect>
+
+=item C<InitEx>
+
+=item C<Exit>
+
+The cumulative time spent connecting and disconnecting from the VMware
+server, which can also be very slow.
+
+=back
+
=head1 SUPPORTED VERSIONS OF VDDK
This plugin requires VDDK E<ge> 5.5.5, which in turn means that it
--
2.31.1

95
SOURCES/0005-luks-Disable-filter-with-old-GnuTLS-in-Debian-10.patch Normal file
View File

@ -0,0 +1,95 @@
From 66daae1a7daf680e06f884e9af6a14830263c932 Mon Sep 17 00:00:00 2001
From: "Richard W.M. Jones" <rjones@redhat.com>
Date: Sun, 8 May 2022 12:13:39 +0100
Subject: [PATCH] luks: Disable filter with old GnuTLS in Debian 10
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
On Debian 10:
luks.c: In function ‘parse_cipher_strings’:
luks.c:574:26: error: ‘GNUTLS_CIPHER_AES_128_XTS’ undeclared (first use in this function); did you mean ‘GNUTLS_CIPHER_AES_128_CCM’?
h->gnutls_cipher = GNUTLS_CIPHER_AES_128_XTS;
^~~~~~~~~~~~~~~~~~~~~~~~~
GNUTLS_CIPHER_AES_128_CCM
luks.c:574:26: note: each undeclared identifier is reported only once for each function it appears in
luks.c:577:26: error: ‘GNUTLS_CIPHER_AES_256_XTS’ undeclared (first use in this function); did you mean ‘GNUTLS_CIPHER_AES_256_CCM’?
h->gnutls_cipher = GNUTLS_CIPHER_AES_256_XTS;
^~~~~~~~~~~~~~~~~~~~~~~~~
GNUTLS_CIPHER_AES_256_CCM
luks.c: In function ‘try_passphrase_in_keyslot’:
luks.c:728:7: error: implicit declaration of function ‘gnutls_pbkdf2’; did you mean ‘gnutls_prf’? [-Werror=implicit-function-declaration]
r = gnutls_pbkdf2 (h->hash_alg, &key, &salt, ks->password_iterations,
^~~~~~~~~~~~~
gnutls_prf
Because gnutls_pbkdf2 is missing there's no chance of making this
filter work on this platform so it's best to compile it out.
Fixes: commit 468919dce6c5eb57503eacac0f67e5dd87c58e6c
(cherry picked from commit f9f67e483f4aad19ad6101163d32562f13504ca7)
---
configure.ac | 5 ++++-
filters/luks/Makefile.am | 2 +-
tests/Makefile.am | 2 +-
3 files changed, 6 insertions(+), 3 deletions(-)
diff --git a/configure.ac b/configure.ac
index de85b4da..1d209f67 100644
--- a/configure.ac
+++ b/configure.ac
@@ -636,12 +636,15 @@ AS_IF([test "x$GNUTLS_LIBS" != "x"],[
gnutls_certificate_set_known_dh_params \
gnutls_group_get \
gnutls_group_get_name \
+ gnutls_pbkdf2 \
gnutls_session_set_verify_cert \
gnutls_srp_server_get_username \
gnutls_transport_is_ktls_enabled \
])
LIBS="$old_LIBS"
])
+AM_CONDITIONAL([HAVE_GNUTLS_PBKDF2],
+ [test "x$GNUTLS_LIBS" != "x" && test "x$ac_cv_func_gnutls_pbkdf2" = xyes])
AC_ARG_ENABLE([linuxdisk],
[AS_HELP_STRING([--disable-linuxdisk],
@@ -1484,7 +1487,7 @@ echo "Optional filters:"
echo
feature "ext2" test "x$HAVE_EXT2_TRUE" = "x"
feature "gzip" test "x$HAVE_ZLIB_TRUE" = "x"
-feature "LUKS" test "x$HAVE_GNUTLS_TRUE" != "x"
+feature "luks" test "x$HAVE_GNUTLS_PBKDF2_TRUE" = "x"
feature "xz" test "x$HAVE_LIBLZMA_TRUE" = "x"
echo
diff --git a/filters/luks/Makefile.am b/filters/luks/Makefile.am
index 30089621..622e5c3d 100644
--- a/filters/luks/Makefile.am
+++ b/filters/luks/Makefile.am
@@ -33,7 +33,7 @@ include $(top_srcdir)/common-rules.mk
EXTRA_DIST = nbdkit-luks-filter.pod
-if HAVE_GNUTLS
+if HAVE_GNUTLS_PBKDF2
filter_LTLIBRARIES = nbdkit-luks-filter.la
diff --git a/tests/Makefile.am b/tests/Makefile.am
index c29453ba..5585b3b7 100644
--- a/tests/Makefile.am
+++ b/tests/Makefile.am
@@ -1597,7 +1597,7 @@ EXTRA_DIST += \
$(NULL)
# luks filter test.
-if HAVE_GNUTLS
+if HAVE_GNUTLS_PBKDF2
TESTS += \
test-luks-info.sh \
test-luks-copy.sh \
--
2.31.1

141
SOURCES/0005-vddk-Include-VDDK-major-library-version-in-dump-plug.patch
View File

@ -1,141 +0,0 @@
From d994773724266dd5f0a8b4282cc604f6b75e077c Mon Sep 17 00:00:00 2001
From: "Richard W.M. Jones" <rjones@redhat.com>
Date: Sat, 23 Oct 2021 16:16:39 +0100
Subject: [PATCH] vddk: Include VDDK major library version in --dump-plugin
output
Although it doesn't seem to be possible to get the precise VDDK
version, With a relatively simple change we can at least return the
VDDK major version. Currently this can be 5, 6 or 7.
(cherry picked from commit 8700649d147948897f3b97810a1dff37924bdd6e)
---
plugins/vddk/nbdkit-vddk-plugin.pod | 4 ++++
plugins/vddk/vddk.c | 29 +++++++++++++++++++----------
tests/test-vddk-real-dump-plugin.sh | 2 ++
3 files changed, 25 insertions(+), 10 deletions(-)
diff --git a/plugins/vddk/nbdkit-vddk-plugin.pod b/plugins/vddk/nbdkit-vddk-plugin.pod
index 5a426135..bc3c3c94 100644
--- a/plugins/vddk/nbdkit-vddk-plugin.pod
+++ b/plugins/vddk/nbdkit-vddk-plugin.pod
@@ -422,6 +422,10 @@ at runtime.
If this is printed then the C<nfchostport=PORT> parameter is supported
by this build.
+=item C<vddk_library_version=...>
+
+The VDDK major library version: 5, 6, 7, ...
+
=item C<vddk_dll=...>
Prints the full path to the VDDK shared library. Since this requires
diff --git a/plugins/vddk/vddk.c b/plugins/vddk/vddk.c
index 993f2d76..d74a484d 100644
--- a/plugins/vddk/vddk.c
+++ b/plugins/vddk/vddk.c
@@ -81,6 +81,7 @@ NBDKIT_DLL_PUBLIC int vddk_debug_stats;
static void *dl; /* dlopen handle */
static bool init_called; /* was InitEx called */
static __thread int error_suppression; /* threadlocal error suppression */
+static int library_version; /* VDDK major: 5, 6, 7, ... */
static enum { NONE = 0, ZLIB, FASTLZ, SKIPZ } compression; /* compression */
static char *config; /* config */
@@ -405,7 +406,10 @@ vddk_config (const char *key, const char *value)
static void
load_library (bool load_error_is_fatal)
{
- static const char *sonames[] = {
+ static struct {
+ const char *soname;
+ int library_version;
+ } libs[] = {
/* Prefer the newest library in case multiple exist. Check two
* possible directories: the usual VDDK installation puts .so
* files in an arch-specific subdirectory of $libdir (our minimum
@@ -413,12 +417,13 @@ load_library (bool load_error_is_fatal)
* but our testsuite is easier to write if we point libdir
* directly to a stub .so.
*/
- "lib64/libvixDiskLib.so.7",
- "libvixDiskLib.so.7",
- "lib64/libvixDiskLib.so.6",
- "libvixDiskLib.so.6",
- "lib64/libvixDiskLib.so.5",
- "libvixDiskLib.so.5",
+ { "lib64/libvixDiskLib.so.7", 7 },
+ { "libvixDiskLib.so.7", 7 },
+ { "lib64/libvixDiskLib.so.6", 6 },
+ { "libvixDiskLib.so.6", 6 },
+ { "lib64/libvixDiskLib.so.5", 5 },
+ { "libvixDiskLib.so.5", 5 },
+ { NULL }
};
size_t i;
CLEANUP_FREE char *orig_error = NULL;
@@ -431,19 +436,20 @@ load_library (bool load_error_is_fatal)
}
}
- for (i = 0; i < sizeof sonames / sizeof sonames[0]; ++i) {
+ for (i = 0; libs[i].soname != NULL; ++i) {
CLEANUP_FREE char *path;
/* Set the full path so that dlopen will preferentially load the
* system libraries from the same directory.
*/
- if (asprintf (&path, "%s/%s", libdir, sonames[i]) == -1) {
+ if (asprintf (&path, "%s/%s", libdir, libs[i].soname) == -1) {
nbdkit_error ("asprintf: %m");
exit (EXIT_FAILURE);
}
dl = dlopen (path, RTLD_NOW);
if (dl != NULL) {
+ library_version = libs[i].library_version;
/* Now that we found the library, ensure that LD_LIBRARY_PATH
* includes its directory for all future loads. This may modify
* path in-place and/or re-exec nbdkit, but that's okay.
@@ -464,10 +470,12 @@ load_library (bool load_error_is_fatal)
"If '%s' is located on a non-standard path you may need to\n"
"set libdir=/path/to/vmware-vix-disklib-distrib.\n\n"
"See nbdkit-vddk-plugin(1) man page section \"LIBRARY LOCATION\" for details.",
- orig_error ? : "(unknown error)", sonames[0]);
+ orig_error ? : "(unknown error)", libs[0].soname);
exit (EXIT_FAILURE);
}
+ assert (library_version >= 5);
+
/* Load symbols. */
#define STUB(fn,ret,args) \
do { \
@@ -583,6 +591,7 @@ vddk_dump_plugin (void)
printf ("vddk_default_libdir=%s\n", VDDK_LIBDIR);
printf ("vddk_has_nfchostport=1\n");
+ printf ("vddk_library_version=%d\n", library_version);
#if defined(HAVE_DLADDR)
/* It would be nice to print the version of VDDK from the shared
diff --git a/tests/test-vddk-real-dump-plugin.sh b/tests/test-vddk-real-dump-plugin.sh
index 2cb7724e..0a079c6c 100755
--- a/tests/test-vddk-real-dump-plugin.sh
+++ b/tests/test-vddk-real-dump-plugin.sh
@@ -58,10 +58,12 @@ rm -f $files
cleanup_fn rm -f $files
nbdkit -f -v vddk libdir="$vddkdir" --dump-plugin > $out
+cat $out
# Check the vddk_* entries are set.
grep ^vddk_default_libdir= $out
grep ^vddk_has_nfchostport= $out
+grep ^vddk_library_version= $out
grep ^vddk_dll= $out
dll="$(grep ^vddk_dll $out | cut -d= -f2)"
--
2.31.1

71
SOURCES/0006-luks-Various-fixes-for-Clang.patch Normal file
View File

@ -0,0 +1,71 @@
From b3c05065801c723966a3e8d93c9b84e808ff38b9 Mon Sep 17 00:00:00 2001
From: "Richard W.M. Jones" <rjones@redhat.com>
Date: Sun, 8 May 2022 12:30:09 +0100
Subject: [PATCH] luks: Various fixes for Clang
With Clang:
luks.c:728:25: error: implicit conversion from enumeration type 'gnutls_digest_algorithm_t' to different enumeration type 'gnutls_mac_algorithm_t' [-Werror,-Wenum-conversion]
r = gnutls_pbkdf2 (h->hash_alg, &key, &salt, ks->password_iterations,
~~~~~~~~~~~~~ ~~~^~~~~~~~
luks.c:764:25: error: implicit conversion from enumeration type 'gnutls_digest_algorithm_t' to different enumeration type 'gnutls_mac_algorithm_t' [-Werror,-Wenum-conversion]
r = gnutls_pbkdf2 (h->hash_alg, &mkey, &msalt,
~~~~~~~~~~~~~ ~~~^~~~~~~~
luks.c:886:35: error: result of comparison of constant 18446744073709551615 with expression of type 'uint32_t' (aka 'unsigned int') is always false [-Werror,-Wtautological-constant-out-of-range-compare]
if (ks->password_iterations > ULONG_MAX) {
~~~~~~~~~~~~~~~~~~~~~~~ ^ ~~~~~~~~~
Fixes: commit 468919dce6c5eb57503eacac0f67e5dd87c58e6c
(cherry picked from commit 87d488ede9101a2effc71cd1851bf4a4caa521d2)
---
filters/luks/luks.c | 13 ++++++-------
1 file changed, 6 insertions(+), 7 deletions(-)
diff --git a/filters/luks/luks.c b/filters/luks/luks.c
index 706a9bd2..cc619698 100644
--- a/filters/luks/luks.c
+++ b/filters/luks/luks.c
@@ -693,6 +693,10 @@ key_material_length_in_sectors (struct handle *h, size_t i)
static int
try_passphrase_in_keyslot (nbdkit_next *next, struct handle *h, size_t i)
{
+ /* I believe this is supposed to be safe, looking at the GnuTLS
+ * header file.
+ */
+ const gnutls_mac_algorithm_t mac = (gnutls_mac_algorithm_t) h->hash_alg;
struct luks_keyslot *ks = &h->phdr.keyslot[i];
size_t split_key_len;
CLEANUP_FREE uint8_t *split_key = NULL;
@@ -725,7 +729,7 @@ try_passphrase_in_keyslot (nbdkit_next *next, struct handle *h, size_t i)
}
/* Hash the passphrase to make a possible masterkey. */
- r = gnutls_pbkdf2 (h->hash_alg, &key, &salt, ks->password_iterations,
+ r = gnutls_pbkdf2 (mac, &key, &salt, ks->password_iterations,
masterkey, h->phdr.master_key_len);
if (r != 0) {
nbdkit_error ("gnutls_pbkdf2: %s", gnutls_strerror (r));
@@ -761,7 +765,7 @@ try_passphrase_in_keyslot (nbdkit_next *next, struct handle *h, size_t i)
/* Check if the masterkey is correct by comparing hash of the
* masterkey with LUKS header.
*/
- r = gnutls_pbkdf2 (h->hash_alg, &mkey, &msalt,
+ r = gnutls_pbkdf2 (mac, &mkey, &msalt,
h->phdr.master_key_digest_iterations,
key_digest, LUKS_DIGESTSIZE);
if (r != 0) {
@@ -883,11 +887,6 @@ luks_prepare (nbdkit_next *next, void *handle, int readonly)
"points beyond the end of the disk", i);
return -1;
}
- if (ks->password_iterations > ULONG_MAX) {
- nbdkit_error ("bad LUKSv1 header: key slot %zu "
- "iterations too large", i);
- return -1;
- }
/*FALLTHROUGH*/
case LUKS_KEY_DISABLED:
break;
--
2.31.1

52
SOURCES/0006-vddk-Add-logical-and-physical-sector-size-to-D-vddk..patch
View File

@ -1,52 +0,0 @@
From 4c80b474a2c2a552e5bdfcaabfa2981540afe8d8 Mon Sep 17 00:00:00 2001
From: "Richard W.M. Jones" <rjones@redhat.com>
Date: Sat, 23 Oct 2021 16:24:27 +0100
Subject: [PATCH] vddk: Add logical and physical sector size to -D
vddk.diskinfo output
In VDDK >= 7 it is possible to display the logical and physical sector
size in debug output.
This commit also extends the test since this flag was not tested
before.
(cherry picked from commit 5bb8f0586e1faabcbf4f43d722a3b3cb5b352e33)
---
plugins/vddk/vddk.c | 6 ++++++
tests/test-vddk-real.sh | 3 ++-
2 files changed, 8 insertions(+), 1 deletion(-)
diff --git a/plugins/vddk/vddk.c b/plugins/vddk/vddk.c
index d74a484d..50bdde26 100644
--- a/plugins/vddk/vddk.c
+++ b/plugins/vddk/vddk.c
@@ -822,6 +822,12 @@ vddk_get_size (void *handle)
info->parentFileNameHint ? : "NULL");
nbdkit_debug ("disk info: uuid: %s",
info->uuid ? : "NULL");
+ if (library_version >= 7) {
+ nbdkit_debug ("disk info: sectory size: "
+ "logical %" PRIu32 " physical %" PRIu32,
+ info->logicalSectorSize,
+ info->physicalSectorSize);
+ }
}
VDDK_CALL_START (VixDiskLib_FreeInfo, "info")
diff --git a/tests/test-vddk-real.sh b/tests/test-vddk-real.sh
index a6aceac9..ae965245 100755
--- a/tests/test-vddk-real.sh
+++ b/tests/test-vddk-real.sh
@@ -89,7 +89,8 @@ if grep 'cannot open shared object file' $log; then
fi
# Now run nbdkit for the test.
-start_nbdkit -P $pid -U $sock -D vddk.stats=1 vddk libdir="$vddkdir" $vmdk
+start_nbdkit -P $pid -U $sock -D vddk.stats=1 -D vddk.diskinfo=1 \
+ vddk libdir="$vddkdir" $vmdk
uri="nbd+unix:///?socket=$sock"
# VDDK < 6.0 did not support flush, so disable flush test there. Also
--
2.31.1

43
SOURCES/0007-luks-Link-with-libcompat-on-Windows.patch Normal file
View File

@ -0,0 +1,43 @@
From 9416effd73a5cb2e1c929449fca88fd7152aa1be Mon Sep 17 00:00:00 2001
From: "Richard W.M. Jones" <rjones@redhat.com>
Date: Sun, 8 May 2022 12:38:00 +0100
Subject: [PATCH] luks: Link with libcompat on Windows
/usr/lib/gcc/x86_64-w64-mingw32/11.2.1/../../../../x86_64-w64-mingw32/bin/ld: ../../common/utils/.libs/libutils.a(libutils_la-full-rw.o): in function `full_pread':
/builds/nbdkit/nbdkit/common/utils/full-rw.c:53: undefined reference to `pread'
/usr/lib/gcc/x86_64-w64-mingw32/11.2.1/../../../../x86_64-w64-mingw32/bin/ld: ../../common/utils/.libs/libutils.a(libutils_la-full-rw.o): in function `full_pwrite':
/builds/nbdkit/nbdkit/common/utils/full-rw.c:76: undefined reference to `pwrite'
/usr/lib/gcc/x86_64-w64-mingw32/11.2.1/../../../../x86_64-w64-mingw32/bin/ld: ../../common/utils/.libs/libutils.a(libutils_la-vector.o): in function `generic_vector_reserve_page_aligned':
/builds/nbdkit/nbdkit/common/utils/vector.c:112: undefined reference to `sysconf'
/usr/lib/gcc/x86_64-w64-mingw32/11.2.1/../../../../x86_64-w64-mingw32/bin/ld: /builds/nbdkit/nbdkit/common/utils/vector.c:134: undefined reference to `posix_memalign'
collect2: error: ld returned 1 exit status
Fixes: commit 468919dce6c5eb57503eacac0f67e5dd87c58e6c
(cherry picked from commit 4a28c4c46aedf270929a62a1c5ecf2c1129cd456)
---
filters/luks/Makefile.am | 2 ++
1 file changed, 2 insertions(+)
diff --git a/filters/luks/Makefile.am b/filters/luks/Makefile.am
index 622e5c3d..2688f696 100644
--- a/filters/luks/Makefile.am
+++ b/filters/luks/Makefile.am
@@ -45,6 +45,7 @@ nbdkit_luks_filter_la_SOURCES = \
nbdkit_luks_filter_la_CPPFLAGS = \
-I$(top_srcdir)/include \
-I$(top_srcdir)/common/include \
+ -I$(top_srcdir)/common/replacements \
-I$(top_srcdir)/common/utils \
$(NULL)
nbdkit_luks_filter_la_CFLAGS = \
@@ -53,6 +54,7 @@ nbdkit_luks_filter_la_CFLAGS = \
$(NULL)
nbdkit_luks_filter_la_LIBADD = \
$(top_builddir)/common/utils/libutils.la \
+ $(top_builddir)/common/replacements/libcompat.la \
$(IMPORT_LIBRARY_ON_WINDOWS) \
$(GNUTLS_LIBS) \
$(NULL)
--
2.31.1

27
SOURCES/0007-vddk-Fix-typo-in-debug-message.patch
View File

@ -1,27 +0,0 @@
From 4b0d278f3851baf37affa26d34e52963dc8c7c04 Mon Sep 17 00:00:00 2001
From: "Richard W.M. Jones" <rjones@redhat.com>
Date: Sat, 23 Oct 2021 19:41:07 +0100
Subject: [PATCH] vddk: Fix typo in debug message
Fixes: commit 5bb8f0586e1faabcbf4f43d722a3b3cb5b352e33
(cherry picked from commit 343dadeb7340d7b8c5730e2bbab33c829b569122)
---
plugins/vddk/vddk.c | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/plugins/vddk/vddk.c b/plugins/vddk/vddk.c
index 50bdde26..65399a91 100644
--- a/plugins/vddk/vddk.c
+++ b/plugins/vddk/vddk.c
@@ -823,7 +823,7 @@ vddk_get_size (void *handle)
nbdkit_debug ("disk info: uuid: %s",
info->uuid ? : "NULL");
if (library_version >= 7) {
- nbdkit_debug ("disk info: sectory size: "
+ nbdkit_debug ("disk info: sector size: "
"logical %" PRIu32 " physical %" PRIu32,
info->logicalSectorSize,
info->physicalSectorSize);
--
2.31.1

2096
SOURCES/0008-luks-Refactor-the-filter.patch Normal file
View File

File diff suppressed because it is too large Load Diff

55
SOURCES/0008-vddk-Only-print-vddk_library_version-when-we-managed.patch
View File

@ -1,55 +0,0 @@
From 670c1ddb6591046256511a680605c5e2349746e8 Mon Sep 17 00:00:00 2001
From: "Richard W.M. Jones" <rjones@redhat.com>
Date: Sat, 23 Oct 2021 19:50:52 +0100
Subject: [PATCH] vddk: Only print vddk_library_version when we managed to load
the library
Because --dump-plugin calls load_library (false) it won't fail if we
didn't manage to load the library. This results in library_version
being 0, which we printed incorrectly.
Resolve this problem by not printing the vddk_library_version entry in
this case.
Fixes: commit 8700649d147948897f3b97810a1dff37924bdd6e
(cherry picked from commit a3fba12c3e9c2113009f556360ae0bd04c45f6bb)
---
plugins/vddk/nbdkit-vddk-plugin.pod | 1 +
plugins/vddk/vddk.c | 9 ++++++++-
2 files changed, 9 insertions(+), 1 deletion(-)
diff --git a/plugins/vddk/nbdkit-vddk-plugin.pod b/plugins/vddk/nbdkit-vddk-plugin.pod
index bc3c3c94..49e3d75d 100644
--- a/plugins/vddk/nbdkit-vddk-plugin.pod
+++ b/plugins/vddk/nbdkit-vddk-plugin.pod
@@ -425,6 +425,7 @@ by this build.
=item C<vddk_library_version=...>
The VDDK major library version: 5, 6, 7, ...
+If this is omitted it means the library could not be loaded.
=item C<vddk_dll=...>
diff --git a/plugins/vddk/vddk.c b/plugins/vddk/vddk.c
index 65399a91..39a7d261 100644
--- a/plugins/vddk/vddk.c
+++ b/plugins/vddk/vddk.c
@@ -591,7 +591,14 @@ vddk_dump_plugin (void)
printf ("vddk_default_libdir=%s\n", VDDK_LIBDIR);
printf ("vddk_has_nfchostport=1\n");
- printf ("vddk_library_version=%d\n", library_version);
+
+ /* Because load_library (false) we might not have loaded VDDK, in
+ * which case we didn't set library_version. Note this cannot
+ * happen in the normal (non-debug-plugin) path because there we use
+ * load_library (true).
+ */
+ if (library_version > 0)
+ printf ("vddk_library_version=%d\n", library_version);
#if defined(HAVE_DLADDR)
/* It would be nice to print the version of VDDK from the shared
--
2.31.1

101
SOURCES/0009-tests-luks-Reduce-time-taken-to-run-these-tests.patch Normal file
View File

@ -0,0 +1,101 @@
From 387bd4c6fee8ab339fd04e0b841b0c67e6020c8a Mon Sep 17 00:00:00 2001
From: "Richard W.M. Jones" <rjones@redhat.com>
Date: Sun, 8 May 2022 18:05:45 +0100
Subject: [PATCH] tests: luks: Reduce time taken to run these tests
Under valgrind they ran very slowly. Turns out valgrinding over
GnuTLS hashing code is not pretty. About half the time seems to be
taken opening the keyslot, and the rest copying the data.
This change reduces the time (under valgrind) from 15 minutes 45 seconds
to about 6 mins 30 seconds.
(cherry picked from commit 7320ae5dba476171a024ca44b889b3474302dc40)
---
tests/test-luks-copy.sh | 18 +++++++++---------
tests/test-luks-info.sh | 6 +++---
2 files changed, 12 insertions(+), 12 deletions(-)
diff --git a/tests/test-luks-copy.sh b/tests/test-luks-copy.sh
index 99f300d0..01801811 100755
--- a/tests/test-luks-copy.sh
+++ b/tests/test-luks-copy.sh
@@ -60,8 +60,8 @@ rm -f $encrypt_disk $plain_disk $pid $sock
qemu-img create -f luks \
--object secret,data=123456,id=sec0 \
-o key-secret=sec0 \
- $encrypt_disk 10M
-truncate -s 10M $plain_disk
+ $encrypt_disk 1M
+truncate -s 1M $plain_disk
qemu-img convert --target-image-opts -n \
--object secret,data=123456,id=sec0 \
$plain_disk \
@@ -74,11 +74,11 @@ start_nbdkit -P $pid -U $sock \
uri="nbd+unix:///?socket=$sock"
# Copy the whole disk out. It should be empty.
-nbdcopy "$uri" $plain_disk
+nbdcopy -C 1 "$uri" $plain_disk
if [ "$(hexdump -C $plain_disk)" != '00000000 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................|
*
-00a00000' ]; then
+00100000' ]; then
echo "$0: expected plaintext disk to be empty"
exit 1
fi
@@ -88,14 +88,14 @@ fi
nbdsh -u "$uri" \
-c 'h.pwrite(b"1"*65536, 0)' \
-c 'h.pwrite(b"2"*65536, 128*1024)' \
- -c 'h.pwrite(b"3"*65536, 9*1024*1024)' \
+ -c 'h.pwrite(b"3"*65536, 900*1024)' \
-c 'buf = h.pread(65536, 0)' \
-c 'assert buf == b"1"*65536' \
-c 'buf = h.pread(65536, 65536)' \
-c 'assert buf == bytearray(65536)' \
-c 'buf = h.pread(65536, 128*1024)' \
-c 'assert buf == b"2"*65536' \
- -c 'buf = h.pread(65536, 9*1024*1024)' \
+ -c 'buf = h.pread(65536, 900*1024)' \
-c 'assert buf == b"3"*65536' \
-c 'h.flush()'
@@ -115,11 +115,11 @@ if [ "$(hexdump -C $plain_disk)" != '00000000 31 31 31 31 31 31 31 31 31 31 31
*
00030000 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................|
*
-00900000 33 33 33 33 33 33 33 33 33 33 33 33 33 33 33 33 |3333333333333333|
+000e1000 33 33 33 33 33 33 33 33 33 33 33 33 33 33 33 33 |3333333333333333|
*
-00910000 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................|
+000f1000 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................|
*
-00a00000' ]; then
+00100000' ]; then
echo "$0: unexpected content"
exit 1
fi
diff --git a/tests/test-luks-info.sh b/tests/test-luks-info.sh
index 3eff657b..ef141ecd 100755
--- a/tests/test-luks-info.sh
+++ b/tests/test-luks-info.sh
@@ -46,11 +46,11 @@ rm -f $disk $info
qemu-img create -f luks \
--object secret,data=123456,id=sec0 \
-o key-secret=sec0 \
- $disk 10M
+ $disk 1M
nbdkit -U - file $disk --filter=luks passphrase=123456 \
--run 'nbdinfo $uri' > $info
cat $info
-# Check the size is 10M (so it doesn't include the LUKS header).
-grep "10485760" $info
+# Check the size is 1M (so it doesn't include the LUKS header).
+grep "1048576" $info
--
2.31.1

72
SOURCES/0009-vddk-Print-one-line-in-dump-plugin-output-for-each-V.patch
View File

@ -1,72 +0,0 @@
From 21d6c2f8f29f0d7f98852b72ee33751814be49fe Mon Sep 17 00:00:00 2001
From: "Richard W.M. Jones" <rjones@redhat.com>
Date: Mon, 25 Oct 2021 08:36:53 +0100
Subject: [PATCH] vddk: Print one line in --dump-plugin output for each VDDK
API
Helps when detecting if certain optional features are being used, such
as flush and extents.
(cherry picked from commit 4ee13559e46cf622410d0bdd7db29bb00908b40a)
---
plugins/vddk/nbdkit-vddk-plugin.pod | 9 +++++++++
plugins/vddk/vddk.c | 10 ++++++++++
tests/test-vddk-real-dump-plugin.sh | 1 +
3 files changed, 20 insertions(+)
diff --git a/plugins/vddk/nbdkit-vddk-plugin.pod b/plugins/vddk/nbdkit-vddk-plugin.pod
index 49e3d75d..0702aa75 100644
--- a/plugins/vddk/nbdkit-vddk-plugin.pod
+++ b/plugins/vddk/nbdkit-vddk-plugin.pod
@@ -432,6 +432,15 @@ If this is omitted it means the library could not be loaded.
Prints the full path to the VDDK shared library. Since this requires
a glibc extension it may not be available in all builds of the plugin.
+=item C<VixDiskLib_...=1>
+
+For each VDDK API that the plugin uses I<and> which is present in the
+VDDK library that was loaded, we print the name of the API
+(eg. C<VixDiskLib_Open=1>). This lets you see which optional APIs are
+available, such as C<VixDiskLib_Flush> and
+C<VixDiskLib_QueryAllocatedBlocks>. If the library could not be
+loaded then these lines are not printed.
+
=back
=head1 NOTES
diff --git a/plugins/vddk/vddk.c b/plugins/vddk/vddk.c
index 39a7d261..096b04bf 100644
--- a/plugins/vddk/vddk.c
+++ b/plugins/vddk/vddk.c
@@ -616,6 +616,16 @@ vddk_dump_plugin (void)
printf ("vddk_dll=%s\n", p);
}
#endif
+
+ /* Note we print all VDDK APIs found here, not just the optional
+ * ones. That is so if we update the baseline VDDK in future and
+ * make optional into required APIs, the output doesn't change.
+ */
+#define STUB(fn,ret,args) if (fn != NULL) printf ("%s=1\n", #fn);
+#define OPTIONAL_STUB(fn,ret,args) STUB(fn,ret,args)
+#include "vddk-stubs.h"
+#undef STUB
+#undef OPTIONAL_STUB
}
/* The rules on threads and VDDK are here:
diff --git a/tests/test-vddk-real-dump-plugin.sh b/tests/test-vddk-real-dump-plugin.sh
index 0a079c6c..e37c8b54 100755
--- a/tests/test-vddk-real-dump-plugin.sh
+++ b/tests/test-vddk-real-dump-plugin.sh
@@ -65,6 +65,7 @@ grep ^vddk_default_libdir= $out
grep ^vddk_has_nfchostport= $out
grep ^vddk_library_version= $out
grep ^vddk_dll= $out
+grep ^VixDiskLib_Open=1 $out
dll="$(grep ^vddk_dll $out | cut -d= -f2)"
test -f "$dll"
--
2.31.1

112
SOURCES/0010-Add-nbdkit.parse_size-Python-function.patch Normal file
View File

@ -0,0 +1,112 @@
From 52ee1dab95436128b44c37cc495022ff90108b2e Mon Sep 17 00:00:00 2001
From: Nikolaus Rath <Nikolaus@rath.org>
Date: Mon, 9 May 2022 10:04:30 +0100
Subject: [PATCH] Add nbdkit.parse_size() Python function.
This enables Python plugins to parse sizes the same way as C plugins.
I'm not sure about the best way to test this - input is appreciated.
I'm not too happy with the way this code is tested. It workes, but putting the tests into
test-python-plugin.py feels misplaced: this file is intended to support the unit tests in
test_python.py, not run its own unit tests.
(cherry picked from commit 1b7d72542be68e254c1ef86ecb1a82b05c78ff63)
---
plugins/python/modfunctions.c | 21 +++++++++++++++++++++
plugins/python/nbdkit-python-plugin.pod | 5 +++++
tests/test-python-plugin.py | 19 +++++++++++++++++++
3 files changed, 45 insertions(+)
diff --git a/plugins/python/modfunctions.c b/plugins/python/modfunctions.c
index fffbaab2..46b0c904 100644
--- a/plugins/python/modfunctions.c
+++ b/plugins/python/modfunctions.c
@@ -93,11 +93,32 @@ do_shutdown (PyObject *self, PyObject *args)
Py_RETURN_NONE;
}
+/* nbdkit.parse_size */
+static PyObject *
+parse_size (PyObject *self, PyObject *args)
+{
+ const char *s;
+ if (!PyArg_ParseTuple (args, "s", &s)) {
+ PyErr_SetString (PyExc_TypeError, "Expected string, got something else");
+ return NULL;
+ }
+
+ int64_t size = nbdkit_parse_size(s);
+ if (size == -1) {
+ PyErr_SetString (PyExc_ValueError, "Unable to parse string as size");
+ return NULL;
+ }
+
+ return PyLong_FromSize_t((size_t)size);
+}
+
static PyMethodDef NbdkitMethods[] = {
{ "debug", debug, METH_VARARGS,
"Print a debug message" },
{ "export_name", export_name, METH_VARARGS,
"Return the optional export name negotiated with the client" },
+ { "parse_size", parse_size, METH_VARARGS,
+ "Parse human-readable size strings into bytes" },
{ "set_error", set_error, METH_VARARGS,
"Store an errno value prior to throwing an exception" },
{ "shutdown", do_shutdown, METH_VARARGS,
diff --git a/plugins/python/nbdkit-python-plugin.pod b/plugins/python/nbdkit-python-plugin.pod
index 051b0237..ccc9406f 100644
--- a/plugins/python/nbdkit-python-plugin.pod
+++ b/plugins/python/nbdkit-python-plugin.pod
@@ -131,6 +131,11 @@ Record C<err> as the reason you are about to throw an exception. C<err>
should correspond to usual errno values, where it may help to
C<import errno>.
+=head3 C<nbdkit.parse_size(str)>
+
+Parse a string (such as "100M") into a size in bytes. Wraps the
+C<nbdkit_parse_size()> C function.
+
=head3 C<nbdkit.shutdown()>
Request asynchronous server shutdown.
diff --git a/tests/test-python-plugin.py b/tests/test-python-plugin.py
index 0b34d532..d4f379fc 100644
--- a/tests/test-python-plugin.py
+++ b/tests/test-python-plugin.py
@@ -34,12 +34,31 @@
import nbdkit
import pickle
import base64
+import unittest
API_VERSION = 2
cfg = {}
+# Not nice, but there doesn't seem to be a better way of putting this
+class TestAPI(unittest.TestCase):
+
+ def test_parse_size(self):
+ self.assertEqual(nbdkit.parse_size('511'), 511)
+ self.assertEqual(nbdkit.parse_size('7k'), 7*1024)
+ self.assertEqual(nbdkit.parse_size('17M'), 17*1024*1024)
+
+ with self.assertRaises(TypeError):
+ nbdkit.parse_size(17)
+
+ with self.assertRaises(ValueError):
+ nbdkit.parse_size('foo')
+
+
+TestAPI().test_parse_size()
+
+
def config(k, v):
global cfg
if k == "cfg":
--
2.31.1

147
SOURCES/0010-vddk-Move-minimum-version-to-VDDK-6.5.patch
View File

@ -1,147 +0,0 @@
From f4379f04ea27e25c00e98db2e60d0fdb647442e9 Mon Sep 17 00:00:00 2001
From: "Richard W.M. Jones" <rjones@redhat.com>
Date: Tue, 26 Oct 2021 19:46:32 +0100
Subject: [PATCH] vddk: Move minimum version to VDDK 6.5
Drop support for VDDK 5.5.5 (released in 2015) and 6.0 (released the
same year). Move minimum supported version to 6.5 (released Nov
2016). This is so we can use asynchronous operations.
Acked-by: Laszlo Ersek <lersek@redhat.com>
(cherry picked from commit 5ed23616762a72e039531a9a7cd81353cd4f436e)
---
plugins/vddk/nbdkit-vddk-plugin.pod | 10 +++-------
plugins/vddk/vddk-stubs.h | 3 +--
plugins/vddk/vddk.c | 24 ++++++++++++++++--------
tests/dummy-vddk.c | 6 ++++++
4 files changed, 26 insertions(+), 17 deletions(-)
diff --git a/plugins/vddk/nbdkit-vddk-plugin.pod b/plugins/vddk/nbdkit-vddk-plugin.pod
index 0702aa75..1c16d096 100644
--- a/plugins/vddk/nbdkit-vddk-plugin.pod
+++ b/plugins/vddk/nbdkit-vddk-plugin.pod
@@ -424,7 +424,7 @@ by this build.
=item C<vddk_library_version=...>
-The VDDK major library version: 5, 6, 7, ...
+The VDDK major library version: 6, 7, ...
If this is omitted it means the library could not be loaded.
=item C<vddk_dll=...>
@@ -548,16 +548,12 @@ server, which can also be very slow.
=head1 SUPPORTED VERSIONS OF VDDK
-This plugin requires VDDK E<ge> 5.5.5, which in turn means that it
-is only supported on x64-64 platforms.
+This plugin requires VDDK E<ge> 6.5 (released Nov 2016). It is only
+supported on the x64-64 archtecture.
It has been tested with all versions up to 7.0.3 (but should work with
future versions).
-VDDK E<ge> 6.0 should be used if possible. This is the first version
-which added Flush support which is crucial for data integrity when
-writing.
-
VDDK 6.7 was the first version that supported the
C<VixDiskLib_QueryAllocatedBlocks> API, required to provide extent
information over NBD.
diff --git a/plugins/vddk/vddk-stubs.h b/plugins/vddk/vddk-stubs.h
index 5e70238d..a94df9cd 100644
--- a/plugins/vddk/vddk-stubs.h
+++ b/plugins/vddk/vddk-stubs.h
@@ -40,8 +40,7 @@
*/
/* Required stubs, present in all versions of VDDK that we support. I
- * have checked that all these exist in at least VDDK 5.5.5 (2015)
- * which is the earliest version of VDDK that we support.
+ * have checked that all these exist in at least VDDK 5.5.5 (2015).
*/
STUB (VixDiskLib_InitEx,
diff --git a/plugins/vddk/vddk.c b/plugins/vddk/vddk.c
index 096b04bf..babffc28 100644
--- a/plugins/vddk/vddk.c
+++ b/plugins/vddk/vddk.c
@@ -75,13 +75,13 @@ NBDKIT_DLL_PUBLIC int vddk_debug_stats;
#undef OPTIONAL_STUB
/* Parameters passed to InitEx. */
-#define VDDK_MAJOR 5
+#define VDDK_MAJOR 6
#define VDDK_MINOR 5
static void *dl; /* dlopen handle */
static bool init_called; /* was InitEx called */
static __thread int error_suppression; /* threadlocal error suppression */
-static int library_version; /* VDDK major: 5, 6, 7, ... */
+static int library_version; /* VDDK major: 6, 7, ... */
static enum { NONE = 0, ZLIB, FASTLZ, SKIPZ } compression; /* compression */
static char *config; /* config */
@@ -413,16 +413,14 @@ load_library (bool load_error_is_fatal)
/* Prefer the newest library in case multiple exist. Check two
* possible directories: the usual VDDK installation puts .so
* files in an arch-specific subdirectory of $libdir (our minimum
- * supported version is VDDK 5.5.5, which only supports x64-64);
- * but our testsuite is easier to write if we point libdir
- * directly to a stub .so.
+ * supported version is VDDK 6.5, which only supports x64-64); but
+ * our testsuite is easier to write if we point libdir directly to
+ * a stub .so.
*/
{ "lib64/libvixDiskLib.so.7", 7 },
{ "libvixDiskLib.so.7", 7 },
{ "lib64/libvixDiskLib.so.6", 6 },
{ "libvixDiskLib.so.6", 6 },
- { "lib64/libvixDiskLib.so.5", 5 },
- { "libvixDiskLib.so.5", 5 },
{ NULL }
};
size_t i;
@@ -474,7 +472,7 @@ load_library (bool load_error_is_fatal)
exit (EXIT_FAILURE);
}
- assert (library_version >= 5);
+ assert (library_version >= 6);
/* Load symbols. */
#define STUB(fn,ret,args) \
@@ -490,6 +488,16 @@ load_library (bool load_error_is_fatal)
#include "vddk-stubs.h"
#undef STUB
#undef OPTIONAL_STUB
+
+ /* Additionally, VDDK version must be >= 6.5. This was the first
+ * version which introduced VixDiskLib_Wait symbol so we can check
+ * for that.
+ */
+ if (VixDiskLib_Wait == NULL) {
+ nbdkit_error ("VDDK version must be >= 6.5. "
+ "See nbdkit-vddk-plugin(1) man page section \"SUPPORTED VERSIONS OF VDDK\".");
+ exit (EXIT_FAILURE);
+ }
}
static int
diff --git a/tests/dummy-vddk.c b/tests/dummy-vddk.c
index 9b5ae0a2..cb88380c 100644
--- a/tests/dummy-vddk.c
+++ b/tests/dummy-vddk.c
@@ -198,3 +198,9 @@ VixDiskLib_Write (VixDiskLibHandle handle,
memcpy (disk + offset, buf, nr_sectors * VIXDISKLIB_SECTOR_SIZE);
return VIX_OK;
}
+
+NBDKIT_DLL_PUBLIC VixError
+VixDiskLib_Wait (VixDiskLibHandle handle)
+{
+ return VIX_OK;
+}
--
2.31.1

34
SOURCES/0011-cache-Fix-cross-reference-nbdkit-readahead-filter.patch Normal file
View File

@ -0,0 +1,34 @@
From 644e0ed6333cf5fe2c1e39da157e8f1ce97267b9 Mon Sep 17 00:00:00 2001
From: "Richard W.M. Jones" <rjones@redhat.com>
Date: Sat, 14 May 2022 13:47:19 +0100
Subject: [PATCH] cache: Fix cross-reference nbdkit-readahead-filter
After the readahead filter was reimplemented so that it only issues
cache requests, the two filters should be used together, not as
alternatives. Update the documentation of the cache filter to make
this clear.
Fixes: commit 2ff548d66ad3eae87868402ec5b3319edd12090f
(cherry picked from commit 894771f39a8fd2632caad00e497146d69cac4bac)
---
filters/cache/nbdkit-cache-filter.pod | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/filters/cache/nbdkit-cache-filter.pod b/filters/cache/nbdkit-cache-filter.pod
index d85fef09..f4234e1a 100644
--- a/filters/cache/nbdkit-cache-filter.pod
+++ b/filters/cache/nbdkit-cache-filter.pod
@@ -28,8 +28,8 @@ loss, as the name suggests).
This filter only caches image contents. To cache image metadata, use
L<nbdkit-cacheextents-filter(1)> between this filter and the plugin.
-To accelerate sequential reads, use L<nbdkit-readahead-filter(1)>
-instead.
+To accelerate sequential reads, use L<nbdkit-readahead-filter(1)> on
+top of this filter.
=head1 PARAMETERS
--
2.31.1

72
SOURCES/0011-vddk-Add-read-write-and-wait-asynchronous-functions.patch
View File

@ -1,72 +0,0 @@
From 90dc3311582784f8b078a30a7207c15c6298b1e2 Mon Sep 17 00:00:00 2001
From: "Richard W.M. Jones" <rjones@redhat.com>
Date: Wed, 27 Oct 2021 11:57:35 +0100
Subject: [PATCH] vddk: Add read, write and wait asynchronous functions
These functions added in VDDK 6.0 - 6.5 implement asynchronous read
and write.
Acked-by: Laszlo Ersek <lersek@redhat.com>
(cherry picked from commit ad53e7becafed6ca3573795a79c534281fe9c274)
---
plugins/vddk/vddk-structs.h | 3 +++
plugins/vddk/vddk-stubs.h | 19 ++++++++++++++++++-
2 files changed, 21 insertions(+), 1 deletion(-)
diff --git a/plugins/vddk/vddk-structs.h b/plugins/vddk/vddk-structs.h
index aeb5bfd0..e97f017c 100644
--- a/plugins/vddk/vddk-structs.h
+++ b/plugins/vddk/vddk-structs.h
@@ -43,6 +43,7 @@
typedef uint64_t VixError;
#define VIX_OK 0
+#define VIX_ASYNC 25000
#define VIXDISKLIB_FLAG_OPEN_UNBUFFERED 1
#define VIXDISKLIB_FLAG_OPEN_SINGLE_LINK 2
@@ -61,6 +62,8 @@ typedef void *VixDiskLibHandle;
typedef void VixDiskLibGenericLogFunc (const char *fmt, va_list args);
+typedef void (*VixDiskLibCompletionCB) (void *data, VixError result);
+
enum VixDiskLibCredType {
VIXDISKLIB_CRED_UID = 1,
VIXDISKLIB_CRED_SESSIONID = 2,
diff --git a/plugins/vddk/vddk-stubs.h b/plugins/vddk/vddk-stubs.h
index a94df9cd..66353691 100644
--- a/plugins/vddk/vddk-stubs.h
+++ b/plugins/vddk/vddk-stubs.h
@@ -103,10 +103,27 @@ STUB (VixDiskLib_Write,
uint64_t start_sector, uint64_t nr_sectors,
const unsigned char *buf));
-/* Added in VDDK 6.0, this will be NULL in earlier versions. */
+/* Added in VDDK 6.0, these will be NULL in earlier versions. */
OPTIONAL_STUB (VixDiskLib_Flush,
VixError,
(VixDiskLibHandle handle));
+OPTIONAL_STUB (VixDiskLib_ReadAsync,
+ VixError,
+ (VixDiskLibHandle handle,
+ uint64_t start_sector, uint64_t nr_sectors,
+ unsigned char *buf,
+ VixDiskLibCompletionCB callback, void *data));
+OPTIONAL_STUB (VixDiskLib_WriteAsync,
+ VixError,
+ (VixDiskLibHandle handle,
+ uint64_t start_sector, uint64_t nr_sectors,
+ const unsigned char *buf,
+ VixDiskLibCompletionCB callback, void *data));
+
+/* Added in VDDK 6.5, this will be NULL in earlier versions. */
+OPTIONAL_STUB (VixDiskLib_Wait,
+ VixError,
+ (VixDiskLibHandle handle));
/* Added in VDDK 6.7, these will be NULL for earlier versions: */
OPTIONAL_STUB (VixDiskLib_QueryAllocatedBlocks,
--
2.31.1

48
SOURCES/0012-curl-Don-t-document-curl-plugin-readahead-filter.patch Normal file
View File

@ -0,0 +1,48 @@
From 4a7e5169935c8850fddcea8da79639ded907c549 Mon Sep 17 00:00:00 2001
From: "Richard W.M. Jones" <rjones@redhat.com>
Date: Sat, 14 May 2022 14:00:16 +0100
Subject: [PATCH] curl: Don't document curl plugin + readahead filter
nbdkit readahead filter does not support plugins which do not use the
parallel thread model.
Fixes: commit 2ff548d66ad3eae87868402ec5b3319edd12090f
(cherry picked from commit 92fbb76d11b9f17c527debd803aa2505f3642783)
---
docs/nbdkit-captive.pod | 7 -------
plugins/curl/nbdkit-curl-plugin.pod | 1 -
2 files changed, 8 deletions(-)
diff --git a/docs/nbdkit-captive.pod b/docs/nbdkit-captive.pod
index eafe36d8..d41a824d 100644
--- a/docs/nbdkit-captive.pod
+++ b/docs/nbdkit-captive.pod
@@ -110,13 +110,6 @@ an embedded disk image. To copy it out:
nbdkit -U - example1 --run 'qemu-img convert $nbd disk.img'
-If plugin requests have a high overhead (for example making HTTP
-requests to a remote server), adding L<nbdkit-readahead-filter(1)> may
-help performance:
-
- nbdkit -U - --filter=readahead curl https://example.com/disk.img \
- --run 'qemu-img convert $nbd disk.img'
-
If the source suffers from temporary network failures
L<nbdkit-retry-filter(1)> or L<nbdkit-retry-request-filter(1)> may
help.
diff --git a/plugins/curl/nbdkit-curl-plugin.pod b/plugins/curl/nbdkit-curl-plugin.pod
index 54fce66c..fc422ca2 100644
--- a/plugins/curl/nbdkit-curl-plugin.pod
+++ b/plugins/curl/nbdkit-curl-plugin.pod
@@ -509,7 +509,6 @@ L<CURLOPT_VERBOSE(3)>,
L<nbdkit(1)>,
L<nbdkit-extentlist-filter(1)>,
L<nbdkit-file-plugin(1)>,
-L<nbdkit-readahead-filter(1)>,
L<nbdkit-retry-filter(1)>,
L<nbdkit-retry-request-filter(1)>,
L<nbdkit-ssh-plugin(1)>,
--
2.31.1

259
SOURCES/0012-vddk-Start-to-split-VDDK-over-several-files.patch
View File

@ -1,259 +0,0 @@
From c9e432e08e889d9e6edea52344b2452f0141f56b Mon Sep 17 00:00:00 2001
From: "Richard W.M. Jones" <rjones@redhat.com>
Date: Wed, 27 Oct 2021 12:20:31 +0100
Subject: [PATCH] vddk: Start to split VDDK over several files
This change doesn't do anything except move some definitions into the
header file vddk.h, but it allows future commits to split up the very
large vddk.c file.
Acked-by: Laszlo Ersek <lersek@redhat.com>
(cherry picked from commit 117634dccf4e29394e8718a8d62e93a9edf0a39c)
---
plugins/vddk/vddk.c | 91 +++++++++++++--------------------------------
plugins/vddk/vddk.h | 89 +++++++++++++++++++++++++++++++++++++++++++-
2 files changed, 112 insertions(+), 68 deletions(-)
diff --git a/plugins/vddk/vddk.c b/plugins/vddk/vddk.c
index babffc28..041bff1a 100644
--- a/plugins/vddk/vddk.c
+++ b/plugins/vddk/vddk.c
@@ -50,14 +50,12 @@
#include <nbdkit-plugin.h>
#include "cleanup.h"
-#include "isaligned.h"
#include "minmax.h"
#include "rounding.h"
#include "tvdiff.h"
#include "vector.h"
#include "vddk.h"
-#include "vddk-structs.h"
/* Debug flags. */
NBDKIT_DLL_PUBLIC int vddk_debug_diskinfo;
@@ -65,11 +63,11 @@ NBDKIT_DLL_PUBLIC int vddk_debug_extents;
NBDKIT_DLL_PUBLIC int vddk_debug_datapath = 1;
NBDKIT_DLL_PUBLIC int vddk_debug_stats;
-/* For each VDDK API define a static global variable. These globals
- * are initialized when the plugin is loaded (by vddk_get_ready).
+/* For each VDDK API define a global variable. These globals are
+ * initialized when the plugin is loaded (by vddk_get_ready).
*/
-#define STUB(fn,ret,args) static ret (*fn) args
-#define OPTIONAL_STUB(fn,ret,args) static ret (*fn) args
+#define STUB(fn,ret,args) ret (*fn) args
+#define OPTIONAL_STUB(fn,ret,args) ret (*fn) args
#include "vddk-stubs.h"
#undef STUB
#undef OPTIONAL_STUB
@@ -78,28 +76,28 @@ NBDKIT_DLL_PUBLIC int vddk_debug_stats;
#define VDDK_MAJOR 6
#define VDDK_MINOR 5
-static void *dl; /* dlopen handle */
-static bool init_called; /* was InitEx called */
-static __thread int error_suppression; /* threadlocal error suppression */
-static int library_version; /* VDDK major: 6, 7, ... */
+void *dl; /* dlopen handle */
+bool init_called; /* was InitEx called */
+__thread int error_suppression; /* threadlocal error suppression */
+int library_version; /* VDDK major: 6, 7, ... */
+bool is_remote; /* true if remote connection */
-static enum { NONE = 0, ZLIB, FASTLZ, SKIPZ } compression; /* compression */
-static char *config; /* config */
-static const char *cookie; /* cookie */
-static const char *filename; /* file */
-char *libdir; /* libdir */
-static uint16_t nfc_host_port; /* nfchostport */
-char *password; /* password */
-static uint16_t port; /* port */
-static const char *server_name; /* server */
-static bool single_link; /* single-link */
-static const char *snapshot_moref; /* snapshot */
-static const char *thumb_print; /* thumbprint */
-static const char *transport_modes; /* transports */
-static bool unbuffered; /* unbuffered */
-static const char *username; /* user */
-static const char *vmx_spec; /* vm */
-static bool is_remote;
+enum compression_type compression; /* compression */
+char *config; /* config */
+const char *cookie; /* cookie */
+const char *filename; /* file */
+char *libdir; /* libdir */
+uint16_t nfc_host_port; /* nfchostport */
+char *password; /* password */
+uint16_t port; /* port */
+const char *server_name; /* server */
+bool single_link; /* single-link */
+const char *snapshot_moref; /* snapshot */
+const char *thumb_print; /* thumbprint */
+const char *transport_modes; /* transports */
+bool unbuffered; /* unbuffered */
+const char *username; /* user */
+const char *vmx_spec; /* vm */
/* For each VDDK API define a variable to store the time taken (used
* to implement -D vddk.stats=1).
@@ -120,45 +118,6 @@ static void display_stats (void);
#undef STUB
#undef OPTIONAL_STUB
-/* Macros to bracket each VDDK API call, for printing debugging
- * information and collecting statistics.
- */
-#define VDDK_CALL_START(fn, fs, ...) \
- do { \
- struct timeval start_t, end_t; \
- /* GCC can optimize this away at compile time: */ \
- const bool datapath = \
- strcmp (#fn, "VixDiskLib_Read") == 0 || \
- strcmp (#fn, "VixDiskLib_Write") == 0; \
- if (vddk_debug_stats) \
- gettimeofday (&start_t, NULL); \
- if (!datapath || vddk_debug_datapath) \
- nbdkit_debug ("VDDK call: %s (" fs ")", #fn, ##__VA_ARGS__); \
- do
-#define VDDK_CALL_END(fn, bytes_) \
- while (0); \
- if (vddk_debug_stats) { \
- gettimeofday (&end_t, NULL); \
- ACQUIRE_LOCK_FOR_CURRENT_SCOPE (&stats_lock); \
- stats_##fn.usecs += tvdiff_usec (&start_t, &end_t); \
- stats_##fn.calls++; \
- stats_##fn.bytes += bytes_; \
- } \
- } while (0)
-
-/* Print VDDK errors. */
-#define VDDK_ERROR(err, fs, ...) \
- do { \
- char *vddk_err_msg; \
- VDDK_CALL_START (VixDiskLib_GetErrorText, "%lu", err) \
- vddk_err_msg = VixDiskLib_GetErrorText ((err), NULL); \
- VDDK_CALL_END (VixDiskLib_GetErrorText, 0); \
- nbdkit_error (fs ": %s", ##__VA_ARGS__, vddk_err_msg); \
- VDDK_CALL_START (VixDiskLib_FreeErrorText, "") \
- VixDiskLib_FreeErrorText (vddk_err_msg); \
- VDDK_CALL_END (VixDiskLib_FreeErrorText, 0); \
- } while (0)
-
/* Unload the plugin. */
static void
vddk_unload (void)
diff --git a/plugins/vddk/vddk.h b/plugins/vddk/vddk.h
index 8c63b4ee..29775eb4 100644
--- a/plugins/vddk/vddk.h
+++ b/plugins/vddk/vddk.h
@@ -1,5 +1,5 @@
/* nbdkit
- * Copyright (C) 2013-2020 Red Hat Inc.
+ * Copyright (C) 2013-2021 Red Hat Inc.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are
@@ -33,11 +33,96 @@
#ifndef NBDKIT_VDDK_H
#define NBDKIT_VDDK_H
+#include <stdbool.h>
+#include <stdint.h>
+#include <sys/time.h>
+
+#include <pthread.h>
+
+#include "isaligned.h"
+#include "tvdiff.h"
+#include "vector.h"
+
+#include "vddk-structs.h"
+
+enum compression_type { NONE = 0, ZLIB, FASTLZ, SKIPZ };
+
+extern void *dl;
+extern bool init_called;
+extern __thread int error_suppression;
+extern int library_version;
+extern bool is_remote;
+
+extern enum compression_type compression;
+extern char *config;
+extern const char *cookie;
+extern const char *filename;
extern char *libdir;
+extern uint16_t nfc_host_port;
extern char *password;
+extern uint16_t port;
+extern const char *server_name;
+extern bool single_link;
+extern const char *snapshot_moref;
+extern const char *thumb_print;
+extern const char *transport_modes;
+extern bool unbuffered;
+extern const char *username;
+extern const char *vmx_spec;
+
+extern int vddk_debug_diskinfo;
+extern int vddk_debug_extents;
+extern int vddk_debug_datapath;
+extern int vddk_debug_stats;
+
+#define STUB(fn,ret,args) extern ret (*fn) args
+#define OPTIONAL_STUB(fn,ret,args) extern ret (*fn) args
+#include "vddk-stubs.h"
+#undef STUB
+#undef OPTIONAL_STUB
+
+/* Macros to bracket each VDDK API call, for printing debugging
+ * information and collecting statistics.
+ */
+#define VDDK_CALL_START(fn, fs, ...) \
+ do { \
+ struct timeval start_t, end_t; \
+ /* GCC can optimize this away at compile time: */ \
+ const bool datapath = \
+ strcmp (#fn, "VixDiskLib_Read") == 0 || \
+ strcmp (#fn, "VixDiskLib_Write") == 0; \
+ if (vddk_debug_stats) \
+ gettimeofday (&start_t, NULL); \
+ if (!datapath || vddk_debug_datapath) \
+ nbdkit_debug ("VDDK call: %s (" fs ")", #fn, ##__VA_ARGS__); \
+ do
+#define VDDK_CALL_END(fn, bytes_) \
+ while (0); \
+ if (vddk_debug_stats) { \
+ gettimeofday (&end_t, NULL); \
+ ACQUIRE_LOCK_FOR_CURRENT_SCOPE (&stats_lock); \
+ stats_##fn.usecs += tvdiff_usec (&start_t, &end_t); \
+ stats_##fn.calls++; \
+ stats_##fn.bytes += bytes_; \
+ } \
+ } while (0)
+
+/* Print VDDK errors. */
+#define VDDK_ERROR(err, fs, ...) \
+ do { \
+ char *vddk_err_msg; \
+ VDDK_CALL_START (VixDiskLib_GetErrorText, "%lu", err) \
+ vddk_err_msg = VixDiskLib_GetErrorText ((err), NULL); \
+ VDDK_CALL_END (VixDiskLib_GetErrorText, 0); \
+ nbdkit_error (fs ": %s", ##__VA_ARGS__, vddk_err_msg); \
+ VDDK_CALL_START (VixDiskLib_FreeErrorText, "") \
+ VixDiskLib_FreeErrorText (vddk_err_msg); \
+ VDDK_CALL_END (VixDiskLib_FreeErrorText, 0); \
+ } while (0)
+
+/* reexec.c */
extern bool noreexec;
extern char *reexeced;
-
extern void reexec_if_needed (const char *prepend);
extern int restore_ld_library_path (void);
--
2.31.1

1021
SOURCES/0013-New-filter-scan.patch Normal file
View File

File diff suppressed because it is too large Load Diff

287
SOURCES/0013-vddk-Refactor-D-vddk.stats-1-into-a-new-file.patch
View File

@ -1,287 +0,0 @@
From 66945d24e9192a67af421eecbb1835d42636ab93 Mon Sep 17 00:00:00 2001
From: "Richard W.M. Jones" <rjones@redhat.com>
Date: Wed, 27 Oct 2021 12:30:41 +0100
Subject: [PATCH] vddk: Refactor -D vddk.stats=1 into a new file
Acked-by: Laszlo Ersek <lersek@redhat.com>
(cherry picked from commit dcd5bc51ed7710c32d956345ea8da14ba15ef8f5)
---
plugins/vddk/Makefile.am | 1 +
plugins/vddk/stats.c | 118 +++++++++++++++++++++++++++++++++++++++
plugins/vddk/vddk.c | 78 +-------------------------
plugins/vddk/vddk.h | 15 +++++
4 files changed, 135 insertions(+), 77 deletions(-)
create mode 100644 plugins/vddk/stats.c
diff --git a/plugins/vddk/Makefile.am b/plugins/vddk/Makefile.am
index 232aaedd..4f470ff9 100644
--- a/plugins/vddk/Makefile.am
+++ b/plugins/vddk/Makefile.am
@@ -46,6 +46,7 @@ nbdkit_vddk_plugin_la_SOURCES = \
vddk.c \
vddk.h \
reexec.c \
+ stats.c \
vddk-structs.h \
vddk-stubs.h \
$(top_srcdir)/include/nbdkit-plugin.h \
diff --git a/plugins/vddk/stats.c b/plugins/vddk/stats.c
new file mode 100644
index 00000000..18a42714
--- /dev/null
+++ b/plugins/vddk/stats.c
@@ -0,0 +1,118 @@
+/* nbdkit
+ * Copyright (C) 2013-2021 Red Hat Inc.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * * Neither the name of Red Hat nor the names of its contributors may be
+ * used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY RED HAT AND CONTRIBUTORS ''AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+ * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL RED HAT OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
+ * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+ * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <config.h>
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <stdint.h>
+#include <inttypes.h>
+
+#include <pthread.h>
+
+#define NBDKIT_API_VERSION 2
+#include <nbdkit-plugin.h>
+
+#include "vector.h"
+
+#include "vddk.h"
+
+/* Debug flags. */
+NBDKIT_DLL_PUBLIC int vddk_debug_stats;
+
+pthread_mutex_t stats_lock = PTHREAD_MUTEX_INITIALIZER;
+
+/* For each VDDK API define a variable to store the time taken (used
+ * to implement -D vddk.stats=1).
+ */
+#define STUB(fn,ret,args) struct vddk_stat stats_##fn = { .name = #fn }
+#define OPTIONAL_STUB(fn,ret,args) STUB(fn,ret,args)
+#include "vddk-stubs.h"
+#undef STUB
+#undef OPTIONAL_STUB
+
+DEFINE_VECTOR_TYPE(statlist, struct vddk_stat)
+
+static int
+stat_compare (const void *vp1, const void *vp2)
+{
+ const struct vddk_stat *st1 = vp1;
+ const struct vddk_stat *st2 = vp2;
+
+ /* Note: sorts in reverse order of time spent in each API call. */
+ if (st1->usecs < st2->usecs) return 1;
+ else if (st1->usecs > st2->usecs) return -1;
+ else return 0;
+}
+
+static const char *
+api_name_without_prefix (const char *name)
+{
+ return strncmp (name, "VixDiskLib_", 11) == 0 ? name + 11 : name;
+}
+
+void
+display_stats (void)
+{
+ statlist stats = empty_vector;
+ size_t i;
+
+ if (!vddk_debug_stats) return;
+
+#define STUB(fn,ret,args) statlist_append (&stats, stats_##fn)
+#define OPTIONAL_STUB(fn,ret,args) statlist_append (&stats, stats_##fn)
+#include "vddk-stubs.h"
+#undef STUB
+#undef OPTIONAL_STUB
+
+ qsort (stats.ptr, stats.size, sizeof stats.ptr[0], stat_compare);
+
+ nbdkit_debug ("VDDK function stats (-D vddk.stats=1):");
+ nbdkit_debug ("%-24s %15s %5s %15s",
+ "VixDiskLib_...", "µs", "calls", "bytes");
+ for (i = 0; i < stats.size; ++i) {
+ if (stats.ptr[i].usecs) {
+ if (stats.ptr[i].bytes > 0)
+ nbdkit_debug (" %-22s %15" PRIi64 " %5" PRIu64 " %15" PRIu64,
+ api_name_without_prefix (stats.ptr[i].name),
+ stats.ptr[i].usecs,
+ stats.ptr[i].calls,
+ stats.ptr[i].bytes);
+ else
+ nbdkit_debug (" %-22s %15" PRIi64 " %5" PRIu64,
+ api_name_without_prefix (stats.ptr[i].name),
+ stats.ptr[i].usecs,
+ stats.ptr[i].calls);
+ }
+ }
+ statlist_reset (&stats);
+}
diff --git a/plugins/vddk/vddk.c b/plugins/vddk/vddk.c
index 041bff1a..67ac775c 100644
--- a/plugins/vddk/vddk.c
+++ b/plugins/vddk/vddk.c
@@ -61,7 +61,6 @@
NBDKIT_DLL_PUBLIC int vddk_debug_diskinfo;
NBDKIT_DLL_PUBLIC int vddk_debug_extents;
NBDKIT_DLL_PUBLIC int vddk_debug_datapath = 1;
-NBDKIT_DLL_PUBLIC int vddk_debug_stats;
/* For each VDDK API define a global variable. These globals are
* initialized when the plugin is loaded (by vddk_get_ready).
@@ -99,25 +98,6 @@ bool unbuffered; /* unbuffered */
const char *username; /* user */
const char *vmx_spec; /* vm */
-/* For each VDDK API define a variable to store the time taken (used
- * to implement -D vddk.stats=1).
- */
-struct vddk_stat {
- const char *name; /* function name */
- int64_t usecs; /* total number of usecs consumed */
- uint64_t calls; /* number of times called */
- uint64_t bytes; /* bytes transferred, datapath calls only */
-};
-static pthread_mutex_t stats_lock = PTHREAD_MUTEX_INITIALIZER;
-static void display_stats (void);
-#define STUB(fn,ret,args) \
- static struct vddk_stat stats_##fn = { .name = #fn }
-#define OPTIONAL_STUB(fn,ret,args) \
- static struct vddk_stat stats_##fn = { .name = #fn }
-#include "vddk-stubs.h"
-#undef STUB
-#undef OPTIONAL_STUB
-
/* Unload the plugin. */
static void
vddk_unload (void)
@@ -130,69 +110,13 @@ vddk_unload (void)
if (dl)
dlclose (dl);
- if (vddk_debug_stats)
- display_stats ();
+ display_stats ();
free (config);
free (libdir);
free (password);
}
-DEFINE_VECTOR_TYPE(statlist, struct vddk_stat)
-
-static int
-stat_compare (const void *vp1, const void *vp2)
-{
- const struct vddk_stat *st1 = vp1;
- const struct vddk_stat *st2 = vp2;
-
- /* Note: sorts in reverse order of time spent in each API call. */
- if (st1->usecs < st2->usecs) return 1;
- else if (st1->usecs > st2->usecs) return -1;
- else return 0;
-}
-
-static const char *
-api_name_without_prefix (const char *name)
-{
- return strncmp (name, "VixDiskLib_", 11) == 0 ? name + 11 : name;
-}
-
-static void
-display_stats (void)
-{
- statlist stats = empty_vector;
- size_t i;
-
-#define STUB(fn,ret,args) statlist_append (&stats, stats_##fn)
-#define OPTIONAL_STUB(fn,ret,args) statlist_append (&stats, stats_##fn)
-#include "vddk-stubs.h"
-#undef STUB
-#undef OPTIONAL_STUB
-
- qsort (stats.ptr, stats.size, sizeof stats.ptr[0], stat_compare);
-
- nbdkit_debug ("VDDK function stats (-D vddk.stats=1):");
- nbdkit_debug ("%-24s %15s %5s %15s",
- "VixDiskLib_...", "µs", "calls", "bytes");
- for (i = 0; i < stats.size; ++i) {
- if (stats.ptr[i].usecs) {
- if (stats.ptr[i].bytes > 0)
- nbdkit_debug (" %-22s %15" PRIi64 " %5" PRIu64 " %15" PRIu64,
- api_name_without_prefix (stats.ptr[i].name),
- stats.ptr[i].usecs,
- stats.ptr[i].calls,
- stats.ptr[i].bytes);
- else
- nbdkit_debug (" %-22s %15" PRIi64 " %5" PRIu64,
- api_name_without_prefix (stats.ptr[i].name),
- stats.ptr[i].usecs,
- stats.ptr[i].calls);
- }
- }
- statlist_reset (&stats);
-}
-
static void
trim (char *str)
{
diff --git a/plugins/vddk/vddk.h b/plugins/vddk/vddk.h
index 29775eb4..1400589d 100644
--- a/plugins/vddk/vddk.h
+++ b/plugins/vddk/vddk.h
@@ -126,4 +126,19 @@ extern char *reexeced;
extern void reexec_if_needed (const char *prepend);
extern int restore_ld_library_path (void);
+/* stats.c */
+struct vddk_stat {
+ const char *name; /* function name */
+ int64_t usecs; /* total number of usecs consumed */
+ uint64_t calls; /* number of times called */
+ uint64_t bytes; /* bytes transferred, datapath calls only */
+};
+extern pthread_mutex_t stats_lock;
+#define STUB(fn,ret,args) extern struct vddk_stat stats_##fn;
+#define OPTIONAL_STUB(fn,ret,args) STUB(fn,ret,args)
+#include "vddk-stubs.h"
+#undef STUB
+#undef OPTIONAL_STUB
+extern void display_stats (void);
+
#endif /* NBDKIT_VDDK_H */
--
2.31.1

67
SOURCES/0014-scan-Remove-condition-variable.patch Normal file
View File

@ -0,0 +1,67 @@
From 91677241184ab1aa77adadd612fa069d084863ec Mon Sep 17 00:00:00 2001
From: "Richard W.M. Jones" <rjones@redhat.com>
Date: Sat, 14 May 2022 18:54:32 +0100
Subject: [PATCH] scan: Remove condition variable
This was copied in from the readahead filter code, but is not actually
needed in this filter because it never has to sleep waiting for a
command.
Fixes: commit 65c20a09ceacb4431986a2982f2c2e746df63fcb
(cherry picked from commit 43ad586698347997cdfa1bd56bfed0292f89f134)
---
filters/scan/scan.c | 6 ------
filters/scan/scan.h | 1 -
2 files changed, 7 deletions(-)
diff --git a/filters/scan/scan.c b/filters/scan/scan.c
index ac5b18d2..8a966577 100644
--- a/filters/scan/scan.c
+++ b/filters/scan/scan.c
@@ -136,9 +136,6 @@ send_command_to_background_thread (struct bgthread_ctrl *ctrl,
ACQUIRE_LOCK_FOR_CURRENT_SCOPE (&ctrl->lock);
if (command_queue_append (&ctrl->cmds, cmd) == -1)
return -1;
- /* Signal the thread if it could be sleeping on an empty queue. */
- if (ctrl->cmds.len == 1)
- pthread_cond_signal (&ctrl->cond);
return 0;
}
@@ -199,13 +196,11 @@ scan_prepare (nbdkit_next *next, void *handle, int readonly)
/* Create the background thread. */
h->ctrl.cmds = (command_queue) empty_vector;
pthread_mutex_init (&h->ctrl.lock, NULL);
- pthread_cond_init (&h->ctrl.cond, NULL);
err = pthread_create (&h->thread, NULL, scan_thread, &h->ctrl);
if (err != 0) {
errno = err;
nbdkit_error ("pthread_create: %m");
- pthread_cond_destroy (&h->ctrl.cond);
pthread_mutex_destroy (&h->ctrl.lock);
return -1;
}
@@ -227,7 +222,6 @@ scan_finalize (nbdkit_next *next, void *handle)
send_command_to_background_thread (&h->ctrl, quit_cmd);
pthread_join (h->thread, NULL);
- pthread_cond_destroy (&h->ctrl.cond);
pthread_mutex_destroy (&h->ctrl.lock);
command_queue_reset (&h->ctrl.cmds);
h->running = false;
diff --git a/filters/scan/scan.h b/filters/scan/scan.h
index 7ff39310..98c0228b 100644
--- a/filters/scan/scan.h
+++ b/filters/scan/scan.h
@@ -54,7 +54,6 @@ DEFINE_VECTOR_TYPE(command_queue, struct command);
struct bgthread_ctrl {
command_queue cmds; /* Command queue. */
pthread_mutex_t lock; /* Lock for queue. */
- pthread_cond_t cond; /* Condition queue size 0 -> 1. */
nbdkit_next *next; /* For sending cache operations. */
};
--
2.31.1

1287
SOURCES/0014-vddk-Implement-parallel-thread-model.patch
View File

File diff suppressed because it is too large Load Diff

57
SOURCES/0015-scan-Small-typographical-fix-in-manual.patch Normal file
View File

@ -0,0 +1,57 @@
From c191f45530d4dd7f978803c0bfa402ca0fc950df Mon Sep 17 00:00:00 2001
From: "Richard W.M. Jones" <rjones@redhat.com>
Date: Sat, 14 May 2022 19:02:48 +0100
Subject: [PATCH] scan: Small typographical fix in manual
Fixes: commit 65c20a09ceacb4431986a2982f2c2e746df63fcb
(cherry picked from commit 67d4e3437d2e28fa3ce1c4b3818d2b1e7939c5ec)
---
filters/scan/nbdkit-scan-filter.pod | 12 ++++++------
1 file changed, 6 insertions(+), 6 deletions(-)
diff --git a/filters/scan/nbdkit-scan-filter.pod b/filters/scan/nbdkit-scan-filter.pod
index 4a8d0ef9..2fe9bb80 100644
--- a/filters/scan/nbdkit-scan-filter.pod
+++ b/filters/scan/nbdkit-scan-filter.pod
@@ -26,8 +26,8 @@ below (after) this filter, giving approximately the same effect.
L<nbdkit-cow-filter(1)> can be used instead of nbdkit-cache-filter, if
you add the C<cow-on-cache=true> option.
-Various C<scan-*> parameters can be used to tune scanning, although
-the defaults should be suitable in most cases.
+Various parameters can be used to tune scanning, although the defaults
+should be suitable in most cases.
A similar filter is L<nbdkit-readahead-filter(1)>.
@@ -38,23 +38,23 @@ filter will print a warning message if this happens.
=over 4
-=item Thread model must be parallel *
+=item Thread model must be parallel*
For example L<nbdkit-curl-plugin(1)> only supports
C<serialize_requests>, and so this filter cannot perform prefetches in
parallel with the read requests.
-=item Only scans while clients are connected *
+=item Only scans while clients are connected*
The current filter only scans while there is at least one client
connected.
-=item Only scans the default export *
+=item Only scans the default export*
The current filter only scans the default export and ignores all
clients connecting to the non-default export name.
-* We may be able to lift these restrictions in future.
+*We may be able to lift these restrictions in future.
=item Underlying filters or plugin must support C<.cache> (prefetch)
--
2.31.1

57
SOURCES/0015-vddk-Assume-that-VixDiskLib_Flush-is-available.patch
View File

@ -1,57 +0,0 @@
From c91ac233f6474b07ef181a08093c5d0f2f4ec4c3 Mon Sep 17 00:00:00 2001
From: "Richard W.M. Jones" <rjones@redhat.com>
Date: Fri, 29 Oct 2021 20:56:55 +0100
Subject: [PATCH] vddk: Assume that VixDiskLib_Flush is available
Since we now require and check that VDDK >= 6.5, we can assume that
VixDiskLib_Flush is always available.
(cherry picked from commit e3685e6f0d0b71ab24b96fe85430a3b75da58736)
---
plugins/vddk/vddk.c | 15 +++++++++++----
1 file changed, 11 insertions(+), 4 deletions(-)
diff --git a/plugins/vddk/vddk.c b/plugins/vddk/vddk.c
index 9f223db0..f967e2d9 100644
--- a/plugins/vddk/vddk.c
+++ b/plugins/vddk/vddk.c
@@ -378,6 +378,12 @@ load_library (bool load_error_is_fatal)
"See nbdkit-vddk-plugin(1) man page section \"SUPPORTED VERSIONS OF VDDK\".");
exit (EXIT_FAILURE);
}
+
+ /* Added in VDDK 6.0 so it must always be present. Since we are
+ * going to call this function unconditionally, fail early and hard
+ * if for some reason it's not present.
+ */
+ assert (VixDiskLib_Flush != NULL);
}
static int
@@ -725,18 +731,19 @@ vddk_get_size (void *handle)
return (int64_t) size;
}
+/* The Flush call was added in VDDK 6.0, since we support minimum 6.5
+ * we are always able to do FUA / flush.
+ */
static int
vddk_can_fua (void *handle)
{
- /* The Flush call was not available in VDDK < 6.0. */
- return VixDiskLib_Flush != NULL ? NBDKIT_FUA_NATIVE : NBDKIT_FUA_NONE;
+ return NBDKIT_FUA_NATIVE;
}
static int
vddk_can_flush (void *handle)
{
- /* The Flush call was not available in VDDK < 6.0. */
- return VixDiskLib_Flush != NULL;
+ return 1;
}
/* Read data from the file.
--
2.31.1

34
SOURCES/0016-ssh-Don-t-reference-readahead-or-scan-filters-from-t.patch Normal file
View File

@ -0,0 +1,34 @@
From 651045d703804d7dafab04a0387ca92573f52467 Mon Sep 17 00:00:00 2001
From: "Richard W.M. Jones" <rjones@redhat.com>
Date: Sat, 14 May 2022 20:57:38 +0100
Subject: [PATCH] ssh: Don't reference readahead or scan filters from this
plugin
These filters do not support plugins which do not use the parallel
thread model.
Fixes: commit 2ff548d66ad3eae87868402ec5b3319edd12090f
Fixes: commit 65c20a09ceacb4431986a2982f2c2e746df63fcb
See-also: commit 92fbb76d11b9f17c527debd803aa2505f3642783
(cherry picked from commit 7eb356719376c4d0b2379cea5d39c81602d2d304)
---
plugins/ssh/nbdkit-ssh-plugin.pod | 2 --
1 file changed, 2 deletions(-)
diff --git a/plugins/ssh/nbdkit-ssh-plugin.pod b/plugins/ssh/nbdkit-ssh-plugin.pod
index 214957d6..bb922d37 100644
--- a/plugins/ssh/nbdkit-ssh-plugin.pod
+++ b/plugins/ssh/nbdkit-ssh-plugin.pod
@@ -347,9 +347,7 @@ C<nbdkit-ssh-plugin> first appeared in nbdkit 1.12.
L<nbdkit(1)>,
L<nbdkit-curl-plugin(1)>,
L<nbdkit-extentlist-filter(1)>,
-L<nbdkit-readahead-filter(1)>,
L<nbdkit-retry-filter(1)>,
-L<nbdkit-scan-filter(1)>,
L<nbdkit-plugin(3)>,
L<ssh(1)>,
L<ssh-agent(1)>,
--
2.31.1

186
SOURCES/0016-vddk-Simplify-detection-of-VDDK-symbols-and-baseline.patch
View File

@ -1,186 +0,0 @@
From 984e95fcbdb19c2495851322a4c33f34291ecfab Mon Sep 17 00:00:00 2001
From: "Richard W.M. Jones" <rjones@redhat.com>
Date: Fri, 29 Oct 2021 21:02:54 +0100
Subject: [PATCH] vddk: Simplify detection of VDDK symbols and baseline 6.5
Make all symbols from VDDK 6.5 into required symbols and use a single
error message function if one of these is missing. The new error is:
nbdkit: error: required VDDK symbol "VixDiskLib_Wait" is
missing. VDDK version must be >= 6.5. See nbdkit-vddk-plugin(1) man
page section "SUPPORTED VERSIONS OF VDDK". Original dlopen error:
vmware-vix-disklib-distrib/lib64/libvixDiskLib.so.6: undefined
symbol: VixDiskLib_Wait
Remove the extra check and assert.
Be more consistent about #define OPTIONAL_STUB(fn,ret,args) STUB(fn,ret,args)
when we want the optional and required stubs to do the same thing.
(cherry picked from commit ec0d22e61881efa39a69d02ccb9e4ede8bf95e75)
---
plugins/vddk/stats.c | 2 +-
plugins/vddk/vddk-stubs.h | 45 ++++++++++++++++++---------------------
plugins/vddk/vddk.c | 36 ++++++++++++-------------------
plugins/vddk/vddk.h | 2 +-
4 files changed, 37 insertions(+), 48 deletions(-)
diff --git a/plugins/vddk/stats.c b/plugins/vddk/stats.c
index 18a42714..76e0c244 100644
--- a/plugins/vddk/stats.c
+++ b/plugins/vddk/stats.c
@@ -89,7 +89,7 @@ display_stats (void)
if (!vddk_debug_stats) return;
#define STUB(fn,ret,args) statlist_append (&stats, stats_##fn)
-#define OPTIONAL_STUB(fn,ret,args) statlist_append (&stats, stats_##fn)
+#define OPTIONAL_STUB(fn,ret,args) STUB(fn,ret,args)
#include "vddk-stubs.h"
#undef STUB
#undef OPTIONAL_STUB
diff --git a/plugins/vddk/vddk-stubs.h b/plugins/vddk/vddk-stubs.h
index 66353691..7d8644c3 100644
--- a/plugins/vddk/vddk-stubs.h
+++ b/plugins/vddk/vddk-stubs.h
@@ -39,10 +39,7 @@
* function name, return value, arguments.
*/
-/* Required stubs, present in all versions of VDDK that we support. I
- * have checked that all these exist in at least VDDK 5.5.5 (2015).
- */
-
+/* Required stubs, present in all versions of VDDK since 6.5 (Nov 2016). */
STUB (VixDiskLib_InitEx,
VixError,
(uint32_t major, uint32_t minor,
@@ -103,27 +100,27 @@ STUB (VixDiskLib_Write,
uint64_t start_sector, uint64_t nr_sectors,
const unsigned char *buf));
-/* Added in VDDK 6.0, these will be NULL in earlier versions. */
-OPTIONAL_STUB (VixDiskLib_Flush,
- VixError,
- (VixDiskLibHandle handle));
-OPTIONAL_STUB (VixDiskLib_ReadAsync,
- VixError,
- (VixDiskLibHandle handle,
- uint64_t start_sector, uint64_t nr_sectors,
- unsigned char *buf,
- VixDiskLibCompletionCB callback, void *data));
-OPTIONAL_STUB (VixDiskLib_WriteAsync,
- VixError,
- (VixDiskLibHandle handle,
- uint64_t start_sector, uint64_t nr_sectors,
- const unsigned char *buf,
- VixDiskLibCompletionCB callback, void *data));
+/* Added in VDDK 6.0. */
+STUB (VixDiskLib_Flush,
+ VixError,
+ (VixDiskLibHandle handle));
+STUB (VixDiskLib_ReadAsync,
+ VixError,
+ (VixDiskLibHandle handle,
+ uint64_t start_sector, uint64_t nr_sectors,
+ unsigned char *buf,
+ VixDiskLibCompletionCB callback, void *data));
+STUB (VixDiskLib_WriteAsync,
+ VixError,
+ (VixDiskLibHandle handle,
+ uint64_t start_sector, uint64_t nr_sectors,
+ const unsigned char *buf,
+ VixDiskLibCompletionCB callback, void *data));
-/* Added in VDDK 6.5, this will be NULL in earlier versions. */
-OPTIONAL_STUB (VixDiskLib_Wait,
- VixError,
- (VixDiskLibHandle handle));
+/* Added in VDDK 6.5. */
+STUB (VixDiskLib_Wait,
+ VixError,
+ (VixDiskLibHandle handle));
/* Added in VDDK 6.7, these will be NULL for earlier versions: */
OPTIONAL_STUB (VixDiskLib_QueryAllocatedBlocks,
diff --git a/plugins/vddk/vddk.c b/plugins/vddk/vddk.c
index f967e2d9..271b5ee0 100644
--- a/plugins/vddk/vddk.c
+++ b/plugins/vddk/vddk.c
@@ -63,7 +63,7 @@ NBDKIT_DLL_PUBLIC int vddk_debug_datapath = 1;
* initialized when the plugin is loaded (by vddk_get_ready).
*/
#define STUB(fn,ret,args) ret (*fn) args
-#define OPTIONAL_STUB(fn,ret,args) ret (*fn) args
+#define OPTIONAL_STUB(fn,ret,args) STUB(fn,ret,args)
#include "vddk-stubs.h"
#undef STUB
#undef OPTIONAL_STUB
@@ -282,6 +282,17 @@ vddk_config (const char *key, const char *value)
return 0;
}
+static void
+missing_required_symbol (const char *fn)
+{
+ nbdkit_error ("required VDDK symbol \"%s\" is missing. "
+ "VDDK version must be >= 6.5. "
+ "See nbdkit-vddk-plugin(1) man page section \"SUPPORTED VERSIONS OF VDDK\". "
+ "Original dlopen error: %s\n",
+ fn, dlerror ());
+ exit (EXIT_FAILURE);
+}
+
/* Load the VDDK library. */
static void
load_library (bool load_error_is_fatal)
@@ -358,32 +369,13 @@ load_library (bool load_error_is_fatal)
#define STUB(fn,ret,args) \
do { \
fn = dlsym (dl, #fn); \
- if (fn == NULL) { \
- nbdkit_error ("required VDDK symbol \"%s\" is missing: %s", \
- #fn, dlerror ()); \
- exit (EXIT_FAILURE); \
- } \
+ if (fn == NULL) \
+ missing_required_symbol (#fn); \
} while (0)
#define OPTIONAL_STUB(fn,ret,args) fn = dlsym (dl, #fn)
#include "vddk-stubs.h"
#undef STUB
#undef OPTIONAL_STUB
-
- /* Additionally, VDDK version must be >= 6.5. This was the first
- * version which introduced VixDiskLib_Wait symbol so we can check
- * for that.
- */
- if (VixDiskLib_Wait == NULL) {
- nbdkit_error ("VDDK version must be >= 6.5. "
- "See nbdkit-vddk-plugin(1) man page section \"SUPPORTED VERSIONS OF VDDK\".");
- exit (EXIT_FAILURE);
- }
-
- /* Added in VDDK 6.0 so it must always be present. Since we are
- * going to call this function unconditionally, fail early and hard
- * if for some reason it's not present.
- */
- assert (VixDiskLib_Flush != NULL);
}
static int
diff --git a/plugins/vddk/vddk.h b/plugins/vddk/vddk.h
index be0b3492..0e3dd79e 100644
--- a/plugins/vddk/vddk.h
+++ b/plugins/vddk/vddk.h
@@ -76,7 +76,7 @@ extern int vddk_debug_datapath;
extern int vddk_debug_stats;
#define STUB(fn,ret,args) extern ret (*fn) args
-#define OPTIONAL_STUB(fn,ret,args) extern ret (*fn) args
+#define OPTIONAL_STUB(fn,ret,args) STUB(fn,ret,args)
#include "vddk-stubs.h"
#undef STUB
#undef OPTIONAL_STUB
--
2.31.1

56
SOURCES/0017-scan-Fix-bound-so-we-don-t-try-to-prefetch-beyond-en.patch Normal file
View File

@ -0,0 +1,56 @@
From f58d2a04338edc647e2334ff58b49508424e3f3b Mon Sep 17 00:00:00 2001
From: "Richard W.M. Jones" <rjones@redhat.com>
Date: Tue, 17 May 2022 13:20:17 +0100
Subject: [PATCH] scan: Fix bound so we don't try to prefetch beyond end of
disk
An off-by-one error in the bound could cause the filter to try to
prefetch beyond the end of the underlying plugin. This would cause
nbdkit to crash with this assertion failure:
nbdkit: backend.c:782: backend_cache: Assertion `backend_valid_range (c, offset, count)' failed.
The sequence of events was:
- scan filter background thread started
- client reads to the end of the disk
- background thread skips ahead to end of disk (offset == size)
- background thread tries to prefetch from this point
In the final step the calculations caused to the background thread to
prefetch a scan-size block beyond the end of the plugin.
Fixes: commit 65c20a09ceacb4431986a2982f2c2e746df63fcb
(cherry picked from commit 953643429b8c57b4dd20a6c0e5b83704ae9a0e88)
---
filters/scan/bgthread.c | 10 +++++-----
1 file changed, 5 insertions(+), 5 deletions(-)
diff --git a/filters/scan/bgthread.c b/filters/scan/bgthread.c
index 384e79b6..5fa5f27f 100644
--- a/filters/scan/bgthread.c
+++ b/filters/scan/bgthread.c
@@ -113,12 +113,12 @@ scan_thread (void *vp)
}
adjust_clock (offset);
- if (offset > size)
- continue;
- /* Issue the next prefetch. */
- n = MIN (scan_size, size - offset);
- ctrl->next->cache (ctrl->next, n, offset, 0, NULL);
+ if (offset < size) {
+ /* Issue the next prefetch. */
+ n = MIN (scan_size, size - offset);
+ ctrl->next->cache (ctrl->next, n, offset, 0, NULL);
+ }
}
if (scan_forever) {
--
2.31.1

40
SOURCES/0017-vddk-Remove-some-whitespace-from-a-couple-of-functio.patch
View File

@ -1,40 +0,0 @@
From 342efed6bb9f8f0c8d2cb4aa2b09da64ed2e7ed4 Mon Sep 17 00:00:00 2001
From: "Richard W.M. Jones" <rjones@redhat.com>
Date: Sat, 30 Oct 2021 08:34:28 +0100
Subject: [PATCH] vddk: Remove some whitespace from a couple of functions
(cherry picked from commit 974dce2c2ef84fc096ee319f340054234a29df91)
---
plugins/vddk/vddk.c | 9 ++-------
1 file changed, 2 insertions(+), 7 deletions(-)
diff --git a/plugins/vddk/vddk.c b/plugins/vddk/vddk.c
index 271b5ee0..184f1a9c 100644
--- a/plugins/vddk/vddk.c
+++ b/plugins/vddk/vddk.c
@@ -792,9 +792,7 @@ static int
vddk_flush (void *handle, uint32_t flags)
{
struct vddk_handle *h = handle;
- struct command flush_cmd = {
- .type = FLUSH,
- };
+ struct command flush_cmd = { .type = FLUSH };
return send_command_and_wait (h, &flush_cmd);
}
@@ -804,10 +802,7 @@ vddk_can_extents (void *handle)
{
struct vddk_handle *h = handle;
int ret;
- struct command can_extents_cmd = {
- .type = CAN_EXTENTS,
- .ptr = &ret,
- };
+ struct command can_extents_cmd = { .type = CAN_EXTENTS, .ptr = &ret };
if (send_command_and_wait (h, &can_extents_cmd) == -1)
return -1;
--
2.31.1

110
SOURCES/0018-tests-Add-a-regression-test-for-LUKS-zeroing-crash.patch Normal file
View File

@ -0,0 +1,110 @@
From d1d2f43223bcda062d10c8e68776590956892f71 Mon Sep 17 00:00:00 2001
From: "Richard W.M. Jones" <rjones@redhat.com>
Date: Fri, 10 Jun 2022 22:11:44 +0100
Subject: [PATCH] tests: Add a regression test for LUKS zeroing crash
https://listman.redhat.com/archives/libguestfs/2022-June/029188.html
(cherry picked from commit 7ab2ef96803bfc385f786be82ebfdd4cc977d504)
---
tests/Makefile.am | 2 ++
tests/test-luks-copy-zero.sh | 70 ++++++++++++++++++++++++++++++++++++
2 files changed, 72 insertions(+)
create mode 100755 tests/test-luks-copy-zero.sh
diff --git a/tests/Makefile.am b/tests/Makefile.am
index 799aa6c2..0f4b0746 100644
--- a/tests/Makefile.am
+++ b/tests/Makefile.am
@@ -1601,11 +1601,13 @@ if HAVE_GNUTLS_PBKDF2
TESTS += \
test-luks-info.sh \
test-luks-copy.sh \
+ test-luks-copy-zero.sh \
$(NULL)
endif
EXTRA_DIST += \
test-luks-info.sh \
test-luks-copy.sh \
+ test-luks-copy-zero.sh \
$(NULL)
# multi-conn filter test.
diff --git a/tests/test-luks-copy-zero.sh b/tests/test-luks-copy-zero.sh
new file mode 100755
index 00000000..6ff560e3
--- /dev/null
+++ b/tests/test-luks-copy-zero.sh
@@ -0,0 +1,70 @@
+#!/usr/bin/env bash
+# nbdkit
+# Copyright (C) 2018-2022 Red Hat Inc.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#
+# * Redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in the
+# documentation and/or other materials provided with the distribution.
+#
+# * Neither the name of Red Hat nor the names of its contributors may be
+# used to endorse or promote products derived from this software without
+# specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY RED HAT AND CONTRIBUTORS ''AS IS'' AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+# PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL RED HAT OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
+# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+# SUCH DAMAGE.
+
+# Regression test for:
+# https://listman.redhat.com/archives/libguestfs/2022-June/029188.html
+
+source ./functions.sh
+set -e
+set -x
+
+requires qemu-img --version
+requires nbdcopy --version
+requires truncate --version
+requires file --version
+requires_filter luks
+
+encrypt_disk=luks-copy-zero1.img
+zero_disk=luks-copy-zero2.img
+cleanup_fn rm -f $encrypt_disk $zero_disk
+rm -f $encrypt_disk $zero_disk
+
+# Create an empty encrypted disk container.
+qemu-img create -f luks \
+ --object secret,data=123456,id=sec0 \
+ -o key-secret=sec0 \
+ $encrypt_disk 100M
+
+# Create an all zeroes disk of the same size.
+truncate -s 100M $zero_disk
+
+# Using nbdkit-luks-filter, write the zero disk into the encrypted
+# disk. nbdcopy will do this using NBD_CMD_ZERO operations.
+nbdkit -U - -fv \
+ file $encrypt_disk --filter=luks passphrase=123456 \
+ --run "nbdcopy -C 1 $zero_disk \$nbd"
+
+# Check that the encrypted disk is still a LUKS disk. If zeroing is
+# wrong in the filter it's possible that it writes through to the
+# underlying disk, erasing the container.
+file $encrypt_disk
+file $encrypt_disk | grep "LUKS encrypted file"
--
2.31.1

338
SOURCES/0018-vddk-Move-config-debug-error-and-utility-functions-a.patch
View File

@ -1,338 +0,0 @@
From edbded52b144ce3c8c45c7ef352f8969a1f5d1bb Mon Sep 17 00:00:00 2001
From: "Richard W.M. Jones" <rjones@redhat.com>
Date: Sat, 30 Oct 2021 08:27:39 +0100
Subject: [PATCH] vddk: Move config, debug/error and utility functions around
Move the functions so they are nearer to where they are used.
Introduce a utils.c file for utility functions.
This is just code rearrangement with no other effects.
(cherry picked from commit c59be086210a06688b9195e0b91f8603a668654a)
---
plugins/vddk/Makefile.am | 1 +
plugins/vddk/utils.c | 51 ++++++++++
plugins/vddk/vddk.c | 201 +++++++++++++++++++--------------------
plugins/vddk/vddk.h | 3 +
4 files changed, 151 insertions(+), 105 deletions(-)
create mode 100644 plugins/vddk/utils.c
diff --git a/plugins/vddk/Makefile.am b/plugins/vddk/Makefile.am
index f8382fc9..02113da0 100644
--- a/plugins/vddk/Makefile.am
+++ b/plugins/vddk/Makefile.am
@@ -47,6 +47,7 @@ nbdkit_vddk_plugin_la_SOURCES = \
vddk.h \
reexec.c \
stats.c \
+ utils.c \
vddk-structs.h \
vddk-stubs.h \
worker.c \
diff --git a/plugins/vddk/utils.c b/plugins/vddk/utils.c
new file mode 100644
index 00000000..f0c19950
--- /dev/null
+++ b/plugins/vddk/utils.c
@@ -0,0 +1,51 @@
+/* nbdkit
+ * Copyright (C) 2013-2021 Red Hat Inc.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * * Neither the name of Red Hat nor the names of its contributors may be
+ * used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY RED HAT AND CONTRIBUTORS ''AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+ * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL RED HAT OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
+ * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+ * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <config.h>
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#define NBDKIT_API_VERSION 2
+#include <nbdkit-plugin.h>
+
+#include "vddk.h"
+
+void
+trim (char *str)
+{
+ size_t len = strlen (str);
+
+ if (len > 0 && str[len-1] == '\n')
+ str[len-1] = '\0';
+}
diff --git a/plugins/vddk/vddk.c b/plugins/vddk/vddk.c
index 184f1a9c..31e5e23b 100644
--- a/plugins/vddk/vddk.c
+++ b/plugins/vddk/vddk.c
@@ -114,61 +114,6 @@ vddk_unload (void)
free (password);
}
-static void
-trim (char *str)
-{
- size_t len = strlen (str);
-
- if (len > 0 && str[len-1] == '\n')
- str[len-1] = '\0';
-}
-
-/* Turn log messages from the library into nbdkit_debug. */
-static void
-debug_function (const char *fs, va_list args)
-{
- CLEANUP_FREE char *str = NULL;
-
- if (vasprintf (&str, fs, args) == -1) {
- nbdkit_debug ("lost debug message: %s", fs);
- return;
- }
-
- trim (str);
-
- nbdkit_debug ("%s", str);
-}
-
-/* Turn error messages from the library into nbdkit_error. */
-static void
-error_function (const char *fs, va_list args)
-{
- CLEANUP_FREE char *str = NULL;
-
- /* If the thread-local error_suppression flag is non-zero then we
- * will suppress error messages from VDDK in this thread.
- */
- if (error_suppression) return;
-
- if (vasprintf (&str, fs, args) == -1) {
- nbdkit_error ("lost error message: %s", fs);
- return;
- }
-
- trim (str);
-
- /* VDDK 7 added a useless error message about their "phone home"
- * system called CEIP which only panics users. Demote it to a debug
- * statement. https://bugzilla.redhat.com/show_bug.cgi?id=1834267
- */
- if (strstr (str, "Get CEIP status failed") != NULL) {
- nbdkit_debug ("%s", str);
- return;
- }
-
- nbdkit_error ("%s", str);
-}
-
/* Configuration. */
static int
vddk_config (const char *key, const char *value)
@@ -282,6 +227,56 @@ vddk_config (const char *key, const char *value)
return 0;
}
+static int
+vddk_config_complete (void)
+{
+ if (filename == NULL) {
+ nbdkit_error ("you must supply the file=<FILENAME> parameter "
+ "after the plugin name on the command line");
+ return -1;
+ }
+
+ /* For remote connections, check all the parameters have been
+ * passed. Note that VDDK will segfault if parameters that it
+ * expects are NULL (and there's no real way to tell what parameters
+ * it is expecting). This implements the same test that the VDDK
+ * sample program does.
+ */
+ is_remote =
+ vmx_spec ||
+ server_name ||
+ username ||
+ password ||
+ cookie ||
+ thumb_print ||
+ port ||
+ nfc_host_port;
+
+ if (is_remote) {
+#define missing(test, param) \
+ if (test) { \
+ nbdkit_error ("remote connection requested, missing parameter: %s", \
+ param); \
+ return -1; \
+ }
+ missing (!server_name, "server");
+ missing (!username, "user");
+ missing (!password, "password");
+ missing (!vmx_spec, "vm");
+#undef missing
+ }
+
+ /* Restore original LD_LIBRARY_PATH after reexec. */
+ if (restore_ld_library_path () == -1)
+ return -1;
+
+ return 0;
+}
+
+#define vddk_config_help \
+ "[file=]<FILENAME> (required) The filename (eg. VMDK file) to serve.\n" \
+ "Many optional parameters are supported, see nbdkit-vddk-plugin(1)."
+
static void
missing_required_symbol (const char *fn)
{
@@ -378,56 +373,6 @@ load_library (bool load_error_is_fatal)
#undef OPTIONAL_STUB
}
-static int
-vddk_config_complete (void)
-{
- if (filename == NULL) {
- nbdkit_error ("you must supply the file=<FILENAME> parameter "
- "after the plugin name on the command line");
- return -1;
- }
-
- /* For remote connections, check all the parameters have been
- * passed. Note that VDDK will segfault if parameters that it
- * expects are NULL (and there's no real way to tell what parameters
- * it is expecting). This implements the same test that the VDDK
- * sample program does.
- */
- is_remote =
- vmx_spec ||
- server_name ||
- username ||
- password ||
- cookie ||
- thumb_print ||
- port ||
- nfc_host_port;
-
- if (is_remote) {
-#define missing(test, param) \
- if (test) { \
- nbdkit_error ("remote connection requested, missing parameter: %s", \
- param); \
- return -1; \
- }
- missing (!server_name, "server");
- missing (!username, "user");
- missing (!password, "password");
- missing (!vmx_spec, "vm");
-#undef missing
- }
-
- /* Restore original LD_LIBRARY_PATH after reexec. */
- if (restore_ld_library_path () == -1)
- return -1;
-
- return 0;
-}
-
-#define vddk_config_help \
- "[file=]<FILENAME> (required) The filename (eg. VMDK file) to serve.\n" \
- "Many optional parameters are supported, see nbdkit-vddk-plugin(1)."
-
static int
vddk_get_ready (void)
{
@@ -435,6 +380,52 @@ vddk_get_ready (void)
return 0;
}
+/* Turn log messages from the library into nbdkit_debug. */
+static void
+debug_function (const char *fs, va_list args)
+{
+ CLEANUP_FREE char *str = NULL;
+
+ if (vasprintf (&str, fs, args) == -1) {
+ nbdkit_debug ("lost debug message: %s", fs);
+ return;
+ }
+
+ trim (str);
+
+ nbdkit_debug ("%s", str);
+}
+
+/* Turn error messages from the library into nbdkit_error. */
+static void
+error_function (const char *fs, va_list args)
+{
+ CLEANUP_FREE char *str = NULL;
+
+ /* If the thread-local error_suppression flag is non-zero then we
+ * will suppress error messages from VDDK in this thread.
+ */
+ if (error_suppression) return;
+
+ if (vasprintf (&str, fs, args) == -1) {
+ nbdkit_error ("lost error message: %s", fs);
+ return;
+ }
+
+ trim (str);
+
+ /* VDDK 7 added a useless error message about their "phone home"
+ * system called CEIP which only panics users. Demote it to a debug
+ * statement. https://bugzilla.redhat.com/show_bug.cgi?id=1834267
+ */
+ if (strstr (str, "Get CEIP status failed") != NULL) {
+ nbdkit_debug ("%s", str);
+ return;
+ }
+
+ nbdkit_error ("%s", str);
+}
+
/* Defer VDDK initialization until after fork because it is known to
* create background threads from VixDiskLib_InitEx. Unfortunately
* error reporting from this callback is difficult, but we have
diff --git a/plugins/vddk/vddk.h b/plugins/vddk/vddk.h
index 0e3dd79e..d99b6f4b 100644
--- a/plugins/vddk/vddk.h
+++ b/plugins/vddk/vddk.h
@@ -183,6 +183,9 @@ extern pthread_mutex_t stats_lock;
#undef OPTIONAL_STUB
extern void display_stats (void);
+/* utils.c */
+extern void trim (char *str);
+
/* worker.c */
extern const char *command_type_string (enum command_type type);
extern int send_command_and_wait (struct vddk_handle *h, struct command *cmd);
--
2.31.1

245
SOURCES/0019-common-utils-test-vector.c-Add-vector-benchmarks.patch
View File

@ -1,245 +0,0 @@
From 239df6ee9583bc520e9a3e18f0c0d8e58602fb5c Mon Sep 17 00:00:00 2001
From: Nir Soffer <nsoffer@redhat.com>
Date: Fri, 5 Nov 2021 20:36:42 +0200
Subject: [PATCH] common/utils/test-vector.c: Add vector benchmarks
The generic vector reallocs on every append. Add benchmarks to measure
the cost with uint32 vector (used for copying extents) and the effect of
reserving space upfront.
The tests show that realloc is pretty efficient, but calling reserve
before the appends speeds the appends up significantly.
NBDKIT_BENCH=1 ./test-vector
bench_reserve: 1000000 appends in 0.004503 s
bench_append: 1000000 appends in 0.014986 s
The new benchmarks do not run by default to avoid trouble in CI on
overloaded machines or under qemu emulation.
A new target added to run all benchmaks:
make bench
Ported from libnbd:
- commit dc9ae0174ab1384081a57a8d54b10f8147ea6430
- commit f6c06a3b4d87fe976a96ea04f8da1f22b2531dbd
(cherry picked from commit a227af7921c9a51c4f1ab699a3b9f06a9a645126)
---
Makefile.am | 5 +++
README | 7 ++++
common/utils/Makefile.am | 5 ++-
common/utils/bench.h | 72 ++++++++++++++++++++++++++++++++++++++
common/utils/test-vector.c | 55 +++++++++++++++++++++++++++--
5 files changed, 141 insertions(+), 3 deletions(-)
create mode 100644 common/utils/bench.h
diff --git a/Makefile.am b/Makefile.am
index b21d69ed..49f5d91c 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -102,6 +102,11 @@ check-root:
check-vddk:
$(MAKE) -C tests check-vddk
+bench: all
+ @for d in common/utils; do \
+ $(MAKE) -C $$d bench || exit 1; \
+ done
+
#----------------------------------------------------------------------
# Maintainers only!
diff --git a/README b/README
index a04325be..b001620c 100644
--- a/README
+++ b/README
@@ -274,6 +274,13 @@ nbdkit-vddk-plugin against the library like this:
make check-vddk vddkdir=vmware-vix-disklib-distrib
+Running the benchmarks
+----------------------
+
+To run benchmarks:
+
+ make bench
+
DOWNLOAD TARBALLS
=================
diff --git a/common/utils/Makefile.am b/common/utils/Makefile.am
index c33811fc..b2f08cb4 100644
--- a/common/utils/Makefile.am
+++ b/common/utils/Makefile.am
@@ -101,6 +101,9 @@ test_quotes_SOURCES = test-quotes.c quote.c utils.h
test_quotes_CPPFLAGS = -I$(srcdir)
test_quotes_CFLAGS = $(WARNINGS_CFLAGS)
-test_vector_SOURCES = test-vector.c vector.c vector.h
+test_vector_SOURCES = test-vector.c vector.c vector.h bench.h
test_vector_CPPFLAGS = -I$(srcdir)
test_vector_CFLAGS = $(WARNINGS_CFLAGS)
+
+bench: test-vector
+ NBDKIT_BENCH=1 ./test-vector
diff --git a/common/utils/bench.h b/common/utils/bench.h
new file mode 100644
index 00000000..496a3614
--- /dev/null
+++ b/common/utils/bench.h
@@ -0,0 +1,72 @@
+/* libnbd
+ * Copyright (C) 2021 Red Hat Inc.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * * Neither the name of Red Hat nor the names of its contributors may be
+ * used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY RED HAT AND CONTRIBUTORS ''AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+ * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL RED HAT OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
+ * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+ * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#ifndef LIBNBD_BENCH_H
+#define LIBNBD_BENCH_H
+
+#include <sys/time.h>
+
+#define MICROSECONDS 1000000
+
+struct bench {
+ struct timeval start, stop;
+};
+
+static inline void
+bench_start(struct bench *b)
+{
+ gettimeofday (&b->start, NULL);
+}
+
+static inline void
+bench_stop(struct bench *b)
+{
+ gettimeofday (&b->stop, NULL);
+}
+
+static inline double
+bench_sec(struct bench *b)
+{
+ struct timeval dt;
+
+ dt.tv_sec = b->stop.tv_sec - b->start.tv_sec;
+ dt.tv_usec = b->stop.tv_usec - b->start.tv_usec;
+
+ if (dt.tv_usec < 0) {
+ dt.tv_sec -= 1;
+ dt.tv_usec += MICROSECONDS;
+ }
+
+ return ((double)dt.tv_sec * MICROSECONDS + dt.tv_usec) / MICROSECONDS;
+}
+
+#endif /* LIBNBD_BENCH_H */
diff --git a/common/utils/test-vector.c b/common/utils/test-vector.c
index 94b2aeb7..28af59b8 100644
--- a/common/utils/test-vector.c
+++ b/common/utils/test-vector.c
@@ -38,9 +38,13 @@
#undef NDEBUG /* Keep test strong even for nbdkit built without assertions */
#include <assert.h>
+#include "bench.h"
#include "vector.h"
+#define APPENDS 1000000
+
DEFINE_VECTOR_TYPE(int64_vector, int64_t);
+DEFINE_VECTOR_TYPE(uint32_vector, uint32_t);
DEFINE_VECTOR_TYPE(string_vector, char *);
static int
@@ -113,10 +117,57 @@ test_string_vector (void)
free (v.ptr);
}
+static void
+bench_reserve (void)
+{
+ uint32_vector v = empty_vector;
+ struct bench b;
+
+ bench_start(&b);
+
+ uint32_vector_reserve(&v, APPENDS);
+
+ for (uint32_t i = 0; i < APPENDS; i++) {
+ uint32_vector_append (&v, i);
+ }
+
+ bench_stop(&b);
+
+ assert (v.ptr[APPENDS - 1] == APPENDS - 1);
+ free (v.ptr);
+
+ printf ("bench_reserve: %d appends in %.6f s\n", APPENDS, bench_sec (&b));
+}
+
+static void
+bench_append (void)
+{
+ uint32_vector v = empty_vector;
+ struct bench b;
+
+ bench_start(&b);
+
+ for (uint32_t i = 0; i < APPENDS; i++) {
+ uint32_vector_append (&v, i);
+ }
+
+ bench_stop(&b);
+
+ assert (v.ptr[APPENDS - 1] == APPENDS - 1);
+ free (v.ptr);
+
+ printf ("bench_append: %d appends in %.6f s\n", APPENDS, bench_sec (&b));
+}
+
int
main (int argc, char *argv[])
{
- test_int64_vector ();
- test_string_vector ();
+ if (getenv("NBDKIT_BENCH")) {
+ bench_reserve ();
+ bench_append ();
+ } else {
+ test_int64_vector ();
+ test_string_vector ();
+ }
return 0;
}
--
2.31.1

121
SOURCES/0019-rate-Allow-burstiness-to-be-controlled.patch Normal file
View File

@ -0,0 +1,121 @@
From c1a7c87fb9710fb29d699d1f39d0da19caf98da0 Mon Sep 17 00:00:00 2001
From: "Richard W.M. Jones" <rjones@redhat.com>
Date: Sat, 11 Jun 2022 12:34:02 +0100
Subject: [PATCH] rate: Allow burstiness to be controlled
Previously it was fixed at 2.0 seconds. Allowing it to be adjusted
upwards could help with large, lumpy requests.
(cherry picked from commit f79e951c20510381d5cd83c203c670874a4978f4)
---
filters/rate/nbdkit-rate-filter.pod | 12 ++++++++++--
filters/rate/rate.c | 20 +++++++++++++-------
tests/test-rate.sh | 2 +-
3 files changed, 24 insertions(+), 10 deletions(-)
diff --git a/filters/rate/nbdkit-rate-filter.pod b/filters/rate/nbdkit-rate-filter.pod
index 8956e641..09ce7dbc 100644
--- a/filters/rate/nbdkit-rate-filter.pod
+++ b/filters/rate/nbdkit-rate-filter.pod
@@ -9,6 +9,7 @@ nbdkit-rate-filter - limit bandwidth by connection or server
[connection-rate=BITSPERSEC]
[rate-file=FILENAME]
[connection-rate-file=FILENAME]
+ [burstiness=SECS]
=head1 DESCRIPTION
@@ -63,6 +64,13 @@ Limit total bandwidth across all connections to C<BITSPERSEC>.
Adjust the per-connection or total bandwidth dynamically by writing
C<BITSPERSEC> into C<FILENAME>. See L</DYNAMIC ADJUSTMENT> below.
+=item B<burstiness=>SECS
+
+Control the bucket capacity, expressed as a length of time in
+"rate-equivalent seconds" that the client is allowed to burst for
+after a period of inactivity. The default is 2.0 seconds. It's not
+recommended to set this smaller than the default.
+
=back
C<BITSPERSEC> can be specified as a simple number, or you can use a
@@ -105,8 +113,8 @@ If the size of requests made by your client is much larger than the
rate limit then you can see long, lumpy sleeps in this filter. In the
future we may modify the filter to break up large requests
automatically in order to limit the length of sleeps. Placing the
-L<nbdkit-blocksize-filter(1)> in front of this filter may help in the
-meantime.
+L<nbdkit-blocksize-filter(1)> in front of this filter, or adjusting
+C<burstiness> upwards may help.
=head1 FILES
diff --git a/filters/rate/rate.c b/filters/rate/rate.c
index 1a70d212..26082f8c 100644
--- a/filters/rate/rate.c
+++ b/filters/rate/rate.c
@@ -68,10 +68,9 @@ static char *rate_file = NULL;
/* Bucket capacity controls the burst rate. It is expressed as the
* length of time in "rate-equivalent seconds" that the client can
- * burst for after a period of inactivity. This could be adjustable
- * in future.
+ * burst for after a period of inactivity.
*/
-#define BUCKET_CAPACITY 2.0
+static double bucket_capacity = 2.0 /* seconds */;
/* Global read and write buckets. */
static struct bucket read_bucket;
@@ -142,6 +141,13 @@ rate_config (nbdkit_next_config *next, nbdkit_backend *nxdata,
return -1;
return 0;
}
+ else if (strcmp (key, "burstiness") == 0) {
+ if (sscanf (value, "%lg", &bucket_capacity) != 1) {
+ nbdkit_error ("burstiness must be a floating point number (seconds)");
+ return -1;
+ }
+ return 0;
+ }
else
return next (nxdata, key, value);
}
@@ -150,8 +156,8 @@ static int
rate_get_ready (int thread_model)
{
/* Initialize the global buckets. */
- bucket_init (&read_bucket, rate, BUCKET_CAPACITY);
- bucket_init (&write_bucket, rate, BUCKET_CAPACITY);
+ bucket_init (&read_bucket, rate, bucket_capacity);
+ bucket_init (&write_bucket, rate, bucket_capacity);
return 0;
}
@@ -178,8 +184,8 @@ rate_open (nbdkit_next_open *next, nbdkit_context *nxdata,
return NULL;
}
- bucket_init (&h->read_bucket, connection_rate, BUCKET_CAPACITY);
- bucket_init (&h->write_bucket, connection_rate, BUCKET_CAPACITY);
+ bucket_init (&h->read_bucket, connection_rate, bucket_capacity);
+ bucket_init (&h->write_bucket, connection_rate, bucket_capacity);
pthread_mutex_init (&h->read_bucket_lock, NULL);
pthread_mutex_init (&h->write_bucket_lock, NULL);
diff --git a/tests/test-rate.sh b/tests/test-rate.sh
index 7305c928..ff781c21 100755
--- a/tests/test-rate.sh
+++ b/tests/test-rate.sh
@@ -56,7 +56,7 @@ nbdkit -U - \
--filter=blocksize --filter=rate \
pattern 25M \
maxdata=65536 \
- rate=10M \
+ rate=10M burstiness=2.0 \
--run 'nbdcopy "$uri" rate.img'
end_t=$SECONDS
--
2.31.1

54
SOURCES/0020-common-urils-vector.c-Optimize-vector-append.patch
View File

@ -1,54 +0,0 @@
From e544d86c797edec613673c7272f8d4f8b05d87f8 Mon Sep 17 00:00:00 2001
From: Nir Soffer <nsoffer@redhat.com>
Date: Fri, 5 Nov 2021 22:16:26 +0200
Subject: [PATCH] common/urils/vector.c: Optimize vector append
Minimize reallocs by growing the backing array by factor of 1.5.
Testing show that now append() is fast without calling reserve()
upfront, simplifying code using vector.
NBDKIT_BENCH=1 ./test-vector
bench_reserve: 1000000 appends in 0.004496 s
bench_append: 1000000 appends in 0.004180 s
This can make a difference in code appending millions of items.
Ported from libnbd commit 985dfa72ae2e41901f0af21e7205ef85428cd4bd.
(cherry picked from commit 12356fa97a840de19bb61e0abedd6e7c7e578e5a)
---
common/utils/vector.c | 14 ++++++++++++--
1 file changed, 12 insertions(+), 2 deletions(-)
diff --git a/common/utils/vector.c b/common/utils/vector.c
index 00cd2546..7df17e1b 100644
--- a/common/utils/vector.c
+++ b/common/utils/vector.c
@@ -41,11 +41,21 @@ int
generic_vector_reserve (struct generic_vector *v, size_t n, size_t itemsize)
{
void *newptr;
+ size_t reqalloc, newalloc;
- newptr = realloc (v->ptr, (n + v->alloc) * itemsize);
+ reqalloc = v->alloc + n;
+ if (reqalloc < v->alloc)
+ return -1; /* overflow */
+
+ newalloc = (v->alloc * 3 + 1) / 2;
+
+ if (newalloc < reqalloc)
+ newalloc = reqalloc;
+
+ newptr = realloc (v->ptr, newalloc * itemsize);
if (newptr == NULL)
return -1;
v->ptr = newptr;
- v->alloc += n;
+ v->alloc = newalloc;
return 0;
}
--
2.31.1

104
SOURCES/0020-luks-Check-return-values-from-malloc-more-carefully.patch Normal file
View File

@ -0,0 +1,104 @@
From 4e8599886ba4802fef1683811a725e7c4bc4fe72 Mon Sep 17 00:00:00 2001
From: "Richard W.M. Jones" <rjones@redhat.com>
Date: Tue, 12 Jul 2022 18:00:38 +0100
Subject: [PATCH] luks: Check return values from malloc more carefully
Found by Coverity:
Error: GCC_ANALYZER_WARNING (CWE-688): [#def53]
nbdkit-1.30.7/filters/luks/luks-encryption.c: scope_hint: In function 'calculate_iv'
nbdkit-1.30.7/filters/luks/luks-encryption.c:175:5: warning[-Wanalyzer-possible-null-argument]: use of possibly-NULL 'iv' where non-null expected
nbdkit-1.30.7/filters/luks/luks-encryption.c:39: included_from: Included from here.
/usr/include/string.h:43:14: note: argument 1 of 'memcpy' must be non-null
# 173| sector32 = (uint32_t) sector; /* truncate to only lower bits */
# 174| sector32 = htole32 (sector32);
# 175|-> memcpy (iv, &sector32, prefixlen);
# 176| memset (iv + prefixlen, 0, ivlen - prefixlen);
# 177| break;
Error: GCC_ANALYZER_WARNING (CWE-688): [#def54]
nbdkit-1.30.7/filters/luks/luks-encryption.c:184:5: warning[-Wanalyzer-possible-null-argument]: use of possibly-NULL 'iv' where non-null expected
nbdkit-1.30.7/filters/luks/luks-encryption.c:39: included_from: Included from here.
/usr/include/string.h:43:14: note: argument 1 of 'memcpy' must be non-null
# 182| prefixlen = ivlen;
# 183| sector = htole64 (sector);
# 184|-> memcpy (iv, &sector, prefixlen);
# 185| memset (iv + prefixlen, 0, ivlen - prefixlen);
# 186| break;
Error: NULL_RETURNS (CWE-476): [#def55]
nbdkit-1.30.7/filters/luks/luks-encryption.c:498: returned_null: "malloc" returns "NULL" (checked 86 out of 94 times).
nbdkit-1.30.7/filters/luks/luks-encryption.c:498: var_assigned: Assigning: "temp" = "NULL" return value from "malloc".
nbdkit-1.30.7/filters/luks/luks-encryption.c:523: dereference: Dereferencing a pointer that might be "NULL" "temp" when calling "memcpy". [Note: The source code implementation of the function has been overridden by a builtin model.]
# 521| gnutls_hash_deinit (hash, temp);
# 522|
# 523|-> memcpy (&block[i*digest_bytes], temp, blen);
# 524| }
# 525|
Fixes: commit 468919dce6c5eb57503eacac0f67e5dd87c58e6c
(cherry picked from commit 00c8bbd9e321681843140f697985505de7177f34)
---
filters/luks/luks-encryption.c | 28 +++++++++++++++++++++++-----
1 file changed, 23 insertions(+), 5 deletions(-)
diff --git a/filters/luks/luks-encryption.c b/filters/luks/luks-encryption.c
index 8ee0eb35..19aaf06a 100644
--- a/filters/luks/luks-encryption.c
+++ b/filters/luks/luks-encryption.c
@@ -495,9 +495,15 @@ af_hash (gnutls_digest_algorithm_t hash_alg, uint8_t *block, size_t len)
size_t digest_bytes = gnutls_hash_get_len (hash_alg);
size_t nr_blocks, last_block_len;
size_t i;
- CLEANUP_FREE uint8_t *temp = malloc (digest_bytes);
int r;
gnutls_hash_hd_t hash;
+ CLEANUP_FREE uint8_t *temp;
+
+ temp = malloc (digest_bytes);
+ if (!temp) {
+ nbdkit_error ("malloc: %m");
+ return -1;
+ }
nr_blocks = len / digest_bytes;
last_block_len = len % digest_bytes;
@@ -874,9 +880,15 @@ int
do_decrypt (struct luks_data *h, gnutls_cipher_hd_t cipher,
uint64_t sector, uint8_t *buf, size_t nr_sectors)
{
- const size_t ivlen = cipher_alg_iv_len (h->cipher_alg, h->cipher_mode);
- CLEANUP_FREE uint8_t *iv = malloc (ivlen);
int r;
+ const size_t ivlen = cipher_alg_iv_len (h->cipher_alg, h->cipher_mode);
+ CLEANUP_FREE uint8_t *iv;
+
+ iv = malloc (ivlen);
+ if (!iv) {
+ nbdkit_error ("malloc: %m");
+ return -1;
+ }
while (nr_sectors) {
calculate_iv (h->ivgen_alg, iv, ivlen, sector);
@@ -902,9 +914,15 @@ int
do_encrypt (struct luks_data *h, gnutls_cipher_hd_t cipher,
uint64_t sector, uint8_t *buf, size_t nr_sectors)
{
- const size_t ivlen = cipher_alg_iv_len (h->cipher_alg, h->cipher_mode);
- CLEANUP_FREE uint8_t *iv = malloc (ivlen);
int r;
+ const size_t ivlen = cipher_alg_iv_len (h->cipher_alg, h->cipher_mode);
+ CLEANUP_FREE uint8_t *iv;
+
+ iv = malloc (ivlen);
+ if (!iv) {
+ nbdkit_error ("malloc: %m");
+ return -1;
+ }
while (nr_sectors) {
calculate_iv (h->ivgen_alg, iv, ivlen, sector);
--
2.31.1

188
SOURCES/0021-common-utils-vector-Rename-alloc-to-cap.patch
View File

@ -1,188 +0,0 @@
From 24e2694b302f6602e0fc7808a53a766cb983dfb4 Mon Sep 17 00:00:00 2001
From: Nir Soffer <nsoffer@redhat.com>
Date: Fri, 5 Nov 2021 22:59:38 +0200
Subject: [PATCH] common/utils/vector: Rename `alloc` to `cap`
The `alloc` field is the maximum number of items you can append to a
vector before it need to be resized. This may confuse users with the
size of the `ptr` array which is `alloc * itemsize`. Rename to "cap",
common term for this property in many languages (e.g C++, Rust, Go).
Tested with "make check". Tests requiring root or external libraries
(vddk) not tested.
Ported from libnbd commit e3c7f02a2a844295564c832108d36c939c4e4ecf.
(cherry picked from commit 75a44237c4463524dbf7951bb62b59c373c85865)
---
common/allocators/malloc.c | 24 ++++++++++++------------
common/utils/vector.c | 16 ++++++++--------
common/utils/vector.h | 12 ++++++------
plugins/vddk/reexec.c | 2 +-
4 files changed, 27 insertions(+), 27 deletions(-)
diff --git a/common/allocators/malloc.c b/common/allocators/malloc.c
index 59409c24..f7474465 100644
--- a/common/allocators/malloc.c
+++ b/common/allocators/malloc.c
@@ -88,16 +88,16 @@ extend (struct m_alloc *ma, uint64_t new_size)
ACQUIRE_WRLOCK_FOR_CURRENT_SCOPE (&ma->lock);
size_t old_size, n;
- if (ma->ba.alloc < new_size) {
- old_size = ma->ba.alloc;
- n = new_size - ma->ba.alloc;
+ if (ma->ba.cap < new_size) {
+ old_size = ma->ba.cap;
+ n = new_size - ma->ba.cap;
#ifdef HAVE_MUNLOCK
/* Since the memory might be moved by realloc, we must unlock the
* original array.
*/
if (ma->use_mlock)
- munlock (ma->ba.ptr, ma->ba.alloc);
+ munlock (ma->ba.ptr, ma->ba.cap);
#endif
if (bytearray_reserve (&ma->ba, n) == -1) {
@@ -110,7 +110,7 @@ extend (struct m_alloc *ma, uint64_t new_size)
#ifdef HAVE_MLOCK
if (ma->use_mlock) {
- if (mlock (ma->ba.ptr, ma->ba.alloc) == -1) {
+ if (mlock (ma->ba.ptr, ma->ba.cap) == -1) {
nbdkit_error ("allocator=malloc: mlock: %m");
return -1;
}
@@ -138,11 +138,11 @@ m_alloc_read (struct allocator *a, void *buf,
/* Avoid reading beyond the end of the allocated array. Return
* zeroes for that part.
*/
- if (offset >= ma->ba.alloc)
+ if (offset >= ma->ba.cap)
memset (buf, 0, count);
- else if (offset + count > ma->ba.alloc) {
- memcpy (buf, ma->ba.ptr + offset, ma->ba.alloc - offset);
- memset (buf + ma->ba.alloc - offset, 0, offset + count - ma->ba.alloc);
+ else if (offset + count > ma->ba.cap) {
+ memcpy (buf, ma->ba.ptr + offset, ma->ba.cap - offset);
+ memset (buf + ma->ba.cap - offset, 0, offset + count - ma->ba.cap);
}
else
memcpy (buf, ma->ba.ptr + offset, count);
@@ -191,9 +191,9 @@ m_alloc_zero (struct allocator *a, uint64_t count, uint64_t offset)
/* Try to avoid extending the array, since the unallocated part
* always reads as zero.
*/
- if (offset < ma->ba.alloc) {
- if (offset + count > ma->ba.alloc)
- memset (ma->ba.ptr + offset, 0, ma->ba.alloc - offset);
+ if (offset < ma->ba.cap) {
+ if (offset + count > ma->ba.cap)
+ memset (ma->ba.ptr + offset, 0, ma->ba.cap - offset);
else
memset (ma->ba.ptr + offset, 0, count);
}
diff --git a/common/utils/vector.c b/common/utils/vector.c
index 7df17e1b..a4b43ce7 100644
--- a/common/utils/vector.c
+++ b/common/utils/vector.c
@@ -41,21 +41,21 @@ int
generic_vector_reserve (struct generic_vector *v, size_t n, size_t itemsize)
{
void *newptr;
- size_t reqalloc, newalloc;
+ size_t reqcap, newcap;
- reqalloc = v->alloc + n;
- if (reqalloc < v->alloc)
+ reqcap = v->cap + n;
+ if (reqcap < v->cap)
return -1; /* overflow */
- newalloc = (v->alloc * 3 + 1) / 2;
+ newcap = (v->cap * 3 + 1) / 2;
- if (newalloc < reqalloc)
- newalloc = reqalloc;
+ if (newcap < reqcap)
+ newcap = reqcap;
- newptr = realloc (v->ptr, newalloc * itemsize);
+ newptr = realloc (v->ptr, newcap * itemsize);
if (newptr == NULL)
return -1;
v->ptr = newptr;
- v->alloc = newalloc;
+ v->cap = newcap;
return 0;
}
diff --git a/common/utils/vector.h b/common/utils/vector.h
index f6a0af78..782dcba6 100644
--- a/common/utils/vector.h
+++ b/common/utils/vector.h
@@ -86,7 +86,7 @@
struct name { \
type *ptr; /* Pointer to array of items. */ \
size_t size; /* Number of valid items in the array. */ \
- size_t alloc; /* Number of items allocated. */ \
+ size_t cap; /* Maximum number of items. */ \
}; \
typedef struct name name; \
\
@@ -106,7 +106,7 @@
name##_insert (name *v, type elem, size_t i) \
{ \
assert (i <= v->size); \
- if (v->size >= v->alloc) { \
+ if (v->size >= v->cap) { \
if (name##_reserve (v, 1) == -1) return -1; \
} \
memmove (&v->ptr[i+1], &v->ptr[i], (v->size-i) * sizeof (elem)); \
@@ -137,7 +137,7 @@
{ \
free (v->ptr); \
v->ptr = NULL; \
- v->size = v->alloc = 0; \
+ v->size = v->cap = 0; \
} \
\
/* Iterate over the vector, calling f() on each element. */ \
@@ -181,17 +181,17 @@
if (newptr == NULL) return -1; \
memcpy (newptr, vptr, len); \
copy->ptr = newptr; \
- copy->size = copy->alloc = v->size; \
+ copy->size = copy->cap = v->size; \
return 0; \
} \
\
-#define empty_vector { .ptr = NULL, .size = 0, .alloc = 0 }
+#define empty_vector { .ptr = NULL, .size = 0, .cap = 0 }
struct generic_vector {
void *ptr;
size_t size;
- size_t alloc;
+ size_t cap;
};
extern int generic_vector_reserve (struct generic_vector *v,
diff --git a/plugins/vddk/reexec.c b/plugins/vddk/reexec.c
index 46acdb62..9e87025e 100644
--- a/plugins/vddk/reexec.c
+++ b/plugins/vddk/reexec.c
@@ -116,7 +116,7 @@ perform_reexec (const char *env, const char *prepend)
nbdkit_error ("realloc: %m");
exit (EXIT_FAILURE);
}
- r = read (fd, buf.ptr + buf.size, buf.alloc - buf.size);
+ r = read (fd, buf.ptr + buf.size, buf.cap - buf.size);
if (r == -1) {
nbdkit_error ("read: %s: %m", cmdline_file);
exit (EXIT_FAILURE);
--
2.31.1

57
SOURCES/0021-luks-Avoid-potential-overflow-when-computing-key-mat.patch Normal file
View File

@ -0,0 +1,57 @@
From 1d593a76796574845d7e32aaadd9f7d1ed4e7987 Mon Sep 17 00:00:00 2001
From: "Richard W.M. Jones" <rjones@redhat.com>
Date: Tue, 12 Jul 2022 18:07:25 +0100
Subject: [PATCH] luks: Avoid potential overflow when computing key material
offset and length
Found by Coverity:
Error: OVERFLOW_BEFORE_WIDEN (CWE-190): [#def58]
nbdkit-1.30.7/filters/luks/luks-encryption.c:558: overflow_before_widen: Potentially overflowing expression "h->phdr.master_key_len * h->phdr.keyslot[i].stripes" with type "unsigned int" (32 bits, unsigned) is evaluated using 32-bit arithmetic, and then used in a context that expects an expression of type "uint64_t" (64 bits, unsigned).
nbdkit-1.30.7/filters/luks/luks-encryption.c:558: remediation: To avoid overflow, cast either "h->phdr.master_key_len" or "h->phdr.keyslot[i].stripes" to type "uint64_t".
# 556| uint64_t len, r;
# 557|
# 558|-> len = h->phdr.master_key_len * h->phdr.keyslot[i].stripes;
# 559| r = DIV_ROUND_UP (len, LUKS_SECTOR_SIZE);
# 560| r = ROUND_UP (r, LUKS_ALIGN_KEYSLOTS / LUKS_SECTOR_SIZE);
Error: OVERFLOW_BEFORE_WIDEN (CWE-190): [#def62]
nbdkit-1.30.7/filters/luks/luks-encryption.c:616: overflow_before_widen: Potentially overflowing expression "ks->key_material_offset * 512U" with type "unsigned int" (32 bits, unsigned) is evaluated using 32-bit arithmetic, and then used in a context that expects an expression of type "uint64_t" (64 bits, unsigned).
nbdkit-1.30.7/filters/luks/luks-encryption.c:616: remediation: To avoid overflow, cast either "ks->key_material_offset" or "512U" to type "uint64_t".
# 614|
# 615| /* Read master key material from plugin. */
# 616|-> start = ks->key_material_offset * LUKS_SECTOR_SIZE;
# 617| if (next->pread (next, split_key, split_key_len, start, 0, &err) == -1) {
# 618| errno = err;
Fixes: commit 468919dce6c5eb57503eacac0f67e5dd87c58e6c
(cherry picked from commit 808d88fbc7b58b7c95e05f41fec729cba92ef518)
---
filters/luks/luks-encryption.c | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/filters/luks/luks-encryption.c b/filters/luks/luks-encryption.c
index 19aaf06a..06435b27 100644
--- a/filters/luks/luks-encryption.c
+++ b/filters/luks/luks-encryption.c
@@ -561,7 +561,7 @@ key_material_length_in_sectors (struct luks_data *h, size_t i)
{
uint64_t len, r;
- len = h->phdr.master_key_len * h->phdr.keyslot[i].stripes;
+ len = (uint64_t) h->phdr.master_key_len * h->phdr.keyslot[i].stripes;
r = DIV_ROUND_UP (len, LUKS_SECTOR_SIZE);
r = ROUND_UP (r, LUKS_ALIGN_KEYSLOTS / LUKS_SECTOR_SIZE);
return r;
@@ -619,7 +619,7 @@ try_passphrase_in_keyslot (nbdkit_next *next, struct luks_data *h,
}
/* Read master key material from plugin. */
- start = ks->key_material_offset * LUKS_SECTOR_SIZE;
+ start = (uint64_t) ks->key_material_offset * LUKS_SECTOR_SIZE;
if (next->pread (next, split_key, split_key_len, start, 0, &err) == -1) {
errno = err;
return -1;
--
2.31.1

1747
SOURCES/0022-common-utils-vector-Rename-size-to-len.patch
View File

File diff suppressed because it is too large Load Diff

36
SOURCES/0022-luks-Avoid-memory-leak-on-error-path.patch Normal file
View File

@ -0,0 +1,36 @@
From ee25c1be953bf385caf23f96384a9834c1f1c250 Mon Sep 17 00:00:00 2001
From: "Richard W.M. Jones" <rjones@redhat.com>
Date: Tue, 12 Jul 2022 18:10:30 +0100
Subject: [PATCH] luks: Avoid memory leak on error path
Found by Coverity:
Error: CPPCHECK_WARNING (CWE-401): [#def65] [important]
nbdkit-1.30.7/filters/luks/luks-encryption.c:707: error[memleak]: Memory leak: h
# 705| if (memcmp (h->phdr.magic, expected_magic, LUKS_MAGIC_LEN) != 0) {
# 706| nbdkit_error ("this disk does not contain a LUKS header");
# 707|-> return NULL;
# 708| }
# 709| h->phdr.version = be16toh (h->phdr.version);
Fixes: commit 468919dce6c5eb57503eacac0f67e5dd87c58e6c
(cherry picked from commit a345cff137763f105f07bb8942c1bbefd0959cff)
---
filters/luks/luks-encryption.c | 1 +
1 file changed, 1 insertion(+)
diff --git a/filters/luks/luks-encryption.c b/filters/luks/luks-encryption.c
index 06435b27..207a4e46 100644
--- a/filters/luks/luks-encryption.c
+++ b/filters/luks/luks-encryption.c
@@ -710,6 +710,7 @@ load_header (nbdkit_next *next, const char *passphrase)
if (memcmp (h->phdr.magic, expected_magic, LUKS_MAGIC_LEN) != 0) {
nbdkit_error ("this disk does not contain a LUKS header");
+ free (h);
return NULL;
}
h->phdr.version = be16toh (h->phdr.version);
--
2.31.1

32
SOURCES/0023-podwrapper.pl.in-Use-short-commit-date.patch
View File

@ -1,32 +0,0 @@
From 2df98ef35c3b023a44983583f65379793599e57f Mon Sep 17 00:00:00 2001
From: Nir Soffer <nsoffer@redhat.com>
Date: Mon, 8 Nov 2021 19:47:57 +0200
Subject: [PATCH] podwrapper.pl.in: Use short commit date
We can use git short commit date format $cs. Maybe it was not available
when podwrapper.pl was created.
Signed-off-by: Nir Soffer <nsoffer@redhat.com>
(cherry picked from libnbd commit 0306fdcb08e8dc5957a9e344b54200711fca1220)
(cherry picked from commit 7a1e79c6b5ca4adcef47fc0929d25d54610fc417)
---
podwrapper.pl.in | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
diff --git a/podwrapper.pl.in b/podwrapper.pl.in
index abad578d..63c1025a 100755
--- a/podwrapper.pl.in
+++ b/podwrapper.pl.in
@@ -233,8 +233,7 @@ my $date;
my $filename = "$abs_top_srcdir/.git";
if (!$date && -d $filename) {
local $ENV{GIT_DIR} = $filename;
- $_ = `git show -O/dev/null -s --format=%ci`;
- $date = $1 if /^(\d+-\d+-\d+)\s/;
+ $date = `git show -O/dev/null -s --format=%cs`;
}
if (!$date) {
my ($day, $month, $year) = (gmtime($ENV{SOURCE_DATE_EPOCH} || time))[3,4,5];
--
2.31.1

48
SOURCES/0023-tests-Hoist-some-EXTRA_DIST-out-of-automake-conditio.patch Normal file
View File

@ -0,0 +1,48 @@
From 5ccf1068703d300c8b5579b3a6ef0e409b5a713e Mon Sep 17 00:00:00 2001
From: "Richard W.M. Jones" <rjones@redhat.com>
Date: Tue, 19 Jul 2022 11:56:47 +0100
Subject: [PATCH] tests: Hoist some EXTRA_DIST out of automake conditionals
We can fail to add some test files (test.tcl, test.lua) to the tarball
if compiling with those languages disabled, which would cause knock-on
failures when the tarball was used with the languages enabled. We
already fixed this for Ruby etc, this commit fixes it for Tcl and Lua.
(cherry picked from commit 3b6763c82909c95431ff57c2fe9be1b98316b057)
---
tests/Makefile.am | 6 ++++--
1 file changed, 4 insertions(+), 2 deletions(-)
diff --git a/tests/Makefile.am b/tests/Makefile.am
index 0f4b0746..2667be32 100644
--- a/tests/Makefile.am
+++ b/tests/Makefile.am
@@ -1203,10 +1203,11 @@ EXTRA_DIST += \
$(NULL)
# Tcl plugin test.
+EXTRA_DIST += test.tcl
+
if HAVE_TCL
LIBGUESTFS_TESTS += test-tcl
-EXTRA_DIST += test.tcl
test_tcl_SOURCES = test-lang-plugins.c test.h
test_tcl_CFLAGS = \
@@ -1219,10 +1220,11 @@ test_tcl_LDADD = libtest.la $(LIBGUESTFS_LIBS)
endif HAVE_TCL
# Lua plugin test.
+EXTRA_DIST += test.lua
+
if HAVE_LUA
LIBGUESTFS_TESTS += test-lua
-EXTRA_DIST += test.lua
test_lua_SOURCES = test-lang-plugins.c test.h
test_lua_CFLAGS = \
--
2.31.1

89
SOURCES/0024-ocaml-Replace-noalloc-with-noalloc-annotation.patch
View File

@ -1,89 +0,0 @@
From e9f77e9da946c963e4ec5d82dfd144305f79ebb5 Mon Sep 17 00:00:00 2001
From: "Richard W.M. Jones" <rjones@redhat.com>
Date: Tue, 9 Nov 2021 09:07:42 +0000
Subject: [PATCH] ocaml: Replace "noalloc" with [@@noalloc] annotation
This requires OCaml >= 4.03 (released April 2016). The previous
minimum version was 4.02.2.
(cherry picked from commit d15dd73845065cc9ca04aa785e2be994f76bf832)
---
README | 2 +-
plugins/ocaml/NBDKit.ml | 16 ++++++++--------
plugins/ocaml/nbdkit-ocaml-plugin.pod | 5 +----
3 files changed, 10 insertions(+), 13 deletions(-)
diff --git a/README b/README
index b001620c..160856b6 100644
--- a/README
+++ b/README
@@ -155,7 +155,7 @@ For the Python plugin:
For the OCaml plugin:
- - OCaml >= 4.02.2
+ - OCaml >= 4.03
For the Tcl plugin:
diff --git a/plugins/ocaml/NBDKit.ml b/plugins/ocaml/NBDKit.ml
index 75823ba5..4d45cc0c 100644
--- a/plugins/ocaml/NBDKit.ml
+++ b/plugins/ocaml/NBDKit.ml
@@ -152,11 +152,11 @@ let default_callbacks = {
export_description = None;
}
-external set_name : string -> unit = "ocaml_nbdkit_set_name" "noalloc"
-external set_longname : string -> unit = "ocaml_nbdkit_set_longname" "noalloc"
-external set_version : string -> unit = "ocaml_nbdkit_set_version" "noalloc"
-external set_description : string -> unit = "ocaml_nbdkit_set_description" "noalloc"
-external set_config_help : string -> unit = "ocaml_nbdkit_set_config_help" "noalloc"
+external set_name : string -> unit = "ocaml_nbdkit_set_name" [@@noalloc]
+external set_longname : string -> unit = "ocaml_nbdkit_set_longname" [@@noalloc]
+external set_version : string -> unit = "ocaml_nbdkit_set_version" [@@noalloc]
+external set_description : string -> unit = "ocaml_nbdkit_set_description" [@@noalloc]
+external set_config_help : string -> unit = "ocaml_nbdkit_set_config_help" [@@noalloc]
external set_field : string -> 'a -> unit = "ocaml_nbdkit_set_field"
@@ -220,7 +220,7 @@ let register_plugin plugin =
(* Bindings to nbdkit server functions. *)
-external _set_error : int -> unit = "ocaml_nbdkit_set_error" "noalloc"
+external _set_error : int -> unit = "ocaml_nbdkit_set_error" [@@noalloc]
let set_error unix_error =
(* There's an awkward triple translation going on here, because
@@ -250,9 +250,9 @@ external read_password : string -> string = "ocaml_nbdkit_read_password"
external realpath : string -> string = "ocaml_nbdkit_realpath"
external nanosleep : int -> int -> unit = "ocaml_nbdkit_nanosleep"
external export_name : unit -> string = "ocaml_nbdkit_export_name"
-external shutdown : unit -> unit = "ocaml_nbdkit_shutdown" "noalloc"
+external shutdown : unit -> unit = "ocaml_nbdkit_shutdown" [@@noalloc]
-external _debug : string -> unit = "ocaml_nbdkit_debug" "noalloc"
+external _debug : string -> unit = "ocaml_nbdkit_debug" [@@noalloc]
let debug fs =
ksprintf _debug fs
diff --git a/plugins/ocaml/nbdkit-ocaml-plugin.pod b/plugins/ocaml/nbdkit-ocaml-plugin.pod
index 2bd0af25..293f8143 100644
--- a/plugins/ocaml/nbdkit-ocaml-plugin.pod
+++ b/plugins/ocaml/nbdkit-ocaml-plugin.pod
@@ -11,10 +11,7 @@ nbdkit-ocaml-plugin - writing nbdkit plugins in OCaml
=head1 DESCRIPTION
This manual page describes how to write nbdkit plugins in natively
-compiled OCaml code.
-
-Note this requires OCaml E<ge> 4.02.2, which has support for shared
-libraries. See L<http://caml.inria.fr/mantis/view.php?id=6693>
+compiled OCaml code. This requires OCaml E<ge> 4.03.
=head1 WRITING AN OCAML NBDKIT PLUGIN
--
2.31.1

39
SOURCES/0025-vddk-Drop-obsolete-documentation-related-to-thread-m.patch
View File

@ -1,39 +0,0 @@
From 5da14da22c1e26aff24baf41fb2ae0f2832acae1 Mon Sep 17 00:00:00 2001
From: "Richard W.M. Jones" <rjones@redhat.com>
Date: Sat, 27 Nov 2021 16:44:41 +0000
Subject: [PATCH] vddk: Drop obsolete documentation related to thread model
Since commit 1eecf15fc3 ("vddk: Implement parallel thread model") we
have implemented a parallel thread model in this plugin, and thread
handling is believed to be safe and in conformity with the VDDK
documentation. Remove obsolete documentation contradicting this.
Reported-by: Ming Xie
Fixes: commit 1eecf15fc3d8ea253ccec4f5883fdbb9aa6f8c2b
(cherry picked from commit 370ecb711c23f9143c933e13468e11d688d0d651)
---
plugins/vddk/nbdkit-vddk-plugin.pod | 8 --------
1 file changed, 8 deletions(-)
diff --git a/plugins/vddk/nbdkit-vddk-plugin.pod b/plugins/vddk/nbdkit-vddk-plugin.pod
index ce82a734..acec0bd2 100644
--- a/plugins/vddk/nbdkit-vddk-plugin.pod
+++ b/plugins/vddk/nbdkit-vddk-plugin.pod
@@ -452,14 +452,6 @@ sector boundaries. This is because the VDDK Read and Write APIs only
take sector numbers. If your client needs finer granularity, you can
use L<nbdkit-blocksize-filter(1)> with the setting C<minblock=512>.
-=head2 Threads
-
-Handling threads in the VDDK API is complex and does not map well to
-any of the thread models offered by nbdkit (see
-L<nbdkit-plugin(3)/THREADS>). The plugin uses the nbdkit
-C<SERIALIZE_REQUESTS> model, but technically even this is not
-completely safe. This is a subject of future work.
-
=head2 Out of memory errors
In the verbose log you may see errors like:
--
2.31.1

32
SOURCES/0026-Revert-podwrapper.pl.in-Use-short-commit-date.patch
View File

@ -1,32 +0,0 @@
From b986f25be4f013eb02cd327826fa225c8202571e Mon Sep 17 00:00:00 2001
From: "Richard W.M. Jones" <rjones@redhat.com>
Date: Sat, 20 Nov 2021 17:50:25 +0000
Subject: [PATCH] Revert "podwrapper.pl.in: Use short commit date"
This commit breaks man page output because there is an extra newline
after the date which wasn't being removed.
This reverts commit 7a1e79c6b5ca4adcef47fc0929d25d54610fc417.
(cherry picked from commit 750ad5972bb082d188f17f8f71ef1ec0c616c676)
---
podwrapper.pl.in | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/podwrapper.pl.in b/podwrapper.pl.in
index 63c1025a..abad578d 100755
--- a/podwrapper.pl.in
+++ b/podwrapper.pl.in
@@ -233,7 +233,8 @@ my $date;
my $filename = "$abs_top_srcdir/.git";
if (!$date && -d $filename) {
local $ENV{GIT_DIR} = $filename;
- $date = `git show -O/dev/null -s --format=%cs`;
+ $_ = `git show -O/dev/null -s --format=%ci`;
+ $date = $1 if /^(\d+-\d+-\d+)\s/;
}
if (!$date) {
my ($day, $month, $year) = (gmtime($ENV{SOURCE_DATE_EPOCH} || time))[3,4,5];
--
2.31.1

31
SOURCES/0027-Fix-podwrapper.pl.in-Use-short-commit-date.patch
View File

@ -1,31 +0,0 @@
From 0c430f02eec2671155c001c8a1d2f964b42022e5 Mon Sep 17 00:00:00 2001
From: Eric Blake <eblake@redhat.com>
Date: Tue, 30 Nov 2021 12:42:01 -0600
Subject: [PATCH] Fix "podwrapper.pl.in: Use short commit date"
This reverts commit 750ad5972bb082d188f17f8f71ef1ec0c616c676, then
fixes the broken newline as suggested in the thread at
https://listman.redhat.com/archives/libguestfs/2021-November/msg00275.html.
(cherry picked from commit 80036dbb0b8f9e0aab5994d80de6321c2a55c669)
---
podwrapper.pl.in | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
diff --git a/podwrapper.pl.in b/podwrapper.pl.in
index abad578d..6f256ba8 100755
--- a/podwrapper.pl.in
+++ b/podwrapper.pl.in
@@ -233,8 +233,7 @@ my $date;
my $filename = "$abs_top_srcdir/.git";
if (!$date && -d $filename) {
local $ENV{GIT_DIR} = $filename;
- $_ = `git show -O/dev/null -s --format=%ci`;
- $date = $1 if /^(\d+-\d+-\d+)\s/;
+ $date = `git show -O/dev/null -s --format=format:%cs`;
}
if (!$date) {
my ($day, $month, $year) = (gmtime($ENV{SOURCE_DATE_EPOCH} || time))[3,4,5];
--
2.31.1

154
SOURCES/0028-scripts-Add-simple-script-for-automating-VDDK-disk-c.patch
View File

@ -1,154 +0,0 @@
From e00a8f2709fdf238daa195da03d8ea2aec9b05e1 Mon Sep 17 00:00:00 2001
From: "Richard W.M. Jones" <rjones@redhat.com>
Date: Tue, 30 Nov 2021 17:56:02 +0000
Subject: [PATCH] scripts: Add simple script for automating VDDK disk
connections
It's tedious to work out how to do this by hand every time. Include a
developer script to make connecting to a guest disk easy.
(cherry picked from commit 44ee90ee01677032a14d5b71118b7af0651db3d5)
---
.gitignore | 1 +
Makefile.am | 2 +-
configure.ac | 2 +
scripts/vddk-open.sh.in | 89 +++++++++++++++++++++++++++++++++++++++++
4 files changed, 93 insertions(+), 1 deletion(-)
create mode 100755 scripts/vddk-open.sh.in
diff --git a/.gitignore b/.gitignore
index 847b72dd..6565600f 100644
--- a/.gitignore
+++ b/.gitignore
@@ -89,6 +89,7 @@ plugins/*/*.3
/plugins/S3/nbdkit-S3-plugin
/plugins/tmpdisk/default-command.c
/podwrapper.pl
+/scripts/vddk-open.sh
/server/libnbdkit.a
/server/local/nbdkit.pc
/server/nbdkit
diff --git a/Makefile.am b/Makefile.am
index 49f5d91c..6df5eba0 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -46,7 +46,7 @@ EXTRA_DIST = \
SECURITY \
$(NULL)
-CLEANFILES += html/*.html
+CLEANFILES += html/*.html scripts/*~
if !ENABLE_LIBFUZZER
# NB: This is not the real nbdkit binary. It's a wrapper that allows
diff --git a/configure.ac b/configure.ac
index 1b737fc1..08c307e9 100644
--- a/configure.ac
+++ b/configure.ac
@@ -1249,6 +1249,8 @@ dnl Produce output files.
AC_CONFIG_HEADERS([config.h])
AC_CONFIG_FILES([podwrapper.pl],
[chmod +x,-w podwrapper.pl])
+AC_CONFIG_FILES([scripts/vddk-open.sh],
+ [chmod +x,-w scripts/vddk-open.sh])
AC_CONFIG_FILES([common/protocol/generate-protostrings.sh],
[chmod +x,-w common/protocol/generate-protostrings.sh])
AC_CONFIG_FILES([Makefile
diff --git a/scripts/vddk-open.sh.in b/scripts/vddk-open.sh.in
new file mode 100755
index 00000000..218bc93c
--- /dev/null
+++ b/scripts/vddk-open.sh.in
@@ -0,0 +1,89 @@
+#!/bin/bash -
+# @configure_input@
+# Copyright (C) 2013-2021 Red Hat Inc.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#
+# * Redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in the
+# documentation and/or other materials provided with the distribution.
+#
+# * Neither the name of Red Hat nor the names of its contributors may be
+# used to endorse or promote products derived from this software without
+# specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY RED HAT AND CONTRIBUTORS ''AS IS'' AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+# PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL RED HAT OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
+# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+# SUCH DAMAGE.
+
+# Open an nbdkit-vddk-plugin connection to the first disk of a guest
+# on a VMware ESXi server. This script automates the tedious bits of
+# getting the disk name, moref, etc. However please read the
+# nbdkit-vddk-plugin documentation as well.
+#
+# Usage:
+# scripts/vddk-open.sh SERVER GUEST -r -f -v libdir=/path/to/vmware-vix-disklib-distrib [...]
+#
+# where SERVER is the hostname or IP address of the ESXi server and
+# GUEST is the name of the guest.
+#
+# These two required parameters are followed by any extra nbdkit
+# parameters you want to use, such as VDDK libdir, flags, filters etc.
+#
+# Note that the script runs ./nbdkit (ie. the wrapper in the top build
+# directory).
+
+nbdkit="@abs_top_builddir@/nbdkit"
+
+server="$1"
+guest="$2"
+shift 2
+
+# Get the libvirt XML, filename and moref.
+echo -n "root password? "
+xml="$( virsh -c "esx://root@$server/?no_verify=1" dumpxml "$guest" )"
+echo
+
+file="$( echo "$xml" | grep '<source file=' | head -1 |
+ sed -e "s/.*'\(.*\)'.*/\1/" )"
+moref="$( echo "$xml" | grep '<vmware:moref' |
+ sed -e 's,.*>\(.*\)<.*,\1,' )"
+
+#echo file="$file"
+#echo moref="$moref"
+
+# Get the thumbprint.
+thumbprint="$( openssl s_client -connect "$server:443" </dev/null 2>/dev/null |
+ openssl x509 -in /dev/stdin -fingerprint -sha1 -noout 2>/dev/null |
+ grep '^sha1 Fingerprint=' |
+ sed 's/.*Fingerprint=\([A-F0-9:]\+\)/\1/' )"
+
+#echo thumbprint="$thumbprint"
+
+# Construct the nbdkit command line.
+declare -a args
+
+args[${#args[@]}]="$nbdkit"
+args[${#args[@]}]="vddk"
+args[${#args[@]}]="file=$file"
+args[${#args[@]}]="vm=moref=$moref"
+args[${#args[@]}]="server=$server"
+args[${#args[@]}]="thumbprint=$thumbprint"
+args[${#args[@]}]="user=root"
+
+echo "${args[@]}" "$@"
+"${args[@]}" "$@"
--
2.31.1

181
SOURCES/0029-file-Fix-implementation-of-cache-none-for-writes.patch
View File

@ -1,181 +0,0 @@
From 5cb4adb94a6ff4325205fea3512c037c91579263 Mon Sep 17 00:00:00 2001
From: "Richard W.M. Jones" <rjones@redhat.com>
Date: Tue, 7 Dec 2021 21:08:26 +0000
Subject: [PATCH] file: Fix implementation of cache=none for writes
When testing virt-v2v we found that cache=none had very pessimal
performance in its current implementation when writing. See:
https://github.com/libguestfs/virt-v2v/commit/ac59d3b2310511b1537d408b675b19ec9a5d384e
However we know of a much better implementation - the one in nbdcopy.
This commit copies that implementation (for writes only).
A simple test is to do:
$ ./nbdkit file out.img cache=none --run 'nbdcopy fedora-33.img $uri'
and then check the cache usage of the output file, which should be
around 0% (using https://github.com/Feh/nocache):
$ cachestats out.img
pages in cache: 409/1572864 (0.0%) [filesize=6291456.0K, pagesize=4K]
For modular virt-v2v doing a local disk to local disk conversion:
- before this change, without cache=none
virt-v2v took 93.7 seconds, 19.1% pages cached in output file
- before this change, enabling cache=none
virt-v2v took 125.4 seconds, 0.0% pages cached in output file
^^^ this is the bad case which caused the investigation
- after this change, without cache=none
virt-v2v took 93.2 seconds, 19.1% pages cached in output file
- after this change, enabling cache=none
virt-v2v took 97.9 seconds, 0.1% pages cached in output file
I tried to adjust NR_WINDOWS to find an optimum. Increasing it made
no difference in performance but predictably caused a slight increase
in cached pages. Reducing it slowed performance slightly. So I
conclude that 8 is about right, but it probably depends on the
hardware.
(cherry picked from commit a956e2e75d6c88eeefecd967505667c9f176e3af)
---
plugins/file/file.c | 79 +++++++++++++++++++++++++----
plugins/file/nbdkit-file-plugin.pod | 3 ++
2 files changed, 72 insertions(+), 10 deletions(-)
diff --git a/plugins/file/file.c b/plugins/file/file.c
index 35270a24..caf24b2c 100644
--- a/plugins/file/file.c
+++ b/plugins/file/file.c
@@ -85,6 +85,69 @@ static int fadvise_mode =
/* cache mode */
static enum { cache_default, cache_none } cache_mode = cache_default;
+/* Define EVICT_WRITES if we are going to evict the page cache
+ * (cache=none) after writing. This is only known to work on Linux.
+ */
+#ifdef __linux__
+#define EVICT_WRITES 1
+#endif
+
+#ifdef EVICT_WRITES
+/* Queue writes so they will be evicted from the cache. See
+ * libnbd.git copy/file-ops.c for the rationale behind this.
+ */
+#define NR_WINDOWS 8
+
+struct write_window {
+ int fd;
+ uint64_t offset;
+ size_t len;
+};
+
+static pthread_mutex_t window_lock = PTHREAD_MUTEX_INITIALIZER;
+static struct write_window window[NR_WINDOWS];
+
+static void
+evict_writes (int fd, uint64_t offset, size_t len)
+{
+ ACQUIRE_LOCK_FOR_CURRENT_SCOPE (&window_lock);
+
+ /* Evict the oldest window from the page cache. */
+ if (window[0].len > 0) {
+ sync_file_range (window[0].fd, window[0].offset, window[0].len,
+ SYNC_FILE_RANGE_WAIT_BEFORE|SYNC_FILE_RANGE_WRITE|
+ SYNC_FILE_RANGE_WAIT_AFTER);
+ posix_fadvise (window[0].fd, window[0].offset, window[0].len,
+ POSIX_FADV_DONTNEED);
+ }
+
+ /* Move the Nth window to N-1. */
+ memmove (&window[0], &window[1], sizeof window[0] * (NR_WINDOWS-1));
+
+ /* Set up the current window and tell Linux to start writing it out
+ * to disk (asynchronously).
+ */
+ sync_file_range (fd, offset, len, SYNC_FILE_RANGE_WRITE);
+ window[NR_WINDOWS-1].fd = fd;
+ window[NR_WINDOWS-1].offset = offset;
+ window[NR_WINDOWS-1].len = len;
+}
+
+/* When we close the handle we must remove any windows which are still
+ * associated. They missed the boat, oh well :-(
+ */
+static void
+remove_fd_from_window (int fd)
+{
+ ACQUIRE_LOCK_FOR_CURRENT_SCOPE (&window_lock);
+ size_t i;
+
+ for (i = 0; i < NR_WINDOWS; ++i)
+ if (window[i].len > 0 && window[i].fd == fd)
+ window[i].len = 0;
+}
+#endif /* EVICT_WRITES */
+
/* Any callbacks using lseek must be protected by this lock. */
static pthread_mutex_t lseek_lock = PTHREAD_MUTEX_INITIALIZER;
@@ -431,6 +494,9 @@ file_close (void *handle)
{
struct handle *h = handle;
+#ifdef EVICT_WRITES
+ remove_fd_from_window (h->fd);
+#endif
close (h->fd);
free (h);
}
@@ -583,15 +649,9 @@ file_pwrite (void *handle, const void *buf, uint32_t count, uint64_t offset,
{
struct handle *h = handle;
-#if defined (HAVE_POSIX_FADVISE) && defined (POSIX_FADV_DONTNEED)
+#if EVICT_WRITES
uint32_t orig_count = count;
uint64_t orig_offset = offset;
-
- /* If cache=none we want to force pages we have just written to the
- * file to be flushed to disk so we can immediately evict them from
- * the page cache.
- */
- if (cache_mode == cache_none) flags |= NBDKIT_FLAG_FUA;
#endif
while (count > 0) {
@@ -608,10 +668,9 @@ file_pwrite (void *handle, const void *buf, uint32_t count, uint64_t offset,
if ((flags & NBDKIT_FLAG_FUA) && file_flush (handle, 0) == -1)
return -1;
-#if defined (HAVE_POSIX_FADVISE) && defined (POSIX_FADV_DONTNEED)
- /* On Linux this will evict the pages we just wrote from the page cache. */
+#if EVICT_WRITES
if (cache_mode == cache_none)
- posix_fadvise (h->fd, orig_offset, orig_count, POSIX_FADV_DONTNEED);
+ evict_writes (h->fd, orig_offset, orig_count);
#endif
return 0;
diff --git a/plugins/file/nbdkit-file-plugin.pod b/plugins/file/nbdkit-file-plugin.pod
index 0ac0ee53..f8f0e198 100644
--- a/plugins/file/nbdkit-file-plugin.pod
+++ b/plugins/file/nbdkit-file-plugin.pod
@@ -117,6 +117,9 @@ cache:
nbdkit file disk.img fadvise=sequential cache=none
+Only use fadvise=sequential if reading, and the reads are mainly
+sequential.
+
=head2 Files on tmpfs
If you want to expose a file that resides on a file system known to
--
2.31.1

95
SOURCES/0030-tests-Add-configure-disable-libguestfs-tests-flag.patch
View File

@ -1,95 +0,0 @@
From 92773e6852719354a136d31519948436f9adf7e9 Mon Sep 17 00:00:00 2001
From: "Richard W.M. Jones" <rjones@redhat.com>
Date: Sat, 18 Dec 2021 20:31:10 +0000
Subject: [PATCH] tests: Add configure --disable-libguestfs-tests flag
This can be used to disable tests which need libguestfs*. We were
already doing that in a hackish way in the Fedora build on some
architectures. This makes it more supportable.
Note that you can use
./configure --enable-libguestfs --disable-libguestfs-tests
to enable the bindings but disable the tests.
The difference between without and with the new flag on an otherwise
fully configured Fedora machine:
# TOTAL: 286
# PASS: 273
# SKIP: 13
# TOTAL: 263
# PASS: 251
# SKIP: 12
* except for those which directly test for requirements using
expressions like:
requires guestfish --version
(cherry picked from commit c09ae98ff3b4b786565de4aa173274531a753d30)
---
configure.ac | 17 ++++++++++++++++-
tests/Makefile.am | 2 ++
2 files changed, 18 insertions(+), 1 deletion(-)
diff --git a/configure.ac b/configure.ac
index 08c307e9..96d738d9 100644
--- a/configure.ac
+++ b/configure.ac
@@ -1146,7 +1146,8 @@ AS_IF([test "$with_libzstd" != "no"],[
])
AM_CONDITIONAL([HAVE_LIBZSTD],[test "x$LIBZSTD_LIBS" != "x"])
-dnl Check for libguestfs (only for the guestfs plugin and the test suite).
+dnl Check for libguestfs (only for the guestfs plugin and parts of
+dnl the test suite).
AC_ARG_WITH([libguestfs],
[AS_HELP_STRING([--without-libguestfs],
[disable guestfs plugin and tests @<:@default=check@:>@])],
@@ -1173,6 +1174,17 @@ AS_IF([test "$with_libguestfs" != "no"],[
])
AM_CONDITIONAL([HAVE_LIBGUESTFS],[test "x$LIBGUESTFS_LIBS" != "x"])
+dnl Disable tests which need libguestfs.
+AC_ARG_ENABLE([libguestfs-tests],
+ [AS_HELP_STRING([--disable-libguestfs-tests],
+ [disable tests which need libguestfs])],
+ [],
+ [enable_libguestfs_tests=check]
+)
+AM_CONDITIONAL([USE_LIBGUESTFS_FOR_TESTS],
+ [test "x$LIBGUESTFS_LIBS" != "x" && \
+ test "x$enable_libguestfs_tests" != "xno"])
+
dnl Check for ext2fs and com_err, for the ext2 filter.
AC_ARG_WITH([ext2],
[AS_HELP_STRING([--without-ext2],
@@ -1447,6 +1459,9 @@ echo "Other optional features:"
echo
feature "allocator=zstd ......................... " \
test "x$HAVE_LIBZSTD_TRUE" = "x"
+feature "tests using libguestfs ................. " \
+ test "x$HAVE_LIBGUESTFS_TRUE" = "x" -a \
+ "x$USE_LIBGUESTFS_FOR_TESTS_TRUE" = "x"
echo
echo "If any optional component is configured ‘no’ when you expected ‘yes’"
diff --git a/tests/Makefile.am b/tests/Makefile.am
index 2b7ae9f3..43b60943 100644
--- a/tests/Makefile.am
+++ b/tests/Makefile.am
@@ -1888,6 +1888,8 @@ TESTS += $(LIBNBD_TESTS)
endif HAVE_LIBNBD
if HAVE_LIBGUESTFS
+if USE_LIBGUESTFS_FOR_TESTS
check_PROGRAMS += $(LIBGUESTFS_TESTS)
TESTS += $(LIBGUESTFS_TESTS)
+endif USE_LIBGUESTFS_FOR_TESTS
endif HAVE_LIBGUESTFS
--
2.31.1

538
SOURCES/0031-vddk-Implement-VMDK-creation.patch
View File

@ -1,538 +0,0 @@
From cf58241f19ed179e48c53f4d6c71df47dd2f5931 Mon Sep 17 00:00:00 2001
From: "Richard W.M. Jones" <rjones@redhat.com>
Date: Tue, 18 Jan 2022 08:58:15 +0000
Subject: [PATCH] vddk: Implement VMDK creation
Add the create=(true|false) parameter. Setting this to true causes
the VMDK local file to be created. Currently this is done on first
connection, but we might change that in future. Various other
parameters can be used to control aspects of the VMDK file.
(cherry picked from commit a39d5773afc3ebab7e5768118a2bccb89a654585)
---
plugins/vddk/nbdkit-vddk-plugin.pod | 102 ++++++++++++++++++++++-
plugins/vddk/vddk-structs.h | 32 +++++++
plugins/vddk/vddk-stubs.h | 7 ++
plugins/vddk/vddk.c | 125 ++++++++++++++++++++++++++++
plugins/vddk/vddk.h | 5 ++
tests/Makefile.am | 6 +-
tests/dummy-vddk.c | 10 +++
tests/test-vddk-real-create.sh | 70 ++++++++++++++++
8 files changed, 354 insertions(+), 3 deletions(-)
create mode 100755 tests/test-vddk-real-create.sh
diff --git a/plugins/vddk/nbdkit-vddk-plugin.pod b/plugins/vddk/nbdkit-vddk-plugin.pod
index acec0bd2..b96192d0 100644
--- a/plugins/vddk/nbdkit-vddk-plugin.pod
+++ b/plugins/vddk/nbdkit-vddk-plugin.pod
@@ -6,7 +6,11 @@ nbdkit-vddk-plugin - nbdkit VMware VDDK plugin
nbdkit vddk [file=]FILENAME
[compression=none|zlib|fastlz|skipz]
- [config=FILENAME] [cookie=COOKIE] [libdir=LIBRARY]
+ [config=FILENAME] [cookie=COOKIE]
+ [create=true] [create-adapter-type=ide|scsi-buslogic|...]
+ [create-hwversion=workstation4|workstation5|...]
+ [create-size=...] [create-type=monolithic-sparse|...]
+ [libdir=LIBRARY]
[nfchostport=PORT] [single-link=true]
[password=PASSWORD | password=- | password=+FILENAME |
password=-FD]
@@ -26,7 +30,7 @@ yourself (see L</LIBRARY LOCATION> below).
=head1 EXAMPLES
-=head2 Open a local VMDK file
+=head2 Open an existing local VMDK file
nbdkit vddk /absolute/path/to/file.vmdk
@@ -38,6 +42,18 @@ I<-r> option):
nbdkit -r vddk /absolute/path/to/file.vmdk
+=head2 Create a new local VMDK file
+
+You can use VDDK to create a VMDK file and fill it with the contents
+of a disk image. Note the C<create-size> parameter is the virtual
+size of the final VMDK disk image and must be at least as large as the
+input disk:
+
+ nbdkit -U - vddk \
+ /absolute/path/to/output.vmdk \
+ create=1 create-size=100M \
+ --run 'qemu-img convert input.qcow2 $uri'
+
=head2 Open a file on a remote VMware ESXi hypervisor
Connect directly to a VMware ESXi hypervisor and export a particular
@@ -136,6 +152,88 @@ C<VIXDISKLIB_CRED_SESSIONID> which can improve performance. The
cookie can be found by connecting to a VCenter Server over HTTPS and
retrieving the C<vmware_soap_session> cookie.
+=item B<create=true>
+
+(nbdkit E<ge> 1.30)
+
+Create a new, local VMDK file. Instead of opening an existing VMDK
+file, a new VMDK file is created and opened. The filename is given by
+the C<file> parameter (see below). The file must not exist already.
+It is not possible to create a remote file using nbdkit.
+
+If this is used, the C<create-size> parameter is required to specify
+the virtual size of the disk. Other C<create-*> parameters (see
+below) can be used to control the VMDK sub-format.
+
+=item B<create-adapter-type=ide>
+
+=item B<create-adapter-type=scsi-buslogic>
+
+=item B<create-adapter-type=scsi-lsilogic>
+
+(nbdkit E<ge> 1.30)
+
+Specify the VMDK disk adapter type. The default is C<scsi-buslogic>.
+
+=item B<create-hwversion=workstation4>
+
+=item B<create-hwversion=workstation5>
+
+=item B<create-hwversion=workstation6>
+
+=item B<create-hwversion=esx30>
+
+=item B<create-hwversion=esx4x>
+
+=item B<create-hwversion=esx50>
+
+=item B<create-hwversion=esx51>
+
+=item B<create-hwversion=esx55>
+
+=item B<create-hwversion=esx60>
+
+=item B<create-hwversion=esx65>
+
+(nbdkit E<ge> 1.30)
+
+Specify the VMDK virtual hardware version. The default is
+C<workstation5>.
+
+=item B<create-size=>SIZE
+
+(nbdkit E<ge> 1.30)
+
+Specify the virtual size of the created disk. The C<SIZE> can use
+modifiers like C<100M> etc. It must be a multiple of 512 bytes
+because VMware only supports sector sizes.
+
+If you use C<create=true> then this parameter is required.
+
+=item B<create-type=monolithic-sparse>
+
+=item B<create-type=monolithic-flat>
+
+=item B<create-type=split-sparse>
+
+=item B<create-type=split-flat>
+
+=item B<create-type=vmfs-flat>
+
+=item B<create-type=stream-optimized>
+
+=item B<create-type=vmfs-thin>
+
+=item B<create-type=vmfs-sparse>
+
+(nbdkit E<ge> 1.30)
+
+Specify the VMDK sub-format. The default is C<monolithic-sparse>.
+
+Some VMDK sub-formats use multiple files, where the C<file> parameter
+specifies the "Disk Descriptor File" and the disk contents are stored
+in adjacent files.
+
=item [B<file=>]FILENAME
=item [B<file=>]B<[>datastoreB<] >vmname/vmnameB<.vmdk>
diff --git a/plugins/vddk/vddk-structs.h b/plugins/vddk/vddk-structs.h
index e97f017c..799c4aec 100644
--- a/plugins/vddk/vddk-structs.h
+++ b/plugins/vddk/vddk-structs.h
@@ -43,6 +43,7 @@
typedef uint64_t VixError;
#define VIX_OK 0
+#define VIX_E_NOT_SUPPORTED 6
#define VIX_ASYNC 25000
#define VIXDISKLIB_FLAG_OPEN_UNBUFFERED 1
@@ -54,6 +55,28 @@ typedef uint64_t VixError;
#define VIXDISKLIB_SECTOR_SIZE 512
+enum VixDiskLibDiskType {
+ VIXDISKLIB_DISK_MONOLITHIC_SPARSE = 1,
+ VIXDISKLIB_DISK_MONOLITHIC_FLAT = 2,
+ VIXDISKLIB_DISK_SPLIT_SPARSE = 3,
+ VIXDISKLIB_DISK_SPLIT_FLAT = 4,
+ VIXDISKLIB_DISK_VMFS_FLAT = 5,
+ VIXDISKLIB_DISK_STREAM_OPTIMIZED = 6,
+ VIXDISKLIB_DISK_VMFS_THIN = 7,
+ VIXDISKLIB_DISK_VMFS_SPARSE = 8
+};
+
+#define VIXDISKLIB_HWVERSION_WORKSTATION_4 3
+#define VIXDISKLIB_HWVERSION_WORKSTATION_5 4
+#define VIXDISKLIB_HWVERSION_WORKSTATION_6 6
+#define VIXDISKLIB_HWVERSION_ESX30 4
+#define VIXDISKLIB_HWVERSION_ESX4X 7
+#define VIXDISKLIB_HWVERSION_ESX50 8
+#define VIXDISKLIB_HWVERSION_ESX51 9
+#define VIXDISKLIB_HWVERSION_ESX55 10
+#define VIXDISKLIB_HWVERSION_ESX60 11
+#define VIXDISKLIB_HWVERSION_ESX65 13
+
#define VIXDISKLIB_MIN_CHUNK_SIZE 128
#define VIXDISKLIB_MAX_CHUNK_NUMBER (512*1024)
@@ -148,4 +171,13 @@ typedef struct {
VixDiskLibBlock blocks[1];
} VixDiskLibBlockList;
+typedef struct {
+ enum VixDiskLibDiskType diskType;
+ enum VixDiskLibAdapterType adapterType;
+ uint16_t hwVersion;
+ uint64_t capacity;
+ uint32_t logicalSectorSize;
+ uint32_t physicalSectorSize;
+} VixDiskLibCreateParams;
+
#endif /* NBDKIT_VDDK_STRUCTS_H */
diff --git a/plugins/vddk/vddk-stubs.h b/plugins/vddk/vddk-stubs.h
index 7d8644c3..d5affa10 100644
--- a/plugins/vddk/vddk-stubs.h
+++ b/plugins/vddk/vddk-stubs.h
@@ -99,6 +99,13 @@ STUB (VixDiskLib_Write,
(VixDiskLibHandle handle,
uint64_t start_sector, uint64_t nr_sectors,
const unsigned char *buf));
+STUB (VixDiskLib_Create,
+ VixError,
+ (const VixDiskLibConnection connection,
+ const char *path,
+ const VixDiskLibCreateParams *create_params,
+ void *progress_function_unused,
+ void *progress_data_unused));
/* Added in VDDK 6.0. */
STUB (VixDiskLib_Flush,
diff --git a/plugins/vddk/vddk.c b/plugins/vddk/vddk.c
index 31e5e23b..5ebf9a2c 100644
--- a/plugins/vddk/vddk.c
+++ b/plugins/vddk/vddk.c
@@ -81,6 +81,14 @@ bool is_remote; /* true if remote connection */
enum compression_type compression; /* compression */
char *config; /* config */
const char *cookie; /* cookie */
+bool create; /* create */
+enum VixDiskLibAdapterType create_adapter_type =
+ VIXDISKLIB_ADAPTER_SCSI_BUSLOGIC; /* create-adapter-type */
+uint16_t create_hwversion =
+ VIXDISKLIB_HWVERSION_WORKSTATION_5; /* create-hwversion */
+uint64_t create_size; /* create-size */
+enum VixDiskLibDiskType create_type =
+ VIXDISKLIB_DISK_MONOLITHIC_SPARSE; /* create-type */
const char *filename; /* file */
char *libdir; /* libdir */
uint16_t nfc_host_port; /* nfchostport */
@@ -119,6 +127,7 @@ static int
vddk_config (const char *key, const char *value)
{
int r;
+ int64_t r64;
if (strcmp (key, "compression") == 0) {
if (strcmp (value, "zlib") == 0)
@@ -144,6 +153,82 @@ vddk_config (const char *key, const char *value)
else if (strcmp (key, "cookie") == 0) {
cookie = value;
}
+ else if (strcmp (key, "create") == 0) {
+ r = nbdkit_parse_bool (value);
+ if (r == -1)
+ return -1;
+ create = r;
+ }
+ else if (strcmp (key, "create-adapter-type") == 0) {
+ if (strcmp (value, "ide") == 0)
+ create_adapter_type = VIXDISKLIB_ADAPTER_IDE;
+ else if (strcmp (value, "scsi-buslogic") == 0)
+ create_adapter_type = VIXDISKLIB_ADAPTER_SCSI_BUSLOGIC;
+ else if (strcmp (value, "scsi-lsilogic") == 0)
+ create_adapter_type = VIXDISKLIB_ADAPTER_SCSI_LSILOGIC;
+ else {
+ nbdkit_error ("unknown create-adapter-type: %s", value);
+ return -1;
+ }
+ }
+ else if (strcmp (key, "create-hwversion") == 0) {
+ if (strcmp (value, "workstation4") == 0)
+ create_hwversion = VIXDISKLIB_HWVERSION_WORKSTATION_4;
+ else if (strcmp (value, "workstation5") == 0)
+ create_hwversion = VIXDISKLIB_HWVERSION_WORKSTATION_5;
+ else if (strcmp (value, "workstation6") == 0)
+ create_hwversion = VIXDISKLIB_HWVERSION_WORKSTATION_6;
+ else if (strcmp (value, "esx30") == 0)
+ create_hwversion = VIXDISKLIB_HWVERSION_ESX30;
+ else if (strcmp (value, "esx4x") == 0)
+ create_hwversion = VIXDISKLIB_HWVERSION_ESX4X;
+ else if (strcmp (value, "esx50") == 0)
+ create_hwversion = VIXDISKLIB_HWVERSION_ESX50;
+ else if (strcmp (value, "esx51") == 0)
+ create_hwversion = VIXDISKLIB_HWVERSION_ESX51;
+ else if (strcmp (value, "esx55") == 0)
+ create_hwversion = VIXDISKLIB_HWVERSION_ESX55;
+ else if (strcmp (value, "esx60") == 0)
+ create_hwversion = VIXDISKLIB_HWVERSION_ESX60;
+ else if (strcmp (value, "esx65") == 0)
+ create_hwversion = VIXDISKLIB_HWVERSION_ESX65;
+ else {
+ nbdkit_error ("unknown create-hwversion: %s", value);
+ return -1;
+ }
+ }
+ else if (strcmp (key, "create-size") == 0) {
+ r64 = nbdkit_parse_size (value);
+ if (r64 == -1)
+ return -1;
+ if (r64 <= 0 || (r64 & 511) != 0) {
+ nbdkit_error ("create-size must be greater than zero and a multiple of 512");
+ return -1;
+ }
+ create_size = r64;
+ }
+ else if (strcmp (key, "create-type") == 0) {
+ if (strcmp (value, "monolithic-sparse") == 0)
+ create_type = VIXDISKLIB_DISK_MONOLITHIC_SPARSE;
+ else if (strcmp (value, "monolithic-flat") == 0)
+ create_type = VIXDISKLIB_DISK_MONOLITHIC_FLAT;
+ else if (strcmp (value, "split-sparse") == 0)
+ create_type = VIXDISKLIB_DISK_SPLIT_SPARSE;
+ else if (strcmp (value, "split-flat") == 0)
+ create_type = VIXDISKLIB_DISK_SPLIT_FLAT;
+ else if (strcmp (value, "vmfs-flat") == 0)
+ create_type = VIXDISKLIB_DISK_VMFS_FLAT;
+ else if (strcmp (value, "stream-optimized") == 0)
+ create_type = VIXDISKLIB_DISK_STREAM_OPTIMIZED;
+ else if (strcmp (value, "vmfs-thin") == 0)
+ create_type = VIXDISKLIB_DISK_VMFS_THIN;
+ else if (strcmp (value, "vmfs-sparse") == 0)
+ create_type = VIXDISKLIB_DISK_VMFS_SPARSE;
+ else {
+ nbdkit_error ("unknown create-type: %s", value);
+ return -1;
+ }
+ }
else if (strcmp (key, "file") == 0) {
/* NB: Don't convert this to an absolute path, because in the
* remote case this can be a path located on the VMware server.
@@ -266,6 +351,18 @@ vddk_config_complete (void)
#undef missing
}
+ if (create) {
+ if (is_remote) {
+ nbdkit_error ("create=true can only be used to create local VMDK files");
+ return -1;
+ }
+
+ if (create_size == 0) {
+ nbdkit_error ("if using create=true you must specify the size using the create-size parameter");
+ return -1;
+ }
+ }
+
/* Restore original LD_LIBRARY_PATH after reexec. */
if (restore_ld_library_path () == -1)
return -1;
@@ -618,6 +715,34 @@ vddk_open (int readonly)
goto err1;
}
+ /* Creating a disk? The first time the connection is opened we will
+ * create it here (we need h->connection). Then set create=false so
+ * we don't create it again. This is all serialized through
+ * open_close_lock so it is safe.
+ */
+ if (create) {
+ VixDiskLibCreateParams cparams = {
+ .diskType = create_type,
+ .adapterType = create_adapter_type,
+ .hwVersion = create_hwversion,
+ .capacity = create_size / VIXDISKLIB_SECTOR_SIZE,
+ .logicalSectorSize = 0,
+ .physicalSectorSize = 0
+ };
+
+ VDDK_CALL_START (VixDiskLib_Create,
+ "h->connection, %s, &cparams, NULL, NULL",
+ filename)
+ err = VixDiskLib_Create (h->connection, filename, &cparams, NULL, NULL);
+ VDDK_CALL_END (VixDiskLib_Create, 0);
+ if (err != VIX_OK) {
+ VDDK_ERROR (err, "VixDiskLib_Create: %s", filename);
+ goto err2;
+ }
+
+ create = false; /* Don't create it again. */
+ }
+
flags = 0;
if (readonly)
flags |= VIXDISKLIB_FLAG_OPEN_READ_ONLY;
diff --git a/plugins/vddk/vddk.h b/plugins/vddk/vddk.h
index d99b6f4b..3a808013 100644
--- a/plugins/vddk/vddk.h
+++ b/plugins/vddk/vddk.h
@@ -56,6 +56,11 @@ extern bool is_remote;
extern enum compression_type compression;
extern char *config;
extern const char *cookie;
+extern bool create;
+extern enum VixDiskLibAdapterType create_adapter_type;
+extern uint16_t create_hwversion;
+extern uint64_t create_size;
+extern enum VixDiskLibDiskType create_type;
extern const char *filename;
extern char *libdir;
extern uint16_t nfc_host_port;
diff --git a/tests/Makefile.am b/tests/Makefile.am
index 43b60943..ad2d43b9 100644
--- a/tests/Makefile.am
+++ b/tests/Makefile.am
@@ -171,7 +171,9 @@ if HAVE_VDDK
#
# make check-vddk vddkdir=vmware-vix-disklib-distrib
check-vddk:
- $(MAKE) check TESTS="test-vddk-real.sh test-vddk-real-dump-plugin.sh"
+ $(MAKE) check TESTS="test-vddk-real.sh
+ test-vddk-real-dump-plugin.sh
+ test-vddk-real-create.sh"
endif HAVE_VDDK
#----------------------------------------------------------------------
@@ -1033,6 +1035,7 @@ TESTS += \
test-vddk-dump-plugin.sh \
test-vddk-password-fd.sh \
test-vddk-password-interactive.sh \
+ test-vddk-real-create.sh \
test-vddk-real-dump-plugin.sh \
test-vddk-real.sh \
test-vddk-reexec.sh \
@@ -1063,6 +1066,7 @@ EXTRA_DIST += \
test-vddk-dump-plugin.sh \
test-vddk-password-fd.sh \
test-vddk-password-interactive.sh \
+ test-vddk-real-create.sh \
test-vddk-real-dump-plugin.sh \
test-vddk-real.sh \
test-vddk-reexec.sh \
diff --git a/tests/dummy-vddk.c b/tests/dummy-vddk.c
index b6f12042..0c5e505f 100644
--- a/tests/dummy-vddk.c
+++ b/tests/dummy-vddk.c
@@ -236,3 +236,13 @@ VixDiskLib_Wait (VixDiskLibHandle handle)
{
return VIX_OK;
}
+
+NBDKIT_DLL_PUBLIC VixError
+VixDiskLib_Create (const VixDiskLibConnection connection,
+ const char *path,
+ const VixDiskLibCreateParams *create_params,
+ void *progress_function_unused,
+ void *progress_data_unused)
+{
+ return VIX_E_NOT_SUPPORTED;
+}
diff --git a/tests/test-vddk-real-create.sh b/tests/test-vddk-real-create.sh
new file mode 100755
index 00000000..8f39a4c9
--- /dev/null
+++ b/tests/test-vddk-real-create.sh
@@ -0,0 +1,70 @@
+#!/usr/bin/env bash
+# nbdkit
+# Copyright (C) 2018-2022 Red Hat Inc.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#
+# * Redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in the
+# documentation and/or other materials provided with the distribution.
+#
+# * Neither the name of Red Hat nor the names of its contributors may be
+# used to endorse or promote products derived from this software without
+# specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY RED HAT AND CONTRIBUTORS ''AS IS'' AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+# PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL RED HAT OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
+# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+# SUCH DAMAGE.
+
+source ./functions.sh
+set -e
+set -x
+
+requires test "x$vddkdir" != "x"
+requires test -d "$vddkdir"
+requires test -f "$vddkdir/lib64/libvixDiskLib.so"
+requires test -f disk
+requires nbdcopy --version
+requires stat --version
+
+# Testing $LD_LIBRARY_PATH stuff breaks valgrind, so skip the rest of
+# this test if valgrinding.
+if [ "x$NBDKIT_VALGRIND" = "x1" ]; then
+ echo "$0: skipped LD_LIBRARY_PATH test when doing valgrind"
+ exit 77
+fi
+
+# VDDK > 5.1.1 only supports x86_64.
+if [ `uname -m` != "x86_64" ]; then
+ echo "$0: unsupported architecture"
+ exit 77
+fi
+
+vmdk=$PWD/test-vddk-real-create.vmdk ;# note must be an absolute path
+files="$vmdk"
+rm -f $files
+cleanup_fn rm -f $files
+
+size="$(stat -c %s disk)"
+
+nbdkit -fv -U - vddk libdir="$vddkdir" $vmdk \
+ create=true create-size=$size \
+ --run 'nbdcopy disk $uri'
+
+# Check the VMDK file was created and looks reasonable.
+test -f $vmdk
+file $vmdk | grep 'VMware'
--
2.31.1

29
SOURCES/0032-vddk-Fix-documentation-of-new-create-flag.patch
View File

@ -1,29 +0,0 @@
From eb5d5a628968c7fd5401cf7e73a6cff6c43994aa Mon Sep 17 00:00:00 2001
From: "Richard W.M. Jones" <rjones@redhat.com>
Date: Tue, 18 Jan 2022 13:14:01 +0000
Subject: [PATCH] vddk: Fix documentation of new create flag
create=1 works, but for consistency use create=true
Fixes: commit a39d5773afc3ebab7e5768118a2bccb89a654585
(cherry picked from commit 0b21897b64a6a1d97a8a7361e8f781ae743dedca)
---
plugins/vddk/nbdkit-vddk-plugin.pod | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/plugins/vddk/nbdkit-vddk-plugin.pod b/plugins/vddk/nbdkit-vddk-plugin.pod
index b96192d0..6c7ae989 100644
--- a/plugins/vddk/nbdkit-vddk-plugin.pod
+++ b/plugins/vddk/nbdkit-vddk-plugin.pod
@@ -51,7 +51,7 @@ input disk:
nbdkit -U - vddk \
/absolute/path/to/output.vmdk \
- create=1 create-size=100M \
+ create=true create-size=100M \
--run 'qemu-img convert input.qcow2 $uri'
=head2 Open a file on a remote VMware ESXi hypervisor
--
2.31.1

55
SOURCES/0033-vddk-Allow-create-hwversion-to-be-specified-as-a-num.patch
View File

@ -1,55 +0,0 @@
From c8cdce47bc38d2f59ecc4b75d6db7f032b63d527 Mon Sep 17 00:00:00 2001
From: "Richard W.M. Jones" <rjones@redhat.com>
Date: Tue, 18 Jan 2022 20:49:21 +0000
Subject: [PATCH] vddk: Allow create-hwversion to be specified as a number
This gives us a bit of future-proofing so we don't always need to add
new hardware versions immediately. Another reason for this is that
VDDK allows you to specify seemingly any number here without
complaint.
Updates: commit a39d5773afc3ebab7e5768118a2bccb89a654585
(cherry picked from commit 071e32927237c2c00d78684c8a0f2e5fbca9963e)
---
plugins/vddk/nbdkit-vddk-plugin.pod | 8 ++++++--
plugins/vddk/vddk.c | 3 ++-
2 files changed, 8 insertions(+), 3 deletions(-)
diff --git a/plugins/vddk/nbdkit-vddk-plugin.pod b/plugins/vddk/nbdkit-vddk-plugin.pod
index 6c7ae989..e6972900 100644
--- a/plugins/vddk/nbdkit-vddk-plugin.pod
+++ b/plugins/vddk/nbdkit-vddk-plugin.pod
@@ -195,10 +195,14 @@ Specify the VMDK disk adapter type. The default is C<scsi-buslogic>.
=item B<create-hwversion=esx65>
+=item B<create-hwversion=>N
+
(nbdkit E<ge> 1.30)
-Specify the VMDK virtual hardware version. The default is
-C<workstation5>.
+Specify the VMDK virtual hardware version. You can give either the
+named version or the equivalent 16 bit number.
+
+The default is C<workstation5> (N = 4).
=item B<create-size=>SIZE
diff --git a/plugins/vddk/vddk.c b/plugins/vddk/vddk.c
index 5ebf9a2c..bab8de6f 100644
--- a/plugins/vddk/vddk.c
+++ b/plugins/vddk/vddk.c
@@ -192,7 +192,8 @@ vddk_config (const char *key, const char *value)
create_hwversion = VIXDISKLIB_HWVERSION_ESX60;
else if (strcmp (value, "esx65") == 0)
create_hwversion = VIXDISKLIB_HWVERSION_ESX65;
- else {
+ else if (nbdkit_parse_uint16_t ("create-hwversion", value,
+ &create_hwversion) == -1) {
nbdkit_error ("unknown create-hwversion: %s", value);
return -1;
}
--
2.31.1

31
SOURCES/0034-tests-Fix-VDDK-tests.patch
View File

@ -1,31 +0,0 @@
From 84c5bc4664607fdf1f051e9e52ac6d0e4f0be049 Mon Sep 17 00:00:00 2001
From: "Richard W.M. Jones" <rjones@redhat.com>
Date: Tue, 18 Jan 2022 21:02:32 +0000
Subject: [PATCH] tests: Fix VDDK tests
We need to use quoting for the subcommand split across lines.
Fixes: commit a39d5773afc3ebab7e5768118a2bccb89a654585
(cherry picked from commit 4df525566b38202ed8a7485ac8e7f06edd5ee49a)
---
tests/Makefile.am | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/tests/Makefile.am b/tests/Makefile.am
index ad2d43b9..62a6f05b 100644
--- a/tests/Makefile.am
+++ b/tests/Makefile.am
@@ -171,8 +171,8 @@ if HAVE_VDDK
#
# make check-vddk vddkdir=vmware-vix-disklib-distrib
check-vddk:
- $(MAKE) check TESTS="test-vddk-real.sh
- test-vddk-real-dump-plugin.sh
+ $(MAKE) check TESTS="test-vddk-real.sh \
+ test-vddk-real-dump-plugin.sh \
test-vddk-real-create.sh"
endif HAVE_VDDK
--
2.31.1

204
SOURCES/0035-server-sockets-get-rid-of-AI_ADDRCONFIG.patch
View File

@ -1,204 +0,0 @@
From 222bce6b83421db1afdad24cf4e8ab7b1aa7b273 Mon Sep 17 00:00:00 2001
From: Laszlo Ersek <lersek@redhat.com>
Date: Tue, 18 Jan 2022 14:48:33 +0100
Subject: [PATCH] server/sockets: get rid of AI_ADDRCONFIG
The AI_ADDRCONFIG hint of getaddrinfo() is supposed to restrict the name
resolution to such address families (IPv4 vs. IPv6) for which the
resolving host has publicly routable addresses assigned.
The main problem with AI_ADDRCONFIG can be shown with the following
command line:
$ nbdkit -f -p 32776 -P pidfile -i ::1 --exit-with-parent null
On a host where ::1 is the only IPv6 address assigned (namely to the
loopback interface), the command fails with
> nbdkit: getaddrinfo: ::1: 32776: Address family for hostname not
> supported
due to the "publicly routable" requirement.
Remove AI_ADDRCONFIG from the getaddrinfo() hints, and as a replacement,
introduce the "-4" and "-6" options, similarly to netcat and ssh.
(1) This makes options of the form:
-i 127.0.0.1
-i ::1
work regardless of "public" IPv6 / IPv4 connectivity;
(2) options of the form
-i localhost
-i FQDN
will bind both IPv4 and IPv6 addresses of the desired interface(s);
(3) omitting the option "-i" will bind both IPv4 and IPv6 wildcard
addresses (0.0.0.0 and ::);
(4) the configurations in (2) and (3) can be restricted to IPv4 or IPv6
addresses by adding the "-4" or "-6" option, respectively.
Importantly, this change allows the "connect-tcp6" test case of libnbd to
pass on such hosts that have no IPv6 connectivity (i.e., where the only
assigned IPv6 address is ::1, namely on the loopback interface).
Ref: https://listman.redhat.com/archives/libguestfs/2022-January/msg00110.html
Signed-off-by: Laszlo Ersek <lersek@redhat.com>
Message-Id: <20220118134833.13246-3-lersek@redhat.com>
[lersek@redhat.com: fix typo in "--exit-with-parent" (Eric)]
Acked-by: Eric Blake <eblake@redhat.com>
(cherry picked from commit 9eec2335d630ae8ef947a927c1922d725d482f4a)
---
common/utils/windows-compat.h | 7 -------
docs/nbdkit.pod | 20 +++++++++++++++++++-
docs/synopsis.txt | 3 ++-
server/internal.h | 1 +
server/main.c | 9 +++++++++
server/options.h | 4 +++-
server/sockets.c | 3 ++-
7 files changed, 36 insertions(+), 11 deletions(-)
diff --git a/common/utils/windows-compat.h b/common/utils/windows-compat.h
index 7695bf7e..658c1d8b 100644
--- a/common/utils/windows-compat.h
+++ b/common/utils/windows-compat.h
@@ -75,13 +75,6 @@ struct sockaddr_un
#define O_NOCTTY 0
#endif
-/* AI_ADDRCONFIG is not available on Windows. It enables a rather
- * obscure feature of getaddrinfo to do with IPv6.
- */
-#ifndef AI_ADDRCONFIG
-#define AI_ADDRCONFIG 0
-#endif
-
/* Windows <errno.h> lacks certain errnos, so replace them here as
* best we can.
*/
diff --git a/docs/nbdkit.pod b/docs/nbdkit.pod
index 99cfb362..042607fb 100644
--- a/docs/nbdkit.pod
+++ b/docs/nbdkit.pod
@@ -173,6 +173,24 @@ Display information about nbdkit or a specific plugin:
Display brief command line usage information and exit.
+=item B<-4>
+
+=item B<--ipv4-only>
+
+=item B<-6>
+
+=item B<--ipv6-only>
+
+When a non-numeric argument is passed to the I<-i> option (such as a
+Fully Qualified Domain Name, or a host name from C</etc/hosts>),
+restrict the name resolution to IPv4 or IPv6 addresses.
+
+When the I<-i> option is omitted, listen on only the IPv4 or IPv6
+address of all interfaces (C<0.0.0.0> or C<::>, respectively).
+
+When both I<-4> and I<-6> options are present on the command line, the
+last one takes effect.
+
=item B<-D> PLUGIN.FLAG=N
=item B<-D> FILTER.FLAG=N
@@ -265,7 +283,7 @@ See also I<-u>.
=item B<--ipaddr> IPADDR
Listen on the specified interface. The default is to listen on all
-interfaces. See also I<-p>.
+interfaces. See also I<-4>, I<-6>, and I<-p>.
=item B<--log=stderr>
diff --git a/docs/synopsis.txt b/docs/synopsis.txt
index 07b9dcff..6154bb2e 100644
--- a/docs/synopsis.txt
+++ b/docs/synopsis.txt
@@ -1,4 +1,5 @@
-nbdkit [-D|--debug PLUGIN|FILTER|nbdkit.FLAG=N]
+nbdkit [-4|--ipv4-only] [-6|--ipv6-only]
+ [-D|--debug PLUGIN|FILTER|nbdkit.FLAG=N]
[-e|--exportname EXPORTNAME] [--exit-with-parent]
[--filter FILTER ...] [-f|--foreground]
[-g|--group GROUP] [-i|--ipaddr IPADDR]
diff --git a/server/internal.h b/server/internal.h
index bc81b786..46fcdd46 100644
--- a/server/internal.h
+++ b/server/internal.h
@@ -113,6 +113,7 @@ enum log_to {
LOG_TO_NULL, /* --log=null forced on the command line */
};
+extern int tcpip_sock_af;
extern struct debug_flag *debug_flags;
extern const char *export_name;
extern bool foreground;
diff --git a/server/main.c b/server/main.c
index 225258de..8e7ac149 100644
--- a/server/main.c
+++ b/server/main.c
@@ -86,6 +86,7 @@ static void error_if_stdio_closed (void);
static void switch_stdio (void);
static void winsock_init (void);
+int tcpip_sock_af = AF_UNSPEC; /* -4, -6 */
struct debug_flag *debug_flags; /* -D */
bool exit_with_parent; /* --exit-with-parent */
const char *export_name; /* -e */
@@ -367,6 +368,14 @@ main (int argc, char *argv[])
exit (EXIT_FAILURE);
#endif
+ case '4':
+ tcpip_sock_af = AF_INET;
+ break;
+
+ case '6':
+ tcpip_sock_af = AF_INET6;
+ break;
+
case 'D':
add_debug_flag (optarg);
break;
diff --git a/server/options.h b/server/options.h
index e59ef17f..39299b9d 100644
--- a/server/options.h
+++ b/server/options.h
@@ -59,8 +59,10 @@ enum {
VSOCK_OPTION,
};
-static const char *short_options = "D:e:fg:i:nop:P:rst:u:U:vV";
+static const char *short_options = "46D:e:fg:i:nop:P:rst:u:U:vV";
static const struct option long_options[] = {
+ { "ipv4-only", no_argument, NULL, '4' },
+ { "ipv6-only", no_argument, NULL, '6' },
{ "debug", required_argument, NULL, 'D' },
{ "dump-config", no_argument, NULL, DUMP_CONFIG_OPTION },
{ "dump-plugin", no_argument, NULL, DUMP_PLUGIN_OPTION },
diff --git a/server/sockets.c b/server/sockets.c
index 15a26f69..4e4ccbc4 100644
--- a/server/sockets.c
+++ b/server/sockets.c
@@ -179,7 +179,8 @@ bind_tcpip_socket (sockets *socks)
port = "10809";
memset (&hints, 0, sizeof hints);
- hints.ai_flags = AI_PASSIVE | AI_ADDRCONFIG;
+ hints.ai_flags = AI_PASSIVE;
+ hints.ai_family = tcpip_sock_af;
hints.ai_socktype = SOCK_STREAM;
err = getaddrinfo (ipaddr, port, &hints, &ai);
--
2.31.1

2
SOURCES/copy-patches.sh
View File

@ -6,7 +6,7 @@ set -e
# directory. Use it like this:
# ./copy-patches.sh
rhel_version=9.0
rhel_version=9.1
# Check we're in the right directory.
if [ ! -f nbdkit.spec ]; then

17
SOURCES/nbdkit-1.28.5.tar.gz.sig
View File

@ -1,17 +0,0 @@
-----BEGIN PGP SIGNATURE-----
iQJFBAABCAAvFiEE93dPsa0HSn6Mh2fqkXOPc+G3aKAFAmHumIgRHHJpY2hAYW5u
ZXhpYS5vcmcACgkQkXOPc+G3aKAt/RAAlUnU/dJ6d1bec5ggmfTgWYwHcXqc6bGi
Tp0vxG+2LiE/D52xhGvaRWQwd2mR4vvHiDLlBitlBZNHItyuB9wM9r5SAIRw5PcR
1kCILAKGQN2dLSS+HUyib6cnlughFwIeSt4XuhwgedLlIBZ+2d38IRC3ia4ShbIX
btZMwRmpD/06Py8A+tM856zb1YgjpA3nQIC56r/ne25vLyAY4LE8T2BtjlkmBxBy
Lswg0KM3+SBsMWwbo0aCfyTOW9lpVa2WnLgu/9nsfIjA+m9kcjfpjfmhduV6lfrx
KRFGtKQnl+RWfhhfmxeCWo5/mXqrqOga7VIWltxRkjQ916TrzTWwMnWBtOvVfRSL
QS1tlJYbClRGHHkM9YHjvV0v+xHcUTt9VAd+RruVjnz2H4hZrEi8klAHWjOUe/1m
37PEoLAh9+ox+zcSODc+MWWOA98oJoXyS0PZvPOzzlSokxaLEY/TRUMrILGBJSyz
hIwdMi63gM1KzIw0ysNJ639Nvu0n/PiIgpPheXK81fNyNpzsThS1uEqonAMC9+Hr
QIMBgfRMdvbG791lVo7WRHSdGpSO+hun4scla+3VZpszqSwFX2+O/ji0+5cos4RR
dxfzV4gt/1FH72OhWGHMSmmTpd+G2ZjXFsjNTmzMYbS0kMeXdIXalVXJrfDroSoU
ITXkfk5uFtU=
=Wlxf
-----END PGP SIGNATURE-----

17
SOURCES/nbdkit-1.30.8.tar.gz.sig Normal file
View File

@ -0,0 +1,17 @@
-----BEGIN PGP SIGNATURE-----
iQJFBAABCAAvFiEE93dPsa0HSn6Mh2fqkXOPc+G3aKAFAmLWguQRHHJpY2hAYW5u
ZXhpYS5vcmcACgkQkXOPc+G3aKBpTQ//SBnebP9miDMyAnMS0zgK1LevsGMWwnLm
cZcXgvx+hvdomTlFElcxBjSPN51rhywI0vyN8fM1h/eckBkGH/fC8Ta+pQHBxFec
KZZfLZ+EhsBv1JAid3t284xxFkahtZ0UWpPmVlZG2EelGiYHpL+nckf0hTS6R2a2
cWDgpSwxIlPNEvDmDqZyUPCTY4vdxtdlI4JwRcqYn7nH6TbBLWK0/pjQH6NtZ/bu
cBQJQpi+ne3mIU6P8GocFJhNEMO1//aZRJsAX5uURuIaAToH+QDtcXCVPTZDogqh
5BPL/nGQu+LbXFlRTGFIwNRyfSLDGFPWnZ/356NssaPpX4cYB9zsEmdnf9enz/Ez
cYrwoxQbilzkvRH0oovb+2UL4V/dEoUgdND6amHWF+3uuT+c/j9CsIk5IrL5y9Ly
VhkttvYL96rOGQoIeZfg0+Icjx6aSau6L7qRvjZBP6egZN3VUVhMFn7knynENXlv
DxDg4XwqWP6OKsBiBteED48Hn2nF0jniq/6Mx/fVrblGlYGxOSLIupdTXl5n10OK
9vwsp7qSYJP1160Mt5r3MFNH0DygmAHbH5jqzDFn5H6TqSV31PuF4XZg/DnBedaJ
+PI9qOnXjkM9gYeVO3FOosf70kh7zfjP6/kOOJ4BGiaZ5T9Y12oJAwxqkjZhUHHF
QDvUE7KDOM8=
=I81A
-----END PGP SIGNATURE-----

23
SOURCES/nbdkit-find-provides Executable file
View File

@ -0,0 +1,23 @@
#!/bin/bash -
# Generate RPM provides automatically for nbdkit packages and filters.
# Copyright (C) 2009-2022 Red Hat Inc.
# To test:
# find /usr/lib64/nbdkit/plugins | ./nbdkit-find-provides VER REL
# find /usr/lib64/nbdkit/filters | ./nbdkit-find-provides VER REL
ver="$1"
rel="$2"
function process_file
{
if [[ $1 =~ /plugins/nbdkit-.*-plugin ]] ||
[[ $1 =~ /filters/nbdkit-.*-filter ]]; then
echo "Provides:" "$(basename $1 .so)" "=" "$ver-$rel"
fi
}
while read line; do
process_file "$line"
done

3
SOURCES/nbdkit.attr Normal file
View File

@ -0,0 +1,3 @@
%__nbdkit_provides %{_rpmconfigdir}/nbdkit-find-provides %{version} %{release}
%__nbdkit_path %{_libdir}/nbdkit/(plugins|filters)/nbdkit-.*-(plugin|filter)(\.so)?$
%__nbdkit_flags exeonly

210
SPECS/nbdkit.spec
View File

@ -1,6 +1,9 @@
%undefine _package_note_flags
%global _hardened_build 1
%ifarch %{kernel_arches}
# ppc64le broken in rawhide:
# https://bugzilla.redhat.com/show_bug.cgi?id=2006709
# riscv64 tests fail with
# qemu-system-riscv64: invalid accelerator kvm
# qemu-system-riscv64: falling back to tcg
@ -8,7 +11,7 @@
# This seems to require changes in libguestfs and/or qemu to support
# -cpu max or -cpu virt.
# s390x builders can't run libguestfs
%ifnarch riscv64 s390 s390x
%ifnarch %{power64} riscv64 s390 s390x
%global have_libguestfs 1
%endif
%endif
@ -29,8 +32,6 @@
# If the test suite is broken on a particular architecture, document
# it as a bug and add it to this list.
#
# armv7, aarch64: https://bugzilla.redhat.com/show_bug.cgi?id=1893892
%global broken_test_arches NONE
%if 0%{?rhel} == 7
@ -48,10 +49,10 @@ ExclusiveArch: x86_64
%global patches_touch_autotools 1
# The source directory.
%global source_directory 1.28-stable
%global source_directory 1.30-stable
Name: nbdkit
Version: 1.28.5
Version: 1.30.8
Release: 1%{?dist}
Summary: NBD server
@ -74,44 +75,37 @@ Source2: libguestfs.keyring
Source3: copy-patches.sh
# Patches come from the upstream repository:
# https://gitlab.com/nbdkit/nbdkit/-/commits/rhel-9.0/
# https://gitlab.com/nbdkit/nbdkit/-/commits/rhel-9.1/
# Patches.
Patch0001: 0001-vddk-Refactor-how-D-vddk.stats-1-is-collected.patch
Patch0002: 0002-vddk-Extend-D-vddk.stats-1-to-show-number-of-calls-a.patch
Patch0003: 0003-vddk-Simplify-and-consolidate-VDDK_CALL_START-END-ma.patch
Patch0004: 0004-vddk-Document-troubleshooting-performance-problems.patch
Patch0005: 0005-vddk-Include-VDDK-major-library-version-in-dump-plug.patch
Patch0006: 0006-vddk-Add-logical-and-physical-sector-size-to-D-vddk..patch
Patch0007: 0007-vddk-Fix-typo-in-debug-message.patch
Patch0008: 0008-vddk-Only-print-vddk_library_version-when-we-managed.patch
Patch0009: 0009-vddk-Print-one-line-in-dump-plugin-output-for-each-V.patch
Patch0010: 0010-vddk-Move-minimum-version-to-VDDK-6.5.patch
Patch0011: 0011-vddk-Add-read-write-and-wait-asynchronous-functions.patch
Patch0012: 0012-vddk-Start-to-split-VDDK-over-several-files.patch
Patch0013: 0013-vddk-Refactor-D-vddk.stats-1-into-a-new-file.patch
Patch0014: 0014-vddk-Implement-parallel-thread-model.patch
Patch0015: 0015-vddk-Assume-that-VixDiskLib_Flush-is-available.patch
Patch0016: 0016-vddk-Simplify-detection-of-VDDK-symbols-and-baseline.patch
Patch0017: 0017-vddk-Remove-some-whitespace-from-a-couple-of-functio.patch
Patch0018: 0018-vddk-Move-config-debug-error-and-utility-functions-a.patch
Patch0019: 0019-common-utils-test-vector.c-Add-vector-benchmarks.patch
Patch0020: 0020-common-urils-vector.c-Optimize-vector-append.patch
Patch0021: 0021-common-utils-vector-Rename-alloc-to-cap.patch
Patch0022: 0022-common-utils-vector-Rename-size-to-len.patch
Patch0023: 0023-podwrapper.pl.in-Use-short-commit-date.patch
Patch0024: 0024-ocaml-Replace-noalloc-with-noalloc-annotation.patch
Patch0025: 0025-vddk-Drop-obsolete-documentation-related-to-thread-m.patch
Patch0026: 0026-Revert-podwrapper.pl.in-Use-short-commit-date.patch
Patch0027: 0027-Fix-podwrapper.pl.in-Use-short-commit-date.patch
Patch0028: 0028-scripts-Add-simple-script-for-automating-VDDK-disk-c.patch
Patch0029: 0029-file-Fix-implementation-of-cache-none-for-writes.patch
Patch0030: 0030-tests-Add-configure-disable-libguestfs-tests-flag.patch
Patch0031: 0031-vddk-Implement-VMDK-creation.patch
Patch0032: 0032-vddk-Fix-documentation-of-new-create-flag.patch
Patch0033: 0033-vddk-Allow-create-hwversion-to-be-specified-as-a-num.patch
Patch0034: 0034-tests-Fix-VDDK-tests.patch
Patch0035: 0035-server-sockets-get-rid-of-AI_ADDRCONFIG.patch
Patch0001: 0001-ssh-Allow-the-remote-file-to-be-created.patch
Patch0002: 0002-readahead-Rewrite-this-filter-so-it-prefetches-using.patch
Patch0003: 0003-readahead-Fix-test.patch
Patch0004: 0004-New-filter-luks.patch
Patch0005: 0005-luks-Disable-filter-with-old-GnuTLS-in-Debian-10.patch
Patch0006: 0006-luks-Various-fixes-for-Clang.patch
Patch0007: 0007-luks-Link-with-libcompat-on-Windows.patch
Patch0008: 0008-luks-Refactor-the-filter.patch
Patch0009: 0009-tests-luks-Reduce-time-taken-to-run-these-tests.patch
Patch0010: 0010-Add-nbdkit.parse_size-Python-function.patch
Patch0011: 0011-cache-Fix-cross-reference-nbdkit-readahead-filter.patch
Patch0012: 0012-curl-Don-t-document-curl-plugin-readahead-filter.patch
Patch0013: 0013-New-filter-scan.patch
Patch0014: 0014-scan-Remove-condition-variable.patch
Patch0015: 0015-scan-Small-typographical-fix-in-manual.patch
Patch0016: 0016-ssh-Don-t-reference-readahead-or-scan-filters-from-t.patch
Patch0017: 0017-scan-Fix-bound-so-we-don-t-try-to-prefetch-beyond-en.patch
Patch0018: 0018-tests-Add-a-regression-test-for-LUKS-zeroing-crash.patch
Patch0019: 0019-rate-Allow-burstiness-to-be-controlled.patch
Patch0020: 0020-luks-Check-return-values-from-malloc-more-carefully.patch
Patch0021: 0021-luks-Avoid-potential-overflow-when-computing-key-mat.patch
Patch0022: 0022-luks-Avoid-memory-leak-on-error-path.patch
Patch0023: 0023-tests-Hoist-some-EXTRA_DIST-out-of-automake-conditio.patch
# For automatic RPM Provides generation.
# See: https://rpm-software-management.github.io/rpm/manual/dependency_generators.html
Source4: nbdkit.attr
Source5: nbdkit-find-provides
BuildRequires: make
%if 0%{patches_touch_autotools}
@ -150,9 +144,7 @@ BuildRequires: python3-boto3
%endif
%if !0%{?rhel}
%if 0%{?have_ocaml}
# Requires OCaml 4.02.2 which contains fix for
# http://caml.inria.fr/mantis/view.php?id=6693
BuildRequires: ocaml >= 4.02.2
BuildRequires: ocaml >= 4.03
BuildRequires: ocaml-ocamldoc
%endif
BuildRequires: ruby-devel
@ -185,6 +177,11 @@ BuildRequires: %{_sbindir}/ss
BuildRequires: %{_bindir}/stat
BuildRequires: %{_bindir}/ssh-keygen
# This package has RPM rules that create the automatic Provides: for
# nbdkit plugins and filters. This means nbdkit build depends on
# itself, but it's a simple noarch package so easy to install.
BuildRequires: nbdkit-srpm-macros >= 1.30.0
# nbdkit is a metapackage pulling the server and a useful subset
# of the plugins and filters.
Requires: nbdkit-server%{?_isa} = %{version}-%{release}
@ -226,7 +223,6 @@ reading the nbdkit(1) and nbdkit-plugin(3) manual pages.
%package server
Summary: The %{name} server
License: BSD
Provides: %{name}-null-plugin = %{version}-%{release}
%description server
This package contains the %{name} server with only the null plugin
@ -239,21 +235,6 @@ the metapackage "nbdkit".
Summary: Basic plugins for %{name}
License: BSD
Requires: %{name}-server%{?_isa} = %{version}-%{release}
Provides: %{name}-data-plugin = %{version}-%{release}
Provides: %{name}-eval-plugin = %{version}-%{release}
Provides: %{name}-file-plugin = %{version}-%{release}
Provides: %{name}-floppy-plugin = %{version}-%{release}
Provides: %{name}-full-plugin = %{version}-%{release}
Provides: %{name}-info-plugin = %{version}-%{release}
Provides: %{name}-memory-plugin = %{version}-%{release}
Provides: %{name}-ondemand-plugin = %{version}-%{release}
Provides: %{name}-pattern-plugin = %{version}-%{release}
Provides: %{name}-partitioning-plugin = %{version}-%{release}
Provides: %{name}-random-plugin = %{version}-%{release}
Provides: %{name}-sh-plugin = %{version}-%{release}
Provides: %{name}-sparse-random-plugin = %{version}-%{release}
Provides: %{name}-split-plugin = %{version}-%{release}
Provides: %{name}-zero-plugin = %{version}-%{release}
%description basic-plugins
@ -549,38 +530,6 @@ VMware VDDK for accessing VMware disks and servers.
Summary: Basic filters for %{name}
License: BSD
Requires: %{name}-server%{?_isa} = %{version}-%{release}
Provides: %{name}-blocksize-filter = %{version}-%{release}
Provides: %{name}-cache-filter = %{version}-%{release}
Provides: %{name}-cacheextents-filter = %{version}-%{release}
Provides: %{name}-checkwrite-filter = %{version}-%{release}
Provides: %{name}-cow-filter = %{version}-%{release}
Provides: %{name}-ddrescue-filter = %{version}-%{release}
Provides: %{name}-delay-filter = %{version}-%{release}
Provides: %{name}-error-filter = %{version}-%{release}
Provides: %{name}-exitlast-filter = %{version}-%{release}
Provides: %{name}-exitwhen-filter = %{version}-%{release}
Provides: %{name}-exportname-filter = %{version}-%{release}
Provides: %{name}-extentlist-filter = %{version}-%{release}
Provides: %{name}-fua-filter = %{version}-%{release}
Provides: %{name}-ip-filter = %{version}-%{release}
Provides: %{name}-limit-filter = %{version}-%{release}
Provides: %{name}-log-filter = %{version}-%{release}
Provides: %{name}-multi-conn-filter = %{version}-%{release}
Provides: %{name}-nocache-filter = %{version}-%{release}
Provides: %{name}-noextents-filter = %{version}-%{release}
Provides: %{name}-nofilter-filter = %{version}-%{release}
Provides: %{name}-noparallel-filter = %{version}-%{release}
Provides: %{name}-nozero-filter = %{version}-%{release}
Provides: %{name}-offset-filter = %{version}-%{release}
Provides: %{name}-partition-filter = %{version}-%{release}
Provides: %{name}-pause-filter = %{version}-%{release}
Provides: %{name}-rate-filter = %{version}-%{release}
Provides: %{name}-readahead-filter = %{version}-%{release}
Provides: %{name}-retry-filter = %{version}-%{release}
Provides: %{name}-stats-filter = %{version}-%{release}
Provides: %{name}-swab-filter = %{version}-%{release}
Provides: %{name}-tls-fallback-filter = %{version}-%{release}
Provides: %{name}-truncate-filter = %{version}-%{release}
%description basic-filters
This package contains filters for %{name} which only depend on simple
@ -589,6 +538,8 @@ complex dependencies are packaged separately.
nbdkit-blocksize-filter Adjust block size of requests sent to plugins.
nbdkit-blocksize-policy-filter Set block size constraints and policy.
nbdkit-cache-filter Server-side cache.
nbdkit-cacheextents-filter Cache extents.
@ -619,6 +570,8 @@ nbdkit-limit-filter Limit nr clients that can connect concurrently.
nbdkit-log-filter Log all transactions to a file.
nbdkit-luks-filter Read and write LUKS-encrypted disks.
nbdkit-multi-conn-filter Enable, emulate or disable multi-conn.
nbdkit-nocache-filter Disable cache requests in the underlying plugin.
@ -637,12 +590,18 @@ nbdkit-partition-filter Serve a single partition.
nbdkit-pause-filter Pause NBD requests.
nbdkit-protect-filter Write-protect parts of a plugin.
nbdkit-rate-filter Limit bandwidth by connection or server.
nbdkit-readahead-filter Prefetch data when reading sequentially.
nbdkit-retry-filter Reopen connection on error.
nbdkit-retry-request-filter Retry single requests on error.
nbdkit-scan-filter Prefetch data ahead of sequential reads.
nbdkit-stats-filter Display statistics about operations.
nbdkit-swab-filter Filter for swapping byte order.
@ -705,8 +664,19 @@ for %{name}. Install this package if you want to develop
plugins for %{name}.
%package srpm-macros
Summary: RPM Provides rules for %{name} plugins and filters
License: BSD
BuildArch: noarch
%description srpm-macros
This package contains RPM rules that create the automatic Provides:
for %{name} plugins and filters found in the plugins directory.
%package bash-completion
Summary: Bash tab-completion for %{name}
License: BSD
BuildArch: noarch
Requires: bash-completion >= 2.0
Requires: %{name}-server = %{version}-%{release}
@ -779,10 +749,6 @@ find $RPM_BUILD_ROOT -name '*.la' -delete
# rust plugin is built. Delete it if this happens.
rm -f $RPM_BUILD_ROOT%{_mandir}/man3/nbdkit-rust-plugin.3*
# Remove the streaming plugin (deprecated upstream).
rm $RPM_BUILD_ROOT%{_libdir}/%{name}/plugins/nbdkit-streaming-plugin.so
rm $RPM_BUILD_ROOT%{_mandir}/man1/nbdkit-streaming-plugin.1*
%if 0%{?rhel}
# In RHEL, remove some plugins we cannot --disable.
for f in cc cdi torrent; do
@ -793,6 +759,11 @@ rm -f $RPM_BUILD_ROOT%{_libdir}/%{name}/plugins/nbdkit-S3-plugin
rm -f $RPM_BUILD_ROOT%{_mandir}/man1/nbdkit-S3-plugin.1*
%endif
# Install RPM dependency generator.
mkdir -p $RPM_BUILD_ROOT%{_rpmconfigdir}/fileattrs/
install -m 0644 %{SOURCE4} $RPM_BUILD_ROOT%{_rpmconfigdir}/fileattrs/
install -m 0755 %{SOURCE5} $RPM_BUILD_ROOT%{_rpmconfigdir}/
%check
%ifnarch %{broken_test_arches}
@ -1085,6 +1056,7 @@ export LIBGUESTFS_TRACE=1
%doc README
%license LICENSE
%{_libdir}/%{name}/filters/nbdkit-blocksize-filter.so
%{_libdir}/%{name}/filters/nbdkit-blocksize-policy-filter.so
%{_libdir}/%{name}/filters/nbdkit-cache-filter.so
%{_libdir}/%{name}/filters/nbdkit-cacheextents-filter.so
%{_libdir}/%{name}/filters/nbdkit-checkwrite-filter.so
@ -1100,6 +1072,7 @@ export LIBGUESTFS_TRACE=1
%{_libdir}/%{name}/filters/nbdkit-ip-filter.so
%{_libdir}/%{name}/filters/nbdkit-limit-filter.so
%{_libdir}/%{name}/filters/nbdkit-log-filter.so
%{_libdir}/%{name}/filters/nbdkit-luks-filter.so
%{_libdir}/%{name}/filters/nbdkit-multi-conn-filter.so
%{_libdir}/%{name}/filters/nbdkit-nocache-filter.so
%{_libdir}/%{name}/filters/nbdkit-noextents-filter.so
@ -1109,14 +1082,18 @@ export LIBGUESTFS_TRACE=1
%{_libdir}/%{name}/filters/nbdkit-offset-filter.so
%{_libdir}/%{name}/filters/nbdkit-partition-filter.so
%{_libdir}/%{name}/filters/nbdkit-pause-filter.so
%{_libdir}/%{name}/filters/nbdkit-protect-filter.so
%{_libdir}/%{name}/filters/nbdkit-rate-filter.so
%{_libdir}/%{name}/filters/nbdkit-readahead-filter.so
%{_libdir}/%{name}/filters/nbdkit-retry-filter.so
%{_libdir}/%{name}/filters/nbdkit-retry-request-filter.so
%{_libdir}/%{name}/filters/nbdkit-scan-filter.so
%{_libdir}/%{name}/filters/nbdkit-stats-filter.so
%{_libdir}/%{name}/filters/nbdkit-swab-filter.so
%{_libdir}/%{name}/filters/nbdkit-tls-fallback-filter.so
%{_libdir}/%{name}/filters/nbdkit-truncate-filter.so
%{_mandir}/man1/nbdkit-blocksize-filter.1*
%{_mandir}/man1/nbdkit-blocksize-policy-filter.1*
%{_mandir}/man1/nbdkit-cache-filter.1*
%{_mandir}/man1/nbdkit-cacheextents-filter.1*
%{_mandir}/man1/nbdkit-checkwrite-filter.1*
@ -1132,6 +1109,7 @@ export LIBGUESTFS_TRACE=1
%{_mandir}/man1/nbdkit-ip-filter.1*
%{_mandir}/man1/nbdkit-limit-filter.1*
%{_mandir}/man1/nbdkit-log-filter.1*
%{_mandir}/man1/nbdkit-luks-filter.1*
%{_mandir}/man1/nbdkit-multi-conn-filter.1*
%{_mandir}/man1/nbdkit-nocache-filter.1*
%{_mandir}/man1/nbdkit-noextents-filter.1*
@ -1141,9 +1119,12 @@ export LIBGUESTFS_TRACE=1
%{_mandir}/man1/nbdkit-offset-filter.1*
%{_mandir}/man1/nbdkit-partition-filter.1*
%{_mandir}/man1/nbdkit-pause-filter.1*
%{_mandir}/man1/nbdkit-protect-filter.1*
%{_mandir}/man1/nbdkit-rate-filter.1*
%{_mandir}/man1/nbdkit-readahead-filter.1*
%{_mandir}/man1/nbdkit-retry-filter.1*
%{_mandir}/man1/nbdkit-retry-request-filter.1*
%{_mandir}/man1/nbdkit-scan-filter.1*
%{_mandir}/man1/nbdkit-stats-filter.1*
%{_mandir}/man1/nbdkit-swab-filter.1*
%{_mandir}/man1/nbdkit-tls-fallback-filter.1*
@ -1214,6 +1195,12 @@ export LIBGUESTFS_TRACE=1
%{_libdir}/pkgconfig/nbdkit.pc
%files srpm-macros
%license LICENSE
%{_rpmconfigdir}/fileattrs/nbdkit.attr
%{_rpmconfigdir}/nbdkit-find-provides
%files bash-completion
%license LICENSE
%dir %{_datadir}/bash-completion/completions
@ -1221,6 +1208,37 @@ export LIBGUESTFS_TRACE=1
%changelog
* Tue Jul 19 2022 Richard W.M. Jones <rjones@redhat.com> - 1.30.8-1
- Rebase to new stable branch version 1.30.8
resolves: rhbz#2059289
- Add automatic provides generator and subpackage nbdkit-srpm-macros
resolves: rhbz#2059291
- New filters: blocksize-policy, protect, retry-request
- Fix license of bash-completion subpackage
- vddk: Fix use of uninitialized memory when computing block size
resolves: rhbz#2066655
- Skip vsock tests unless the vsock_loopback module is loaded
resolves: rhbz#2069558
- Add support for ssh create remote file.
- Suppress excess messages from nbdkit-nbd-plugin
resolves: rhbz#2083498
- Suppress incorrect VDDK error when converting guests from vCenter
resolves: rhbz#2083617
- Backport new readahead filter from 1.32.
- Backport new LUKS filter from 1.32.
- Backport new scan filter from 1.32.
- Add new Python binding for nbdkit_parse_size from 1.32
- Add new rate filter burstiness setting from 1.32
- vddk: Suppress new VDDK "phone home" messages
resolves: rhbz#2104720
- vddk: Clearer error message when thumbprint is wrong
resolves: rhbz#1905772
- Fix memory allocator=malloc,mlock=true
resolves: rhbz#2044432
- Fix multiple Coverity problems
- Fix bounds error in nbdkit-checkwrite-filter
resolves: rhbz#2108545
* Mon Jan 24 2022 Richard W.M. Jones <rjones@redhat.com> - 1.28.5-1
- Rebase to new stable branch version 1.28.5
resolves: rhbz#2011709