rpms/nbdkit
5
0
Fork 0
Code Issues Pull Requests Releases Activity

import nbdkit-1.24.0-4.module+el8.6.0+14480+c0a3aa0f

Browse Source
This commit is contained in:
CentOS Sources 2022-05-10 03:08:30 -04:00 committed by Stepan Oksanichenko
parent 003a44a973
commit f07d6b31a4
29 changed files with 1017 additions and 4568 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.gitignore vendored
View File

@ -1,2 +1,2 @@
SOURCES/libguestfs.keyring
SOURCES/nbdkit-1.16.2.tar.gz
SOURCES/nbdkit-1.24.0.tar.gz

2
.nbdkit.metadata
View File

@ -1,2 +1,2 @@
1bbc40f501a7fef9eef2a39b701a71aee2fea7c4 SOURCES/libguestfs.keyring
42a5761cd3403c02c43cdf7d541ff3faaf8b4769 SOURCES/nbdkit-1.16.2.tar.gz
069720cc0d1502b007652101d293a57d7b4d7c41 SOURCES/nbdkit-1.24.0.tar.gz

82
SOURCES/0001-cache-cow-Fix-data-corruption-in-zero-and-trim-on-un.patch Normal file
View File

@ -0,0 +1,82 @@
From 99788909d9ec36e3210cf85976fe5b18da690ddd Mon Sep 17 00:00:00 2001
From: "Richard W.M. Jones" <rjones@redhat.com>
Date: Wed, 4 Aug 2021 20:24:59 +0100
Subject: [PATCH] cache, cow: Fix data corruption in zero and trim on unaligned
tail
Commit eb6009b092 ("cache, cow: Reduce use of bounce-buffer") first
introduced in nbdkit 1.14 added an optimization of the
read-modify-write mechanism used for unaligned heads and tails when
zeroing in the cache layer.
Unfortunately the part applied to the tail contained a mistake: It
zeroes the end of the buffer rather than the beginning. This causes
data corruption when you use the zero or trim function with an offset
and count which is not aligned to the block size.
Although the bug has been around for years, a recent change made it
more likely to happen. Commit c1905b0a28 ("cache, cow: Use a 64K
block size by default") increased the default block size from 4K to
64K. Most filesystems use a 4K block size so operations like fstrim
will make 4K-aligned requests, and with a 4K block size also in the
cache or cow filter the unaligned case would never have been hit
before.
We can demonstrate the bug simply by filling a buffer with data
(100000 bytes in the example), and then trimming that data, which
ought to zero it out.
Before this commit there is data visible after the trim:
$ nbdkit --filter=cow data "0x21 * 100000" --run 'nbdsh -u $uri -c "h.trim(100000, 0)" ; nbdcopy $uri - | hexdump -C'
00000000 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................|
*
00018000 21 21 21 21 21 21 21 21 21 21 21 21 21 21 21 21 |!!!!!!!!!!!!!!!!|
*
000186a0
After this commit the trim completely clears the data:
$ nbdkit --filter=cow data "0x21 * 100000" --run 'nbdsh -u $uri -c "h.trim(100000, 0)" ; nbdcopy $uri - | hexdump -C'
00000000 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................|
*
000186a0
Thanks: Ming Xie for finding the bug
Fixes: commit eb6009b092ae642ed25f133d487dd40ef7bf70f8
(cherry picked from commit a0ae7b2158598ce48ac31706319007f716d01c87)
(cherry picked from commit c0b15574647672cb5c48178333acdd07424692ef)
---
filters/cache/cache.c | 2 +-
filters/cow/cow.c | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/filters/cache/cache.c b/filters/cache/cache.c
index 91dcc43d..0616cc7b 100644
--- a/filters/cache/cache.c
+++ b/filters/cache/cache.c
@@ -493,7 +493,7 @@ cache_zero (struct nbdkit_next_ops *next_ops, void *nxdata,
ACQUIRE_LOCK_FOR_CURRENT_SCOPE (&lock);
r = blk_read (next_ops, nxdata, blknum, block, err);
if (r != -1) {
- memset (&block[count], 0, blksize - count);
+ memset (block, 0, count);
r = blk_write (next_ops, nxdata, blknum, block, flags, err);
}
if (r == -1)
diff --git a/filters/cow/cow.c b/filters/cow/cow.c
index 51ca64a4..1cfcc4e7 100644
--- a/filters/cow/cow.c
+++ b/filters/cow/cow.c
@@ -419,7 +419,7 @@ cow_zero (struct nbdkit_next_ops *next_ops, void *nxdata,
ACQUIRE_LOCK_FOR_CURRENT_SCOPE (&lock);
r = blk_read (next_ops, nxdata, blknum, block, err);
if (r != -1) {
- memset (&block[count], 0, BLKSIZE - count);
+ memset (block, 0, count);
r = blk_write (blknum, block, err);
}
if (r == -1)
--
2.31.1

75
SOURCES/0001-server-Allow-D-nbdkit.-debug-flags-for-the-core-serv.patch
View File

@ -1,75 +0,0 @@
From d7836fb0a7131c725e3c02be7e48e99c671637c3 Mon Sep 17 00:00:00 2001
From: "Richard W.M. Jones" <rjones@redhat.com>
Date: Thu, 12 Dec 2019 08:57:15 +0000
Subject: [PATCH 01/19] server: Allow -D nbdkit.* debug flags for the core
server.
These work like plugin/filter debug flags, but apply to the internals
of the server.
(cherry picked from commit 3b45db234a691f8ff926a6fef583e11c3601f112)
---
docs/nbdkit.pod | 7 +++++++
docs/synopsis.txt | 2 +-
server/main.c | 3 +++
server/nbdkit.syms | 2 ++
4 files changed, 13 insertions(+), 1 deletion(-)
diff --git a/docs/nbdkit.pod b/docs/nbdkit.pod
index a2e72b13..346d8332 100644
--- a/docs/nbdkit.pod
+++ b/docs/nbdkit.pod
@@ -177,6 +177,13 @@ Display brief command line usage information and exit.
Set the plugin or filter Debug Flag called C<FLAG> to the integer
value C<N>. See L<nbdkit-plugin(3)/Debug Flags>.
+=item B<-D> nbdkit.FLAG=N
+
+=item B<--debug> nbdkit.FLAG=N
+
+Set the nbdkit server Debug Flag called C<FLAG> to the integer value
+C<N>.
+
=item B<--dump-config>
Dump out the compile-time configuration values and exit.
diff --git a/docs/synopsis.txt b/docs/synopsis.txt
index 3c239373..c3675422 100644
--- a/docs/synopsis.txt
+++ b/docs/synopsis.txt
@@ -1,4 +1,4 @@
-nbdkit [-D|--debug PLUGIN|FILTER.FLAG=N]
+nbdkit [-D|--debug PLUGIN|FILTER|nbdkit.FLAG=N]
[-e|--exportname EXPORTNAME] [--exit-with-parent]
[--filter FILTER ...] [-f|--foreground]
[-g|--group GROUP] [-i|--ipaddr IPADDR]
diff --git a/server/main.c b/server/main.c
index d39941b1..11ba1e6d 100644
--- a/server/main.c
+++ b/server/main.c
@@ -563,6 +563,9 @@ main (int argc, char *argv[])
free (t);
}
+ /* Apply nbdkit.* flags for the server. */
+ apply_debug_flags (NULL, "nbdkit");
+
/* Check all debug flags were used, and free them. */
free_debug_flags ();
diff --git a/server/nbdkit.syms b/server/nbdkit.syms
index 390972e2..96c22c07 100644
--- a/server/nbdkit.syms
+++ b/server/nbdkit.syms
@@ -67,6 +67,8 @@
nbdkit_vdebug;
nbdkit_verror;
+ nbdkit_debug_*;
+
# Everything else is hidden.
local: *;
};
--
2.18.2

67
SOURCES/0002-server-Allow-D-debug-flags-to-contain-dots-for-names.patch
View File

@ -1,67 +0,0 @@
From e5d2d44fff9214725506cbc84e7b3c035ec0eae9 Mon Sep 17 00:00:00 2001
From: "Richard W.M. Jones" <rjones@redhat.com>
Date: Thu, 12 Dec 2019 11:06:36 +0000
Subject: [PATCH 02/19] server: Allow -D debug flags to contain dots for
namespacing.
This is just a convenience. Either of:
-D myplugin.foo_bar=1
-D myplugin.foo.bar=1
correspond to the same plugin variable "myplugin_debug_foo_bar".
(cherry picked from commit a895fa84aaa50f52af68319523020046394c789f)
---
docs/nbdkit-plugin.pod | 8 ++++++++
server/debug-flags.c | 10 +++++++++-
2 files changed, 17 insertions(+), 1 deletion(-)
diff --git a/docs/nbdkit-plugin.pod b/docs/nbdkit-plugin.pod
index b69cb825..879ddf09 100644
--- a/docs/nbdkit-plugin.pod
+++ b/docs/nbdkit-plugin.pod
@@ -1298,6 +1298,14 @@ You should only use this feature for debug settings. For general
settings use ordinary plugin parameters. Debug Flags can only be C
ints. They are not supported by non-C language plugins.
+For convenience C<'.'> characters are replaced with C<'_'> characters
+in the variable name, so both of these parameters:
+
+ -D myplugin.foo_bar=1
+ -D myplugin.foo.bar=1
+
+correspond to the plugin variable C<myplugin_debug_foo_bar>.
+
=head1 INSTALLING THE PLUGIN
The plugin is a C<*.so> file and possibly a manual page. You can of
diff --git a/server/debug-flags.c b/server/debug-flags.c
index 9344d85c..5e06f5ed 100644
--- a/server/debug-flags.c
+++ b/server/debug-flags.c
@@ -56,12 +56,20 @@ static char *
symbol_of_debug_flag (const char *name, const char *flag)
{
char *var;
+ size_t i;
+ int len;
- if (asprintf (&var, "%s_debug_%s", name, flag) == -1) {
+ if ((len = asprintf (&var, "%s_debug_%s", name, flag)) == -1) {
perror ("asprintf");
exit (EXIT_FAILURE);
}
+ /* If there are any '.'s remaining in the name, convert them to '_'. */
+ for (i = 0; i < (size_t) len; ++i) {
+ if (var[i] == '.')
+ var[i] = '_';
+ }
+
return var; /* caller frees */
}
--
2.18.2

94
SOURCES/0002-server-CVE-2021-3716-reset-structured-replies-on-sta.patch Normal file
View File

@ -0,0 +1,94 @@
From 6b9d4380df9bd0be91f49aad8c4f47b4e672adde Mon Sep 17 00:00:00 2001
From: Eric Blake <eblake@redhat.com>
Date: Mon, 16 Aug 2021 13:43:29 -0500
Subject: [PATCH] server: CVE-2021-3716 reset structured replies on starttls
https://nostarttls.secvuln.info/ pointed out a series of CVEs in
common implementation flaw in various SMTP and IMAP clients and
servers, all with a common thread of improperly caching plaintext
state across the STARTTLS encryption boundary; and recommended that
other protocols with a STARTTLS operation perform a similar audit.
It turns out that nbdkit has the same vulnerability in regards to the
NBD protocol: when nbdkit is run in opportunistic TLS mode, an
attacker is able to inject a plaintext NBD_OPT_STRUCTURED_REPLY before
proxying everything else a client sends to the server; if the server
then acts on that plaintext request (as nbdkit did before this patch),
then the server ends up sending structured replies to at least
NBD_CMD_READ, even though the client was assuming that the transition
to TLS has ruled out a MitM attack.
On the bright side, nbdkit's behavior on a second
NBD_OPT_STRUCTURED_REPLY was to still reply with success, so a client
that always requests structured replies after starting TLS sees no
difference in behavior (that is, qemu 2.12 and later are immune) (had
nbdkit given an error to the second request, that may have caused
confusion to more clients). And there is always the mitigation of
using --tls=require, which lets nbdkit reject the MitM message
pre-encryption. However, nbd-client 3.15 to the present do not
understand structured replies, and I have confirmed that a MitM
attacker can thus cause a denial-of-service attack that does not
trigger until the client does its first encrypted NBD_CMD_READ.
The NBD spec has been recently tightened to declare the nbdkit
behavior to be a security hole:
https://github.com/NetworkBlockDevice/nbd/commit/77e55378096aa
Fixes: eaa4c6e9a2c4bd (server: Minimal implementation of NBD Structured Replies.)
(cherry picked from commit 09a13dafb7bb3a38ab52eb5501cba786365ba7fd)
(cherry picked from commit 6185b15a81e6915734d678f0781e31d45a7941a1)
---
docs/nbdkit-security.pod | 11 +++++++++--
server/protocol-handshake-newstyle.c | 3 ++-
2 files changed, 11 insertions(+), 3 deletions(-)
diff --git a/docs/nbdkit-security.pod b/docs/nbdkit-security.pod
index 3a28e54d..5a4e6da8 100644
--- a/docs/nbdkit-security.pod
+++ b/docs/nbdkit-security.pod
@@ -10,7 +10,7 @@ For how to report new security issues, see the C<SECURITY> file in the
top level source directory, also available online here:
L<https://github.com/libguestfs/nbdkit/blob/master/SECURITY>
-=head2 CVE-2019-14850
+=head2 CVE-2019-14850
denial of service due to premature opening of back-end connection
See the full announcement and links to mitigation, tests and fixes
@@ -26,6 +26,13 @@ See the full announcement and links to mitigation, tests and fixes
here:
https://www.redhat.com/archives/libguestfs/2019-September/msg00272.html
+=head2 CVE-2021-3716
+structured read denial of service attack against starttls
+
+See the full announcement and links to mitigation, tests and fixes
+here:
+https://www.redhat.com/archives/libguestfs/2021-August/msg00083.html
+
=head1 SEE ALSO
L<nbdkit(1)>.
@@ -38,4 +45,4 @@ Richard W.M. Jones
=head1 COPYRIGHT
-Copyright (C) 2013-2020 Red Hat Inc.
+Copyright (C) 2013-2021 Red Hat Inc.
diff --git a/server/protocol-handshake-newstyle.c b/server/protocol-handshake-newstyle.c
index 0a76a814..b94950e2 100644
--- a/server/protocol-handshake-newstyle.c
+++ b/server/protocol-handshake-newstyle.c
@@ -495,7 +495,8 @@ negotiate_handshake_newstyle_options (void)
return -1;
conn->using_tls = true;
debug ("using TLS on this connection");
- /* Wipe out any cached default export name. */
+ /* Wipe out any cached state. */
+ conn->structured_replies = false;
for_each_backend (b) {
struct handle *h = get_handle (conn, b->i);
free (h->default_exportname);
--
2.31.1

451
SOURCES/0003-server-Add-D-nbdkit.backend.controlpath-and-D-nbdkit.patch
View File

@ -1,451 +0,0 @@
From 83c72d9bf9d6a9ccf6939b4ebd0028b62673a78a Mon Sep 17 00:00:00 2001
From: "Richard W.M. Jones" <rjones@redhat.com>
Date: Thu, 12 Dec 2019 10:57:52 +0000
Subject: [PATCH 03/19] server: Add -D nbdkit.backend.controlpath and -D
nbdkit.backend.datapath.
These can be used to suppress verbose debugging messages from the
backend.
BugLink: https://bugzilla.redhat.com/1782868
Cherry picked from commit 231717e8cd5f27d76631be6651062d5a5ccf7fdc.
Remove use of nofilter from the test.
---
docs/nbdkit.pod | 35 ++++++++++++-
server/backend.c | 83 ++++++++++++++++++------------
tests/Makefile.am | 4 ++
tests/test-nbdkit-backend-debug.sh | 70 +++++++++++++++++++++++++
4 files changed, 158 insertions(+), 34 deletions(-)
create mode 100755 tests/test-nbdkit-backend-debug.sh
diff --git a/docs/nbdkit.pod b/docs/nbdkit.pod
index 346d8332..38e6bfca 100644
--- a/docs/nbdkit.pod
+++ b/docs/nbdkit.pod
@@ -182,7 +182,7 @@ value C<N>. See L<nbdkit-plugin(3)/Debug Flags>.
=item B<--debug> nbdkit.FLAG=N
Set the nbdkit server Debug Flag called C<FLAG> to the integer value
-C<N>.
+C<N>. See L</SERVER DEBUG FLAGS> below.
=item B<--dump-config>
@@ -527,6 +527,39 @@ languages. The file should be executable. For example:
(see L<nbdkit-perl-plugin(3)> for a full example).
+=head1 SERVER DEBUG FLAGS
+
+As well as enabling or disabling debugging in the server using
+I<--verbose> you can control extra debugging in the server using the
+C<-D nbdkit.*> flags listed in this section. Note these flags are an
+internal implementation detail of the server and may be changed or
+removed at any time in the future.
+
+=over 4
+
+=item B<-D nbdkit.backend.controlpath=0>
+
+=item B<-D nbdkit.backend.controlpath=1>
+
+=item B<-D nbdkit.backend.datapath=0>
+
+=item B<-D nbdkit.backend.datapath=1>
+
+These flags control the verbosity of nbdkit backend debugging messages
+(the ones which show every request processed by the server). The
+default for both settings is C<1> (normal debugging) but you can set
+them to C<0> to suppress these messages.
+
+C<-D nbdkit.backend.datapath=0> is the more useful setting which lets you
+suppress messages about pread, pwrite, zero, trim, etc. commands.
+When transferring large amounts of data these messages are numerous
+and not usually very interesting.
+
+C<-D nbdkit.backend.controlpath=0> suppresses the non-datapath
+commands (config, open, close, can_write, etc.)
+
+=back
+
=head1 SIGNALS
nbdkit responds to the following signals:
diff --git a/server/backend.c b/server/backend.c
index b9fe2a21..208c07b1 100644
--- a/server/backend.c
+++ b/server/backend.c
@@ -46,6 +46,22 @@
/* Helpers for registering a new backend. */
+/* Use:
+ * -D nbdkit.backend.controlpath=0 to suppress control path debugging.
+ * -D nbdkit.backend.datapath=0 to suppress data path debugging.
+ */
+int nbdkit_debug_backend_controlpath = 1;
+int nbdkit_debug_backend_datapath = 1;
+
+#define controlpath_debug(fs, ...) \
+ do { \
+ if (nbdkit_debug_backend_controlpath) debug ((fs), ##__VA_ARGS__); \
+ } while (0)
+#define datapath_debug(fs, ...) \
+ do { \
+ if (nbdkit_debug_backend_datapath) debug ((fs), ##__VA_ARGS__); \
+ } while (0)
+
void
backend_init (struct backend *b, struct backend *next, size_t index,
const char *filename, void *dl, const char *type)
@@ -108,7 +124,7 @@ backend_load (struct backend *b, const char *name, void (*load) (void))
apply_debug_flags (b->dl, name);
/* Call the on-load callback if it exists. */
- debug ("%s: load", name);
+ controlpath_debug ("%s: load", name);
if (load)
load ();
}
@@ -121,7 +137,7 @@ backend_unload (struct backend *b, void (*unload) (void))
*/
lock_unload ();
- debug ("%s: unload %s", b->name, b->type);
+ controlpath_debug ("%s: unload %s", b->name, b->type);
if (unload)
unload ();
@@ -139,7 +155,7 @@ backend_open (struct backend *b, struct connection *conn, int readonly)
{
struct b_conn_handle *h = &conn->handles[b->i];
- debug ("%s: open readonly=%d", b->name, readonly);
+ controlpath_debug ("%s: open readonly=%d", b->name, readonly);
assert (h->handle == NULL);
assert ((h->state & HANDLE_OPEN) == 0);
@@ -151,7 +167,7 @@ backend_open (struct backend *b, struct connection *conn, int readonly)
* inner-to-outer ordering.
*/
h->handle = b->open (b, conn, readonly);
- debug ("%s: open returned handle %p", b->name, h->handle);
+ controlpath_debug ("%s: open returned handle %p", b->name, h->handle);
if (h->handle == NULL) {
if (b->i) /* Do not strand backend if this layer failed */
@@ -179,7 +195,7 @@ backend_prepare (struct backend *b, struct connection *conn)
if (b->i && backend_prepare (b->next, conn) == -1)
return -1;
- debug ("%s: prepare readonly=%d", b->name, h->can_write == 0);
+ controlpath_debug ("%s: prepare readonly=%d", b->name, h->can_write == 0);
if (b->prepare (b, conn, h->handle, h->can_write == 0) == -1)
return -1;
@@ -196,7 +212,7 @@ backend_finalize (struct backend *b, struct connection *conn)
* filter furthest away from the plugin, and matching .close order.
*/
- debug ("%s: finalize", b->name);
+ controlpath_debug ("%s: finalize", b->name);
/* Once finalize fails, we can do nothing further on this connection */
if (h->state & HANDLE_FAILED)
@@ -223,7 +239,7 @@ backend_close (struct backend *b, struct connection *conn)
struct b_conn_handle *h = &conn->handles[b->i];
/* outer-to-inner order, opposite .open */
- debug ("%s: close", b->name);
+ controlpath_debug ("%s: close", b->name);
if (h->handle) {
assert (h->state & HANDLE_OPEN);
@@ -252,7 +268,7 @@ backend_valid_range (struct backend *b, struct connection *conn,
int
backend_reopen (struct backend *b, struct connection *conn, int readonly)
{
- debug ("%s: reopen readonly=%d", b->name, readonly);
+ controlpath_debug ("%s: reopen readonly=%d", b->name, readonly);
if (backend_finalize (b, conn) == -1)
return -1;
@@ -274,7 +290,7 @@ backend_get_size (struct backend *b, struct connection *conn)
{
struct b_conn_handle *h = &conn->handles[b->i];
- debug ("%s: get_size", b->name);
+ controlpath_debug ("%s: get_size", b->name);
assert (h->handle && (h->state & HANDLE_CONNECTED));
if (h->exportsize == -1)
@@ -287,7 +303,7 @@ backend_can_write (struct backend *b, struct connection *conn)
{
struct b_conn_handle *h = &conn->handles[b->i];
- debug ("%s: can_write", b->name);
+ controlpath_debug ("%s: can_write", b->name);
assert (h->handle && (h->state & HANDLE_CONNECTED));
if (h->can_write == -1)
@@ -300,7 +316,7 @@ backend_can_flush (struct backend *b, struct connection *conn)
{
struct b_conn_handle *h = &conn->handles[b->i];
- debug ("%s: can_flush", b->name);
+ controlpath_debug ("%s: can_flush", b->name);
assert (h->handle && (h->state & HANDLE_CONNECTED));
if (h->can_flush == -1)
@@ -313,7 +329,7 @@ backend_is_rotational (struct backend *b, struct connection *conn)
{
struct b_conn_handle *h = &conn->handles[b->i];
- debug ("%s: is_rotational", b->name);
+ controlpath_debug ("%s: is_rotational", b->name);
assert (h->handle && (h->state & HANDLE_CONNECTED));
if (h->is_rotational == -1)
@@ -327,7 +343,7 @@ backend_can_trim (struct backend *b, struct connection *conn)
struct b_conn_handle *h = &conn->handles[b->i];
int r;
- debug ("%s: can_trim", b->name);
+ controlpath_debug ("%s: can_trim", b->name);
assert (h->handle && (h->state & HANDLE_CONNECTED));
if (h->can_trim == -1) {
@@ -347,7 +363,7 @@ backend_can_zero (struct backend *b, struct connection *conn)
struct b_conn_handle *h = &conn->handles[b->i];
int r;
- debug ("%s: can_zero", b->name);
+ controlpath_debug ("%s: can_zero", b->name);
assert (h->handle && (h->state & HANDLE_CONNECTED));
if (h->can_zero == -1) {
@@ -367,7 +383,7 @@ backend_can_fast_zero (struct backend *b, struct connection *conn)
struct b_conn_handle *h = &conn->handles[b->i];
int r;
- debug ("%s: can_fast_zero", b->name);
+ controlpath_debug ("%s: can_fast_zero", b->name);
assert (h->handle && (h->state & HANDLE_CONNECTED));
if (h->can_fast_zero == -1) {
@@ -386,7 +402,7 @@ backend_can_extents (struct backend *b, struct connection *conn)
{
struct b_conn_handle *h = &conn->handles[b->i];
- debug ("%s: can_extents", b->name);
+ controlpath_debug ("%s: can_extents", b->name);
assert (h->handle && (h->state & HANDLE_CONNECTED));
if (h->can_extents == -1)
@@ -400,7 +416,7 @@ backend_can_fua (struct backend *b, struct connection *conn)
struct b_conn_handle *h = &conn->handles[b->i];
int r;
- debug ("%s: can_fua", b->name);
+ controlpath_debug ("%s: can_fua", b->name);
assert (h->handle && (h->state & HANDLE_CONNECTED));
if (h->can_fua == -1) {
@@ -420,7 +436,7 @@ backend_can_multi_conn (struct backend *b, struct connection *conn)
struct b_conn_handle *h = &conn->handles[b->i];
assert (h->handle && (h->state & HANDLE_CONNECTED));
- debug ("%s: can_multi_conn", b->name);
+ controlpath_debug ("%s: can_multi_conn", b->name);
if (h->can_multi_conn == -1)
h->can_multi_conn = b->can_multi_conn (b, conn, h->handle);
@@ -432,7 +448,7 @@ backend_can_cache (struct backend *b, struct connection *conn)
{
struct b_conn_handle *h = &conn->handles[b->i];
- debug ("%s: can_cache", b->name);
+ controlpath_debug ("%s: can_cache", b->name);
assert (h->handle && (h->state & HANDLE_CONNECTED));
if (h->can_cache == -1)
@@ -451,8 +467,8 @@ backend_pread (struct backend *b, struct connection *conn,
assert (h->handle && (h->state & HANDLE_CONNECTED));
assert (backend_valid_range (b, conn, offset, count));
assert (flags == 0);
- debug ("%s: pread count=%" PRIu32 " offset=%" PRIu64,
- b->name, count, offset);
+ datapath_debug ("%s: pread count=%" PRIu32 " offset=%" PRIu64,
+ b->name, count, offset);
r = b->pread (b, conn, h->handle, buf, count, offset, flags, err);
if (r == -1)
@@ -475,8 +491,8 @@ backend_pwrite (struct backend *b, struct connection *conn,
assert (!(flags & ~NBDKIT_FLAG_FUA));
if (fua)
assert (h->can_fua > NBDKIT_FUA_NONE);
- debug ("%s: pwrite count=%" PRIu32 " offset=%" PRIu64 " fua=%d",
- b->name, count, offset, fua);
+ datapath_debug ("%s: pwrite count=%" PRIu32 " offset=%" PRIu64 " fua=%d",
+ b->name, count, offset, fua);
r = b->pwrite (b, conn, h->handle, buf, count, offset, flags, err);
if (r == -1)
@@ -494,7 +510,7 @@ backend_flush (struct backend *b, struct connection *conn,
assert (h->handle && (h->state & HANDLE_CONNECTED));
assert (h->can_flush == 1);
assert (flags == 0);
- debug ("%s: flush", b->name);
+ datapath_debug ("%s: flush", b->name);
r = b->flush (b, conn, h->handle, flags, err);
if (r == -1)
@@ -518,8 +534,8 @@ backend_trim (struct backend *b, struct connection *conn,
assert (!(flags & ~NBDKIT_FLAG_FUA));
if (fua)
assert (h->can_fua > NBDKIT_FUA_NONE);
- debug ("%s: trim count=%" PRIu32 " offset=%" PRIu64 " fua=%d",
- b->name, count, offset, fua);
+ datapath_debug ("%s: trim count=%" PRIu32 " offset=%" PRIu64 " fua=%d",
+ b->name, count, offset, fua);
r = b->trim (b, conn, h->handle, count, offset, flags, err);
if (r == -1)
@@ -547,9 +563,10 @@ backend_zero (struct backend *b, struct connection *conn,
assert (h->can_fua > NBDKIT_FUA_NONE);
if (fast)
assert (h->can_fast_zero == 1);
- debug ("%s: zero count=%" PRIu32 " offset=%" PRIu64
- " may_trim=%d fua=%d fast=%d",
- b->name, count, offset, !!(flags & NBDKIT_FLAG_MAY_TRIM), fua, fast);
+ datapath_debug ("%s: zero count=%" PRIu32 " offset=%" PRIu64
+ " may_trim=%d fua=%d fast=%d",
+ b->name, count, offset,
+ !!(flags & NBDKIT_FLAG_MAY_TRIM), fua, fast);
r = b->zero (b, conn, h->handle, count, offset, flags, err);
if (r == -1) {
@@ -572,8 +589,8 @@ backend_extents (struct backend *b, struct connection *conn,
assert (h->can_extents >= 0);
assert (backend_valid_range (b, conn, offset, count));
assert (!(flags & ~NBDKIT_FLAG_REQ_ONE));
- debug ("%s: extents count=%" PRIu32 " offset=%" PRIu64 " req_one=%d",
- b->name, count, offset, !!(flags & NBDKIT_FLAG_REQ_ONE));
+ datapath_debug ("%s: extents count=%" PRIu32 " offset=%" PRIu64 " req_one=%d",
+ b->name, count, offset, !!(flags & NBDKIT_FLAG_REQ_ONE));
if (h->can_extents == 0) {
/* By default it is safe assume that everything in the range is
@@ -602,8 +619,8 @@ backend_cache (struct backend *b, struct connection *conn,
assert (h->can_cache > NBDKIT_CACHE_NONE);
assert (backend_valid_range (b, conn, offset, count));
assert (flags == 0);
- debug ("%s: cache count=%" PRIu32 " offset=%" PRIu64,
- b->name, count, offset);
+ datapath_debug ("%s: cache count=%" PRIu32 " offset=%" PRIu64,
+ b->name, count, offset);
if (h->can_cache == NBDKIT_CACHE_EMULATE) {
static char buf[MAX_REQUEST_SIZE]; /* data sink, never read */
diff --git a/tests/Makefile.am b/tests/Makefile.am
index 01341973..d225cc63 100644
--- a/tests/Makefile.am
+++ b/tests/Makefile.am
@@ -135,6 +135,7 @@ EXTRA_DIST = \
test-nbd-extents.sh \
test-nbd-tls.sh \
test-nbd-tls-psk.sh \
+ test-nbdkit-backend-debug.sh \
test-nozero.sh \
test-null-extents.sh \
test_ocaml_plugin.ml \
@@ -746,6 +747,9 @@ endif HAVE_VDDK
# zero plugin test.
TESTS += test-zero.sh
+# -D nbdkit.backend.* settings.
+TESTS += test-nbdkit-backend-debug.sh
+
#----------------------------------------------------------------------
# Tests of language plugins.
diff --git a/tests/test-nbdkit-backend-debug.sh b/tests/test-nbdkit-backend-debug.sh
new file mode 100755
index 00000000..69a69a7c
--- /dev/null
+++ b/tests/test-nbdkit-backend-debug.sh
@@ -0,0 +1,70 @@
+#!/usr/bin/env bash
+# nbdkit
+# Copyright (C) 2019 Red Hat Inc.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#
+# * Redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in the
+# documentation and/or other materials provided with the distribution.
+#
+# * Neither the name of Red Hat nor the names of its contributors may be
+# used to endorse or promote products derived from this software without
+# specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY RED HAT AND CONTRIBUTORS ''AS IS'' AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+# PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL RED HAT OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
+# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+# SUCH DAMAGE.
+
+source ./functions.sh
+set -x
+set -e
+
+requires qemu-img --version
+
+out="test-nbdkit-backend-debug.out"
+debug="test-nbdkit-backend-debug.debug"
+files="$out $debug"
+rm -f $files
+cleanup_fn rm -f $files
+
+nbdkit -U - \
+ -v \
+ memory 10M \
+ --run "qemu-img convert \$nbd $out" |& tee $debug
+
+# Should contain all debugging messages.
+grep '^nbdkit:.*debug: memory: open' $debug
+grep '^nbdkit:.*debug: memory: pread' $debug
+
+nbdkit -U - \
+ -v -D nbdkit.backend.controlpath=0 \
+ memory 10M \
+ --run "qemu-img convert \$nbd $out" |& tee $debug
+
+# Should contain only datapath messages.
+grep -v '^nbdkit:.*debug: memory: open' $debug
+grep '^nbdkit:.*debug: memory: pread' $debug
+
+nbdkit -U - \
+ -v -D nbdkit.backend.datapath=0 \
+ memory 10M \
+ --run "qemu-img convert \$nbd $out" |& tee $debug
+
+# Should contain only controlpath messages.
+grep '^nbdkit:.*debug: memory: open' $debug
+grep -v '^nbdkit:.*debug: memory: pread' $debug
--
2.18.2

40
SOURCES/0003-server-reset-meta-context-replies-on-starttls.patch Normal file
View File

@ -0,0 +1,40 @@
From add9b794b9dc697a1b52115c997fcfb6e06bf64c Mon Sep 17 00:00:00 2001
From: Eric Blake <eblake@redhat.com>
Date: Mon, 16 Aug 2021 13:43:29 -0500
Subject: [PATCH] server: reset meta context replies on starttls
Related to CVE-2021-3716, but not as severe. No compliant client will
send NBD_CMD_BLOCK_STATUS unless it first negotiates
NBD_OPT_SET_META_CONTEXT. If an attacker injects a premature
SET_META_CONTEXT, either the client will never notice (because it
never uses BLOCK_STATUS), or the client will overwrite the attacker's
attempt with the client's own SET_META_CONTEXT request after
encryption is enabled. So I don't class this as having the potential
to trigger denial-of-service due to any protocol mismatch between
compliant client and server (I don't care what happens with
non-compliant clients).
Fixes: 26455d45 (server: protocol: Implement Block Status "base:allocation".)
(cherry picked from commit 6c5faac6a37077cf2366388a80862bb00616d0d8)
(cherry picked from commit 814d8103fb4b581dc01dfd25d2cd81596576f211)
---
server/protocol-handshake-newstyle.c | 3 +++
1 file changed, 3 insertions(+)
diff --git a/server/protocol-handshake-newstyle.c b/server/protocol-handshake-newstyle.c
index b94950e2..eb0f3961 100644
--- a/server/protocol-handshake-newstyle.c
+++ b/server/protocol-handshake-newstyle.c
@@ -497,6 +497,9 @@ negotiate_handshake_newstyle_options (void)
debug ("using TLS on this connection");
/* Wipe out any cached state. */
conn->structured_replies = false;
+ free (conn->exportname_from_set_meta_context);
+ conn->exportname_from_set_meta_context = NULL;
+ conn->meta_context_base_allocation = false;
for_each_backend (b) {
struct handle *h = get_handle (conn, b->i);
free (h->default_exportname);
--
2.31.1

59
SOURCES/0004-cow-Fix-for-qemu-6.1-which-requires-backing-format.patch Normal file
View File

@ -0,0 +1,59 @@
From 3c2879a38c299b725091cea45329879e3f46fc99 Mon Sep 17 00:00:00 2001
From: "Richard W.M. Jones" <rjones@redhat.com>
Date: Tue, 31 Aug 2021 11:23:27 +0100
Subject: [PATCH] cow: Fix for qemu 6.1 which requires backing format
The diffing example in the manual created a qcow2 file with a backing
file but did not specify the backing format. However qemu 6.1 now
requires this and fails with:
qemu-img: cow-diff.qcow2: Backing file specified without backing format
or:
qemu-img: Could not change the backing file to 'cow-base.img': backing format must be specified
Fix the example by adding the -F option to the command line.
Also there was a test of this rebasing sequence which failed, so this
commit updates the test too.
(cherry picked from commit 618290ef33ce13b75c1a79fea1f1ffb327b5ba07)
---
filters/cow/nbdkit-cow-filter.pod | 4 ++--
tests/test-cow.sh | 4 ++--
2 files changed, 4 insertions(+), 4 deletions(-)
diff --git a/filters/cow/nbdkit-cow-filter.pod b/filters/cow/nbdkit-cow-filter.pod
index 4d5ae856..510bdd40 100644
--- a/filters/cow/nbdkit-cow-filter.pod
+++ b/filters/cow/nbdkit-cow-filter.pod
@@ -101,8 +101,8 @@ At the end, disconnect the client.
Run these C<qemu-img> commands to construct a qcow2 file containing
the differences:
- qemu-img create -f qcow2 -b nbd:localhost diff.qcow2
- qemu-img rebase -b disk.img diff.qcow2
+ qemu-img create -F raw -b nbd:localhost -f qcow2 diff.qcow2
+ qemu-img rebase -F raw -b disk.img -f qcow2 diff.qcow2
F<diff.qcow2> now contains the differences between the base
(F<disk.img>) and the changes stored in nbdkit-cow-filter. C<nbdkit>
diff --git a/tests/test-cow.sh b/tests/test-cow.sh
index 8772afd7..edc4c223 100755
--- a/tests/test-cow.sh
+++ b/tests/test-cow.sh
@@ -72,8 +72,8 @@ fi
# If we have qemu-img, try the hairy rebase operation documented
# in the nbdkit-cow-filter manual.
if qemu-img --version >/dev/null 2>&1; then
- qemu-img create -f qcow2 -b nbd:unix:$sock cow-diff.qcow2
- time qemu-img rebase -b cow-base.img cow-diff.qcow2
+ qemu-img create -F raw -b nbd:unix:$sock -f qcow2 cow-diff.qcow2
+ time qemu-img rebase -F raw -b cow-base.img -f qcow2 cow-diff.qcow2
qemu-img info cow-diff.qcow2
# This checks the file we created exists.
--
2.31.1

65
SOURCES/0004-python-Add-various-constants-to-the-API.patch
View File

@ -1,65 +0,0 @@
From b646050b8da51c39cf21f95fa847c12784a1169c Mon Sep 17 00:00:00 2001
From: "Richard W.M. Jones" <rjones@redhat.com>
Date: Thu, 21 Nov 2019 15:02:44 +0000
Subject: [PATCH 04/19] python: Add various constants to the API.
These are accessible from the plugin by:
import nbdkit
if flags & nbdkit.FLAG_MAY_TRIM:
&c.
Many (all?) of these are not yet useful for plugins, some will never
be useful, but they only consume a tiny bit of memory and it's nice to
have the complete set available for future use.
(cherry picked from commit 14b7fe2e0de881e3dfc8803484ade29a61e323c9)
---
plugins/python/python.c | 30 ++++++++++++++++++++++++++++++
1 file changed, 30 insertions(+)
diff --git a/plugins/python/python.c b/plugins/python/python.c
index 7052aac0..47da0838 100644
--- a/plugins/python/python.c
+++ b/plugins/python/python.c
@@ -231,6 +231,36 @@ create_nbdkit_module (void)
nbdkit_error ("could not create the nbdkit API module");
exit (EXIT_FAILURE);
}
+
+ /* Constants corresponding to various flags. */
+#define ADD_INT_CONSTANT(name) \
+ if (PyModule_AddIntConstant (m, #name, NBDKIT_##name) == -1) { \
+ nbdkit_error ("could not add constant %s to nbdkit API module", \
+ #name); \
+ exit (EXIT_FAILURE); \
+ }
+ ADD_INT_CONSTANT (THREAD_MODEL_SERIALIZE_CONNECTIONS);
+ ADD_INT_CONSTANT (THREAD_MODEL_SERIALIZE_ALL_REQUESTS);
+ ADD_INT_CONSTANT (THREAD_MODEL_SERIALIZE_REQUESTS);
+ ADD_INT_CONSTANT (THREAD_MODEL_PARALLEL);
+
+ ADD_INT_CONSTANT (FLAG_MAY_TRIM);
+ ADD_INT_CONSTANT (FLAG_FUA);
+ ADD_INT_CONSTANT (FLAG_REQ_ONE);
+ ADD_INT_CONSTANT (FLAG_FAST_ZERO);
+
+ ADD_INT_CONSTANT (FUA_NONE);
+ ADD_INT_CONSTANT (FUA_EMULATE);
+ ADD_INT_CONSTANT (FUA_NATIVE);
+
+ ADD_INT_CONSTANT (CACHE_NONE);
+ ADD_INT_CONSTANT (CACHE_EMULATE);
+ ADD_INT_CONSTANT (CACHE_NATIVE);
+
+ ADD_INT_CONSTANT (EXTENT_HOLE);
+ ADD_INT_CONSTANT (EXTENT_ZERO);
+#undef ADD_INT_CONSTANT
+
return m;
}
--
2.18.2

558
SOURCES/0005-python-Implement-nbdkit-API-version-2.patch
View File

@ -1,558 +0,0 @@
From 49ef7e7d7c3602cc8e53d2052fce9d3a12840ea2 Mon Sep 17 00:00:00 2001
From: "Richard W.M. Jones" <rjones@redhat.com>
Date: Thu, 21 Nov 2019 15:44:39 +0000
Subject: [PATCH 05/19] python: Implement nbdkit API version 2.
To avoid breaking existing plugins, Python plugins wishing to use
version 2 of the API must opt in by declaring:
API_VERSION = 2
(Plugins which do not do this are assumed to want API version 1).
For v2 API, we also avoid a copy by passing a buffer into pread.
It's more efficient if we pass the C buffer directly to Python code.
In some cases the Python code will be able to write directly into the
C buffer using functions like file.readinto and socket.recv_into.
This avoids an extra copy.
Thanks: Nir Soffer
https://www.redhat.com/archives/libguestfs/2019-November/thread.html#00220
(cherry picked from commit a9b2637cf4f00fb8a25ffaf31ee83be5fe019ae2)
---
plugins/python/example.py | 20 +++-
plugins/python/nbdkit-python-plugin.pod | 69 +++++++-----
plugins/python/python.c | 139 +++++++++++++++++++-----
tests/python-exception.py | 4 +-
tests/shebang.py | 5 +-
tests/test.py | 28 +++--
6 files changed, 190 insertions(+), 75 deletions(-)
diff --git a/plugins/python/example.py b/plugins/python/example.py
index 60f9d7f0..c04b7e29 100644
--- a/plugins/python/example.py
+++ b/plugins/python/example.py
@@ -34,6 +34,12 @@ import errno
disk = bytearray(1024 * 1024)
+# There are several variants of the API. nbdkit will call this
+# function first to determine which one you want to use. This is the
+# latest version at the time this example was written.
+API_VERSION = 2
+
+
# This just prints the extra command line parameters, but real plugins
# should parse them and reject any unknown parameters.
def config(key, value):
@@ -54,20 +60,22 @@ def get_size(h):
return len(disk)
-def pread(h, count, offset):
+def pread(h, buf, offset, flags):
global disk
- return disk[offset:offset+count]
+ end = offset + len(buf)
+ buf[:] = disk[offset:end]
+ # or if reading from a file you can use:
+ #f.readinto(buf)
-
-def pwrite(h, buf, offset):
+def pwrite(h, buf, offset, flags):
global disk
end = offset + len(buf)
disk[offset:end] = buf
-def zero(h, count, offset, may_trim):
+def zero(h, count, offset, flags):
global disk
- if may_trim:
+ if flags & nbdkit.FLAG_MAY_TRIM:
disk[offset:offset+count] = bytearray(count)
else:
nbdkit.set_error(errno.EOPNOTSUPP)
diff --git a/plugins/python/nbdkit-python-plugin.pod b/plugins/python/nbdkit-python-plugin.pod
index 3680fd65..4923d9da 100644
--- a/plugins/python/nbdkit-python-plugin.pod
+++ b/plugins/python/nbdkit-python-plugin.pod
@@ -33,11 +33,12 @@ To write a Python nbdkit plugin, you create a Python file which
contains at least the following required functions (in the top level
C<__main__> module):
+ API_VERSION = 2
def open(readonly):
# see below
def get_size(h):
# see below
- def pread(h, count, offset):
+ def pread(h, buf, offset, flags):
# see below
Note that the subroutines must have those literal names (like C<open>),
@@ -82,6 +83,18 @@ I<--dump-plugin> option, eg:
python_version=3.7.0
python_pep_384_abi_version=3
+=head2 API versions
+
+The nbdkit API has evolved and new versions are released periodically.
+To ensure backwards compatibility plugins have to opt in to the new
+version. From Python you do this by declaring a constant in your
+module:
+
+ API_VERSION = 2
+
+(where 2 is the latest version at the time this documentation was
+written). All newly written Python modules must have this constant.
+
=head2 Executable script
If you want you can make the script executable and include a "shebang"
@@ -199,16 +212,12 @@ contents will be garbage collected.
(Required)
- def pread(h, count, offset):
- # construct a buffer of length count bytes and return it
+ def pread(h, buf, offset, flags):
+ # read into the buffer
-The body of your C<pread> function should construct a buffer of length
-(at least) C<count> bytes. You should read C<count> bytes from the
-disk starting at C<offset>.
-
-The returned buffer can be any type compatible with the Python 3
-buffer protocol, such as bytearray, bytes or memoryview
-(L<https://docs.python.org/3/c-api/buffer.html>)
+The body of your C<pread> function should read exactly C<len(buf)>
+bytes of data starting at disk C<offset> and write it into the buffer
+C<buf>. C<flags> is always 0.
NBD only supports whole reads, so your function should try to read
the whole region (perhaps requiring a loop). If the read fails or
@@ -219,13 +228,13 @@ C<nbdkit.set_error> first.
(Optional)
- def pwrite(h, buf, offset):
+ def pwrite(h, buf, offset, flags):
length = len (buf)
# no return value
The body of your C<pwrite> function should write the buffer C<buf> to
the disk. You should write C<count> bytes to the disk starting at
-C<offset>.
+C<offset>. C<flags> may contain C<nbdkit.FLAG_FUA>.
NBD only supports whole writes, so your function should try to
write the whole region (perhaps requiring a loop). If the write
@@ -236,11 +245,12 @@ fails or is partial, your function should throw an exception,
(Optional)
- def flush(h):
+ def flush(h, flags):
# no return value
The body of your C<flush> function should do a L<sync(2)> or
L<fdatasync(2)> or equivalent on the backing store.
+C<flags> is always 0.
If the flush fails, your function should throw an exception, optionally
using C<nbdkit.set_error> first.
@@ -249,32 +259,35 @@ using C<nbdkit.set_error> first.
(Optional)
- def trim(h, count, offset):
+ def trim(h, count, offset, flags):
# no return value
-The body of your C<trim> function should "punch a hole" in the
-backing store. If the trim fails, your function should throw an
-exception, optionally using C<nbdkit.set_error> first.
+The body of your C<trim> function should "punch a hole" in the backing
+store. C<flags> may contain C<nbdkit.FLAG_FUA>. If the trim fails,
+your function should throw an exception, optionally using
+C<nbdkit.set_error> first.
=item C<zero>
(Optional)
- def zero(h, count, offset, may_trim):
+ def zero(h, count, offset, flags):
# no return value
-The body of your C<zero> function should ensure that C<count> bytes
-of the disk, starting at C<offset>, will read back as zero. If
-C<may_trim> is true, the operation may be optimized as a trim as long
-as subsequent reads see zeroes.
+The body of your C<zero> function should ensure that C<count> bytes of
+the disk, starting at C<offset>, will read back as zero. C<flags> is
+a bitmask which may include C<nbdkit.FLAG_MAY_TRIM>,
+C<nbdkit.FLAG_FUA>, C<nbdkit.FLAG_FAST_ZERO>.
NBD only supports whole writes, so your function should try to
-write the whole region (perhaps requiring a loop). If the write
-fails or is partial, your function should throw an exception,
-optionally using C<nbdkit.set_error> first. In particular, if
-you would like to automatically fall back to C<pwrite> (perhaps
-because there is nothing to optimize if C<may_trim> is false),
-use C<nbdkit.set_error(errno.EOPNOTSUPP)>.
+write the whole region (perhaps requiring a loop).
+
+If the write fails or is partial, your function should throw an
+exception, optionally using C<nbdkit.set_error> first. In particular,
+if you would like to automatically fall back to C<pwrite> (perhaps
+because there is nothing to optimize if
+S<C<flags & nbdkit.FLAG_MAY_TRIM>> is false), use
+S<C<nbdkit.set_error (errno.EOPNOTSUPP)>>.
=back
diff --git a/plugins/python/python.c b/plugins/python/python.c
index 47da0838..0f28595f 100644
--- a/plugins/python/python.c
+++ b/plugins/python/python.c
@@ -46,6 +46,8 @@
#define PY_SSIZE_T_CLEAN 1
#include <Python.h>
+#define NBDKIT_API_VERSION 2
+
#include <nbdkit-plugin.h>
#include "cleanup.h"
@@ -60,6 +62,7 @@
*/
static const char *script;
static PyObject *module;
+static int py_api_version = 1;
static int last_error;
@@ -285,9 +288,14 @@ py_dump_plugin (void)
PyObject *fn;
PyObject *r;
+ /* Python version and ABI. */
printf ("python_version=%s\n", PY_VERSION);
printf ("python_pep_384_abi_version=%d\n", PYTHON_ABI_VERSION);
+ /* Maximum nbdkit API version supported. */
+ printf ("nbdkit_python_maximum_api_version=%d\n", NBDKIT_API_VERSION);
+
+ /* If the script has a dump_plugin function, call it. */
if (script && callback_defined ("dump_plugin", &fn)) {
PyErr_Clear ();
@@ -297,6 +305,30 @@ py_dump_plugin (void)
}
}
+static int
+get_py_api_version (void)
+{
+ PyObject *obj;
+ long value;
+
+ obj = PyObject_GetAttrString (module, "API_VERSION");
+ if (obj == NULL)
+ return 1; /* Default to API version 1. */
+
+ value = PyLong_AsLong (obj);
+ Py_DECREF (obj);
+
+ if (value < 1 || value > NBDKIT_API_VERSION) {
+ nbdkit_error ("%s: API_VERSION requested unknown version: %ld. "
+ "This plugin supports API versions between 1 and %d.",
+ script, value, NBDKIT_API_VERSION);
+ return -1;
+ }
+
+ nbdkit_debug ("module requested API_VERSION %ld", value);
+ return (int) value;
+}
+
static int
py_config (const char *key, const char *value)
{
@@ -359,6 +391,11 @@ py_config (const char *key, const char *value)
"nbdkit requires these callbacks.", script);
return -1;
}
+
+ /* Get the API version. */
+ py_api_version = get_py_api_version ();
+ if (py_api_version == -1)
+ return -1;
}
else if (callback_defined ("config", &fn)) {
/* Other parameters are passed to the Python .config callback. */
@@ -469,8 +506,8 @@ py_get_size (void *handle)
}
static int
-py_pread (void *handle, void *buf,
- uint32_t count, uint64_t offset)
+py_pread (void *handle, void *buf, uint32_t count, uint64_t offset,
+ uint32_t flags)
{
PyObject *obj = handle;
PyObject *fn;
@@ -485,24 +522,40 @@ py_pread (void *handle, void *buf,
PyErr_Clear ();
- r = PyObject_CallFunction (fn, "OiL", obj, count, offset);
+ switch (py_api_version) {
+ case 1:
+ r = PyObject_CallFunction (fn, "OiL", obj, count, offset);
+ break;
+ case 2:
+ r = PyObject_CallFunction (fn, "ONLI", obj,
+ PyMemoryView_FromMemory ((char *)buf, count, PyBUF_WRITE),
+ offset, flags);
+ break;
+ default: abort ();
+ }
Py_DECREF (fn);
if (check_python_failure ("pread") == -1)
return ret;
- if (PyObject_GetBuffer (r, &view, PyBUF_SIMPLE) == -1) {
- nbdkit_error ("%s: value returned from pread does not support the "
- "buffer protocol",
- script);
- goto out;
- }
+ if (py_api_version == 1) {
+ /* In API v1 the Python pread function had to return a buffer
+ * protocol compatible function. In API v2+ it writes directly to
+ * the C buffer so this code is not used.
+ */
+ if (PyObject_GetBuffer (r, &view, PyBUF_SIMPLE) == -1) {
+ nbdkit_error ("%s: value returned from pread does not support the "
+ "buffer protocol",
+ script);
+ goto out;
+ }
- if (view.len < count) {
- nbdkit_error ("%s: buffer returned from pread is too small", script);
- goto out;
- }
+ if (view.len < count) {
+ nbdkit_error ("%s: buffer returned from pread is too small", script);
+ goto out;
+ }
- memcpy (buf, view.buf, count);
+ memcpy (buf, view.buf, count);
+ }
ret = 0;
out:
@@ -515,8 +568,8 @@ out:
}
static int
-py_pwrite (void *handle, const void *buf,
- uint32_t count, uint64_t offset)
+py_pwrite (void *handle, const void *buf, uint32_t count, uint64_t offset,
+ uint32_t flags)
{
PyObject *obj = handle;
PyObject *fn;
@@ -525,9 +578,19 @@ py_pwrite (void *handle, const void *buf,
if (callback_defined ("pwrite", &fn)) {
PyErr_Clear ();
- r = PyObject_CallFunction (fn, "ONL", obj,
+ switch (py_api_version) {
+ case 1:
+ r = PyObject_CallFunction (fn, "ONL", obj,
PyMemoryView_FromMemory ((char *)buf, count, PyBUF_READ),
offset);
+ break;
+ case 2:
+ r = PyObject_CallFunction (fn, "ONLI", obj,
+ PyMemoryView_FromMemory ((char *)buf, count, PyBUF_READ),
+ offset, flags);
+ break;
+ default: abort ();
+ }
Py_DECREF (fn);
if (check_python_failure ("pwrite") == -1)
return -1;
@@ -542,7 +605,7 @@ py_pwrite (void *handle, const void *buf,
}
static int
-py_flush (void *handle)
+py_flush (void *handle, uint32_t flags)
{
PyObject *obj = handle;
PyObject *fn;
@@ -551,7 +614,15 @@ py_flush (void *handle)
if (callback_defined ("flush", &fn)) {
PyErr_Clear ();
- r = PyObject_CallFunctionObjArgs (fn, obj, NULL);
+ switch (py_api_version) {
+ case 1:
+ r = PyObject_CallFunctionObjArgs (fn, obj, NULL);
+ break;
+ case 2:
+ r = PyObject_CallFunction (fn, "OI", obj, flags);
+ break;
+ default: abort ();
+ }
Py_DECREF (fn);
if (check_python_failure ("flush") == -1)
return -1;
@@ -566,7 +637,7 @@ py_flush (void *handle)
}
static int
-py_trim (void *handle, uint32_t count, uint64_t offset)
+py_trim (void *handle, uint32_t count, uint64_t offset, uint32_t flags)
{
PyObject *obj = handle;
PyObject *fn;
@@ -575,7 +646,15 @@ py_trim (void *handle, uint32_t count, uint64_t offset)
if (callback_defined ("trim", &fn)) {
PyErr_Clear ();
- r = PyObject_CallFunction (fn, "OiL", obj, count, offset);
+ switch (py_api_version) {
+ case 1:
+ r = PyObject_CallFunction (fn, "OiL", obj, count, offset);
+ break;
+ case 2:
+ r = PyObject_CallFunction (fn, "OiLI", obj, count, offset, flags);
+ break;
+ default: abort ();
+ }
Py_DECREF (fn);
if (check_python_failure ("trim") == -1)
return -1;
@@ -590,7 +669,7 @@ py_trim (void *handle, uint32_t count, uint64_t offset)
}
static int
-py_zero (void *handle, uint32_t count, uint64_t offset, int may_trim)
+py_zero (void *handle, uint32_t count, uint64_t offset, uint32_t flags)
{
PyObject *obj = handle;
PyObject *fn;
@@ -600,9 +679,19 @@ py_zero (void *handle, uint32_t count, uint64_t offset, int may_trim)
PyErr_Clear ();
last_error = 0;
- r = PyObject_CallFunction (fn, "OiLO",
- obj, count, offset,
- may_trim ? Py_True : Py_False);
+ switch (py_api_version) {
+ case 1: {
+ int may_trim = flags & NBDKIT_FLAG_MAY_TRIM;
+ r = PyObject_CallFunction (fn, "OiLO",
+ obj, count, offset,
+ may_trim ? Py_True : Py_False);
+ break;
+ }
+ case 2:
+ r = PyObject_CallFunction (fn, "OiLI", obj, count, offset, flags);
+ break;
+ default: abort ();
+ }
Py_DECREF (fn);
if (last_error == EOPNOTSUPP || last_error == ENOTSUP) {
/* When user requests this particular error, we want to
diff --git a/tests/python-exception.py b/tests/python-exception.py
index d0c79bb0..ee4a3f3a 100644
--- a/tests/python-exception.py
+++ b/tests/python-exception.py
@@ -62,5 +62,5 @@ def get_size(h):
return 0
-def pread(h, count, offset):
- return ""
+def pread(h, buf, offset):
+ buf[:] = bytearray(len(buf))
diff --git a/tests/shebang.py b/tests/shebang.py
index 6f336230..0634589a 100755
--- a/tests/shebang.py
+++ b/tests/shebang.py
@@ -13,6 +13,7 @@ def get_size(h):
return len(disk)
-def pread(h, count, offset):
+def pread(h, buf, offset):
global disk
- return disk[offset:offset+count]
+ end = offset + len(buf)
+ buf[:] = disk[offset:end]
diff --git a/tests/test.py b/tests/test.py
index 9a2e947d..4db56623 100644
--- a/tests/test.py
+++ b/tests/test.py
@@ -3,6 +3,9 @@ import nbdkit
disk = bytearray(1024*1024)
+API_VERSION = 2
+
+
def config_complete():
print ("set_error = %r" % nbdkit.set_error)
@@ -32,25 +35,26 @@ def can_trim(h):
return True
-def pread(h, count, offset):
+def pread(h, buf, offset, flags):
global disk
- return disk[offset:offset+count]
+ end = offset + len(buf)
+ buf[:] = disk[offset:end]
-def pwrite(h, buf, offset):
+def pwrite(h, buf, offset, flags):
global disk
end = offset + len(buf)
disk[offset:end] = buf
-def zero(h, count, offset, may_trim=False):
+def flush(h, flags):
+ pass
+
+
+def trim(h, count, offset, flags):
+ pass
+
+
+def zero(h, count, offset, flags):
global disk
disk[offset:offset+count] = bytearray(count)
-
-
-def flush(h):
- pass
-
-
-def trim(h, count, offset):
- pass
--
2.18.2

98
SOURCES/0006-python-Implement-cache.patch
View File

@ -1,98 +0,0 @@
From c5b1fac4c67078f0164bd23eab6d4d2b8c9830b0 Mon Sep 17 00:00:00 2001
From: "Richard W.M. Jones" <rjones@redhat.com>
Date: Thu, 21 Nov 2019 16:42:02 +0000
Subject: [PATCH 06/19] python: Implement cache.
However this does not implement can_cache, since that is not a simple
boolean.
(cherry picked from commit e61ffb73c7a0af0c383184fdb8f08d30784a195e)
---
plugins/python/nbdkit-python-plugin.pod | 14 ++++++++++-
plugins/python/python.c | 31 +++++++++++++++++++++++++
2 files changed, 44 insertions(+), 1 deletion(-)
diff --git a/plugins/python/nbdkit-python-plugin.pod b/plugins/python/nbdkit-python-plugin.pod
index 4923d9da..0ea8deef 100644
--- a/plugins/python/nbdkit-python-plugin.pod
+++ b/plugins/python/nbdkit-python-plugin.pod
@@ -289,6 +289,19 @@ because there is nothing to optimize if
S<C<flags & nbdkit.FLAG_MAY_TRIM>> is false), use
S<C<nbdkit.set_error (errno.EOPNOTSUPP)>>.
+=item C<cache>
+
+(Optional)
+
+ def cache(h, count, offset, flags):
+ # no return value
+
+The body of your C<cache> function should prefetch data in the
+indicated range.
+
+If the cache operation fails, your function should throw an exception,
+optionally using C<nbdkit.set_error> first.
+
=back
=head2 Missing callbacks
@@ -317,7 +330,6 @@ C<can_zero>,
C<can_fast_zero>,
C<can_extents>,
C<can_multi_conn>,
-C<cache>,
C<extents>.
These are not yet supported.
diff --git a/plugins/python/python.c b/plugins/python/python.c
index 0f28595f..c5cf38e5 100644
--- a/plugins/python/python.c
+++ b/plugins/python/python.c
@@ -714,6 +714,36 @@ py_zero (void *handle, uint32_t count, uint64_t offset, uint32_t flags)
return -1;
}
+static int
+py_cache (void *handle, uint32_t count, uint64_t offset, uint32_t flags)
+{
+ PyObject *obj = handle;
+ PyObject *fn;
+ PyObject *r;
+
+ if (callback_defined ("cache", &fn)) {
+ PyErr_Clear ();
+
+ switch (py_api_version) {
+ case 1:
+ case 2:
+ r = PyObject_CallFunction (fn, "OiLI", obj, count, offset, flags, NULL);
+ break;
+ default: abort ();
+ }
+ Py_DECREF (fn);
+ if (check_python_failure ("cache") == -1)
+ return -1;
+ Py_DECREF (r);
+ }
+ else {
+ nbdkit_error ("%s not implemented", "cache");
+ return -1;
+ }
+
+ return 0;
+}
+
static int
boolean_callback (void *handle, const char *can_fn, const char *plain_fn)
{
@@ -799,6 +829,7 @@ static struct nbdkit_plugin plugin = {
.flush = py_flush,
.trim = py_trim,
.zero = py_zero,
+ .cache = py_cache,
};
NBDKIT_REGISTER_PLUGIN (plugin)
--
2.18.2

80
SOURCES/0007-python-Implement-can_zero-can_fast_zero.patch
View File

@ -1,80 +0,0 @@
From 17721b316dd66b0a1ed792eeccd2489fb97828df Mon Sep 17 00:00:00 2001
From: "Richard W.M. Jones" <rjones@redhat.com>
Date: Thu, 21 Nov 2019 16:42:59 +0000
Subject: [PATCH 07/19] python: Implement can_zero, can_fast_zero.
(cherry picked from commit 039f600d2ad7a9ff04523a165eb2fe41b9c87c01)
---
plugins/python/nbdkit-python-plugin.pod | 16 ++++++++++++++--
plugins/python/python.c | 14 ++++++++++++++
2 files changed, 28 insertions(+), 2 deletions(-)
diff --git a/plugins/python/nbdkit-python-plugin.pod b/plugins/python/nbdkit-python-plugin.pod
index 0ea8deef..1f1c30f6 100644
--- a/plugins/python/nbdkit-python-plugin.pod
+++ b/plugins/python/nbdkit-python-plugin.pod
@@ -208,6 +208,20 @@ contents will be garbage collected.
def can_trim(h):
# return a boolean
+=item C<can_zero>
+
+(Optional)
+
+ def can_zero(h):
+ # return a boolean
+
+=item C<can_fast_zero>
+
+(Optional)
+
+ def can_fast_zero(h):
+ # return a boolean
+
=item C<pread>
(Required)
@@ -326,8 +340,6 @@ C<config_help>,
C<magic_config_key>,
C<can_fua>,
C<can_cache>,
-C<can_zero>,
-C<can_fast_zero>,
C<can_extents>,
C<can_multi_conn>,
C<extents>.
diff --git a/plugins/python/python.c b/plugins/python/python.c
index c5cf38e5..38fc1193 100644
--- a/plugins/python/python.c
+++ b/plugins/python/python.c
@@ -797,6 +797,18 @@ py_can_trim (void *handle)
return boolean_callback (handle, "can_trim", "trim");
}
+static int
+py_can_zero (void *handle)
+{
+ return boolean_callback (handle, "can_zero", "zero");
+}
+
+static int
+py_can_fast_zero (void *handle)
+{
+ return boolean_callback (handle, "can_fast_zero", NULL);
+}
+
#define py_config_help \
"script=<FILENAME> (required) The Python plugin to run.\n" \
"[other arguments may be used by the plugin that you load]"
@@ -823,6 +835,8 @@ static struct nbdkit_plugin plugin = {
.can_write = py_can_write,
.can_flush = py_can_flush,
.can_trim = py_can_trim,
+ .can_zero = py_can_zero,
+ .can_fast_zero = py_can_fast_zero,
.pread = py_pread,
.pwrite = py_pwrite,
--
2.18.2

65
SOURCES/0008-python-Implement-can_multi_conn.patch
View File

@ -1,65 +0,0 @@
From 2a85ce81ad95eb2f9b2f29666480b814ea0f80d9 Mon Sep 17 00:00:00 2001
From: "Richard W.M. Jones" <rjones@redhat.com>
Date: Thu, 21 Nov 2019 16:46:11 +0000
Subject: [PATCH 08/19] python: Implement can_multi_conn.
(cherry picked from commit 21dd7bf49d3238c7e75918d4bf324b617f458d83)
---
plugins/python/nbdkit-python-plugin.pod | 8 +++++++-
plugins/python/python.c | 7 +++++++
2 files changed, 14 insertions(+), 1 deletion(-)
diff --git a/plugins/python/nbdkit-python-plugin.pod b/plugins/python/nbdkit-python-plugin.pod
index 1f1c30f6..b92bb56a 100644
--- a/plugins/python/nbdkit-python-plugin.pod
+++ b/plugins/python/nbdkit-python-plugin.pod
@@ -187,6 +187,13 @@ contents will be garbage collected.
def is_rotational(h):
# return a boolean
+=item C<can_multi_conn>
+
+(Optional)
+
+ def can_multi_conn(h):
+ # return a boolean
+
=item C<can_write>
(Optional)
@@ -341,7 +348,6 @@ C<magic_config_key>,
C<can_fua>,
C<can_cache>,