rpms/nbdkit
7
0
Fork 0
Code Issues Pull Requests Releases Activity

Rebase along stable branch to 1.26.5

Browse Source 
resolves: rhbz#1995327
Add nbdkit-vddk-plugin -D vddk.stats=1 flag
resolves: rhbz#1995329
Add nbdkit-cow-filter cow-block-size parameter
resolves: rhbz#1995332
Fix CVE-2021-3716 nbdkit: NBD_OPT_STRUCTURED_REPLY injection on STARTTLS
Update keyring
This commit is contained in:
Richard W.M. Jones 2021-08-19 08:53:56 +01:00
parent b56fe54678
commit a86e62e68a
33 changed files with 1665 additions and 637 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

39
0001-server-reset-meta-context-replies-on-starttls.patch Normal file
View File

@ -0,0 +1,39 @@
From 89ef17c90996c0e212e3a17c8d26ff930ab464ea Mon Sep 17 00:00:00 2001
From: Eric Blake <eblake@redhat.com>
Date: Mon, 16 Aug 2021 13:43:29 -0500
Subject: [PATCH] server: reset meta context replies on starttls
Related to CVE-2021-3716, but not as severe. No compliant client will
send NBD_CMD_BLOCK_STATUS unless it first negotiates
NBD_OPT_SET_META_CONTEXT. If an attacker injects a premature
SET_META_CONTEXT, either the client will never notice (because it
never uses BLOCK_STATUS), or the client will overwrite the attacker's
attempt with the client's own SET_META_CONTEXT request after
encryption is enabled. So I don't class this as having the potential
to trigger denial-of-service due to any protocol mismatch between
compliant client and server (I don't care what happens with
non-compliant clients).
Fixes: 26455d45 (server: protocol: Implement Block Status "base:allocation".)
(cherry picked from commit 6c5faac6a37077cf2366388a80862bb00616d0d8)
---
server/protocol-handshake-newstyle.c | 3 +++
1 file changed, 3 insertions(+)
diff --git a/server/protocol-handshake-newstyle.c b/server/protocol-handshake-newstyle.c
index 7e6b7b1b..79b2c8ce 100644
--- a/server/protocol-handshake-newstyle.c
+++ b/server/protocol-handshake-newstyle.c
@@ -497,6 +497,9 @@ negotiate_handshake_newstyle_options (void)
debug ("using TLS on this connection");
/* Wipe out any cached state. */
conn->structured_replies = false;
+ free (conn->exportname_from_set_meta_context);
+ conn->exportname_from_set_meta_context = NULL;
+ conn->meta_context_base_allocation = false;
for_each_backend (b) {
free (conn->default_exportname[b->i]);
conn->default_exportname[b->i] = NULL;
--
2.31.1

2
0001-cache-Reduce-verbosity-of-debugging.patch → 0002-cache-Reduce-verbosity-of-debugging.patch
View File

@ -1,4 +1,4 @@
From 8785f90134fa912e31e72190d217db9c39754fcf Mon Sep 17 00:00:00 2001 From 4b576a8e0eb99ec1a79ca432350fb7ac27a5c089 Mon Sep 17 00:00:00 2001
From: "Richard W.M. Jones" <rjones@redhat.com> From: "Richard W.M. Jones" <rjones@redhat.com>
Date: Mon, 26 Jul 2021 11:59:43 +0100 Date: Mon, 26 Jul 2021 11:59:43 +0100
Subject: [PATCH] cache: Reduce verbosity of debugging Subject: [PATCH] cache: Reduce verbosity of debugging

10
0002-cache-cow-Add-blk_read_multiple-function.patch → 0003-cache-cow-Add-blk_read_multiple-function.patch
View File

@ -1,4 +1,4 @@
From f6d831cd517851157e822ee96de5894e0b37c22d Mon Sep 17 00:00:00 2001 From b5dc8577c5c6d1205e2106b629fad327c3a409ea Mon Sep 17 00:00:00 2001
From: "Richard W.M. Jones" <rjones@redhat.com> From: "Richard W.M. Jones" <rjones@redhat.com>
Date: Mon, 26 Jul 2021 13:55:21 +0100 Date: Mon, 26 Jul 2021 13:55:21 +0100
Subject: [PATCH] cache, cow: Add blk_read_multiple function Subject: [PATCH] cache, cow: Add blk_read_multiple function
@ -224,7 +224,7 @@ index 745f552d..14cc03f2 100644
/* Unaligned tail */ /* Unaligned tail */
diff --git a/filters/cow/blk.c b/filters/cow/blk.c diff --git a/filters/cow/blk.c b/filters/cow/blk.c
index 0f12d510..9e6c8879 100644 index b7c4d7f1..4ec8d1b8 100644
--- a/filters/cow/blk.c --- a/filters/cow/blk.c
+++ b/filters/cow/blk.c +++ b/filters/cow/blk.c
@@ -79,6 +79,7 @@ @@ -79,6 +79,7 @@
@ -235,7 +235,7 @@ index 0f12d510..9e6c8879 100644
#include <errno.h> #include <errno.h>
#include <sys/types.h> #include <sys/types.h>
@@ -223,33 +224,48 @@ blk_status (uint64_t blknum, bool *present, bool *trimmed) @@ -219,33 +220,48 @@ blk_status (uint64_t blknum, bool *present, bool *trimmed)
*trimmed = state == BLOCK_TRIMMED; *trimmed = state == BLOCK_TRIMMED;
} }
@ -295,7 +295,7 @@ index 0f12d510..9e6c8879 100644
if (offset + n > size) { if (offset + n > size) {
tail = offset + n - size; tail = offset + n - size;
@@ -264,20 +280,35 @@ blk_read (nbdkit_next *next, @@ -260,20 +276,35 @@ blk_read (nbdkit_next *next,
* zeroing the tail. * zeroing the tail.
*/ */
memset (block + n, 0, tail); memset (block + n, 0, tail);
@ -354,7 +354,7 @@ index e6fd7417..b066c602 100644
enum cache_mode { enum cache_mode {
BLK_CACHE_IGNORE, /* Do nothing */ BLK_CACHE_IGNORE, /* Do nothing */
diff --git a/filters/cow/cow.c b/filters/cow/cow.c diff --git a/filters/cow/cow.c b/filters/cow/cow.c
index 6cefee36..e939f23f 100644 index f30b7505..78daca22 100644
--- a/filters/cow/cow.c --- a/filters/cow/cow.c
+++ b/filters/cow/cow.c +++ b/filters/cow/cow.c
@@ -210,7 +210,7 @@ cow_pread (nbdkit_next *next, @@ -210,7 +210,7 @@ cow_pread (nbdkit_next *next,

10
0003-cache-cow-Use-full-pread-pwrite-operations.patch → 0004-cache-cow-Use-full-pread-pwrite-operations.patch
View File

@ -1,4 +1,4 @@
From 1b21e41dfd69b0a5f51657bc6f8b1cbc49663ee2 Mon Sep 17 00:00:00 2001 From 5bd332a683811586039f99f31c01d4f2f7181334 Mon Sep 17 00:00:00 2001
From: "Richard W.M. Jones" <rjones@redhat.com> From: "Richard W.M. Jones" <rjones@redhat.com>
Date: Mon, 26 Jul 2021 15:21:18 +0100 Date: Mon, 26 Jul 2021 15:21:18 +0100
Subject: [PATCH] cache, cow: Use full pread/pwrite operations Subject: [PATCH] cache, cow: Use full pread/pwrite operations
@ -180,10 +180,10 @@ index f85ada35..42bd3779 100644
nbdkit_error ("pwrite: %m"); nbdkit_error ("pwrite: %m");
return -1; return -1;
diff --git a/filters/cow/blk.c b/filters/cow/blk.c diff --git a/filters/cow/blk.c b/filters/cow/blk.c
index 9e6c8879..cebd9454 100644 index 4ec8d1b8..121b0dd4 100644
--- a/filters/cow/blk.c --- a/filters/cow/blk.c
+++ b/filters/cow/blk.c +++ b/filters/cow/blk.c
@@ -282,7 +282,7 @@ blk_read_multiple (nbdkit_next *next, @@ -278,7 +278,7 @@ blk_read_multiple (nbdkit_next *next,
memset (block + n, 0, tail); memset (block + n, 0, tail);
} }
else if (state == BLOCK_ALLOCATED) { /* Read overlay. */ else if (state == BLOCK_ALLOCATED) { /* Read overlay. */
@ -192,7 +192,7 @@ index 9e6c8879..cebd9454 100644
*err = errno; *err = errno;
nbdkit_error ("pread: %m"); nbdkit_error ("pread: %m");
return -1; return -1;
@@ -357,7 +357,7 @@ blk_cache (nbdkit_next *next, @@ -353,7 +353,7 @@ blk_cache (nbdkit_next *next,
memset (block + n, 0, tail); memset (block + n, 0, tail);
if (mode == BLK_CACHE_COW) { if (mode == BLK_CACHE_COW) {
@ -201,7 +201,7 @@ index 9e6c8879..cebd9454 100644
*err = errno; *err = errno;
nbdkit_error ("pwrite: %m"); nbdkit_error ("pwrite: %m");
return -1; return -1;
@@ -376,7 +376,7 @@ blk_write (uint64_t blknum, const uint8_t *block, int *err) @@ -372,7 +372,7 @@ blk_write (uint64_t blknum, const uint8_t *block, int *err)
nbdkit_debug ("cow: blk_write block %" PRIu64 " (offset %" PRIu64 ")", nbdkit_debug ("cow: blk_write block %" PRIu64 " (offset %" PRIu64 ")",
blknum, (uint64_t) offset); blknum, (uint64_t) offset);

2
0004-cache-Implement-cache-on-read-PATH.patch → 0005-cache-Implement-cache-on-read-PATH.patch
View File

@ -1,4 +1,4 @@
From dac1245e839d64cc5ee22ae4f804a950e487ff5d Mon Sep 17 00:00:00 2001 From 4db23fd29af0488aa9c7e01577a5be9565a4465e Mon Sep 17 00:00:00 2001
From: "Richard W.M. Jones" <rjones@redhat.com> From: "Richard W.M. Jones" <rjones@redhat.com>
Date: Mon, 26 Jul 2021 16:16:15 +0100 Date: Mon, 26 Jul 2021 16:16:15 +0100
Subject: [PATCH] cache: Implement cache-on-read=/PATH Subject: [PATCH] cache: Implement cache-on-read=/PATH

2
0005-cache-Add-cache-min-block-size-parameter.patch → 0006-cache-Add-cache-min-block-size-parameter.patch
View File

@ -1,4 +1,4 @@
From 9d1d3bb689a6a88bcc376b699d9668a3a623faee Mon Sep 17 00:00:00 2001 From f7f4b71d559dc6950bc795742f64e8eaeeadf3ec Mon Sep 17 00:00:00 2001
From: "Richard W.M. Jones" <rjones@redhat.com> From: "Richard W.M. Jones" <rjones@redhat.com>
Date: Mon, 26 Jul 2021 16:30:26 +0100 Date: Mon, 26 Jul 2021 16:30:26 +0100
Subject: [PATCH] cache: Add cache-min-block-size parameter Subject: [PATCH] cache: Add cache-min-block-size parameter

2
0006-cache-cow-Use-a-64K-block-size-by-default.patch → 0007-cache-cow-Use-a-64K-block-size-by-default.patch
View File

@ -1,4 +1,4 @@
From e0f76fe019449c81297de39ddd6a12006d94481a Mon Sep 17 00:00:00 2001 From 83e1167e1a350bd08ac6245f47a5877438408492 Mon Sep 17 00:00:00 2001
From: "Richard W.M. Jones" <rjones@redhat.com> From: "Richard W.M. Jones" <rjones@redhat.com>
Date: Mon, 26 Jul 2021 17:39:23 +0100 Date: Mon, 26 Jul 2021 17:39:23 +0100
Subject: [PATCH] cache, cow: Use a 64K block size by default Subject: [PATCH] cache, cow: Use a 64K block size by default

2
0007-cache-Refactor-printing-state-into-new-function.patch → 0008-cache-Refactor-printing-state-into-new-function.patch
View File

@ -1,4 +1,4 @@
From 9b4f5045c92f6a666c8d4b08379c34fb8e677dcd Mon Sep 17 00:00:00 2001 From 2592bb42051b3e6d17240badc814b9b16f121c1d Mon Sep 17 00:00:00 2001
From: "Richard W.M. Jones" <rjones@redhat.com> From: "Richard W.M. Jones" <rjones@redhat.com>
Date: Tue, 27 Jul 2021 21:16:30 +0100 Date: Tue, 27 Jul 2021 21:16:30 +0100
Subject: [PATCH] cache: Refactor printing state into new function Subject: [PATCH] cache: Refactor printing state into new function

2
0008-tests-cache-Test-cache-on-read-option-really-caches.patch → 0009-tests-cache-Test-cache-on-read-option-really-caches.patch
View File

@ -1,4 +1,4 @@
From fcd113cf7bf5587f8cac6d331ec1e427be0a830f Mon Sep 17 00:00:00 2001 From 315948e75e06d038bd8afa319a41e3fde33b4174 Mon Sep 17 00:00:00 2001
From: "Richard W.M. Jones" <rjones@redhat.com> From: "Richard W.M. Jones" <rjones@redhat.com>
Date: Thu, 29 Jul 2021 20:16:43 +0100 Date: Thu, 29 Jul 2021 20:16:43 +0100
Subject: [PATCH] tests: cache: Test cache-on-read option really caches Subject: [PATCH] tests: cache: Test cache-on-read option really caches

14
0009-cow-Implement-cow-on-read.patch → 0010-cow-Implement-cow-on-read.patch
View File

@ -1,4 +1,4 @@
From e7ff9f833cc11e47db90868a02fee95900f62a84 Mon Sep 17 00:00:00 2001 From 57f9bd29f9d7432ad5a70620c373b28db768a314 Mon Sep 17 00:00:00 2001
From: "Richard W.M. Jones" <rjones@redhat.com> From: "Richard W.M. Jones" <rjones@redhat.com>
Date: Tue, 27 Jul 2021 23:01:52 +0100 Date: Tue, 27 Jul 2021 23:01:52 +0100
Subject: [PATCH] cow: Implement cow-on-read Subject: [PATCH] cow: Implement cow-on-read
@ -19,10 +19,10 @@ This is very similar to the nbdkit-cache-filter cache-on-read flag.
create mode 100755 tests/test-cow-on-read.sh create mode 100755 tests/test-cow-on-read.sh
diff --git a/filters/cow/blk.c b/filters/cow/blk.c diff --git a/filters/cow/blk.c b/filters/cow/blk.c
index cebd9454..9d42b5fc 100644 index 121b0dd4..4f84e092 100644
--- a/filters/cow/blk.c --- a/filters/cow/blk.c
+++ b/filters/cow/blk.c +++ b/filters/cow/blk.c
@@ -230,7 +230,7 @@ blk_status (uint64_t blknum, bool *present, bool *trimmed) @@ -226,7 +226,7 @@ blk_status (uint64_t blknum, bool *present, bool *trimmed)
int int
blk_read_multiple (nbdkit_next *next, blk_read_multiple (nbdkit_next *next,
uint64_t blknum, uint64_t nrblocks, uint64_t blknum, uint64_t nrblocks,
@ -31,7 +31,7 @@ index cebd9454..9d42b5fc 100644
{ {
off_t offset = blknum * BLKSIZE; off_t offset = blknum * BLKSIZE;
enum bm_entry state; enum bm_entry state;
@@ -280,6 +280,19 @@ blk_read_multiple (nbdkit_next *next, @@ -276,6 +276,19 @@ blk_read_multiple (nbdkit_next *next,
* zeroing the tail. * zeroing the tail.
*/ */
memset (block + n, 0, tail); memset (block + n, 0, tail);
@ -51,7 +51,7 @@ index cebd9454..9d42b5fc 100644
} }
else if (state == BLOCK_ALLOCATED) { /* Read overlay. */ else if (state == BLOCK_ALLOCATED) { /* Read overlay. */
if (full_pread (fd, block, BLKSIZE * runblocks, offset) == -1) { if (full_pread (fd, block, BLKSIZE * runblocks, offset) == -1) {
@@ -301,14 +314,14 @@ blk_read_multiple (nbdkit_next *next, @@ -297,14 +310,14 @@ blk_read_multiple (nbdkit_next *next,
blknum + runblocks, blknum + runblocks,
nrblocks - runblocks, nrblocks - runblocks,
block + BLKSIZE * runblocks, block + BLKSIZE * runblocks,
@ -95,7 +95,7 @@ index 1bc85283..b7e6f092 100644
/* Cache mode for blocks not already in overlay */ /* Cache mode for blocks not already in overlay */
enum cache_mode { enum cache_mode {
diff --git a/filters/cow/cow.c b/filters/cow/cow.c diff --git a/filters/cow/cow.c b/filters/cow/cow.c
index e939f23f..6ad42eec 100644 index 78daca22..6efb39f2 100644
--- a/filters/cow/cow.c --- a/filters/cow/cow.c
+++ b/filters/cow/cow.c +++ b/filters/cow/cow.c
@@ -38,6 +38,7 @@ @@ -38,6 +38,7 @@
@ -108,7 +108,7 @@ index e939f23f..6ad42eec 100644
#include <pthread.h> #include <pthread.h>
@@ -59,6 +60,15 @@ static pthread_mutex_t rmw_lock = PTHREAD_MUTEX_INITIALIZER; @@ -59,6 +60,15 @@ static pthread_mutex_t rmw_lock = PTHREAD_MUTEX_INITIALIZER;
bool cow_on_cache; static bool cow_on_cache;
+/* Cache on read ("cow-on-read") mode. */ +/* Cache on read ("cow-on-read") mode. */
+extern enum cor_mode { +extern enum cor_mode {

42
0010-delay-Add-delay-open-and-delay-close.patch → 0011-delay-Add-delay-open-and-delay-close.patch
View File

@ -1,4 +1,4 @@
From a37d5a2176af0362ad3e4234fe1fc14a47978261 Mon Sep 17 00:00:00 2001 From a7e7af18d64164fac42581452f6dc3c07650fcae Mon Sep 17 00:00:00 2001
From: "Richard W.M. Jones" <rjones@redhat.com> From: "Richard W.M. Jones" <rjones@redhat.com>
Date: Fri, 30 Jul 2021 10:19:57 +0100 Date: Fri, 30 Jul 2021 10:19:57 +0100
Subject: [PATCH] delay: Add delay-open and delay-close Subject: [PATCH] delay: Add delay-open and delay-close
@ -7,24 +7,24 @@ Useful for simulating VDDK which has very slow connection.
(cherry picked from commit de8dcd3a34a38b088a0f9a6f8ca754702ad1f598) (cherry picked from commit de8dcd3a34a38b088a0f9a6f8ca754702ad1f598)
--- ---
filters/delay/delay.c | 60 ++++++++++++++++++++++++++- filters/delay/delay.c | 58 ++++++++++++++++++++++++++-
filters/delay/nbdkit-delay-filter.pod | 27 ++++++++++-- filters/delay/nbdkit-delay-filter.pod | 27 +++++++++++--
2 files changed, 82 insertions(+), 5 deletions(-) 2 files changed, 80 insertions(+), 5 deletions(-)
diff --git a/filters/delay/delay.c b/filters/delay/delay.c diff --git a/filters/delay/delay.c b/filters/delay/delay.c
index 7e7fe195..5bd21321 100644 index 5a925aa4..df3729a7 100644
--- a/filters/delay/delay.c --- a/filters/delay/delay.c
+++ b/filters/delay/delay.c +++ b/filters/delay/delay.c
@@ -49,6 +49,8 @@ static int delay_trim_ms = 0; /* trim delay (milliseconds) */ @@ -48,6 +48,8 @@ static unsigned delay_zero_ms = 0; /* zero delay (milliseconds) */
static int delay_extents_ms = 0;/* extents delay (milliseconds) */ static unsigned delay_trim_ms = 0; /* trim delay (milliseconds) */
static int delay_cache_ms = 0; /* cache delay (milliseconds) */ static unsigned delay_extents_ms = 0;/* extents delay (milliseconds) */
static int delay_fast_zero = 1; /* whether delaying zero includes fast zero */ static unsigned delay_cache_ms = 0; /* cache delay (milliseconds) */
+static int delay_open_ms = 0; /* open delay (milliseconds) */ +static unsigned delay_open_ms = 0; /* open delay (milliseconds) */
+static int delay_close_ms = 0; /* close delay (milliseconds) */ +static unsigned delay_close_ms = 0; /* close delay (milliseconds) */
static int static int delay_fast_zero = 1; /* whether delaying zero includes fast zero */
parse_delay (const char *key, const char *value)
@@ -128,6 +130,18 @@ cache_delay (int *err) @@ -126,6 +128,18 @@ cache_delay (int *err)
return delay (delay_cache_ms, err); return delay (delay_cache_ms, err);
} }
@ -43,26 +43,24 @@ index 7e7fe195..5bd21321 100644
/* Called for each key=value passed on the command line. */ /* Called for each key=value passed on the command line. */
static int static int
delay_config (nbdkit_next_config *next, nbdkit_backend *nxdata, delay_config (nbdkit_next_config *next, nbdkit_backend *nxdata,
@@ -191,6 +205,18 @@ delay_config (nbdkit_next_config *next, nbdkit_backend *nxdata, @@ -182,6 +196,16 @@ delay_config (nbdkit_next_config *next, nbdkit_backend *nxdata,
return -1; return -1;
return 0; return 0;
} }
+ else if (strcmp (key, "delay-open") == 0) { + else if (strcmp (key, "delay-open") == 0) {
+ delay_open_ms = parse_delay (key, value); + if (parse_delay (key, value, &delay_open_ms) == -1)
+ if (delay_open_ms == -1)
+ return -1; + return -1;
+ return 0; + return 0;
+ } + }
+ else if (strcmp (key, "delay-close") == 0) { + else if (strcmp (key, "delay-close") == 0) {
+ delay_close_ms = parse_delay (key, value); + if (parse_delay (key, value, &delay_close_ms) == -1)
+ if (delay_close_ms == -1)
+ return -1; + return -1;
+ return 0; + return 0;
+ } + }
else else
return next (nxdata, key, value); return next (nxdata, key, value);
} }
@@ -204,7 +230,9 @@ delay_config (nbdkit_next_config *next, nbdkit_backend *nxdata, @@ -195,7 +219,9 @@ delay_config (nbdkit_next_config *next, nbdkit_backend *nxdata,
"delay-extents=<NN>[ms] Extents delay in seconds/milliseconds.\n" \ "delay-extents=<NN>[ms] Extents delay in seconds/milliseconds.\n" \
"delay-cache=<NN>[ms] Cache delay in seconds/milliseconds.\n" \ "delay-cache=<NN>[ms] Cache delay in seconds/milliseconds.\n" \
"wdelay=<NN>[ms] Write, zero and trim delay in secs/msecs.\n" \ "wdelay=<NN>[ms] Write, zero and trim delay in secs/msecs.\n" \
@ -73,7 +71,7 @@ index 7e7fe195..5bd21321 100644
/* Override the plugin's .can_fast_zero if needed */ /* Override the plugin's .can_fast_zero if needed */
static int static int
@@ -217,6 +245,34 @@ delay_can_fast_zero (nbdkit_next *next, @@ -208,6 +234,34 @@ delay_can_fast_zero (nbdkit_next *next,
return next->can_fast_zero (next); return next->can_fast_zero (next);
} }
@ -108,7 +106,7 @@ index 7e7fe195..5bd21321 100644
/* Read data. */ /* Read data. */
static int static int
delay_pread (nbdkit_next *next, delay_pread (nbdkit_next *next,
@@ -294,6 +350,8 @@ static struct nbdkit_filter filter = { @@ -285,6 +339,8 @@ static struct nbdkit_filter filter = {
.config = delay_config, .config = delay_config,
.config_help = delay_config_help, .config_help = delay_config_help,
.can_fast_zero = delay_can_fast_zero, .can_fast_zero = delay_can_fast_zero,

2
0011-python-Implement-.cleanup-method.patch → 0012-python-Implement-.cleanup-method.patch
View File

@ -1,4 +1,4 @@
From 77310049efc37daaf7bd993298000f8f3638497e Mon Sep 17 00:00:00 2001 From 17a912a449fa75b5c12ac3acab596b476699c671 Mon Sep 17 00:00:00 2001
From: "Richard W.M. Jones" <rjones@redhat.com> From: "Richard W.M. Jones" <rjones@redhat.com>
Date: Tue, 3 Aug 2021 14:19:38 +0100 Date: Tue, 3 Aug 2021 14:19:38 +0100
Subject: [PATCH] python: Implement .cleanup() method Subject: [PATCH] python: Implement .cleanup() method

157
0012-server-Make-debug-messages-atomic.patch
View File

@ -1,157 +0,0 @@
From 453a7611b625fc6f306a47ccc61bf3b83d75e522 Mon Sep 17 00:00:00 2001
From: "Richard W.M. Jones" <rjones@redhat.com>
Date: Thu, 5 Aug 2021 10:13:53 +0100
Subject: [PATCH] server: Make debug messages atomic
Previously we used flockfile around the separate fprintf statements
used to emit debug messages. However this does not make debug
messages atomic (only atomic with respect to other nbdkit threads).
It's easy to see this in strace:
1915655 write(2, "nbdkit: ", 8) = 8
1915655 write(2, "debug: ", 7) = 7
1915655 write(2, "nbdkit 1.27.3 (nbdkit-1.27.3-1.f"..., 36) = 36
1915655 write(2, "\n", 1) = 1
We require open_memstream already, so use this to print messages to a
string buffer which we can emit in a single write.
This fixes various tests which grep the log file looking for sentinel
messages. In particular a recent Fedora build failure in
test-nbdkit-backend-debug.sh was caused by this, but there are other
tests that fail occasionally for the same reason.
This change also tries harder to set errno correctly before expanding
the format string (for debug messages that contain %m).
Note that libnbd already emits atomic messages.
(cherry picked from commit 30eef3bf2d93b12072f91f95987bae33f9e3fe1a)
---
server/debug.c | 72 ++++++++++++++++++++++++++++++++------------------
1 file changed, 46 insertions(+), 26 deletions(-)
diff --git a/server/debug.c b/server/debug.c
index eede5e16..07710581 100644
--- a/server/debug.c
+++ b/server/debug.c
@@ -40,23 +40,22 @@
#include "internal.h"
-/* Called with flockfile (stderr) taken. */
static void
-prologue (void)
+prologue (FILE *fp)
{
const char *name = threadlocal_get_name ();
size_t instance_num = threadlocal_get_instance_num ();
- fprintf (stderr, "%s: ", program_name);
+ fprintf (fp, "%s: ", program_name);
if (name) {
- fprintf (stderr, "%s", name);
+ fprintf (fp, "%s", name);
if (instance_num > 0)
- fprintf (stderr, "[%zu]", instance_num);
- fprintf (stderr, ": ");
+ fprintf (fp, "[%zu]", instance_num);
+ fprintf (fp, ": ");
}
- fprintf (stderr, "debug: ");
+ fprintf (fp, "debug: ");
}
/* Note: preserves the previous value of errno. */
@@ -64,20 +63,30 @@ NBDKIT_DLL_PUBLIC void
nbdkit_vdebug (const char *fs, va_list args)
{
int err = errno;
+ CLEANUP_FREE char *str = NULL;
+ size_t len = 0;
+ FILE *fp;
if (!verbose)
return;
-#ifdef HAVE_FLOCKFILE
- flockfile (stderr);
-#endif
- prologue ();
- vfprintf (stderr, fs, args);
+ fp = open_memstream (&str, &len);
+ if (fp == NULL) {
+ /* Try to emit what we can. */
+ errno = err;
+ vfprintf (stderr, fs, args);
+ return;
+ }
- fprintf (stderr, "\n");
-#ifdef HAVE_FUNLOCKFILE
- funlockfile (stderr);
-#endif
+ prologue (fp);
+
+ errno = err;
+ vfprintf (fp, fs, args);
+
+ fprintf (fp, "\n");
+ fclose (fp);
+
+ fputs (str, stderr);
errno = err;
}
@@ -86,25 +95,36 @@ nbdkit_vdebug (const char *fs, va_list args)
NBDKIT_DLL_PUBLIC void
nbdkit_debug (const char *fs, ...)
{
+ int err = errno;
va_list args;
- int err = errno;
+ CLEANUP_FREE char *str = NULL;
+ size_t len = 0;
+ FILE *fp;
if (!verbose)
return;
-#ifdef HAVE_FLOCKFILE
- flockfile (stderr);
-#endif
- prologue ();
+ fp = open_memstream (&str, &len);
+ if (fp == NULL) {
+ /* Try to emit what we can. */
+ va_start (args, fs);
+ errno = err;
+ vfprintf (stderr, fs, args);
+ va_end (args);
+ return;
+ }
+
+ prologue (fp);
va_start (args, fs);
- vfprintf (stderr, fs, args);
+ errno = err;
+ vfprintf (fp, fs, args);
va_end (args);
- fprintf (stderr, "\n");
-#ifdef HAVE_FUNLOCKFILE
- funlockfile (stderr);
-#endif
+ fprintf (fp, "\n");
+ fclose (fp);
+
+ fputs (str, stderr);
errno = err;
}
--
2.31.1

2
0013-cow-General-revision-and-updates-to-the-manual.patch
View File

@ -1,4 +1,4 @@
From d15e320d20260c14973ef84172ae8cbe337a2b48 Mon Sep 17 00:00:00 2001 From e9abe97b40fef6f9bd9028a2520f45203bba0749 Mon Sep 17 00:00:00 2001
From: "Richard W.M. Jones" <rjones@redhat.com> From: "Richard W.M. Jones" <rjones@redhat.com>
Date: Thu, 5 Aug 2021 18:18:34 +0100 Date: Thu, 5 Aug 2021 18:18:34 +0100
Subject: [PATCH] cow: General revision and updates to the manual Subject: [PATCH] cow: General revision and updates to the manual

2
0014-cache-Move-plugin-args-in-synopsis-earlier.patch
View File

@ -1,4 +1,4 @@
From 583d5308ea8d26248e521b76afb380432d2084bc Mon Sep 17 00:00:00 2001 From c8c1e74a8c1c112b83646ac09fe7f9bde097a52a Mon Sep 17 00:00:00 2001
From: "Richard W.M. Jones" <rjones@redhat.com> From: "Richard W.M. Jones" <rjones@redhat.com>
Date: Thu, 5 Aug 2021 18:20:37 +0100 Date: Thu, 5 Aug 2021 18:20:37 +0100
Subject: [PATCH] cache: Move plugin-args in synopsis earlier Subject: [PATCH] cache: Move plugin-args in synopsis earlier

2
0015-data-Improve-the-example-with-a-diagram.patch
View File

@ -1,4 +1,4 @@
From 72b87c985dc9324d896333f9ddaf317cece8a812 Mon Sep 17 00:00:00 2001 From 0eae7ebf6f714fb339f4a476b65e070b528824ec Mon Sep 17 00:00:00 2001
From: "Richard W.M. Jones" <rjones@redhat.com> From: "Richard W.M. Jones" <rjones@redhat.com>
Date: Sun, 8 Aug 2021 16:32:38 +0100 Date: Sun, 8 Aug 2021 16:32:38 +0100
Subject: [PATCH] data: Improve the example with a diagram Subject: [PATCH] data: Improve the example with a diagram

8
0016-cow-Add-some-more-debugging-especially-for-blk_read_.patch
View File

@ -1,4 +1,4 @@
From d2ed77d1b8fedcba3aedadeed883553886f4bb56 Mon Sep 17 00:00:00 2001 From a22248e3075e782d28542f8f6acd046c9dfa8998 Mon Sep 17 00:00:00 2001
From: "Richard W.M. Jones" <rjones@redhat.com> From: "Richard W.M. Jones" <rjones@redhat.com>
Date: Mon, 9 Aug 2021 14:09:31 +0100 Date: Mon, 9 Aug 2021 14:09:31 +0100
Subject: [PATCH] cow: Add some more debugging especially for blk_read_multiple Subject: [PATCH] cow: Add some more debugging especially for blk_read_multiple
@ -12,10 +12,10 @@ Only activated when we use -D cow.verbose=1
1 file changed, 9 insertions(+), 2 deletions(-) 1 file changed, 9 insertions(+), 2 deletions(-)
diff --git a/filters/cow/blk.c b/filters/cow/blk.c diff --git a/filters/cow/blk.c b/filters/cow/blk.c
index 9d42b5fc..879f471a 100644 index 4f84e092..c22d5886 100644
--- a/filters/cow/blk.c --- a/filters/cow/blk.c
+++ b/filters/cow/blk.c +++ b/filters/cow/blk.c
@@ -258,8 +258,10 @@ blk_read_multiple (nbdkit_next *next, @@ -254,8 +254,10 @@ blk_read_multiple (nbdkit_next *next,
if (cow_debug_verbose) if (cow_debug_verbose)
nbdkit_debug ("cow: blk_read_multiple block %" PRIu64 nbdkit_debug ("cow: blk_read_multiple block %" PRIu64
@ -28,7 +28,7 @@ index 9d42b5fc..879f471a 100644
if (state == BLOCK_NOT_ALLOCATED) { /* Read underlying plugin. */ if (state == BLOCK_NOT_ALLOCATED) { /* Read underlying plugin. */
unsigned n, tail = 0; unsigned n, tail = 0;
@@ -285,6 +287,11 @@ blk_read_multiple (nbdkit_next *next, @@ -281,6 +283,11 @@ blk_read_multiple (nbdkit_next *next,
* set them as allocated. * set them as allocated.
*/ */
if (cow_on_read) { if (cow_on_read) {

2
0021-delay-Fix-delay-close.patch → 0017-delay-Fix-delay-close.patch
View File

@ -1,4 +1,4 @@
From a8ea42d2aa3738c680a932e2c42257ce4a880a47 Mon Sep 17 00:00:00 2001 From be7252bada79ee542356dffaf5f3c568a5c7fec3 Mon Sep 17 00:00:00 2001
From: "Richard W.M. Jones" <rjones@redhat.com> From: "Richard W.M. Jones" <rjones@redhat.com>
Date: Tue, 10 Aug 2021 08:39:15 +0100 Date: Tue, 10 Aug 2021 08:39:15 +0100
Subject: [PATCH] delay: Fix delay-close Subject: [PATCH] delay: Fix delay-close

74
0017-tests-test-debug-flags.sh-Don-t-use-port-10809-durin.patch
View File

@ -1,74 +0,0 @@
From ad2b4d2c07def233b2192c2a7ff925d1b6b823e7 Mon Sep 17 00:00:00 2001
From: "Richard W.M. Jones" <rjones@redhat.com>
Date: Mon, 9 Aug 2021 14:48:15 +0100
Subject: [PATCH] tests/test-debug-flags.sh: Don't use port 10809 during test
Use a temporary Unix domain socket (-U -) instead.
Reported-by: Ming Xie
Fixes: commit 0e3a54f78f4ab0cbe4bee2b965ec0e610c399a6e
(cherry picked from commit 31f4e5c31825485e7bb39d170a7102ddbc4043c2)
---
tests/test-debug-flags.sh | 22 +++++++++++-----------
1 file changed, 11 insertions(+), 11 deletions(-)
diff --git a/tests/test-debug-flags.sh b/tests/test-debug-flags.sh
index 73be9904..2be799a1 100755
--- a/tests/test-debug-flags.sh
+++ b/tests/test-debug-flags.sh
@@ -46,7 +46,7 @@ rm -f $files
cleanup_fn rm -f $files
# This should work and show the "extra debugging" line in debug output.
-nbdkit -f -v -D example2.extra=1 example2 file=disk \
+nbdkit -U - -f -v -D example2.extra=1 example2 file=disk \
--run 'nbdinfo "$uri"' 2>debug-flags.out
cat debug-flags.out
if ! grep -sq 'extra debugging:' debug-flags.out ; then
@@ -82,33 +82,33 @@ check_warning ()
# This is expected to fail because we didn't set the file= parameter,
# but it should not fail because of the debug flag.
-nbdkit -f -D example2.extra=1 example2 2>debug-flags.out && expected_failure
+nbdkit -U - -f -D example2.extra=1 example2 2>debug-flags.out && expected_failure
check_error "you must supply the file="
# This should fail because we didn't set the file= parameter, but it
# should also print a warning about the unknown -D flag.
-nbdkit -f -D example2.unknown=1 example2 2>debug-flags.out && expected_failure
+nbdkit -U - -f -D example2.unknown=1 example2 2>debug-flags.out && expected_failure
check_error "you must supply the file="
check_warning "does not contain a global variable called example2_debug_unknown"
# This should fail because we didn't set the file= parameter, but it
# should also print a warning because the -D flag is unused.
-nbdkit -f -D example1.foo=1 example2 2>debug-flags.out && expected_failure
+nbdkit -U - -f -D example1.foo=1 example2 2>debug-flags.out && expected_failure
check_error "you must supply the file="
check_warning "was not used"
# These should fail because the -D flag has a bad format.
-nbdkit -f -D = example2 2>debug-flags.out && expected_failure
+nbdkit -U - -f -D = example2 2>debug-flags.out && expected_failure
check_error "must have the format"
-nbdkit -f -D . example2 2>debug-flags.out && expected_failure
+nbdkit -U - -f -D . example2 2>debug-flags.out && expected_failure
check_error "must have the format"
-nbdkit -f -D =. example2 2>debug-flags.out && expected_failure
+nbdkit -U - -f -D =. example2 2>debug-flags.out && expected_failure
check_error "must have the format"
-nbdkit -f -D .= example2 2>debug-flags.out && expected_failure
+nbdkit -U - -f -D .= example2 2>debug-flags.out && expected_failure
check_error "must have the format"
-nbdkit -f -D .extra=1 example2 2>debug-flags.out && expected_failure
+nbdkit -U - -f -D .extra=1 example2 2>debug-flags.out && expected_failure
check_error "must have the format"
-nbdkit -f -D example2.=1 example2 2>debug-flags.out && expected_failure
+nbdkit -U - -f -D example2.=1 example2 2>debug-flags.out && expected_failure
check_error "must have the format"
-nbdkit -f -D example2.extra= example2 2>debug-flags.out && expected_failure
+nbdkit -U - -f -D example2.extra= example2 2>debug-flags.out && expected_failure
check_error "must have the format"
--
2.31.1

182
0018-delay-Improve-parsing-and-representation-of-delay-ti.patch
View File

@ -1,182 +0,0 @@
From 810c2449cb519100bf9ea50d743162a391eac873 Mon Sep 17 00:00:00 2001
From: "Richard W.M. Jones" <rjones@redhat.com>
Date: Mon, 9 Aug 2021 16:16:36 +0100
Subject: [PATCH] delay: Improve parsing and representation of delay times
We used sscanf to parse the delay settings for both NNms and NN
(seconds). In the first case we need to use sscanf otherwise the code
is awkward. In the second case using sscanf meant that bogus suffices
were not ignored, eg:
$ nbdkit null --filter=delay delay-open=10SECS --run 'nbdinfo $uri'
Use nbdkit_parse_unsigned instead. This now results in an error:
nbdkit: error: delay-open: could not parse number: "10SECS": trailing garbage
Also, previously we stored delay times as an int. This makes no sense
since they cannot be negative, so use an unsigned instead.
Reported-by: Ming Xie
(cherry picked from commit 58187831a4346b44e398f105163abac8f3dfb7f0)
---
filters/delay/delay.c | 73 ++++++++++++++++++-------------------------
1 file changed, 31 insertions(+), 42 deletions(-)
diff --git a/filters/delay/delay.c b/filters/delay/delay.c
index 5bd21321..df3729a7 100644
--- a/filters/delay/delay.c
+++ b/filters/delay/delay.c
@@ -42,25 +42,28 @@
#include <nbdkit-filter.h>
-static int delay_read_ms = 0; /* read delay (milliseconds) */
-static int delay_write_ms = 0; /* write delay (milliseconds) */
-static int delay_zero_ms = 0; /* zero delay (milliseconds) */
-static int delay_trim_ms = 0; /* trim delay (milliseconds) */
-static int delay_extents_ms = 0;/* extents delay (milliseconds) */
-static int delay_cache_ms = 0; /* cache delay (milliseconds) */
+static unsigned delay_read_ms = 0; /* read delay (milliseconds) */
+static unsigned delay_write_ms = 0; /* write delay (milliseconds) */
+static unsigned delay_zero_ms = 0; /* zero delay (milliseconds) */
+static unsigned delay_trim_ms = 0; /* trim delay (milliseconds) */
+static unsigned delay_extents_ms = 0;/* extents delay (milliseconds) */
+static unsigned delay_cache_ms = 0; /* cache delay (milliseconds) */
+static unsigned delay_open_ms = 0; /* open delay (milliseconds) */
+static unsigned delay_close_ms = 0; /* close delay (milliseconds) */
+
static int delay_fast_zero = 1; /* whether delaying zero includes fast zero */
-static int delay_open_ms = 0; /* open delay (milliseconds) */
-static int delay_close_ms = 0; /* close delay (milliseconds) */
static int
-parse_delay (const char *key, const char *value)
+parse_delay (const char *key, const char *value, unsigned *r)
{
size_t len = strlen (value);
- int r;
if (len > 2 && strcmp (&value[len-2], "ms") == 0) {
- if (sscanf (value, "%d", &r) == 1 && r >= 0)
- return r;
+ /* We have to use sscanf here instead of nbdkit_parse_unsigned
+ * because that function will reject the "ms" suffix.
+ */
+ if (sscanf (value, "%u", r) == 1)
+ return 0;
else {
nbdkit_error ("cannot parse %s in milliseconds parameter: %s",
key, value);
@@ -68,24 +71,19 @@ parse_delay (const char *key, const char *value)
}
}
else {
- if (sscanf (value, "%d", &r) == 1 && r >= 0) {
- if (r * 1000LL > INT_MAX) {
- nbdkit_error ("seconds parameter %s is too large: %s",
- key, value);
- return -1;
- }
- return r * 1000;
- }
- else {
- nbdkit_error ("cannot parse %s in seconds parameter: %s",
- key, value);
+ if (nbdkit_parse_unsigned (key, value, r) == -1)
+ return -1;
+ if (*r * 1000U > UINT_MAX) {
+ nbdkit_error ("seconds parameter %s is too large: %s", key, value);
return -1;
}
+ *r *= 1000;
+ return 0;
}
}
static int
-delay (int ms, int *err)
+delay (unsigned ms, int *err)
{
if (ms > 0 && nbdkit_nanosleep (ms / 1000, (ms % 1000) * 1000000) == -1) {
*err = errno;
@@ -150,14 +148,12 @@ delay_config (nbdkit_next_config *next, nbdkit_backend *nxdata,
if (strcmp (key, "rdelay") == 0 ||
strcmp (key, "delay-read") == 0 ||
strcmp (key, "delay-reads") == 0) {
- delay_read_ms = parse_delay (key, value);
- if (delay_read_ms == -1)
+ if (parse_delay (key, value, &delay_read_ms) == -1)
return -1;
return 0;
}
else if (strcmp (key, "wdelay") == 0) {
- delay_write_ms = parse_delay (key, value);
- if (delay_write_ms == -1)
+ if (parse_delay (key, value, &delay_write_ms) == -1)
return -1;
/* Historically wdelay set all write-related delays. */
delay_zero_ms = delay_trim_ms = delay_write_ms;
@@ -165,15 +161,13 @@ delay_config (nbdkit_next_config *next, nbdkit_backend *nxdata,
}
else if (strcmp (key, "delay-write") == 0 ||
strcmp (key, "delay-writes") == 0) {
- delay_write_ms = parse_delay (key, value);
- if (delay_write_ms == -1)
+ if (parse_delay (key, value, &delay_write_ms) == -1)
return -1;
return 0;
}
else if (strcmp (key, "delay-zero") == 0 ||
strcmp (key, "delay-zeroes") == 0) {
- delay_zero_ms = parse_delay (key, value);
- if (delay_zero_ms == -1)
+ if (parse_delay (key, value, &delay_zero_ms) == -1)
return -1;
return 0;
}
@@ -181,21 +175,18 @@ delay_config (nbdkit_next_config *next, nbdkit_backend *nxdata,
strcmp (key, "delay-trims") == 0 ||
strcmp (key, "delay-discard") == 0 ||
strcmp (key, "delay-discards") == 0) {
- delay_trim_ms = parse_delay (key, value);
- if (delay_trim_ms == -1)
+ if (parse_delay (key, value, &delay_trim_ms) == -1)
return -1;
return 0;
}
else if (strcmp (key, "delay-extent") == 0 ||
strcmp (key, "delay-extents") == 0) {
- delay_extents_ms = parse_delay (key, value);
- if (delay_extents_ms == -1)
+ if (parse_delay (key, value, &delay_extents_ms) == -1)
return -1;
return 0;
}
else if (strcmp (key, "delay-cache") == 0) {
- delay_cache_ms = parse_delay (key, value);
- if (delay_cache_ms == -1)
+ if (parse_delay (key, value, &delay_cache_ms) == -1)
return -1;
return 0;
}
@@ -206,14 +197,12 @@ delay_config (nbdkit_next_config *next, nbdkit_backend *nxdata,
return 0;
}
else if (strcmp (key, "delay-open") == 0) {
- delay_open_ms = parse_delay (key, value);
- if (delay_open_ms == -1)
+ if (parse_delay (key, value, &delay_open_ms) == -1)
return -1;
return 0;
}
else if (strcmp (key, "delay-close") == 0) {
- delay_close_ms = parse_delay (key, value);
- if (delay_close_ms == -1)
+ if (parse_delay (key, value, &delay_close_ms) == -1)
return -1;
return 0;
}
--
2.31.1

2
0022-delay-Test-delay-open-and-delay-close.patch → 0018-delay-Test-delay-open-and-delay-close.patch
View File

@ -1,4 +1,4 @@
From 1f2acea1e6bd0a1120bf6b48854202ec8680f5c0 Mon Sep 17 00:00:00 2001 From 838ec052abe63056434c08ea80f4609e697dad0f Mon Sep 17 00:00:00 2001
From: "Richard W.M. Jones" <rjones@redhat.com> From: "Richard W.M. Jones" <rjones@redhat.com>
Date: Tue, 10 Aug 2021 09:11:43 +0100 Date: Tue, 10 Aug 2021 09:11:43 +0100
Subject: [PATCH] delay: Test delay-open and delay-close Subject: [PATCH] delay: Test delay-open and delay-close

41
0019-server-Return-from-nbdkit_nanosleep-early-if-the-soc.patch
View File

@ -1,41 +0,0 @@
From 33318699bf1255aef0c6ee4863236d26d7b326ec Mon Sep 17 00:00:00 2001
From: "Richard W.M. Jones" <rjones@redhat.com>
Date: Mon, 9 Aug 2021 20:11:41 +0100
Subject: [PATCH] server: Return from nbdkit_nanosleep early if the socket
closes
https://bugzilla.redhat.com/show_bug.cgi?id=1991652#c2
Reported-by: Ming Xie
Fixes: https://bugzilla.redhat.com/show_bug.cgi?id=1991652
(cherry picked from commit 87a88f8c52a0d2fd392c35d37f8b048bcede1382)
---
server/public.c | 5 ++++-
1 file changed, 4 insertions(+), 1 deletion(-)
diff --git a/server/public.c b/server/public.c
index 3362f1ab..4870e2d3 100644
--- a/server/public.c
+++ b/server/public.c
@@ -693,6 +693,8 @@ nbdkit_nanosleep (unsigned sec, unsigned nsec)
* - the current connection is multi-threaded and another thread detects
* NBD_CMD_DISC or a problem with the connection
* - the input socket detects POLLRDHUP/POLLHUP/POLLERR
+ * - the input socket is invalid (POLLNVAL, probably closed by
+ * another thread)
*/
struct connection *conn = threadlocal_get_conn ();
struct pollfd fds[] = {
@@ -724,7 +726,8 @@ nbdkit_nanosleep (unsigned sec, unsigned nsec)
*/
assert (quit ||
(conn && conn->nworkers > 0 && connection_get_status () < 1) ||
- (conn && (fds[2].revents & (POLLRDHUP | POLLHUP | POLLERR))));
+ (conn && (fds[2].revents & (POLLRDHUP | POLLHUP | POLLERR |
+ POLLNVAL))));
nbdkit_error ("aborting sleep to shut down");
errno = ESHUTDOWN;
return -1;
--
2.31.1

74
0019-vddk-Implement-can_flush-and-can_fua.patch Normal file
View File

@ -0,0 +1,74 @@
From 2104686eb708bf87070c21e7af0e70e0317306b6 Mon Sep 17 00:00:00 2001
From: "Richard W.M. Jones" <rjones@redhat.com>
Date: Mon, 5 Jul 2021 21:36:41 +0100
Subject: [PATCH] vddk: Implement can_flush and can_fua
VDDK < 6.0 doesn't support flush. Previously we advertised flush and
FUA but ignored them if VDDK didn't support it. Instead, correctly
set these flags in the NBD protocol according to what VDDK supports.
(cherry picked from commit 04b05274414a8cf4615eb2d6f46d5658814509c1)
---
plugins/vddk/vddk.c | 28 ++++++++++++++++++++--------
1 file changed, 20 insertions(+), 8 deletions(-)
diff --git a/plugins/vddk/vddk.c b/plugins/vddk/vddk.c
index 76faa768..b5bce9a0 100644
--- a/plugins/vddk/vddk.c
+++ b/plugins/vddk/vddk.c
@@ -772,12 +772,28 @@ vddk_pwrite (void *handle, const void *buf, uint32_t count, uint64_t offset,
return -1;
}
- if (fua && vddk_flush (handle, 0) == -1)
- return -1;
+ if (fua) {
+ if (vddk_flush (handle, 0) == -1)
+ return -1;
+ }
return 0;
}
+static int
+vddk_can_fua (void *handle)
+{
+ /* The Flush call was not available in VDDK < 6.0. */
+ return VixDiskLib_Flush != NULL ? NBDKIT_FUA_NATIVE : NBDKIT_FUA_NONE;
+}
+
+static int
+vddk_can_flush (void *handle)
+{
+ /* The Flush call was not available in VDDK < 6.0. */
+ return VixDiskLib_Flush != NULL;
+}
+
/* Flush data to the file. */
static int
vddk_flush (void *handle, uint32_t flags)
@@ -785,12 +801,6 @@ vddk_flush (void *handle, uint32_t flags)
struct vddk_handle *h = handle;
VixError err;
- /* The Flush call was not available in VDDK < 6.0 so this is simply
- * ignored on earlier versions.
- */
- if (VixDiskLib_Flush == NULL)
- return 0;
-
DEBUG_CALL ("VixDiskLib_Flush", "handle");
err = VixDiskLib_Flush (h->handle);
if (err != VIX_OK) {
@@ -985,6 +995,8 @@ static struct nbdkit_plugin plugin = {
.get_size = vddk_get_size,
.pread = vddk_pread,
.pwrite = vddk_pwrite,
+ .can_fua = vddk_can_fua,
+ .can_flush = vddk_can_flush,
.flush = vddk_flush,
.can_extents = vddk_can_extents,
.extents = vddk_extents,
--
2.31.1

43
0020-server-nanosleep-Change-error-for-early-end-of-sleep.patch
View File

@ -1,43 +0,0 @@
From 032531cd5d402119a81efbaf07d781123c5b02af Mon Sep 17 00:00:00 2001
From: "Richard W.M. Jones" <rjones@redhat.com>
Date: Tue, 10 Aug 2021 08:30:43 +0100
Subject: [PATCH] server: nanosleep: Change error for early end of sleep
At the moment nbdkit_nanosleep gives an incorrect error message if it
aborts early. Even in the case when the server is not actually
shutting down it will say:
$ nbdkit --filter=delay null delay-close=3 --run 'nbdinfo --size $uri; nbdinfo --size $uri'
0
nbdkit: null[1]: error: aborting sleep to shut down
0
nbdkit: null[2]: error: aborting sleep to shut down
This commit changes the error so we only talk about shut down when the
server is actually shutting down, and use a different message in other
cases.
(cherry picked from commit cd24d9c418992e6f2c721c7deec70e564c23ab83)
---
server/public.c | 5 ++++-
1 file changed, 4 insertions(+), 1 deletion(-)
diff --git a/server/public.c b/server/public.c
index 4870e2d3..d9ed0d9c 100644
--- a/server/public.c
+++ b/server/public.c
@@ -728,7 +728,10 @@ nbdkit_nanosleep (unsigned sec, unsigned nsec)
(conn && conn->nworkers > 0 && connection_get_status () < 1) ||
(conn && (fds[2].revents & (POLLRDHUP | POLLHUP | POLLERR |
POLLNVAL))));
- nbdkit_error ("aborting sleep to shut down");
+ if (quit)
+ nbdkit_error ("aborting sleep because of server shut down");
+ else
+ nbdkit_error ("aborting sleep because of connection close or error");
errno = ESHUTDOWN;
return -1;
--
2.31.1

319
0020-vddk-Replace-DEBUG_CALL-with-bracketed-VDDK_CALL_STA.patch Normal file
View File

@ -0,0 +1,319 @@
From 51713e7702d389fd55d5721c4773fca40e3e89f6 Mon Sep 17 00:00:00 2001
From: "Richard W.M. Jones" <rjones@redhat.com>
Date: Wed, 18 Aug 2021 14:26:30 +0100
Subject: [PATCH] vddk: Replace DEBUG_CALL with bracketed VDDK_CALL_START/END
macros
This is neutral refactoring, but allows us in the next commit to
collect statistics about the amount of time spent in these calls.
(cherry picked from commit 1335ebfb5637bf5a44403d0b152da7272fdd3e54)
---
plugins/vddk/vddk.c | 175 +++++++++++++++++++++++++-------------------
1 file changed, 99 insertions(+), 76 deletions(-)
diff --git a/plugins/vddk/vddk.c b/plugins/vddk/vddk.c
index b5bce9a0..888009ab 100644
--- a/plugins/vddk/vddk.c
+++ b/plugins/vddk/vddk.c
@@ -104,19 +104,23 @@ static bool is_remote;
VixDiskLib_FreeErrorText (vddk_err_msg); \
} while (0)
-#define DEBUG_CALL(fn, fs, ...) \
- nbdkit_debug ("VDDK call: %s (" fs ")", fn, ##__VA_ARGS__)
-#define DEBUG_CALL_DATAPATH(fn, fs, ...) \
- if (vddk_debug_datapath) \
- nbdkit_debug ("VDDK call: %s (" fs ")", fn, ##__VA_ARGS__)
+#define VDDK_CALL_START(fn, fs, ...) \
+ nbdkit_debug ("VDDK call: %s (" fs ")", #fn, ##__VA_ARGS__); \
+ do
+#define VDDK_CALL_START_DATAPATH(fn, fs, ...) \
+ if (vddk_debug_datapath) \
+ nbdkit_debug ("VDDK call: %s (" fs ")", #fn, ##__VA_ARGS__); \
+ do
+#define VDDK_CALL_END(fn) while (0)
/* Unload the plugin. */
static void
vddk_unload (void)
{
if (init_called) {
- DEBUG_CALL ("VixDiskLib_Exit", "");
- VixDiskLib_Exit ();
+ VDDK_CALL_START (VixDiskLib_Exit, "") {
+ VixDiskLib_Exit ();
+ } VDDK_CALL_END (VixDiskLib_Exit);
}
if (dl)
dlclose (dl);
@@ -449,15 +453,16 @@ vddk_after_fork (void)
VixError err;
/* Initialize VDDK library. */
- DEBUG_CALL ("VixDiskLib_InitEx",
- "%d, %d, &debug_fn, &error_fn, &error_fn, %s, %s",
- VDDK_MAJOR, VDDK_MINOR,
- libdir, config ? : "NULL");
- err = VixDiskLib_InitEx (VDDK_MAJOR, VDDK_MINOR,
- &debug_function, /* log function */
- &error_function, /* warn function */
- &error_function, /* panic function */
- libdir, config);
+ VDDK_CALL_START (VixDiskLib_InitEx,
+ "%d, %d, &debug_fn, &error_fn, &error_fn, %s, %s",
+ VDDK_MAJOR, VDDK_MINOR,
+ libdir, config ? : "NULL") {
+ err = VixDiskLib_InitEx (VDDK_MAJOR, VDDK_MINOR,
+ &debug_function, /* log function */
+ &error_function, /* warn function */
+ &error_function, /* panic function */
+ libdir, config);
+ } VDDK_CALL_END (VixDiskLib_InitEx);
if (err != VIX_OK) {
VDDK_ERROR (err, "VixDiskLib_InitEx");
exit (EXIT_FAILURE);
@@ -519,8 +524,9 @@ allocate_connect_params (void)
VixDiskLibConnectParams *ret;
if (VixDiskLib_AllocateConnectParams != NULL) {
- DEBUG_CALL ("VixDiskLib_AllocateConnectParams", "");
- ret = VixDiskLib_AllocateConnectParams ();
+ VDDK_CALL_START (VixDiskLib_AllocateConnectParams, "") {
+ ret = VixDiskLib_AllocateConnectParams ();
+ } VDDK_CALL_END (VixDiskLib_AllocateConnectParams);
}
else
ret = calloc (1, sizeof (VixDiskLibConnectParams));
@@ -535,8 +541,9 @@ free_connect_params (VixDiskLibConnectParams *params)
* originally called. Otherwise use free.
*/
if (VixDiskLib_AllocateConnectParams != NULL) {
- DEBUG_CALL ("VixDiskLib_FreeConnectParams", "params");
- VixDiskLib_FreeConnectParams (params);
+ VDDK_CALL_START (VixDiskLib_FreeConnectParams, "params") {
+ VixDiskLib_FreeConnectParams (params);
+ } VDDK_CALL_END (VixDiskLib_FreeConnectParams);
}
else
free (params);
@@ -589,16 +596,17 @@ vddk_open (int readonly)
* either ESXi or vCenter servers.
*/
- DEBUG_CALL ("VixDiskLib_ConnectEx",
- "h->params, %d, %s, %s, &connection",
- readonly,
- snapshot_moref ? : "NULL",
- transport_modes ? : "NULL");
- err = VixDiskLib_ConnectEx (h->params,
- readonly,
- snapshot_moref,
- transport_modes,
- &h->connection);
+ VDDK_CALL_START (VixDiskLib_ConnectEx,
+ "h->params, %d, %s, %s, &connection",
+ readonly,
+ snapshot_moref ? : "NULL",
+ transport_modes ? : "NULL") {
+ err = VixDiskLib_ConnectEx (h->params,
+ readonly,
+ snapshot_moref,
+ transport_modes,
+ &h->connection);
+ } VDDK_CALL_END (VixDiskLib_ConnectEx);
if (err != VIX_OK) {
VDDK_ERROR (err, "VixDiskLib_ConnectEx");
goto err1;
@@ -618,9 +626,10 @@ vddk_open (int readonly)
case NONE: break;
}
- DEBUG_CALL ("VixDiskLib_Open",
- "connection, %s, %d, &handle", filename, flags);
- err = VixDiskLib_Open (h->connection, filename, flags, &h->handle);
+ VDDK_CALL_START (VixDiskLib_Open,
+ "connection, %s, %d, &handle", filename, flags) {
+ err = VixDiskLib_Open (h->connection, filename, flags, &h->handle);
+ } VDDK_CALL_END (VixDiskLib_Open);
if (err != VIX_OK) {
VDDK_ERROR (err, "VixDiskLib_Open: %s", filename);
goto err2;
@@ -632,8 +641,9 @@ vddk_open (int readonly)
return h;
err2:
- DEBUG_CALL ("VixDiskLib_Disconnect", "connection");
- VixDiskLib_Disconnect (h->connection);
+ VDDK_CALL_START (VixDiskLib_Disconnect, "connection") {
+ VixDiskLib_Disconnect (h->connection);
+ } VDDK_CALL_END (VixDiskLib_Disconnect);
err1:
free_connect_params (h->params);
err0:
@@ -648,10 +658,13 @@ vddk_close (void *handle)
ACQUIRE_LOCK_FOR_CURRENT_SCOPE (&open_close_lock);
struct vddk_handle *h = handle;
- DEBUG_CALL ("VixDiskLib_Close", "handle");
- VixDiskLib_Close (h->handle);
- DEBUG_CALL ("VixDiskLib_Disconnect", "connection");
- VixDiskLib_Disconnect (h->connection);
+ VDDK_CALL_START (VixDiskLib_Close, "handle") {
+ VixDiskLib_Close (h->handle);
+ } VDDK_CALL_END (VixDiskLib_Close);
+ VDDK_CALL_START (VixDiskLib_Disconnect, "connection") {
+ VixDiskLib_Disconnect (h->connection);
+ } VDDK_CALL_END (VixDiskLib_Disconnect);
+
free_connect_params (h->params);
free (h);
}
@@ -665,8 +678,9 @@ vddk_get_size (void *handle)
VixError err;
uint64_t size;
- DEBUG_CALL ("VixDiskLib_GetInfo", "handle, &info");
- err = VixDiskLib_GetInfo (h->handle, &info);
+ VDDK_CALL_START (VixDiskLib_GetInfo, "handle, &info") {
+ err = VixDiskLib_GetInfo (h->handle, &info);
+ } VDDK_CALL_END (VixDiskLib_GetInfo);
if (err != VIX_OK) {
VDDK_ERROR (err, "VixDiskLib_GetInfo");
return -1;
@@ -694,8 +708,9 @@ vddk_get_size (void *handle)
info->uuid ? : "NULL");
}
- DEBUG_CALL ("VixDiskLib_FreeInfo", "info");
- VixDiskLib_FreeInfo (info);
+ VDDK_CALL_START (VixDiskLib_FreeInfo, "info") {
+ VixDiskLib_FreeInfo (info);
+ } VDDK_CALL_END (VixDiskLib_FreeInfo);
return (int64_t) size;
}
@@ -723,11 +738,12 @@ vddk_pread (void *handle, void *buf, uint32_t count, uint64_t offset,
offset /= VIXDISKLIB_SECTOR_SIZE;
count /= VIXDISKLIB_SECTOR_SIZE;
- DEBUG_CALL_DATAPATH ("VixDiskLib_Read",
- "handle, %" PRIu64 " sectors, "
- "%" PRIu32 " sectors, buffer",
- offset, count);
- err = VixDiskLib_Read (h->handle, offset, count, buf);
+ VDDK_CALL_START_DATAPATH (VixDiskLib_Read,
+ "handle, %" PRIu64 " sectors, "
+ "%" PRIu32 " sectors, buffer",
+ offset, count) {
+ err = VixDiskLib_Read (h->handle, offset, count, buf);
+ } VDDK_CALL_END (VixDiskLib_Read);
if (err != VIX_OK) {
VDDK_ERROR (err, "VixDiskLib_Read");
return -1;
@@ -762,11 +778,12 @@ vddk_pwrite (void *handle, const void *buf, uint32_t count, uint64_t offset,
offset /= VIXDISKLIB_SECTOR_SIZE;
count /= VIXDISKLIB_SECTOR_SIZE;
- DEBUG_CALL_DATAPATH ("VixDiskLib_Write",
- "handle, %" PRIu64 " sectors, "
- "%" PRIu32 " sectors, buffer",
- offset, count);
- err = VixDiskLib_Write (h->handle, offset, count, buf);
+ VDDK_CALL_START_DATAPATH (VixDiskLib_Write,
+ "handle, %" PRIu64 " sectors, "
+ "%" PRIu32 " sectors, buffer",
+ offset, count) {
+ err = VixDiskLib_Write (h->handle, offset, count, buf);
+ } VDDK_CALL_END (VixDiskLib_Write);
if (err != VIX_OK) {
VDDK_ERROR (err, "VixDiskLib_Write");
return -1;
@@ -801,8 +818,9 @@ vddk_flush (void *handle, uint32_t flags)
struct vddk_handle *h = handle;
VixError err;
- DEBUG_CALL ("VixDiskLib_Flush", "handle");
- err = VixDiskLib_Flush (h->handle);
+ VDDK_CALL_START (VixDiskLib_Flush, "handle") {
+ err = VixDiskLib_Flush (h->handle);
+ } VDDK_CALL_END (VixDiskLib_Flush);
if (err != VIX_OK) {
VDDK_ERROR (err, "VixDiskLib_Flush");
return -1;
@@ -836,17 +854,19 @@ vddk_can_extents (void *handle)
* the best thing we can do here is to try the call and if it's
* non-functional return false.
*/
- DEBUG_CALL ("VixDiskLib_QueryAllocatedBlocks",
- "handle, 0, %d sectors, %d sectors",
- VIXDISKLIB_MIN_CHUNK_SIZE, VIXDISKLIB_MIN_CHUNK_SIZE);
- err = VixDiskLib_QueryAllocatedBlocks (h->handle,
- 0, VIXDISKLIB_MIN_CHUNK_SIZE,
- VIXDISKLIB_MIN_CHUNK_SIZE,
- &block_list);
+ VDDK_CALL_START (VixDiskLib_QueryAllocatedBlocks,
+ "handle, 0, %d sectors, %d sectors",
+ VIXDISKLIB_MIN_CHUNK_SIZE, VIXDISKLIB_MIN_CHUNK_SIZE) {
+ err = VixDiskLib_QueryAllocatedBlocks (h->handle,
+ 0, VIXDISKLIB_MIN_CHUNK_SIZE,
+ VIXDISKLIB_MIN_CHUNK_SIZE,
+ &block_list);
+ } VDDK_CALL_END (VixDiskLib_QueryAllocatedBlocks);
error_suppression = 0;
if (err == VIX_OK) {
- DEBUG_CALL ("VixDiskLib_FreeBlockList", "block_list");
- VixDiskLib_FreeBlockList (block_list);
+ VDDK_CALL_START (VixDiskLib_FreeBlockList, "block_list") {
+ VixDiskLib_FreeBlockList (block_list);
+ } VDDK_CALL_END (VixDiskLib_FreeBlockList);
}
if (err != VIX_OK) {
char *errmsg = VixDiskLib_GetErrorText (err, NULL);
@@ -923,14 +943,15 @@ vddk_extents (void *handle, uint32_t count, uint64_t offset, uint32_t flags,
nr_chunks = MIN (nr_chunks, VIXDISKLIB_MAX_CHUNK_NUMBER);
nr_sectors = nr_chunks * VIXDISKLIB_MIN_CHUNK_SIZE;
- DEBUG_CALL ("VixDiskLib_QueryAllocatedBlocks",
- "handle, %" PRIu64 " sectors, %" PRIu64 " sectors, "
- "%d sectors",
- start_sector, nr_sectors, VIXDISKLIB_MIN_CHUNK_SIZE);
- err = VixDiskLib_QueryAllocatedBlocks (h->handle,
- start_sector, nr_sectors,
- VIXDISKLIB_MIN_CHUNK_SIZE,
- &block_list);
+ VDDK_CALL_START (VixDiskLib_QueryAllocatedBlocks,
+ "handle, %" PRIu64 " sectors, %" PRIu64 " sectors, "
+ "%d sectors",
+ start_sector, nr_sectors, VIXDISKLIB_MIN_CHUNK_SIZE) {
+ err = VixDiskLib_QueryAllocatedBlocks (h->handle,
+ start_sector, nr_sectors,
+ VIXDISKLIB_MIN_CHUNK_SIZE,
+ &block_list);
+ } VDDK_CALL_END (VixDiskLib_QueryAllocatedBlocks);
if (err != VIX_OK) {
VDDK_ERROR (err, "VixDiskLib_QueryAllocatedBlocks");
return -1;
@@ -949,13 +970,15 @@ vddk_extents (void *handle, uint32_t count, uint64_t offset, uint32_t flags,
add_extent (extents, &position, blk_offset, true) == -1) ||
(add_extent (extents,
&position, blk_offset + blk_length, false) == -1)) {
- DEBUG_CALL ("VixDiskLib_FreeBlockList", "block_list");
- VixDiskLib_FreeBlockList (block_list);
+ VDDK_CALL_START (VixDiskLib_FreeBlockList, "block_list") {
+ VixDiskLib_FreeBlockList (block_list);
+ } VDDK_CALL_END (VixDiskLib_FreeBlockList);
return -1;
}
}
- DEBUG_CALL ("VixDiskLib_FreeBlockList", "block_list");
- VixDiskLib_FreeBlockList (block_list);
+ VDDK_CALL_START (VixDiskLib_FreeBlockList, "block_list") {
+ VixDiskLib_FreeBlockList (block_list);
+ } VDDK_CALL_END (VixDiskLib_FreeBlockList);
/* There's an implicit hole after the returned list of blocks, up
* to the end of the QueryAllocatedBlocks request.
--
2.31.1

103
0021-tests-Add-a-better-test-of-real-VDDK.patch Normal file
View File

@ -0,0 +1,103 @@
From bd181ea739ebfafbf7239b5fa89e98becdb8cb72 Mon Sep 17 00:00:00 2001
From: "Richard W.M. Jones" <rjones@redhat.com>
Date: Mon, 5 Jul 2021 22:03:10 +0100
Subject: [PATCH] tests: Add a better test of real VDDK
The previous test only tested reading and maybe extents, and used an
all-zero disk. I'm fairly convinced the test only worked accidentally
since you must use an absolute path when opening a local file and the
test did not do that.
Add a more comprehensive test that tests writing and flush too.
(cherry picked from commit a6ca4f24593008bb2d8efb177e7f424cff51dfbf)
---
tests/test-vddk-real.sh | 55 ++++++++++++++++++++++++++++-------------
1 file changed, 38 insertions(+), 17 deletions(-)
diff --git a/tests/test-vddk-real.sh b/tests/test-vddk-real.sh
index df486ba1..f848db44 100755
--- a/tests/test-vddk-real.sh
+++ b/tests/test-vddk-real.sh
@@ -37,8 +37,12 @@ set -x
requires test "x$vddkdir" != "x"
requires test -d "$vddkdir"
requires test -f "$vddkdir/lib64/libvixDiskLib.so"
+requires test -r /dev/urandom
+requires cmp --version
+requires dd --version
requires qemu-img --version
requires nbdcopy --version
+requires nbdinfo --version
requires stat --version
# VDDK > 5.1.1 only supports x86_64.
@@ -47,31 +51,48 @@ if [ `uname -m` != "x86_64" ]; then
exit 77
fi
-files="test-vddk-real.vmdk test-vddk-real.out test-vddk-real.log"
-rm -f $files
-cleanup_fn rm -f $files
-
-qemu-img create -f vmdk test-vddk-real.vmdk 100M
-
# Since we are comparing error messages below, let's make sure we're
# not translating errors.
export LANG=C
-fail=0
-nbdkit -f -v -U - \
- --filter=readahead \
- vddk libdir="$vddkdir" test-vddk-real.vmdk \
- --run 'nbdcopy "$uri" test-vddk-real.out' \
- > test-vddk-real.log 2>&1 || fail=1
+pid=test-vddk-real.pid
+sock=$(mktemp -u /tmp/nbdkit-test-sock.XXXXXX)
+vmdk=$PWD/test-vddk-real.vmdk ;# note must be an absolute path
+raw=test-vddk-real.raw
+raw2=test-vddk-real.raw2
+log=test-vddk-real.log
+files="$pid $sock $vmdk $raw $raw2 $log"
+rm -f $files
+cleanup_fn rm -f $files
+
+qemu-img create -f vmdk $vmdk 10M
+
+# Check first that the VDDK library can be fully loaded. We have to
+# check the log file for missing modules since they may not show up as
+# errors.
+nbdkit -fv -U - vddk libdir="$vddkdir" $vmdk --run 'nbdinfo "$uri"' >$log 2>&1
# Check the log for missing modules
-cat test-vddk-real.log
+cat $log
if grep 'cannot open shared object file' test-vddk-real.log; then
exit 1
fi
-# Check the raw output file has exactly the right size.
-size="$(stat -c '%s' test-vddk-real.out)"
-test "$size" -eq $((100 * 1024 * 1024))
+# Now run nbdkit for the test.
+start_nbdkit -P $pid -U $sock vddk libdir="$vddkdir" $vmdk
+uri="nbd+unix:///?socket=$sock"
-exit $fail
+# VDDK < 6.0 did not support flush, so disable flush test there. Also
+# if nbdinfo doesn't support the --can flush syntax (added in libnbd
+# 1.10) then this is disabled.
+if nbdinfo --can flush "$uri"; then flush="--flush"; else flush=""; fi
+
+# Copy in and out some data. This should exercise read, write,
+# extents and flushing.
+dd if=/dev/urandom of=$raw count=5 bs=$((1024*1024))
+truncate -s 10M $raw
+
+nbdcopy $flush $raw "$uri"
+nbdcopy "$uri" $raw2
+
+cmp $raw $raw2
--
2.31.1

245
0022-vddk-Add-stats-about-the-amount-of-time-spent-in-VDD.patch Normal file
View File

@ -0,0 +1,245 @@
From 45db64d72bf03fece8a7fb994887360954905a3b Mon Sep 17 00:00:00 2001
From: "Richard W.M. Jones" <rjones@redhat.com>
Date: Wed, 18 Aug 2021 14:47:58 +0100
Subject: [PATCH] vddk: Add stats about the amount of time spent in VDDK calls
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
If you use -D vddk.stats=1 then when the plugin is unloaded it will
display the amount of time spent in each VDDK call. The output will
look something like this:
nbdkit: debug: VDDK function stats (-D vddk.stats=1):
nbdkit: debug: µs
nbdkit: debug: VixDiskLib_Exit 1001154
nbdkit: debug: VixDiskLib_InitEx 96008
nbdkit: debug: VixDiskLib_Flush 15722
nbdkit: debug: VixDiskLib_Write 12081
nbdkit: debug: VixDiskLib_Open 6029
nbdkit: debug: VixDiskLib_Read 1364
nbdkit: debug: VixDiskLib_Close 605
nbdkit: debug: VixDiskLib_QueryAllocatedBlocks 191
nbdkit: debug: VixDiskLib_ConnectEx 134
nbdkit: debug: VixDiskLib_Disconnect 76
nbdkit: debug: VixDiskLib_FreeConnectParams 57
nbdkit: debug: VixDiskLib_GetInfo 56
nbdkit: debug: VixDiskLib_GetTransportMode 43
nbdkit: debug: VixDiskLib_FreeInfo 42
nbdkit: debug: VixDiskLib_FreeBlockList 32
nbdkit: debug: VixDiskLib_AllocateConnectParams 28
VDDK APIs which are never called are not printed.
(cherry picked from commit f2dfc7d74ee650bdf2cc930a07b1c5bcb509976c)
---
plugins/vddk/nbdkit-vddk-plugin.pod | 5 ++
plugins/vddk/vddk.c | 107 +++++++++++++++++++++++++---
tests/test-vddk-real.sh | 2 +-
3 files changed, 103 insertions(+), 11 deletions(-)
diff --git a/plugins/vddk/nbdkit-vddk-plugin.pod b/plugins/vddk/nbdkit-vddk-plugin.pod
index b783f13a..2a1b17dc 100644
--- a/plugins/vddk/nbdkit-vddk-plugin.pod
+++ b/plugins/vddk/nbdkit-vddk-plugin.pod
@@ -510,6 +510,11 @@ Debug extents returned by C<QueryAllocatedBlocks>.
Suppress debugging of datapath calls (C<Read> and C<Write>).
+=item B<-D vddk.stats=1>
+
+When the plugin exits print some statistics about the amount of time
+spent waiting on each VDDK call.
+
=back
=head1 FILES
diff --git a/plugins/vddk/vddk.c b/plugins/vddk/vddk.c
index 888009ab..fce96d9a 100644
--- a/plugins/vddk/vddk.c
+++ b/plugins/vddk/vddk.c
@@ -42,6 +42,7 @@
#include <assert.h>
#include <dlfcn.h>
#include <libgen.h>
+#include <sys/time.h>
#include <pthread.h>
@@ -52,6 +53,8 @@
#include "isaligned.h"
#include "minmax.h"
#include "rounding.h"
+#include "tvdiff.h"
+#include "vector.h"
#include "vddk.h"
#include "vddk-structs.h"
@@ -60,6 +63,7 @@
NBDKIT_DLL_PUBLIC int vddk_debug_diskinfo;
NBDKIT_DLL_PUBLIC int vddk_debug_extents;
NBDKIT_DLL_PUBLIC int vddk_debug_datapath = 1;
+NBDKIT_DLL_PUBLIC int vddk_debug_stats;
/* For each VDDK API define a static global variable. These globals
* are initialized when the plugin is loaded (by vddk_get_ready).
@@ -96,22 +100,52 @@ static const char *username; /* user */
static const char *vmx_spec; /* vm */
static bool is_remote;
-#define VDDK_ERROR(err, fs, ...) \
- do { \
- char *vddk_err_msg; \
- vddk_err_msg = VixDiskLib_GetErrorText ((err), NULL); \
- nbdkit_error (fs ": %s", ##__VA_ARGS__, vddk_err_msg); \
- VixDiskLib_FreeErrorText (vddk_err_msg); \
- } while (0)
+/* For each VDDK API define a variable to store the time taken (used
+ * to implement -D vddk.stats=1).
+ */
+static pthread_mutex_t stats_lock = PTHREAD_MUTEX_INITIALIZER;
+static void display_stats (void);
+#define STUB(fn,ret,args) static int64_t stats_##fn;
+#define OPTIONAL_STUB(fn,ret,args) static int64_t stats_##fn;
+#include "vddk-stubs.h"
+#undef STUB
+#undef OPTIONAL_STUB
#define VDDK_CALL_START(fn, fs, ...) \
+ do { \
+ struct timeval start_t, end_t; \
+ if (vddk_debug_stats) \
+ gettimeofday (&start_t, NULL); \
nbdkit_debug ("VDDK call: %s (" fs ")", #fn, ##__VA_ARGS__); \
do
#define VDDK_CALL_START_DATAPATH(fn, fs, ...) \
+ do { \
+ struct timeval start_t, end_t; \
+ if (vddk_debug_stats) \
+ gettimeofday (&start_t, NULL); \
if (vddk_debug_datapath) \
nbdkit_debug ("VDDK call: %s (" fs ")", #fn, ##__VA_ARGS__); \
do
-#define VDDK_CALL_END(fn) while (0)
+#define VDDK_CALL_END(fn) \
+ while (0); \
+ if (vddk_debug_stats) { \
+ gettimeofday (&end_t, NULL); \
+ ACQUIRE_LOCK_FOR_CURRENT_SCOPE (&stats_lock); \
+ stats_##fn += tvdiff_usec (&start_t, &end_t); \
+ } \
+ } while (0)
+
+#define VDDK_ERROR(err, fs, ...) \
+ do { \
+ char *vddk_err_msg; \
+ VDDK_CALL_START (VixDiskLib_GetErrorText, "%lu", err) { \
+ vddk_err_msg = VixDiskLib_GetErrorText ((err), NULL); \
+ } VDDK_CALL_END (VixDiskLib_GetErrorText); \
+ nbdkit_error (fs ": %s", ##__VA_ARGS__, vddk_err_msg); \
+ VDDK_CALL_START (VixDiskLib_FreeErrorText, "") { \
+ VixDiskLib_FreeErrorText (vddk_err_msg); \
+ } VDDK_CALL_END (VixDiskLib_FreeErrorText); \
+ } while (0)
/* Unload the plugin. */
static void
@@ -124,11 +158,61 @@ vddk_unload (void)
}
if (dl)
dlclose (dl);
+
+ if (vddk_debug_stats)
+ display_stats ();
+
free (config);
free (libdir);
free (password);
}
+struct vddk_stat {
+ const char *fn;
+ int64_t usecs;
+};
+DEFINE_VECTOR_TYPE(statlist, struct vddk_stat)
+
+static int
+stat_compare (const void *vp1, const void *vp2)
+{
+ const struct vddk_stat *st1 = vp1;
+ const struct vddk_stat *st2 = vp2;
+
+ /* Note: sorts in reverse order. */
+ if (st1->usecs < st2->usecs) return 1;
+ else if (st1->usecs > st2->usecs) return -1;
+ else return 0;
+}
+
+static void
+display_stats (void)
+{
+ statlist stats = empty_vector;
+ struct vddk_stat st;
+ size_t i;
+
+#define ADD_ONE_STAT(fn_, usecs_) \
+ st.fn = fn_; \
+ st.usecs = usecs_; \
+ statlist_append (&stats, st)
+#define STUB(fn,ret,args) ADD_ONE_STAT (#fn, stats_##fn);
+#define OPTIONAL_STUB(fn,ret,args) ADD_ONE_STAT (#fn, stats_##fn);
+#include "vddk-stubs.h"
+#undef STUB
+#undef OPTIONAL_STUB
+#undef ADD_ONE_STAT
+
+ qsort (stats.ptr, stats.size, sizeof stats.ptr[0], stat_compare);
+
+ nbdkit_debug ("VDDK function stats (-D vddk.stats=1):");
+ nbdkit_debug ("%-40s %9s", "", "µs");
+ for (i = 0; i < stats.size; ++i) {
+ if (stats.ptr[i].usecs)
+ nbdkit_debug ("%-40s %9" PRIi64, stats.ptr[i].fn, stats.ptr[i].usecs);
+ }
+}
+
static void
trim (char *str)
{
@@ -557,6 +641,7 @@ vddk_open (int readonly)
struct vddk_handle *h;
VixError err;
uint32_t flags;
+ const char *transport_mode;
h = malloc (sizeof *h);
if (h == NULL) {
@@ -635,8 +720,10 @@ vddk_open (int readonly)
goto err2;
}
- nbdkit_debug ("transport mode: %s",
- VixDiskLib_GetTransportMode (h->handle));
+ VDDK_CALL_START (VixDiskLib_GetTransportMode, "handle") {
+ transport_mode = VixDiskLib_GetTransportMode (h->handle);
+ } VDDK_CALL_END (VixDiskLib_GetTransportMode);
+ nbdkit_debug ("transport mode: %s", transport_mode);
return h;
diff --git a/tests/test-vddk-real.sh b/tests/test-vddk-real.sh
index f848db44..3c8b4262 100755
--- a/tests/test-vddk-real.sh
+++ b/tests/test-vddk-real.sh
@@ -79,7 +79,7 @@ if grep 'cannot open shared object file' test-vddk-real.log; then
fi
# Now run nbdkit for the test.
-start_nbdkit -P $pid -U $sock vddk libdir="$vddkdir" $vmdk
+start_nbdkit -P $pid -U $sock -D vddk.stats=1 vddk libdir="$vddkdir" $vmdk
uri="nbd+unix:///?socket=$sock"
# VDDK < 6.0 did not support flush, so disable flush test there. Also
--
2.31.1

62
0023-common-allocators-malloc.c-Remove-bogus-kernel-hints.patch
View File

@ -1,62 +0,0 @@
From 85aea60685b493eac5e7664581c1887ede987461 Mon Sep 17 00:00:00 2001
From: "Richard W.M. Jones" <rjones@redhat.com>
Date: Wed, 11 Aug 2021 05:54:15 -0400
Subject: [PATCH] common/allocators/malloc.c: Remove bogus kernel hints
These kernel hints are wrong in several ways.
MADV_DONTFORK should not be used because when we use captive nbdkit
(the --run option) we do actually fork and run nbdkit as the child.
However the kernel does not have to provide the mallocd memory to this
child process so it disappears.
Even if the hints were not wrong, setting them using
madvise (ma->ba.ptr ...) would be wrong because the allocator buffer
can be extended at any time using realloc and could move in memory.
The hints would then apply to unrelated glibc allocations. I believe
this is what caused the crash I observed.
For some reason the bug was only seen on s390x where it caused memory
corruption in glibc followed by a crash, but I don't believe this bug
is specific to s390x, it's just something about that architecture that
made it more likely to happen.
Fixes: https://bugzilla.redhat.com/show_bug.cgi?id=1992542
(cherry picked from commit 557a7a85c944dcd247feb0d670b0deca8da46576)
---
common/allocators/malloc.c | 19 -------------------
1 file changed, 19 deletions(-)
diff --git a/common/allocators/malloc.c b/common/allocators/malloc.c
index 9dc25a8e..59409c24 100644
--- a/common/allocators/malloc.c
+++ b/common/allocators/malloc.c
@@ -105,25 +105,6 @@ extend (struct m_alloc *ma, uint64_t new_size)
return -1;
}
- /* Hints to the kernel. Doesn't matter if these fail.
- * XXX Consider in future: MADV_MERGEABLE (tunable)
- */
-#ifdef MADV_RANDOM
- madvise (ma->ba.ptr, ma->ba.alloc, MADV_RANDOM);
-#endif
-#ifdef MADV_WILLNEED
- madvise (ma->ba.ptr, ma->ba.alloc, MADV_WILLNEED);
-#endif
-#ifdef MADV_DONTFORK
- madvise (ma->ba.ptr, ma->ba.alloc, MADV_DONTFORK);
-#endif
-#ifdef MADV_HUGEPAGE
- madvise (ma->ba.ptr, ma->ba.alloc, MADV_HUGEPAGE);
-#endif
-#ifdef MADV_DONTDUMP
- madvise (ma->ba.ptr, ma->ba.alloc, MADV_DONTDUMP);
-#endif
-
/* Initialize the newly allocated memory to 0. */
memset (ma->ba.ptr + old_size, 0, n);
--
2.31.1

772
0023-cow-Make-the-block-size-configurable.patch Normal file
View File

@ -0,0 +1,772 @@
From 0be4847cdec9effd6128da03ea42a4953e5a6343 Mon Sep 17 00:00:00 2001
From: "Richard W.M. Jones" <rjones@redhat.com>
Date: Tue, 17 Aug 2021 22:03:11 +0100
Subject: [PATCH] cow: Make the block size configurable
Commit c1905b0a28 ("cache, cow: Use a 64K block size by default")
changed the nbdkit-cow-filter block size to 64K, but it was still a
fixed size. In contrast the cache filter allows the block size to be
adjusted.
Allow the block size in this filter to be adjusted up or down with a
new cow-block-size=N parameter.
When using the VDDK plugin, adjusting this setting can make a
difference. The following timings come from a modified virt-v2v which
sets cow-block-size and was used to convert from a VMware server to
-o null (this is also using cow-on-read=true):
cow-block-size=64K: 18m18
cow-block-size=256K: 14m13
cow-block-size=1M: 14m19
cow-block-size=4M: 37m33
As you can see it's not obvious how to choose a good block size, but
at least by allowing adjustment we can tune things.
(cherry picked from commit 7182c47d04d2b68005fceadefc0c14bfaa61a533)
---
filters/cow/blk.c | 35 +++----
filters/cow/blk.h | 5 -
filters/cow/cow.c | 150 +++++++++++++++++-------------
filters/cow/cow.h | 39 ++++++++
filters/cow/nbdkit-cow-filter.pod | 5 +
tests/Makefile.am | 2 +
tests/test-cow-block-size.sh | 72 ++++++++++++++
7 files changed, 221 insertions(+), 87 deletions(-)
create mode 100644 filters/cow/cow.h
create mode 100755 tests/test-cow-block-size.sh
diff --git a/filters/cow/blk.c b/filters/cow/blk.c
index c22d5886..f9341dc1 100644
--- a/filters/cow/blk.c
+++ b/filters/cow/blk.c
@@ -99,6 +99,7 @@
#include "pwrite.h"
#include "utils.h"
+#include "cow.h"
#include "blk.h"
/* The temporary overlay. */
@@ -137,7 +138,7 @@ blk_init (void)
size_t len;
char *template;
- bitmap_init (&bm, BLKSIZE, 2 /* bits per block */);
+ bitmap_init (&bm, blksize, 2 /* bits per block */);
tmpdir = getenv ("TMPDIR");
if (!tmpdir)
@@ -199,7 +200,7 @@ blk_set_size (uint64_t new_size)
if (bitmap_resize (&bm, size) == -1)
return -1;
- if (ftruncate (fd, ROUND_UP (size, BLKSIZE)) == -1) {
+ if (ftruncate (fd, ROUND_UP (size, blksize)) == -1) {
nbdkit_error ("ftruncate: %m");
return -1;
}
@@ -228,7 +229,7 @@ blk_read_multiple (nbdkit_next *next,
uint64_t blknum, uint64_t nrblocks,
uint8_t *block, bool cow_on_read, int *err)
{
- off_t offset = blknum * BLKSIZE;
+ off_t offset = blknum * blksize;
enum bm_entry state;
uint64_t b, runblocks;
@@ -262,8 +263,8 @@ blk_read_multiple (nbdkit_next *next,
if (state == BLOCK_NOT_ALLOCATED) { /* Read underlying plugin. */
unsigned n, tail = 0;
- assert (BLKSIZE * runblocks <= UINT_MAX);
- n = BLKSIZE * runblocks;
+ assert (blksize * runblocks <= UINT_MAX);
+ n = blksize * runblocks;
if (offset + n > size) {
tail = offset + n - size;
@@ -288,7 +289,7 @@ blk_read_multiple (nbdkit_next *next,
"at offset %" PRIu64 " into the cache",
runblocks, offset);
- if (full_pwrite (fd, block, BLKSIZE * runblocks, offset) == -1) {
+ if (full_pwrite (fd, block, blksize * runblocks, offset) == -1) {
*err = errno;
nbdkit_error ("pwrite: %m");
return -1;
@@ -298,14 +299,14 @@ blk_read_multiple (nbdkit_next *next,
}
}
else if (state == BLOCK_ALLOCATED) { /* Read overlay. */
- if (full_pread (fd, block, BLKSIZE * runblocks, offset) == -1) {
+ if (full_pread (fd, block, blksize * runblocks, offset) == -1) {
*err = errno;
nbdkit_error ("pread: %m");
return -1;
}
}
else /* state == BLOCK_TRIMMED */ {
- memset (block, 0, BLKSIZE * runblocks);
+ memset (block, 0, blksize * runblocks);
}
/* If all done, return. */
@@ -316,7 +317,7 @@ blk_read_multiple (nbdkit_next *next,
return blk_read_multiple (next,
blknum + runblocks,
nrblocks - runblocks,
- block + BLKSIZE * runblocks,
+ block + blksize * runblocks,
cow_on_read, err);
}
@@ -333,9 +334,9 @@ blk_cache (nbdkit_next *next,
{
/* XXX Could make this lock more fine-grained with some thought. */
ACQUIRE_LOCK_FOR_CURRENT_SCOPE (&lock);
- off_t offset = blknum * BLKSIZE;
+ off_t offset = blknum * blksize;
enum bm_entry state = bitmap_get_blk (&bm, blknum, BLOCK_NOT_ALLOCATED);
- unsigned n = BLKSIZE, tail = 0;
+ unsigned n = blksize, tail = 0;
if (offset + n > size) {
tail = offset + n - size;
@@ -348,7 +349,7 @@ blk_cache (nbdkit_next *next,
if (state == BLOCK_ALLOCATED) {
#if HAVE_POSIX_FADVISE
- int r = posix_fadvise (fd, offset, BLKSIZE, POSIX_FADV_WILLNEED);
+ int r = posix_fadvise (fd, offset, blksize, POSIX_FADV_WILLNEED);
if (r) {
errno = r;
nbdkit_error ("posix_fadvise: %m");
@@ -373,7 +374,7 @@ blk_cache (nbdkit_next *next,
memset (block + n, 0, tail);
if (mode == BLK_CACHE_COW) {
- if (full_pwrite (fd, block, BLKSIZE, offset) == -1) {
+ if (full_pwrite (fd, block, blksize, offset) == -1) {
*err = errno;
nbdkit_error ("pwrite: %m");
return -1;
@@ -386,13 +387,13 @@ blk_cache (nbdkit_next *next,
int
blk_write (uint64_t blknum, const uint8_t *block, int *err)
{
- off_t offset = blknum * BLKSIZE;
+ off_t offset = blknum * blksize;
if (cow_debug_verbose)
nbdkit_debug ("cow: blk_write block %" PRIu64 " (offset %" PRIu64 ")",
blknum, (uint64_t) offset);
- if (full_pwrite (fd, block, BLKSIZE, offset) == -1) {
+ if (full_pwrite (fd, block, blksize, offset) == -1) {
*err = errno;
nbdkit_error ("pwrite: %m");
return -1;
@@ -407,14 +408,14 @@ blk_write (uint64_t blknum, const uint8_t *block, int *err)
int
blk_trim (uint64_t blknum, int *err)
{
- off_t offset = blknum * BLKSIZE;
+ off_t offset = blknum * blksize;
if (cow_debug_verbose)
nbdkit_debug ("cow: blk_trim block %" PRIu64 " (offset %" PRIu64 ")",
blknum, (uint64_t) offset);
/* XXX As an optimization we could punch a whole in the overlay
- * here. However it's not trivial since BLKSIZE is unrelated to the
+ * here. However it's not trivial since blksize is unrelated to the
* overlay filesystem block size.
*/
ACQUIRE_LOCK_FOR_CURRENT_SCOPE (&lock);
diff --git a/filters/cow/blk.h b/filters/cow/blk.h
index b7e6f092..62fb5416 100644
--- a/filters/cow/blk.h
+++ b/filters/cow/blk.h
@@ -33,11 +33,6 @@
#ifndef NBDKIT_BLK_H
#define NBDKIT_BLK_H
-/* Size of a block in the overlay. A 4K block size means that we need
- * 64 MB of memory to store the bitmap for a 1 TB underlying image.
- */
-#define BLKSIZE 65536
-
/* Initialize the overlay and bitmap. */
extern int blk_init (void);
diff --git a/filters/cow/cow.c b/filters/cow/cow.c
index 6efb39f2..1c62c857 100644
--- a/filters/cow/cow.c
+++ b/filters/cow/cow.c
@@ -40,6 +40,7 @@
#include <string.h>
#include <unistd.h>
#include <errno.h>
+#include <limits.h>
#include <pthread.h>
@@ -47,9 +48,11 @@
#include "cleanup.h"
#include "isaligned.h"
+#include "ispowerof2.h"
#include "minmax.h"
#include "rounding.h"
+#include "cow.h"
#include "blk.h"
/* Read-modify-write requests are serialized through this global lock.
@@ -58,6 +61,8 @@
*/
static pthread_mutex_t rmw_lock = PTHREAD_MUTEX_INITIALIZER;
+unsigned blksize = 65536; /* block size */
+
static bool cow_on_cache;
/* Cache on read ("cow-on-read") mode. */
@@ -69,13 +74,6 @@ extern enum cor_mode {
enum cor_mode cor_mode = COR_OFF;
const char *cor_path;
-static void
-cow_load (void)
-{
- if (blk_init () == -1)
- exit (EXIT_FAILURE);
-}
-
static void
cow_unload (void)
{
@@ -86,7 +84,19 @@ static int
cow_config (nbdkit_next_config *next, nbdkit_backend *nxdata,
const char *key, const char *value)
{
- if (strcmp (key, "cow-on-cache") == 0) {
+ if (strcmp (key, "cow-block-size") == 0) {
+ int64_t r = nbdkit_parse_size (value);
+ if (r == -1)
+ return -1;
+ if (r <= 4096 || r > UINT_MAX || !is_power_of_2 (r)) {
+ nbdkit_error ("cow-block-size is out of range (4096..2G) "
+ "or not a power of 2");
+ return -1;
+ }
+ blksize = r;
+ return 0;
+ }
+ else if (strcmp (key, "cow-on-cache") == 0) {
int r;
r = nbdkit_parse_bool (value);
@@ -114,9 +124,19 @@ cow_config (nbdkit_next_config *next, nbdkit_backend *nxdata,
}
#define cow_config_help \
+ "cow-block-size=<N> Set COW block size.\n" \
"cow-on-cache=<BOOL> Copy cache (prefetch) requests to the overlay.\n" \
"cow-on-read=<BOOL>|/PATH Copy read requests to the overlay."
+static int
+cow_get_ready (int thread_model)
+{
+ if (blk_init () == -1)
+ return -1;
+
+ return 0;
+}
+
/* Decide if cow-on-read is currently on or off. */
bool
cow_on_read (void)
@@ -249,8 +269,8 @@ cow_pread (nbdkit_next *next,
uint64_t blknum, blkoffs, nrblocks;
int r;
- if (!IS_ALIGNED (count | offset, BLKSIZE)) {
- block = malloc (BLKSIZE);
+ if (!IS_ALIGNED (count | offset, blksize)) {
+ block = malloc (blksize);
if (block == NULL) {
*err = errno;
nbdkit_error ("malloc: %m");
@@ -258,12 +278,12 @@ cow_pread (nbdkit_next *next,
}
}
- blknum = offset / BLKSIZE; /* block number */
- blkoffs = offset % BLKSIZE; /* offset within the block */
+ blknum = offset / blksize; /* block number */
+ blkoffs = offset % blksize; /* offset within the block */
/* Unaligned head */
if (blkoffs) {
- uint64_t n = MIN (BLKSIZE - blkoffs, count);
+ uint64_t n = MIN (blksize - blkoffs, count);
assert (block);
r = blk_read (next, blknum, block, cow_on_read (), err);
@@ -279,15 +299,15 @@ cow_pread (nbdkit_next *next,
}
/* Aligned body */
- nrblocks = count / BLKSIZE;
+ nrblocks = count / blksize;
if (nrblocks > 0) {
r = blk_read_multiple (next, blknum, nrblocks, buf, cow_on_read (), err);
if (r == -1)
return -1;
- buf += nrblocks * BLKSIZE;
- count -= nrblocks * BLKSIZE;
- offset += nrblocks * BLKSIZE;
+ buf += nrblocks * blksize;
+ count -= nrblocks * blksize;
+ offset += nrblocks * blksize;
blknum += nrblocks;
}
@@ -314,8 +334,8 @@ cow_pwrite (nbdkit_next *next,
uint64_t blknum, blkoffs;
int r;
- if (!IS_ALIGNED (count | offset, BLKSIZE)) {
- block = malloc (BLKSIZE);
+ if (!IS_ALIGNED (count | offset, blksize)) {
+ block = malloc (blksize);
if (block == NULL) {
*err = errno;
nbdkit_error ("malloc: %m");
@@ -323,12 +343,12 @@ cow_pwrite (nbdkit_next *next,
}
}
- blknum = offset / BLKSIZE; /* block number */
- blkoffs = offset % BLKSIZE; /* offset within the block */
+ blknum = offset / blksize; /* block number */
+ blkoffs = offset % blksize; /* offset within the block */
/* Unaligned head */
if (blkoffs) {
- uint64_t n = MIN (BLKSIZE - blkoffs, count);
+ uint64_t n = MIN (blksize - blkoffs, count);
/* Do a read-modify-write operation on the current block.
* Hold the rmw_lock over the whole operation.
@@ -350,14 +370,14 @@ cow_pwrite (nbdkit_next *next,
}
/* Aligned body */
- while (count >= BLKSIZE) {
+ while (count >= blksize) {
r = blk_write (blknum, buf, err);
if (r == -1)
return -1;
- buf += BLKSIZE;
- count -= BLKSIZE;
- offset += BLKSIZE;
+ buf += blksize;
+ count -= blksize;
+ offset += blksize;
blknum++;
}
@@ -397,19 +417,19 @@ cow_zero (nbdkit_next *next,
return -1;
}
- block = malloc (BLKSIZE);
+ block = malloc (blksize);
if (block == NULL) {
*err = errno;
nbdkit_error ("malloc: %m");
return -1;
}
- blknum = offset / BLKSIZE; /* block number */
- blkoffs = offset % BLKSIZE; /* offset within the block */
+ blknum = offset / blksize; /* block number */
+ blkoffs = offset % blksize; /* offset within the block */
/* Unaligned head */
if (blkoffs) {
- uint64_t n = MIN (BLKSIZE - blkoffs, count);
+ uint64_t n = MIN (blksize - blkoffs, count);
/* Do a read-modify-write operation on the current block.
* Hold the rmw_lock over the whole operation.
@@ -429,9 +449,9 @@ cow_zero (nbdkit_next *next,
}
/* Aligned body */
- if (count >= BLKSIZE)
- memset (block, 0, BLKSIZE);
- while (count >= BLKSIZE) {
+ if (count >= blksize)
+ memset (block, 0, blksize);
+ while (count >= blksize) {
/* XXX There is the possibility of optimizing this: since this loop is
* writing a whole, aligned block, we should use FALLOC_FL_ZERO_RANGE.
*/
@@ -439,8 +459,8 @@ cow_zero (nbdkit_next *next,
if (r == -1)
return -1;
- count -= BLKSIZE;
- offset += BLKSIZE;
+ count -= blksize;
+ offset += blksize;
blknum++;
}
@@ -471,8 +491,8 @@ cow_trim (nbdkit_next *next,
uint64_t blknum, blkoffs;
int r;
- if (!IS_ALIGNED (count | offset, BLKSIZE)) {
- block = malloc (BLKSIZE);
+ if (!IS_ALIGNED (count | offset, blksize)) {
+ block = malloc (blksize);
if (block == NULL) {
*err = errno;
nbdkit_error ("malloc: %m");
@@ -480,12 +500,12 @@ cow_trim (nbdkit_next *next,
}
}
- blknum = offset / BLKSIZE; /* block number */
- blkoffs = offset % BLKSIZE; /* offset within the block */
+ blknum = offset / blksize; /* block number */
+ blkoffs = offset % blksize; /* offset within the block */
/* Unaligned head */
if (blkoffs) {
- uint64_t n = MIN (BLKSIZE - blkoffs, count);
+ uint64_t n = MIN (blksize - blkoffs, count);
/* Do a read-modify-write operation on the current block.
* Hold the lock over the whole operation.
@@ -505,13 +525,13 @@ cow_trim (nbdkit_next *next,
}
/* Aligned body */
- while (count >= BLKSIZE) {
+ while (count >= blksize) {
r = blk_trim (blknum, err);
if (r == -1)
return -1;
- count -= BLKSIZE;
- offset += BLKSIZE;
+ count -= blksize;
+ offset += blksize;
blknum++;
}
@@ -568,22 +588,22 @@ cow_cache (nbdkit_next *next,
mode = BLK_CACHE_COW;
assert (!flags);
- block = malloc (BLKSIZE);
+ block = malloc (blksize);
if (block == NULL) {
*err = errno;
nbdkit_error ("malloc: %m");
return -1;
}
- blknum = offset / BLKSIZE; /* block number */
- blkoffs = offset % BLKSIZE; /* offset within the block */
+ blknum = offset / blksize; /* block number */
+ blkoffs = offset % blksize; /* offset within the block */
/* Unaligned head */
remaining += blkoffs;
offset -= blkoffs;
/* Unaligned tail */
- remaining = ROUND_UP (remaining, BLKSIZE);
+ remaining = ROUND_UP (remaining, blksize);
/* Aligned body */
while (remaining) {
@@ -591,8 +611,8 @@ cow_cache (nbdkit_next *next,
if (r == -1)
return -1;
- remaining -= BLKSIZE;
- offset += BLKSIZE;
+ remaining -= blksize;
+ offset += blksize;
blknum++;
}
@@ -616,13 +636,13 @@ cow_extents (nbdkit_next *next,
* value so rounding up is safe here.
*/
end = offset + count;
- offset = ROUND_DOWN (offset, BLKSIZE);
- end = ROUND_UP (end, BLKSIZE);
+ offset = ROUND_DOWN (offset, blksize);
+ end = ROUND_UP (end, blksize);
count = end - offset;
- blknum = offset / BLKSIZE;
+ blknum = offset / blksize;
- assert (IS_ALIGNED (offset, BLKSIZE));
- assert (IS_ALIGNED (count, BLKSIZE));
+ assert (IS_ALIGNED (offset, blksize));
+ assert (IS_ALIGNED (count, blksize));
assert (count > 0); /* We must make forward progress. */
while (count > 0) {
@@ -634,7 +654,7 @@ cow_extents (nbdkit_next *next,
/* Present in the overlay. */
if (present) {
e.offset = offset;
- e.length = BLKSIZE;
+ e.length = blksize;
if (trimmed)
e.type = NBDKIT_EXTENT_HOLE|NBDKIT_EXTENT_ZERO;
@@ -647,8 +667,8 @@ cow_extents (nbdkit_next *next,
}
blknum++;
- offset += BLKSIZE;
- count -= BLKSIZE;
+ offset += blksize;
+ count -= blksize;
}
/* Not present in the overlay, but we can ask the plugin. */
@@ -667,12 +687,12 @@ cow_extents (nbdkit_next *next,
* (range_count), but count is a 64 bit quantity, so don't
* overflow range_count here.
*/
- if (range_count >= UINT32_MAX - BLKSIZE + 1) break;
+ if (range_count >= UINT32_MAX - blksize + 1) break;
blknum++;
- offset += BLKSIZE;
- count -= BLKSIZE;
- range_count += BLKSIZE;
+ offset += blksize;
+ count -= blksize;
+ range_count += blksize;
if (count == 0) break;
blk_status (blknum, &present, &trimmed);
@@ -706,7 +726,7 @@ cow_extents (nbdkit_next *next,
/* Otherwise assume the block is non-sparse. */
else {
e.offset = offset;
- e.length = BLKSIZE;
+ e.length = blksize;
e.type = 0;
if (nbdkit_add_extent (extents, e.offset, e.length, e.type) == -1) {
@@ -715,8 +735,8 @@ cow_extents (nbdkit_next *next,
}
blknum++;
- offset += BLKSIZE;
- count -= BLKSIZE;
+ offset += blksize;
+ count -= blksize;
}
/* If the caller only wanted the first extent, and we've managed
@@ -734,11 +754,11 @@ cow_extents (nbdkit_next *next,
static struct nbdkit_filter filter = {
.name = "cow",
.longname = "nbdkit copy-on-write (COW) filter",
- .load = cow_load,
.unload = cow_unload,
.open = cow_open,
.config = cow_config,
.config_help = cow_config_help,
+ .get_ready = cow_get_ready,
.prepare = cow_prepare,
.get_size = cow_get_size,
.can_write = cow_can_write,
diff --git a/filters/cow/cow.h b/filters/cow/cow.h
new file mode 100644
index 00000000..d46dbe91
--- /dev/null
+++ b/filters/cow/cow.h
@@ -0,0 +1,39 @@
+/* nbdkit
+ * Copyright (C) 2018-2021 Red Hat Inc.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * * Neither the name of Red Hat nor the names of its contributors may be
+ * used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY RED HAT AND CONTRIBUTORS ''AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+ * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL RED HAT OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
+ * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+ * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#ifndef NBDKIT_COW_H
+#define NBDKIT_COW_H
+
+/* Size of a block in the cache. */
+extern unsigned blksize;
+
+#endif /* NBDKIT_COW_H */
diff --git a/filters/cow/nbdkit-cow-filter.pod b/filters/cow/nbdkit-cow-filter.pod
index 7f861140..997c9097 100644
--- a/filters/cow/nbdkit-cow-filter.pod
+++ b/filters/cow/nbdkit-cow-filter.pod
@@ -5,6 +5,7 @@ nbdkit-cow-filter - nbdkit copy-on-write (COW) filter
=head1 SYNOPSIS
nbdkit --filter=cow plugin [plugin-args...]
+ [cow-block-size=N]
[cow-on-cache=false|true]
[cow-on-read=false|true|/PATH]
@@ -42,6 +43,10 @@ serve the same data to each client.
=over 4
+=item B<cow-block-size=>N
+
+Set the block size used by the filter. The default is 64K.
+
=item B<cow-on-cache=false>
Do not save data from cache (prefetch) requests in the overlay. This
diff --git a/tests/Makefile.am b/tests/Makefile.am
index e61c5829..d93f848f 100644
--- a/tests/Makefile.am
+++ b/tests/Makefile.am
@@ -1404,6 +1404,7 @@ EXTRA_DIST += \
if HAVE_MKE2FS_WITH_D
TESTS += \
test-cow.sh \
+ test-cow-block-size.sh \
test-cow-extents1.sh \
test-cow-extents2.sh \
test-cow-extents-large.sh \
@@ -1415,6 +1416,7 @@ endif
TESTS += test-cow-null.sh
EXTRA_DIST += \
test-cow.sh \
+ test-cow-block-size.sh \
test-cow-extents1.sh \
test-cow-extents2.sh \
test-cow-extents-large.sh \
diff --git a/tests/test-cow-block-size.sh b/tests/test-cow-block-size.sh
new file mode 100755
index 00000000..6de1c068
--- /dev/null
+++ b/tests/test-cow-block-size.sh
@@ -0,0 +1,72 @@
+#!/usr/bin/env bash
+# nbdkit
+# Copyright (C) 2018-2021 Red Hat Inc.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#
+# * Redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in the
+# documentation and/or other materials provided with the distribution.
+#
+# * Neither the name of Red Hat nor the names of its contributors may be
+# used to endorse or promote products derived from this software without
+# specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY RED HAT AND CONTRIBUTORS ''AS IS'' AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+# PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL RED HAT OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
+# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+# SUCH DAMAGE.
+
+source ./functions.sh
+set -e
+set -x
+
+requires_plugin linuxdisk
+requires guestfish --version
+requires nbdcopy --version
+requires qemu-img --version
+
+sock=$(mktemp -u /tmp/nbdkit-test-sock.XXXXXX)
+files="cow-block-size-base.img $sock cow-block-size.pid"
+rm -f $files
+cleanup_fn rm -f $files
+
+# Create a base image which is partitioned with an empty filesystem.
+rm -rf cow-block-size.d
+mkdir cow-block-size.d
+cleanup_fn rm -rf cow-block-size.d
+nbdkit -fv -U - linuxdisk cow-block-size.d size=100M \
+ --run 'nbdcopy "$uri" cow-block-size-base.img'
+lastmod="$(stat -c "%y" cow-block-size-base.img)"
+
+# Run nbdkit with a COW overlay, 4M block size and copy on read.
+start_nbdkit -P cow-block-size.pid -U $sock \
+ --filter=cow file cow-block-size-base.img \
+ cow-block-size=4M cow-on-read=true
+
+# Write some data into the overlay.
+guestfish --format=raw -a "nbd://?socket=$sock" -m /dev/sda1 <<EOF
+ fill-pattern "abcde" 128K /large
+ write /hello "hello, world"
+EOF
+
+# The original file must not be modified.
+currmod="$(stat -c "%y" cow-block-size-base.img)"
+
+if [ "$lastmod" != "$currmod" ]; then
+ echo "$0: FAILED last modified time of base file changed"
+ exit 1
+fi
--
2.31.1

26
0024-cow-Ship-cow.h-header.patch Normal file
View File

@ -0,0 +1,26 @@
From 8d2ef02bd4de988e20ad1efba8038d311cd59665 Mon Sep 17 00:00:00 2001
From: Eric Blake <eblake@redhat.com>
Date: Wed, 18 Aug 2021 19:16:43 -0500
Subject: [PATCH] cow: Ship cow.h header
Fixes: 7182c47d0 (cow: Make the block size configurable)
(cherry picked from commit 75ff1b8b1afb3744b21a306c62e4973c90d386be)
---
filters/cow/Makefile.am | 1 +
1 file changed, 1 insertion(+)
diff --git a/filters/cow/Makefile.am b/filters/cow/Makefile.am
index a80ccd8f..88cda497 100644
--- a/filters/cow/Makefile.am
+++ b/filters/cow/Makefile.am
@@ -39,6 +39,7 @@ nbdkit_cow_filter_la_SOURCES = \
blk.c \
blk.h \
cow.c \
+ cow.h \
$(top_srcdir)/include/nbdkit-filter.h \
$(NULL)
--
2.31.1

BIN
libguestfs.keyring
View File

Binary file not shown.

53
nbdkit.spec
View File

@ -50,8 +50,8 @@ ExclusiveArch: x86_64
%global source_directory 1.26-stable %global source_directory 1.26-stable
Name: nbdkit Name: nbdkit
Version: 1.26.3 Version: 1.26.5
Release: 4%{?dist} Release: 1%{?dist}
Summary: NBD server Summary: NBD server
License: BSD License: BSD
@ -76,29 +76,30 @@ Source3: copy-patches.sh
# https://gitlab.com/nbdkit/nbdkit/-/commits/rhel-9.0/ # https://gitlab.com/nbdkit/nbdkit/-/commits/rhel-9.0/
# Patches. # Patches.
Patch0001: 0001-cache-Reduce-verbosity-of-debugging.patch Patch0001: 0001-server-reset-meta-context-replies-on-starttls.patch
Patch0002: 0002-cache-cow-Add-blk_read_multiple-function.patch Patch0002: 0002-cache-Reduce-verbosity-of-debugging.patch
Patch0003: 0003-cache-cow-Use-full-pread-pwrite-operations.patch Patch0003: 0003-cache-cow-Add-blk_read_multiple-function.patch
Patch0004: 0004-cache-Implement-cache-on-read-PATH.patch Patch0004: 0004-cache-cow-Use-full-pread-pwrite-operations.patch
Patch0005: 0005-cache-Add-cache-min-block-size-parameter.patch Patch0005: 0005-cache-Implement-cache-on-read-PATH.patch
Patch0006: 0006-cache-cow-Use-a-64K-block-size-by-default.patch Patch0006: 0006-cache-Add-cache-min-block-size-parameter.patch
Patch0007: 0007-cache-Refactor-printing-state-into-new-function.patch Patch0007: 0007-cache-cow-Use-a-64K-block-size-by-default.patch
Patch0008: 0008-tests-cache-Test-cache-on-read-option-really-caches.patch Patch0008: 0008-cache-Refactor-printing-state-into-new-function.patch
Patch0009: 0009-cow-Implement-cow-on-read.patch Patch0009: 0009-tests-cache-Test-cache-on-read-option-really-caches.patch
Patch0010: 0010-delay-Add-delay-open-and-delay-close.patch Patch0010: 0010-cow-Implement-cow-on-read.patch
Patch0011: 0011-python-Implement-.cleanup-method.patch Patch0011: 0011-delay-Add-delay-open-and-delay-close.patch
Patch0012: 0012-server-Make-debug-messages-atomic.patch Patch0012: 0012-python-Implement-.cleanup-method.patch
Patch0013: 0013-cow-General-revision-and-updates-to-the-manual.patch Patch0013: 0013-cow-General-revision-and-updates-to-the-manual.patch
Patch0014: 0014-cache-Move-plugin-args-in-synopsis-earlier.patch Patch0014: 0014-cache-Move-plugin-args-in-synopsis-earlier.patch
Patch0015: 0015-data-Improve-the-example-with-a-diagram.patch Patch0015: 0015-data-Improve-the-example-with-a-diagram.patch
Patch0016: 0016-cow-Add-some-more-debugging-especially-for-blk_read_.patch Patch0016: 0016-cow-Add-some-more-debugging-especially-for-blk_read_.patch
Patch0017: 0017-tests-test-debug-flags.sh-Don-t-use-port-10809-durin.patch Patch0017: 0017-delay-Fix-delay-close.patch
Patch0018: 0018-delay-Improve-parsing-and-representation-of-delay-ti.patch Patch0018: 0018-delay-Test-delay-open-and-delay-close.patch
Patch0019: 0019-server-Return-from-nbdkit_nanosleep-early-if-the-soc.patch Patch0019: 0019-vddk-Implement-can_flush-and-can_fua.patch
Patch0020: 0020-server-nanosleep-Change-error-for-early-end-of-sleep.patch Patch0020: 0020-vddk-Replace-DEBUG_CALL-with-bracketed-VDDK_CALL_STA.patch
Patch0021: 0021-delay-Fix-delay-close.patch Patch0021: 0021-tests-Add-a-better-test-of-real-VDDK.patch
Patch0022: 0022-delay-Test-delay-open-and-delay-close.patch Patch0022: 0022-vddk-Add-stats-about-the-amount-of-time-spent-in-VDD.patch
Patch0023: 0023-common-allocators-malloc.c-Remove-bogus-kernel-hints.patch Patch0023: 0023-cow-Make-the-block-size-configurable.patch
Patch0024: 0024-cow-Ship-cow.h-header.patch
BuildRequires: make BuildRequires: make
%if 0%{patches_touch_autotools} %if 0%{patches_touch_autotools}
@ -1269,6 +1270,16 @@ export LIBGUESTFS_TRACE=1
%changelog %changelog
* Thu Aug 19 2021 Richard W.M. Jones <rjones@redhat.com> - 1.26.5-1
- Rebase along stable branch to 1.26.5
resolves: rhbz#1995327
- Add nbdkit-vddk-plugin -D vddk.stats=1 flag
resolves: rhbz#1995329
- Add nbdkit-cow-filter cow-block-size parameter
resolves: rhbz#1995332
- Fix CVE-2021-3716 nbdkit: NBD_OPT_STRUCTURED_REPLY injection on STARTTLS
- Update keyring
* Wed Aug 11 2021 Richard W.M. Jones <rjones@redhat.com> - 1.26.3-4 * Wed Aug 11 2021 Richard W.M. Jones <rjones@redhat.com> - 1.26.3-4
- Remove bogus kernel hints in allocator=malloc - Remove bogus kernel hints in allocator=malloc
resolves: rhbz#1992542 resolves: rhbz#1992542

4
sources
View File

@ -1,2 +1,2 @@
SHA512 (nbdkit-1.26.3.tar.gz) = 7a85b1f35aabbe9cf9f5b45f2cd6458df1e7acf25b4753cfeae13196cd16d83c8d91b491d3f4b4ee24f567f8432a82e1ddda0c3c4a145429eefe43e68aa9f0c7 SHA512 (nbdkit-1.26.5.tar.gz) = 966bee315e7e9535eadf5ee392fd3c7b8ee33e0bc910ac903cb4f1e2748e6bcff3c459594665534693a24f64a2ff440ac30c2c673a2ce39ccddcb557b7afc4ce
SHA512 (nbdkit-1.26.3.tar.gz.sig) = 34c61897a8f36a53f88c062de5da42ef12c21f43e6fe39b874da3c862be2e86ef075b02bd079195d91da43bd7aced126ae4e4d7763152329ff38ceee3906423c SHA512 (nbdkit-1.26.5.tar.gz.sig) = 8bd9279896e747c417a2417bae89cfae8eccf9a0bf445a510b69c682fa9342329996cff5e0ed6394741edd2a842deef05456c33642a4b48c44a0848b08e1f97f