rpms/nbdkit
7
0
Fork 0
Code Issues Pull Requests Releases Activity

import nbdkit-1.28.2-1.el9

Browse Source
This commit is contained in:
CentOS Sources 2021-12-07 12:38:55 -05:00 committed by Stepan Oksanichenko
parent ed82013cf7
commit 2193c3b856
53 changed files with 6104 additions and 4484 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.gitignore vendored
View File

@ -1,2 +1,2 @@
SOURCES/libguestfs.keyring SOURCES/libguestfs.keyring
SOURCES/nbdkit-1.26.5.tar.gz SOURCES/nbdkit-1.28.2.tar.gz

2
.nbdkit.metadata
View File

@ -1,2 +1,2 @@
cc1b37b9cfafa515aab3eefd345ecc59aac2ce7b SOURCES/libguestfs.keyring cc1b37b9cfafa515aab3eefd345ecc59aac2ce7b SOURCES/libguestfs.keyring
52a221954f374f2a3f1adcc5c5e3e6f7332d906d SOURCES/nbdkit-1.26.5.tar.gz ee726a0b63e9a52b787de1462b11e1af34ca51ab SOURCES/nbdkit-1.28.2.tar.gz

39
SOURCES/0001-server-reset-meta-context-replies-on-starttls.patch
View File

@ -1,39 +0,0 @@
From 89ef17c90996c0e212e3a17c8d26ff930ab464ea Mon Sep 17 00:00:00 2001
From: Eric Blake <eblake@redhat.com>
Date: Mon, 16 Aug 2021 13:43:29 -0500
Subject: [PATCH] server: reset meta context replies on starttls
Related to CVE-2021-3716, but not as severe. No compliant client will
send NBD_CMD_BLOCK_STATUS unless it first negotiates
NBD_OPT_SET_META_CONTEXT. If an attacker injects a premature
SET_META_CONTEXT, either the client will never notice (because it
never uses BLOCK_STATUS), or the client will overwrite the attacker's
attempt with the client's own SET_META_CONTEXT request after
encryption is enabled. So I don't class this as having the potential
to trigger denial-of-service due to any protocol mismatch between
compliant client and server (I don't care what happens with
non-compliant clients).
Fixes: 26455d45 (server: protocol: Implement Block Status "base:allocation".)
(cherry picked from commit 6c5faac6a37077cf2366388a80862bb00616d0d8)
---
server/protocol-handshake-newstyle.c | 3 +++
1 file changed, 3 insertions(+)
diff --git a/server/protocol-handshake-newstyle.c b/server/protocol-handshake-newstyle.c
index 7e6b7b1b..79b2c8ce 100644
--- a/server/protocol-handshake-newstyle.c
+++ b/server/protocol-handshake-newstyle.c
@@ -497,6 +497,9 @@ negotiate_handshake_newstyle_options (void)
debug ("using TLS on this connection");
/* Wipe out any cached state. */
conn->structured_replies = false;
+ free (conn->exportname_from_set_meta_context);
+ conn->exportname_from_set_meta_context = NULL;
+ conn->meta_context_base_allocation = false;
for_each_backend (b) {
free (conn->default_exportname[b->i]);
conn->default_exportname[b->i] = NULL;
--
2.31.1

117
SOURCES/0001-vddk-Refactor-how-D-vddk.stats-1-is-collected.patch Normal file
View File

@ -0,0 +1,117 @@
From 96ee8f6f2844bceb8e27ffb442359a2b7521c950 Mon Sep 17 00:00:00 2001
From: "Richard W.M. Jones" <rjones@redhat.com>
Date: Thu, 21 Oct 2021 14:49:52 +0100
Subject: [PATCH] vddk: Refactor how -D vddk.stats=1 is collected
In order to allow us to collect more per-API stats, introduce a global
struct per API for storing these stats.
(cherry picked from commit 3d8657f3d9a2c1b59284333566428b4c7ce32a74)
---
plugins/vddk/vddk.c | 36 ++++++++++++++++++------------------
1 file changed, 18 insertions(+), 18 deletions(-)
diff --git a/plugins/vddk/vddk.c b/plugins/vddk/vddk.c
index 80f5870e..3d751544 100644
--- a/plugins/vddk/vddk.c
+++ b/plugins/vddk/vddk.c
@@ -1,5 +1,5 @@
/* nbdkit
- * Copyright (C) 2013-2020 Red Hat Inc.
+ * Copyright (C) 2013-2021 Red Hat Inc.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are
@@ -103,14 +103,23 @@ static bool is_remote;
/* For each VDDK API define a variable to store the time taken (used
* to implement -D vddk.stats=1).
*/
+struct vddk_stat {
+ const char *name; /* function name */
+ int64_t usecs; /* total number of usecs consumed */
+};
static pthread_mutex_t stats_lock = PTHREAD_MUTEX_INITIALIZER;
static void display_stats (void);
-#define STUB(fn,ret,args) static int64_t stats_##fn;
-#define OPTIONAL_STUB(fn,ret,args) static int64_t stats_##fn;
+#define STUB(fn,ret,args) \
+ static struct vddk_stat stats_##fn = { .name = #fn }
+#define OPTIONAL_STUB(fn,ret,args) \
+ static struct vddk_stat stats_##fn = { .name = #fn }
#include "vddk-stubs.h"
#undef STUB
#undef OPTIONAL_STUB
+/* Macros to bracket each VDDK API call, for printing debugging
+ * information and collecting statistics.
+ */
#define VDDK_CALL_START(fn, fs, ...) \
do { \
struct timeval start_t, end_t; \
@@ -131,10 +140,11 @@ static void display_stats (void);
if (vddk_debug_stats) { \
gettimeofday (&end_t, NULL); \
ACQUIRE_LOCK_FOR_CURRENT_SCOPE (&stats_lock); \
- stats_##fn += tvdiff_usec (&start_t, &end_t); \
+ stats_##fn.usecs += tvdiff_usec (&start_t, &end_t); \
} \
} while (0)
+/* Print VDDK errors. */
#define VDDK_ERROR(err, fs, ...) \
do { \
char *vddk_err_msg; \
@@ -167,10 +177,6 @@ vddk_unload (void)
free (password);
}
-struct vddk_stat {
- const char *fn;
- int64_t usecs;
-};
DEFINE_VECTOR_TYPE(statlist, struct vddk_stat)
static int
@@ -179,7 +185,7 @@ stat_compare (const void *vp1, const void *vp2)
const struct vddk_stat *st1 = vp1;
const struct vddk_stat *st2 = vp2;
- /* Note: sorts in reverse order. */
+ /* Note: sorts in reverse order of time spent in each API call. */
if (st1->usecs < st2->usecs) return 1;
else if (st1->usecs > st2->usecs) return -1;
else return 0;
@@ -189,19 +195,13 @@ static void
display_stats (void)
{
statlist stats = empty_vector;
- struct vddk_stat st;
size_t i;
-#define ADD_ONE_STAT(fn_, usecs_) \
- st.fn = fn_; \
- st.usecs = usecs_; \
- statlist_append (&stats, st)
-#define STUB(fn,ret,args) ADD_ONE_STAT (#fn, stats_##fn);
-#define OPTIONAL_STUB(fn,ret,args) ADD_ONE_STAT (#fn, stats_##fn);
+#define STUB(fn,ret,args) statlist_append (&stats, stats_##fn)
+#define OPTIONAL_STUB(fn,ret,args) statlist_append (&stats, stats_##fn)
#include "vddk-stubs.h"
#undef STUB
#undef OPTIONAL_STUB
-#undef ADD_ONE_STAT
qsort (stats.ptr, stats.size, sizeof stats.ptr[0], stat_compare);
@@ -209,7 +209,7 @@ display_stats (void)
nbdkit_debug ("%-40s %9s", "", "µs");
for (i = 0; i < stats.size; ++i) {
if (stats.ptr[i].usecs)
- nbdkit_debug ("%-40s %9" PRIi64, stats.ptr[i].fn, stats.ptr[i].usecs);
+ nbdkit_debug ("%-40s %9" PRIi64, stats.ptr[i].name, stats.ptr[i].usecs);
}
statlist_reset (&stats);
}
--
2.31.1

124
SOURCES/0002-cache-Reduce-verbosity-of-debugging.patch
View File

@ -1,124 +0,0 @@
From 4b576a8e0eb99ec1a79ca432350fb7ac27a5c089 Mon Sep 17 00:00:00 2001
From: "Richard W.M. Jones" <rjones@redhat.com>
Date: Mon, 26 Jul 2021 11:59:43 +0100
Subject: [PATCH] cache: Reduce verbosity of debugging
The cache filter is very verbose in its debugging. Reduce the default
level. Use -D cache.verbose=1 to restore original debugging.
Compare commit 745a0f13662031c2b9c9b69f62b4ae3a6b2f38f0.
(cherry picked from commit 6be735edf7d5fb3fb8350c72e6d9525badbab14d)
---
filters/cache/blk.c | 53 +++++++++++++++++++++++++++------------------
1 file changed, 32 insertions(+), 21 deletions(-)
diff --git a/filters/cache/blk.c b/filters/cache/blk.c
index 12e8407e..f52f30e3 100644
--- a/filters/cache/blk.c
+++ b/filters/cache/blk.c
@@ -93,6 +93,9 @@ enum bm_entry {
BLOCK_DIRTY = 3,
};
+/* Extra debugging (-D cache.verbose=1). */
+NBDKIT_DLL_PUBLIC int cache_debug_verbose = 0;
+
int
blk_init (void)
{
@@ -199,12 +202,14 @@ blk_read (nbdkit_next *next,
reclaim (fd, &bm);
- nbdkit_debug ("cache: blk_read block %" PRIu64 " (offset %" PRIu64 ") is %s",
- blknum, (uint64_t) offset,
- state == BLOCK_NOT_CACHED ? "not cached" :
- state == BLOCK_CLEAN ? "clean" :
- state == BLOCK_DIRTY ? "dirty" :
- "unknown");
+ if (cache_debug_verbose)
+ nbdkit_debug ("cache: blk_read block %" PRIu64
+ " (offset %" PRIu64 ") is %s",
+ blknum, (uint64_t) offset,
+ state == BLOCK_NOT_CACHED ? "not cached" :
+ state == BLOCK_CLEAN ? "clean" :
+ state == BLOCK_DIRTY ? "dirty" :
+ "unknown");
if (state == BLOCK_NOT_CACHED) { /* Read underlying plugin. */
unsigned n = blksize, tail = 0;
@@ -225,9 +230,10 @@ blk_read (nbdkit_next *next,
/* If cache-on-read, copy the block to the cache. */
if (cache_on_read) {
- nbdkit_debug ("cache: cache-on-read block %" PRIu64
- " (offset %" PRIu64 ")",
- blknum, (uint64_t) offset);
+ if (cache_debug_verbose)
+ nbdkit_debug ("cache: cache-on-read block %" PRIu64
+ " (offset %" PRIu64 ")",
+ blknum, (uint64_t) offset);
if (pwrite (fd, block, blksize, offset) == -1) {
*err = errno;
@@ -259,12 +265,14 @@ blk_cache (nbdkit_next *next,
reclaim (fd, &bm);
- nbdkit_debug ("cache: blk_cache block %" PRIu64 " (offset %" PRIu64 ") is %s",
- blknum, (uint64_t) offset,
- state == BLOCK_NOT_CACHED ? "not cached" :
- state == BLOCK_CLEAN ? "clean" :
- state == BLOCK_DIRTY ? "dirty" :
- "unknown");
+ if (cache_debug_verbose)
+ nbdkit_debug ("cache: blk_cache block %" PRIu64
+ " (offset %" PRIu64 ") is %s",
+ blknum, (uint64_t) offset,
+ state == BLOCK_NOT_CACHED ? "not cached" :
+ state == BLOCK_CLEAN ? "clean" :
+ state == BLOCK_DIRTY ? "dirty" :
+ "unknown");
if (state == BLOCK_NOT_CACHED) {
/* Read underlying plugin, copy to cache regardless of cache-on-read. */
@@ -284,8 +292,9 @@ blk_cache (nbdkit_next *next,
*/
memset (block + n, 0, tail);
- nbdkit_debug ("cache: cache block %" PRIu64 " (offset %" PRIu64 ")",
- blknum, (uint64_t) offset);
+ if (cache_debug_verbose)
+ nbdkit_debug ("cache: cache block %" PRIu64 " (offset %" PRIu64 ")",
+ blknum, (uint64_t) offset);
if (pwrite (fd, block, blksize, offset) == -1) {
*err = errno;
@@ -324,8 +333,9 @@ blk_writethrough (nbdkit_next *next,
reclaim (fd, &bm);
- nbdkit_debug ("cache: writethrough block %" PRIu64 " (offset %" PRIu64 ")",
- blknum, (uint64_t) offset);
+ if (cache_debug_verbose)
+ nbdkit_debug ("cache: writethrough block %" PRIu64 " (offset %" PRIu64 ")",
+ blknum, (uint64_t) offset);
if (pwrite (fd, block, blksize, offset) == -1) {
*err = errno;
@@ -357,8 +367,9 @@ blk_write (nbdkit_next *next,
reclaim (fd, &bm);
- nbdkit_debug ("cache: writeback block %" PRIu64 " (offset %" PRIu64 ")",
- blknum, (uint64_t) offset);
+ if (cache_debug_verbose)
+ nbdkit_debug ("cache: writeback block %" PRIu64 " (offset %" PRIu64 ")",
+ blknum, (uint64_t) offset);
if (pwrite (fd, block, blksize, offset) == -1) {
*err = errno;
--
2.31.1

140
SOURCES/0002-vddk-Extend-D-vddk.stats-1-to-show-number-of-calls-a.patch Normal file
View File

@ -0,0 +1,140 @@
From f388c9b6c983d395ced0d4f467980b182d0a1b84 Mon Sep 17 00:00:00 2001
From: "Richard W.M. Jones" <rjones@redhat.com>
Date: Thu, 21 Oct 2021 15:10:00 +0100
Subject: [PATCH] vddk: Extend -D vddk.stats=1 to show number of calls and
bytes transferred
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
The new output looks like this:
nbdkit: debug: VDDK function stats (-D vddk.stats=1):
nbdkit: debug: VixDiskLib_... µs calls bytes
nbdkit: debug: Exit 1000854 1
nbdkit: debug: InitEx 79304 1
nbdkit: debug: Flush 13577 1
nbdkit: debug: Write 12534 21 10485760
nbdkit: debug: Open 4753 3
nbdkit: debug: Read 966 20 5242880
nbdkit: debug: Close 574 3
nbdkit: debug: QueryAllocatedBlocks 116 4
nbdkit: debug: ConnectEx 103 3
nbdkit: debug: Disconnect 88 3
nbdkit: debug: GetTransportMode 68 3
nbdkit: debug: GetInfo 46 3
nbdkit: debug: FreeConnectParams 36 3
nbdkit: debug: FreeInfo 36 3
nbdkit: debug: FreeBlockList 22 4
nbdkit: debug: AllocateConnectParams 22 3
(cherry picked from commit 5c80f0d290db45a679d55baf37ff39bacb8ce7ec)
---
plugins/vddk/nbdkit-vddk-plugin.pod | 3 +--
plugins/vddk/vddk.c | 41 +++++++++++++++++++++++++----
2 files changed, 37 insertions(+), 7 deletions(-)
diff --git a/plugins/vddk/nbdkit-vddk-plugin.pod b/plugins/vddk/nbdkit-vddk-plugin.pod
index 078badcc..e53d3286 100644
--- a/plugins/vddk/nbdkit-vddk-plugin.pod
+++ b/plugins/vddk/nbdkit-vddk-plugin.pod
@@ -517,8 +517,7 @@ Suppress debugging of datapath calls (C<Read> and C<Write>).
=item B<-D vddk.stats=1>
-When the plugin exits print some statistics about the amount of time
-spent waiting on each VDDK call.
+When the plugin exits print some statistics about each VDDK call.
=back
diff --git a/plugins/vddk/vddk.c b/plugins/vddk/vddk.c
index 3d751544..5f1d223b 100644
--- a/plugins/vddk/vddk.c
+++ b/plugins/vddk/vddk.c
@@ -106,6 +106,8 @@ static bool is_remote;
struct vddk_stat {
const char *name; /* function name */
int64_t usecs; /* total number of usecs consumed */
+ uint64_t calls; /* number of times called */
+ uint64_t bytes; /* bytes transferred, datapath calls only */
};
static pthread_mutex_t stats_lock = PTHREAD_MUTEX_INITIALIZER;
static void display_stats (void);
@@ -141,6 +143,17 @@ static void display_stats (void);
gettimeofday (&end_t, NULL); \
ACQUIRE_LOCK_FOR_CURRENT_SCOPE (&stats_lock); \
stats_##fn.usecs += tvdiff_usec (&start_t, &end_t); \
+ stats_##fn.calls++; \
+ } \
+ } while (0)
+#define VDDK_CALL_END_DATAPATH(fn, bytes_) \
+ while (0); \
+ if (vddk_debug_stats) { \
+ gettimeofday (&end_t, NULL); \
+ ACQUIRE_LOCK_FOR_CURRENT_SCOPE (&stats_lock); \
+ stats_##fn.usecs += tvdiff_usec (&start_t, &end_t); \
+ stats_##fn.calls++; \
+ stats_##fn.bytes += bytes_; \
} \
} while (0)
@@ -191,6 +204,12 @@ stat_compare (const void *vp1, const void *vp2)
else return 0;
}
+static const char *
+api_name_without_prefix (const char *name)
+{
+ return strncmp (name, "VixDiskLib_", 11) == 0 ? name + 11 : name;
+}
+
static void
display_stats (void)
{
@@ -206,10 +225,22 @@ display_stats (void)
qsort (stats.ptr, stats.size, sizeof stats.ptr[0], stat_compare);
nbdkit_debug ("VDDK function stats (-D vddk.stats=1):");
- nbdkit_debug ("%-40s %9s", "", "µs");
+ nbdkit_debug ("%-24s %15s %5s %15s",
+ "VixDiskLib_...", "µs", "calls", "bytes");
for (i = 0; i < stats.size; ++i) {
- if (stats.ptr[i].usecs)
- nbdkit_debug ("%-40s %9" PRIi64, stats.ptr[i].name, stats.ptr[i].usecs);
+ if (stats.ptr[i].usecs) {
+ if (stats.ptr[i].bytes > 0)
+ nbdkit_debug (" %-22s %15" PRIi64 " %5" PRIu64 " %15" PRIu64,
+ api_name_without_prefix (stats.ptr[i].name),
+ stats.ptr[i].usecs,
+ stats.ptr[i].calls,
+ stats.ptr[i].bytes);
+ else
+ nbdkit_debug (" %-22s %15" PRIi64 " %5" PRIu64,
+ api_name_without_prefix (stats.ptr[i].name),
+ stats.ptr[i].usecs,
+ stats.ptr[i].calls);
+ }
}
statlist_reset (&stats);
}
@@ -831,7 +862,7 @@ vddk_pread (void *handle, void *buf, uint32_t count, uint64_t offset,
"%" PRIu32 " sectors, buffer",
offset, count) {
err = VixDiskLib_Read (h->handle, offset, count, buf);
- } VDDK_CALL_END (VixDiskLib_Read);
+ } VDDK_CALL_END_DATAPATH (VixDiskLib_Read, count * VIXDISKLIB_SECTOR_SIZE);
if (err != VIX_OK) {
VDDK_ERROR (err, "VixDiskLib_Read");
return -1;
@@ -871,7 +902,7 @@ vddk_pwrite (void *handle, const void *buf, uint32_t count, uint64_t offset,
"%" PRIu32 " sectors, buffer",
offset, count) {
err = VixDiskLib_Write (h->handle, offset, count, buf);
- } VDDK_CALL_END (VixDiskLib_Write);
+ } VDDK_CALL_END_DATAPATH (VixDiskLib_Write, count * VIXDISKLIB_SECTOR_SIZE);
if (err != VIX_OK) {
VDDK_ERROR (err, "VixDiskLib_Write");
return -1;
--
2.31.1

400
SOURCES/0003-cache-cow-Add-blk_read_multiple-function.patch
View File

@ -1,400 +0,0 @@
From b5dc8577c5c6d1205e2106b629fad327c3a409ea Mon Sep 17 00:00:00 2001
From: "Richard W.M. Jones" <rjones@redhat.com>
Date: Mon, 26 Jul 2021 13:55:21 +0100
Subject: [PATCH] cache, cow: Add blk_read_multiple function
Currently the cache and cow filters break up large requests into many
single block-sized requests to the underlying plugin. For some
plugins (eg. curl) this is very inefficient and causes huge
slow-downs.
For example I tested nbdkit + curl vs nbdkit + cache + curl against a
slow, remote VMware server. A simple run of virt-inspector was at
least 6-7 times slower with the cache filter. (It was so slow that I
didn't actually let it run to completion - I am estimating the
slowdown multiple using interim debug messages).
Implement a new blk_read_multiple function in the cache filter. It
does not break up "runs" of blocks which all have the same cache
state. The cache .pread method uses the new function to read the
block-aligned part of the request.
(cherry picked from commit ab661ccef5b3369fa22c33d0289baddc251b73bf)
---
filters/cache/blk.c | 83 ++++++++++++++++++++++++++++++++-----------
filters/cache/blk.h | 6 ++++
filters/cache/cache.c | 21 +++++------
filters/cow/blk.c | 63 +++++++++++++++++++++++---------
filters/cow/blk.h | 6 ++++
filters/cow/cow.c | 21 +++++------
6 files changed, 138 insertions(+), 62 deletions(-)
diff --git a/filters/cache/blk.c b/filters/cache/blk.c
index f52f30e3..f85ada35 100644
--- a/filters/cache/blk.c
+++ b/filters/cache/blk.c
@@ -44,6 +44,7 @@
#include <string.h>
#include <unistd.h>
#include <fcntl.h>
+#include <limits.h>
#include <errno.h>
#ifdef HAVE_SYS_STATVFS_H
@@ -193,26 +194,40 @@ blk_set_size (uint64_t new_size)
return 0;
}
-int
-blk_read (nbdkit_next *next,
- uint64_t blknum, uint8_t *block, int *err)
+static int
+_blk_read_multiple (nbdkit_next *next,
+ uint64_t blknum, uint64_t nrblocks,
+ uint8_t *block, int *err)
{
off_t offset = blknum * blksize;
- enum bm_entry state = bitmap_get_blk (&bm, blknum, BLOCK_NOT_CACHED);
+ bool not_cached =
+ bitmap_get_blk (&bm, blknum, BLOCK_NOT_CACHED) == BLOCK_NOT_CACHED;
+ uint64_t b, runblocks;
- reclaim (fd, &bm);
+ assert (nrblocks > 0);
if (cache_debug_verbose)
- nbdkit_debug ("cache: blk_read block %" PRIu64
+ nbdkit_debug ("cache: blk_read_multiple block %" PRIu64
" (offset %" PRIu64 ") is %s",
blknum, (uint64_t) offset,
- state == BLOCK_NOT_CACHED ? "not cached" :
- state == BLOCK_CLEAN ? "clean" :
- state == BLOCK_DIRTY ? "dirty" :
- "unknown");
+ not_cached ? "not cached" : "cached");
- if (state == BLOCK_NOT_CACHED) { /* Read underlying plugin. */
- unsigned n = blksize, tail = 0;
+ /* Find out how many of the following blocks form a "run" with the
+ * same cached/not-cached state. We can process that many blocks in
+ * one go.
+ */
+ for (b = 1, runblocks = 1; b < nrblocks; ++b, ++runblocks) {
+ bool s =
+ bitmap_get_blk (&bm, blknum + b, BLOCK_NOT_CACHED) == BLOCK_NOT_CACHED;
+ if (not_cached != s)
+ break;
+ }
+
+ if (not_cached) { /* Read underlying plugin. */
+ unsigned n, tail = 0;
+
+ assert (blksize * runblocks <= UINT_MAX);
+ n = blksize * runblocks;
if (offset + n > size) {
tail = offset + n - size;
@@ -228,32 +243,60 @@ blk_read (nbdkit_next *next,
*/
memset (block + n, 0, tail);
- /* If cache-on-read, copy the block to the cache. */
+ /* If cache-on-read, copy the blocks to the cache. */
if (cache_on_read) {
if (cache_debug_verbose)
nbdkit_debug ("cache: cache-on-read block %" PRIu64
" (offset %" PRIu64 ")",
blknum, (uint64_t) offset);
- if (pwrite (fd, block, blksize, offset) == -1) {
+ if (pwrite (fd, block, blksize * runblocks, offset) == -1) {
*err = errno;
nbdkit_error ("pwrite: %m");
return -1;
}
- bitmap_set_blk (&bm, blknum, BLOCK_CLEAN);
- lru_set_recently_accessed (blknum);
+ for (b = 0; b < runblocks; ++b) {
+ bitmap_set_blk (&bm, blknum + b, BLOCK_CLEAN);
+ lru_set_recently_accessed (blknum + b);
+ }
}
- return 0;
}
else { /* Read cache. */
- if (pread (fd, block, blksize, offset) == -1) {
+ if (pread (fd, block, blksize * runblocks, offset) == -1) {
*err = errno;
nbdkit_error ("pread: %m");
return -1;
}
- lru_set_recently_accessed (blknum);
- return 0;
+ for (b = 0; b < runblocks; ++b)
+ lru_set_recently_accessed (blknum + b);
}
+
+ /* If all done, return. */
+ if (runblocks == nrblocks)
+ return 0;
+
+ /* Recurse to read remaining blocks. */
+ return _blk_read_multiple (next,
+ blknum + runblocks,
+ nrblocks - runblocks,
+ block + blksize * runblocks,
+ err);
+}
+
+int
+blk_read_multiple (nbdkit_next *next,
+ uint64_t blknum, uint64_t nrblocks,
+ uint8_t *block, int *err)
+{
+ reclaim (fd, &bm);
+ return _blk_read_multiple (next, blknum, nrblocks, block, err);
+}
+
+int
+blk_read (nbdkit_next *next,
+ uint64_t blknum, uint8_t *block, int *err)
+{
+ return blk_read_multiple (next, blknum, 1, block, err);
}
int
diff --git a/filters/cache/blk.h b/filters/cache/blk.h
index 87c753e2..1ee33ed7 100644
--- a/filters/cache/blk.h
+++ b/filters/cache/blk.h
@@ -55,6 +55,12 @@ extern int blk_read (nbdkit_next *next,
uint64_t blknum, uint8_t *block, int *err)
__attribute__((__nonnull__ (1, 3, 4)));
+/* As above, but read multiple blocks. */
+extern int blk_read_multiple (nbdkit_next *next,
+ uint64_t blknum, uint64_t nrblocks,
+ uint8_t *block, int *err)
+ __attribute__((__nonnull__ (1, 4, 5)));
+
/* If a single block is not cached, copy it from the plugin. */
extern int blk_cache (nbdkit_next *next,
uint64_t blknum, uint8_t *block, int *err)
diff --git a/filters/cache/cache.c b/filters/cache/cache.c
index 745f552d..14cc03f2 100644
--- a/filters/cache/cache.c
+++ b/filters/cache/cache.c
@@ -313,7 +313,7 @@ cache_pread (nbdkit_next *next,
uint32_t flags, int *err)
{
CLEANUP_FREE uint8_t *block = NULL;
- uint64_t blknum, blkoffs;
+ uint64_t blknum, blkoffs, nrblocks;
int r;
assert (!flags);
@@ -348,22 +348,17 @@ cache_pread (nbdkit_next *next,
}
/* Aligned body */
- /* XXX This breaks up large read requests into smaller ones, which
- * is a problem for plugins which have a large, fixed per-request
- * overhead (hello, curl). We should try to keep large requests
- * together as much as possible, but that requires us to be much
- * smarter here.
- */
- while (count >= blksize) {
+ nrblocks = count / blksize;
+ if (nrblocks > 0) {
ACQUIRE_LOCK_FOR_CURRENT_SCOPE (&lock);
- r = blk_read (next, blknum, buf, err);
+ r = blk_read_multiple (next, blknum, nrblocks, buf, err);
if (r == -1)
return -1;
- buf += blksize;
- count -= blksize;
- offset += blksize;
- blknum++;
+ buf += nrblocks * blksize;
+ count -= nrblocks * blksize;
+ offset += nrblocks * blksize;
+ blknum += nrblocks;
}
/* Unaligned tail */
diff --git a/filters/cow/blk.c b/filters/cow/blk.c
index b7c4d7f1..4ec8d1b8 100644
--- a/filters/cow/blk.c
+++ b/filters/cow/blk.c
@@ -79,6 +79,7 @@
#include <inttypes.h>
#include <unistd.h>
#include <fcntl.h>
+#include <limits.h>
#include <errno.h>
#include <sys/types.h>
@@ -219,33 +220,48 @@ blk_status (uint64_t blknum, bool *present, bool *trimmed)
*trimmed = state == BLOCK_TRIMMED;
}
-/* These are the block operations. They always read or write a single
- * whole block of size ‘blksize’.
+/* These are the block operations. They always read or write whole
+ * blocks of size ‘blksize’.
*/
int
-blk_read (nbdkit_next *next,
- uint64_t blknum, uint8_t *block, int *err)
+blk_read_multiple (nbdkit_next *next,
+ uint64_t blknum, uint64_t nrblocks,
+ uint8_t *block, int *err)
{
off_t offset = blknum * BLKSIZE;
enum bm_entry state;
+ uint64_t b, runblocks;
- /* The state might be modified from another thread - for example
- * another thread might write (BLOCK_NOT_ALLOCATED ->
- * BLOCK_ALLOCATED) while we are reading from the plugin, returning
- * the old data. However a read issued after the write returns
- * should always return the correct data.
+ /* Find out how many of the following blocks form a "run" with the
+ * same state. We can process that many blocks in one go.
+ *
+ * About the locking: The state might be modified from another
+ * thread - for example another thread might write
+ * (BLOCK_NOT_ALLOCATED -> BLOCK_ALLOCATED) while we are reading
+ * from the plugin, returning the old data. However a read issued
+ * after the write returns should always return the correct data.
*/
{
ACQUIRE_LOCK_FOR_CURRENT_SCOPE (&lock);
state = bitmap_get_blk (&bm, blknum, BLOCK_NOT_ALLOCATED);
+
+ for (b = 1, runblocks = 1; b < nrblocks; ++b, ++runblocks) {
+ enum bm_entry s = bitmap_get_blk (&bm, blknum + b, BLOCK_NOT_ALLOCATED);
+ if (state != s)
+ break;
+ }
}
if (cow_debug_verbose)
- nbdkit_debug ("cow: blk_read block %" PRIu64 " (offset %" PRIu64 ") is %s",
+ nbdkit_debug ("cow: blk_read_multiple block %" PRIu64
+ " (offset %" PRIu64 ") is %s",
blknum, (uint64_t) offset, state_to_string (state));
if (state == BLOCK_NOT_ALLOCATED) { /* Read underlying plugin. */
- unsigned n = BLKSIZE, tail = 0;
+ unsigned n, tail = 0;
+
+ assert (BLKSIZE * runblocks <= UINT_MAX);
+ n = BLKSIZE * runblocks;
if (offset + n > size) {
tail = offset + n - size;
@@ -260,20 +276,35 @@ blk_read (nbdkit_next *next,
* zeroing the tail.
*/
memset (block + n, 0, tail);
- return 0;
}
else if (state == BLOCK_ALLOCATED) { /* Read overlay. */
- if (pread (fd, block, BLKSIZE, offset) == -1) {
+ if (pread (fd, block, BLKSIZE * runblocks, offset) == -1) {
*err = errno;
nbdkit_error ("pread: %m");
return -1;
}
- return 0;
}
else /* state == BLOCK_TRIMMED */ {
- memset (block, 0, BLKSIZE);
- return 0;
+ memset (block, 0, BLKSIZE * runblocks);
}
+
+ /* If all done, return. */
+ if (runblocks == nrblocks)
+ return 0;
+
+ /* Recurse to read remaining blocks. */
+ return blk_read_multiple (next,
+ blknum + runblocks,
+ nrblocks - runblocks,
+ block + BLKSIZE * runblocks,
+ err);
+}
+
+int
+blk_read (nbdkit_next *next,
+ uint64_t blknum, uint8_t *block, int *err)
+{
+ return blk_read_multiple (next, blknum, 1, block, err);
}
int
diff --git a/filters/cow/blk.h b/filters/cow/blk.h
index e6fd7417..b066c602 100644
--- a/filters/cow/blk.h
+++ b/filters/cow/blk.h
@@ -55,6 +55,12 @@ extern int blk_read (nbdkit_next *next,
uint64_t blknum, uint8_t *block, int *err)
__attribute__((__nonnull__ (1, 3, 4)));
+/* Read multiple blocks from the overlay or plugin. */
+extern int blk_read_multiple (nbdkit_next *next,
+ uint64_t blknum, uint64_t nrblocks,
+ uint8_t *block, int *err)
+ __attribute__((__nonnull__ (1, 4, 5)));
+
/* Cache mode for blocks not already in overlay */
enum cache_mode {
BLK_CACHE_IGNORE, /* Do nothing */
diff --git a/filters/cow/cow.c b/filters/cow/cow.c
index f30b7505..78daca22 100644
--- a/filters/cow/cow.c
+++ b/filters/cow/cow.c
@@ -210,7 +210,7 @@ cow_pread (nbdkit_next *next,
uint32_t flags, int *err)
{
CLEANUP_FREE uint8_t *block = NULL;
- uint64_t blknum, blkoffs;
+ uint64_t blknum, blkoffs, nrblocks;
int r;
if (!IS_ALIGNED (count | offset, BLKSIZE)) {
@@ -243,21 +243,16 @@ cow_pread (nbdkit_next *next,
}
/* Aligned body */
- /* XXX This breaks up large read requests into smaller ones, which
- * is a problem for plugins which have a large, fixed per-request
- * overhead (hello, curl). We should try to keep large requests
- * together as much as possible, but that requires us to be much
- * smarter here.
- */
- while (count >= BLKSIZE) {
- r = blk_read (next, blknum, buf, err);
+ nrblocks = count / BLKSIZE;
+ if (nrblocks > 0) {
+ r = blk_read_multiple (next, blknum, nrblocks, buf, err);
if (r == -1)
return -1;
- buf += BLKSIZE;
- count -= BLKSIZE;
- offset += BLKSIZE;
- blknum++;
+ buf += nrblocks * BLKSIZE;
+ count -= nrblocks * BLKSIZE;
+ offset += nrblocks * BLKSIZE;
+ blknum += nrblocks;
}
/* Unaligned tail */
--
2.31.1

320
SOURCES/0003-vddk-Simplify-and-consolidate-VDDK_CALL_START-END-ma.patch Normal file
View File

@ -0,0 +1,320 @@
From cc1c3b4ab57a1662bf87766161167fac40a78c0e Mon Sep 17 00:00:00 2001
From: "Richard W.M. Jones" <rjones@redhat.com>
Date: Thu, 21 Oct 2021 22:55:17 +0100
Subject: [PATCH] vddk: Simplify and consolidate VDDK_CALL_START/END macros
We don't need the VDDK_CALL_*_DATAPATH versions of these macros
because the compiler is able to optimize some static strcmps.
Furthermore we can remove extra { .. } when the macros are applied.
(cherry picked from commit 3ea0ed6582faa8f800b7a2a15d58032917a21bd5)
---
plugins/vddk/vddk.c | 124 ++++++++++++++++++++------------------------
1 file changed, 56 insertions(+), 68 deletions(-)
diff --git a/plugins/vddk/vddk.c b/plugins/vddk/vddk.c
index 5f1d223b..993f2d76 100644
--- a/plugins/vddk/vddk.c
+++ b/plugins/vddk/vddk.c
@@ -125,28 +125,16 @@ static void display_stats (void);
#define VDDK_CALL_START(fn, fs, ...) \
do { \
struct timeval start_t, end_t; \
+ /* GCC can optimize this away at compile time: */ \
+ const bool datapath = \
+ strcmp (#fn, "VixDiskLib_Read") == 0 || \
+ strcmp (#fn, "VixDiskLib_Write") == 0; \
if (vddk_debug_stats) \
gettimeofday (&start_t, NULL); \
- nbdkit_debug ("VDDK call: %s (" fs ")", #fn, ##__VA_ARGS__); \
- do
-#define VDDK_CALL_START_DATAPATH(fn, fs, ...) \
- do { \
- struct timeval start_t, end_t; \
- if (vddk_debug_stats) \
- gettimeofday (&start_t, NULL); \
- if (vddk_debug_datapath) \
+ if (!datapath || vddk_debug_datapath) \
nbdkit_debug ("VDDK call: %s (" fs ")", #fn, ##__VA_ARGS__); \
do
-#define VDDK_CALL_END(fn) \
- while (0); \
- if (vddk_debug_stats) { \
- gettimeofday (&end_t, NULL); \
- ACQUIRE_LOCK_FOR_CURRENT_SCOPE (&stats_lock); \
- stats_##fn.usecs += tvdiff_usec (&start_t, &end_t); \
- stats_##fn.calls++; \
- } \
- } while (0)
-#define VDDK_CALL_END_DATAPATH(fn, bytes_) \
+#define VDDK_CALL_END(fn, bytes_) \
while (0); \
if (vddk_debug_stats) { \
gettimeofday (&end_t, NULL); \
@@ -161,13 +149,13 @@ static void display_stats (void);
#define VDDK_ERROR(err, fs, ...) \
do { \
char *vddk_err_msg; \
- VDDK_CALL_START (VixDiskLib_GetErrorText, "%lu", err) { \
+ VDDK_CALL_START (VixDiskLib_GetErrorText, "%lu", err) \
vddk_err_msg = VixDiskLib_GetErrorText ((err), NULL); \
- } VDDK_CALL_END (VixDiskLib_GetErrorText); \
+ VDDK_CALL_END (VixDiskLib_GetErrorText, 0); \
nbdkit_error (fs ": %s", ##__VA_ARGS__, vddk_err_msg); \
- VDDK_CALL_START (VixDiskLib_FreeErrorText, "") { \
+ VDDK_CALL_START (VixDiskLib_FreeErrorText, "") \
VixDiskLib_FreeErrorText (vddk_err_msg); \
- } VDDK_CALL_END (VixDiskLib_FreeErrorText); \
+ VDDK_CALL_END (VixDiskLib_FreeErrorText, 0); \
} while (0)
/* Unload the plugin. */
@@ -175,9 +163,9 @@ static void
vddk_unload (void)
{
if (init_called) {
- VDDK_CALL_START (VixDiskLib_Exit, "") {
+ VDDK_CALL_START (VixDiskLib_Exit, "")
VixDiskLib_Exit ();
- } VDDK_CALL_END (VixDiskLib_Exit);
+ VDDK_CALL_END (VixDiskLib_Exit, 0);
}
if (dl)
dlclose (dl);
@@ -572,13 +560,13 @@ vddk_after_fork (void)
VDDK_CALL_START (VixDiskLib_InitEx,
"%d, %d, &debug_fn, &error_fn, &error_fn, %s, %s",
VDDK_MAJOR, VDDK_MINOR,
- libdir, config ? : "NULL") {
+ libdir, config ? : "NULL")
err = VixDiskLib_InitEx (VDDK_MAJOR, VDDK_MINOR,
&debug_function, /* log function */
&error_function, /* warn function */
&error_function, /* panic function */
libdir, config);
- } VDDK_CALL_END (VixDiskLib_InitEx);
+ VDDK_CALL_END (VixDiskLib_InitEx, 0);
if (err != VIX_OK) {
VDDK_ERROR (err, "VixDiskLib_InitEx");
exit (EXIT_FAILURE);
@@ -640,9 +628,9 @@ allocate_connect_params (void)
VixDiskLibConnectParams *ret;
if (VixDiskLib_AllocateConnectParams != NULL) {
- VDDK_CALL_START (VixDiskLib_AllocateConnectParams, "") {
+ VDDK_CALL_START (VixDiskLib_AllocateConnectParams, "")
ret = VixDiskLib_AllocateConnectParams ();
- } VDDK_CALL_END (VixDiskLib_AllocateConnectParams);
+ VDDK_CALL_END (VixDiskLib_AllocateConnectParams, 0);
}
else
ret = calloc (1, sizeof (VixDiskLibConnectParams));
@@ -657,9 +645,9 @@ free_connect_params (VixDiskLibConnectParams *params)
* originally called. Otherwise use free.
*/
if (VixDiskLib_AllocateConnectParams != NULL) {
- VDDK_CALL_START (VixDiskLib_FreeConnectParams, "params") {
+ VDDK_CALL_START (VixDiskLib_FreeConnectParams, "params")
VixDiskLib_FreeConnectParams (params);
- } VDDK_CALL_END (VixDiskLib_FreeConnectParams);
+ VDDK_CALL_END (VixDiskLib_FreeConnectParams, 0);
}
else
free (params);
@@ -716,13 +704,13 @@ vddk_open (int readonly)
"h->params, %d, %s, %s, &connection",
readonly,
snapshot_moref ? : "NULL",
- transport_modes ? : "NULL") {
+ transport_modes ? : "NULL")
err = VixDiskLib_ConnectEx (h->params,
readonly,
snapshot_moref,
transport_modes,
&h->connection);
- } VDDK_CALL_END (VixDiskLib_ConnectEx);
+ VDDK_CALL_END (VixDiskLib_ConnectEx, 0);
if (err != VIX_OK) {
VDDK_ERROR (err, "VixDiskLib_ConnectEx");
goto err1;
@@ -743,25 +731,25 @@ vddk_open (int readonly)
}
VDDK_CALL_START (VixDiskLib_Open,
- "connection, %s, %d, &handle", filename, flags) {
+ "connection, %s, %d, &handle", filename, flags)
err = VixDiskLib_Open (h->connection, filename, flags, &h->handle);
- } VDDK_CALL_END (VixDiskLib_Open);
+ VDDK_CALL_END (VixDiskLib_Open, 0);
if (err != VIX_OK) {
VDDK_ERROR (err, "VixDiskLib_Open: %s", filename);
goto err2;
}
- VDDK_CALL_START (VixDiskLib_GetTransportMode, "handle") {
+ VDDK_CALL_START (VixDiskLib_GetTransportMode, "handle")
transport_mode = VixDiskLib_GetTransportMode (h->handle);
- } VDDK_CALL_END (VixDiskLib_GetTransportMode);
+ VDDK_CALL_END (VixDiskLib_GetTransportMode, 0);
nbdkit_debug ("transport mode: %s", transport_mode);
return h;
err2:
- VDDK_CALL_START (VixDiskLib_Disconnect, "connection") {
+ VDDK_CALL_START (VixDiskLib_Disconnect, "connection")
VixDiskLib_Disconnect (h->connection);
- } VDDK_CALL_END (VixDiskLib_Disconnect);
+ VDDK_CALL_END (VixDiskLib_Disconnect, 0);
err1:
free_connect_params (h->params);
err0:
@@ -776,12 +764,12 @@ vddk_close (void *handle)
ACQUIRE_LOCK_FOR_CURRENT_SCOPE (&open_close_lock);
struct vddk_handle *h = handle;
- VDDK_CALL_START (VixDiskLib_Close, "handle") {
+ VDDK_CALL_START (VixDiskLib_Close, "handle")
VixDiskLib_Close (h->handle);
- } VDDK_CALL_END (VixDiskLib_Close);
- VDDK_CALL_START (VixDiskLib_Disconnect, "connection") {
+ VDDK_CALL_END (VixDiskLib_Close, 0);
+ VDDK_CALL_START (VixDiskLib_Disconnect, "connection")
VixDiskLib_Disconnect (h->connection);
- } VDDK_CALL_END (VixDiskLib_Disconnect);
+ VDDK_CALL_END (VixDiskLib_Disconnect, 0);
free_connect_params (h->params);
free (h);
@@ -796,9 +784,9 @@ vddk_get_size (void *handle)
VixError err;
uint64_t size;
- VDDK_CALL_START (VixDiskLib_GetInfo, "handle, &info") {
+ VDDK_CALL_START (VixDiskLib_GetInfo, "handle, &info")
err = VixDiskLib_GetInfo (h->handle, &info);
- } VDDK_CALL_END (VixDiskLib_GetInfo);
+ VDDK_CALL_END (VixDiskLib_GetInfo, 0);
if (err != VIX_OK) {
VDDK_ERROR (err, "VixDiskLib_GetInfo");
return -1;
@@ -827,9 +815,9 @@ vddk_get_size (void *handle)
info->uuid ? : "NULL");
}
- VDDK_CALL_START (VixDiskLib_FreeInfo, "info") {
+ VDDK_CALL_START (VixDiskLib_FreeInfo, "info")
VixDiskLib_FreeInfo (info);
- } VDDK_CALL_END (VixDiskLib_FreeInfo);
+ VDDK_CALL_END (VixDiskLib_FreeInfo, 0);
return (int64_t) size;
}
@@ -857,12 +845,12 @@ vddk_pread (void *handle, void *buf, uint32_t count, uint64_t offset,
offset /= VIXDISKLIB_SECTOR_SIZE;
count /= VIXDISKLIB_SECTOR_SIZE;
- VDDK_CALL_START_DATAPATH (VixDiskLib_Read,
- "handle, %" PRIu64 " sectors, "
- "%" PRIu32 " sectors, buffer",
- offset, count) {
+ VDDK_CALL_START (VixDiskLib_Read,
+ "handle, %" PRIu64 " sectors, "
+ "%" PRIu32 " sectors, buffer",
+ offset, count)
err = VixDiskLib_Read (h->handle, offset, count, buf);
- } VDDK_CALL_END_DATAPATH (VixDiskLib_Read, count * VIXDISKLIB_SECTOR_SIZE);
+ VDDK_CALL_END (VixDiskLib_Read, count * VIXDISKLIB_SECTOR_SIZE);
if (err != VIX_OK) {
VDDK_ERROR (err, "VixDiskLib_Read");
return -1;
@@ -897,12 +885,12 @@ vddk_pwrite (void *handle, const void *buf, uint32_t count, uint64_t offset,
offset /= VIXDISKLIB_SECTOR_SIZE;
count /= VIXDISKLIB_SECTOR_SIZE;
- VDDK_CALL_START_DATAPATH (VixDiskLib_Write,
- "handle, %" PRIu64 " sectors, "
- "%" PRIu32 " sectors, buffer",
- offset, count) {
+ VDDK_CALL_START (VixDiskLib_Write,
+ "handle, %" PRIu64 " sectors, "
+ "%" PRIu32 " sectors, buffer",
+ offset, count)
err = VixDiskLib_Write (h->handle, offset, count, buf);
- } VDDK_CALL_END_DATAPATH (VixDiskLib_Write, count * VIXDISKLIB_SECTOR_SIZE);
+ VDDK_CALL_END (VixDiskLib_Write, count * VIXDISKLIB_SECTOR_SIZE);
if (err != VIX_OK) {
VDDK_ERROR (err, "VixDiskLib_Write");
return -1;
@@ -945,9 +933,9 @@ vddk_flush (void *handle, uint32_t flags)
* file so it appears to be the correct call to use here.
*/
- VDDK_CALL_START (VixDiskLib_Flush, "handle") {
+ VDDK_CALL_START (VixDiskLib_Flush, "handle")
err = VixDiskLib_Flush (h->handle);
- } VDDK_CALL_END (VixDiskLib_Flush);
+ VDDK_CALL_END (VixDiskLib_Flush, 0);
if (err != VIX_OK) {
VDDK_ERROR (err, "VixDiskLib_Flush");
return -1;
@@ -983,17 +971,17 @@ vddk_can_extents (void *handle)
*/
VDDK_CALL_START (VixDiskLib_QueryAllocatedBlocks,
"handle, 0, %d sectors, %d sectors",
- VIXDISKLIB_MIN_CHUNK_SIZE, VIXDISKLIB_MIN_CHUNK_SIZE) {
+ VIXDISKLIB_MIN_CHUNK_SIZE, VIXDISKLIB_MIN_CHUNK_SIZE)
err = VixDiskLib_QueryAllocatedBlocks (h->handle,
0, VIXDISKLIB_MIN_CHUNK_SIZE,
VIXDISKLIB_MIN_CHUNK_SIZE,
&block_list);
- } VDDK_CALL_END (VixDiskLib_QueryAllocatedBlocks);
+ VDDK_CALL_END (VixDiskLib_QueryAllocatedBlocks, 0);
error_suppression = 0;
if (err == VIX_OK) {
- VDDK_CALL_START (VixDiskLib_FreeBlockList, "block_list") {
+ VDDK_CALL_START (VixDiskLib_FreeBlockList, "block_list")
VixDiskLib_FreeBlockList (block_list);
- } VDDK_CALL_END (VixDiskLib_FreeBlockList);
+ VDDK_CALL_END (VixDiskLib_FreeBlockList, 0);
}
if (err != VIX_OK) {
char *errmsg = VixDiskLib_GetErrorText (err, NULL);
@@ -1073,12 +1061,12 @@ vddk_extents (void *handle, uint32_t count, uint64_t offset, uint32_t flags,
VDDK_CALL_START (VixDiskLib_QueryAllocatedBlocks,
"handle, %" PRIu64 " sectors, %" PRIu64 " sectors, "
"%d sectors",
- start_sector, nr_sectors, VIXDISKLIB_MIN_CHUNK_SIZE) {
+ start_sector, nr_sectors, VIXDISKLIB_MIN_CHUNK_SIZE)
err = VixDiskLib_QueryAllocatedBlocks (h->handle,
start_sector, nr_sectors,
VIXDISKLIB_MIN_CHUNK_SIZE,
&block_list);
- } VDDK_CALL_END (VixDiskLib_QueryAllocatedBlocks);
+ VDDK_CALL_END (VixDiskLib_QueryAllocatedBlocks, 0);
if (err != VIX_OK) {
VDDK_ERROR (err, "VixDiskLib_QueryAllocatedBlocks");
return -1;
@@ -1097,15 +1085,15 @@ vddk_extents (void *handle, uint32_t count, uint64_t offset, uint32_t flags,
add_extent (extents, &position, blk_offset, true) == -1) ||
(add_extent (extents,
&position, blk_offset + blk_length, false) == -1)) {
- VDDK_CALL_START (VixDiskLib_FreeBlockList, "block_list") {
+ VDDK_CALL_START (VixDiskLib_FreeBlockList, "block_list")
VixDiskLib_FreeBlockList (block_list);
- } VDDK_CALL_END (VixDiskLib_FreeBlockList);
+ VDDK_CALL_END (VixDiskLib_FreeBlockList, 0);
return -1;
}
}
- VDDK_CALL_START (VixDiskLib_FreeBlockList, "block_list") {
+ VDDK_CALL_START (VixDiskLib_FreeBlockList, "block_list")
VixDiskLib_FreeBlockList (block_list);
- } VDDK_CALL_END (VixDiskLib_FreeBlockList);
+ VDDK_CALL_END (VixDiskLib_FreeBlockList, 0);
/* There's an implicit hole after the returned list of blocks, up
* to the end of the QueryAllocatedBlocks request.
--
2.31.1

215
SOURCES/0004-cache-cow-Use-full-pread-pwrite-operations.patch
View File

@ -1,215 +0,0 @@
From 5bd332a683811586039f99f31c01d4f2f7181334 Mon Sep 17 00:00:00 2001
From: "Richard W.M. Jones" <rjones@redhat.com>
Date: Mon, 26 Jul 2021 15:21:18 +0100
Subject: [PATCH] cache, cow: Use full pread/pwrite operations
Although it probably cannot happen on Linux, POSIX allows pread/pwrite
to return or write fewer bytes than requested. The cache and cow
filters didn't handle this situation. Replace the raw
pread(2)/pwrite(2) syscalls with alternate versions which can handle
this.
(cherry picked from commit ce0db9d7736dd28dd0f10951ce65853e50b35e41)
---
common/utils/Makefile.am | 1 +
common/utils/full-rw.c | 81 ++++++++++++++++++++++++++++++++++++++++
common/utils/utils.h | 2 +
filters/cache/blk.c | 10 ++---
filters/cow/blk.c | 6 +--
5 files changed, 92 insertions(+), 8 deletions(-)
create mode 100644 common/utils/full-rw.c
diff --git a/common/utils/Makefile.am b/common/utils/Makefile.am
index 1708a4c8..14e9dfc4 100644
--- a/common/utils/Makefile.am
+++ b/common/utils/Makefile.am
@@ -40,6 +40,7 @@ libutils_la_SOURCES = \
cleanup-nbdkit.c \
cleanup.h \
environ.c \
+ full-rw.c \
quote.c \
utils.c \
utils.h \
diff --git a/common/utils/full-rw.c b/common/utils/full-rw.c
new file mode 100644
index 00000000..55b32cdd
--- /dev/null
+++ b/common/utils/full-rw.c
@@ -0,0 +1,81 @@
+/* nbdkit
+ * Copyright (C) 2021 Red Hat Inc.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * * Neither the name of Red Hat nor the names of its contributors may be
+ * used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY RED HAT AND CONTRIBUTORS ''AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+ * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL RED HAT OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
+ * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+ * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+/* These functions are like pread(2)/pwrite(2) but they always read or
+ * write the full amount, or fail.
+ */
+
+#include <config.h>
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <errno.h>
+
+ssize_t
+full_pread (int fd, void *buf, size_t count, off_t offset)
+{
+ ssize_t ret = 0, r;
+
+ while (count > 0) {
+ r = pread (fd, buf, count, offset);
+ if (r == -1) return -1;
+ if (r == 0) {
+ /* Presumably the caller wasn't expecting end-of-file here, so
+ * return an error.
+ */
+ errno = EIO;
+ return -1;
+ }
+ ret += r;
+ offset += r;
+ count -= r;
+ }
+
+ return ret;
+}
+
+ssize_t
+full_pwrite (int fd, const void *buf, size_t count, off_t offset)
+{
+ ssize_t ret = 0, r;
+
+ while (count > 0) {
+ r = pwrite (fd, buf, count, offset);
+ if (r == -1) return -1;
+ ret += r;
+ offset += r;
+ count -= r;
+ }
+
+ return ret;
+}
diff --git a/common/utils/utils.h b/common/utils/utils.h
index f8f70212..83397ae1 100644
--- a/common/utils/utils.h
+++ b/common/utils/utils.h
@@ -40,5 +40,7 @@ extern int set_cloexec (int fd);
extern int set_nonblock (int fd);
extern char **copy_environ (char **env, ...) __attribute__((__sentinel__));
extern char *make_temporary_directory (void);
+extern ssize_t full_pread (int fd, void *buf, size_t count, off_t offset);
+extern ssize_t full_pwrite (int fd, const void *buf, size_t count, off_t offset);
#endif /* NBDKIT_UTILS_H */
diff --git a/filters/cache/blk.c b/filters/cache/blk.c
index f85ada35..42bd3779 100644
--- a/filters/cache/blk.c
+++ b/filters/cache/blk.c
@@ -250,7 +250,7 @@ _blk_read_multiple (nbdkit_next *next,
" (offset %" PRIu64 ")",
blknum, (uint64_t) offset);
- if (pwrite (fd, block, blksize * runblocks, offset) == -1) {
+ if (full_pwrite (fd, block, blksize * runblocks, offset) == -1) {
*err = errno;
nbdkit_error ("pwrite: %m");
return -1;
@@ -262,7 +262,7 @@ _blk_read_multiple (nbdkit_next *next,
}
}
else { /* Read cache. */
- if (pread (fd, block, blksize * runblocks, offset) == -1) {
+ if (full_pread (fd, block, blksize * runblocks, offset) == -1) {
*err = errno;
nbdkit_error ("pread: %m");
return -1;
@@ -339,7 +339,7 @@ blk_cache (nbdkit_next *next,
nbdkit_debug ("cache: cache block %" PRIu64 " (offset %" PRIu64 ")",
blknum, (uint64_t) offset);
- if (pwrite (fd, block, blksize, offset) == -1) {
+ if (full_pwrite (fd, block, blksize, offset) == -1) {
*err = errno;
nbdkit_error ("pwrite: %m");
return -1;
@@ -380,7 +380,7 @@ blk_writethrough (nbdkit_next *next,
nbdkit_debug ("cache: writethrough block %" PRIu64 " (offset %" PRIu64 ")",
blknum, (uint64_t) offset);
- if (pwrite (fd, block, blksize, offset) == -1) {
+ if (full_pwrite (fd, block, blksize, offset) == -1) {
*err = errno;
nbdkit_error ("pwrite: %m");
return -1;
@@ -414,7 +414,7 @@ blk_write (nbdkit_next *next,
nbdkit_debug ("cache: writeback block %" PRIu64 " (offset %" PRIu64 ")",
blknum, (uint64_t) offset);
- if (pwrite (fd, block, blksize, offset) == -1) {
+ if (full_pwrite (fd, block, blksize, offset) == -1) {
*err = errno;
nbdkit_error ("pwrite: %m");
return -1;
diff --git a/filters/cow/blk.c b/filters/cow/blk.c
index 4ec8d1b8..121b0dd4 100644
--- a/filters/cow/blk.c
+++ b/filters/cow/blk.c
@@ -278,7 +278,7 @@ blk_read_multiple (nbdkit_next *next,
memset (block + n, 0, tail);
}
else if (state == BLOCK_ALLOCATED) { /* Read overlay. */
- if (pread (fd, block, BLKSIZE * runblocks, offset) == -1) {
+ if (full_pread (fd, block, BLKSIZE * runblocks, offset) == -1) {
*err = errno;
nbdkit_error ("pread: %m");
return -1;
@@ -353,7 +353,7 @@ blk_cache (nbdkit_next *next,
memset (block + n, 0, tail);
if (mode == BLK_CACHE_COW) {
- if (pwrite (fd, block, BLKSIZE, offset) == -1) {
+ if (full_pwrite (fd, block, BLKSIZE, offset) == -1) {
*err = errno;
nbdkit_error ("pwrite: %m");
return -1;
@@ -372,7 +372,7 @@ blk_write (uint64_t blknum, const uint8_t *block, int *err)
nbdkit_debug ("cow: blk_write block %" PRIu64 " (offset %" PRIu64 ")",
blknum, (uint64_t) offset);
- if (pwrite (fd, block, BLKSIZE, offset) == -1) {
+ if (full_pwrite (fd, block, BLKSIZE, offset) == -1) {
*err = errno;
nbdkit_error ("pwrite: %m");
return -1;
--
2.31.1

84
SOURCES/0004-vddk-Document-troubleshooting-performance-problems.patch Normal file
View File

@ -0,0 +1,84 @@
From 4bd9926c0e506fdb04976d348b1c7614865c8b06 Mon Sep 17 00:00:00 2001
From: "Richard W.M. Jones" <rjones@redhat.com>
Date: Fri, 22 Oct 2021 18:00:27 +0100
Subject: [PATCH] vddk: Document troubleshooting performance problems
Document how to use -D vddk.stats=1 to diagnose performance problems
with VDDK.
(cherry picked from commit e491978c193f49010cc28ad344d0fb3c1b5ede35)
---
plugins/vddk/nbdkit-vddk-plugin.pod | 57 +++++++++++++++++++++++++++++
1 file changed, 57 insertions(+)
diff --git a/plugins/vddk/nbdkit-vddk-plugin.pod b/plugins/vddk/nbdkit-vddk-plugin.pod
index e53d3286..5a426135 100644
--- a/plugins/vddk/nbdkit-vddk-plugin.pod
+++ b/plugins/vddk/nbdkit-vddk-plugin.pod
@@ -475,6 +475,63 @@ and restarting the C<hostd> service:
For more information see L<https://bugzilla.redhat.com/1614276>.
+=head2 Troubleshooting performance problems
+
+VDDK has very uneven performance with some operations being very slow.
+This plugin has options to allow you to debug performance issues. If
+your application has a debug or diagnostic setting, add the following
+nbdkit command line options:
+
+ -v -D nbdkit.backend.datapath=0 -D vddk.datapath=0 -D vddk.stats=1
+
+C<-v> enables verbose messages and the two datapath options I<disable>
+the very verbose per-read/-write messages. C<-D vddk.stats=1> enables
+a summary when nbdkit exits of the cumulative time taken in each VDDK
+function, the number of times each function was called, and (for read
+and write) the number of bytes transferred. An example of what those
+stats look like can be found here:
+L<https://gitlab.com/nbdkit/nbdkit/-/commit/5c80f0d290db45a679d55baf37ff39bacb8ce7ec>
+
+You can interpret the stats as follows:
+
+=over 4
+
+=item C<Read>
+
+The cumulative time spent waiting for VDDK to return from
+C<VixDiskLib_Read> calls, the number of times this function was
+called, and the total bytes read. You can use this to determine the
+read bandwidth to the VMware server.
+
+=item C<Write>
+
+=item C<Flush>
+
+Same as above, but for writing and flushing writes.
+
+=item C<QueryAllocatedBlocks>
+
+This call is used to query information about the sparseness of the
+remote disk. It is only available in VDDK E<ge> 6.7. The call is
+notably very slow in all versions of VMware we have tested.
+
+=item C<Open>
+
+=item C<Close>
+
+=item C<ConnectEx>
+
+=item C<Disconnect>
+
+=item C<InitEx>
+
+=item C<Exit>
+
+The cumulative time spent connecting and disconnecting from the VMware
+server, which can also be very slow.
+
+=back
+
=head1 SUPPORTED VERSIONS OF VDDK
This plugin requires VDDK E<ge> 5.5.5, which in turn means that it
--
2.31.1

151
SOURCES/0005-cache-Implement-cache-on-read-PATH.patch
View File

@ -1,151 +0,0 @@
From 4db23fd29af0488aa9c7e01577a5be9565a4465e Mon Sep 17 00:00:00 2001
From: "Richard W.M. Jones" <rjones@redhat.com>
Date: Mon, 26 Jul 2021 16:16:15 +0100
Subject: [PATCH] cache: Implement cache-on-read=/PATH
For virt-v2v we will need to be able to turn cache-on-read on while
performing inspection and modification of the guest, and off when
doing the bulk copy. To do that allow the cache-on-read parameter to
refer to a path where the existence of the path toggles the feature.
(We could restart nbdkit between these phases, but this change avoids
doing that.)
(cherry picked from commit c8b575241b15b3bf0adaf15313e67e5ed4270b5a)
---
filters/cache/blk.c | 2 +-
filters/cache/cache.c | 33 ++++++++++++++++++++-------
filters/cache/cache.h | 10 ++++++--
filters/cache/nbdkit-cache-filter.pod | 11 ++++++++-
4 files changed, 44 insertions(+), 12 deletions(-)
diff --git a/filters/cache/blk.c b/filters/cache/blk.c
index 42bd3779..19f79605 100644
--- a/filters/cache/blk.c
+++ b/filters/cache/blk.c
@@ -244,7 +244,7 @@ _blk_read_multiple (nbdkit_next *next,
memset (block + n, 0, tail);
/* If cache-on-read, copy the blocks to the cache. */
- if (cache_on_read) {
+ if (cache_on_read ()) {
if (cache_debug_verbose)
nbdkit_debug ("cache: cache-on-read block %" PRIu64
" (offset %" PRIu64 ")",
diff --git a/filters/cache/cache.c b/filters/cache/cache.c
index 14cc03f2..44da0008 100644
--- a/filters/cache/cache.c
+++ b/filters/cache/cache.c
@@ -74,7 +74,8 @@ unsigned blksize;
enum cache_mode cache_mode = CACHE_MODE_WRITEBACK;
int64_t max_size = -1;
unsigned hi_thresh = 95, lo_thresh = 80;
-bool cache_on_read = false;
+enum cor_mode cor_mode = COR_OFF;
+const char *cor_path;
static int cache_flush (nbdkit_next *next, void *handle, uint32_t flags,
int *err);
@@ -161,12 +162,16 @@ cache_config (nbdkit_next_config *next, nbdkit_backend *nxdata,
}
#endif /* !HAVE_CACHE_RECLAIM */
else if (strcmp (key, "cache-on-read") == 0) {
- int r;
-
- r = nbdkit_parse_bool (value);
- if (r == -1)
- return -1;
- cache_on_read = r;
+ if (value[0] == '/') {
+ cor_path = value;
+ cor_mode = COR_PATH;
+ }
+ else {
+ int r = nbdkit_parse_bool (value);
+ if (r == -1)
+ return -1;
+ cor_mode = r ? COR_ON : COR_OFF;
+ }
return 0;
}
else {
@@ -177,7 +182,7 @@ cache_config (nbdkit_next_config *next, nbdkit_backend *nxdata,
#define cache_config_help_common \
"cache=MODE Set cache MODE, one of writeback (default),\n" \
" writethrough, or unsafe.\n" \
- "cache-on-read=BOOL Set to true to cache on reads (default false).\n"
+ "cache-on-read=BOOL|/PATH Set to true to cache on reads (default false).\n"
#ifndef HAVE_CACHE_RECLAIM
#define cache_config_help cache_config_help_common
#else
@@ -187,6 +192,18 @@ cache_config (nbdkit_next_config *next, nbdkit_backend *nxdata,
"cache-low-threshold=PCT Percentage of max size where reclaim ends.\n"
#endif
+/* Decide if cache-on-read is currently on or off. */
+bool
+cache_on_read (void)
+{
+ switch (cor_mode) {
+ case COR_ON: return true;
+ case COR_OFF: return false;
+ case COR_PATH: return access (cor_path, F_OK) == 0;
+ default: abort ();
+ }
+}
+
static int
cache_config_complete (nbdkit_next_config_complete *next,
nbdkit_backend *nxdata)
diff --git a/filters/cache/cache.h b/filters/cache/cache.h
index 2b72221f..a559adef 100644
--- a/filters/cache/cache.h
+++ b/filters/cache/cache.h
@@ -49,7 +49,13 @@ extern unsigned blksize;
extern int64_t max_size;
extern unsigned hi_thresh, lo_thresh;
-/* Cache read requests. */
-extern bool cache_on_read;
+/* Cache on read mode. */
+extern enum cor_mode {
+ COR_OFF,
+ COR_ON,
+ COR_PATH,
+} cor_mode;
+extern const char *cor_path;
+extern bool cache_on_read (void);
#endif /* NBDKIT_CACHE_H */
diff --git a/filters/cache/nbdkit-cache-filter.pod b/filters/cache/nbdkit-cache-filter.pod
index ebcf1d10..f20cb9ce 100644
--- a/filters/cache/nbdkit-cache-filter.pod
+++ b/filters/cache/nbdkit-cache-filter.pod
@@ -8,7 +8,7 @@ nbdkit-cache-filter - nbdkit caching filter
[cache-max-size=SIZE]
[cache-high-threshold=N]
[cache-low-threshold=N]
- [cache-on-read=true|false]
+ [cache-on-read=true|false|/PATH]
[plugin-args...]
=head1 DESCRIPTION
@@ -87,6 +87,15 @@ the plugin.
Do not cache read requests (this is the default).
+=item B<cache-on-read=/PATH>
+
+(nbdkit E<ge> 1.28)
+
+When F</PATH> (which must be an absolute path) exists, this behaves
+like C<cache-on-read=true>, and when it does not exist like
+C<cache-on-read=false>. This allows you to control the cache-on-read
+behaviour while nbdkit is running.
+
=back
=head1 CACHE MAXIMUM SIZE
--
2.31.1

141
SOURCES/0005-vddk-Include-VDDK-major-library-version-in-dump-plug.patch Normal file
View File

@ -0,0 +1,141 @@
From eb6ccb03d0ca12ef19e5705cd96f81824910087b Mon Sep 17 00:00:00 2001
From: "Richard W.M. Jones" <rjones@redhat.com>
Date: Sat, 23 Oct 2021 16:16:39 +0100
Subject: [PATCH] vddk: Include VDDK major library version in --dump-plugin
output
Although it doesn't seem to be possible to get the precise VDDK
version, With a relatively simple change we can at least return the
VDDK major version. Currently this can be 5, 6 or 7.
(cherry picked from commit 8700649d147948897f3b97810a1dff37924bdd6e)
---
plugins/vddk/nbdkit-vddk-plugin.pod | 4 ++++
plugins/vddk/vddk.c | 29 +++++++++++++++++++----------
tests/test-vddk-real-dump-plugin.sh | 2 ++
3 files changed, 25 insertions(+), 10 deletions(-)
diff --git a/plugins/vddk/nbdkit-vddk-plugin.pod b/plugins/vddk/nbdkit-vddk-plugin.pod
index 5a426135..bc3c3c94 100644
--- a/plugins/vddk/nbdkit-vddk-plugin.pod
+++ b/plugins/vddk/nbdkit-vddk-plugin.pod
@@ -422,6 +422,10 @@ at runtime.
If this is printed then the C<nfchostport=PORT> parameter is supported
by this build.
+=item C<vddk_library_version=...>
+
+The VDDK major library version: 5, 6, 7, ...
+
=item C<vddk_dll=...>
Prints the full path to the VDDK shared library. Since this requires
diff --git a/plugins/vddk/vddk.c b/plugins/vddk/vddk.c
index 993f2d76..d74a484d 100644
--- a/plugins/vddk/vddk.c
+++ b/plugins/vddk/vddk.c
@@ -81,6 +81,7 @@ NBDKIT_DLL_PUBLIC int vddk_debug_stats;
static void *dl; /* dlopen handle */
static bool init_called; /* was InitEx called */
static __thread int error_suppression; /* threadlocal error suppression */
+static int library_version; /* VDDK major: 5, 6, 7, ... */
static enum { NONE = 0, ZLIB, FASTLZ, SKIPZ } compression; /* compression */
static char *config; /* config */
@@ -405,7 +406,10 @@ vddk_config (const char *key, const char *value)
static void
load_library (bool load_error_is_fatal)
{
- static const char *sonames[] = {
+ static struct {
+ const char *soname;
+ int library_version;
+ } libs[] = {
/* Prefer the newest library in case multiple exist. Check two
* possible directories: the usual VDDK installation puts .so
* files in an arch-specific subdirectory of $libdir (our minimum
@@ -413,12 +417,13 @@ load_library (bool load_error_is_fatal)
* but our testsuite is easier to write if we point libdir
* directly to a stub .so.
*/
- "lib64/libvixDiskLib.so.7",
- "libvixDiskLib.so.7",
- "lib64/libvixDiskLib.so.6",
- "libvixDiskLib.so.6",
- "lib64/libvixDiskLib.so.5",
- "libvixDiskLib.so.5",
+ { "lib64/libvixDiskLib.so.7", 7 },
+ { "libvixDiskLib.so.7", 7 },
+ { "lib64/libvixDiskLib.so.6", 6 },
+ { "libvixDiskLib.so.6", 6 },
+ { "lib64/libvixDiskLib.so.5", 5 },
+ { "libvixDiskLib.so.5", 5 },
+ { NULL }
};
size_t i;
CLEANUP_FREE char *orig_error = NULL;
@@ -431,19 +436,20 @@ load_library (bool load_error_is_fatal)
}
}
- for (i = 0; i < sizeof sonames / sizeof sonames[0]; ++i) {
+ for (i = 0; libs[i].soname != NULL; ++i) {
CLEANUP_FREE char *path;
/* Set the full path so that dlopen will preferentially load the
* system libraries from the same directory.
*/
- if (asprintf (&path, "%s/%s", libdir, sonames[i]) == -1) {
+ if (asprintf (&path, "%s/%s", libdir, libs[i].soname) == -1) {
nbdkit_error ("asprintf: %m");
exit (EXIT_FAILURE);
}
dl = dlopen (path, RTLD_NOW);
if (dl != NULL) {
+ library_version = libs[i].library_version;
/* Now that we found the library, ensure that LD_LIBRARY_PATH
* includes its directory for all future loads. This may modify
* path in-place and/or re-exec nbdkit, but that's okay.
@@ -464,10 +470,12 @@ load_library (bool load_error_is_fatal)
"If '%s' is located on a non-standard path you may need to\n"
"set libdir=/path/to/vmware-vix-disklib-distrib.\n\n"
"See nbdkit-vddk-plugin(1) man page section \"LIBRARY LOCATION\" for details.",
- orig_error ? : "(unknown error)", sonames[0]);
+ orig_error ? : "(unknown error)", libs[0].soname);
exit (EXIT_FAILURE);
}
+ assert (library_version >= 5);
+
/* Load symbols. */
#define STUB(fn,ret,args) \
do { \
@@ -583,6 +591,7 @@ vddk_dump_plugin (void)
printf ("vddk_default_libdir=%s\n", VDDK_LIBDIR);
printf ("vddk_has_nfchostport=1\n");
+ printf ("vddk_library_version=%d\n", library_version);
#if defined(HAVE_DLADDR)
/* It would be nice to print the version of VDDK from the shared
diff --git a/tests/test-vddk-real-dump-plugin.sh b/tests/test-vddk-real-dump-plugin.sh
index 2cb7724e..0a079c6c 100755
--- a/tests/test-vddk-real-dump-plugin.sh
+++ b/tests/test-vddk-real-dump-plugin.sh
@@ -58,10 +58,12 @@ rm -f $files
cleanup_fn rm -f $files
nbdkit -f -v vddk libdir="$vddkdir" --dump-plugin > $out
+cat $out
# Check the vddk_* entries are set.
grep ^vddk_default_libdir= $out
grep ^vddk_has_nfchostport= $out
+grep ^vddk_library_version= $out
grep ^vddk_dll= $out
dll="$(grep ^vddk_dll $out | cut -d= -f2)"
--
2.31.1

278
SOURCES/0006-cache-Add-cache-min-block-size-parameter.patch
View File

@ -1,278 +0,0 @@
From f7f4b71d559dc6950bc795742f64e8eaeeadf3ec Mon Sep 17 00:00:00 2001
From: "Richard W.M. Jones" <rjones@redhat.com>
Date: Mon, 26 Jul 2021 16:30:26 +0100
Subject: [PATCH] cache: Add cache-min-block-size parameter
This allows you to choose a larger block size. I found experimentally
that this improves performance because of locality in access patterns.
The idea came from qcow2 which implicitly does the same thing because
of the relatively large cluster size (32K).
nbdkit + cache-filter with 4K block size + cache-on-read + curl
(to a very slow remote site):
=> virt-inspector took 22 mins
same with 64K block size:
=> virt-inspector took 19 mins
However compared to a qcow2 file using qemu's copy-on-read, backed
with nbdkit + curl we are still a lot slower, possibly because having
the cache inside virt-inspector greatly reduces round trip overhead:
=> virt-inspector took 13 mins
(cherry picked from commit 4ceacb6caa64e12bd78af5f90e86ee591e055944)
---
filters/cache/blk.c | 2 +-
filters/cache/cache.c | 36 ++++++++++----
filters/cache/cache.h | 3 ++
filters/cache/nbdkit-cache-filter.pod | 9 ++++
tests/Makefile.am | 2 +
tests/test-cache-block-size.sh | 70 +++++++++++++++++++++++++++
6 files changed, 112 insertions(+), 10 deletions(-)
create mode 100755 tests/test-cache-block-size.sh
diff --git a/filters/cache/blk.c b/filters/cache/blk.c
index 19f79605..6276985f 100644
--- a/filters/cache/blk.c
+++ b/filters/cache/blk.c
@@ -149,7 +149,7 @@ blk_init (void)
nbdkit_error ("fstatvfs: %s: %m", tmpdir);
return -1;
}
- blksize = MAX (4096, statvfs.f_bsize);
+ blksize = MAX (min_block_size, statvfs.f_bsize);
nbdkit_debug ("cache: block size: %u", blksize);
bitmap_init (&bm, blksize, 2 /* bits per block */);
diff --git a/filters/cache/cache.c b/filters/cache/cache.c
index 44da0008..109ac89e 100644
--- a/filters/cache/cache.c
+++ b/filters/cache/cache.c
@@ -40,6 +40,7 @@
#include <inttypes.h>
#include <unistd.h>
#include <fcntl.h>
+#include <limits.h>
#include <errno.h>
#include <assert.h>
#include <sys/types.h>
@@ -62,6 +63,7 @@
#include "blk.h"
#include "reclaim.h"
#include "isaligned.h"
+#include "ispowerof2.h"
#include "minmax.h"
#include "rounding.h"
@@ -70,7 +72,8 @@
*/
static pthread_mutex_t lock = PTHREAD_MUTEX_INITIALIZER;
-unsigned blksize;
+unsigned blksize; /* actual block size (picked by blk.c) */
+unsigned min_block_size = 4096;
enum cache_mode cache_mode = CACHE_MODE_WRITEBACK;
int64_t max_size = -1;
unsigned hi_thresh = 95, lo_thresh = 80;
@@ -80,13 +83,6 @@ const char *cor_path;
static int cache_flush (nbdkit_next *next, void *handle, uint32_t flags,
int *err);
-static void
-cache_load (void)
-{
- if (blk_init () == -1)
- exit (EXIT_FAILURE);
-}
-
static void
cache_unload (void)
{
@@ -116,6 +112,19 @@ cache_config (nbdkit_next_config *next, nbdkit_backend *nxdata,
return -1;
}
}
+ else if (strcmp (key, "cache-min-block-size") == 0) {
+ int64_t r;
+
+ r = nbdkit_parse_size (value);
+ if (r == -1)
+ return -1;
+ if (r < 4096 || !is_power_of_2 (r) || r > UINT_MAX) {
+ nbdkit_error ("cache-min-block-size is not a power of 2, or is too small or too large");
+ return -1;
+ }
+ min_block_size = r;
+ return 0;
+ }
#ifdef HAVE_CACHE_RECLAIM
else if (strcmp (key, "cache-max-size") == 0) {
int64_t r;
@@ -220,6 +229,15 @@ cache_config_complete (nbdkit_next_config_complete *next,
return next (nxdata);
}
+static int
+cache_get_ready (int thread_model)
+{
+ if (blk_init () == -1)
+ return -1;
+
+ return 0;
+}
+
/* Get the file size, set the cache size. */
static int64_t
cache_get_size (nbdkit_next *next,
@@ -691,11 +709,11 @@ cache_cache (nbdkit_next *next,
static struct nbdkit_filter filter = {
.name = "cache",
.longname = "nbdkit caching filter",
- .load = cache_load,
.unload = cache_unload,
.config = cache_config,
.config_complete = cache_config_complete,
.config_help = cache_config_help,
+ .get_ready = cache_get_ready,
.prepare = cache_prepare,
.get_size = cache_get_size,
.can_cache = cache_can_cache,
diff --git a/filters/cache/cache.h b/filters/cache/cache.h
index a559adef..5c32c37c 100644
--- a/filters/cache/cache.h
+++ b/filters/cache/cache.h
@@ -45,6 +45,9 @@ extern enum cache_mode {
/* Size of a block in the cache. */
extern unsigned blksize;
+/* Minimum block size (cache-min-block-size parameter). */
+extern unsigned min_block_size;
+
/* Maximum size of the cache and high/low thresholds. */
extern int64_t max_size;
extern unsigned hi_thresh, lo_thresh;
diff --git a/filters/cache/nbdkit-cache-filter.pod b/filters/cache/nbdkit-cache-filter.pod
index f20cb9ce..6cbd1c08 100644
--- a/filters/cache/nbdkit-cache-filter.pod
+++ b/filters/cache/nbdkit-cache-filter.pod
@@ -5,6 +5,7 @@ nbdkit-cache-filter - nbdkit caching filter
=head1 SYNOPSIS
nbdkit --filter=cache plugin [cache=writeback|writethrough|unsafe]
+ [cache-min-block-size=SIZE]
[cache-max-size=SIZE]
[cache-high-threshold=N]
[cache-low-threshold=N]
@@ -59,6 +60,14 @@ This is dangerous and can cause data loss, but this may be acceptable
if you only use it for testing or with data that you don't care about
or can cheaply reconstruct.
+=item B<cache-min-block-size=>SIZE
+
+Set the minimum block size used by the cache. This must be a power of
+2 and E<ge> 4096.
+
+The default is 4096, or the block size of the filesystem which
+contains the temporary file storing the cache (whichever is larger).
+
=item B<cache-max-size=>SIZE
=item B<cache-high-threshold=>N
diff --git a/tests/Makefile.am b/tests/Makefile.am
index 9630205d..a038eabc 100644
--- a/tests/Makefile.am
+++ b/tests/Makefile.am
@@ -1371,12 +1371,14 @@ EXTRA_DIST += test-blocksize.sh test-blocksize-extents.sh
# cache filter test.
TESTS += \
test-cache.sh \
+ test-cache-block-size.sh \
test-cache-on-read.sh \
test-cache-max-size.sh \
test-cache-unaligned.sh \
$(NULL)
EXTRA_DIST += \
test-cache.sh \
+ test-cache-block-size.sh \
test-cache-on-read.sh \
test-cache-max-size.sh \
test-cache-unaligned.sh \
diff --git a/tests/test-cache-block-size.sh b/tests/test-cache-block-size.sh
new file mode 100755
index 00000000..a2a27407
--- /dev/null
+++ b/tests/test-cache-block-size.sh
@@ -0,0 +1,70 @@
+#!/usr/bin/env bash
+# nbdkit
+# Copyright (C) 2018-2021 Red Hat Inc.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#
+# * Redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in the
+# documentation and/or other materials provided with the distribution.
+#
+# * Neither the name of Red Hat nor the names of its contributors may be
+# used to endorse or promote products derived from this software without
+# specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY RED HAT AND CONTRIBUTORS ''AS IS'' AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+# PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL RED HAT OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
+# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+# SUCH DAMAGE.
+
+source ./functions.sh
+set -e
+set -x
+
+requires_filter cache
+requires_nbdsh_uri
+
+sock=$(mktemp -u /tmp/nbdkit-test-sock.XXXXXX)
+files="cache-block-size.img $sock cache-block-size.pid"
+rm -f $files
+cleanup_fn rm -f $files
+
+# Create an empty base image.
+truncate -s 128K cache-block-size.img
+
+# Run nbdkit with the caching filter.
+start_nbdkit -P cache-block-size.pid -U $sock --filter=cache \
+ file cache-block-size.img cache-min-block-size=64K
+
+nbdsh --connect "nbd+unix://?socket=$sock" \
+ -c '
+# Write some pattern data to the overlay and check it reads back OK.
+buf = b"abcd" * 16384
+h.pwrite(buf, 32768)
+zero = h.pread(32768, 0)
+assert zero == bytearray(32768)
+buf2 = h.pread(65536, 32768)
+assert buf == buf2
+
+# Flushing should write through to the underlying file.
+h.flush()
+
+with open("cache-block-size.img", "rb") as file:
+ zero = file.read(32768)
+ assert zero == bytearray(32768)
+ buf2 = file.read(65536)
+ assert buf == buf2
+'
--
2.31.1

52
SOURCES/0006-vddk-Add-logical-and-physical-sector-size-to-D-vddk..patch Normal file
View File

@ -0,0 +1,52 @@
From 0139f1815e9259fa789d84d2f32d30ee59bd728c Mon Sep 17 00:00:00 2001
From: "Richard W.M. Jones" <rjones@redhat.com>
Date: Sat, 23 Oct 2021 16:24:27 +0100
Subject: [PATCH] vddk: Add logical and physical sector size to -D
vddk.diskinfo output
In VDDK >= 7 it is possible to display the logical and physical sector
size in debug output.
This commit also extends the test since this flag was not tested
before.
(cherry picked from commit 5bb8f0586e1faabcbf4f43d722a3b3cb5b352e33)
---
plugins/vddk/vddk.c | 6 ++++++
tests/test-vddk-real.sh | 3 ++-
2 files changed, 8 insertions(+), 1 deletion(-)
diff --git a/plugins/vddk/vddk.c b/plugins/vddk/vddk.c
index d74a484d..50bdde26 100644
--- a/plugins/vddk/vddk.c
+++ b/plugins/vddk/vddk.c
@@ -822,6 +822,12 @@ vddk_get_size (void *handle)
info->parentFileNameHint ? : "NULL");
nbdkit_debug ("disk info: uuid: %s",
info->uuid ? : "NULL");
+ if (library_version >= 7) {
+ nbdkit_debug ("disk info: sectory size: "
+ "logical %" PRIu32 " physical %" PRIu32,
+ info->logicalSectorSize,
+ info->physicalSectorSize);
+ }
}
VDDK_CALL_START (VixDiskLib_FreeInfo, "info")
diff --git a/tests/test-vddk-real.sh b/tests/test-vddk-real.sh
index a6aceac9..ae965245 100755
--- a/tests/test-vddk-real.sh
+++ b/tests/test-vddk-real.sh
@@ -89,7 +89,8 @@ if grep 'cannot open shared object file' $log; then
fi
# Now run nbdkit for the test.
-start_nbdkit -P $pid -U $sock -D vddk.stats=1 vddk libdir="$vddkdir" $vmdk
+start_nbdkit -P $pid -U $sock -D vddk.stats=1 -D vddk.diskinfo=1 \
+ vddk libdir="$vddkdir" $vmdk
uri="nbd+unix:///?socket=$sock"
# VDDK < 6.0 did not support flush, so disable flush test there. Also
--
2.31.1

138
SOURCES/0007-cache-cow-Use-a-64K-block-size-by-default.patch
View File

@ -1,138 +0,0 @@
From 83e1167e1a350bd08ac6245f47a5877438408492 Mon Sep 17 00:00:00 2001
From: "Richard W.M. Jones" <rjones@redhat.com>
Date: Mon, 26 Jul 2021 17:39:23 +0100
Subject: [PATCH] cache, cow: Use a 64K block size by default
Based on the results presented in the previous commit, use a 64K block
size by default in both the cache and cow filters. For the cache
filter you could go back to a 4K block size if you wanted by using the
cache-min-block-size=4K parameter. For cow it is compiled in so
cannot be adjusted.
(cherry picked from commit c1905b0a28677d961babdb16d6f30ae61042c825)
---
filters/cache/cache.c | 2 +-
filters/cache/nbdkit-cache-filter.pod | 4 ++--
filters/cow/blk.h | 2 +-
tests/test-cache-block-size.sh | 2 +-
tests/test-cow-extents1.sh | 33 +++++++++++++++------------
5 files changed, 23 insertions(+), 20 deletions(-)
diff --git a/filters/cache/cache.c b/filters/cache/cache.c
index 109ac89e..c912c5fb 100644
--- a/filters/cache/cache.c
+++ b/filters/cache/cache.c
@@ -73,7 +73,7 @@
static pthread_mutex_t lock = PTHREAD_MUTEX_INITIALIZER;
unsigned blksize; /* actual block size (picked by blk.c) */
-unsigned min_block_size = 4096;
+unsigned min_block_size = 65536;
enum cache_mode cache_mode = CACHE_MODE_WRITEBACK;
int64_t max_size = -1;
unsigned hi_thresh = 95, lo_thresh = 80;
diff --git a/filters/cache/nbdkit-cache-filter.pod b/filters/cache/nbdkit-cache-filter.pod
index 6cbd1c08..df9c1f99 100644
--- a/filters/cache/nbdkit-cache-filter.pod
+++ b/filters/cache/nbdkit-cache-filter.pod
@@ -65,8 +65,8 @@ or can cheaply reconstruct.
Set the minimum block size used by the cache. This must be a power of
2 and E<ge> 4096.
-The default is 4096, or the block size of the filesystem which
-contains the temporary file storing the cache (whichever is larger).
+The default is 64K, or the block size of the filesystem which contains
+the temporary file storing the cache (whichever is larger).
=item B<cache-max-size=>SIZE
diff --git a/filters/cow/blk.h b/filters/cow/blk.h
index b066c602..1bc85283 100644
--- a/filters/cow/blk.h
+++ b/filters/cow/blk.h
@@ -36,7 +36,7 @@
/* Size of a block in the overlay. A 4K block size means that we need
* 64 MB of memory to store the bitmap for a 1 TB underlying image.
*/
-#define BLKSIZE 4096
+#define BLKSIZE 65536
/* Initialize the overlay and bitmap. */
extern int blk_init (void);
diff --git a/tests/test-cache-block-size.sh b/tests/test-cache-block-size.sh
index a2a27407..d20cc940 100755
--- a/tests/test-cache-block-size.sh
+++ b/tests/test-cache-block-size.sh
@@ -47,7 +47,7 @@ truncate -s 128K cache-block-size.img
# Run nbdkit with the caching filter.
start_nbdkit -P cache-block-size.pid -U $sock --filter=cache \
- file cache-block-size.img cache-min-block-size=64K
+ file cache-block-size.img cache-min-block-size=4K
nbdsh --connect "nbd+unix://?socket=$sock" \
-c '
diff --git a/tests/test-cow-extents1.sh b/tests/test-cow-extents1.sh
index 8e0e0383..ebfd83f6 100755
--- a/tests/test-cow-extents1.sh
+++ b/tests/test-cow-extents1.sh
@@ -65,7 +65,7 @@ cleanup_fn rm -f $files
# Create a base file which is half allocated, half sparse.
dd if=/dev/urandom of=$base count=128 bs=1K
-truncate -s 256K $base
+truncate -s 4M $base
lastmod="$(stat -c "%y" $base)"
# Run nbdkit with a COW overlay.
@@ -76,30 +76,33 @@ uri="nbd+unix:///?socket=$sock"
nbdinfo --map "$uri" > $out
cat $out
if [ "$(tr -s ' ' < $out | cut -d' ' -f 1-4)" != " 0 131072 0
- 131072 131072 3" ]; then
+ 131072 4063232 3" ]; then
echo "$0: unexpected initial file map"
exit 1
fi
# Punch some holes.
nbdsh -u "$uri" \
- -c 'h.trim(4096, 4096)' \
- -c 'h.trim(4098, 16383)' \
- -c 'h.pwrite(b"1"*4096, 65536)' \
- -c 'h.trim(8192, 131072)' \
- -c 'h.pwrite(b"2"*8192, 196608)'
+ -c 'bs = 65536' \
+ -c 'h.trim(bs, bs)' \
+ -c 'h.trim(bs+2, 4*bs-1)' \
+ -c 'h.pwrite(b"1"*bs, 16*bs)' \
+ -c 'h.trim(2*bs, 32*bs)' \
+ -c 'h.pwrite(b"2"*(2*bs), 48*bs)'
# The extents map should be fully allocated.
nbdinfo --map "$uri" > $out
cat $out
-if [ "$(tr -s ' ' < $out | cut -d' ' -f 1-4)" != " 0 4096 0
- 4096 4096 3
- 8192 8192 0
- 16384 4096 3
- 20480 110592 0
- 131072 65536 3
- 196608 8192 0
- 204800 57344 3" ]; then
+if [ "$(tr -s ' ' < $out | cut -d' ' -f 1-4)" != " 0 65536 0
+ 65536 131072 3
+ 196608 65536 0
+ 262144 65536 3
+ 327680 65536 0
+ 393216 655360 3
+ 1048576 65536 0
+ 1114112 2031616 3
+ 3145728 131072 0
+ 3276800 917504 3" ]; then
echo "$0: unexpected trimmed file map"
exit 1
fi
--
2.31.1

27
SOURCES/0007-vddk-Fix-typo-in-debug-message.patch Normal file
View File

@ -0,0 +1,27 @@
From a5f73cbcbb6891d2e3c2cb541d47b44a236785ce Mon Sep 17 00:00:00 2001
From: "Richard W.M. Jones" <rjones@redhat.com>
Date: Sat, 23 Oct 2021 19:41:07 +0100
Subject: [PATCH] vddk: Fix typo in debug message
Fixes: commit 5bb8f0586e1faabcbf4f43d722a3b3cb5b352e33
(cherry picked from commit 343dadeb7340d7b8c5730e2bbab33c829b569122)
---
plugins/vddk/vddk.c | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/plugins/vddk/vddk.c b/plugins/vddk/vddk.c
index 50bdde26..65399a91 100644
--- a/plugins/vddk/vddk.c
+++ b/plugins/vddk/vddk.c
@@ -823,7 +823,7 @@ vddk_get_size (void *handle)
nbdkit_debug ("disk info: uuid: %s",
info->uuid ? : "NULL");
if (library_version >= 7) {
- nbdkit_debug ("disk info: sectory size: "
+ nbdkit_debug ("disk info: sector size: "
"logical %" PRIu32 " physical %" PRIu32,
info->logicalSectorSize,
info->physicalSectorSize);
--
2.31.1

50
SOURCES/0008-cache-Refactor-printing-state-into-new-function.patch
View File

@ -1,50 +0,0 @@
From 2592bb42051b3e6d17240badc814b9b16f121c1d Mon Sep 17 00:00:00 2001
From: "Richard W.M. Jones" <rjones@redhat.com>
Date: Tue, 27 Jul 2021 21:16:30 +0100
Subject: [PATCH] cache: Refactor printing state into new function
This minor refactoring just makes the cache and cow filters' blk.c a
little bit more similar.
(cherry picked from commit bdb86ea14c00a950f2a2d34071ac1e0799d29132)
---
filters/cache/blk.c | 16 ++++++++++++----
1 file changed, 12 insertions(+), 4 deletions(-)
diff --git a/filters/cache/blk.c b/filters/cache/blk.c
index 6276985f..e50a7f24 100644
--- a/filters/cache/blk.c
+++ b/filters/cache/blk.c
@@ -94,6 +94,17 @@ enum bm_entry {
BLOCK_DIRTY = 3,
};
+static const char *
+state_to_string (enum bm_entry state)
+{
+ switch (state) {
+ case BLOCK_NOT_CACHED: return "not cached";
+ case BLOCK_CLEAN: return "clean";
+ case BLOCK_DIRTY: return "dirty";
+ default: abort ();
+ }
+}
+
/* Extra debugging (-D cache.verbose=1). */
NBDKIT_DLL_PUBLIC int cache_debug_verbose = 0;
@@ -312,10 +323,7 @@ blk_cache (nbdkit_next *next,
nbdkit_debug ("cache: blk_cache block %" PRIu64
" (offset %" PRIu64 ") is %s",
blknum, (uint64_t) offset,
- state == BLOCK_NOT_CACHED ? "not cached" :
- state == BLOCK_CLEAN ? "clean" :
- state == BLOCK_DIRTY ? "dirty" :
- "unknown");
+ state_to_string (state));
if (state == BLOCK_NOT_CACHED) {
/* Read underlying plugin, copy to cache regardless of cache-on-read. */
--
2.31.1

55
SOURCES/0008-vddk-Only-print-vddk_library_version-when-we-managed.patch Normal file
View File

@ -0,0 +1,55 @@
From 1cb810a416e1bdd78a8e5df886a3185d3cfa54d0 Mon Sep 17 00:00:00 2001
From: "Richard W.M. Jones" <rjones@redhat.com>
Date: Sat, 23 Oct 2021 19:50:52 +0100
Subject: [PATCH] vddk: Only print vddk_library_version when we managed to load
the library
Because --dump-plugin calls load_library (false) it won't fail if we
didn't manage to load the library. This results in library_version
being 0, which we printed incorrectly.
Resolve this problem by not printing the vddk_library_version entry in
this case.
Fixes: commit 8700649d147948897f3b97810a1dff37924bdd6e
(cherry picked from commit a3fba12c3e9c2113009f556360ae0bd04c45f6bb)
---
plugins/vddk/nbdkit-vddk-plugin.pod | 1 +
plugins/vddk/vddk.c | 9 ++++++++-
2 files changed, 9 insertions(+), 1 deletion(-)
diff --git a/plugins/vddk/nbdkit-vddk-plugin.pod b/plugins/vddk/nbdkit-vddk-plugin.pod
index bc3c3c94..49e3d75d 100644
--- a/plugins/vddk/nbdkit-vddk-plugin.pod
+++ b/plugins/vddk/nbdkit-vddk-plugin.pod
@@ -425,6 +425,7 @@ by this build.
=item C<vddk_library_version=...>
The VDDK major library version: 5, 6, 7, ...
+If this is omitted it means the library could not be loaded.
=item C<vddk_dll=...>
diff --git a/plugins/vddk/vddk.c b/plugins/vddk/vddk.c
index 65399a91..39a7d261 100644
--- a/plugins/vddk/vddk.c
+++ b/plugins/vddk/vddk.c
@@ -591,7 +591,14 @@ vddk_dump_plugin (void)
printf ("vddk_default_libdir=%s\n", VDDK_LIBDIR);
printf ("vddk_has_nfchostport=1\n");
- printf ("vddk_library_version=%d\n", library_version);
+
+ /* Because load_library (false) we might not have loaded VDDK, in
+ * which case we didn't set library_version. Note this cannot
+ * happen in the normal (non-debug-plugin) path because there we use
+ * load_library (true).
+ */
+ if (library_version > 0)
+ printf ("vddk_library_version=%d\n", library_version);
#if defined(HAVE_DLADDR)
/* It would be nice to print the version of VDDK from the shared
--
2.31.1

147
SOURCES/0009-tests-cache-Test-cache-on-read-option-really-caches.patch
View File

@ -1,147 +0,0 @@
From 315948e75e06d038bd8afa319a41e3fde33b4174 Mon Sep 17 00:00:00 2001
From: "Richard W.M. Jones" <rjones@redhat.com>
Date: Thu, 29 Jul 2021 20:16:43 +0100
Subject: [PATCH] tests: cache: Test cache-on-read option really caches
By making use of the delay filter to add a penalty for hitting the
plugin we can check whether or not the cache-on-read option is
working.
(cherry picked from commit 3ae7aa533bb9322ab6dc6deecb687ded76634ab4)
---
tests/Makefile.am | 2 +
tests/test-cache-on-read-caches.sh | 87 ++++++++++++++++++++++++++++++
tests/test-cache-on-read.sh | 5 --
3 files changed, 89 insertions(+), 5 deletions(-)
create mode 100755 tests/test-cache-on-read-caches.sh
diff --git a/tests/Makefile.am b/tests/Makefile.am
index a038eabc..51ca913a 100644
--- a/tests/Makefile.am
+++ b/tests/Makefile.am
@@ -1373,6 +1373,7 @@ TESTS += \
test-cache.sh \
test-cache-block-size.sh \
test-cache-on-read.sh \
+ test-cache-on-read-caches.sh \
test-cache-max-size.sh \
test-cache-unaligned.sh \
$(NULL)
@@ -1380,6 +1381,7 @@ EXTRA_DIST += \
test-cache.sh \
test-cache-block-size.sh \
test-cache-on-read.sh \
+ test-cache-on-read-caches.sh \
test-cache-max-size.sh \
test-cache-unaligned.sh \
$(NULL)
diff --git a/tests/test-cache-on-read-caches.sh b/tests/test-cache-on-read-caches.sh
new file mode 100755
index 00000000..80b34159
--- /dev/null
+++ b/tests/test-cache-on-read-caches.sh
@@ -0,0 +1,87 @@
+#!/usr/bin/env bash
+# nbdkit
+# Copyright (C) 2018-2021 Red Hat Inc.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#
+# * Redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in the
+# documentation and/or other materials provided with the distribution.
+#
+# * Neither the name of Red Hat nor the names of its contributors may be
+# used to endorse or promote products derived from this software without
+# specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY RED HAT AND CONTRIBUTORS ''AS IS'' AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+# PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL RED HAT OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
+# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+# SUCH DAMAGE.
+
+source ./functions.sh
+set -e
+set -x
+
+requires_filter cache
+requires_filter delay
+requires_nbdsh_uri
+
+sock=$(mktemp -u /tmp/nbdkit-test-sock.XXXXXX)
+files="$sock cache-on-read-caches.pid"
+rm -f $files
+cleanup_fn rm -f $files
+
+# Run nbdkit with the cache filter, cache-on-read and a read delay.
+start_nbdkit -P cache-on-read-caches.pid -U $sock \
+ --filter=cache --filter=delay \
+ memory 64K cache-on-read=true rdelay=10
+
+nbdsh --connect "nbd+unix://?socket=$sock" \
+ -c '
+from time import time
+
+# First read should suffer a penalty. Because we are reading
+# a single 64K block (same size as the cache block), we should
+# only suffer one penalty of approx. 10 seconds.
+st = time()
+zb = h.pread(65536, 0)
+et = time()
+el = et-st
+print("elapsed time: %g" % el)
+assert et-st >= 10
+assert zb == bytearray(65536)
+
+# Second read should not suffer a penalty.
+st = time()
+zb = h.pread(65536, 0)
+et = time()
+el = et-st
+print("elapsed time: %g" % el)
+assert el < 10
+assert zb == bytearray(65536)
+
+# Write something.
+buf = b"abcd" * 16384
+h.pwrite(buf, 0)
+
+# Reading back should be quick since it is stored in the overlay.
+st = time()
+buf2 = h.pread(65536, 0)
+et = time()
+el = et-st
+print("elapsed time: %g" % el)
+assert el < 10
+assert buf == buf2
+'
diff --git a/tests/test-cache-on-read.sh b/tests/test-cache-on-read.sh
index f8584dcd..85ca83d4 100755
--- a/tests/test-cache-on-read.sh
+++ b/tests/test-cache-on-read.sh
@@ -56,9 +56,4 @@ zero = h.pread(32768, 0)
assert zero == bytearray(32768)
buf2 = h.pread(65536, 32768)
assert buf == buf2
-
-# XXX Suggestion to improve this test: Use the delay filter below the
-# cache filter, and time reads to prove that the second read is faster
-# because it is not going through the delay filter and plugin.
-# XXX second h.pread here ...
'
--
2.31.1

72
SOURCES/0009-vddk-Print-one-line-in-dump-plugin-output-for-each-V.patch Normal file
View File

@ -0,0 +1,72 @@
From 8780009ec092d9cc5a408b7597d88aa54db13639 Mon Sep 17 00:00:00 2001
From: "Richard W.M. Jones" <rjones@redhat.com>
Date: Mon, 25 Oct 2021 08:36:53 +0100
Subject: [PATCH] vddk: Print one line in --dump-plugin output for each VDDK
API
Helps when detecting if certain optional features are being used, such
as flush and extents.
(cherry picked from commit 4ee13559e46cf622410d0bdd7db29bb00908b40a)
---
plugins/vddk/nbdkit-vddk-plugin.pod | 9 +++++++++
plugins/vddk/vddk.c | 10 ++++++++++
tests/test-vddk-real-dump-plugin.sh | 1 +
3 files changed, 20 insertions(+)
diff --git a/plugins/vddk/nbdkit-vddk-plugin.pod b/plugins/vddk/nbdkit-vddk-plugin.pod
index 49e3d75d..0702aa75 100644
--- a/plugins/vddk/nbdkit-vddk-plugin.pod
+++ b/plugins/vddk/nbdkit-vddk-plugin.pod
@@ -432,6 +432,15 @@ If this is omitted it means the library could not be loaded.
Prints the full path to the VDDK shared library. Since this requires
a glibc extension it may not be available in all builds of the plugin.
+=item C<VixDiskLib_...=1>
+
+For each VDDK API that the plugin uses I<and> which is present in the
+VDDK library that was loaded, we print the name of the API
+(eg. C<VixDiskLib_Open=1>). This lets you see which optional APIs are
+available, such as C<VixDiskLib_Flush> and
+C<VixDiskLib_QueryAllocatedBlocks>. If the library could not be
+loaded then these lines are not printed.
+
=back
=head1 NOTES
diff --git a/plugins/vddk/vddk.c b/plugins/vddk/vddk.c
index 39a7d261..096b04bf 100644
--- a/plugins/vddk/vddk.c
+++ b/plugins/vddk/vddk.c
@@ -616,6 +616,16 @@ vddk_dump_plugin (void)
printf ("vddk_dll=%s\n", p);
}
#endif
+
+ /* Note we print all VDDK APIs found here, not just the optional
+ * ones. That is so if we update the baseline VDDK in future and
+ * make optional into required APIs, the output doesn't change.
+ */
+#define STUB(fn,ret,args) if (fn != NULL) printf ("%s=1\n", #fn);
+#define OPTIONAL_STUB(fn,ret,args) STUB(fn,ret,args)
+#include "vddk-stubs.h"
+#undef STUB
+#undef OPTIONAL_STUB
}
/* The rules on threads and VDDK are here:
diff --git a/tests/test-vddk-real-dump-plugin.sh b/tests/test-vddk-real-dump-plugin.sh
index 0a079c6c..e37c8b54 100755
--- a/tests/test-vddk-real-dump-plugin.sh
+++ b/tests/test-vddk-real-dump-plugin.sh
@@ -65,6 +65,7 @@ grep ^vddk_default_libdir= $out
grep ^vddk_has_nfchostport= $out
grep ^vddk_library_version= $out
grep ^vddk_dll= $out
+grep ^VixDiskLib_Open=1 $out
dll="$(grep ^vddk_dll $out | cut -d= -f2)"
test -f "$dll"
--
2.31.1

457
SOURCES/0010-cow-Implement-cow-on-read.patch
View File

@ -1,457 +0,0 @@
From 57f9bd29f9d7432ad5a70620c373b28db768a314 Mon Sep 17 00:00:00 2001
From: "Richard W.M. Jones" <rjones@redhat.com>
Date: Tue, 27 Jul 2021 23:01:52 +0100
Subject: [PATCH] cow: Implement cow-on-read
This is very similar to the nbdkit-cache-filter cache-on-read flag.
(cherry picked from commit bd93b3f27246f917de48a6cc2525d9c424c07976)
---
filters/cow/blk.c | 21 ++++++--
filters/cow/blk.h | 10 ++--
filters/cow/cow.c | 56 ++++++++++++++++----
filters/cow/nbdkit-cow-filter.pod | 17 ++++++
tests/Makefile.am | 4 ++
tests/test-cow-on-read-caches.sh | 87 +++++++++++++++++++++++++++++++
tests/test-cow-on-read.sh | 59 +++++++++++++++++++++
7 files changed, 236 insertions(+), 18 deletions(-)
create mode 100755 tests/test-cow-on-read-caches.sh
create mode 100755 tests/test-cow-on-read.sh
diff --git a/filters/cow/blk.c b/filters/cow/blk.c
index 121b0dd4..4f84e092 100644
--- a/filters/cow/blk.c
+++ b/filters/cow/blk.c
@@ -226,7 +226,7 @@ blk_status (uint64_t blknum, bool *present, bool *trimmed)
int
blk_read_multiple (nbdkit_next *next,
uint64_t blknum, uint64_t nrblocks,
- uint8_t *block, int *err)
+ uint8_t *block, bool cow_on_read, int *err)
{
off_t offset = blknum * BLKSIZE;
enum bm_entry state;
@@ -276,6 +276,19 @@ blk_read_multiple (nbdkit_next *next,
* zeroing the tail.
*/
memset (block + n, 0, tail);
+
+ /* If cow-on-read is true then copy the blocks to the cache and
+ * set them as allocated.
+ */
+ if (cow_on_read) {
+ if (full_pwrite (fd, block, BLKSIZE * runblocks, offset) == -1) {
+ *err = errno;
+ nbdkit_error ("pwrite: %m");
+ return -1;
+ }
+ for (b = 0; b < runblocks; ++b)
+ bitmap_set_blk (&bm, blknum+b, BLOCK_ALLOCATED);
+ }
}
else if (state == BLOCK_ALLOCATED) { /* Read overlay. */
if (full_pread (fd, block, BLKSIZE * runblocks, offset) == -1) {
@@ -297,14 +310,14 @@ blk_read_multiple (nbdkit_next *next,
blknum + runblocks,
nrblocks - runblocks,
block + BLKSIZE * runblocks,
- err);
+ cow_on_read, err);
}
int
blk_read (nbdkit_next *next,
- uint64_t blknum, uint8_t *block, int *err)
+ uint64_t blknum, uint8_t *block, bool cow_on_read, int *err)
{
- return blk_read_multiple (next, blknum, 1, block, err);
+ return blk_read_multiple (next, blknum, 1, block, cow_on_read, err);
}
int
diff --git a/filters/cow/blk.h b/filters/cow/blk.h
index 1bc85283..b7e6f092 100644
--- a/filters/cow/blk.h
+++ b/filters/cow/blk.h
@@ -52,14 +52,16 @@ extern void blk_status (uint64_t blknum, bool *present, bool *trimmed);
/* Read a single block from the overlay or plugin. */
extern int blk_read (nbdkit_next *next,
- uint64_t blknum, uint8_t *block, int *err)
- __attribute__((__nonnull__ (1, 3, 4)));
+ uint64_t blknum, uint8_t *block,
+ bool cow_on_read, int *err)
+ __attribute__((__nonnull__ (1, 3, 5)));
/* Read multiple blocks from the overlay or plugin. */
extern int blk_read_multiple (nbdkit_next *next,
uint64_t blknum, uint64_t nrblocks,
- uint8_t *block, int *err)
- __attribute__((__nonnull__ (1, 4, 5)));
+ uint8_t *block,
+ bool cow_on_read, int *err)
+ __attribute__((__nonnull__ (1, 4, 6)));
/* Cache mode for blocks not already in overlay */
enum cache_mode {
diff --git a/filters/cow/cow.c b/filters/cow/cow.c
index 78daca22..6efb39f2 100644
--- a/filters/cow/cow.c
+++ b/filters/cow/cow.c
@@ -38,6 +38,7 @@
#include <stdbool.h>
#include <inttypes.h>
#include <string.h>
+#include <unistd.h>
#include <errno.h>
#include <pthread.h>
@@ -59,6 +60,15 @@ static pthread_mutex_t rmw_lock = PTHREAD_MUTEX_INITIALIZER;
static bool cow_on_cache;
+/* Cache on read ("cow-on-read") mode. */
+extern enum cor_mode {
+ COR_OFF,
+ COR_ON,
+ COR_PATH,
+} cor_mode;
+enum cor_mode cor_mode = COR_OFF;
+const char *cor_path;
+
static void
cow_load (void)
{
@@ -85,13 +95,39 @@ cow_config (nbdkit_next_config *next, nbdkit_backend *nxdata,
cow_on_cache = r;
return 0;
}
+ else if (strcmp (key, "cow-on-read") == 0) {
+ if (value[0] == '/') {
+ cor_path = value;
+ cor_mode = COR_PATH;
+ }
+ else {
+ int r = nbdkit_parse_bool (value);
+ if (r == -1)
+ return -1;
+ cor_mode = r ? COR_ON : COR_OFF;
+ }
+ return 0;
+ }
else {
return next (nxdata, key, value);
}
}
#define cow_config_help \
- "cow-on-cache=<BOOL> Set to true to treat client cache requests as writes.\n"
+ "cow-on-cache=<BOOL> Copy cache (prefetch) requests to the overlay.\n" \
+ "cow-on-read=<BOOL>|/PATH Copy read requests to the overlay."
+
+/* Decide if cow-on-read is currently on or off. */
+bool
+cow_on_read (void)
+{
+ switch (cor_mode) {
+ case COR_ON: return true;
+ case COR_OFF: return false;
+ case COR_PATH: return access (cor_path, F_OK) == 0;
+ default: abort ();
+ }
+}
static void *
cow_open (nbdkit_next_open *next, nbdkit_context *nxdata,
@@ -230,7 +266,7 @@ cow_pread (nbdkit_next *next,
uint64_t n = MIN (BLKSIZE - blkoffs, count);
assert (block);
- r = blk_read (next, blknum, block, err);
+ r = blk_read (next, blknum, block, cow_on_read (), err);
if (r == -1)
return -1;
@@ -245,7 +281,7 @@ cow_pread (nbdkit_next *next,
/* Aligned body */
nrblocks = count / BLKSIZE;
if (nrblocks > 0) {
- r = blk_read_multiple (next, blknum, nrblocks, buf, err);
+ r = blk_read_multiple (next, blknum, nrblocks, buf, cow_on_read (), err);
if (r == -1)
return -1;
@@ -258,7 +294,7 @@ cow_pread (nbdkit_next *next,
/* Unaligned tail */
if (count) {
assert (block);
- r = blk_read (next, blknum, block, err);
+ r = blk_read (next, blknum, block, cow_on_read (), err);
if (r == -1)
return -1;
@@ -299,7 +335,7 @@ cow_pwrite (nbdkit_next *next,
*/
assert (block);
ACQUIRE_LOCK_FOR_CURRENT_SCOPE (&rmw_lock);
- r = blk_read (next, blknum, block, err);
+ r = blk_read (next, blknum, block, cow_on_read (), err);
if (r != -1) {
memcpy (&block[blkoffs], buf, n);
r = blk_write (blknum, block, err);
@@ -329,7 +365,7 @@ cow_pwrite (nbdkit_next *next,
if (count) {
assert (block);
ACQUIRE_LOCK_FOR_CURRENT_SCOPE (&rmw_lock);
- r = blk_read (next, blknum, block, err);
+ r = blk_read (next, blknum, block, cow_on_read (), err);
if (r != -1) {
memcpy (block, buf, count);
r = blk_write (blknum, block, err);
@@ -379,7 +415,7 @@ cow_zero (nbdkit_next *next,
* Hold the rmw_lock over the whole operation.
*/
ACQUIRE_LOCK_FOR_CURRENT_SCOPE (&rmw_lock);
- r = blk_read (next, blknum, block, err);
+ r = blk_read (next, blknum, block, cow_on_read (), err);
if (r != -1) {
memset (&block[blkoffs], 0, n);
r = blk_write (blknum, block, err);
@@ -411,7 +447,7 @@ cow_zero (nbdkit_next *next,
/* Unaligned tail */
if (count) {
ACQUIRE_LOCK_FOR_CURRENT_SCOPE (&rmw_lock);
- r = blk_read (next, blknum, block, err);
+ r = blk_read (next, blknum, block, cow_on_read (), err);
if (r != -1) {
memset (block, 0, count);
r = blk_write (blknum, block, err);
@@ -455,7 +491,7 @@ cow_trim (nbdkit_next *next,
* Hold the lock over the whole operation.
*/
ACQUIRE_LOCK_FOR_CURRENT_SCOPE (&rmw_lock);
- r = blk_read (next, blknum, block, err);
+ r = blk_read (next, blknum, block, cow_on_read (), err);
if (r != -1) {
memset (&block[blkoffs], 0, n);
r = blk_write (blknum, block, err);
@@ -482,7 +518,7 @@ cow_trim (nbdkit_next *next,
/* Unaligned tail */
if (count) {
ACQUIRE_LOCK_FOR_CURRENT_SCOPE (&rmw_lock);
- r = blk_read (next, blknum, block, err);
+ r = blk_read (next, blknum, block, cow_on_read (), err);
if (r != -1) {
memset (block, 0, count);
r = blk_write (blknum, block, err);
diff --git a/filters/cow/nbdkit-cow-filter.pod b/filters/cow/nbdkit-cow-filter.pod
index 571189e7..01261429 100644
--- a/filters/cow/nbdkit-cow-filter.pod
+++ b/filters/cow/nbdkit-cow-filter.pod
@@ -62,6 +62,23 @@ the data from the plugin into the overlay.
Do not save data from cache (prefetch) requests in the overlay. This
leaves the overlay as small as possible. This is the default.
+=item B<cow-on-read=true>
+
+When the client issues a read request, copy the data into the overlay
+so that the same data can be served more quickly later.
+
+=item B<cow-on-read=false>
+
+Do not save data from read requests in the overlay. This leaves the
+overlay as small as possible. This is the default.
+
+=item B<cow-on-read=/PATH>
+
+When F</PATH> (which must be an absolute path) exists, this behaves
+like C<cow-on-read=true>, and when it does not exist like
+C<cow-on-read=false>. This allows you to control the C<cow-on-read>
+behaviour while nbdkit is running.
+
=back
=head1 EXAMPLES
diff --git a/tests/Makefile.am b/tests/Makefile.am
index 51ca913a..edc8d66d 100644
--- a/tests/Makefile.am
+++ b/tests/Makefile.am
@@ -1407,6 +1407,8 @@ TESTS += \
test-cow-extents1.sh \
test-cow-extents2.sh \
test-cow-extents-large.sh \
+ test-cow-on-read.sh \
+ test-cow-on-read-caches.sh \
test-cow-unaligned.sh \
$(NULL)
endif
@@ -1417,6 +1419,8 @@ EXTRA_DIST += \
test-cow-extents2.sh \
test-cow-extents-large.sh \
test-cow-null.sh \
+ test-cow-on-read.sh \
+ test-cow-on-read-caches.sh \
test-cow-unaligned.sh \
$(NULL)
diff --git a/tests/test-cow-on-read-caches.sh b/tests/test-cow-on-read-caches.sh
new file mode 100755
index 00000000..c5b60198
--- /dev/null
+++ b/tests/test-cow-on-read-caches.sh
@@ -0,0 +1,87 @@
+#!/usr/bin/env bash
+# nbdkit
+# Copyright (C) 2018-2021 Red Hat Inc.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#
+# * Redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in the
+# documentation and/or other materials provided with the distribution.
+#
+# * Neither the name of Red Hat nor the names of its contributors may be
+# used to endorse or promote products derived from this software without
+# specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY RED HAT AND CONTRIBUTORS ''AS IS'' AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+# PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL RED HAT OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
+# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+# SUCH DAMAGE.
+
+source ./functions.sh
+set -e
+set -x
+
+requires_filter cow
+requires_filter delay
+requires_nbdsh_uri
+
+sock=$(mktemp -u /tmp/nbdkit-test-sock.XXXXXX)
+files="$sock cow-on-read-caches.pid"
+rm -f $files
+cleanup_fn rm -f $files
+
+# Run nbdkit with the cow filter, cow-on-read and a read delay.
+start_nbdkit -P cow-on-read-caches.pid -U $sock \
+ --filter=cow --filter=delay \
+ memory 64K cow-on-read=true rdelay=10
+
+nbdsh --connect "nbd+unix://?socket=$sock" \
+ -c '
+from time import time
+
+# First read should suffer a penalty. Because we are reading
+# a single 64K block (same size as the COW block), we should
+# only suffer one penalty of approx. 10 seconds.
+st = time()
+zb = h.pread(65536, 0)
+et = time()
+el = et-st
+print("elapsed time: %g" % el)
+assert et-st >= 10
+assert zb == bytearray(65536)
+
+# Second read should not suffer a penalty.
+st = time()
+zb = h.pread(65536, 0)
+et = time()
+el = et-st
+print("elapsed time: %g" % el)
+assert el < 10
+assert zb == bytearray(65536)
+
+# Write something.
+buf = b"abcd" * 16384
+h.pwrite(buf, 0)
+
+# Reading back should be quick since it is stored in the overlay.
+st = time()
+buf2 = h.pread(65536, 0)
+et = time()
+el = et-st
+print("elapsed time: %g" % el)
+assert el < 10
+assert buf == buf2
+'
diff --git a/tests/test-cow-on-read.sh b/tests/test-cow-on-read.sh
new file mode 100755
index 00000000..4f58b33b
--- /dev/null
+++ b/tests/test-cow-on-read.sh
@@ -0,0 +1,59 @@
+#!/usr/bin/env bash
+# nbdkit
+# Copyright (C) 2018-2021 Red Hat Inc.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#
+# * Redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in the
+# documentation and/or other materials provided with the distribution.
+#
+# * Neither the name of Red Hat nor the names of its contributors may be
+# used to endorse or promote products derived from this software without
+# specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY RED HAT AND CONTRIBUTORS ''AS IS'' AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+# PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL RED HAT OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
+# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+# SUCH DAMAGE.
+
+source ./functions.sh
+set -e
+set -x
+
+requires_filter cow
+requires_nbdsh_uri
+
+sock=$(mktemp -u /tmp/nbdkit-test-sock.XXXXXX)
+files="$sock cow-on-read.pid"
+rm -f $files
+cleanup_fn rm -f $files
+
+# Run nbdkit with the cow filter and cow-on-read.
+start_nbdkit -P cow-on-read.pid -U $sock \
+ --filter=cow \
+ memory 128K cow-on-read=true
+
+nbdsh --connect "nbd+unix://?socket=$sock" \
+ -c '
+# Write some pattern data to the overlay and check it reads back OK.
+buf = b"abcd" * 16384
+h.pwrite(buf, 32768)
+zero = h.pread(32768, 0)
+assert zero == bytearray(32768)
+buf2 = h.pread(65536, 32768)
+assert buf == buf2
+'
--
2.31.1

147
SOURCES/0010-vddk-Move-minimum-version-to-VDDK-6.5.patch Normal file
View File

@ -0,0 +1,147 @@
From e34016cbba4340b25f9a52c98db918aa72b38a7c Mon Sep 17 00:00:00 2001
From: "Richard W.M. Jones" <rjones@redhat.com>
Date: Tue, 26 Oct 2021 19:46:32 +0100
Subject: [PATCH] vddk: Move minimum version to VDDK 6.5
Drop support for VDDK 5.5.5 (released in 2015) and 6.0 (released the
same year). Move minimum supported version to 6.5 (released Nov
2016). This is so we can use asynchronous operations.
Acked-by: Laszlo Ersek <lersek@redhat.com>
(cherry picked from commit 5ed23616762a72e039531a9a7cd81353cd4f436e)
---
plugins/vddk/nbdkit-vddk-plugin.pod | 10 +++-------
plugins/vddk/vddk-stubs.h | 3 +--
plugins/vddk/vddk.c | 24 ++++++++++++++++--------
tests/dummy-vddk.c | 6 ++++++
4 files changed, 26 insertions(+), 17 deletions(-)
diff --git a/plugins/vddk/nbdkit-vddk-plugin.pod b/plugins/vddk/nbdkit-vddk-plugin.pod
index 0702aa75..1c16d096 100644
--- a/plugins/vddk/nbdkit-vddk-plugin.pod
+++ b/plugins/vddk/nbdkit-vddk-plugin.pod
@@ -424,7 +424,7 @@ by this build.
=item C<vddk_library_version=...>
-The VDDK major library version: 5, 6, 7, ...
+The VDDK major library version: 6, 7, ...
If this is omitted it means the library could not be loaded.
=item C<vddk_dll=...>
@@ -548,16 +548,12 @@ server, which can also be very slow.
=head1 SUPPORTED VERSIONS OF VDDK
-This plugin requires VDDK E<ge> 5.5.5, which in turn means that it
-is only supported on x64-64 platforms.
+This plugin requires VDDK E<ge> 6.5 (released Nov 2016). It is only
+supported on the x64-64 archtecture.
It has been tested with all versions up to 7.0.3 (but should work with
future versions).
-VDDK E<ge> 6.0 should be used if possible. This is the first version
-which added Flush support which is crucial for data integrity when
-writing.
-
VDDK 6.7 was the first version that supported the
C<VixDiskLib_QueryAllocatedBlocks> API, required to provide extent
information over NBD.
diff --git a/plugins/vddk/vddk-stubs.h b/plugins/vddk/vddk-stubs.h
index 5e70238d..a94df9cd 100644
--- a/plugins/vddk/vddk-stubs.h
+++ b/plugins/vddk/vddk-stubs.h
@@ -40,8 +40,7 @@
*/
/* Required stubs, present in all versions of VDDK that we support. I
- * have checked that all these exist in at least VDDK 5.5.5 (2015)
- * which is the earliest version of VDDK that we support.
+ * have checked that all these exist in at least VDDK 5.5.5 (2015).
*/
STUB (VixDiskLib_InitEx,
diff --git a/plugins/vddk/vddk.c b/plugins/vddk/vddk.c
index 096b04bf..babffc28 100644
--- a/plugins/vddk/vddk.c
+++ b/plugins/vddk/vddk.c
@@ -75,13 +75,13 @@ NBDKIT_DLL_PUBLIC int vddk_debug_stats;
#undef OPTIONAL_STUB
/* Parameters passed to InitEx. */
-#define VDDK_MAJOR 5
+#define VDDK_MAJOR 6
#define VDDK_MINOR 5
static void *dl; /* dlopen handle */
static bool init_called; /* was InitEx called */
static __thread int error_suppression; /* threadlocal error suppression */
-static int library_version; /* VDDK major: 5, 6, 7, ... */
+static int library_version; /* VDDK major: 6, 7, ... */
static enum { NONE = 0, ZLIB, FASTLZ, SKIPZ } compression; /* compression */
static char *config; /* config */
@@ -413,16 +413,14 @@ load_library (bool load_error_is_fatal)
/* Prefer the newest library in case multiple exist. Check two
* possible directories: the usual VDDK installation puts .so
* files in an arch-specific subdirectory of $libdir (our minimum
- * supported version is VDDK 5.5.5, which only supports x64-64);
- * but our testsuite is easier to write if we point libdir
- * directly to a stub .so.
+ * supported version is VDDK 6.5, which only supports x64-64); but
+ * our testsuite is easier to write if we point libdir directly to
+ * a stub .so.
*/
{ "lib64/libvixDiskLib.so.7", 7 },
{ "libvixDiskLib.so.7", 7 },
{ "lib64/libvixDiskLib.so.6", 6 },
{ "libvixDiskLib.so.6", 6 },
- { "lib64/libvixDiskLib.so.5", 5 },
- { "libvixDiskLib.so.5", 5 },
{ NULL }
};
size_t i;
@@ -474,7 +472,7 @@ load_library (bool load_error_is_fatal)
exit (EXIT_FAILURE);
}
- assert (library_version >= 5);
+ assert (library_version >= 6);
/* Load symbols. */
#define STUB(fn,ret,args) \
@@ -490,6 +488,16 @@ load_library (bool load_error_is_fatal)
#include "vddk-stubs.h"
#undef STUB
#undef OPTIONAL_STUB
+
+ /* Additionally, VDDK version must be >= 6.5. This was the first
+ * version which introduced VixDiskLib_Wait symbol so we can check
+ * for that.
+ */
+ if (VixDiskLib_Wait == NULL) {
+ nbdkit_error ("VDDK version must be >= 6.5. "
+ "See nbdkit-vddk-plugin(1) man page section \"SUPPORTED VERSIONS OF VDDK\".");
+ exit (EXIT_FAILURE);
+ }
}
static int
diff --git a/tests/dummy-vddk.c b/tests/dummy-vddk.c
index 9b5ae0a2..cb88380c 100644
--- a/tests/dummy-vddk.c
+++ b/tests/dummy-vddk.c
@@ -198,3 +198,9 @@ VixDiskLib_Write (VixDiskLibHandle handle,
memcpy (disk + offset, buf, nr_sectors * VIXDISKLIB_SECTOR_SIZE);
return VIX_OK;
}
+
+NBDKIT_DLL_PUBLIC VixError
+VixDiskLib_Wait (VixDiskLibHandle handle)
+{
+ return VIX_OK;
+}
--
2.31.1

170
SOURCES/0011-delay-Add-delay-open-and-delay-close.patch
View File

@ -1,170 +0,0 @@
From a7e7af18d64164fac42581452f6dc3c07650fcae Mon Sep 17 00:00:00 2001
From: "Richard W.M. Jones" <rjones@redhat.com>
Date: Fri, 30 Jul 2021 10:19:57 +0100
Subject: [PATCH] delay: Add delay-open and delay-close
Useful for simulating VDDK which has very slow connection.
(cherry picked from commit de8dcd3a34a38b088a0f9a6f8ca754702ad1f598)
---
filters/delay/delay.c | 58 ++++++++++++++++++++++++++-
filters/delay/nbdkit-delay-filter.pod | 27 +++++++++++--
2 files changed, 80 insertions(+), 5 deletions(-)
diff --git a/filters/delay/delay.c b/filters/delay/delay.c
index 5a925aa4..df3729a7 100644
--- a/filters/delay/delay.c
+++ b/filters/delay/delay.c
@@ -48,6 +48,8 @@ static unsigned delay_zero_ms = 0; /* zero delay (milliseconds) */
static unsigned delay_trim_ms = 0; /* trim delay (milliseconds) */
static unsigned delay_extents_ms = 0;/* extents delay (milliseconds) */
static unsigned delay_cache_ms = 0; /* cache delay (milliseconds) */
+static unsigned delay_open_ms = 0; /* open delay (milliseconds) */
+static unsigned delay_close_ms = 0; /* close delay (milliseconds) */
static int delay_fast_zero = 1; /* whether delaying zero includes fast zero */
@@ -126,6 +128,18 @@ cache_delay (int *err)
return delay (delay_cache_ms, err);
}
+static int
+open_delay (int *err)
+{
+ return delay (delay_open_ms, err);
+}
+
+static int
+close_delay (int *err)
+{
+ return delay (delay_close_ms, err);
+}
+
/* Called for each key=value passed on the command line. */
static int
delay_config (nbdkit_next_config *next, nbdkit_backend *nxdata,
@@ -182,6 +196,16 @@ delay_config (nbdkit_next_config *next, nbdkit_backend *nxdata,
return -1;
return 0;
}
+ else if (strcmp (key, "delay-open") == 0) {
+ if (parse_delay (key, value, &delay_open_ms) == -1)
+ return -1;
+ return 0;
+ }
+ else if (strcmp (key, "delay-close") == 0) {
+ if (parse_delay (key, value, &delay_close_ms) == -1)
+ return -1;
+ return 0;
+ }
else
return next (nxdata, key, value);
}
@@ -195,7 +219,9 @@ delay_config (nbdkit_next_config *next, nbdkit_backend *nxdata,
"delay-extents=<NN>[ms] Extents delay in seconds/milliseconds.\n" \
"delay-cache=<NN>[ms] Cache delay in seconds/milliseconds.\n" \
"wdelay=<NN>[ms] Write, zero and trim delay in secs/msecs.\n" \
- "delay-fast-zero=<BOOL> Delay fast zero requests (default true).\n"
+ "delay-fast-zero=<BOOL> Delay fast zero requests (default true).\n" \
+ "delay-open=<NN>[ms] Open delay in seconds/milliseconds.\n" \
+ "delay-close=<NN>[ms] Close delay in seconds/milliseconds."
/* Override the plugin's .can_fast_zero if needed */
static int
@@ -208,6 +234,34 @@ delay_can_fast_zero (nbdkit_next *next,
return next->can_fast_zero (next);
}
+/* Open connection. */
+static void *
+delay_open (nbdkit_next_open *next, nbdkit_context *nxdata,
+ int readonly, const char *exportname, int is_tls)
+{
+ int err;
+
+ if (open_delay (&err) == -1) {
+ errno = err;
+ nbdkit_error ("delay: %m");
+ return NULL;
+ }
+
+ if (next (nxdata, readonly, exportname) == -1)
+ return NULL;
+
+ return NBDKIT_HANDLE_NOT_NEEDED;
+}
+
+/* Close connection. */
+static void
+delay_close (void *handle)
+{
+ int err;
+
+ close_delay (&err);
+}
+
/* Read data. */
static int
delay_pread (nbdkit_next *next,
@@ -285,6 +339,8 @@ static struct nbdkit_filter filter = {
.config = delay_config,
.config_help = delay_config_help,
.can_fast_zero = delay_can_fast_zero,
+ .open = delay_open,
+ .close = delay_close,
.pread = delay_pread,
.pwrite = delay_pwrite,
.zero = delay_zero,
diff --git a/filters/delay/nbdkit-delay-filter.pod b/filters/delay/nbdkit-delay-filter.pod
index d6961a9e..11ae544b 100644
--- a/filters/delay/nbdkit-delay-filter.pod
+++ b/filters/delay/nbdkit-delay-filter.pod
@@ -9,10 +9,15 @@ nbdkit-delay-filter - nbdkit delay filter
nbdkit --filter=delay plugin rdelay=NNms wdelay=NNms [plugin-args...]
nbdkit --filter=delay plugin [plugin-args ...]
- delay-read=(SECS|NNms) delay-write=(SECS|NNms)
- delay-zero=(SECS|NNms) delay-trim=(SECS|NNms)
- delay-extents=(SECS|NNms) delay-cache=(SECS|NNms)
+ delay-read=(SECS|NNms)
+ delay-write=(SECS|NNms)
+ delay-zero=(SECS|NNms)
+ delay-trim=(SECS|NNms)
+ delay-extents=(SECS|NNms)
+ delay-cache=(SECS|NNms)
delay-fast-zero=BOOL
+ delay-open=(SECS|NNms)
+ delay-close=(SECS|NNms)
=head1 DESCRIPTION
@@ -108,6 +113,20 @@ delay as any other zero request; but setting this parameter to false
instantly fails a fast zero response without waiting for or consulting
the plugin.
+=item B<delay-open=>SECS
+
+=item B<delay-open=>NNB<ms>
+
+=item B<delay-close=>SECS
+
+=item B<delay-close=>NNB<ms>
+
+(nbdkit E<ge> 1.28)
+
+Delay open and close operations by C<SECS> seconds or C<NN>
+milliseconds. Open corresponds to client connection. Close may not
+be visible to clients if they abruptly disconnect.
+
=back
=head1 FILES
@@ -140,4 +159,4 @@ Richard W.M. Jones
=head1 COPYRIGHT
-Copyright (C) 2018 Red Hat Inc.
+Copyright (C) 2018-2021 Red Hat Inc.
--
2.31.1

72
SOURCES/0011-vddk-Add-read-write-and-wait-asynchronous-functions.patch Normal file
View File

@ -0,0 +1,72 @@
From 69b989b37c8e33f52d928c7202146e9e11a2a93c Mon Sep 17 00:00:00 2001
From: "Richard W.M. Jones" <rjones@redhat.com>
Date: Wed, 27 Oct 2021 11:57:35 +0100
Subject: [PATCH] vddk: Add read, write and wait asynchronous functions
These functions added in VDDK 6.0 - 6.5 implement asynchronous read
and write.
Acked-by: Laszlo Ersek <lersek@redhat.com>
(cherry picked from commit ad53e7becafed6ca3573795a79c534281fe9c274)
---
plugins/vddk/vddk-structs.h | 3 +++
plugins/vddk/vddk-stubs.h | 19 ++++++++++++++++++-
2 files changed, 21 insertions(+), 1 deletion(-)
diff --git a/plugins/vddk/vddk-structs.h b/plugins/vddk/vddk-structs.h
index aeb5bfd0..e97f017c 100644
--- a/plugins/vddk/vddk-structs.h
+++ b/plugins/vddk/vddk-structs.h
@@ -43,6 +43,7 @@
typedef uint64_t VixError;
#define VIX_OK 0
+#define VIX_ASYNC 25000
#define VIXDISKLIB_FLAG_OPEN_UNBUFFERED 1
#define VIXDISKLIB_FLAG_OPEN_SINGLE_LINK 2
@@ -61,6 +62,8 @@ typedef void *VixDiskLibHandle;
typedef void VixDiskLibGenericLogFunc (const char *fmt, va_list args);
+typedef void (*VixDiskLibCompletionCB) (void *data, VixError result);
+
enum VixDiskLibCredType {
VIXDISKLIB_CRED_UID = 1,
VIXDISKLIB_CRED_SESSIONID = 2,
diff --git a/plugins/vddk/vddk-stubs.h b/plugins/vddk/vddk-stubs.h
index a94df9cd..66353691 100644
--- a/plugins/vddk/vddk-stubs.h
+++ b/plugins/vddk/vddk-stubs.h
@@ -103,10 +103,27 @@ STUB (VixDiskLib_Write,
uint64_t start_sector, uint64_t nr_sectors,
const unsigned char *buf));
-/* Added in VDDK 6.0, this will be NULL in earlier versions. */
+/* Added in VDDK 6.0, these will be NULL in earlier versions. */
OPTIONAL_STUB (VixDiskLib_Flush,
VixError,
(VixDiskLibHandle handle));
+OPTIONAL_STUB (VixDiskLib_ReadAsync,
+ VixError,
+ (VixDiskLibHandle handle,
+ uint64_t start_sector, uint64_t nr_sectors,
+ unsigned char *buf,
+ VixDiskLibCompletionCB callback, void *data));
+OPTIONAL_STUB (VixDiskLib_WriteAsync,
+ VixError,
+ (VixDiskLibHandle handle,
+ uint64_t start_sector, uint64_t nr_sectors,
+ const unsigned char *buf,
+ VixDiskLibCompletionCB callback, void *data));
+
+/* Added in VDDK 6.5, this will be NULL in earlier versions. */
+OPTIONAL_STUB (VixDiskLib_Wait,
+ VixError,
+ (VixDiskLibHandle handle));
/* Added in VDDK 6.7, these will be NULL for earlier versions: */
OPTIONAL_STUB (VixDiskLib_QueryAllocatedBlocks,
--
2.31.1

90
SOURCES/0012-python-Implement-.cleanup-method.patch
View File

@ -1,90 +0,0 @@
From 17a912a449fa75b5c12ac3acab596b476699c671 Mon Sep 17 00:00:00 2001
From: "Richard W.M. Jones" <rjones@redhat.com>
Date: Tue, 3 Aug 2021 14:19:38 +0100
Subject: [PATCH] python: Implement .cleanup() method
(cherry picked from commit f2fe99e4b0f54467ab8028eaf2d039cf918b2961)
---
plugins/python/nbdkit-python-plugin.pod | 20 +++++++++++++++++---
plugins/python/plugin.c | 19 +++++++++++++++++++
2 files changed, 36 insertions(+), 3 deletions(-)
diff --git a/plugins/python/nbdkit-python-plugin.pod b/plugins/python/nbdkit-python-plugin.pod
index 6f5f2c00..a92a557f 100644
--- a/plugins/python/nbdkit-python-plugin.pod
+++ b/plugins/python/nbdkit-python-plugin.pod
@@ -257,6 +257,12 @@ There are no arguments or return value.
There are no arguments or return value.
+=item C<cleanup>
+
+(Optional, nbdkit E<ge> 1.28)
+
+There are no arguments or return value.
+
=item C<list_exports>
(Optional)
@@ -498,10 +504,18 @@ optionally using C<nbdkit.set_error> first.
=over 4
-=item Missing: C<load> and C<unload>
+=item Missing: C<load>
-These are not needed because you can just use ordinary Python
-constructs.
+This is not needed since you can use regular Python mechanisms like
+top level statements to run code when the module is loaded.
+
+=item Missing: C<unload>
+
+This is missing, but in nbdkit E<ge> 1.28 you can put code in the
+C<cleanup()> function to have it run when nbdkit exits. In earlier
+versions of nbdkit, using a Python
+L<atexit|https://docs.python.org/3/library/atexit.html> handler is
+recommended.
=item Missing:
C<name>,
diff --git a/plugins/python/plugin.c b/plugins/python/plugin.c
index 64430a1a..f85512b4 100644
--- a/plugins/python/plugin.c
+++ b/plugins/python/plugin.c
@@ -298,6 +298,24 @@ py_after_fork (void)
return 0;
}
+static void
+py_cleanup (void)
+{
+ ACQUIRE_PYTHON_GIL_FOR_CURRENT_SCOPE;
+ PyObject *fn;
+ PyObject *r;
+
+ if (callback_defined ("cleanup", &fn)) {
+ PyErr_Clear ();
+
+ r = PyObject_CallObject (fn, NULL);
+ Py_DECREF (fn);
+ if (check_python_failure ("cleanup") == -1)
+ return;
+ Py_DECREF (r);
+ }
+}
+
static int
py_list_exports (int readonly, int is_tls, struct nbdkit_exports *exports)
{
@@ -1039,6 +1057,7 @@ static struct nbdkit_plugin plugin = {
.thread_model = py_thread_model,
.get_ready = py_get_ready,
.after_fork = py_after_fork,
+ .cleanup = py_cleanup,
.list_exports = py_list_exports,
.default_export = py_default_export,
--
2.31.1

259
SOURCES/0012-vddk-Start-to-split-VDDK-over-several-files.patch Normal file
View File

@ -0,0 +1,259 @@
From 98a499c0e9d08f208474759012ec3ed823ce2335 Mon Sep 17 00:00:00 2001
From: "Richard W.M. Jones" <rjones@redhat.com>
Date: Wed, 27 Oct 2021 12:20:31 +0100
Subject: [PATCH] vddk: Start to split VDDK over several files
This change doesn't do anything except move some definitions into the
header file vddk.h, but it allows future commits to split up the very
large vddk.c file.
Acked-by: Laszlo Ersek <lersek@redhat.com>
(cherry picked from commit 117634dccf4e29394e8718a8d62e93a9edf0a39c)
---
plugins/vddk/vddk.c | 91 +++++++++++++--------------------------------
plugins/vddk/vddk.h | 89 +++++++++++++++++++++++++++++++++++++++++++-
2 files changed, 112 insertions(+), 68 deletions(-)
diff --git a/plugins/vddk/vddk.c b/plugins/vddk/vddk.c
index babffc28..041bff1a 100644
--- a/plugins/vddk/vddk.c
+++ b/plugins/vddk/vddk.c
@@ -50,14 +50,12 @@
#include <nbdkit-plugin.h>
#include "cleanup.h"
-#include "isaligned.h"
#include "minmax.h"
#include "rounding.h"
#include "tvdiff.h"
#include "vector.h"
#include "vddk.h"
-#include "vddk-structs.h"
/* Debug flags. */
NBDKIT_DLL_PUBLIC int vddk_debug_diskinfo;
@@ -65,11 +63,11 @@ NBDKIT_DLL_PUBLIC int vddk_debug_extents;
NBDKIT_DLL_PUBLIC int vddk_debug_datapath = 1;
NBDKIT_DLL_PUBLIC int vddk_debug_stats;
-/* For each VDDK API define a static global variable. These globals
- * are initialized when the plugin is loaded (by vddk_get_ready).
+/* For each VDDK API define a global variable. These globals are
+ * initialized when the plugin is loaded (by vddk_get_ready).
*/
-#define STUB(fn,ret,args) static ret (*fn) args
-#define OPTIONAL_STUB(fn,ret,args) static ret (*fn) args
+#define STUB(fn,ret,args) ret (*fn) args
+#define OPTIONAL_STUB(fn,ret,args) ret (*fn) args
#include "vddk-stubs.h"
#undef STUB
#undef OPTIONAL_STUB
@@ -78,28 +76,28 @@ NBDKIT_DLL_PUBLIC int vddk_debug_stats;
#define VDDK_MAJOR 6
#define VDDK_MINOR 5
-static void *dl; /* dlopen handle */
-static bool init_called; /* was InitEx called */
-static __thread int error_suppression; /* threadlocal error suppression */
-static int library_version; /* VDDK major: 6, 7, ... */
+void *dl; /* dlopen handle */
+bool init_called; /* was InitEx called */
+__thread int error_suppression; /* threadlocal error suppression */
+int library_version; /* VDDK major: 6, 7, ... */
+bool is_remote; /* true if remote connection */
-static enum { NONE = 0, ZLIB, FASTLZ, SKIPZ } compression; /* compression */
-static char *config; /* config */
-static const char *cookie; /* cookie */
-static const char *filename; /* file */
-char *libdir; /* libdir */
-static uint16_t nfc_host_port; /* nfchostport */
-char *password; /* password */
-static uint16_t port; /* port */
-static const char *server_name; /* server */
-static bool single_link; /* single-link */
-static const char *snapshot_moref; /* snapshot */
-static const char *thumb_print; /* thumbprint */
-static const char *transport_modes; /* transports */
-static bool unbuffered; /* unbuffered */
-static const char *username; /* user */
-static const char *vmx_spec; /* vm */
-static bool is_remote;
+enum compression_type compression; /* compression */
+char *config; /* config */
+const char *cookie; /* cookie */
+const char *filename; /* file */
+char *libdir; /* libdir */
+uint16_t nfc_host_port; /* nfchostport */
+char *password; /* password */
+uint16_t port; /* port */
+const char *server_name; /* server */
+bool single_link; /* single-link */
+const char *snapshot_moref; /* snapshot */
+const char *thumb_print; /* thumbprint */
+const char *transport_modes; /* transports */
+bool unbuffered; /* unbuffered */
+const char *username; /* user */
+const char *vmx_spec; /* vm */
/* For each VDDK API define a variable to store the time taken (used
* to implement -D vddk.stats=1).
@@ -120,45 +118,6 @@ static void display_stats (void);
#undef STUB
#undef OPTIONAL_STUB
-/* Macros to bracket each VDDK API call, for printing debugging
- * information and collecting statistics.
- */
-#define VDDK_CALL_START(fn, fs, ...) \
- do { \
- struct timeval start_t, end_t; \
- /* GCC can optimize this away at compile time: */ \
- const bool datapath = \
- strcmp (#fn, "VixDiskLib_Read") == 0 || \
- strcmp (#fn, "VixDiskLib_Write") == 0; \
- if (vddk_debug_stats) \
- gettimeofday (&start_t, NULL); \
- if (!datapath || vddk_debug_datapath) \
- nbdkit_debug ("VDDK call: %s (" fs ")", #fn, ##__VA_ARGS__); \
- do
-#define VDDK_CALL_END(fn, bytes_) \
- while (0); \
- if (vddk_debug_stats) { \
- gettimeofday (&end_t, NULL); \
- ACQUIRE_LOCK_FOR_CURRENT_SCOPE (&stats_lock); \
- stats_##fn.usecs += tvdiff_usec (&start_t, &end_t); \
- stats_##fn.calls++; \
- stats_##fn.bytes += bytes_; \
- } \
- } while (0)
-
-/* Print VDDK errors. */
-#define VDDK_ERROR(err, fs, ...) \
- do { \
- char *vddk_err_msg; \
- VDDK_CALL_START (VixDiskLib_GetErrorText, "%lu", err) \
- vddk_err_msg = VixDiskLib_GetErrorText ((err), NULL); \
- VDDK_CALL_END (VixDiskLib_GetErrorText, 0); \
- nbdkit_error (fs ": %s", ##__VA_ARGS__, vddk_err_msg); \
- VDDK_CALL_START (VixDiskLib_FreeErrorText, "") \
- VixDiskLib_FreeErrorText (vddk_err_msg); \
- VDDK_CALL_END (VixDiskLib_FreeErrorText, 0); \
- } while (0)
-
/* Unload the plugin. */
static void
vddk_unload (void)
diff --git a/plugins/vddk/vddk.h b/plugins/vddk/vddk.h
index 8c63b4ee..29775eb4 100644
--- a/plugins/vddk/vddk.h
+++ b/plugins/vddk/vddk.h
@@ -1,5 +1,5 @@
/* nbdkit
- * Copyright (C) 2013-2020 Red Hat Inc.
+ * Copyright (C) 2013-2021 Red Hat Inc.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are
@@ -33,11 +33,96 @@
#ifndef NBDKIT_VDDK_H
#define NBDKIT_VDDK_H
+#include <stdbool.h>
+#include <stdint.h>
+#include <sys/time.h>
+
+#include <pthread.h>
+
+#include "isaligned.h"
+#include "tvdiff.h"
+#include "vector.h"
+
+#include "vddk-structs.h"
+
+enum compression_type { NONE = 0, ZLIB, FASTLZ, SKIPZ };
+
+extern void *dl;
+extern bool init_called;
+extern __thread int error_suppression;
+extern int library_version;
+extern bool is_remote;
+
+extern enum compression_type compression;
+extern char *config;
+extern const char *cookie;
+extern const char *filename;
extern char *libdir;
+extern uint16_t nfc_host_port;
extern char *password;
+extern uint16_t port;
+extern const char *server_name;
+extern bool single_link;
+extern const char *snapshot_moref;
+extern const char *thumb_print;
+extern const char *transport_modes;
+extern bool unbuffered;
+extern const char *username;
+extern const char *vmx_spec;
+
+extern int vddk_debug_diskinfo;
+extern int vddk_debug_extents;
+extern int vddk_debug_datapath;
+extern int vddk_debug_stats;
+
+#define STUB(fn,ret,args) extern ret (*fn) args
+#define OPTIONAL_STUB(fn,ret,args) extern ret (*fn) args
+#include "vddk-stubs.h"
+#undef STUB
+#undef OPTIONAL_STUB
+
+/* Macros to bracket each VDDK API call, for printing debugging
+ * information and collecting statistics.
+ */
+#define VDDK_CALL_START(fn, fs, ...) \
+ do { \
+ struct timeval start_t, end_t; \
+ /* GCC can optimize this away at compile time: */ \
+ const bool datapath = \
+ strcmp (#fn, "VixDiskLib_Read") == 0 || \
+ strcmp (#fn, "VixDiskLib_Write") == 0; \
+ if (vddk_debug_stats) \
+ gettimeofday (&start_t, NULL); \
+ if (!datapath || vddk_debug_datapath) \
+ nbdkit_debug ("VDDK call: %s (" fs ")", #fn, ##__VA_ARGS__); \
+ do
+#define VDDK_CALL_END(fn, bytes_) \
+ while (0); \
+ if (vddk_debug_stats) { \
+ gettimeofday (&end_t, NULL); \
+ ACQUIRE_LOCK_FOR_CURRENT_SCOPE (&stats_lock); \
+ stats_##fn.usecs += tvdiff_usec (&start_t, &end_t); \
+ stats_##fn.calls++; \
+ stats_##fn.bytes += bytes_; \
+ } \
+ } while (0)
+
+/* Print VDDK errors. */
+#define VDDK_ERROR(err, fs, ...) \
+ do { \
+ char *vddk_err_msg; \
+ VDDK_CALL_START (VixDiskLib_GetErrorText, "%lu", err) \
+ vddk_err_msg = VixDiskLib_GetErrorText ((err), NULL); \
+ VDDK_CALL_END (VixDiskLib_GetErrorText, 0); \
+ nbdkit_error (fs ": %s", ##__VA_ARGS__, vddk_err_msg); \
+ VDDK_CALL_START (VixDiskLib_FreeErrorText, "") \
+ VixDiskLib_FreeErrorText (vddk_err_msg); \
+ VDDK_CALL_END (VixDiskLib_FreeErrorText, 0); \
+ } while (0)
+
+/* reexec.c */
extern bool noreexec;
extern char *reexeced;
-
extern void reexec_if_needed (const char *prepend);
extern int restore_ld_library_path (void);
--
2.31.1

150
SOURCES/0013-cow-General-revision-and-updates-to-the-manual.patch
View File

@ -1,150 +0,0 @@
From e9abe97b40fef6f9bd9028a2520f45203bba0749 Mon Sep 17 00:00:00 2001
From: "Richard W.M. Jones" <rjones@redhat.com>
Date: Thu, 5 Aug 2021 18:18:34 +0100
Subject: [PATCH] cow: General revision and updates to the manual
(cherry picked from commit ba5517b81307c228577cf3c54a651d044ac91a25)
---
filters/cow/nbdkit-cow-filter.pod | 74 ++++++++++++++++---------------
1 file changed, 39 insertions(+), 35 deletions(-)
diff --git a/filters/cow/nbdkit-cow-filter.pod b/filters/cow/nbdkit-cow-filter.pod
index 01261429..7f861140 100644
--- a/filters/cow/nbdkit-cow-filter.pod
+++ b/filters/cow/nbdkit-cow-filter.pod
@@ -5,33 +5,23 @@ nbdkit-cow-filter - nbdkit copy-on-write (COW) filter
=head1 SYNOPSIS
nbdkit --filter=cow plugin [plugin-args...]
+ [cow-on-cache=false|true]
+ [cow-on-read=false|true|/PATH]
=head1 DESCRIPTION
C<nbdkit-cow-filter> is a filter that makes a temporary writable copy
-on top of a read-only plugin. It can be used to enable writes for
-plugins which only implement read-only access. Note that:
+on top of a plugin. It can also be used to enable writes for plugins
+which are read-only.
-=over 4
+The underlying plugin is opened read-only. This filter does not pass
+any writes or write-like operations (like trim and zero) through to
+the underlying plugin.
-=item *
-
-B<Anything written is thrown away as soon as nbdkit exits.>
-
-=item *
-
-All connections to the nbdkit instance see the same view of the disk.
-
-This is different from L<nbd-server(1)> where each connection sees its
-own copy-on-write overlay and simply disconnecting the client throws
-that away. It also allows us to create diffs, see below.
-
-=item *
-
-The plugin is opened read-only (as if the I<-r> flag was passed), but
-you should B<not> pass the I<-r> flag to nbdkit.
-
-=back
+B<Note that anything written is thrown away as soon as nbdkit exits.>
+If you want to save changes, either copy out the whole disk using a
+tool like L<nbdcopy(1)>, or use the method described in L</NOTES>
+below to create a diff.
Limitations of the filter include:
@@ -52,26 +42,26 @@ serve the same data to each client.
=over 4
-=item B<cow-on-cache=true>
-
-When the client issues a cache (prefetch) request, preemptively save
-the data from the plugin into the overlay.
-
=item B<cow-on-cache=false>
Do not save data from cache (prefetch) requests in the overlay. This
leaves the overlay as small as possible. This is the default.
-=item B<cow-on-read=true>
+=item B<cow-on-cache=true>
-When the client issues a read request, copy the data into the overlay
-so that the same data can be served more quickly later.
+When the client issues a cache (prefetch) request, preemptively save
+the data from the plugin into the overlay.
=item B<cow-on-read=false>
Do not save data from read requests in the overlay. This leaves the
overlay as small as possible. This is the default.
+=item B<cow-on-read=true>
+
+When the client issues a read request, copy the data into the overlay
+so that the same data can be served more quickly later.
+
=item B<cow-on-read=/PATH>
When F</PATH> (which must be an absolute path) exists, this behaves
@@ -83,18 +73,23 @@ behaviour while nbdkit is running.
=head1 EXAMPLES
+=head2 nbdkit --filter=cow file disk.img
+
Serve the file F<disk.img>, allowing writes, but do not save any
-changes into the file:
+changes into the file.
- nbdkit --filter=cow file disk.img
+=head2 nbdkit --filter=cow --filter=xz file disk.xz cow-on-read=true
L<nbdkit-xz-filter(1)> only supports read access, but you can provide
-temporary write access by doing (although this does B<not> save
-changes to the file):
+temporary write access by using the command above. Because xz
+decompression is slow, using C<cow-on-read=true> causes reads to be
+cached as well as writes, improving performance at the expense of
+using more temporary space. Note that writes are thrown away when
+nbdkit exits and do not get saved into the file.
- nbdkit --filter=cow --filter=xz file disk.xz
+=head1 NOTES
-=head1 CREATING A DIFF WITH QEMU-IMG
+=head2 Creating a diff with qemu-img
Although nbdkit-cow-filter itself cannot save the differences, it is
possible to do this using an obscure feature of L<qemu-img(1)>.
@@ -118,6 +113,14 @@ F<diff.qcow2> now contains the differences between the base
(F<disk.img>) and the changes stored in nbdkit-cow-filter. C<nbdkit>
can now be killed.
+=head2 Compared to nbd-server -c option
+
+All connections to the nbdkit instance see the same view of the disk.
+This is different from L<nbd-server(1)> I<-c> option where each
+connection sees its own copy-on-write overlay and simply disconnecting
+the client throws that away. It also allows us to create diffs as
+above.
+
=head1 ENVIRONMENT VARIABLES
=over 4
@@ -154,6 +157,7 @@ L<nbdkit-cache-filter(1)>,
L<nbdkit-cacheextents-filter(1)>,
L<nbdkit-xz-filter(1)>,
L<nbdkit-filter(3)>,
+L<nbdcopy(1)>,
L<qemu-img(1)>.
=head1 AUTHORS
--
2.31.1

287
SOURCES/0013-vddk-Refactor-D-vddk.stats-1-into-a-new-file.patch Normal file
View File

@ -0,0 +1,287 @@
From d602150dbb5ebacea42c25a0f6c8c26c45766a49 Mon Sep 17 00:00:00 2001
From: "Richard W.M. Jones" <rjones@redhat.com>
Date: Wed, 27 Oct 2021 12:30:41 +0100
Subject: [PATCH] vddk: Refactor -D vddk.stats=1 into a new file
Acked-by: Laszlo Ersek <lersek@redhat.com>
(cherry picked from commit dcd5bc51ed7710c32d956345ea8da14ba15ef8f5)
---
plugins/vddk/Makefile.am | 1 +
plugins/vddk/stats.c | 118 +++++++++++++++++++++++++++++++++++++++
plugins/vddk/vddk.c | 78 +-------------------------
plugins/vddk/vddk.h | 15 +++++
4 files changed, 135 insertions(+), 77 deletions(-)
create mode 100644 plugins/vddk/stats.c
diff --git a/plugins/vddk/Makefile.am b/plugins/vddk/Makefile.am
index 232aaedd..4f470ff9 100644
--- a/plugins/vddk/Makefile.am
+++ b/plugins/vddk/Makefile.am
@@ -46,6 +46,7 @@ nbdkit_vddk_plugin_la_SOURCES = \
vddk.c \
vddk.h \
reexec.c \
+ stats.c \
vddk-structs.h \
vddk-stubs.h \
$(top_srcdir)/include/nbdkit-plugin.h \
diff --git a/plugins/vddk/stats.c b/plugins/vddk/stats.c
new file mode 100644
index 00000000..18a42714
--- /dev/null
+++ b/plugins/vddk/stats.c
@@ -0,0 +1,118 @@
+/* nbdkit
+ * Copyright (C) 2013-2021 Red Hat Inc.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * * Neither the name of Red Hat nor the names of its contributors may be
+ * used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY RED HAT AND CONTRIBUTORS ''AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+ * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL RED HAT OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
+ * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+ * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <config.h>
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <stdint.h>
+#include <inttypes.h>
+
+#include <pthread.h>
+
+#define NBDKIT_API_VERSION 2
+#include <nbdkit-plugin.h>
+
+#include "vector.h"
+
+#include "vddk.h"
+
+/* Debug flags. */
+NBDKIT_DLL_PUBLIC int vddk_debug_stats;
+
+pthread_mutex_t stats_lock = PTHREAD_MUTEX_INITIALIZER;
+
+/* For each VDDK API define a variable to store the time taken (used
+ * to implement -D vddk.stats=1).
+ */
+#define STUB(fn,ret,args) struct vddk_stat stats_##fn = { .name = #fn }
+#define OPTIONAL_STUB(fn,ret,args) STUB(fn,ret,args)
+#include "vddk-stubs.h"
+#undef STUB
+#undef OPTIONAL_STUB
+
+DEFINE_VECTOR_TYPE(statlist, struct vddk_stat)
+
+static int
+stat_compare (const void *vp1, const void *vp2)
+{
+ const struct vddk_stat *st1 = vp1;
+ const struct vddk_stat *st2 = vp2;
+
+ /* Note: sorts in reverse order of time spent in each API call. */
+ if (st1->usecs < st2->usecs) return 1;
+ else if (st1->usecs > st2->usecs) return -1;
+ else return 0;
+}
+
+static const char *
+api_name_without_prefix (const char *name)
+{
+ return strncmp (name, "VixDiskLib_", 11) == 0 ? name + 11 : name;
+}
+
+void
+display_stats (void)
+{
+ statlist stats = empty_vector;
+ size_t i;
+
+ if (!vddk_debug_stats) return;
+
+#define STUB(fn,ret,args) statlist_append (&stats, stats_##fn)
+#define OPTIONAL_STUB(fn,ret,args) statlist_append (&stats, stats_##fn)
+#include "vddk-stubs.h"
+#undef STUB
+#undef OPTIONAL_STUB
+
+ qsort (stats.ptr, stats.size, sizeof stats.ptr[0], stat_compare);
+
+ nbdkit_debug ("VDDK function stats (-D vddk.stats=1):");
+ nbdkit_debug ("%-24s %15s %5s %15s",
+ "VixDiskLib_...", "µs", "calls", "bytes");
+ for (i = 0; i < stats.size; ++i) {
+ if (stats.ptr[i].usecs) {
+ if (stats.ptr[i].bytes > 0)
+ nbdkit_debug (" %-22s %15" PRIi64 " %5" PRIu64 " %15" PRIu64,
+ api_name_without_prefix (stats.ptr[i].name),
+ stats.ptr[i].usecs,
+ stats.ptr[i].calls,
+ stats.ptr[i].bytes);
+ else
+ nbdkit_debug (" %-22s %15" PRIi64 " %5" PRIu64,
+ api_name_without_prefix (stats.ptr[i].name),
+ stats.ptr[i].usecs,
+ stats.ptr[i].calls);
+ }
+ }
+ statlist_reset (&stats);
+}
diff --git a/plugins/vddk/vddk.c b/plugins/vddk/vddk.c
index 041bff1a..67ac775c 100644
--- a/plugins/vddk/vddk.c
+++ b/plugins/vddk/vddk.c
@@ -61,7 +61,6 @@
NBDKIT_DLL_PUBLIC int vddk_debug_diskinfo;
NBDKIT_DLL_PUBLIC int vddk_debug_extents;
NBDKIT_DLL_PUBLIC int vddk_debug_datapath = 1;
-NBDKIT_DLL_PUBLIC int vddk_debug_stats;
/* For each VDDK API define a global variable. These globals are
* initialized when the plugin is loaded (by vddk_get_ready).
@@ -99,25 +98,6 @@ bool unbuffered; /* unbuffered */
const char *username; /* user */
const char *vmx_spec; /* vm */
-/* For each VDDK API define a variable to store the time taken (used
- * to implement -D vddk.stats=1).
- */
-struct vddk_stat {
- const char *name; /* function name */
- int64_t usecs; /* total number of usecs consumed */
- uint64_t calls; /* number of times called */
- uint64_t bytes; /* bytes transferred, datapath calls only */
-};
-static pthread_mutex_t stats_lock = PTHREAD_MUTEX_INITIALIZER;
-static void display_stats (void);
-#define STUB(fn,ret,args) \
- static struct vddk_stat stats_##fn = { .name = #fn }
-#define OPTIONAL_STUB(fn,ret,args) \
- static struct vddk_stat stats_##fn = { .name = #fn }
-#include "vddk-stubs.h"
-#undef STUB
-#undef OPTIONAL_STUB
-
/* Unload the plugin. */
static void
vddk_unload (void)
@@ -130,69 +110,13 @@ vddk_unload (void)
if (dl)
dlclose (dl);
- if (vddk_debug_stats)
- display_stats ();
+ display_stats ();
free (config);
free (libdir);
free (password);
}
-DEFINE_VECTOR_TYPE(statlist, struct vddk_stat)
-
-static int
-stat_compare (const void *vp1, const void *vp2)
-{
- const struct vddk_stat *st1 = vp1;
- const struct vddk_stat *st2 = vp2;
-
- /* Note: sorts in reverse order of time spent in each API call. */
- if (st1->usecs < st2->usecs) return 1;
- else if (st1->usecs > st2->usecs) return -1;
- else return 0;
-}
-
-static const char *
-api_name_without_prefix (const char *name)
-{
- return strncmp (name, "VixDiskLib_", 11) == 0 ? name + 11 : name;
-}
-
-static void
-display_stats (void)
-{
- statlist stats = empty_vector;
- size_t i;
-
-#define STUB(fn,ret,args) statlist_append (&stats, stats_##fn)
-#define OPTIONAL_STUB(fn,ret,args) statlist_append (&stats, stats_##fn)
-#include "vddk-stubs.h"
-#undef STUB
-#undef OPTIONAL_STUB
-
- qsort (stats.ptr, stats.size, sizeof stats.ptr[0], stat_compare);
-
- nbdkit_debug ("VDDK function stats (-D vddk.stats=1):");
- nbdkit_debug ("%-24s %15s %5s %15s",
- "VixDiskLib_...", "µs", "calls", "bytes");
- for (i = 0; i < stats.size; ++i) {
- if (stats.ptr[i].usecs) {
- if (stats.ptr[i].bytes > 0)
- nbdkit_debug (" %-22s %15" PRIi64 " %5" PRIu64 " %15" PRIu64,
- api_name_without_prefix (stats.ptr[i].name),
- stats.ptr[i].usecs,
- stats.ptr[i].calls,
- stats.ptr[i].bytes);
- else
- nbdkit_debug (" %-22s %15" PRIi64 " %5" PRIu64,
- api_name_without_prefix (stats.ptr[i].name),
- stats.ptr[i].usecs,
- stats.ptr[i].calls);
- }
- }
- statlist_reset (&stats);
-}
-
static void
trim (char *str)
{
diff --git a/plugins/vddk/vddk.h b/plugins/vddk/vddk.h
index 29775eb4..1400589d 100644
--- a/plugins/vddk/vddk.h
+++ b/plugins/vddk/vddk.h
@@ -126,4 +126,19 @@ extern char *reexeced;
extern void reexec_if_needed (const char *prepend);
extern int restore_ld_library_path (void);
+/* stats.c */
+struct vddk_stat {
+ const char *name; /* function name */
+ int64_t usecs; /* total number of usecs consumed */
+ uint64_t calls; /* number of times called */
+ uint64_t bytes; /* bytes transferred, datapath calls only */
+};
+extern pthread_mutex_t stats_lock;
+#define STUB(fn,ret,args) extern struct vddk_stat stats_##fn;
+#define OPTIONAL_STUB(fn,ret,args) STUB(fn,ret,args)
+#include "vddk-stubs.h"
+#undef STUB
+#undef OPTIONAL_STUB
+extern void display_stats (void);
+
#endif /* NBDKIT_VDDK_H */
--
2.31.1

35
SOURCES/0014-cache-Move-plugin-args-in-synopsis-earlier.patch
View File

@ -1,35 +0,0 @@
From c8c1e74a8c1c112b83646ac09fe7f9bde097a52a Mon Sep 17 00:00:00 2001
From: "Richard W.M. Jones" <rjones@redhat.com>
Date: Thu, 5 Aug 2021 18:20:37 +0100
Subject: [PATCH] cache: Move plugin-args in synopsis earlier
Makes this page consistent with nbdkit-cow-filter.
(cherry picked from commit f1ddcef468907b0321041b1c4e0a430be46920be)
---
filters/cache/nbdkit-cache-filter.pod | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/filters/cache/nbdkit-cache-filter.pod b/filters/cache/nbdkit-cache-filter.pod
index df9c1f99..d85fef09 100644
--- a/filters/cache/nbdkit-cache-filter.pod
+++ b/filters/cache/nbdkit-cache-filter.pod
@@ -4,13 +4,13 @@ nbdkit-cache-filter - nbdkit caching filter
=head1 SYNOPSIS
- nbdkit --filter=cache plugin [cache=writeback|writethrough|unsafe]
+ nbdkit --filter=cache plugin [plugin-args...]
+ [cache=writeback|writethrough|unsafe]
[cache-min-block-size=SIZE]
[cache-max-size=SIZE]
[cache-high-threshold=N]
[cache-low-threshold=N]
[cache-on-read=true|false|/PATH]
- [plugin-args...]
=head1 DESCRIPTION
--
2.31.1

1287
SOURCES/0014-vddk-Implement-parallel-thread-model.patch Normal file
View File

File diff suppressed because it is too large Load Diff

53
SOURCES/0015-data-Improve-the-example-with-a-diagram.patch
View File

@ -1,53 +0,0 @@
From 0eae7ebf6f714fb339f4a476b65e070b528824ec Mon Sep 17 00:00:00 2001
From: "Richard W.M. Jones" <rjones@redhat.com>
Date: Sun, 8 Aug 2021 16:32:38 +0100
Subject: [PATCH] data: Improve the example with a diagram
And other improvements to readability.
(cherry picked from commit 4e3a9bda2b7a3d141234e26250c69baa6ed5194d)
---
plugins/data/nbdkit-data-plugin.pod | 24 +++++++++++++++---------
1 file changed, 15 insertions(+), 9 deletions(-)
diff --git a/plugins/data/nbdkit-data-plugin.pod b/plugins/data/nbdkit-data-plugin.pod
index 32a450ab..b69435e9 100644
--- a/plugins/data/nbdkit-data-plugin.pod
+++ b/plugins/data/nbdkit-data-plugin.pod
@@ -172,18 +172,24 @@ compact format. It is a string containing a list of bytes which are
written into the disk image sequentially. You can move the virtual
offset where bytes are written using C<@offset>.
-For example:
-
nbdkit data '0 1 2 3 @0x1fe 0x55 0xaa'
-creates a 0x200 = 512 byte (1 sector) image containing the four bytes
-C<0 1 2 3> at the start, and the two bytes C<0x55 0xaa> at the end of
-the sector, with the remaining 506 bytes in the middle being all
-zeroes. In this example the size (512 bytes) is implied by the data.
-But you could additionally use the C<size> parameter to either
-truncate or extend (with zeroes) the disk image.
+creates:
-Whitespace between fields in the string is ignored.
+ total size 0x200 = 512 bytes (1 sector)
+┌──────┬──────┬──────┬──────┬───────── ── ── ───┬──────┬──────┐
+│ 0 │ 1 │ 2 │ 3 │ 0 0 ... 0 │ 0x55 │ 0xaa │
+└──────┴──────┴──────┴──────┴───────── ── ── ───┴──────┴──────┘
+ ↑
+ offset 0x1fe
+
+In this example the size is implied by the data. But you could also
+use the C<size> parameter to either truncate or extend (with zeroes)
+the disk image. Another way to write the same disk would be this,
+where we align the offset to the end of the sector and move back 2
+bytes to write the signature:
+
+ nbdkit data '0 1 2 3 @^0x200 @-2 le16:0xaa55'
Fields in the string can be:
--
2.31.1

57
SOURCES/0015-vddk-Assume-that-VixDiskLib_Flush-is-available.patch Normal file
View File

@ -0,0 +1,57 @@
From eda9dd7f5e610fd4e17019813c5a045f0b3603df Mon Sep 17 00:00:00 2001
From: "Richard W.M. Jones" <rjones@redhat.com>
Date: Fri, 29 Oct 2021 20:56:55 +0100
Subject: [PATCH] vddk: Assume that VixDiskLib_Flush is available
Since we now require and check that VDDK >= 6.5, we can assume that
VixDiskLib_Flush is always available.
(cherry picked from commit e3685e6f0d0b71ab24b96fe85430a3b75da58736)
---
plugins/vddk/vddk.c | 15 +++++++++++----
1 file changed, 11 insertions(+), 4 deletions(-)
diff --git a/plugins/vddk/vddk.c b/plugins/vddk/vddk.c
index 9f223db0..f967e2d9 100644
--- a/plugins/vddk/vddk.c
+++ b/plugins/vddk/vddk.c
@@ -378,6 +378,12 @@ load_library (bool load_error_is_fatal)
"See nbdkit-vddk-plugin(1) man page section \"SUPPORTED VERSIONS OF VDDK\".");
exit (EXIT_FAILURE);
}
+
+ /* Added in VDDK 6.0 so it must always be present. Since we are
+ * going to call this function unconditionally, fail early and hard
+ * if for some reason it's not present.
+ */
+ assert (VixDiskLib_Flush != NULL);
}
static int
@@ -725,18 +731,19 @@ vddk_get_size (void *handle)
return (int64_t) size;
}
+/* The Flush call was added in VDDK 6.0, since we support minimum 6.5
+ * we are always able to do FUA / flush.
+ */
static int
vddk_can_fua (void *handle)
{
- /* The Flush call was not available in VDDK < 6.0. */
- return VixDiskLib_Flush != NULL ? NBDKIT_FUA_NATIVE : NBDKIT_FUA_NONE;
+ return NBDKIT_FUA_NATIVE;
}
static int
vddk_can_flush (void *handle)
{
- /* The Flush call was not available in VDDK < 6.0. */
- return VixDiskLib_Flush != NULL;
+ return 1;
}
/* Read data from the file.
--
2.31.1

45
SOURCES/0016-cow-Add-some-more-debugging-especially-for-blk_read_.patch
View File

@ -1,45 +0,0 @@
From a22248e3075e782d28542f8f6acd046c9dfa8998 Mon Sep 17 00:00:00 2001
From: "Richard W.M. Jones" <rjones@redhat.com>
Date: Mon, 9 Aug 2021 14:09:31 +0100
Subject: [PATCH] cow: Add some more debugging especially for blk_read_multiple
and cow-on-read
Only activated when we use -D cow.verbose=1
(cherry picked from commit 2da1ae0ca966af955d8fcf3feffffc80d07142fd)
---
filters/cow/blk.c | 11 +++++++++--
1 file changed, 9 insertions(+), 2 deletions(-)
diff --git a/filters/cow/blk.c b/filters/cow/blk.c
index 4f84e092..c22d5886 100644
--- a/filters/cow/blk.c
+++ b/filters/cow/blk.c
@@ -254,8 +254,10 @@ blk_read_multiple (nbdkit_next *next,
if (cow_debug_verbose)
nbdkit_debug ("cow: blk_read_multiple block %" PRIu64
- " (offset %" PRIu64 ") is %s",
- blknum, (uint64_t) offset, state_to_string (state));
+ " (offset %" PRIu64 ") run of length %" PRIu64
+ " is %s",
+ blknum, (uint64_t) offset, runblocks,
+ state_to_string (state));
if (state == BLOCK_NOT_ALLOCATED) { /* Read underlying plugin. */
unsigned n, tail = 0;
@@ -281,6 +283,11 @@ blk_read_multiple (nbdkit_next *next,
* set them as allocated.
*/
if (cow_on_read) {
+ if (cow_debug_verbose)
+ nbdkit_debug ("cow: cow-on-read saving %" PRIu64 " blocks "
+ "at offset %" PRIu64 " into the cache",
+ runblocks, offset);
+
if (full_pwrite (fd, block, BLKSIZE * runblocks, offset) == -1) {
*err = errno;
nbdkit_error ("pwrite: %m");
--
2.31.1

186
SOURCES/0016-vddk-Simplify-detection-of-VDDK-symbols-and-baseline.patch Normal file
View File

@ -0,0 +1,186 @@
From 1b2b386c9a254808a25fbfce3640c96bdb8cf9be Mon Sep 17 00:00:00 2001
From: "Richard W.M. Jones" <rjones@redhat.com>
Date: Fri, 29 Oct 2021 21:02:54 +0100
Subject: [PATCH] vddk: Simplify detection of VDDK symbols and baseline 6.5
Make all symbols from VDDK 6.5 into required symbols and use a single
error message function if one of these is missing. The new error is:
nbdkit: error: required VDDK symbol "VixDiskLib_Wait" is
missing. VDDK version must be >= 6.5. See nbdkit-vddk-plugin(1) man
page section "SUPPORTED VERSIONS OF VDDK". Original dlopen error:
vmware-vix-disklib-distrib/lib64/libvixDiskLib.so.6: undefined
symbol: VixDiskLib_Wait
Remove the extra check and assert.
Be more consistent about #define OPTIONAL_STUB(fn,ret,args) STUB(fn,ret,args)
when we want the optional and required stubs to do the same thing.
(cherry picked from commit ec0d22e61881efa39a69d02ccb9e4ede8bf95e75)
---
plugins/vddk/stats.c | 2 +-
plugins/vddk/vddk-stubs.h | 45 ++++++++++++++++++---------------------
plugins/vddk/vddk.c | 36 ++++++++++++-------------------
plugins/vddk/vddk.h | 2 +-
4 files changed, 37 insertions(+), 48 deletions(-)
diff --git a/plugins/vddk/stats.c b/plugins/vddk/stats.c
index 18a42714..76e0c244 100644
--- a/plugins/vddk/stats.c
+++ b/plugins/vddk/stats.c
@@ -89,7 +89,7 @@ display_stats (void)
if (!vddk_debug_stats) return;
#define STUB(fn,ret,args) statlist_append (&stats, stats_##fn)
-#define OPTIONAL_STUB(fn,ret,args) statlist_append (&stats, stats_##fn)
+#define OPTIONAL_STUB(fn,ret,args) STUB(fn,ret,args)
#include "vddk-stubs.h"
#undef STUB
#undef OPTIONAL_STUB
diff --git a/plugins/vddk/vddk-stubs.h b/plugins/vddk/vddk-stubs.h
index 66353691..7d8644c3 100644
--- a/plugins/vddk/vddk-stubs.h
+++ b/plugins/vddk/vddk-stubs.h
@@ -39,10 +39,7 @@
* function name, return value, arguments.
*/
-/* Required stubs, present in all versions of VDDK that we support. I
- * have checked that all these exist in at least VDDK 5.5.5 (2015).
- */
-
+/* Required stubs, present in all versions of VDDK since 6.5 (Nov 2016). */
STUB (VixDiskLib_InitEx,
VixError,
(uint32_t major, uint32_t minor,
@@ -103,27 +100,27 @@ STUB (VixDiskLib_Write,
uint64_t start_sector, uint64_t nr_sectors,
const unsigned char *buf));
-/* Added in VDDK 6.0, these will be NULL in earlier versions. */
-OPTIONAL_STUB (VixDiskLib_Flush,
- VixError,
- (VixDiskLibHandle handle));
-OPTIONAL_STUB (VixDiskLib_ReadAsync,
- VixError,
- (VixDiskLibHandle handle,
- uint64_t start_sector, uint64_t nr_sectors,
- unsigned char *buf,
- VixDiskLibCompletionCB callback, void *data));
-OPTIONAL_STUB (VixDiskLib_WriteAsync,
- VixError,
- (VixDiskLibHandle handle,
- uint64_t start_sector, uint64_t nr_sectors,
- const unsigned char *buf,
- VixDiskLibCompletionCB callback, void *data));
+/* Added in VDDK 6.0. */
+STUB (VixDiskLib_Flush,
+ VixError,
+ (VixDiskLibHandle handle));
+STUB (VixDiskLib_ReadAsync,
+ VixError,
+ (VixDiskLibHandle handle,
+ uint64_t start_sector, uint64_t nr_sectors,
+ unsigned char *buf,
+ VixDiskLibCompletionCB callback, void *data));
+STUB (VixDiskLib_WriteAsync,
+ VixError,
+ (VixDiskLibHandle handle,
+ uint64_t start_sector, uint64_t nr_sectors,
+ const unsigned char *buf,
+ VixDiskLibCompletionCB callback, void *data));
-/* Added in VDDK 6.5, this will be NULL in earlier versions. */
-OPTIONAL_STUB (VixDiskLib_Wait,
- VixError,
- (VixDiskLibHandle handle));
+/* Added in VDDK 6.5. */
+STUB (VixDiskLib_Wait,
+ VixError,
+ (VixDiskLibHandle handle));
/* Added in VDDK 6.7, these will be NULL for earlier versions: */
OPTIONAL_STUB (VixDiskLib_QueryAllocatedBlocks,
diff --git a/plugins/vddk/vddk.c b/plugins/vddk/vddk.c
index f967e2d9..271b5ee0 100644
--- a/plugins/vddk/vddk.c
+++ b/plugins/vddk/vddk.c
@@ -63,7 +63,7 @@ NBDKIT_DLL_PUBLIC int vddk_debug_datapath = 1;
* initialized when the plugin is loaded (by vddk_get_ready).
*/
#define STUB(fn,ret,args) ret (*fn) args
-#define OPTIONAL_STUB(fn,ret,args) ret (*fn) args
+#define OPTIONAL_STUB(fn,ret,args) STUB(fn,ret,args)
#include "vddk-stubs.h"
#undef STUB
#undef OPTIONAL_STUB
@@ -282,6 +282,17 @@ vddk_config (const char *key, const char *value)
return 0;
}
+static void
+missing_required_symbol (const char *fn)
+{
+ nbdkit_error ("required VDDK symbol \"%s\" is missing. "
+ "VDDK version must be >= 6.5. "
+ "See nbdkit-vddk-plugin(1) man page section \"SUPPORTED VERSIONS OF VDDK\". "
+ "Original dlopen error: %s\n",
+ fn, dlerror ());
+ exit (EXIT_FAILURE);
+}
+
/* Load the VDDK library. */
static void
load_library (bool load_error_is_fatal)
@@ -358,32 +369,13 @@ load_library (bool load_error_is_fatal)
#define STUB(fn,ret,args) \
do { \
fn = dlsym (dl, #fn); \
- if (fn == NULL) { \
- nbdkit_error ("required VDDK symbol \"%s\" is missing: %s", \
- #fn, dlerror ()); \
- exit (EXIT_FAILURE); \
- } \
+ if (fn == NULL) \
+ missing_required_symbol (#fn); \
} while (0)
#define OPTIONAL_STUB(fn,ret,args) fn = dlsym (dl, #fn)
#include "vddk-stubs.h"
#undef STUB
#undef OPTIONAL_STUB
-
- /* Additionally, VDDK version must be >= 6.5. This was the first
- * version which introduced VixDiskLib_Wait symbol so we can check
- * for that.
- */
- if (VixDiskLib_Wait == NULL) {
- nbdkit_error ("VDDK version must be >= 6.5. "
- "See nbdkit-vddk-plugin(1) man page section \"SUPPORTED VERSIONS OF VDDK\".");
- exit (EXIT_FAILURE);
- }
-
- /* Added in VDDK 6.0 so it must always be present. Since we are
- * going to call this function unconditionally, fail early and hard
- * if for some reason it's not present.
- */
- assert (VixDiskLib_Flush != NULL);
}
static int
diff --git a/plugins/vddk/vddk.h b/plugins/vddk/vddk.h
index be0b3492..0e3dd79e 100644
--- a/plugins/vddk/vddk.h
+++ b/plugins/vddk/vddk.h
@@ -76,7 +76,7 @@ extern int vddk_debug_datapath;
extern int vddk_debug_stats;
#define STUB(fn,ret,args) extern ret (*fn) args
-#define OPTIONAL_STUB(fn,ret,args) extern ret (*fn) args
+#define OPTIONAL_STUB(fn,ret,args) STUB(fn,ret,args)
#include "vddk-stubs.h"
#undef STUB
#undef OPTIONAL_STUB
--
2.31.1

142
SOURCES/0017-delay-Fix-delay-close.patch
View File

@ -1,142 +0,0 @@
From be7252bada79ee542356dffaf5f3c568a5c7fec3 Mon Sep 17 00:00:00 2001
From: "Richard W.M. Jones" <rjones@redhat.com>
Date: Tue, 10 Aug 2021 08:39:15 +0100
Subject: [PATCH] delay: Fix delay-close
See comments in the code for how this has been fixed.
This only delays clients which use NBD_CMD_DISC (libnbd
nbd_shutdown(3)). Clients which drop the connection obviously cannot
be delayed. For example:
$ nbdkit --filter=delay null delay-close=3 \
--run 'time nbdsh -u $uri -c "h.shutdown()"
time nbdsh -u $uri -c "pass"'
real 0m3.061s # Client used shutdown, was delayed
user 0m0.028s
sys 0m0.030s
real 0m0.058s # Client disconnected, was not delayed
user 0m0.029s
sys 0m0.027s
Reported-by: Ming Xie
Fixes: commit de8dcd3a34a38b088a0f9a6f8ca754702ad1f598
Fixes: https://bugzilla.redhat.com/show_bug.cgi?id=1991652
(cherry picked from commit 0cafebdb67d0d557ba1be8ea306b8acc5d9b2203)
---
filters/delay/delay.c | 42 +++++++++++++++++++--------
filters/delay/nbdkit-delay-filter.pod | 14 +++++++--
2 files changed, 41 insertions(+), 15 deletions(-)
diff --git a/filters/delay/delay.c b/filters/delay/delay.c
index df3729a7..9252b855 100644
--- a/filters/delay/delay.c
+++ b/filters/delay/delay.c
@@ -39,6 +39,7 @@
#include <stdbool.h>
#include <errno.h>
#include <limits.h>
+#include <time.h>
#include <nbdkit-filter.h>
@@ -134,12 +135,6 @@ open_delay (int *err)
return delay (delay_open_ms, err);
}
-static int
-close_delay (int *err)
-{
- return delay (delay_close_ms, err);
-}
-
/* Called for each key=value passed on the command line. */
static int
delay_config (nbdkit_next_config *next, nbdkit_backend *nxdata,
@@ -253,13 +248,36 @@ delay_open (nbdkit_next_open *next, nbdkit_context *nxdata,
return NBDKIT_HANDLE_NOT_NEEDED;
}
-/* Close connection. */
-static void
-delay_close (void *handle)
+/* Close connection.
+ *
+ * We cannot call nbdkit_nanosleep here because the socket may have
+ * been closed and that function will abort and return immediately.
+ * However we want to force a sleep (even if the server is shutting
+ * down) so use regular nanosleep instead.
+ *
+ * We cannot use the .close callback because that happens after the
+ * socket has closed, thus not delaying the client. By using
+ * .finalize we can delay well-behaved clients (those that use
+ * NBD_CMD_DISC). We cannot delay clients that drop the connection.
+ */
+static int
+delay_finalize (nbdkit_next *next, void *handle)
{
- int err;
+ const unsigned ms = delay_close_ms;
- close_delay (&err);
+ if (ms > 0) {
+ struct timespec ts;
+
+ ts.tv_sec = ms / 1000;
+ ts.tv_nsec = (ms % 1000) * 1000000;
+ /* If nanosleep fails we don't really want to interrupt the chain
+ * of finalize calls through the other filters, so ignore any
+ * error here.
+ */
+ nanosleep (&ts, NULL);
+ }
+
+ return next->finalize (next);
}
/* Read data. */
@@ -340,7 +358,7 @@ static struct nbdkit_filter filter = {
.config_help = delay_config_help,
.can_fast_zero = delay_can_fast_zero,
.open = delay_open,
- .close = delay_close,
+ .finalize = delay_finalize,
.pread = delay_pread,
.pwrite = delay_pwrite,
.zero = delay_zero,
diff --git a/filters/delay/nbdkit-delay-filter.pod b/filters/delay/nbdkit-delay-filter.pod
index 11ae544b..76614736 100644
--- a/filters/delay/nbdkit-delay-filter.pod
+++ b/filters/delay/nbdkit-delay-filter.pod
@@ -117,15 +117,23 @@ the plugin.
=item B<delay-open=>NNB<ms>
+(nbdkit E<ge> 1.28)
+
+Delay open (client connection) by C<SECS> seconds or C<NN>
+milliseconds.
+
=item B<delay-close=>SECS
=item B<delay-close=>NNB<ms>
(nbdkit E<ge> 1.28)
-Delay open and close operations by C<SECS> seconds or C<NN>
-milliseconds. Open corresponds to client connection. Close may not
-be visible to clients if they abruptly disconnect.
+Delay close (client disconnection) by C<SECS> seconds or C<NN>
+milliseconds. This can also cause server shutdown to be delayed if
+clients are connected at the time. This only affects clients that
+gracefully disconnect (using C<NBD_CMD_DISC> / libnbd function
+L<nbd_shutdown(3)>). Clients that abruptly disconnect from the server
+cannot be delayed.
=back
--
2.31.1

40
SOURCES/0017-vddk-Remove-some-whitespace-from-a-couple-of-functio.patch Normal file
View File

@ -0,0 +1,40 @@
From 2363e76ab34a2e11b57970d82161f73453a4a8ec Mon Sep 17 00:00:00 2001
From: "Richard W.M. Jones" <rjones@redhat.com>
Date: Sat, 30 Oct 2021 08:34:28 +0100
Subject: [PATCH] vddk: Remove some whitespace from a couple of functions
(cherry picked from commit 974dce2c2ef84fc096ee319f340054234a29df91)
---
plugins/vddk/vddk.c | 9 ++-------
1 file changed, 2 insertions(+), 7 deletions(-)
diff --git a/plugins/vddk/vddk.c b/plugins/vddk/vddk.c
index 271b5ee0..184f1a9c 100644
--- a/plugins/vddk/vddk.c
+++ b/plugins/vddk/vddk.c
@@ -792,9 +792,7 @@ static int
vddk_flush (void *handle, uint32_t flags)
{
struct vddk_handle *h = handle;
- struct command flush_cmd = {
- .type = FLUSH,
- };
+ struct command flush_cmd = { .type = FLUSH };
return send_command_and_wait (h, &flush_cmd);
}
@@ -804,10 +802,7 @@ vddk_can_extents (void *handle)
{
struct vddk_handle *h = handle;
int ret;
- struct command can_extents_cmd = {
- .type = CAN_EXTENTS,
- .ptr = &ret,
- };
+ struct command can_extents_cmd = { .type = CAN_EXTENTS, .ptr = &ret };
if (send_command_and_wait (h, &can_extents_cmd) == -1)
return -1;
--
2.31.1

155
SOURCES/0018-delay-Test-delay-open-and-delay-close.patch
View File

@ -1,155 +0,0 @@
From 838ec052abe63056434c08ea80f4609e697dad0f Mon Sep 17 00:00:00 2001
From: "Richard W.M. Jones" <rjones@redhat.com>
Date: Tue, 10 Aug 2021 09:11:43 +0100
Subject: [PATCH] delay: Test delay-open and delay-close
(cherry picked from commit 3caabaf87ec744b863b50b5bf77a9c1b93a7c3e0)
---
tests/Makefile.am | 12 +++++++--
tests/test-delay-close.sh | 54 +++++++++++++++++++++++++++++++++++++++
tests/test-delay-open.sh | 49 +++++++++++++++++++++++++++++++++++
3 files changed, 113 insertions(+), 2 deletions(-)
create mode 100755 tests/test-delay-close.sh
create mode 100755 tests/test-delay-open.sh
diff --git a/tests/Makefile.am b/tests/Makefile.am
index edc8d66d..e61c5829 100644
--- a/tests/Makefile.am
+++ b/tests/Makefile.am
@@ -1425,8 +1425,16 @@ EXTRA_DIST += \
$(NULL)
# delay filter tests.
-TESTS += test-delay-shutdown.sh
-EXTRA_DIST += test-delay-shutdown.sh
+TESTS += \
+ test-delay-close.sh \
+ test-delay-open.sh \
+ test-delay-shutdown.sh \
+ $(NULL)
+EXTRA_DIST += \
+ test-delay-close.sh \
+ test-delay-open.sh \
+ test-delay-shutdown.sh \
+ $(NULL)
LIBNBD_TESTS += test-delay
test_delay_SOURCES = test-delay.c
diff --git a/tests/test-delay-close.sh b/tests/test-delay-close.sh
new file mode 100755
index 00000000..1de305f5
--- /dev/null
+++ b/tests/test-delay-close.sh
@@ -0,0 +1,54 @@
+#!/usr/bin/env bash
+# nbdkit
+# Copyright (C) 2018-2021 Red Hat Inc.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#
+# * Redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in the
+# documentation and/or other materials provided with the distribution.
+#
+# * Neither the name of Red Hat nor the names of its contributors may be
+# used to endorse or promote products derived from this software without
+# specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY RED HAT AND CONTRIBUTORS ''AS IS'' AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+# PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL RED HAT OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
+# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+# SUCH DAMAGE.
+
+source ./functions.sh
+set -e
+set -x
+
+requires_run
+requires_plugin null
+requires_filter delay
+requires nbdsh --version
+
+# Test delay-close with a well-behaved client.
+
+nbdkit -U - null --filter=delay delay-close=3 \
+ --run '
+start_t=$SECONDS
+nbdsh -u "$uri" -c "h.shutdown()"
+end_t=$SECONDS
+
+if [ $((end_t - start_t)) -lt 3 ]; then
+ echo "$0: delay filter failed: delay-close=3 caused delay < 3 seconds"
+ exit 1
+fi
+'
diff --git a/tests/test-delay-open.sh b/tests/test-delay-open.sh
new file mode 100755
index 00000000..2a74e44c
--- /dev/null
+++ b/tests/test-delay-open.sh
@@ -0,0 +1,49 @@
+#!/usr/bin/env bash
+# nbdkit
+# Copyright (C) 2018-2021 Red Hat Inc.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#
+# * Redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in the
+# documentation and/or other materials provided with the distribution.
+#
+# * Neither the name of Red Hat nor the names of its contributors may be
+# used to endorse or promote products derived from this software without
+# specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY RED HAT AND CONTRIBUTORS ''AS IS'' AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+# PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL RED HAT OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
+# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+# SUCH DAMAGE.
+
+source ./functions.sh
+set -e
+set -x
+
+requires_run
+requires_plugin null
+requires_filter delay
+requires nbdinfo --version
+
+start_t=$SECONDS
+nbdkit -U - null --filter=delay delay-open=3 --run 'nbdinfo "$uri"'
+end_t=$SECONDS
+
+if [ $((end_t - start_t)) -lt 3 ]; then
+ echo "$0: delay filter failed: delay-open=3 caused delay < 3 seconds"
+ exit 1
+fi
--
2.31.1

338
SOURCES/0018-vddk-Move-config-debug-error-and-utility-functions-a.patch Normal file
View File

@ -0,0 +1,338 @@
From 6c0034cf8802d466b170135fec0d6a97d1eb2f2a Mon Sep 17 00:00:00 2001
From: "Richard W.M. Jones" <rjones@redhat.com>
Date: Sat, 30 Oct 2021 08:27:39 +0100
Subject: [PATCH] vddk: Move config, debug/error and utility functions around
Move the functions so they are nearer to where they are used.
Introduce a utils.c file for utility functions.
This is just code rearrangement with no other effects.
(cherry picked from commit c59be086210a06688b9195e0b91f8603a668654a)
---
plugins/vddk/Makefile.am | 1 +
plugins/vddk/utils.c | 51 ++++++++++
plugins/vddk/vddk.c | 201 +++++++++++++++++++--------------------
plugins/vddk/vddk.h | 3 +
4 files changed, 151 insertions(+), 105 deletions(-)
create mode 100644 plugins/vddk/utils.c
diff --git a/plugins/vddk/Makefile.am b/plugins/vddk/Makefile.am
index f8382fc9..02113da0 100644
--- a/plugins/vddk/Makefile.am
+++ b/plugins/vddk/Makefile.am
@@ -47,6 +47,7 @@ nbdkit_vddk_plugin_la_SOURCES = \
vddk.h \
reexec.c \
stats.c \
+ utils.c \
vddk-structs.h \
vddk-stubs.h \
worker.c \
diff --git a/plugins/vddk/utils.c b/plugins/vddk/utils.c
new file mode 100644
index 00000000..f0c19950
--- /dev/null
+++ b/plugins/vddk/utils.c
@@ -0,0 +1,51 @@
+/* nbdkit
+ * Copyright (C) 2013-2021 Red Hat Inc.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * * Neither the name of Red Hat nor the names of its contributors may be
+ * used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY RED HAT AND CONTRIBUTORS ''AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+ * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL RED HAT OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
+ * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+ * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <config.h>
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#define NBDKIT_API_VERSION 2
+#include <nbdkit-plugin.h>
+
+#include "vddk.h"
+
+void
+trim (char *str)
+{
+ size_t len = strlen (str);
+
+ if (len > 0 && str[len-1] == '\n')
+ str[len-1] = '\0';
+}
diff --git a/plugins/vddk/vddk.c b/plugins/vddk/vddk.c
index 184f1a9c..31e5e23b 100644
--- a/plugins/vddk/vddk.c
+++ b/plugins/vddk/vddk.c
@@ -114,61 +114,6 @@ vddk_unload (void)
free (password);
}
-static void
-trim (char *str)
-{
- size_t len = strlen (str);
-
- if (len > 0 && str[len-1] == '\n')
- str[len-1] = '\0';
-}
-
-/* Turn log messages from the library into nbdkit_debug. */
-static void
-debug_function (const char *fs, va_list args)
-{
- CLEANUP_FREE char *str = NULL;
-
- if (vasprintf (&str, fs, args) == -1) {
- nbdkit_debug ("lost debug message: %s", fs);
- return;
- }
-
- trim (str);
-
- nbdkit_debug ("%s", str);
-}
-
-/* Turn error messages from the library into nbdkit_error. */
-static void
-error_function (const char *fs, va_list args)
-{
- CLEANUP_FREE char *str = NULL;
-
- /* If the thread-local error_suppression flag is non-zero then we
- * will suppress error messages from VDDK in this thread.
- */
- if (error_suppression) return;
-
- if (vasprintf (&str, fs, args) == -1) {
- nbdkit_error ("lost error message: %s", fs);
- return;
- }
-
- trim (str);
-
- /* VDDK 7 added a useless error message about their "phone home"
- * system called CEIP which only panics users. Demote it to a debug
- * statement. https://bugzilla.redhat.com/show_bug.cgi?id=1834267
- */
- if (strstr (str, "Get CEIP status failed") != NULL) {
- nbdkit_debug ("%s", str);
- return;
- }
-
- nbdkit_error ("%s", str);
-}
-
/* Configuration. */
static int
vddk_config (const char *key, const char *value)
@@ -282,6 +227,56 @@ vddk_config (const char *key, const char *value)
return 0;
}
+static int
+vddk_config_complete (void)
+{
+ if (filename == NULL) {
+ nbdkit_error ("you must supply the file=<FILENAME> parameter "
+ "after the plugin name on the command line");
+ return -1;
+ }
+
+ /* For remote connections, check all the parameters have been
+ * passed. Note that VDDK will segfault if parameters that it
+ * expects are NULL (and there's no real way to tell what parameters
+ * it is expecting). This implements the same test that the VDDK
+ * sample program does.
+ */
+ is_remote =
+ vmx_spec ||
+ server_name ||
+ username ||
+ password ||
+ cookie ||
+ thumb_print ||
+ port ||
+ nfc_host_port;
+
+ if (is_remote) {
+#define missing(test, param) \
+ if (test) { \
+ nbdkit_error ("remote connection requested, missing parameter: %s", \
+ param); \
+ return -1; \
+ }
+ missing (!server_name, "server");
+ missing (!username, "user");
+ missing (!password, "password");
+ missing (!vmx_spec, "vm");
+#undef missing
+ }
+
+ /* Restore original LD_LIBRARY_PATH after reexec. */
+ if (restore_ld_library_path () == -1)
+ return -1;
+
+ return 0;
+}
+
+#define vddk_config_help \
+ "[file=]<FILENAME> (required) The filename (eg. VMDK file) to serve.\n" \
+ "Many optional parameters are supported, see nbdkit-vddk-plugin(1)."
+
static void
missing_required_symbol (const char *fn)
{
@@ -378,56 +373,6 @@ load_library (bool load_error_is_fatal)
#undef OPTIONAL_STUB
}
-static int
-vddk_config_complete (void)
-{
- if (filename == NULL) {
- nbdkit_error ("you must supply the file=<FILENAME> parameter "
- "after the plugin name on the command line");
- return -1;
- }
-
- /* For remote connections, check all the parameters have been
- * passed. Note that VDDK will segfault if parameters that it
- * expects are NULL (and there's no real way to tell what parameters
- * it is expecting). This implements the same test that the VDDK
- * sample program does.
- */
- is_remote =
- vmx_spec ||
- server_name ||
- username ||
- password ||
- cookie ||
- thumb_print ||
- port ||
- nfc_host_port;
-
- if (is_remote) {
-#define missing(test, param) \
- if (test) { \
- nbdkit_error ("remote connection requested, missing parameter: %s", \
- param); \
- return -1; \
- }
- missing (!server_name, "server");
- missing (!username, "user");
- missing (!password, "password");
- missing (!vmx_spec, "vm");
-#undef missing
- }
-
- /* Restore original LD_LIBRARY_PATH after reexec. */
- if (restore_ld_library_path () == -1)
- return -1;
-
- return 0;
-}
-
-#define vddk_config_help \
- "[file=]<FILENAME> (required) The filename (eg. VMDK file) to serve.\n" \
- "Many optional parameters are supported, see nbdkit-vddk-plugin(1)."
-
static int
vddk_get_ready (void)
{
@@ -435,6 +380,52 @@ vddk_get_ready (void)
return 0;
}
+/* Turn log messages from the library into nbdkit_debug. */
+static void
+debug_function (const char *fs, va_list args)
+{
+ CLEANUP_FREE char *str = NULL;
+
+ if (vasprintf (&str, fs, args) == -1) {
+ nbdkit_debug ("lost debug message: %s", fs);
+ return;
+ }
+
+ trim (str);
+
+ nbdkit_debug ("%s", str);
+}
+
+/* Turn error messages from the library into nbdkit_error. */
+static void
+error_function (const char *fs, va_list args)
+{
+ CLEANUP_FREE char *str = NULL;
+
+ /* If the thread-local error_suppression flag is non-zero then we
+ * will suppress error messages from VDDK in this thread.
+ */
+ if (error_suppression) return;
+
+ if (vasprintf (&str, fs, args) == -1) {
+ nbdkit_error ("lost error message: %s", fs);
+ return;
+ }
+
+ trim (str);
+
+ /* VDDK 7 added a useless error message about their "phone home"
+ * system called CEIP which only panics users. Demote it to a debug
+ * statement. https://bugzilla.redhat.com/show_bug.cgi?id=1834267
+ */
+ if (strstr (str, "Get CEIP status failed") != NULL) {
+ nbdkit_debug ("%s", str);
+ return;
+ }
+
+ nbdkit_error ("%s", str);
+}
+
/* Defer VDDK initialization until after fork because it is known to
* create background threads from VixDiskLib_InitEx. Unfortunately
* error reporting from this callback is difficult, but we have
diff --git a/plugins/vddk/vddk.h b/plugins/vddk/vddk.h
index 0e3dd79e..d99b6f4b 100644
--- a/plugins/vddk/vddk.h
+++ b/plugins/vddk/vddk.h
@@ -183,6 +183,9 @@ extern pthread_mutex_t stats_lock;
#undef OPTIONAL_STUB
extern void display_stats (void);
+/* utils.c */
+extern void trim (char *str);
+
/* worker.c */
extern const char *command_type_string (enum command_type type);
extern int send_command_and_wait (struct vddk_handle *h, struct command *cmd);
--
2.31.1

245
SOURCES/0019-common-utils-test-vector.c-Add-vector-benchmarks.patch Normal file
View File

@ -0,0 +1,245 @@
From 6459704cc66f5fa0a2e6fc1e199458b77327fe52 Mon Sep 17 00:00:00 2001
From: Nir Soffer <nsoffer@redhat.com>
Date: Fri, 5 Nov 2021 20:36:42 +0200
Subject: [PATCH] common/utils/test-vector.c: Add vector benchmarks
The generic vector reallocs on every append. Add benchmarks to measure
the cost with uint32 vector (used for copying extents) and the effect of
reserving space upfront.
The tests show that realloc is pretty efficient, but calling reserve
before the appends speeds the appends up significantly.
NBDKIT_BENCH=1 ./test-vector
bench_reserve: 1000000 appends in 0.004503 s
bench_append: 1000000 appends in 0.014986 s
The new benchmarks do not run by default to avoid trouble in CI on
overloaded machines or under qemu emulation.
A new target added to run all benchmaks:
make bench
Ported from libnbd:
- commit dc9ae0174ab1384081a57a8d54b10f8147ea6430
- commit f6c06a3b4d87fe976a96ea04f8da1f22b2531dbd
(cherry picked from commit a227af7921c9a51c4f1ab699a3b9f06a9a645126)
---
Makefile.am | 5 +++
README | 7 ++++
common/utils/Makefile.am | 5 ++-
common/utils/bench.h | 72 ++++++++++++++++++++++++++++++++++++++
common/utils/test-vector.c | 55 +++++++++++++++++++++++++++--
5 files changed, 141 insertions(+), 3 deletions(-)
create mode 100644 common/utils/bench.h
diff --git a/Makefile.am b/Makefile.am
index b21d69ed..49f5d91c 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -102,6 +102,11 @@ check-root:
check-vddk:
$(MAKE) -C tests check-vddk
+bench: all
+ @for d in common/utils; do \
+ $(MAKE) -C $$d bench || exit 1; \
+ done
+
#----------------------------------------------------------------------
# Maintainers only!
diff --git a/README b/README
index a04325be..b001620c 100644
--- a/README
+++ b/README
@@ -274,6 +274,13 @@ nbdkit-vddk-plugin against the library like this:
make check-vddk vddkdir=vmware-vix-disklib-distrib
+Running the benchmarks
+----------------------
+
+To run benchmarks:
+
+ make bench
+
DOWNLOAD TARBALLS
=================
diff --git a/common/utils/Makefile.am b/common/utils/Makefile.am
index 14e9dfc4..55415535 100644
--- a/common/utils/Makefile.am
+++ b/common/utils/Makefile.am
@@ -100,6 +100,9 @@ test_quotes_SOURCES = test-quotes.c quote.c utils.h
test_quotes_CPPFLAGS = -I$(srcdir)
test_quotes_CFLAGS = $(WARNINGS_CFLAGS)
-test_vector_SOURCES = test-vector.c vector.c vector.h
+test_vector_SOURCES = test-vector.c vector.c vector.h bench.h
test_vector_CPPFLAGS = -I$(srcdir)
test_vector_CFLAGS = $(WARNINGS_CFLAGS)
+
+bench: test-vector
+ NBDKIT_BENCH=1 ./test-vector
diff --git a/common/utils/bench.h b/common/utils/bench.h
new file mode 100644
index 00000000..496a3614
--- /dev/null
+++ b/common/utils/bench.h
@@ -0,0 +1,72 @@
+/* libnbd
+ * Copyright (C) 2021 Red Hat Inc.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * * Neither the name of Red Hat nor the names of its contributors may be
+ * used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY RED HAT AND CONTRIBUTORS ''AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+ * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL RED HAT OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
+ * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+ * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#ifndef LIBNBD_BENCH_H
+#define LIBNBD_BENCH_H
+
+#include <sys/time.h>
+
+#define MICROSECONDS 1000000
+
+struct bench {
+ struct timeval start, stop;
+};
+
+static inline void
+bench_start(struct bench *b)
+{
+ gettimeofday (&b->start, NULL);
+}
+
+static inline void
+bench_stop(struct bench *b)
+{
+ gettimeofday (&b->stop, NULL);
+}
+
+static inline double
+bench_sec(struct bench *b)
+{
+ struct timeval dt;
+
+ dt.tv_sec = b->stop.tv_sec - b->start.tv_sec;
+ dt.tv_usec = b->stop.tv_usec - b->start.tv_usec;
+
+ if (dt.tv_usec < 0) {
+ dt.tv_sec -= 1;
+ dt.tv_usec += MICROSECONDS;
+ }
+
+ return ((double)dt.tv_sec * MICROSECONDS + dt.tv_usec) / MICROSECONDS;
+}
+
+#endif /* LIBNBD_BENCH_H */
diff --git a/common/utils/test-vector.c b/common/utils/test-vector.c
index 94b2aeb7..28af59b8 100644
--- a/common/utils/test-vector.c
+++ b/common/utils/test-vector.c
@@ -38,9 +38,13 @@
#undef NDEBUG /* Keep test strong even for nbdkit built without assertions */
#include <assert.h>
+#include "bench.h"
#include "vector.h"
+#define APPENDS 1000000
+
DEFINE_VECTOR_TYPE(int64_vector, int64_t);
+DEFINE_VECTOR_TYPE(uint32_vector, uint32_t);
DEFINE_VECTOR_TYPE(string_vector, char *);
static int
@@ -113,10 +117,57 @@ test_string_vector (void)
free (v.ptr);
}
+static void
+bench_reserve (void)
+{
+ uint32_vector v = empty_vector;
+ struct bench b;
+
+ bench_start(&b);
+
+ uint32_vector_reserve(&v, APPENDS);
+
+ for (uint32_t i = 0; i < APPENDS; i++) {
+ uint32_vector_append (&v, i);
+ }
+
+ bench_stop(&b);
+
+ assert (v.ptr[APPENDS - 1] == APPENDS - 1);
+ free (v.ptr);
+
+ printf ("bench_reserve: %d appends in %.6f s\n", APPENDS, bench_sec (&b));
+}
+
+static void
+bench_append (void)
+{
+ uint32_vector v = empty_vector;
+ struct bench b;
+
+ bench_start(&b);
+
+ for (uint32_t i = 0; i < APPENDS; i++) {
+ uint32_vector_append (&v, i);
+ }
+
+ bench_stop(&b);
+
+ assert (v.ptr[APPENDS - 1] == APPENDS - 1);
+ free (v.ptr);
+
+ printf ("bench_append: %d appends in %.6f s\n", APPENDS, bench_sec (&b));
+}
+
int
main (int argc, char *argv[])
{
- test_int64_vector ();
- test_string_vector ();
+ if (getenv("NBDKIT_BENCH")) {
+ bench_reserve ();
+ bench_append ();
+ } else {
+ test_int64_vector ();
+ test_string_vector ();
+ }
return 0;
}
--
2.31.1

74
SOURCES/0019-vddk-Implement-can_flush-and-can_fua.patch
View File

@ -1,74 +0,0 @@
From 2104686eb708bf87070c21e7af0e70e0317306b6 Mon Sep 17 00:00:00 2001
From: "Richard W.M. Jones" <rjones@redhat.com>
Date: Mon, 5 Jul 2021 21:36:41 +0100
Subject: [PATCH] vddk: Implement can_flush and can_fua
VDDK < 6.0 doesn't support flush. Previously we advertised flush and
FUA but ignored them if VDDK didn't support it. Instead, correctly
set these flags in the NBD protocol according to what VDDK supports.
(cherry picked from commit 04b05274414a8cf4615eb2d6f46d5658814509c1)
---
plugins/vddk/vddk.c | 28 ++++++++++++++++++++--------
1 file changed, 20 insertions(+), 8 deletions(-)
diff --git a/plugins/vddk/vddk.c b/plugins/vddk/vddk.c
index 76faa768..b5bce9a0 100644
--- a/plugins/vddk/vddk.c
+++ b/plugins/vddk/vddk.c
@@ -772,12 +772,28 @@ vddk_pwrite (void *handle, const void *buf, uint32_t count, uint64_t offset,
return -1;
}
- if (fua && vddk_flush (handle, 0) == -1)
- return -1;
+ if (fua) {
+ if (vddk_flush (handle, 0) == -1)
+ return -1;
+ }
return 0;
}
+static int
+vddk_can_fua (void *handle)
+{
+ /* The Flush call was not available in VDDK < 6.0. */
+ return VixDiskLib_Flush != NULL ? NBDKIT_FUA_NATIVE : NBDKIT_FUA_NONE;
+}
+
+static int
+vddk_can_flush (void *handle)
+{
+ /* The Flush call was not available in VDDK < 6.0. */
+ return VixDiskLib_Flush != NULL;
+}
+
/* Flush data to the file. */
static int
vddk_flush (void *handle, uint32_t flags)
@@ -785,12 +801,6 @@ vddk_flush (void *handle, uint32_t flags)
struct vddk_handle *h = handle;
VixError err;
- /* The Flush call was not available in VDDK < 6.0 so this is simply
- * ignored on earlier versions.
- */
- if (VixDiskLib_Flush == NULL)
- return 0;
-
DEBUG_CALL ("VixDiskLib_Flush", "handle");
err = VixDiskLib_Flush (h->handle);
if (err != VIX_OK) {
@@ -985,6 +995,8 @@ static struct nbdkit_plugin plugin = {
.get_size = vddk_get_size,
.pread = vddk_pread,
.pwrite = vddk_pwrite,
+ .can_fua = vddk_can_fua,
+ .can_flush = vddk_can_flush,
.flush = vddk_flush,
.can_extents = vddk_can_extents,
.extents = vddk_extents,
--
2.31.1

54
SOURCES/0020-common-urils-vector.c-Optimize-vector-append.patch Normal file
View File

@ -0,0 +1,54 @@
From 5454ced7c8cfc2ba278c2635eecb9a5e4841e613 Mon Sep 17 00:00:00 2001
From: Nir Soffer <nsoffer@redhat.com>
Date: Fri, 5 Nov 2021 22:16:26 +0200
Subject: [PATCH] common/urils/vector.c: Optimize vector append
Minimize reallocs by growing the backing array by factor of 1.5.
Testing show that now append() is fast without calling reserve()
upfront, simplifying code using vector.
NBDKIT_BENCH=1 ./test-vector
bench_reserve: 1000000 appends in 0.004496 s
bench_append: 1000000 appends in 0.004180 s
This can make a difference in code appending millions of items.
Ported from libnbd commit 985dfa72ae2e41901f0af21e7205ef85428cd4bd.
(cherry picked from commit 12356fa97a840de19bb61e0abedd6e7c7e578e5a)
---
common/utils/vector.c | 14 ++++++++++++--
1 file changed, 12 insertions(+), 2 deletions(-)
diff --git a/common/utils/vector.c b/common/utils/vector.c
index 00cd2546..7df17e1b 100644
--- a/common/utils/vector.c
+++ b/common/utils/vector.c
@@ -41,11 +41,21 @@ int
generic_vector_reserve (struct generic_vector *v, size_t n, size_t itemsize)
{
void *newptr;
+ size_t reqalloc, newalloc;
- newptr = realloc (v->ptr, (n + v->alloc) * itemsize);
+ reqalloc = v->alloc + n;
+ if (reqalloc < v->alloc)
+ return -1; /* overflow */
+
+ newalloc = (v->alloc * 3 + 1) / 2;
+
+ if (newalloc < reqalloc)
+ newalloc = reqalloc;
+
+ newptr = realloc (v->ptr, newalloc * itemsize);
if (newptr == NULL)
return -1;
v->ptr = newptr;
- v->alloc += n;
+ v->alloc = newalloc;
return 0;
}
--
2.31.1

319
SOURCES/0020-vddk-Replace-DEBUG_CALL-with-bracketed-VDDK_CALL_STA.patch
View File

@ -1,319 +0,0 @@
From 51713e7702d389fd55d5721c4773fca40e3e89f6 Mon Sep 17 00:00:00 2001
From: "Richard W.M. Jones" <rjones@redhat.com>
Date: Wed, 18 Aug 2021 14:26:30 +0100
Subject: [PATCH] vddk: Replace DEBUG_CALL with bracketed VDDK_CALL_START/END
macros
This is neutral refactoring, but allows us in the next commit to
collect statistics about the amount of time spent in these calls.
(cherry picked from commit 1335ebfb5637bf5a44403d0b152da7272fdd3e54)
---
plugins/vddk/vddk.c | 175 +++++++++++++++++++++++++-------------------
1 file changed, 99 insertions(+), 76 deletions(-)
diff --git a/plugins/vddk/vddk.c b/plugins/vddk/vddk.c
index b5bce9a0..888009ab 100644
--- a/plugins/vddk/vddk.c
+++ b/plugins/vddk/vddk.c
@@ -104,19 +104,23 @@ static bool is_remote;
VixDiskLib_FreeErrorText (vddk_err_msg); \
} while (0)
-#define DEBUG_CALL(fn, fs, ...) \
- nbdkit_debug ("VDDK call: %s (" fs ")", fn, ##__VA_ARGS__)
-#define DEBUG_CALL_DATAPATH(fn, fs, ...) \
- if (vddk_debug_datapath) \
- nbdkit_debug ("VDDK call: %s (" fs ")", fn, ##__VA_ARGS__)
+#define VDDK_CALL_START(fn, fs, ...) \
+ nbdkit_debug ("VDDK call: %s (" fs ")", #fn, ##__VA_ARGS__); \
+ do
+#define VDDK_CALL_START_DATAPATH(fn, fs, ...) \
+ if (vddk_debug_datapath) \
+ nbdkit_debug ("VDDK call: %s (" fs ")", #fn, ##__VA_ARGS__); \
+ do
+#define VDDK_CALL_END(fn) while (0)
/* Unload the plugin. */
static void
vddk_unload (void)
{
if (init_called) {
- DEBUG_CALL ("VixDiskLib_Exit", "");
- VixDiskLib_Exit ();
+ VDDK_CALL_START (VixDiskLib_Exit, "") {
+ VixDiskLib_Exit ();
+ } VDDK_CALL_END (VixDiskLib_Exit);
}
if (dl)
dlclose (dl);
@@ -449,15 +453,16 @@ vddk_after_fork (void)
VixError err;
/* Initialize VDDK library. */
- DEBUG_CALL ("VixDiskLib_InitEx",
- "%d, %d, &debug_fn, &error_fn, &error_fn, %s, %s",
- VDDK_MAJOR, VDDK_MINOR,
- libdir, config ? : "NULL");
- err = VixDiskLib_InitEx (VDDK_MAJOR, VDDK_MINOR,
- &debug_function, /* log function */
- &error_function, /* warn function */
- &error_function, /* panic function */
- libdir, config);
+ VDDK_CALL_START (VixDiskLib_InitEx,
+ "%d, %d, &debug_fn, &error_fn, &error_fn, %s, %s",
+ VDDK_MAJOR, VDDK_MINOR,
+ libdir, config ? : "NULL") {
+ err = VixDiskLib_InitEx (VDDK_MAJOR, VDDK_MINOR,
+ &debug_function, /* log function */
+ &error_function, /* warn function */
+ &error_function, /* panic function */
+ libdir, config);
+ } VDDK_CALL_END (VixDiskLib_InitEx);
if (err != VIX_OK) {
VDDK_ERROR (err, "VixDiskLib_InitEx");
exit (EXIT_FAILURE);
@@ -519,8 +524,9 @@ allocate_connect_params (void)
VixDiskLibConnectParams *ret;
if (VixDiskLib_AllocateConnectParams != NULL) {
- DEBUG_CALL ("VixDiskLib_AllocateConnectParams", "");
- ret = VixDiskLib_AllocateConnectParams ();
+ VDDK_CALL_START (VixDiskLib_AllocateConnectParams, "") {
+ ret = VixDiskLib_AllocateConnectParams ();
+ } VDDK_CALL_END (VixDiskLib_AllocateConnectParams);
}
else
ret = calloc (1, sizeof (VixDiskLibConnectParams));
@@ -535,8 +541,9 @@ free_connect_params (VixDiskLibConnectParams *params)
* originally called. Otherwise use free.
*/
if (VixDiskLib_AllocateConnectParams != NULL) {
- DEBUG_CALL ("VixDiskLib_FreeConnectParams", "params");
- VixDiskLib_FreeConnectParams (params);
+ VDDK_CALL_START (VixDiskLib_FreeConnectParams, "params") {
+ VixDiskLib_FreeConnectParams (params);
+ } VDDK_CALL_END (VixDiskLib_FreeConnectParams);
}
else
free (params);
@@ -589,16 +596,17 @@ vddk_open (int readonly)
* either ESXi or vCenter servers.
*/
- DEBUG_CALL ("VixDiskLib_ConnectEx",
- "h->params, %d, %s, %s, &connection",
- readonly,
- snapshot_moref ? : "NULL",
- transport_modes ? : "NULL");
- err = VixDiskLib_ConnectEx (h->params,
- readonly,
- snapshot_moref,
- transport_modes,
- &h->connection);
+ VDDK_CALL_START (VixDiskLib_ConnectEx,
+ "h->params, %d, %s, %s, &connection",
+ readonly,
+ snapshot_moref ? : "NULL",
+ transport_modes ? : "NULL") {
+ err = VixDiskLib_ConnectEx (h->params,
+ readonly,
+ snapshot_moref,
+ transport_modes,
+ &h->connection);
+ } VDDK_CALL_END (VixDiskLib_ConnectEx);
if (err != VIX_OK) {
VDDK_ERROR (err, "VixDiskLib_ConnectEx");
goto err1;
@@ -618,9 +626,10 @@ vddk_open (int readonly)
case NONE: break;
}
- DEBUG_CALL ("VixDiskLib_Open",
- "connection, %s, %d, &handle", filename, flags);
- err = VixDiskLib_Open (h->connection, filename, flags, &h->handle);
+ VDDK_CALL_START (VixDiskLib_Open,
+ "connection, %s, %d, &handle", filename, flags) {
+ err = VixDiskLib_Open (h->connection, filename, flags, &h->handle);
+ } VDDK_CALL_END (VixDiskLib_Open);
if (err != VIX_OK) {
VDDK_ERROR (err, "VixDiskLib_Open: %s", filename);
goto err2;
@@ -632,8 +641,9 @@ vddk_open (int readonly)
return h;
err2:
- DEBUG_CALL ("VixDiskLib_Disconnect", "connection");
- VixDiskLib_Disconnect (h->connection);
+ VDDK_CALL_START (VixDiskLib_Disconnect, "connection") {
+ VixDiskLib_Disconnect (h->connection);
+ } VDDK_CALL_END (VixDiskLib_Disconnect);
err1:
free_connect_params (h->params);
err0:
@@ -648,10 +658,13 @@ vddk_close (void *handle)
ACQUIRE_LOCK_FOR_CURRENT_SCOPE (&open_close_lock);
struct vddk_handle *h = handle;
- DEBUG_CALL ("VixDiskLib_Close", "handle");
- VixDiskLib_Close (h->handle);
- DEBUG_CALL ("VixDiskLib_Disconnect", "connection");
- VixDiskLib_Disconnect (h->connection);
+ VDDK_CALL_START (VixDiskLib_Close, "handle") {
+ VixDiskLib_Close (h->handle);
+ } VDDK_CALL_END (VixDiskLib_Close);
+ VDDK_CALL_START (VixDiskLib_Disconnect, "connection") {
+ VixDiskLib_Disconnect (h->connection);
+ } VDDK_CALL_END (VixDiskLib_Disconnect);
+
free_connect_params (h->params);
free (h);
}
@@ -665,8 +678,9 @@ vddk_get_size (void *handle)
VixError err;
uint64_t size;
- DEBUG_CALL ("VixDiskLib_GetInfo", "handle, &info");
- err = VixDiskLib_GetInfo (h->handle, &info);
+ VDDK_CALL_START (VixDiskLib_GetInfo, "handle, &info") {
+ err = VixDiskLib_GetInfo (h->handle, &info);
+ } VDDK_CALL_END (VixDiskLib_GetInfo);
if (err != VIX_OK) {
VDDK_ERROR (err, "VixDiskLib_GetInfo");
return -1;
@@ -694,8 +708,9 @@ vddk_get_size (void *handle)
info->uuid ? : "NULL");
}
- DEBUG_CALL ("VixDiskLib_FreeInfo", "info");
- VixDiskLib_FreeInfo (info);
+ VDDK_CALL_START (VixDiskLib_FreeInfo, "info") {
+ VixDiskLib_FreeInfo (info);
+ } VDDK_CALL_END (VixDiskLib_FreeInfo);
return (int64_t) size;
}
@@ -723,11 +738,12 @@ vddk_pread (void *handle, void *buf, uint32_t count, uint64_t offset,
offset /= VIXDISKLIB_SECTOR_SIZE;
count /= VIXDISKLIB_SECTOR_SIZE;
- DEBUG_CALL_DATAPATH ("VixDiskLib_Read",
- "handle, %" PRIu64 " sectors, "
- "%" PRIu32 " sectors, buffer",
- offset, count);
- err = VixDiskLib_Read (h->handle, offset, count, buf);
+ VDDK_CALL_START_DATAPATH (VixDiskLib_Read,
+ "handle, %" PRIu64 " sectors, "
+ "%" PRIu32 " sectors, buffer",
+ offset, count) {
+ err = VixDiskLib_Read (h->handle, offset, count, buf);
+ } VDDK_CALL_END (VixDiskLib_Read);
if (err != VIX_OK) {
VDDK_ERROR (err, "VixDiskLib_Read");
return -1;
@@ -762,11 +778,12 @@ vddk_pwrite (void *handle, const void *buf, uint32_t count, uint64_t offset,
offset /= VIXDISKLIB_SECTOR_SIZE;
count /= VIXDISKLIB_SECTOR_SIZE;
- DEBUG_CALL_DATAPATH ("VixDiskLib_Write",
- "handle, %" PRIu64 " sectors, "
- "%" PRIu32 " sectors, buffer",
- offset, count);
- err = VixDiskLib_Write (h->handle, offset, count, buf);
+ VDDK_CALL_START_DATAPATH (VixDiskLib_Write,
+ "handle, %" PRIu64 " sectors, "
+ "%" PRIu32 " sectors, buffer",
+ offset, count) {
+ err = VixDiskLib_Write (h->handle, offset, count, buf);
+ } VDDK_CALL_END (VixDiskLib_Write);
if (err != VIX_OK) {
VDDK_ERROR (err, "VixDiskLib_Write");
return -1;
@@ -801,8 +818,9 @@ vddk_flush (void *handle, uint32_t flags)
struct vddk_handle *h = handle;
VixError err;
- DEBUG_CALL ("VixDiskLib_Flush", "handle");
- err = VixDiskLib_Flush (h->handle);
+ VDDK_CALL_START (VixDiskLib_Flush, "handle") {
+ err = VixDiskLib_Flush (h->handle);
+ } VDDK_CALL_END (VixDiskLib_Flush);
if (err != VIX_OK) {
VDDK_ERROR (err, "VixDiskLib_Flush");
return -1;
@@ -836,17 +854,19 @@ vddk_can_extents (void *handle)
* the best thing we can do here is to try the call and if it's
* non-functional return false.
*/
- DEBUG_CALL ("VixDiskLib_QueryAllocatedBlocks",
- "handle, 0, %d sectors, %d sectors",
- VIXDISKLIB_MIN_CHUNK_SIZE, VIXDISKLIB_MIN_CHUNK_SIZE);
- err = VixDiskLib_QueryAllocatedBlocks (h->handle,
- 0, VIXDISKLIB_MIN_CHUNK_SIZE,
- VIXDISKLIB_MIN_CHUNK_SIZE,
- &block_list);
+ VDDK_CALL_START (VixDiskLib_QueryAllocatedBlocks,
+ "handle, 0, %d sectors, %d sectors",
+ VIXDISKLIB_MIN_CHUNK_SIZE, VIXDISKLIB_MIN_CHUNK_SIZE) {
+ err = VixDiskLib_QueryAllocatedBlocks (h->handle,
+ 0, VIXDISKLIB_MIN_CHUNK_SIZE,
+ VIXDISKLIB_MIN_CHUNK_SIZE,
+ &block_list);
+ } VDDK_CALL_END (VixDiskLib_QueryAllocatedBlocks);
error_suppression = 0;
if (err == VIX_OK) {
- DEBUG_CALL ("VixDiskLib_FreeBlockList", "block_list");
- VixDiskLib_FreeBlockList (block_list);
+ VDDK_CALL_START (VixDiskLib_FreeBlockList, "block_list") {
+ VixDiskLib_FreeBlockList (block_list);
+ } VDDK_CALL_END (VixDiskLib_FreeBlockList);
}
if (err != VIX_OK) {
char *errmsg = VixDiskLib_GetErrorText (err, NULL);
@@ -923,14 +943,15 @@ vddk_extents (void *handle, uint32_t count, uint64_t offset, uint32_t flags,
nr_chunks = MIN (nr_chunks, VIXDISKLIB_MAX_CHUNK_NUMBER);
nr_sectors = nr_chunks * VIXDISKLIB_MIN_CHUNK_SIZE;
- DEBUG_CALL ("VixDiskLib_QueryAllocatedBlocks",
- "handle, %" PRIu64 " sectors, %" PRIu64 " sectors, "
- "%d sectors",
- start_sector, nr_sectors, VIXDISKLIB_MIN_CHUNK_SIZE);
- err = VixDiskLib_QueryAllocatedBlocks (h->handle,
- start_sector, nr_sectors,
- VIXDISKLIB_MIN_CHUNK_SIZE,
- &block_list);
+ VDDK_CALL_START (VixDiskLib_QueryAllocatedBlocks,
+ "handle, %" PRIu64 " sectors, %" PRIu64 " sectors, "
+ "%d sectors",
+ start_sector, nr_sectors, VIXDISKLIB_MIN_CHUNK_SIZE) {
+ err = VixDiskLib_QueryAllocatedBlocks (h->handle,
+ start_sector, nr_sectors,
+ VIXDISKLIB_MIN_CHUNK_SIZE,
+ &block_list);
+ } VDDK_CALL_END (VixDiskLib_QueryAllocatedBlocks);
if (err != VIX_OK) {
VDDK_ERROR (err, "VixDiskLib_QueryAllocatedBlocks");
return -1;
@@ -949,13 +970,15 @@ vddk_extents (void *handle, uint32_t count, uint64_t offset, uint32_t flags,
add_extent (extents, &position, blk_offset, true) == -1) ||
(add_extent (extents,
&position, blk_offset + blk_length, false) == -1)) {
- DEBUG_CALL ("VixDiskLib_FreeBlockList", "block_list");
- VixDiskLib_FreeBlockList (block_list);
+ VDDK_CALL_START (VixDiskLib_FreeBlockList, "block_list") {
+ VixDiskLib_FreeBlockList (block_list);
+ } VDDK_CALL_END (VixDiskLib_FreeBlockList);
return -1;
}
}
- DEBUG_CALL ("VixDiskLib_FreeBlockList", "block_list");
- VixDiskLib_FreeBlockList (block_list);
+ VDDK_CALL_START (VixDiskLib_FreeBlockList, "block_list") {
+ VixDiskLib_FreeBlockList (block_list);
+ } VDDK_CALL_END (VixDiskLib_FreeBlockList);
/* There's an implicit hole after the returned list of blocks, up
* to the end of the QueryAllocatedBlocks request.
--
2.31.1

188
SOURCES/0021-common-utils-vector-Rename-alloc-to-cap.patch Normal file
View File

@ -0,0 +1,188 @@
From 304f180b61fa28421b9901d2173a280e633b55c2 Mon Sep 17 00:00:00 2001
From: Nir Soffer <nsoffer@redhat.com>
Date: Fri, 5 Nov 2021 22:59:38 +0200
Subject: [PATCH] common/utils/vector: Rename `alloc` to `cap`
The `alloc` field is the maximum number of items you can append to a
vector before it need to be resized. This may confuse users with the
size of the `ptr` array which is `alloc * itemsize`. Rename to "cap",
common term for this property in many languages (e.g C++, Rust, Go).
Tested with "make check". Tests requiring root or external libraries
(vddk) not tested.
Ported from libnbd commit e3c7f02a2a844295564c832108d36c939c4e4ecf.
(cherry picked from commit 75a44237c4463524dbf7951bb62b59c373c85865)
---
common/allocators/malloc.c | 24 ++++++++++++------------
common/utils/vector.c | 16 ++++++++--------
common/utils/vector.h | 12 ++++++------
plugins/vddk/reexec.c | 2 +-
4 files changed, 27 insertions(+), 27 deletions(-)
diff --git a/common/allocators/malloc.c b/common/allocators/malloc.c
index 59409c24..f7474465 100644
--- a/common/allocators/malloc.c
+++ b/common/allocators/malloc.c
@@ -88,16 +88,16 @@ extend (struct m_alloc *ma, uint64_t new_size)
ACQUIRE_WRLOCK_FOR_CURRENT_SCOPE (&ma->lock);
size_t old_size, n;
- if (ma->ba.alloc < new_size) {
- old_size = ma->ba.alloc;
- n = new_size - ma->ba.alloc;
+ if (ma->ba.cap < new_size) {
+ old_size = ma->ba.cap;
+ n = new_size - ma->ba.cap;
#ifdef HAVE_MUNLOCK
/* Since the memory might be moved by realloc, we must unlock the
* original array.
*/
if (ma->use_mlock)
- munlock (ma->ba.ptr, ma->ba.alloc);
+ munlock (ma->ba.ptr, ma->ba.cap);
#endif
if (bytearray_reserve (&ma->ba, n) == -1) {
@@ -110,7 +110,7 @@ extend (struct m_alloc *ma, uint64_t new_size)
#ifdef HAVE_MLOCK
if (ma->use_mlock) {
- if (mlock (ma->ba.ptr, ma->ba.alloc) == -1) {
+ if (mlock (ma->ba.ptr, ma->ba.cap) == -1) {
nbdkit_error ("allocator=malloc: mlock: %m");
return -1;
}
@@ -138,11 +138,11 @@ m_alloc_read (struct allocator *a, void *buf,
/* Avoid reading beyond the end of the allocated array. Return
* zeroes for that part.
*/
- if (offset >= ma->ba.alloc)
+ if (offset >= ma->ba.cap)
memset (buf, 0, count);
- else if (offset + count > ma->ba.alloc) {
- memcpy (buf, ma->ba.ptr + offset, ma->ba.alloc - offset);
- memset (buf + ma->ba.alloc - offset, 0, offset + count - ma->ba.alloc);
+ else if (offset + count > ma->ba.cap) {
+ memcpy (buf, ma->ba.ptr + offset, ma->ba.cap - offset);
+ memset (buf + ma->ba.cap - offset, 0, offset + count - ma->ba.cap);
}
else
memcpy (buf, ma->ba.ptr + offset, count);
@@ -191,9 +191,9 @@ m_alloc_zero (struct allocator *a, uint64_t count, uint64_t offset)
/* Try to avoid extending the array, since the unallocated part
* always reads as zero.
*/
- if (offset < ma->ba.alloc) {
- if (offset + count > ma->ba.alloc)
- memset (ma->ba.ptr + offset, 0, ma->ba.alloc - offset);
+ if (offset < ma->ba.cap) {
+ if (offset + count > ma->ba.cap)
+ memset (ma->ba.ptr + offset, 0, ma->ba.cap - offset);
else
memset (ma->ba.ptr + offset, 0, count);
}
diff --git a/common/utils/vector.c b/common/utils/vector.c
index 7df17e1b..a4b43ce7 100644
--- a/common/utils/vector.c
+++ b/common/utils/vector.c
@@ -41,21 +41,21 @@ int
generic_vector_reserve (struct generic_vector *v, size_t n, size_t itemsize)
{
void *newptr;
- size_t reqalloc, newalloc;
+ size_t reqcap, newcap;
- reqalloc = v->alloc + n;
- if (reqalloc < v->alloc)
+ reqcap = v->cap + n;
+ if (reqcap < v->cap)
return -1; /* overflow */
- newalloc = (v->alloc * 3 + 1) / 2;
+ newcap = (v->cap * 3 + 1) / 2;
- if (newalloc < reqalloc)
- newalloc = reqalloc;
+ if (newcap < reqcap)
+ newcap = reqcap;
- newptr = realloc (v->ptr, newalloc * itemsize);
+ newptr = realloc (v->ptr, newcap * itemsize);
if (newptr == NULL)
return -1;
v->ptr = newptr;
- v->alloc = newalloc;
+ v->cap = newcap;
return 0;
}
diff --git a/common/utils/vector.h b/common/utils/vector.h
index f6a0af78..782dcba6 100644
--- a/common/utils/vector.h
+++ b/common/utils/vector.h
@@ -86,7 +86,7 @@
struct name { \
type *ptr; /* Pointer to array of items. */ \
size_t size; /* Number of valid items in the array. */ \
- size_t alloc; /* Number of items allocated. */ \
+ size_t cap; /* Maximum number of items. */ \
}; \
typedef struct name name; \
\
@@ -106,7 +106,7 @@
name##_insert (name *v, type elem, size_t i) \
{ \
assert (i <= v->size); \
- if (v->size >= v->alloc) { \
+ if (v->size >= v->cap) { \
if (name##_reserve (v, 1) == -1) return -1; \
} \
memmove (&v->ptr[i+1], &v->ptr[i], (v->size-i) * sizeof (elem)); \
@@ -137,7 +137,7 @@
{ \
free (v->ptr); \
v->ptr = NULL; \
- v->size = v->alloc = 0; \
+ v->size = v->cap = 0; \
} \
\
/* Iterate over the vector, calling f() on each element. */ \
@@ -181,17 +181,17 @@
if (newptr == NULL) return -1; \
memcpy (newptr, vptr, len); \
copy->ptr = newptr; \
- copy->size = copy->alloc = v->size; \
+ copy->size = copy->cap = v->size; \
return 0; \
} \
\
-#define empty_vector { .ptr = NULL, .size = 0, .alloc = 0 }
+#define empty_vector { .ptr = NULL, .size = 0, .cap = 0 }
struct generic_vector {
void *ptr;
size_t size;
- size_t alloc;
+ size_t cap;
};
extern int generic_vector_reserve (struct generic_vector *v,
diff --git a/plugins/vddk/reexec.c b/plugins/vddk/reexec.c
index 46acdb62..9e87025e 100644
--- a/plugins/vddk/reexec.c
+++ b/plugins/vddk/reexec.c
@@ -116,7 +116,7 @@ perform_reexec (const char *env, const char *prepend)
nbdkit_error ("realloc: %m");
exit (EXIT_FAILURE);
}
- r = read (fd, buf.ptr + buf.size, buf.alloc - buf.size);
+ r = read (fd, buf.ptr + buf.size, buf.cap - buf.size);
if (r == -1) {
nbdkit_error ("read: %s: %m", cmdline_file);
exit (EXIT_FAILURE);
--
2.31.1

103
SOURCES/0021-tests-Add-a-better-test-of-real-VDDK.patch
View File

@ -1,103 +0,0 @@
From bd181ea739ebfafbf7239b5fa89e98becdb8cb72 Mon Sep 17 00:00:00 2001
From: "Richard W.M. Jones" <rjones@redhat.com>
Date: Mon, 5 Jul 2021 22:03:10 +0100
Subject: [PATCH] tests: Add a better test of real VDDK
The previous test only tested reading and maybe extents, and used an
all-zero disk. I'm fairly convinced the test only worked accidentally
since you must use an absolute path when opening a local file and the
test did not do that.
Add a more comprehensive test that tests writing and flush too.
(cherry picked from commit a6ca4f24593008bb2d8efb177e7f424cff51dfbf)
---
tests/test-vddk-real.sh | 55 ++++++++++++++++++++++++++++-------------
1 file changed, 38 insertions(+), 17 deletions(-)
diff --git a/tests/test-vddk-real.sh b/tests/test-vddk-real.sh
index df486ba1..f848db44 100755
--- a/tests/test-vddk-real.sh
+++ b/tests/test-vddk-real.sh
@@ -37,8 +37,12 @@ set -x
requires test "x$vddkdir" != "x"
requires test -d "$vddkdir"
requires test -f "$vddkdir/lib64/libvixDiskLib.so"
+requires test -r /dev/urandom
+requires cmp --version
+requires dd --version
requires qemu-img --version
requires nbdcopy --version
+requires nbdinfo --version
requires stat --version
# VDDK > 5.1.1 only supports x86_64.
@@ -47,31 +51,48 @@ if [ `uname -m` != "x86_64" ]; then
exit 77
fi
-files="test-vddk-real.vmdk test-vddk-real.out test-vddk-real.log"
-rm -f $files
-cleanup_fn rm -f $files
-
-qemu-img create -f vmdk test-vddk-real.vmdk 100M
-
# Since we are comparing error messages below, let's make sure we're
# not translating errors.
export LANG=C
-fail=0
-nbdkit -f -v -U - \
- --filter=readahead \
- vddk libdir="$vddkdir" test-vddk-real.vmdk \
- --run 'nbdcopy "$uri" test-vddk-real.out' \
- > test-vddk-real.log 2>&1 || fail=1
+pid=test-vddk-real.pid
+sock=$(mktemp -u /tmp/nbdkit-test-sock.XXXXXX)
+vmdk=$PWD/test-vddk-real.vmdk ;# note must be an absolute path
+raw=test-vddk-real.raw
+raw2=test-vddk-real.raw2
+log=test-vddk-real.log
+files="$pid $sock $vmdk $raw $raw2 $log"
+rm -f $files
+cleanup_fn rm -f $files
+
+qemu-img create -f vmdk $vmdk 10M
+
+# Check first that the VDDK library can be fully loaded. We have to
+# check the log file for missing modules since they may not show up as
+# errors.
+nbdkit -fv -U - vddk libdir="$vddkdir" $vmdk --run 'nbdinfo "$uri"' >$log 2>&1
# Check the log for missing modules
-cat test-vddk-real.log
+cat $log
if grep 'cannot open shared object file' test-vddk-real.log; then
exit 1
fi
-# Check the raw output file has exactly the right size.
-size="$(stat -c '%s' test-vddk-real.out)"
-test "$size" -eq $((100 * 1024 * 1024))
+# Now run nbdkit for the test.
+start_nbdkit -P $pid -U $sock vddk libdir="$vddkdir" $vmdk
+uri="nbd+unix:///?socket=$sock"
-exit $fail
+# VDDK < 6.0 did not support flush, so disable flush test there. Also
+# if nbdinfo doesn't support the --can flush syntax (added in libnbd
+# 1.10) then this is disabled.
+if nbdinfo --can flush "$uri"; then flush="--flush"; else flush=""; fi
+
+# Copy in and out some data. This should exercise read, write,
+# extents and flushing.
+dd if=/dev/urandom of=$raw count=5 bs=$((1024*1024))
+truncate -s 10M $raw
+
+nbdcopy $flush $raw "$uri"
+nbdcopy "$uri" $raw2
+
+cmp $raw $raw2
--
2.31.1

1747
SOURCES/0022-common-utils-vector-Rename-size-to-len.patch Normal file
View File

File diff suppressed because it is too large Load Diff

245
SOURCES/0022-vddk-Add-stats-about-the-amount-of-time-spent-in-VDD.patch
View File

@ -1,245 +0,0 @@
From 45db64d72bf03fece8a7fb994887360954905a3b Mon Sep 17 00:00:00 2001
From: "Richard W.M. Jones" <rjones@redhat.com>
Date: Wed, 18 Aug 2021 14:47:58 +0100
Subject: [PATCH] vddk: Add stats about the amount of time spent in VDDK calls
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
If you use -D vddk.stats=1 then when the plugin is unloaded it will
display the amount of time spent in each VDDK call. The output will
look something like this:
nbdkit: debug: VDDK function stats (-D vddk.stats=1):
nbdkit: debug: µs
nbdkit: debug: VixDiskLib_Exit 1001154
nbdkit: debug: VixDiskLib_InitEx 96008
nbdkit: debug: VixDiskLib_Flush 15722
nbdkit: debug: VixDiskLib_Write 12081
nbdkit: debug: VixDiskLib_Open 6029
nbdkit: debug: VixDiskLib_Read 1364
nbdkit: debug: VixDiskLib_Close 605
nbdkit: debug: VixDiskLib_QueryAllocatedBlocks 191
nbdkit: debug: VixDiskLib_ConnectEx 134
nbdkit: debug: VixDiskLib_Disconnect 76
nbdkit: debug: VixDiskLib_FreeConnectParams 57
nbdkit: debug: VixDiskLib_GetInfo 56
nbdkit: debug: VixDiskLib_GetTransportMode 43
nbdkit: debug: VixDiskLib_FreeInfo 42
nbdkit: debug: VixDiskLib_FreeBlockList 32
nbdkit: debug: VixDiskLib_AllocateConnectParams 28
VDDK APIs which are never called are not printed.
(cherry picked from commit f2dfc7d74ee650bdf2cc930a07b1c5bcb509976c)
---
plugins/vddk/nbdkit-vddk-plugin.pod | 5 ++
plugins/vddk/vddk.c | 107 +++++++++++++++++++++++++---
tests/test-vddk-real.sh | 2 +-
3 files changed, 103 insertions(+), 11 deletions(-)
diff --git a/plugins/vddk/nbdkit-vddk-plugin.pod b/plugins/vddk/nbdkit-vddk-plugin.pod
index b783f13a..2a1b17dc 100644
--- a/plugins/vddk/nbdkit-vddk-plugin.pod
+++ b/plugins/vddk/nbdkit-vddk-plugin.pod
@@ -510,6 +510,11 @@ Debug extents returned by C<QueryAllocatedBlocks>.
Suppress debugging of datapath calls (C<Read> and C<Write>).
+=item B<-D vddk.stats=1>
+
+When the plugin exits print some statistics about the amount of time
+spent waiting on each VDDK call.
+
=back
=head1 FILES
diff --git a/plugins/vddk/vddk.c b/plugins/vddk/vddk.c
index 888009ab..fce96d9a 100644
--- a/plugins/vddk/vddk.c
+++ b/plugins/vddk/vddk.c
@@ -42,6 +42,7 @@
#include <assert.h>
#include <dlfcn.h>
#include <libgen.h>
+#include <sys/time.h>
#include <pthread.h>
@@ -52,6 +53,8 @@
#include "isaligned.h"
#include "minmax.h"
#include "rounding.h"
+#include "tvdiff.h"
+#include "vector.h"
#include "vddk.h"
#include "vddk-structs.h"
@@ -60,6 +63,7 @@
NBDKIT_DLL_PUBLIC int vddk_debug_diskinfo;
NBDKIT_DLL_PUBLIC int vddk_debug_extents;
NBDKIT_DLL_PUBLIC int vddk_debug_datapath = 1;
+NBDKIT_DLL_PUBLIC int vddk_debug_stats;
/* For each VDDK API define a static global variable. These globals
* are initialized when the plugin is loaded (by vddk_get_ready).
@@ -96,22 +100,52 @@ static const char *username; /* user */
static const char *vmx_spec; /* vm */
static bool is_remote;
-#define VDDK_ERROR(err, fs, ...) \
- do { \
- char *vddk_err_msg; \
- vddk_err_msg = VixDiskLib_GetErrorText ((err), NULL); \
- nbdkit_error (fs ": %s", ##__VA_ARGS__, vddk_err_msg); \
- VixDiskLib_FreeErrorText (vddk_err_msg); \
- } while (0)
+/* For each VDDK API define a variable to store the time taken (used
+ * to implement -D vddk.stats=1).
+ */
+static pthread_mutex_t stats_lock = PTHREAD_MUTEX_INITIALIZER;
+static void display_stats (void);
+#define STUB(fn,ret,args) static int64_t stats_##fn;
+#define OPTIONAL_STUB(fn,ret,args) static int64_t stats_##fn;
+#include "vddk-stubs.h"
+#undef STUB
+#undef OPTIONAL_STUB
#define VDDK_CALL_START(fn, fs, ...) \
+ do { \
+ struct timeval start_t, end_t; \
+ if (vddk_debug_stats) \
+ gettimeofday (&start_t, NULL); \
nbdkit_debug ("VDDK call: %s (" fs ")", #fn, ##__VA_ARGS__); \
do
#define VDDK_CALL_START_DATAPATH(fn, fs, ...) \
+ do { \
+ struct timeval start_t, end_t; \
+ if (vddk_debug_stats) \
+ gettimeofday (&start_t, NULL); \
if (vddk_debug_datapath) \
nbdkit_debug ("VDDK call: %s (" fs ")", #fn, ##__VA_ARGS__); \
do
-#define VDDK_CALL_END(fn) while (0)
+#define VDDK_CALL_END(fn) \
+ while (0); \
+ if (vddk_debug_stats) { \
+ gettimeofday (&end_t, NULL); \
+ ACQUIRE_LOCK_FOR_CURRENT_SCOPE (&stats_lock); \
+ stats_##fn += tvdiff_usec (&start_t, &end_t); \
+ } \
+ } while (0)
+
+#define VDDK_ERROR(err, fs, ...) \
+ do { \
+ char *vddk_err_msg; \
+ VDDK_CALL_START (VixDiskLib_GetErrorText, "%lu", err) { \
+ vddk_err_msg = VixDiskLib_GetErrorText ((err), NULL); \
+ } VDDK_CALL_END (VixDiskLib_GetErrorText); \
+ nbdkit_error (fs ": %s", ##__VA_ARGS__, vddk_err_msg); \
+ VDDK_CALL_START (VixDiskLib_FreeErrorText, "") { \
+ VixDiskLib_FreeErrorText (vddk_err_msg); \
+ } VDDK_CALL_END (VixDiskLib_FreeErrorText); \
+ } while (0)
/* Unload the plugin. */
static void
@@ -124,11 +158,61 @@ vddk_unload (void)
}
if (dl)
dlclose (dl);
+
+ if (vddk_debug_stats)
+ display_stats ();
+
free (config);
free (libdir);
free (password);
}
+struct vddk_stat {
+ const char *fn;
+ int64_t usecs;
+};
+DEFINE_VECTOR_TYPE(statlist, struct vddk_stat)
+
+static int
+stat_compare (const void *vp1, const void *vp2)
+{
+ const struct vddk_stat *st1 = vp1;
+ const struct vddk_stat *st2 = vp2;
+
+ /* Note: sorts in reverse order. */
+ if (st1->usecs < st2->usecs) return 1;
+ else if (st1->usecs > st2->usecs) return -1;
+ else return 0;
+}
+
+static void
+display_stats (void)
+{
+ statlist stats = empty_vector;
+ struct vddk_stat st;
+ size_t i;
+
+#define ADD_ONE_STAT(fn_, usecs_) \
+ st.fn = fn_; \
+ st.usecs = usecs_; \
+ statlist_append (&stats, st)
+#define STUB(fn,ret,args) ADD_ONE_STAT (#fn, stats_##fn);
+#define OPTIONAL_STUB(fn,ret,args) ADD_ONE_STAT (#fn, stats_##fn);
+#include "vddk-stubs.h"
+#undef STUB
+#undef OPTIONAL_STUB
+#undef ADD_ONE_STAT
+
+ qsort (stats.ptr, stats.size, sizeof stats.ptr[0], stat_compare);
+
+ nbdkit_debug ("VDDK function stats (-D vddk.stats=1):");
+ nbdkit_debug ("%-40s %9s", "", "µs");
+ for (i = 0; i < stats.size; ++i) {
+ if (stats.ptr[i].usecs)
+ nbdkit_debug ("%-40s %9" PRIi64, stats.ptr[i].fn, stats.ptr[i].usecs);
+ }
+}
+
static void
trim (char *str)
{
@@ -557,6 +641,7 @@ vddk_open (int readonly)
struct vddk_handle *h;
VixError err;
uint32_t flags;
+ const char *transport_mode;
h = malloc (sizeof *h);
if (h == NULL) {
@@ -635,8 +720,10 @@ vddk_open (int readonly)
goto err2;
}
- nbdkit_debug ("transport mode: %s",
- VixDiskLib_GetTransportMode (h->handle));
+ VDDK_CALL_START (VixDiskLib_GetTransportMode, "handle") {
+ transport_mode = VixDiskLib_GetTransportMode (h->handle);
+ } VDDK_CALL_END (VixDiskLib_GetTransportMode);
+ nbdkit_debug ("transport mode: %s", transport_mode);
return h;
diff --git a/tests/test-vddk-real.sh b/tests/test-vddk-real.sh
index f848db44..3c8b4262 100755
--- a/tests/test-vddk-real.sh
+++ b/tests/test-vddk-real.sh
@@ -79,7 +79,7 @@ if grep 'cannot open shared object file' test-vddk-real.log; then
fi
# Now run nbdkit for the test.
-start_nbdkit -P $pid -U $sock vddk libdir="$vddkdir" $vmdk
+start_nbdkit -P $pid -U $sock -D vddk.stats=1 vddk libdir="$vddkdir" $vmdk
uri="nbd+unix:///?socket=$sock"
# VDDK < 6.0 did not support flush, so disable flush test there. Also
--
2.31.1

772
SOURCES/0023-cow-Make-the-block-size-configurable.patch
View File

@ -1,772 +0,0 @@
From 0be4847cdec9effd6128da03ea42a4953e5a6343 Mon Sep 17 00:00:00 2001
From: "Richard W.M. Jones" <rjones@redhat.com>
Date: Tue, 17 Aug 2021 22:03:11 +0100
Subject: [PATCH] cow: Make the block size configurable
Commit c1905b0a28 ("cache, cow: Use a 64K block size by default")
changed the nbdkit-cow-filter block size to 64K, but it was still a
fixed size. In contrast the cache filter allows the block size to be
adjusted.
Allow the block size in this filter to be adjusted up or down with a
new cow-block-size=N parameter.
When using the VDDK plugin, adjusting this setting can make a
difference. The following timings come from a modified virt-v2v which
sets cow-block-size and was used to convert from a VMware server to
-o null (this is also using cow-on-read=true):
cow-block-size=64K: 18m18
cow-block-size=256K: 14m13
cow-block-size=1M: 14m19
cow-block-size=4M: 37m33
As you can see it's not obvious how to choose a good block size, but
at least by allowing adjustment we can tune things.
(cherry picked from commit 7182c47d04d2b68005fceadefc0c14bfaa61a533)
---
filters/cow/blk.c | 35 +++----
filters/cow/blk.h | 5 -
filters/cow/cow.c | 150 +++++++++++++++++-------------
filters/cow/cow.h | 39 ++++++++
filters/cow/nbdkit-cow-filter.pod | 5 +
tests/Makefile.am | 2 +
tests/test-cow-block-size.sh | 72 ++++++++++++++
7 files changed, 221 insertions(+), 87 deletions(-)
create mode 100644 filters/cow/cow.h
create mode 100755 tests/test-cow-block-size.sh
diff --git a/filters/cow/blk.c b/filters/cow/blk.c
index c22d5886..f9341dc1 100644
--- a/filters/cow/blk.c
+++ b/filters/cow/blk.c
@@ -99,6 +99,7 @@
#include "pwrite.h"
#include "utils.h"
+#include "cow.h"
#include "blk.h"
/* The temporary overlay. */
@@ -137,7 +138,7 @@ blk_init (void)
size_t len;
char *template;
- bitmap_init (&bm, BLKSIZE, 2 /* bits per block */);
+ bitmap_init (&bm, blksize, 2 /* bits per block */);
tmpdir = getenv ("TMPDIR");
if (!tmpdir)
@@ -199,7 +200,7 @@ blk_set_size (uint64_t new_size)
if (bitmap_resize (&bm, size) == -1)
return -1;
- if (ftruncate (fd, ROUND_UP (size, BLKSIZE)) == -1) {
+ if (ftruncate (fd, ROUND_UP (size, blksize)) == -1) {
nbdkit_error ("ftruncate: %m");
return -1;
}
@@ -228,7 +229,7 @@ blk_read_multiple (nbdkit_next *next,
uint64_t blknum, uint64_t nrblocks,
uint8_t *block, bool cow_on_read, int *err)
{
- off_t offset = blknum * BLKSIZE;
+ off_t offset = blknum * blksize;
enum bm_entry state;
uint64_t b, runblocks;
@@ -262,8 +263,8 @@ blk_read_multiple (nbdkit_next *next,
if (state == BLOCK_NOT_ALLOCATED) { /* Read underlying plugin. */
unsigned n, tail = 0;
- assert (BLKSIZE * runblocks <= UINT_MAX);
- n = BLKSIZE * runblocks;
+ assert (blksize * runblocks <= UINT_MAX);
+ n = blksize * runblocks;
if (offset + n > size) {
tail = offset + n - size;
@@ -288,7 +289,7 @@ blk_read_multiple (nbdkit_next *next,
"at offset %" PRIu64 " into the cache",
runblocks, offset);
- if (full_pwrite (fd, block, BLKSIZE * runblocks, offset) == -1) {
+ if (full_pwrite (fd, block, blksize * runblocks, offset) == -1) {
*err = errno;
nbdkit_error ("pwrite: %m");
return -1;
@@ -298,14 +299,14 @@ blk_read_multiple (nbdkit_next *next,
}
}
else if (state == BLOCK_ALLOCATED) { /* Read overlay. */
- if (full_pread (fd, block, BLKSIZE * runblocks, offset) == -1) {
+ if (full_pread (fd, block, blksize * runblocks, offset) == -1) {
*err = errno;
nbdkit_error ("pread: %m");
return -1;
}
}
else /* state == BLOCK_TRIMMED */ {
- memset (block, 0, BLKSIZE * runblocks);
+ memset (block, 0, blksize * runblocks);
}
/* If all done, return. */
@@ -316,7 +317,7 @@ blk_read_multiple (nbdkit_next *next,
return blk_read_multiple (next,
blknum + runblocks,
nrblocks - runblocks,
- block + BLKSIZE * runblocks,
+ block + blksize * runblocks,
cow_on_read, err);
}
@@ -333,9 +334,9 @@ blk_cache (nbdkit_next *next,
{
/* XXX Could make this lock more fine-grained with some thought. */
ACQUIRE_LOCK_FOR_CURRENT_SCOPE (&lock);
- off_t offset = blknum * BLKSIZE;
+ off_t offset = blknum * blksize;
enum bm_entry state = bitmap_get_blk (&bm, blknum, BLOCK_NOT_ALLOCATED);
- unsigned n = BLKSIZE, tail = 0;
+ unsigned n = blksize, tail = 0;
if (offset + n > size) {
tail = offset + n - size;
@@ -348,7 +349,7 @@ blk_cache (nbdkit_next *next,
if (state == BLOCK_ALLOCATED) {
#if HAVE_POSIX_FADVISE
- int r = posix_fadvise (fd, offset, BLKSIZE, POSIX_FADV_WILLNEED);
+ int r = posix_fadvise (fd, offset, blksize, POSIX_FADV_WILLNEED);
if (r) {
errno = r;
nbdkit_error ("posix_fadvise: %m");
@@ -373,7 +374,7 @@ blk_cache (nbdkit_next *next,
memset (block + n, 0, tail);
if (mode == BLK_CACHE_COW) {
- if (full_pwrite (fd, block, BLKSIZE, offset) == -1) {
+ if (full_pwrite (fd, block, blksize, offset) == -1) {
*err = errno;
nbdkit_error ("pwrite: %m");
return -1;
@@ -386,13 +387,13 @@ blk_cache (nbdkit_next *next,
int
blk_write (uint64_t blknum, const uint8_t *block, int *err)
{
- off_t offset = blknum * BLKSIZE;
+ off_t offset = blknum * blksize;
if (cow_debug_verbose)
nbdkit_debug ("cow: blk_write block %" PRIu64 " (offset %" PRIu64 ")",
blknum, (uint64_t) offset);
- if (full_pwrite (fd, block, BLKSIZE, offset) == -1) {
+ if (full_pwrite (fd, block, blksize, offset) == -1) {
*err = errno;
nbdkit_error ("pwrite: %m");
return -1;
@@ -407,14 +408,14 @@ blk_write (uint64_t blknum, const uint8_t *block, int *err)
int
blk_trim (uint64_t blknum, int *err)
{
- off_t offset = blknum * BLKSIZE;
+ off_t offset = blknum * blksize;
if (cow_debug_verbose)
nbdkit_debug ("cow: blk_trim block %" PRIu64 " (offset %" PRIu64 ")",
blknum, (uint64_t) offset);
/* XXX As an optimization we could punch a whole in the overlay
- * here. However it's not trivial since BLKSIZE is unrelated to the
+ * here. However it's not trivial since blksize is unrelated to the
* overlay filesystem block size.
*/
ACQUIRE_LOCK_FOR_CURRENT_SCOPE (&lock);
diff --git a/filters/cow/blk.h b/filters/cow/blk.h
index b7e6f092..62fb5416 100644
--- a/filters/cow/blk.h
+++ b/filters/cow/blk.h
@@ -33,11 +33,6 @@
#ifndef NBDKIT_BLK_H
#define NBDKIT_BLK_H
-/* Size of a block in the overlay. A 4K block size means that we need
- * 64 MB of memory to store the bitmap for a 1 TB underlying image.
- */
-#define BLKSIZE 65536
-
/* Initialize the overlay and bitmap. */
extern int blk_init (void);
diff --git a/filters/cow/cow.c b/filters/cow/cow.c
index 6efb39f2..1c62c857 100644
--- a/filters/cow/cow.c
+++ b/filters/cow/cow.c
@@ -40,6 +40,7 @@
#include <string.h>
#include <unistd.h>
#include <errno.h>
+#include <limits.h>
#include <pthread.h>
@@ -47,9 +48,11 @@
#include "cleanup.h"
#include "isaligned.h"
+#include "ispowerof2.h"
#include "minmax.h"
#include "rounding.h"
+#include "cow.h"
#include "blk.h"
/* Read-modify-write requests are serialized through this global lock.
@@ -58,6 +61,8 @@
*/
static pthread_mutex_t rmw_lock = PTHREAD_MUTEX_INITIALIZER;
+unsigned blksize = 65536; /* block size */
+
static bool cow_on_cache;
/* Cache on read ("cow-on-read") mode. */
@@ -69,13 +74,6 @@ extern enum cor_mode {
enum cor_mode cor_mode = COR_OFF;
const char *cor_path;
-static void
-cow_load (void)
-{
- if (blk_init () == -1)
- exit (EXIT_FAILURE);
-}
-
static void
cow_unload (void)
{
@@ -86,7 +84,19 @@ static int
cow_config (nbdkit_next_config *next, nbdkit_backend *nxdata,
const char *key, const char *value)
{
- if (strcmp (key, "cow-on-cache") == 0) {
+ if (strcmp (key, "cow-block-size") == 0) {
+ int64_t r = nbdkit_parse_size (value);
+ if (r == -1)
+ return -1;
+ if (r <= 4096 || r > UINT_MAX || !is_power_of_2 (r)) {
+ nbdkit_error ("cow-block-size is out of range (4096..2G) "
+ "or not a power of 2");
+ return -1;
+ }
+ blksize = r;
+ return 0;
+ }
+ else if (strcmp (key, "cow-on-cache") == 0) {
int r;
r = nbdkit_parse_bool (value);
@@ -114,9 +124,19 @@ cow_config (nbdkit_next_config *next, nbdkit_backend *nxdata,
}
#define cow_config_help \
+ "cow-block-size=<N> Set COW block size.\n" \
"cow-on-cache=<BOOL> Copy cache (prefetch) requests to the overlay.\n" \
"cow-on-read=<BOOL>|/PATH Copy read requests to the overlay."
+static int
+cow_get_ready (int thread_model)
+{
+ if (blk_init () == -1)
+ return -1;
+
+ return 0;
+}
+
/* Decide if cow-on-read is currently on or off. */
bool
cow_on_read (void)
@@ -249,8 +269,8 @@ cow_pread (nbdkit_next *next,
uint64_t blknum, blkoffs, nrblocks;
int r;
- if (!IS_ALIGNED (count | offset, BLKSIZE)) {
- block = malloc (BLKSIZE);
+ if (!IS_ALIGNED (count | offset, blksize)) {
+ block = malloc (blksize);
if (block == NULL) {
*err = errno;
nbdkit_error ("malloc: %m");
@@ -258,12 +278,12 @@ cow_pread (nbdkit_next *next,
}
}
- blknum = offset / BLKSIZE; /* block number */
- blkoffs = offset % BLKSIZE; /* offset within the block */
+ blknum = offset / blksize; /* block number */
+ blkoffs = offset % blksize; /* offset within the block */
/* Unaligned head */
if (blkoffs) {
- uint64_t n = MIN (BLKSIZE - blkoffs, count);
+ uint64_t n = MIN (blksize - blkoffs, count);
assert (block);
r = blk_read (next, blknum, block, cow_on_read (), err);
@@ -279,15 +299,15 @@ cow_pread (nbdkit_next *next,
}
/* Aligned body */
- nrblocks = count / BLKSIZE;
+ nrblocks = count / blksize;
if (nrblocks > 0) {
r = blk_read_multiple (next, blknum, nrblocks, buf, cow_on_read (), err);
if (r == -1)
return -1;
- buf += nrblocks * BLKSIZE;
- count -= nrblocks * BLKSIZE;
- offset += nrblocks * BLKSIZE;
+ buf += nrblocks * blksize;
+ count -= nrblocks * blksize;
+ offset += nrblocks * blksize;
blknum += nrblocks;
}
@@ -314,8 +334,8 @@ cow_pwrite (nbdkit_next *next,
uint64_t blknum, blkoffs;
int r;
- if (!IS_ALIGNED (count | offset, BLKSIZE)) {
- block = malloc (BLKSIZE);
+ if (!IS_ALIGNED (count | offset, blksize)) {
+ block = malloc (blksize);
if (block == NULL) {
*err = errno;
nbdkit_error ("malloc: %m");
@@ -323,12 +343,12 @@ cow_pwrite (nbdkit_next *next,
}
}
- blknum = offset / BLKSIZE; /* block number */
- blkoffs = offset % BLKSIZE; /* offset within the block */
+ blknum = offset / blksize; /* block number */
+ blkoffs = offset % blksize; /* offset within the block */
/* Unaligned head */
if (blkoffs) {
- uint64_t n = MIN (BLKSIZE - blkoffs, count);
+ uint64_t n = MIN (blksize - blkoffs, count);
/* Do a read-modify-write operation on the current block.
* Hold the rmw_lock over the whole operation.
@@ -350,14 +370,14 @@ cow_pwrite (nbdkit_next *next,
}
/* Aligned body */
- while (count >= BLKSIZE) {
+ while (count >= blksize) {
r = blk_write (blknum, buf, err);
if (r == -1)
return -1;
- buf += BLKSIZE;
- count -= BLKSIZE;
- offset += BLKSIZE;
+ buf += blksize;
+ count -= blksize;
+ offset += blksize;
blknum++;
}
@@ -397,19 +417,19 @@ cow_zero (nbdkit_next *next,
return -1;
}
- block = malloc (BLKSIZE);
+ block = malloc (blksize);
if (block == NULL) {
*err = errno;
nbdkit_error ("malloc: %m");
return -1;
}
- blknum = offset / BLKSIZE; /* block number */
- blkoffs = offset % BLKSIZE; /* offset within the block */
+ blknum = offset / blksize; /* block number */
+ blkoffs = offset % blksize; /* offset within the block */
/* Unaligned head */
if (blkoffs) {
- uint64_t n = MIN (BLKSIZE - blkoffs, count);
+ uint64_t n = MIN (blksize - blkoffs, count);
/* Do a read-modify-write operation on the current block.
* Hold the rmw_lock over the whole operation.
@@ -429,9 +449,9 @@ cow_zero (nbdkit_next *next,
}
/* Aligned body */
- if (count >= BLKSIZE)
- memset (block, 0, BLKSIZE);
- while (count >= BLKSIZE) {
+ if (count >= blksize)
+ memset (block, 0, blksize);
+ while (count >= blksize) {
/* XXX There is the possibility of optimizing this: since this loop is
* writing a whole, aligned block, we should use FALLOC_FL_ZERO_RANGE.
*/
@@ -439,8 +459,8 @@ cow_zero (nbdkit_next *next,
if (r == -1)
return -1;
- count -= BLKSIZE;
- offset += BLKSIZE;
+ count -= blksize;
+ offset += blksize;
blknum++;
}
@@ -471,8 +491,8 @@ cow_trim (nbdkit_next *next,
uint64_t blknum, blkoffs;
int r;
- if (!IS_ALIGNED (count | offset, BLKSIZE)) {
- block = malloc (BLKSIZE);
+ if (!IS_ALIGNED (count | offset, blksize)) {
+ block = malloc (blksize);
if (block == NULL) {
*err = errno;
nbdkit_error ("malloc: %m");
@@ -480,12 +500,12 @@ cow_trim (nbdkit_next *next,
}
}
- blknum = offset / BLKSIZE; /* block number */
- blkoffs = offset % BLKSIZE; /* offset within the block */
+ blknum = offset / blksize; /* block number */
+ blkoffs = offset % blksize; /* offset within the block */
/* Unaligned head */
if (blkoffs) {
- uint64_t n = MIN (BLKSIZE - blkoffs, count);
+ uint64_t n = MIN (blksize - blkoffs, count);
/* Do a read-modify-write operation on the current block.
* Hold the lock over the whole operation.
@@ -505,13 +525,13 @@ cow_trim (nbdkit_next *next,
}
/* Aligned body */
- while (count >= BLKSIZE) {
+ while (count >= blksize) {
r = blk_trim (blknum, err);
if (r == -1)
return -1;
- count -= BLKSIZE;
- offset += BLKSIZE;
+ count -= blksize;
+ offset += blksize;
blknum++;
}
@@ -568,22 +588,22 @@ cow_cache (nbdkit_next *next,
mode = BLK_CACHE_COW;
assert (!flags);
- block = malloc (BLKSIZE);
+ block = malloc (blksize);
if (block == NULL) {
*err = errno;
nbdkit_error ("malloc: %m");
return -1;
}
- blknum = offset / BLKSIZE; /* block number */
- blkoffs = offset % BLKSIZE; /* offset within the block */
+ blknum = offset / blksize; /* block number */
+ blkoffs = offset % blksize; /* offset within the block */
/* Unaligned head */
remaining += blkoffs;
offset -= blkoffs;
/* Unaligned tail */
- remaining = ROUND_UP (remaining, BLKSIZE);
+ remaining = ROUND_UP (remaining, blksize);
/* Aligned body */
while (remaining) {
@@ -591,8 +611,8 @@ cow_cache (nbdkit_next *next,
if (r == -1)
return -1;
- remaining -= BLKSIZE;
- offset += BLKSIZE;
+ remaining -= blksize;
+ offset += blksize;
blknum++;
}
@@ -616,13 +636,13 @@ cow_extents (nbdkit_next *next,
* value so rounding up is safe here.
*/
end = offset + count;
- offset = ROUND_DOWN (offset, BLKSIZE);
- end = ROUND_UP (end, BLKSIZE);
+ offset = ROUND_DOWN (offset, blksize);
+ end = ROUND_UP (end, blksize);
count = end - offset;
- blknum = offset / BLKSIZE;
+ blknum = offset / blksize;
- assert (IS_ALIGNED (offset, BLKSIZE));
- assert (IS_ALIGNED (count, BLKSIZE));
+ assert (IS_ALIGNED (offset, blksize));
+ assert (IS_ALIGNED (count, blksize));
assert (count > 0); /* We must make forward progress. */
while (count > 0) {
@@ -634,7 +654,7 @@ cow_extents (nbdkit_next *next,
/* Present in the overlay. */
if (present) {
e.offset = offset;
- e.length = BLKSIZE;
+ e.length = blksize;
if (trimmed)
e.type = NBDKIT_EXTENT_HOLE|NBDKIT_EXTENT_ZERO;
@@ -647,8 +667,8 @@ cow_extents (nbdkit_next *next,
}
blknum++;
- offset += BLKSIZE;
- count -= BLKSIZE;
+ offset += blksize;
+ count -= blksize;
}
/* Not present in the overlay, but we can ask the plugin. */
@@ -667,12 +687,12 @@ cow_extents (nbdkit_next *next,
* (range_count), but count is a 64 bit quantity, so don't
* overflow range_count here.
*/
- if (range_count >= UINT32_MAX - BLKSIZE + 1) break;
+ if (range_count >= UINT32_MAX - blksize + 1) break;
blknum++;
- offset += BLKSIZE;
- count -= BLKSIZE;
- range_count += BLKSIZE;
+ offset += blksize;
+ count -= blksize;
+ range_count += blksize;
if (count == 0) break;
blk_status (blknum, &present, &trimmed);
@@ -706,7 +726,7 @@ cow_extents (nbdkit_next *next,
/* Otherwise assume the block is non-sparse. */
else {
e.offset = offset;
- e.length = BLKSIZE;
+ e.length = blksize;
e.type = 0;
if (nbdkit_add_extent (extents, e.offset, e.length, e.type) == -1) {
@@ -715,8 +735,8 @@ cow_extents (nbdkit_next *next,
}
blknum++;
- offset += BLKSIZE;
- count -= BLKSIZE;
+ offset += blksize;
+ count -= blksize;
}
/* If the caller only wanted the first extent, and we've managed
@@ -734,11 +754,11 @@ cow_extents (nbdkit_next *next,
static struct nbdkit_filter filter = {
.name = "cow",
.longname = "nbdkit copy-on-write (COW) filter",
- .load = cow_load,
.unload = cow_unload,
.open = cow_open,
.config = cow_config,
.config_help = cow_config_help,
+ .get_ready = cow_get_ready,
.prepare = cow_prepare,
.get_size = cow_get_size,
.can_write = cow_can_write,
diff --git a/filters/cow/cow.h b/filters/cow/cow.h
new file mode 100644
index 00000000..d46dbe91
--- /dev/null
+++ b/filters/cow/cow.h
@@ -0,0 +1,39 @@
+/* nbdkit
+ * Copyright (C) 2018-2021 Red Hat Inc.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * * Neither the name of Red Hat nor the names of its contributors may be
+ * used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY RED HAT AND CONTRIBUTORS ''AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+ * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL RED HAT OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
+ * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+ * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#ifndef NBDKIT_COW_H
+#define NBDKIT_COW_H
+
+/* Size of a block in the cache. */
+extern unsigned blksize;
+
+#endif /* NBDKIT_COW_H */
diff --git a/filters/cow/nbdkit-cow-filter.pod b/filters/cow/nbdkit-cow-filter.pod
index 7f861140..997c9097 100644
--- a/filters/cow/nbdkit-cow-filter.pod
+++ b/filters/cow/nbdkit-cow-filter.pod
@@ -5,6 +5,7 @@ nbdkit-cow-filter - nbdkit copy-on-write (COW) filter
=head1 SYNOPSIS
nbdkit --filter=cow plugin [plugin-args...]
+ [cow-block-size=N]
[cow-on-cache=false|true]
[cow-on-read=false|true|/PATH]
@@ -42,6 +43,10 @@ serve the same data to each client.
=over 4
+=item B<cow-block-size=>N
+
+Set the block size used by the filter. The default is 64K.
+
=item B<cow-on-cache=false>
Do not save data from cache (prefetch) requests in the overlay. This
diff --git a/tests/Makefile.am b/tests/Makefile.am
index e61c5829..d93f848f 100644
--- a/tests/Makefile.am
+++ b/tests/Makefile.am
@@ -1404,6 +1404,7 @@ EXTRA_DIST += \
if HAVE_MKE2FS_WITH_D
TESTS += \
test-cow.sh \
+ test-cow-block-size.sh \
test-cow-extents1.sh \
test-cow-extents2.sh \
test-cow-extents-large.sh \
@@ -1415,6 +1416,7 @@ endif
TESTS += test-cow-null.sh
EXTRA_DIST += \
test-cow.sh \
+ test-cow-block-size.sh \
test-cow-extents1.sh \
test-cow-extents2.sh \
test-cow-extents-large.sh \
diff --git a/tests/test-cow-block-size.sh b/tests/test-cow-block-size.sh
new file mode 100755
index 00000000..6de1c068
--- /dev/null
+++ b/tests/test-cow-block-size.sh
@@ -0,0 +1,72 @@
+#!/usr/bin/env bash
+# nbdkit
+# Copyright (C) 2018-2021 Red Hat Inc.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#
+# * Redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in the
+# documentation and/or other materials provided with the distribution.
+#
+# * Neither the name of Red Hat nor the names of its contributors may be
+# used to endorse or promote products derived from this software without
+# specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY RED HAT AND CONTRIBUTORS ''AS IS'' AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+# PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL RED HAT OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
+# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+# SUCH DAMAGE.
+
+source ./functions.sh
+set -e
+set -x
+
+requires_plugin linuxdisk
+requires guestfish --version
+requires nbdcopy --version
+requires qemu-img --version
+
+sock=$(mktemp -u /tmp/nbdkit-test-sock.XXXXXX)
+files="cow-block-size-base.img $sock cow-block-size.pid"
+rm -f $files
+cleanup_fn rm -f $files
+
+# Create a base image which is partitioned with an empty filesystem.
+rm -rf cow-block-size.d
+mkdir cow-block-size.d
+cleanup_fn rm -rf cow-block-size.d
+nbdkit -fv -U - linuxdisk cow-block-size.d size=100M \
+ --run 'nbdcopy "$uri" cow-block-size-base.img'
+lastmod="$(stat -c "%y" cow-block-size-base.img)"
+
+# Run nbdkit with a COW overlay, 4M block size and copy on read.
+start_nbdkit -P cow-block-size.pid -U $sock \
+ --filter=cow file cow-block-size-base.img \
+ cow-block-size=4M cow-on-read=true
+
+# Write some data into the overlay.
+guestfish --format=raw -a "nbd://?socket=$sock" -m /dev/sda1 <<EOF
+ fill-pattern "abcde" 128K /large
+ write /hello "hello, world"
+EOF
+
+# The original file must not be modified.
+currmod="$(stat -c "%y" cow-block-size-base.img)"
+
+if [ "$lastmod" != "$currmod" ]; then
+ echo "$0: FAILED last modified time of base file changed"
+ exit 1
+fi
--
2.31.1

32
SOURCES/0023-podwrapper.pl.in-Use-short-commit-date.patch Normal file
View File

@ -0,0 +1,32 @@
From ece6d7e1a5827de17e86a20f7dae5f6f853d419b Mon Sep 17 00:00:00 2001
From: Nir Soffer <nsoffer@redhat.com>
Date: Mon, 8 Nov 2021 19:47:57 +0200
Subject: [PATCH] podwrapper.pl.in: Use short commit date
We can use git short commit date format $cs. Maybe it was not available
when podwrapper.pl was created.
Signed-off-by: Nir Soffer <nsoffer@redhat.com>
(cherry picked from libnbd commit 0306fdcb08e8dc5957a9e344b54200711fca1220)
(cherry picked from commit 7a1e79c6b5ca4adcef47fc0929d25d54610fc417)
---
podwrapper.pl.in | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
diff --git a/podwrapper.pl.in b/podwrapper.pl.in
index abad578d..63c1025a 100755
--- a/podwrapper.pl.in
+++ b/podwrapper.pl.in
@@ -233,8 +233,7 @@ my $date;
my $filename = "$abs_top_srcdir/.git";
if (!$date && -d $filename) {
local $ENV{GIT_DIR} = $filename;
- $_ = `git show -O/dev/null -s --format=%ci`;
- $date = $1 if /^(\d+-\d+-\d+)\s/;
+ $date = `git show -O/dev/null -s --format=%cs`;
}
if (!$date) {
my ($day, $month, $year) = (gmtime($ENV{SOURCE_DATE_EPOCH} || time))[3,4,5];
--
2.31.1

26
SOURCES/0024-cow-Ship-cow.h-header.patch
View File

@ -1,26 +0,0 @@
From 8d2ef02bd4de988e20ad1efba8038d311cd59665 Mon Sep 17 00:00:00 2001
From: Eric Blake <eblake@redhat.com>
Date: Wed, 18 Aug 2021 19:16:43 -0500
Subject: [PATCH] cow: Ship cow.h header
Fixes: 7182c47d0 (cow: Make the block size configurable)
(cherry picked from commit 75ff1b8b1afb3744b21a306c62e4973c90d386be)
---
filters/cow/Makefile.am | 1 +
1 file changed, 1 insertion(+)
diff --git a/filters/cow/Makefile.am b/filters/cow/Makefile.am
index a80ccd8f..88cda497 100644
--- a/filters/cow/Makefile.am
+++ b/filters/cow/Makefile.am
@@ -39,6 +39,7 @@ nbdkit_cow_filter_la_SOURCES = \
blk.c \
blk.h \
cow.c \
+ cow.h \
$(top_srcdir)/include/nbdkit-filter.h \
$(NULL)
--
2.31.1

92
SOURCES/0024-ocaml-Replace-noalloc-with-noalloc-annotation.patch Normal file
View File

@ -0,0 +1,92 @@
From 2955179919fc6233427b82d27ae61755b2b5e3d7 Mon Sep 17 00:00:00 2001
From: "Richard W.M. Jones" <rjones@redhat.com>
Date: Tue, 9 Nov 2021 09:07:42 +0000
Subject: [PATCH] ocaml: Replace "noalloc" with [@@noalloc] annotation
This requires OCaml >= 4.03 (released April 2016). The previous
minimum version was 4.02.2.
(cherry picked from commit d15dd73845065cc9ca04aa785e2be994f76bf832)
---
README | 2 +-
plugins/ocaml/NBDKit.ml | 18 +++++++++---------
plugins/ocaml/nbdkit-ocaml-plugin.pod | 5 +----
3 files changed, 11 insertions(+), 14 deletions(-)
diff --git a/README b/README
index b001620c..160856b6 100644
--- a/README
+++ b/README
@@ -155,7 +155,7 @@ For the Python plugin:
For the OCaml plugin:
- - OCaml >= 4.02.2
+ - OCaml >= 4.03
For the Tcl plugin:
diff --git a/plugins/ocaml/NBDKit.ml b/plugins/ocaml/NBDKit.ml
index ecdc00c1..c9ce31b5 100644
--- a/plugins/ocaml/NBDKit.ml
+++ b/plugins/ocaml/NBDKit.ml
@@ -152,13 +152,13 @@ let default_callbacks = {
export_description = None;
}
-external set_name : string -> unit = "ocaml_nbdkit_set_name" "noalloc"
-external set_longname : string -> unit = "ocaml_nbdkit_set_longname" "noalloc"
-external set_version : string -> unit = "ocaml_nbdkit_set_version" "noalloc"
-external set_description : string -> unit = "ocaml_nbdkit_set_description" "noalloc"
-external set_config_help : string -> unit = "ocaml_nbdkit_set_config_help" "noalloc"
+external set_name : string -> unit = "ocaml_nbdkit_set_name" [@@noalloc]
+external set_longname : string -> unit = "ocaml_nbdkit_set_longname" [@@noalloc]
+external set_version : string -> unit = "ocaml_nbdkit_set_version" [@@noalloc]
+external set_description : string -> unit = "ocaml_nbdkit_set_description" [@@noalloc]
+external set_config_help : string -> unit = "ocaml_nbdkit_set_config_help" [@@noalloc]
-external set_field : string -> 'a -> unit = "ocaml_nbdkit_set_field" "noalloc"
+external set_field : string -> 'a -> unit = "ocaml_nbdkit_set_field" [@@noalloc]
let register_plugin plugin =
(* Check the required fields have been set by the caller. *)
@@ -220,7 +220,7 @@ let register_plugin plugin =
(* Bindings to nbdkit server functions. *)
-external _set_error : int -> unit = "ocaml_nbdkit_set_error" "noalloc"
+external _set_error : int -> unit = "ocaml_nbdkit_set_error" [@@noalloc]
let set_error unix_error =
(* There's an awkward triple translation going on here, because
@@ -250,9 +250,9 @@ external read_password : string -> string = "ocaml_nbdkit_read_password"
external realpath : string -> string = "ocaml_nbdkit_realpath"
external nanosleep : int -> int -> unit = "ocaml_nbdkit_nanosleep"
external export_name : unit -> string = "ocaml_nbdkit_export_name"
-external shutdown : unit -> unit = "ocaml_nbdkit_shutdown" "noalloc"
+external shutdown : unit -> unit = "ocaml_nbdkit_shutdown" [@@noalloc]
-external _debug : string -> unit = "ocaml_nbdkit_debug" "noalloc"
+external _debug : string -> unit = "ocaml_nbdkit_debug" [@@noalloc]
let debug fs =
ksprintf _debug fs
diff --git a/plugins/ocaml/nbdkit-ocaml-plugin.pod b/plugins/ocaml/nbdkit-ocaml-plugin.pod
index 2bd0af25..293f8143 100644
--- a/plugins/ocaml/nbdkit-ocaml-plugin.pod
+++ b/plugins/ocaml/nbdkit-ocaml-plugin.pod
@@ -11,10 +11,7 @@ nbdkit-ocaml-plugin - writing nbdkit plugins in OCaml
=head1 DESCRIPTION
This manual page describes how to write nbdkit plugins in natively
-compiled OCaml code.
-
-Note this requires OCaml E<ge> 4.02.2, which has support for shared
-libraries. See L<http://caml.inria.fr/mantis/view.php?id=6693>
+compiled OCaml code. This requires OCaml E<ge> 4.03.
=head1 WRITING AN OCAML NBDKIT PLUGIN
--
2.31.1

11
SOURCES/nbdkit-1.26.5.tar.gz.sig
View File

@ -1,11 +0,0 @@
-----BEGIN PGP SIGNATURE-----
iQEzBAABCAAdFiEEccLMIrHEYCkn0vOqp6FrSiUnQ2oFAmEdoO0ACgkQp6FrSiUn
Q2r9AAf/VN10pDdAapr8W5bPNbgMIJ4dGtGl3sbMKg5mUVNkWwReUpiiUInZwnQW
I+TUjMkvB2jJGkruldgipuRNbAAGROZV3JugE2fMl8qQC1TvL/RMAOHcfKCswzfv
pVik1FmvpI88litCw05csH30TEA1BtFM0TlOR9xoeDkV9e2IUtWcxFJYP6RN5COr
NgTMfouxHWuR+FlVpXkvPl4aOuCavpplobcS0OaKNrqFXMhN+qcKjYgKazUKPB9C
TEExW8/CKTBVdaNpMbcLW/VBEaE85c3mv0xU26YKcEkj+OPAl4AzJ4Z+0MSDXk7t
3nrWSxPX67gBU5XDtKZ1IMUttydvzA==
=NyyV
-----END PGP SIGNATURE-----

17
SOURCES/nbdkit-1.28.2.tar.gz.sig Normal file
View File

@ -0,0 +1,17 @@
-----BEGIN PGP SIGNATURE-----
iQJFBAABCAAvFiEE93dPsa0HSn6Mh2fqkXOPc+G3aKAFAmGKYx0RHHJpY2hAYW5u
ZXhpYS5vcmcACgkQkXOPc+G3aKAw7g//Q/42avSvu3DZQXPCl772S9K7erYzw1LK
PzffbDKWwsR4YN+m4w94gFjb6n2NZOnSBVpDYofkt5mzHM9dwYTlCJdS+J4SxV8e
s23+RQjgX0fQMDv4qfGE7CUNQXX6CLRWUlXSu4dMMbsxbF8y18VHl+eLIN7U15Ec
8E2lwcthQ8ipwz0xuxKE1GUNXzWsJ1SHBaCmo8HH1x1Rs+mWu0QjwFllhTc4+gq9
5yrdZuViv96DxIsk3u7RByqrTJYZi2tK9dU75dVeXI6o74b1WUrXkwqW8uqIR4m5
hZry1+vXPzWIO4G5o1gika+C4xvYuyIShNl/c/aWJsA0ku4fDtUxAkjC/k78D4yR
0FQbNfztiCFgvQNS3UH6M0d63S4GE76m7HgwI7SkUtczlnPl2AhzSjjfXZ5DrHH9
S8SgmgwP4qfwszY6Xs7ISFofRV5YBgMhSIiJc/5yYRLtZZCcRWNvv/OWYNTGdVBp
rRYQRvnhe2VbaaVexOn1Fkzcv/FNeD+96bsTp1B1SailUUFrTOpRZ0yIOxfZZZO/
JcroGuFvq0BywP1U6Rq2PI1IHD4I6x3QLYIfNBP076Rc17BhcXyn2Ei18XkRo+xd
iFTCDpe0zlSCl3tLL7Hvp2EtetYkpiQKHA59g7RoSVQDX/RAml3VFDvwZ2rNBeCL
yegMoVPEmc0=
=c3GR
-----END PGP SIGNATURE-----

139
SPECS/nbdkit.spec
View File

@ -47,10 +47,10 @@ ExclusiveArch: x86_64
%global patches_touch_autotools 1 %global patches_touch_autotools 1
# The source directory. # The source directory.
%global source_directory 1.26-stable %global source_directory 1.28-stable
Name: nbdkit Name: nbdkit
Version: 1.26.5 Version: 1.28.2
Release: 1%{?dist} Release: 1%{?dist}
Summary: NBD server Summary: NBD server
@ -76,30 +76,30 @@ Source3: copy-patches.sh
# https://gitlab.com/nbdkit/nbdkit/-/commits/rhel-9.0/ # https://gitlab.com/nbdkit/nbdkit/-/commits/rhel-9.0/
# Patches. # Patches.
Patch0001: 0001-server-reset-meta-context-replies-on-starttls.patch Patch0001: 0001-vddk-Refactor-how-D-vddk.stats-1-is-collected.patch
Patch0002: 0002-cache-Reduce-verbosity-of-debugging.patch Patch0002: 0002-vddk-Extend-D-vddk.stats-1-to-show-number-of-calls-a.patch
Patch0003: 0003-cache-cow-Add-blk_read_multiple-function.patch Patch0003: 0003-vddk-Simplify-and-consolidate-VDDK_CALL_START-END-ma.patch
Patch0004: 0004-cache-cow-Use-full-pread-pwrite-operations.patch Patch0004: 0004-vddk-Document-troubleshooting-performance-problems.patch
Patch0005: 0005-cache-Implement-cache-on-read-PATH.patch Patch0005: 0005-vddk-Include-VDDK-major-library-version-in-dump-plug.patch
Patch0006: 0006-cache-Add-cache-min-block-size-parameter.patch Patch0006: 0006-vddk-Add-logical-and-physical-sector-size-to-D-vddk..patch
Patch0007: 0007-cache-cow-Use-a-64K-block-size-by-default.patch Patch0007: 0007-vddk-Fix-typo-in-debug-message.patch
Patch0008: 0008-cache-Refactor-printing-state-into-new-function.patch Patch0008: 0008-vddk-Only-print-vddk_library_version-when-we-managed.patch
Patch0009: 0009-tests-cache-Test-cache-on-read-option-really-caches.patch Patch0009: 0009-vddk-Print-one-line-in-dump-plugin-output-for-each-V.patch
Patch0010: 0010-cow-Implement-cow-on-read.patch Patch0010: 0010-vddk-Move-minimum-version-to-VDDK-6.5.patch
Patch0011: 0011-delay-Add-delay-open-and-delay-close.patch Patch0011: 0011-vddk-Add-read-write-and-wait-asynchronous-functions.patch
Patch0012: 0012-python-Implement-.cleanup-method.patch Patch0012: 0012-vddk-Start-to-split-VDDK-over-several-files.patch
Patch0013: 0013-cow-General-revision-and-updates-to-the-manual.patch Patch0013: 0013-vddk-Refactor-D-vddk.stats-1-into-a-new-file.patch
Patch0014: 0014-cache-Move-plugin-args-in-synopsis-earlier.patch Patch0014: 0014-vddk-Implement-parallel-thread-model.patch
Patch0015: 0015-data-Improve-the-example-with-a-diagram.patch Patch0015: 0015-vddk-Assume-that-VixDiskLib_Flush-is-available.patch
Patch0016: 0016-cow-Add-some-more-debugging-especially-for-blk_read_.patch Patch0016: 0016-vddk-Simplify-detection-of-VDDK-symbols-and-baseline.patch
Patch0017: 0017-delay-Fix-delay-close.patch Patch0017: 0017-vddk-Remove-some-whitespace-from-a-couple-of-functio.patch
Patch0018: 0018-delay-Test-delay-open-and-delay-close.patch Patch0018: 0018-vddk-Move-config-debug-error-and-utility-functions-a.patch
Patch0019: 0019-vddk-Implement-can_flush-and-can_fua.patch Patch0019: 0019-common-utils-test-vector.c-Add-vector-benchmarks.patch
Patch0020: 0020-vddk-Replace-DEBUG_CALL-with-bracketed-VDDK_CALL_STA.patch Patch0020: 0020-common-urils-vector.c-Optimize-vector-append.patch
Patch0021: 0021-tests-Add-a-better-test-of-real-VDDK.patch Patch0021: 0021-common-utils-vector-Rename-alloc-to-cap.patch
Patch0022: 0022-vddk-Add-stats-about-the-amount-of-time-spent-in-VDD.patch Patch0022: 0022-common-utils-vector-Rename-size-to-len.patch
Patch0023: 0023-cow-Make-the-block-size-configurable.patch Patch0023: 0023-podwrapper.pl.in-Use-short-commit-date.patch
Patch0024: 0024-cow-Ship-cow.h-header.patch Patch0024: 0024-ocaml-Replace-noalloc-with-noalloc-annotation.patch
BuildRequires: make BuildRequires: make
%if 0%{patches_touch_autotools} %if 0%{patches_touch_autotools}
@ -125,7 +125,7 @@ BuildRequires: libnbd-devel >= 1.3.11
BuildRequires: libssh-devel BuildRequires: libssh-devel
BuildRequires: e2fsprogs, e2fsprogs-devel BuildRequires: e2fsprogs, e2fsprogs-devel
%if !0%{?rhel} %if !0%{?rhel}
BuildRequires: genisoimage BuildRequires: xorriso
BuildRequires: rb_libtorrent-devel BuildRequires: rb_libtorrent-devel
%endif %endif
BuildRequires: bash-completion BuildRequires: bash-completion
@ -217,16 +217,18 @@ reading the nbdkit(1) and nbdkit-plugin(3) manual pages.
%package server %package server
Summary: The %{name} server Summary: The %{name} server
License: BSD License: BSD
Provides: %{name}-null-plugin = %{version}-%{release}
%description server %description server
This package contains the %{name} server with no plugins or filters. This package contains the %{name} server with only the null plugin
and no filters. To install a basic set of plugins and filters you
need to install "nbdkit-basic-plugins", "nbdkit-basic-filters" or
the metapackage "nbdkit".
%package basic-plugins %package basic-plugins
Summary: Basic plugins for %{name} Summary: Basic plugins for %{name}
License: BSD License: BSD
Requires: %{name}-server%{?_isa} = %{version}-%{release} Requires: %{name}-server%{?_isa} = %{version}-%{release}
Provides: %{name}-data-plugin = %{version}-%{release} Provides: %{name}-data-plugin = %{version}-%{release}
Provides: %{name}-eval-plugin = %{version}-%{release} Provides: %{name}-eval-plugin = %{version}-%{release}
@ -235,7 +237,6 @@ Provides: %{name}-floppy-plugin = %{version}-%{release}
Provides: %{name}-full-plugin = %{version}-%{release} Provides: %{name}-full-plugin = %{version}-%{release}
Provides: %{name}-info-plugin = %{version}-%{release} Provides: %{name}-info-plugin = %{version}-%{release}
Provides: %{name}-memory-plugin = %{version}-%{release} Provides: %{name}-memory-plugin = %{version}-%{release}
Provides: %{name}-null-plugin = %{version}-%{release}
Provides: %{name}-ondemand-plugin = %{version}-%{release} Provides: %{name}-ondemand-plugin = %{version}-%{release}
Provides: %{name}-pattern-plugin = %{version}-%{release} Provides: %{name}-pattern-plugin = %{version}-%{release}
Provides: %{name}-partitioning-plugin = %{version}-%{release} Provides: %{name}-partitioning-plugin = %{version}-%{release}
@ -265,8 +266,6 @@ nbdkit-info-plugin Serve client and server information.
nbdkit-memory-plugin A virtual memory plugin. nbdkit-memory-plugin A virtual memory plugin.
nbdkit-null-plugin A null (bitbucket) plugin.
nbdkit-ondemand-plugin Create filesystems on demand. nbdkit-ondemand-plugin Create filesystems on demand.
nbdkit-pattern-plugin Fixed test pattern. nbdkit-pattern-plugin Fixed test pattern.
@ -287,14 +286,12 @@ nbdkit-zero-plugin Zero-length plugin for testing.
%package example-plugins %package example-plugins
Summary: Example plugins for %{name} Summary: Example plugins for %{name}
License: BSD License: BSD
Requires: %{name}-server%{?_isa} = %{version}-%{release} Requires: %{name}-server%{?_isa} = %{version}-%{release}
%if !0%{?rhel} %if !0%{?rhel}
# example4 is written in Perl. # example4 is written in Perl.
Requires: %{name}-perl-plugin Requires: %{name}-perl-plugin
%endif %endif
%description example-plugins %description example-plugins
This package contains example plugins for %{name}. This package contains example plugins for %{name}.
@ -306,12 +303,10 @@ This package contains example plugins for %{name}.
%package cc-plugin %package cc-plugin
Summary: Write small inline C plugins and scripts for %{name} Summary: Write small inline C plugins and scripts for %{name}
License: BSD License: BSD
Requires: %{name}-server%{?_isa} = %{version}-%{release} Requires: %{name}-server%{?_isa} = %{version}-%{release}
Requires: gcc Requires: gcc
Requires: %{_bindir}/cat Requires: %{_bindir}/cat
%description cc-plugin %description cc-plugin
This package contains support for writing inline C plugins and scripts This package contains support for writing inline C plugins and scripts
for %{name}. NOTE this is NOT the right package for writing plugins for %{name}. NOTE this is NOT the right package for writing plugins
@ -323,12 +318,10 @@ in C, install %{name}-devel for that.
%package cdi-plugin %package cdi-plugin
Summary: Containerized Data Import plugin for %{name} Summary: Containerized Data Import plugin for %{name}
License: BSD License: BSD
Requires: %{name}-server%{?_isa} = %{version}-%{release} Requires: %{name}-server%{?_isa} = %{version}-%{release}
Requires: jq Requires: jq
Requires: podman Requires: podman
%description cdi-plugin %description cdi-plugin
This package contains Containerized Data Import support for %{name}. This package contains Containerized Data Import support for %{name}.
%endif %endif
@ -337,10 +330,8 @@ This package contains Containerized Data Import support for %{name}.
%package curl-plugin %package curl-plugin
Summary: HTTP/FTP (cURL) plugin for %{name} Summary: HTTP/FTP (cURL) plugin for %{name}
License: BSD License: BSD
Requires: %{name}-server%{?_isa} = %{version}-%{release} Requires: %{name}-server%{?_isa} = %{version}-%{release}
%description curl-plugin %description curl-plugin
This package contains cURL (HTTP/FTP) support for %{name}. This package contains cURL (HTTP/FTP) support for %{name}.
@ -349,10 +340,8 @@ This package contains cURL (HTTP/FTP) support for %{name}.
%package guestfs-plugin %package guestfs-plugin
Summary: libguestfs plugin for %{name} Summary: libguestfs plugin for %{name}
License: BSD License: BSD
Requires: %{name}-server%{?_isa} = %{version}-%{release} Requires: %{name}-server%{?_isa} = %{version}-%{release}
%description guestfs-plugin %description guestfs-plugin
This package is a libguestfs plugin for %{name}. This package is a libguestfs plugin for %{name}.
%endif %endif
@ -362,10 +351,8 @@ This package is a libguestfs plugin for %{name}.
%package iso-plugin %package iso-plugin
Summary: Virtual ISO 9660 plugin for %{name} Summary: Virtual ISO 9660 plugin for %{name}
License: BSD License: BSD
Requires: %{name}-server%{?_isa} = %{version}-%{release} Requires: %{name}-server%{?_isa} = %{version}-%{release}
Requires: genisoimage Requires: xorriso
%description iso-plugin %description iso-plugin
This package is a virtual ISO 9660 (CD-ROM) plugin for %{name}. This package is a virtual ISO 9660 (CD-ROM) plugin for %{name}.
@ -376,10 +363,8 @@ This package is a virtual ISO 9660 (CD-ROM) plugin for %{name}.
%package libvirt-plugin %package libvirt-plugin
Summary: Libvirt plugin for %{name} Summary: Libvirt plugin for %{name}
License: BSD License: BSD
Requires: %{name}-server%{?_isa} = %{version}-%{release} Requires: %{name}-server%{?_isa} = %{version}-%{release}
%description libvirt-plugin %description libvirt-plugin
This package is a libvirt plugin for %{name}. It lets you access This package is a libvirt plugin for %{name}. It lets you access
libvirt guest disks readonly. It is implemented using the libvirt libvirt guest disks readonly. It is implemented using the libvirt
@ -390,12 +375,10 @@ virDomainBlockPeek API.
%package linuxdisk-plugin %package linuxdisk-plugin
Summary: Virtual Linux disk plugin for %{name} Summary: Virtual Linux disk plugin for %{name}
License: BSD License: BSD
Requires: %{name}-server%{?_isa} = %{version}-%{release} Requires: %{name}-server%{?_isa} = %{version}-%{release}
# for mke2fs # for mke2fs
Requires: e2fsprogs Requires: e2fsprogs
%description linuxdisk-plugin %description linuxdisk-plugin
This package is a virtual Linux disk plugin for %{name}. This package is a virtual Linux disk plugin for %{name}.
@ -404,10 +387,8 @@ This package is a virtual Linux disk plugin for %{name}.
%package lua-plugin %package lua-plugin
Summary: Lua plugin for %{name} Summary: Lua plugin for %{name}
License: BSD License: BSD
Requires: %{name}-server%{?_isa} = %{version}-%{release} Requires: %{name}-server%{?_isa} = %{version}-%{release}
%description lua-plugin %description lua-plugin
This package lets you write Lua plugins for %{name}. This package lets you write Lua plugins for %{name}.
%endif %endif
@ -416,10 +397,8 @@ This package lets you write Lua plugins for %{name}.
%package nbd-plugin %package nbd-plugin
Summary: NBD proxy / forward plugin for %{name} Summary: NBD proxy / forward plugin for %{name}
License: BSD License: BSD
Requires: %{name}-server%{?_isa} = %{version}-%{release} Requires: %{name}-server%{?_isa} = %{version}-%{release}
%description nbd-plugin %description nbd-plugin
This package lets you forward NBD connections from %{name} This package lets you forward NBD connections from %{name}
to another NBD server. to another NBD server.
@ -429,10 +408,8 @@ to another NBD server.
%package ocaml-plugin %package ocaml-plugin
Summary: OCaml plugin for %{name} Summary: OCaml plugin for %{name}
License: BSD License: BSD
Requires: %{name}-server%{?_isa} = %{version}-%{release} Requires: %{name}-server%{?_isa} = %{version}-%{release}
%description ocaml-plugin %description ocaml-plugin
This package lets you run OCaml plugins for %{name}. This package lets you run OCaml plugins for %{name}.
@ -443,11 +420,9 @@ To compile OCaml plugins you will also need to install
%package ocaml-plugin-devel %package ocaml-plugin-devel
Summary: OCaml development environment for %{name} Summary: OCaml development environment for %{name}
License: BSD License: BSD
Requires: %{name}-server%{?_isa} = %{version}-%{release} Requires: %{name}-server%{?_isa} = %{version}-%{release}
Requires: %{name}-ocaml-plugin%{?_isa} = %{version}-%{release} Requires: %{name}-ocaml-plugin%{?_isa} = %{version}-%{release}
%description ocaml-plugin-devel %description ocaml-plugin-devel
This package lets you write OCaml plugins for %{name}. This package lets you write OCaml plugins for %{name}.
%endif %endif
@ -457,10 +432,8 @@ This package lets you write OCaml plugins for %{name}.
%package perl-plugin %package perl-plugin
Summary: Perl plugin for %{name} Summary: Perl plugin for %{name}
License: BSD License: BSD
Requires: %{name}-server%{?_isa} = %{version}-%{release} Requires: %{name}-server%{?_isa} = %{version}-%{release}
%description perl-plugin %description perl-plugin
This package lets you write Perl plugins for %{name}. This package lets you write Perl plugins for %{name}.
%endif %endif
@ -469,10 +442,8 @@ This package lets you write Perl plugins for %{name}.
%package python-plugin %package python-plugin
Summary: Python 3 plugin for %{name} Summary: Python 3 plugin for %{name}
License: BSD License: BSD
Requires: %{name}-server%{?_isa} = %{version}-%{release} Requires: %{name}-server%{?_isa} = %{version}-%{release}
%description python-plugin %description python-plugin
This package lets you write Python 3 plugins for %{name}. This package lets you write Python 3 plugins for %{name}.
@ -481,10 +452,8 @@ This package lets you write Python 3 plugins for %{name}.
%package ruby-plugin %package ruby-plugin
Summary: Ruby plugin for %{name} Summary: Ruby plugin for %{name}
License: BSD License: BSD
Requires: %{name}-server%{?_isa} = %{version}-%{release} Requires: %{name}-server%{?_isa} = %{version}-%{release}
%description ruby-plugin %description ruby-plugin
This package lets you write Ruby plugins for %{name}. This package lets you write Ruby plugins for %{name}.
%endif %endif
@ -496,12 +465,10 @@ This package lets you write Ruby plugins for %{name}.
%package S3-plugin %package S3-plugin
Summary: Amazon S3 and Ceph plugin for %{name} Summary: Amazon S3 and Ceph plugin for %{name}
License: BSD License: BSD
Requires: %{name}-python-plugin >= 1.22 Requires: %{name}-python-plugin >= 1.22
# XXX Should not need to add this. # XXX Should not need to add this.
Requires: python3-boto3 Requires: python3-boto3
%description S3-plugin %description S3-plugin
This package lets you open disk images stored in Amazon S3 This package lets you open disk images stored in Amazon S3
or Ceph using %{name}. or Ceph using %{name}.
@ -511,10 +478,8 @@ or Ceph using %{name}.
%package ssh-plugin %package ssh-plugin
Summary: SSH plugin for %{name} Summary: SSH plugin for %{name}
License: BSD License: BSD
Requires: %{name}-server%{?_isa} = %{version}-%{release} Requires: %{name}-server%{?_isa} = %{version}-%{release}
%description ssh-plugin %description ssh-plugin
This package contains SSH support for %{name}. This package contains SSH support for %{name}.
@ -523,10 +488,8 @@ This package contains SSH support for %{name}.
%package tcl-plugin %package tcl-plugin
Summary: Tcl plugin for %{name} Summary: Tcl plugin for %{name}
License: BSD License: BSD
Requires: %{name}-server%{?_isa} = %{version}-%{release} Requires: %{name}-server%{?_isa} = %{version}-%{release}
%description tcl-plugin %description tcl-plugin
This package lets you write Tcl plugins for %{name}. This package lets you write Tcl plugins for %{name}.
%endif %endif
@ -535,7 +498,6 @@ This package lets you write Tcl plugins for %{name}.
%package tmpdisk-plugin %package tmpdisk-plugin
Summary: Remote temporary filesystem disk plugin for %{name} Summary: Remote temporary filesystem disk plugin for %{name}
License: BSD License: BSD
Requires: %{name}-server%{?_isa} = %{version}-%{release} Requires: %{name}-server%{?_isa} = %{version}-%{release}
# For mkfs and mke2fs (defaults). # For mkfs and mke2fs (defaults).
Requires: util-linux, e2fsprogs Requires: util-linux, e2fsprogs
@ -545,7 +507,6 @@ Suggests: xfsprogs
Suggests: ntfsprogs, dosfstools Suggests: ntfsprogs, dosfstools
%endif %endif
%description tmpdisk-plugin %description tmpdisk-plugin
This package is a remote temporary filesystem disk plugin for %{name}. This package is a remote temporary filesystem disk plugin for %{name}.
@ -554,10 +515,8 @@ This package is a remote temporary filesystem disk plugin for %{name}.
%package torrent-plugin %package torrent-plugin
Summary: BitTorrent plugin for %{name} Summary: BitTorrent plugin for %{name}
License: BSD License: BSD
Requires: %{name}-server%{?_isa} = %{version}-%{release} Requires: %{name}-server%{?_isa} = %{version}-%{release}
%description torrent-plugin %description torrent-plugin
This package is a BitTorrent plugin for %{name}. This package is a BitTorrent plugin for %{name}.
%endif %endif
@ -567,12 +526,10 @@ This package is a BitTorrent plugin for %{name}.
%package vddk-plugin %package vddk-plugin
Summary: VMware VDDK plugin for %{name} Summary: VMware VDDK plugin for %{name}
License: BSD License: BSD
Requires: %{name}-server%{?_isa} = %{version}-%{release} Requires: %{name}-server%{?_isa} = %{version}-%{release}
# https://bugzilla.redhat.com/show_bug.cgi?id=1931818 # https://bugzilla.redhat.com/show_bug.cgi?id=1931818
Requires: libxcrypt-compat Requires: libxcrypt-compat
%description vddk-plugin %description vddk-plugin
This package is a plugin for %{name} which connects to This package is a plugin for %{name} which connects to
VMware VDDK for accessing VMware disks and servers. VMware VDDK for accessing VMware disks and servers.
@ -582,7 +539,6 @@ VMware VDDK for accessing VMware disks and servers.
%package basic-filters %package basic-filters
Summary: Basic filters for %{name} Summary: Basic filters for %{name}
License: BSD License: BSD
Requires: %{name}-server%{?_isa} = %{version}-%{release} Requires: %{name}-server%{?_isa} = %{version}-%{release}
Provides: %{name}-blocksize-filter = %{version}-%{release} Provides: %{name}-blocksize-filter = %{version}-%{release}
Provides: %{name}-cache-filter = %{version}-%{release} Provides: %{name}-cache-filter = %{version}-%{release}
@ -617,7 +573,6 @@ Provides: %{name}-swab-filter = %{version}-%{release}
Provides: %{name}-tls-fallback-filter = %{version}-%{release} Provides: %{name}-tls-fallback-filter = %{version}-%{release}
Provides: %{name}-truncate-filter = %{version}-%{release} Provides: %{name}-truncate-filter = %{version}-%{release}
%description basic-filters %description basic-filters
This package contains filters for %{name} which only depend on simple This package contains filters for %{name} which only depend on simple
C libraries: glibc, gnutls. Other filters for nbdkit with more C libraries: glibc, gnutls. Other filters for nbdkit with more
@ -692,10 +647,8 @@ nbdkit-truncate-filter Truncate, expand, round up or round down size.
%package ext2-filter %package ext2-filter
Summary: ext2, ext3 and ext4 filesystem support for %{name} Summary: ext2, ext3 and ext4 filesystem support for %{name}
License: BSD License: BSD
Requires: %{name}-server%{?_isa} = %{version}-%{release} Requires: %{name}-server%{?_isa} = %{version}-%{release}
%description ext2-filter %description ext2-filter
This package contains ext2, ext3 and ext4 filesystem support for This package contains ext2, ext3 and ext4 filesystem support for
%{name}. %{name}.
@ -705,10 +658,8 @@ This package contains ext2, ext3 and ext4 filesystem support for
%package gzip-filter %package gzip-filter
Summary: GZip filter for %{name} Summary: GZip filter for %{name}
License: BSD License: BSD
Requires: %{name}-server%{?_isa} = %{version}-%{release} Requires: %{name}-server%{?_isa} = %{version}-%{release}
%description gzip-filter %description gzip-filter
This package is a gzip filter for %{name}. This package is a gzip filter for %{name}.
@ -716,13 +667,10 @@ This package is a gzip filter for %{name}.
%package tar-filter %package tar-filter
Summary: Tar archive filter for %{name} Summary: Tar archive filter for %{name}
License: BSD License: BSD
Requires: %{name}-server%{?_isa} = %{version}-%{release} Requires: %{name}-server%{?_isa} = %{version}-%{release}
Requires: tar Requires: tar
Obsoletes: %{name}-tar-plugin < 1.23.9-3 Obsoletes: %{name}-tar-plugin < 1.23.9-3
%description tar-filter %description tar-filter
This package is a tar archive filter for %{name}. This package is a tar archive filter for %{name}.
@ -730,10 +678,8 @@ This package is a tar archive filter for %{name}.
%package xz-filter %package xz-filter
Summary: XZ filter for %{name} Summary: XZ filter for %{name}
License: BSD License: BSD
Requires: %{name}-server%{?_isa} = %{version}-%{release} Requires: %{name}-server%{?_isa} = %{version}-%{release}
%description xz-filter %description xz-filter
This package is the xz filter for %{name}. This package is the xz filter for %{name}.
@ -741,11 +687,9 @@ This package is the xz filter for %{name}.
%package devel %package devel
Summary: Development files and documentation for %{name} Summary: Development files and documentation for %{name}
License: BSD License: BSD
Requires: %{name}-server%{?_isa} = %{version}-%{release} Requires: %{name}-server%{?_isa} = %{version}-%{release}
Requires: pkgconfig Requires: pkgconfig
%description devel %description devel
This package contains development files and documentation This package contains development files and documentation
for %{name}. Install this package if you want to develop for %{name}. Install this package if you want to develop
@ -758,7 +702,6 @@ BuildArch: noarch
Requires: bash-completion >= 2.0 Requires: bash-completion >= 2.0
Requires: %{name}-server = %{version}-%{release} Requires: %{name}-server = %{version}-%{release}
%description bash-completion %description bash-completion
Install this package if you want intelligent bash tab-completion Install this package if you want intelligent bash tab-completion
for %{name}. for %{name}.
@ -907,11 +850,13 @@ export LIBGUESTFS_TRACE=1
%{_sbindir}/nbdkit %{_sbindir}/nbdkit
%dir %{_libdir}/%{name} %dir %{_libdir}/%{name}
%dir %{_libdir}/%{name}/plugins %dir %{_libdir}/%{name}/plugins
%{_libdir}/%{name}/plugins/nbdkit-null-plugin.so
%dir %{_libdir}/%{name}/filters %dir %{_libdir}/%{name}/filters
%{_mandir}/man1/nbdkit.1* %{_mandir}/man1/nbdkit.1*
%{_mandir}/man1/nbdkit-captive.1* %{_mandir}/man1/nbdkit-captive.1*
%{_mandir}/man1/nbdkit-client.1* %{_mandir}/man1/nbdkit-client.1*
%{_mandir}/man1/nbdkit-loop.1* %{_mandir}/man1/nbdkit-loop.1*
%{_mandir}/man1/nbdkit-null-plugin.1*
%{_mandir}/man1/nbdkit-probing.1* %{_mandir}/man1/nbdkit-probing.1*
%{_mandir}/man1/nbdkit-protocol.1* %{_mandir}/man1/nbdkit-protocol.1*
%{_mandir}/man1/nbdkit-service.1* %{_mandir}/man1/nbdkit-service.1*
@ -929,7 +874,6 @@ export LIBGUESTFS_TRACE=1
%{_libdir}/%{name}/plugins/nbdkit-full-plugin.so %{_libdir}/%{name}/plugins/nbdkit-full-plugin.so
%{_libdir}/%{name}/plugins/nbdkit-info-plugin.so %{_libdir}/%{name}/plugins/nbdkit-info-plugin.so
%{_libdir}/%{name}/plugins/nbdkit-memory-plugin.so %{_libdir}/%{name}/plugins/nbdkit-memory-plugin.so
%{_libdir}/%{name}/plugins/nbdkit-null-plugin.so
%{_libdir}/%{name}/plugins/nbdkit-ondemand-plugin.so %{_libdir}/%{name}/plugins/nbdkit-ondemand-plugin.so
%{_libdir}/%{name}/plugins/nbdkit-partitioning-plugin.so %{_libdir}/%{name}/plugins/nbdkit-partitioning-plugin.so
%{_libdir}/%{name}/plugins/nbdkit-pattern-plugin.so %{_libdir}/%{name}/plugins/nbdkit-pattern-plugin.so
@ -945,7 +889,6 @@ export LIBGUESTFS_TRACE=1
%{_mandir}/man1/nbdkit-full-plugin.1* %{_mandir}/man1/nbdkit-full-plugin.1*
%{_mandir}/man1/nbdkit-info-plugin.1* %{_mandir}/man1/nbdkit-info-plugin.1*
%{_mandir}/man1/nbdkit-memory-plugin.1* %{_mandir}/man1/nbdkit-memory-plugin.1*
%{_mandir}/man1/nbdkit-null-plugin.1*
%{_mandir}/man1/nbdkit-ondemand-plugin.1* %{_mandir}/man1/nbdkit-ondemand-plugin.1*
%{_mandir}/man1/nbdkit-partitioning-plugin.1* %{_mandir}/man1/nbdkit-partitioning-plugin.1*
%{_mandir}/man1/nbdkit-pattern-plugin.1* %{_mandir}/man1/nbdkit-pattern-plugin.1*
@ -1123,7 +1066,7 @@ export LIBGUESTFS_TRACE=1
%ifarch x86_64 %ifarch x86_64
%files vddk-plugin %files vddk-plugin
%doc README %doc README plugins/vddk/README.VDDK
%license LICENSE %license LICENSE
%{_libdir}/%{name}/plugins/nbdkit-vddk-plugin.so %{_libdir}/%{name}/plugins/nbdkit-vddk-plugin.so
%{_mandir}/man1/nbdkit-vddk-plugin.1* %{_mandir}/man1/nbdkit-vddk-plugin.1*
@ -1270,6 +1213,16 @@ export LIBGUESTFS_TRACE=1
%changelog %changelog
* Tue Nov 09 2021 Richard W.M. Jones <rjones@redhat.com> - 1.28.2-1
- Move nbdkit-null-plugin to nbdkit-server package
resolves: rhbz#2021154
- Add asynchronous support in nbdkit-vddk-plugin
resolves: rhbz#2018463
- Rebase to new stable branch version 1.28.2
resolves: rhbz#2011709
- Switch to xorriso (instead of genisoimage)
- Distribute README.VDDK in nbdkit-vddk-plugin subpackage
* Thu Aug 19 2021 Richard W.M. Jones <rjones@redhat.com> - 1.26.5-1 * Thu Aug 19 2021 Richard W.M. Jones <rjones@redhat.com> - 1.26.5-1
- Rebase along stable branch to 1.26.5 - Rebase along stable branch to 1.26.5
resolves: rhbz#1995327 resolves: rhbz#1995327