diff --git a/.gitignore b/.gitignore index 99a3396..b6ebf88 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,2 @@ SOURCES/libguestfs.keyring -SOURCES/nbdkit-1.26.5.tar.gz +SOURCES/nbdkit-1.28.2.tar.gz diff --git a/.nbdkit.metadata b/.nbdkit.metadata index 32d5418..2e5f25c 100644 --- a/.nbdkit.metadata +++ b/.nbdkit.metadata @@ -1,2 +1,2 @@ cc1b37b9cfafa515aab3eefd345ecc59aac2ce7b SOURCES/libguestfs.keyring -52a221954f374f2a3f1adcc5c5e3e6f7332d906d SOURCES/nbdkit-1.26.5.tar.gz +ee726a0b63e9a52b787de1462b11e1af34ca51ab SOURCES/nbdkit-1.28.2.tar.gz diff --git a/SOURCES/0001-server-reset-meta-context-replies-on-starttls.patch b/SOURCES/0001-server-reset-meta-context-replies-on-starttls.patch deleted file mode 100644 index fb51c55..0000000 --- a/SOURCES/0001-server-reset-meta-context-replies-on-starttls.patch +++ /dev/null @@ -1,39 +0,0 @@ -From 89ef17c90996c0e212e3a17c8d26ff930ab464ea Mon Sep 17 00:00:00 2001 -From: Eric Blake -Date: Mon, 16 Aug 2021 13:43:29 -0500 -Subject: [PATCH] server: reset meta context replies on starttls - -Related to CVE-2021-3716, but not as severe. No compliant client will -send NBD_CMD_BLOCK_STATUS unless it first negotiates -NBD_OPT_SET_META_CONTEXT. If an attacker injects a premature -SET_META_CONTEXT, either the client will never notice (because it -never uses BLOCK_STATUS), or the client will overwrite the attacker's -attempt with the client's own SET_META_CONTEXT request after -encryption is enabled. So I don't class this as having the potential -to trigger denial-of-service due to any protocol mismatch between -compliant client and server (I don't care what happens with -non-compliant clients). - -Fixes: 26455d45 (server: protocol: Implement Block Status "base:allocation".) -(cherry picked from commit 6c5faac6a37077cf2366388a80862bb00616d0d8) ---- - server/protocol-handshake-newstyle.c | 3 +++ - 1 file changed, 3 insertions(+) - -diff --git a/server/protocol-handshake-newstyle.c b/server/protocol-handshake-newstyle.c -index 7e6b7b1b..79b2c8ce 100644 ---- a/server/protocol-handshake-newstyle.c -+++ b/server/protocol-handshake-newstyle.c -@@ -497,6 +497,9 @@ negotiate_handshake_newstyle_options (void) - debug ("using TLS on this connection"); - /* Wipe out any cached state. */ - conn->structured_replies = false; -+ free (conn->exportname_from_set_meta_context); -+ conn->exportname_from_set_meta_context = NULL; -+ conn->meta_context_base_allocation = false; - for_each_backend (b) { - free (conn->default_exportname[b->i]); - conn->default_exportname[b->i] = NULL; --- -2.31.1 - diff --git a/SOURCES/0001-vddk-Refactor-how-D-vddk.stats-1-is-collected.patch b/SOURCES/0001-vddk-Refactor-how-D-vddk.stats-1-is-collected.patch new file mode 100644 index 0000000..ba66d22 --- /dev/null +++ b/SOURCES/0001-vddk-Refactor-how-D-vddk.stats-1-is-collected.patch @@ -0,0 +1,117 @@ +From 96ee8f6f2844bceb8e27ffb442359a2b7521c950 Mon Sep 17 00:00:00 2001 +From: "Richard W.M. Jones" +Date: Thu, 21 Oct 2021 14:49:52 +0100 +Subject: [PATCH] vddk: Refactor how -D vddk.stats=1 is collected + +In order to allow us to collect more per-API stats, introduce a global +struct per API for storing these stats. + +(cherry picked from commit 3d8657f3d9a2c1b59284333566428b4c7ce32a74) +--- + plugins/vddk/vddk.c | 36 ++++++++++++++++++------------------ + 1 file changed, 18 insertions(+), 18 deletions(-) + +diff --git a/plugins/vddk/vddk.c b/plugins/vddk/vddk.c +index 80f5870e..3d751544 100644 +--- a/plugins/vddk/vddk.c ++++ b/plugins/vddk/vddk.c +@@ -1,5 +1,5 @@ + /* nbdkit +- * Copyright (C) 2013-2020 Red Hat Inc. ++ * Copyright (C) 2013-2021 Red Hat Inc. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are +@@ -103,14 +103,23 @@ static bool is_remote; + /* For each VDDK API define a variable to store the time taken (used + * to implement -D vddk.stats=1). + */ ++struct vddk_stat { ++ const char *name; /* function name */ ++ int64_t usecs; /* total number of usecs consumed */ ++}; + static pthread_mutex_t stats_lock = PTHREAD_MUTEX_INITIALIZER; + static void display_stats (void); +-#define STUB(fn,ret,args) static int64_t stats_##fn; +-#define OPTIONAL_STUB(fn,ret,args) static int64_t stats_##fn; ++#define STUB(fn,ret,args) \ ++ static struct vddk_stat stats_##fn = { .name = #fn } ++#define OPTIONAL_STUB(fn,ret,args) \ ++ static struct vddk_stat stats_##fn = { .name = #fn } + #include "vddk-stubs.h" + #undef STUB + #undef OPTIONAL_STUB + ++/* Macros to bracket each VDDK API call, for printing debugging ++ * information and collecting statistics. ++ */ + #define VDDK_CALL_START(fn, fs, ...) \ + do { \ + struct timeval start_t, end_t; \ +@@ -131,10 +140,11 @@ static void display_stats (void); + if (vddk_debug_stats) { \ + gettimeofday (&end_t, NULL); \ + ACQUIRE_LOCK_FOR_CURRENT_SCOPE (&stats_lock); \ +- stats_##fn += tvdiff_usec (&start_t, &end_t); \ ++ stats_##fn.usecs += tvdiff_usec (&start_t, &end_t); \ + } \ + } while (0) + ++/* Print VDDK errors. */ + #define VDDK_ERROR(err, fs, ...) \ + do { \ + char *vddk_err_msg; \ +@@ -167,10 +177,6 @@ vddk_unload (void) + free (password); + } + +-struct vddk_stat { +- const char *fn; +- int64_t usecs; +-}; + DEFINE_VECTOR_TYPE(statlist, struct vddk_stat) + + static int +@@ -179,7 +185,7 @@ stat_compare (const void *vp1, const void *vp2) + const struct vddk_stat *st1 = vp1; + const struct vddk_stat *st2 = vp2; + +- /* Note: sorts in reverse order. */ ++ /* Note: sorts in reverse order of time spent in each API call. */ + if (st1->usecs < st2->usecs) return 1; + else if (st1->usecs > st2->usecs) return -1; + else return 0; +@@ -189,19 +195,13 @@ static void + display_stats (void) + { + statlist stats = empty_vector; +- struct vddk_stat st; + size_t i; + +-#define ADD_ONE_STAT(fn_, usecs_) \ +- st.fn = fn_; \ +- st.usecs = usecs_; \ +- statlist_append (&stats, st) +-#define STUB(fn,ret,args) ADD_ONE_STAT (#fn, stats_##fn); +-#define OPTIONAL_STUB(fn,ret,args) ADD_ONE_STAT (#fn, stats_##fn); ++#define STUB(fn,ret,args) statlist_append (&stats, stats_##fn) ++#define OPTIONAL_STUB(fn,ret,args) statlist_append (&stats, stats_##fn) + #include "vddk-stubs.h" + #undef STUB + #undef OPTIONAL_STUB +-#undef ADD_ONE_STAT + + qsort (stats.ptr, stats.size, sizeof stats.ptr[0], stat_compare); + +@@ -209,7 +209,7 @@ display_stats (void) + nbdkit_debug ("%-40s %9s", "", "µs"); + for (i = 0; i < stats.size; ++i) { + if (stats.ptr[i].usecs) +- nbdkit_debug ("%-40s %9" PRIi64, stats.ptr[i].fn, stats.ptr[i].usecs); ++ nbdkit_debug ("%-40s %9" PRIi64, stats.ptr[i].name, stats.ptr[i].usecs); + } + statlist_reset (&stats); + } +-- +2.31.1 + diff --git a/SOURCES/0002-cache-Reduce-verbosity-of-debugging.patch b/SOURCES/0002-cache-Reduce-verbosity-of-debugging.patch deleted file mode 100644 index 5ddcebc..0000000 --- a/SOURCES/0002-cache-Reduce-verbosity-of-debugging.patch +++ /dev/null @@ -1,124 +0,0 @@ -From 4b576a8e0eb99ec1a79ca432350fb7ac27a5c089 Mon Sep 17 00:00:00 2001 -From: "Richard W.M. Jones" -Date: Mon, 26 Jul 2021 11:59:43 +0100 -Subject: [PATCH] cache: Reduce verbosity of debugging - -The cache filter is very verbose in its debugging. Reduce the default -level. Use -D cache.verbose=1 to restore original debugging. - -Compare commit 745a0f13662031c2b9c9b69f62b4ae3a6b2f38f0. - -(cherry picked from commit 6be735edf7d5fb3fb8350c72e6d9525badbab14d) ---- - filters/cache/blk.c | 53 +++++++++++++++++++++++++++------------------ - 1 file changed, 32 insertions(+), 21 deletions(-) - -diff --git a/filters/cache/blk.c b/filters/cache/blk.c -index 12e8407e..f52f30e3 100644 ---- a/filters/cache/blk.c -+++ b/filters/cache/blk.c -@@ -93,6 +93,9 @@ enum bm_entry { - BLOCK_DIRTY = 3, - }; - -+/* Extra debugging (-D cache.verbose=1). */ -+NBDKIT_DLL_PUBLIC int cache_debug_verbose = 0; -+ - int - blk_init (void) - { -@@ -199,12 +202,14 @@ blk_read (nbdkit_next *next, - - reclaim (fd, &bm); - -- nbdkit_debug ("cache: blk_read block %" PRIu64 " (offset %" PRIu64 ") is %s", -- blknum, (uint64_t) offset, -- state == BLOCK_NOT_CACHED ? "not cached" : -- state == BLOCK_CLEAN ? "clean" : -- state == BLOCK_DIRTY ? "dirty" : -- "unknown"); -+ if (cache_debug_verbose) -+ nbdkit_debug ("cache: blk_read block %" PRIu64 -+ " (offset %" PRIu64 ") is %s", -+ blknum, (uint64_t) offset, -+ state == BLOCK_NOT_CACHED ? "not cached" : -+ state == BLOCK_CLEAN ? "clean" : -+ state == BLOCK_DIRTY ? "dirty" : -+ "unknown"); - - if (state == BLOCK_NOT_CACHED) { /* Read underlying plugin. */ - unsigned n = blksize, tail = 0; -@@ -225,9 +230,10 @@ blk_read (nbdkit_next *next, - - /* If cache-on-read, copy the block to the cache. */ - if (cache_on_read) { -- nbdkit_debug ("cache: cache-on-read block %" PRIu64 -- " (offset %" PRIu64 ")", -- blknum, (uint64_t) offset); -+ if (cache_debug_verbose) -+ nbdkit_debug ("cache: cache-on-read block %" PRIu64 -+ " (offset %" PRIu64 ")", -+ blknum, (uint64_t) offset); - - if (pwrite (fd, block, blksize, offset) == -1) { - *err = errno; -@@ -259,12 +265,14 @@ blk_cache (nbdkit_next *next, - - reclaim (fd, &bm); - -- nbdkit_debug ("cache: blk_cache block %" PRIu64 " (offset %" PRIu64 ") is %s", -- blknum, (uint64_t) offset, -- state == BLOCK_NOT_CACHED ? "not cached" : -- state == BLOCK_CLEAN ? "clean" : -- state == BLOCK_DIRTY ? "dirty" : -- "unknown"); -+ if (cache_debug_verbose) -+ nbdkit_debug ("cache: blk_cache block %" PRIu64 -+ " (offset %" PRIu64 ") is %s", -+ blknum, (uint64_t) offset, -+ state == BLOCK_NOT_CACHED ? "not cached" : -+ state == BLOCK_CLEAN ? "clean" : -+ state == BLOCK_DIRTY ? "dirty" : -+ "unknown"); - - if (state == BLOCK_NOT_CACHED) { - /* Read underlying plugin, copy to cache regardless of cache-on-read. */ -@@ -284,8 +292,9 @@ blk_cache (nbdkit_next *next, - */ - memset (block + n, 0, tail); - -- nbdkit_debug ("cache: cache block %" PRIu64 " (offset %" PRIu64 ")", -- blknum, (uint64_t) offset); -+ if (cache_debug_verbose) -+ nbdkit_debug ("cache: cache block %" PRIu64 " (offset %" PRIu64 ")", -+ blknum, (uint64_t) offset); - - if (pwrite (fd, block, blksize, offset) == -1) { - *err = errno; -@@ -324,8 +333,9 @@ blk_writethrough (nbdkit_next *next, - - reclaim (fd, &bm); - -- nbdkit_debug ("cache: writethrough block %" PRIu64 " (offset %" PRIu64 ")", -- blknum, (uint64_t) offset); -+ if (cache_debug_verbose) -+ nbdkit_debug ("cache: writethrough block %" PRIu64 " (offset %" PRIu64 ")", -+ blknum, (uint64_t) offset); - - if (pwrite (fd, block, blksize, offset) == -1) { - *err = errno; -@@ -357,8 +367,9 @@ blk_write (nbdkit_next *next, - - reclaim (fd, &bm); - -- nbdkit_debug ("cache: writeback block %" PRIu64 " (offset %" PRIu64 ")", -- blknum, (uint64_t) offset); -+ if (cache_debug_verbose) -+ nbdkit_debug ("cache: writeback block %" PRIu64 " (offset %" PRIu64 ")", -+ blknum, (uint64_t) offset); - - if (pwrite (fd, block, blksize, offset) == -1) { - *err = errno; --- -2.31.1 - diff --git a/SOURCES/0002-vddk-Extend-D-vddk.stats-1-to-show-number-of-calls-a.patch b/SOURCES/0002-vddk-Extend-D-vddk.stats-1-to-show-number-of-calls-a.patch new file mode 100644 index 0000000..1823655 --- /dev/null +++ b/SOURCES/0002-vddk-Extend-D-vddk.stats-1-to-show-number-of-calls-a.patch @@ -0,0 +1,140 @@ +From f388c9b6c983d395ced0d4f467980b182d0a1b84 Mon Sep 17 00:00:00 2001 +From: "Richard W.M. Jones" +Date: Thu, 21 Oct 2021 15:10:00 +0100 +Subject: [PATCH] vddk: Extend -D vddk.stats=1 to show number of calls and + bytes transferred +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +The new output looks like this: + +nbdkit: debug: VDDK function stats (-D vddk.stats=1): +nbdkit: debug: VixDiskLib_... µs calls bytes +nbdkit: debug: Exit 1000854 1 +nbdkit: debug: InitEx 79304 1 +nbdkit: debug: Flush 13577 1 +nbdkit: debug: Write 12534 21 10485760 +nbdkit: debug: Open 4753 3 +nbdkit: debug: Read 966 20 5242880 +nbdkit: debug: Close 574 3 +nbdkit: debug: QueryAllocatedBlocks 116 4 +nbdkit: debug: ConnectEx 103 3 +nbdkit: debug: Disconnect 88 3 +nbdkit: debug: GetTransportMode 68 3 +nbdkit: debug: GetInfo 46 3 +nbdkit: debug: FreeConnectParams 36 3 +nbdkit: debug: FreeInfo 36 3 +nbdkit: debug: FreeBlockList 22 4 +nbdkit: debug: AllocateConnectParams 22 3 +(cherry picked from commit 5c80f0d290db45a679d55baf37ff39bacb8ce7ec) +--- + plugins/vddk/nbdkit-vddk-plugin.pod | 3 +-- + plugins/vddk/vddk.c | 41 +++++++++++++++++++++++++---- + 2 files changed, 37 insertions(+), 7 deletions(-) + +diff --git a/plugins/vddk/nbdkit-vddk-plugin.pod b/plugins/vddk/nbdkit-vddk-plugin.pod +index 078badcc..e53d3286 100644 +--- a/plugins/vddk/nbdkit-vddk-plugin.pod ++++ b/plugins/vddk/nbdkit-vddk-plugin.pod +@@ -517,8 +517,7 @@ Suppress debugging of datapath calls (C and C). + + =item B<-D vddk.stats=1> + +-When the plugin exits print some statistics about the amount of time +-spent waiting on each VDDK call. ++When the plugin exits print some statistics about each VDDK call. + + =back + +diff --git a/plugins/vddk/vddk.c b/plugins/vddk/vddk.c +index 3d751544..5f1d223b 100644 +--- a/plugins/vddk/vddk.c ++++ b/plugins/vddk/vddk.c +@@ -106,6 +106,8 @@ static bool is_remote; + struct vddk_stat { + const char *name; /* function name */ + int64_t usecs; /* total number of usecs consumed */ ++ uint64_t calls; /* number of times called */ ++ uint64_t bytes; /* bytes transferred, datapath calls only */ + }; + static pthread_mutex_t stats_lock = PTHREAD_MUTEX_INITIALIZER; + static void display_stats (void); +@@ -141,6 +143,17 @@ static void display_stats (void); + gettimeofday (&end_t, NULL); \ + ACQUIRE_LOCK_FOR_CURRENT_SCOPE (&stats_lock); \ + stats_##fn.usecs += tvdiff_usec (&start_t, &end_t); \ ++ stats_##fn.calls++; \ ++ } \ ++ } while (0) ++#define VDDK_CALL_END_DATAPATH(fn, bytes_) \ ++ while (0); \ ++ if (vddk_debug_stats) { \ ++ gettimeofday (&end_t, NULL); \ ++ ACQUIRE_LOCK_FOR_CURRENT_SCOPE (&stats_lock); \ ++ stats_##fn.usecs += tvdiff_usec (&start_t, &end_t); \ ++ stats_##fn.calls++; \ ++ stats_##fn.bytes += bytes_; \ + } \ + } while (0) + +@@ -191,6 +204,12 @@ stat_compare (const void *vp1, const void *vp2) + else return 0; + } + ++static const char * ++api_name_without_prefix (const char *name) ++{ ++ return strncmp (name, "VixDiskLib_", 11) == 0 ? name + 11 : name; ++} ++ + static void + display_stats (void) + { +@@ -206,10 +225,22 @@ display_stats (void) + qsort (stats.ptr, stats.size, sizeof stats.ptr[0], stat_compare); + + nbdkit_debug ("VDDK function stats (-D vddk.stats=1):"); +- nbdkit_debug ("%-40s %9s", "", "µs"); ++ nbdkit_debug ("%-24s %15s %5s %15s", ++ "VixDiskLib_...", "µs", "calls", "bytes"); + for (i = 0; i < stats.size; ++i) { +- if (stats.ptr[i].usecs) +- nbdkit_debug ("%-40s %9" PRIi64, stats.ptr[i].name, stats.ptr[i].usecs); ++ if (stats.ptr[i].usecs) { ++ if (stats.ptr[i].bytes > 0) ++ nbdkit_debug (" %-22s %15" PRIi64 " %5" PRIu64 " %15" PRIu64, ++ api_name_without_prefix (stats.ptr[i].name), ++ stats.ptr[i].usecs, ++ stats.ptr[i].calls, ++ stats.ptr[i].bytes); ++ else ++ nbdkit_debug (" %-22s %15" PRIi64 " %5" PRIu64, ++ api_name_without_prefix (stats.ptr[i].name), ++ stats.ptr[i].usecs, ++ stats.ptr[i].calls); ++ } + } + statlist_reset (&stats); + } +@@ -831,7 +862,7 @@ vddk_pread (void *handle, void *buf, uint32_t count, uint64_t offset, + "%" PRIu32 " sectors, buffer", + offset, count) { + err = VixDiskLib_Read (h->handle, offset, count, buf); +- } VDDK_CALL_END (VixDiskLib_Read); ++ } VDDK_CALL_END_DATAPATH (VixDiskLib_Read, count * VIXDISKLIB_SECTOR_SIZE); + if (err != VIX_OK) { + VDDK_ERROR (err, "VixDiskLib_Read"); + return -1; +@@ -871,7 +902,7 @@ vddk_pwrite (void *handle, const void *buf, uint32_t count, uint64_t offset, + "%" PRIu32 " sectors, buffer", + offset, count) { + err = VixDiskLib_Write (h->handle, offset, count, buf); +- } VDDK_CALL_END (VixDiskLib_Write); ++ } VDDK_CALL_END_DATAPATH (VixDiskLib_Write, count * VIXDISKLIB_SECTOR_SIZE); + if (err != VIX_OK) { + VDDK_ERROR (err, "VixDiskLib_Write"); + return -1; +-- +2.31.1 + diff --git a/SOURCES/0003-cache-cow-Add-blk_read_multiple-function.patch b/SOURCES/0003-cache-cow-Add-blk_read_multiple-function.patch deleted file mode 100644 index 5b56637..0000000 --- a/SOURCES/0003-cache-cow-Add-blk_read_multiple-function.patch +++ /dev/null @@ -1,400 +0,0 @@ -From b5dc8577c5c6d1205e2106b629fad327c3a409ea Mon Sep 17 00:00:00 2001 -From: "Richard W.M. Jones" -Date: Mon, 26 Jul 2021 13:55:21 +0100 -Subject: [PATCH] cache, cow: Add blk_read_multiple function - -Currently the cache and cow filters break up large requests into many -single block-sized requests to the underlying plugin. For some -plugins (eg. curl) this is very inefficient and causes huge -slow-downs. - -For example I tested nbdkit + curl vs nbdkit + cache + curl against a -slow, remote VMware server. A simple run of virt-inspector was at -least 6-7 times slower with the cache filter. (It was so slow that I -didn't actually let it run to completion - I am estimating the -slowdown multiple using interim debug messages). - -Implement a new blk_read_multiple function in the cache filter. It -does not break up "runs" of blocks which all have the same cache -state. The cache .pread method uses the new function to read the -block-aligned part of the request. - -(cherry picked from commit ab661ccef5b3369fa22c33d0289baddc251b73bf) ---- - filters/cache/blk.c | 83 ++++++++++++++++++++++++++++++++----------- - filters/cache/blk.h | 6 ++++ - filters/cache/cache.c | 21 +++++------ - filters/cow/blk.c | 63 +++++++++++++++++++++++--------- - filters/cow/blk.h | 6 ++++ - filters/cow/cow.c | 21 +++++------ - 6 files changed, 138 insertions(+), 62 deletions(-) - -diff --git a/filters/cache/blk.c b/filters/cache/blk.c -index f52f30e3..f85ada35 100644 ---- a/filters/cache/blk.c -+++ b/filters/cache/blk.c -@@ -44,6 +44,7 @@ - #include - #include - #include -+#include - #include - - #ifdef HAVE_SYS_STATVFS_H -@@ -193,26 +194,40 @@ blk_set_size (uint64_t new_size) - return 0; - } - --int --blk_read (nbdkit_next *next, -- uint64_t blknum, uint8_t *block, int *err) -+static int -+_blk_read_multiple (nbdkit_next *next, -+ uint64_t blknum, uint64_t nrblocks, -+ uint8_t *block, int *err) - { - off_t offset = blknum * blksize; -- enum bm_entry state = bitmap_get_blk (&bm, blknum, BLOCK_NOT_CACHED); -+ bool not_cached = -+ bitmap_get_blk (&bm, blknum, BLOCK_NOT_CACHED) == BLOCK_NOT_CACHED; -+ uint64_t b, runblocks; - -- reclaim (fd, &bm); -+ assert (nrblocks > 0); - - if (cache_debug_verbose) -- nbdkit_debug ("cache: blk_read block %" PRIu64 -+ nbdkit_debug ("cache: blk_read_multiple block %" PRIu64 - " (offset %" PRIu64 ") is %s", - blknum, (uint64_t) offset, -- state == BLOCK_NOT_CACHED ? "not cached" : -- state == BLOCK_CLEAN ? "clean" : -- state == BLOCK_DIRTY ? "dirty" : -- "unknown"); -+ not_cached ? "not cached" : "cached"); - -- if (state == BLOCK_NOT_CACHED) { /* Read underlying plugin. */ -- unsigned n = blksize, tail = 0; -+ /* Find out how many of the following blocks form a "run" with the -+ * same cached/not-cached state. We can process that many blocks in -+ * one go. -+ */ -+ for (b = 1, runblocks = 1; b < nrblocks; ++b, ++runblocks) { -+ bool s = -+ bitmap_get_blk (&bm, blknum + b, BLOCK_NOT_CACHED) == BLOCK_NOT_CACHED; -+ if (not_cached != s) -+ break; -+ } -+ -+ if (not_cached) { /* Read underlying plugin. */ -+ unsigned n, tail = 0; -+ -+ assert (blksize * runblocks <= UINT_MAX); -+ n = blksize * runblocks; - - if (offset + n > size) { - tail = offset + n - size; -@@ -228,32 +243,60 @@ blk_read (nbdkit_next *next, - */ - memset (block + n, 0, tail); - -- /* If cache-on-read, copy the block to the cache. */ -+ /* If cache-on-read, copy the blocks to the cache. */ - if (cache_on_read) { - if (cache_debug_verbose) - nbdkit_debug ("cache: cache-on-read block %" PRIu64 - " (offset %" PRIu64 ")", - blknum, (uint64_t) offset); - -- if (pwrite (fd, block, blksize, offset) == -1) { -+ if (pwrite (fd, block, blksize * runblocks, offset) == -1) { - *err = errno; - nbdkit_error ("pwrite: %m"); - return -1; - } -- bitmap_set_blk (&bm, blknum, BLOCK_CLEAN); -- lru_set_recently_accessed (blknum); -+ for (b = 0; b < runblocks; ++b) { -+ bitmap_set_blk (&bm, blknum + b, BLOCK_CLEAN); -+ lru_set_recently_accessed (blknum + b); -+ } - } -- return 0; - } - else { /* Read cache. */ -- if (pread (fd, block, blksize, offset) == -1) { -+ if (pread (fd, block, blksize * runblocks, offset) == -1) { - *err = errno; - nbdkit_error ("pread: %m"); - return -1; - } -- lru_set_recently_accessed (blknum); -- return 0; -+ for (b = 0; b < runblocks; ++b) -+ lru_set_recently_accessed (blknum + b); - } -+ -+ /* If all done, return. */ -+ if (runblocks == nrblocks) -+ return 0; -+ -+ /* Recurse to read remaining blocks. */ -+ return _blk_read_multiple (next, -+ blknum + runblocks, -+ nrblocks - runblocks, -+ block + blksize * runblocks, -+ err); -+} -+ -+int -+blk_read_multiple (nbdkit_next *next, -+ uint64_t blknum, uint64_t nrblocks, -+ uint8_t *block, int *err) -+{ -+ reclaim (fd, &bm); -+ return _blk_read_multiple (next, blknum, nrblocks, block, err); -+} -+ -+int -+blk_read (nbdkit_next *next, -+ uint64_t blknum, uint8_t *block, int *err) -+{ -+ return blk_read_multiple (next, blknum, 1, block, err); - } - - int -diff --git a/filters/cache/blk.h b/filters/cache/blk.h -index 87c753e2..1ee33ed7 100644 ---- a/filters/cache/blk.h -+++ b/filters/cache/blk.h -@@ -55,6 +55,12 @@ extern int blk_read (nbdkit_next *next, - uint64_t blknum, uint8_t *block, int *err) - __attribute__((__nonnull__ (1, 3, 4))); - -+/* As above, but read multiple blocks. */ -+extern int blk_read_multiple (nbdkit_next *next, -+ uint64_t blknum, uint64_t nrblocks, -+ uint8_t *block, int *err) -+ __attribute__((__nonnull__ (1, 4, 5))); -+ - /* If a single block is not cached, copy it from the plugin. */ - extern int blk_cache (nbdkit_next *next, - uint64_t blknum, uint8_t *block, int *err) -diff --git a/filters/cache/cache.c b/filters/cache/cache.c -index 745f552d..14cc03f2 100644 ---- a/filters/cache/cache.c -+++ b/filters/cache/cache.c -@@ -313,7 +313,7 @@ cache_pread (nbdkit_next *next, - uint32_t flags, int *err) - { - CLEANUP_FREE uint8_t *block = NULL; -- uint64_t blknum, blkoffs; -+ uint64_t blknum, blkoffs, nrblocks; - int r; - - assert (!flags); -@@ -348,22 +348,17 @@ cache_pread (nbdkit_next *next, - } - - /* Aligned body */ -- /* XXX This breaks up large read requests into smaller ones, which -- * is a problem for plugins which have a large, fixed per-request -- * overhead (hello, curl). We should try to keep large requests -- * together as much as possible, but that requires us to be much -- * smarter here. -- */ -- while (count >= blksize) { -+ nrblocks = count / blksize; -+ if (nrblocks > 0) { - ACQUIRE_LOCK_FOR_CURRENT_SCOPE (&lock); -- r = blk_read (next, blknum, buf, err); -+ r = blk_read_multiple (next, blknum, nrblocks, buf, err); - if (r == -1) - return -1; - -- buf += blksize; -- count -= blksize; -- offset += blksize; -- blknum++; -+ buf += nrblocks * blksize; -+ count -= nrblocks * blksize; -+ offset += nrblocks * blksize; -+ blknum += nrblocks; - } - - /* Unaligned tail */ -diff --git a/filters/cow/blk.c b/filters/cow/blk.c -index b7c4d7f1..4ec8d1b8 100644 ---- a/filters/cow/blk.c -+++ b/filters/cow/blk.c -@@ -79,6 +79,7 @@ - #include - #include - #include -+#include - #include - #include - -@@ -219,33 +220,48 @@ blk_status (uint64_t blknum, bool *present, bool *trimmed) - *trimmed = state == BLOCK_TRIMMED; - } - --/* These are the block operations. They always read or write a single -- * whole block of size ‘blksize’. -+/* These are the block operations. They always read or write whole -+ * blocks of size ‘blksize’. - */ - int --blk_read (nbdkit_next *next, -- uint64_t blknum, uint8_t *block, int *err) -+blk_read_multiple (nbdkit_next *next, -+ uint64_t blknum, uint64_t nrblocks, -+ uint8_t *block, int *err) - { - off_t offset = blknum * BLKSIZE; - enum bm_entry state; -+ uint64_t b, runblocks; - -- /* The state might be modified from another thread - for example -- * another thread might write (BLOCK_NOT_ALLOCATED -> -- * BLOCK_ALLOCATED) while we are reading from the plugin, returning -- * the old data. However a read issued after the write returns -- * should always return the correct data. -+ /* Find out how many of the following blocks form a "run" with the -+ * same state. We can process that many blocks in one go. -+ * -+ * About the locking: The state might be modified from another -+ * thread - for example another thread might write -+ * (BLOCK_NOT_ALLOCATED -> BLOCK_ALLOCATED) while we are reading -+ * from the plugin, returning the old data. However a read issued -+ * after the write returns should always return the correct data. - */ - { - ACQUIRE_LOCK_FOR_CURRENT_SCOPE (&lock); - state = bitmap_get_blk (&bm, blknum, BLOCK_NOT_ALLOCATED); -+ -+ for (b = 1, runblocks = 1; b < nrblocks; ++b, ++runblocks) { -+ enum bm_entry s = bitmap_get_blk (&bm, blknum + b, BLOCK_NOT_ALLOCATED); -+ if (state != s) -+ break; -+ } - } - - if (cow_debug_verbose) -- nbdkit_debug ("cow: blk_read block %" PRIu64 " (offset %" PRIu64 ") is %s", -+ nbdkit_debug ("cow: blk_read_multiple block %" PRIu64 -+ " (offset %" PRIu64 ") is %s", - blknum, (uint64_t) offset, state_to_string (state)); - - if (state == BLOCK_NOT_ALLOCATED) { /* Read underlying plugin. */ -- unsigned n = BLKSIZE, tail = 0; -+ unsigned n, tail = 0; -+ -+ assert (BLKSIZE * runblocks <= UINT_MAX); -+ n = BLKSIZE * runblocks; - - if (offset + n > size) { - tail = offset + n - size; -@@ -260,20 +276,35 @@ blk_read (nbdkit_next *next, - * zeroing the tail. - */ - memset (block + n, 0, tail); -- return 0; - } - else if (state == BLOCK_ALLOCATED) { /* Read overlay. */ -- if (pread (fd, block, BLKSIZE, offset) == -1) { -+ if (pread (fd, block, BLKSIZE * runblocks, offset) == -1) { - *err = errno; - nbdkit_error ("pread: %m"); - return -1; - } -- return 0; - } - else /* state == BLOCK_TRIMMED */ { -- memset (block, 0, BLKSIZE); -- return 0; -+ memset (block, 0, BLKSIZE * runblocks); - } -+ -+ /* If all done, return. */ -+ if (runblocks == nrblocks) -+ return 0; -+ -+ /* Recurse to read remaining blocks. */ -+ return blk_read_multiple (next, -+ blknum + runblocks, -+ nrblocks - runblocks, -+ block + BLKSIZE * runblocks, -+ err); -+} -+ -+int -+blk_read (nbdkit_next *next, -+ uint64_t blknum, uint8_t *block, int *err) -+{ -+ return blk_read_multiple (next, blknum, 1, block, err); - } - - int -diff --git a/filters/cow/blk.h b/filters/cow/blk.h -index e6fd7417..b066c602 100644 ---- a/filters/cow/blk.h -+++ b/filters/cow/blk.h -@@ -55,6 +55,12 @@ extern int blk_read (nbdkit_next *next, - uint64_t blknum, uint8_t *block, int *err) - __attribute__((__nonnull__ (1, 3, 4))); - -+/* Read multiple blocks from the overlay or plugin. */ -+extern int blk_read_multiple (nbdkit_next *next, -+ uint64_t blknum, uint64_t nrblocks, -+ uint8_t *block, int *err) -+ __attribute__((__nonnull__ (1, 4, 5))); -+ - /* Cache mode for blocks not already in overlay */ - enum cache_mode { - BLK_CACHE_IGNORE, /* Do nothing */ -diff --git a/filters/cow/cow.c b/filters/cow/cow.c -index f30b7505..78daca22 100644 ---- a/filters/cow/cow.c -+++ b/filters/cow/cow.c -@@ -210,7 +210,7 @@ cow_pread (nbdkit_next *next, - uint32_t flags, int *err) - { - CLEANUP_FREE uint8_t *block = NULL; -- uint64_t blknum, blkoffs; -+ uint64_t blknum, blkoffs, nrblocks; - int r; - - if (!IS_ALIGNED (count | offset, BLKSIZE)) { -@@ -243,21 +243,16 @@ cow_pread (nbdkit_next *next, - } - - /* Aligned body */ -- /* XXX This breaks up large read requests into smaller ones, which -- * is a problem for plugins which have a large, fixed per-request -- * overhead (hello, curl). We should try to keep large requests -- * together as much as possible, but that requires us to be much -- * smarter here. -- */ -- while (count >= BLKSIZE) { -- r = blk_read (next, blknum, buf, err); -+ nrblocks = count / BLKSIZE; -+ if (nrblocks > 0) { -+ r = blk_read_multiple (next, blknum, nrblocks, buf, err); - if (r == -1) - return -1; - -- buf += BLKSIZE; -- count -= BLKSIZE; -- offset += BLKSIZE; -- blknum++; -+ buf += nrblocks * BLKSIZE; -+ count -= nrblocks * BLKSIZE; -+ offset += nrblocks * BLKSIZE; -+ blknum += nrblocks; - } - - /* Unaligned tail */ --- -2.31.1 - diff --git a/SOURCES/0003-vddk-Simplify-and-consolidate-VDDK_CALL_START-END-ma.patch b/SOURCES/0003-vddk-Simplify-and-consolidate-VDDK_CALL_START-END-ma.patch new file mode 100644 index 0000000..1d51262 --- /dev/null +++ b/SOURCES/0003-vddk-Simplify-and-consolidate-VDDK_CALL_START-END-ma.patch @@ -0,0 +1,320 @@ +From cc1c3b4ab57a1662bf87766161167fac40a78c0e Mon Sep 17 00:00:00 2001 +From: "Richard W.M. Jones" +Date: Thu, 21 Oct 2021 22:55:17 +0100 +Subject: [PATCH] vddk: Simplify and consolidate VDDK_CALL_START/END macros + +We don't need the VDDK_CALL_*_DATAPATH versions of these macros +because the compiler is able to optimize some static strcmps. +Furthermore we can remove extra { .. } when the macros are applied. + +(cherry picked from commit 3ea0ed6582faa8f800b7a2a15d58032917a21bd5) +--- + plugins/vddk/vddk.c | 124 ++++++++++++++++++++------------------------ + 1 file changed, 56 insertions(+), 68 deletions(-) + +diff --git a/plugins/vddk/vddk.c b/plugins/vddk/vddk.c +index 5f1d223b..993f2d76 100644 +--- a/plugins/vddk/vddk.c ++++ b/plugins/vddk/vddk.c +@@ -125,28 +125,16 @@ static void display_stats (void); + #define VDDK_CALL_START(fn, fs, ...) \ + do { \ + struct timeval start_t, end_t; \ ++ /* GCC can optimize this away at compile time: */ \ ++ const bool datapath = \ ++ strcmp (#fn, "VixDiskLib_Read") == 0 || \ ++ strcmp (#fn, "VixDiskLib_Write") == 0; \ + if (vddk_debug_stats) \ + gettimeofday (&start_t, NULL); \ +- nbdkit_debug ("VDDK call: %s (" fs ")", #fn, ##__VA_ARGS__); \ +- do +-#define VDDK_CALL_START_DATAPATH(fn, fs, ...) \ +- do { \ +- struct timeval start_t, end_t; \ +- if (vddk_debug_stats) \ +- gettimeofday (&start_t, NULL); \ +- if (vddk_debug_datapath) \ ++ if (!datapath || vddk_debug_datapath) \ + nbdkit_debug ("VDDK call: %s (" fs ")", #fn, ##__VA_ARGS__); \ + do +-#define VDDK_CALL_END(fn) \ +- while (0); \ +- if (vddk_debug_stats) { \ +- gettimeofday (&end_t, NULL); \ +- ACQUIRE_LOCK_FOR_CURRENT_SCOPE (&stats_lock); \ +- stats_##fn.usecs += tvdiff_usec (&start_t, &end_t); \ +- stats_##fn.calls++; \ +- } \ +- } while (0) +-#define VDDK_CALL_END_DATAPATH(fn, bytes_) \ ++#define VDDK_CALL_END(fn, bytes_) \ + while (0); \ + if (vddk_debug_stats) { \ + gettimeofday (&end_t, NULL); \ +@@ -161,13 +149,13 @@ static void display_stats (void); + #define VDDK_ERROR(err, fs, ...) \ + do { \ + char *vddk_err_msg; \ +- VDDK_CALL_START (VixDiskLib_GetErrorText, "%lu", err) { \ ++ VDDK_CALL_START (VixDiskLib_GetErrorText, "%lu", err) \ + vddk_err_msg = VixDiskLib_GetErrorText ((err), NULL); \ +- } VDDK_CALL_END (VixDiskLib_GetErrorText); \ ++ VDDK_CALL_END (VixDiskLib_GetErrorText, 0); \ + nbdkit_error (fs ": %s", ##__VA_ARGS__, vddk_err_msg); \ +- VDDK_CALL_START (VixDiskLib_FreeErrorText, "") { \ ++ VDDK_CALL_START (VixDiskLib_FreeErrorText, "") \ + VixDiskLib_FreeErrorText (vddk_err_msg); \ +- } VDDK_CALL_END (VixDiskLib_FreeErrorText); \ ++ VDDK_CALL_END (VixDiskLib_FreeErrorText, 0); \ + } while (0) + + /* Unload the plugin. */ +@@ -175,9 +163,9 @@ static void + vddk_unload (void) + { + if (init_called) { +- VDDK_CALL_START (VixDiskLib_Exit, "") { ++ VDDK_CALL_START (VixDiskLib_Exit, "") + VixDiskLib_Exit (); +- } VDDK_CALL_END (VixDiskLib_Exit); ++ VDDK_CALL_END (VixDiskLib_Exit, 0); + } + if (dl) + dlclose (dl); +@@ -572,13 +560,13 @@ vddk_after_fork (void) + VDDK_CALL_START (VixDiskLib_InitEx, + "%d, %d, &debug_fn, &error_fn, &error_fn, %s, %s", + VDDK_MAJOR, VDDK_MINOR, +- libdir, config ? : "NULL") { ++ libdir, config ? : "NULL") + err = VixDiskLib_InitEx (VDDK_MAJOR, VDDK_MINOR, + &debug_function, /* log function */ + &error_function, /* warn function */ + &error_function, /* panic function */ + libdir, config); +- } VDDK_CALL_END (VixDiskLib_InitEx); ++ VDDK_CALL_END (VixDiskLib_InitEx, 0); + if (err != VIX_OK) { + VDDK_ERROR (err, "VixDiskLib_InitEx"); + exit (EXIT_FAILURE); +@@ -640,9 +628,9 @@ allocate_connect_params (void) + VixDiskLibConnectParams *ret; + + if (VixDiskLib_AllocateConnectParams != NULL) { +- VDDK_CALL_START (VixDiskLib_AllocateConnectParams, "") { ++ VDDK_CALL_START (VixDiskLib_AllocateConnectParams, "") + ret = VixDiskLib_AllocateConnectParams (); +- } VDDK_CALL_END (VixDiskLib_AllocateConnectParams); ++ VDDK_CALL_END (VixDiskLib_AllocateConnectParams, 0); + } + else + ret = calloc (1, sizeof (VixDiskLibConnectParams)); +@@ -657,9 +645,9 @@ free_connect_params (VixDiskLibConnectParams *params) + * originally called. Otherwise use free. + */ + if (VixDiskLib_AllocateConnectParams != NULL) { +- VDDK_CALL_START (VixDiskLib_FreeConnectParams, "params") { ++ VDDK_CALL_START (VixDiskLib_FreeConnectParams, "params") + VixDiskLib_FreeConnectParams (params); +- } VDDK_CALL_END (VixDiskLib_FreeConnectParams); ++ VDDK_CALL_END (VixDiskLib_FreeConnectParams, 0); + } + else + free (params); +@@ -716,13 +704,13 @@ vddk_open (int readonly) + "h->params, %d, %s, %s, &connection", + readonly, + snapshot_moref ? : "NULL", +- transport_modes ? : "NULL") { ++ transport_modes ? : "NULL") + err = VixDiskLib_ConnectEx (h->params, + readonly, + snapshot_moref, + transport_modes, + &h->connection); +- } VDDK_CALL_END (VixDiskLib_ConnectEx); ++ VDDK_CALL_END (VixDiskLib_ConnectEx, 0); + if (err != VIX_OK) { + VDDK_ERROR (err, "VixDiskLib_ConnectEx"); + goto err1; +@@ -743,25 +731,25 @@ vddk_open (int readonly) + } + + VDDK_CALL_START (VixDiskLib_Open, +- "connection, %s, %d, &handle", filename, flags) { ++ "connection, %s, %d, &handle", filename, flags) + err = VixDiskLib_Open (h->connection, filename, flags, &h->handle); +- } VDDK_CALL_END (VixDiskLib_Open); ++ VDDK_CALL_END (VixDiskLib_Open, 0); + if (err != VIX_OK) { + VDDK_ERROR (err, "VixDiskLib_Open: %s", filename); + goto err2; + } + +- VDDK_CALL_START (VixDiskLib_GetTransportMode, "handle") { ++ VDDK_CALL_START (VixDiskLib_GetTransportMode, "handle") + transport_mode = VixDiskLib_GetTransportMode (h->handle); +- } VDDK_CALL_END (VixDiskLib_GetTransportMode); ++ VDDK_CALL_END (VixDiskLib_GetTransportMode, 0); + nbdkit_debug ("transport mode: %s", transport_mode); + + return h; + + err2: +- VDDK_CALL_START (VixDiskLib_Disconnect, "connection") { ++ VDDK_CALL_START (VixDiskLib_Disconnect, "connection") + VixDiskLib_Disconnect (h->connection); +- } VDDK_CALL_END (VixDiskLib_Disconnect); ++ VDDK_CALL_END (VixDiskLib_Disconnect, 0); + err1: + free_connect_params (h->params); + err0: +@@ -776,12 +764,12 @@ vddk_close (void *handle) + ACQUIRE_LOCK_FOR_CURRENT_SCOPE (&open_close_lock); + struct vddk_handle *h = handle; + +- VDDK_CALL_START (VixDiskLib_Close, "handle") { ++ VDDK_CALL_START (VixDiskLib_Close, "handle") + VixDiskLib_Close (h->handle); +- } VDDK_CALL_END (VixDiskLib_Close); +- VDDK_CALL_START (VixDiskLib_Disconnect, "connection") { ++ VDDK_CALL_END (VixDiskLib_Close, 0); ++ VDDK_CALL_START (VixDiskLib_Disconnect, "connection") + VixDiskLib_Disconnect (h->connection); +- } VDDK_CALL_END (VixDiskLib_Disconnect); ++ VDDK_CALL_END (VixDiskLib_Disconnect, 0); + + free_connect_params (h->params); + free (h); +@@ -796,9 +784,9 @@ vddk_get_size (void *handle) + VixError err; + uint64_t size; + +- VDDK_CALL_START (VixDiskLib_GetInfo, "handle, &info") { ++ VDDK_CALL_START (VixDiskLib_GetInfo, "handle, &info") + err = VixDiskLib_GetInfo (h->handle, &info); +- } VDDK_CALL_END (VixDiskLib_GetInfo); ++ VDDK_CALL_END (VixDiskLib_GetInfo, 0); + if (err != VIX_OK) { + VDDK_ERROR (err, "VixDiskLib_GetInfo"); + return -1; +@@ -827,9 +815,9 @@ vddk_get_size (void *handle) + info->uuid ? : "NULL"); + } + +- VDDK_CALL_START (VixDiskLib_FreeInfo, "info") { ++ VDDK_CALL_START (VixDiskLib_FreeInfo, "info") + VixDiskLib_FreeInfo (info); +- } VDDK_CALL_END (VixDiskLib_FreeInfo); ++ VDDK_CALL_END (VixDiskLib_FreeInfo, 0); + + return (int64_t) size; + } +@@ -857,12 +845,12 @@ vddk_pread (void *handle, void *buf, uint32_t count, uint64_t offset, + offset /= VIXDISKLIB_SECTOR_SIZE; + count /= VIXDISKLIB_SECTOR_SIZE; + +- VDDK_CALL_START_DATAPATH (VixDiskLib_Read, +- "handle, %" PRIu64 " sectors, " +- "%" PRIu32 " sectors, buffer", +- offset, count) { ++ VDDK_CALL_START (VixDiskLib_Read, ++ "handle, %" PRIu64 " sectors, " ++ "%" PRIu32 " sectors, buffer", ++ offset, count) + err = VixDiskLib_Read (h->handle, offset, count, buf); +- } VDDK_CALL_END_DATAPATH (VixDiskLib_Read, count * VIXDISKLIB_SECTOR_SIZE); ++ VDDK_CALL_END (VixDiskLib_Read, count * VIXDISKLIB_SECTOR_SIZE); + if (err != VIX_OK) { + VDDK_ERROR (err, "VixDiskLib_Read"); + return -1; +@@ -897,12 +885,12 @@ vddk_pwrite (void *handle, const void *buf, uint32_t count, uint64_t offset, + offset /= VIXDISKLIB_SECTOR_SIZE; + count /= VIXDISKLIB_SECTOR_SIZE; + +- VDDK_CALL_START_DATAPATH (VixDiskLib_Write, +- "handle, %" PRIu64 " sectors, " +- "%" PRIu32 " sectors, buffer", +- offset, count) { ++ VDDK_CALL_START (VixDiskLib_Write, ++ "handle, %" PRIu64 " sectors, " ++ "%" PRIu32 " sectors, buffer", ++ offset, count) + err = VixDiskLib_Write (h->handle, offset, count, buf); +- } VDDK_CALL_END_DATAPATH (VixDiskLib_Write, count * VIXDISKLIB_SECTOR_SIZE); ++ VDDK_CALL_END (VixDiskLib_Write, count * VIXDISKLIB_SECTOR_SIZE); + if (err != VIX_OK) { + VDDK_ERROR (err, "VixDiskLib_Write"); + return -1; +@@ -945,9 +933,9 @@ vddk_flush (void *handle, uint32_t flags) + * file so it appears to be the correct call to use here. + */ + +- VDDK_CALL_START (VixDiskLib_Flush, "handle") { ++ VDDK_CALL_START (VixDiskLib_Flush, "handle") + err = VixDiskLib_Flush (h->handle); +- } VDDK_CALL_END (VixDiskLib_Flush); ++ VDDK_CALL_END (VixDiskLib_Flush, 0); + if (err != VIX_OK) { + VDDK_ERROR (err, "VixDiskLib_Flush"); + return -1; +@@ -983,17 +971,17 @@ vddk_can_extents (void *handle) + */ + VDDK_CALL_START (VixDiskLib_QueryAllocatedBlocks, + "handle, 0, %d sectors, %d sectors", +- VIXDISKLIB_MIN_CHUNK_SIZE, VIXDISKLIB_MIN_CHUNK_SIZE) { ++ VIXDISKLIB_MIN_CHUNK_SIZE, VIXDISKLIB_MIN_CHUNK_SIZE) + err = VixDiskLib_QueryAllocatedBlocks (h->handle, + 0, VIXDISKLIB_MIN_CHUNK_SIZE, + VIXDISKLIB_MIN_CHUNK_SIZE, + &block_list); +- } VDDK_CALL_END (VixDiskLib_QueryAllocatedBlocks); ++ VDDK_CALL_END (VixDiskLib_QueryAllocatedBlocks, 0); + error_suppression = 0; + if (err == VIX_OK) { +- VDDK_CALL_START (VixDiskLib_FreeBlockList, "block_list") { ++ VDDK_CALL_START (VixDiskLib_FreeBlockList, "block_list") + VixDiskLib_FreeBlockList (block_list); +- } VDDK_CALL_END (VixDiskLib_FreeBlockList); ++ VDDK_CALL_END (VixDiskLib_FreeBlockList, 0); + } + if (err != VIX_OK) { + char *errmsg = VixDiskLib_GetErrorText (err, NULL); +@@ -1073,12 +1061,12 @@ vddk_extents (void *handle, uint32_t count, uint64_t offset, uint32_t flags, + VDDK_CALL_START (VixDiskLib_QueryAllocatedBlocks, + "handle, %" PRIu64 " sectors, %" PRIu64 " sectors, " + "%d sectors", +- start_sector, nr_sectors, VIXDISKLIB_MIN_CHUNK_SIZE) { ++ start_sector, nr_sectors, VIXDISKLIB_MIN_CHUNK_SIZE) + err = VixDiskLib_QueryAllocatedBlocks (h->handle, + start_sector, nr_sectors, + VIXDISKLIB_MIN_CHUNK_SIZE, + &block_list); +- } VDDK_CALL_END (VixDiskLib_QueryAllocatedBlocks); ++ VDDK_CALL_END (VixDiskLib_QueryAllocatedBlocks, 0); + if (err != VIX_OK) { + VDDK_ERROR (err, "VixDiskLib_QueryAllocatedBlocks"); + return -1; +@@ -1097,15 +1085,15 @@ vddk_extents (void *handle, uint32_t count, uint64_t offset, uint32_t flags, + add_extent (extents, &position, blk_offset, true) == -1) || + (add_extent (extents, + &position, blk_offset + blk_length, false) == -1)) { +- VDDK_CALL_START (VixDiskLib_FreeBlockList, "block_list") { ++ VDDK_CALL_START (VixDiskLib_FreeBlockList, "block_list") + VixDiskLib_FreeBlockList (block_list); +- } VDDK_CALL_END (VixDiskLib_FreeBlockList); ++ VDDK_CALL_END (VixDiskLib_FreeBlockList, 0); + return -1; + } + } +- VDDK_CALL_START (VixDiskLib_FreeBlockList, "block_list") { ++ VDDK_CALL_START (VixDiskLib_FreeBlockList, "block_list") + VixDiskLib_FreeBlockList (block_list); +- } VDDK_CALL_END (VixDiskLib_FreeBlockList); ++ VDDK_CALL_END (VixDiskLib_FreeBlockList, 0); + + /* There's an implicit hole after the returned list of blocks, up + * to the end of the QueryAllocatedBlocks request. +-- +2.31.1 + diff --git a/SOURCES/0004-cache-cow-Use-full-pread-pwrite-operations.patch b/SOURCES/0004-cache-cow-Use-full-pread-pwrite-operations.patch deleted file mode 100644 index 300aeb2..0000000 --- a/SOURCES/0004-cache-cow-Use-full-pread-pwrite-operations.patch +++ /dev/null @@ -1,215 +0,0 @@ -From 5bd332a683811586039f99f31c01d4f2f7181334 Mon Sep 17 00:00:00 2001 -From: "Richard W.M. Jones" -Date: Mon, 26 Jul 2021 15:21:18 +0100 -Subject: [PATCH] cache, cow: Use full pread/pwrite operations - -Although it probably cannot happen on Linux, POSIX allows pread/pwrite -to return or write fewer bytes than requested. The cache and cow -filters didn't handle this situation. Replace the raw -pread(2)/pwrite(2) syscalls with alternate versions which can handle -this. - -(cherry picked from commit ce0db9d7736dd28dd0f10951ce65853e50b35e41) ---- - common/utils/Makefile.am | 1 + - common/utils/full-rw.c | 81 ++++++++++++++++++++++++++++++++++++++++ - common/utils/utils.h | 2 + - filters/cache/blk.c | 10 ++--- - filters/cow/blk.c | 6 +-- - 5 files changed, 92 insertions(+), 8 deletions(-) - create mode 100644 common/utils/full-rw.c - -diff --git a/common/utils/Makefile.am b/common/utils/Makefile.am -index 1708a4c8..14e9dfc4 100644 ---- a/common/utils/Makefile.am -+++ b/common/utils/Makefile.am -@@ -40,6 +40,7 @@ libutils_la_SOURCES = \ - cleanup-nbdkit.c \ - cleanup.h \ - environ.c \ -+ full-rw.c \ - quote.c \ - utils.c \ - utils.h \ -diff --git a/common/utils/full-rw.c b/common/utils/full-rw.c -new file mode 100644 -index 00000000..55b32cdd ---- /dev/null -+++ b/common/utils/full-rw.c -@@ -0,0 +1,81 @@ -+/* nbdkit -+ * Copyright (C) 2021 Red Hat Inc. -+ * -+ * Redistribution and use in source and binary forms, with or without -+ * modification, are permitted provided that the following conditions are -+ * met: -+ * -+ * * Redistributions of source code must retain the above copyright -+ * notice, this list of conditions and the following disclaimer. -+ * -+ * * Redistributions in binary form must reproduce the above copyright -+ * notice, this list of conditions and the following disclaimer in the -+ * documentation and/or other materials provided with the distribution. -+ * -+ * * Neither the name of Red Hat nor the names of its contributors may be -+ * used to endorse or promote products derived from this software without -+ * specific prior written permission. -+ * -+ * THIS SOFTWARE IS PROVIDED BY RED HAT AND CONTRIBUTORS ''AS IS'' AND -+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, -+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A -+ * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL RED HAT OR -+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF -+ * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND -+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT -+ * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF -+ * SUCH DAMAGE. -+ */ -+ -+/* These functions are like pread(2)/pwrite(2) but they always read or -+ * write the full amount, or fail. -+ */ -+ -+#include -+ -+#include -+#include -+#include -+#include -+ -+ssize_t -+full_pread (int fd, void *buf, size_t count, off_t offset) -+{ -+ ssize_t ret = 0, r; -+ -+ while (count > 0) { -+ r = pread (fd, buf, count, offset); -+ if (r == -1) return -1; -+ if (r == 0) { -+ /* Presumably the caller wasn't expecting end-of-file here, so -+ * return an error. -+ */ -+ errno = EIO; -+ return -1; -+ } -+ ret += r; -+ offset += r; -+ count -= r; -+ } -+ -+ return ret; -+} -+ -+ssize_t -+full_pwrite (int fd, const void *buf, size_t count, off_t offset) -+{ -+ ssize_t ret = 0, r; -+ -+ while (count > 0) { -+ r = pwrite (fd, buf, count, offset); -+ if (r == -1) return -1; -+ ret += r; -+ offset += r; -+ count -= r; -+ } -+ -+ return ret; -+} -diff --git a/common/utils/utils.h b/common/utils/utils.h -index f8f70212..83397ae1 100644 ---- a/common/utils/utils.h -+++ b/common/utils/utils.h -@@ -40,5 +40,7 @@ extern int set_cloexec (int fd); - extern int set_nonblock (int fd); - extern char **copy_environ (char **env, ...) __attribute__((__sentinel__)); - extern char *make_temporary_directory (void); -+extern ssize_t full_pread (int fd, void *buf, size_t count, off_t offset); -+extern ssize_t full_pwrite (int fd, const void *buf, size_t count, off_t offset); - - #endif /* NBDKIT_UTILS_H */ -diff --git a/filters/cache/blk.c b/filters/cache/blk.c -index f85ada35..42bd3779 100644 ---- a/filters/cache/blk.c -+++ b/filters/cache/blk.c -@@ -250,7 +250,7 @@ _blk_read_multiple (nbdkit_next *next, - " (offset %" PRIu64 ")", - blknum, (uint64_t) offset); - -- if (pwrite (fd, block, blksize * runblocks, offset) == -1) { -+ if (full_pwrite (fd, block, blksize * runblocks, offset) == -1) { - *err = errno; - nbdkit_error ("pwrite: %m"); - return -1; -@@ -262,7 +262,7 @@ _blk_read_multiple (nbdkit_next *next, - } - } - else { /* Read cache. */ -- if (pread (fd, block, blksize * runblocks, offset) == -1) { -+ if (full_pread (fd, block, blksize * runblocks, offset) == -1) { - *err = errno; - nbdkit_error ("pread: %m"); - return -1; -@@ -339,7 +339,7 @@ blk_cache (nbdkit_next *next, - nbdkit_debug ("cache: cache block %" PRIu64 " (offset %" PRIu64 ")", - blknum, (uint64_t) offset); - -- if (pwrite (fd, block, blksize, offset) == -1) { -+ if (full_pwrite (fd, block, blksize, offset) == -1) { - *err = errno; - nbdkit_error ("pwrite: %m"); - return -1; -@@ -380,7 +380,7 @@ blk_writethrough (nbdkit_next *next, - nbdkit_debug ("cache: writethrough block %" PRIu64 " (offset %" PRIu64 ")", - blknum, (uint64_t) offset); - -- if (pwrite (fd, block, blksize, offset) == -1) { -+ if (full_pwrite (fd, block, blksize, offset) == -1) { - *err = errno; - nbdkit_error ("pwrite: %m"); - return -1; -@@ -414,7 +414,7 @@ blk_write (nbdkit_next *next, - nbdkit_debug ("cache: writeback block %" PRIu64 " (offset %" PRIu64 ")", - blknum, (uint64_t) offset); - -- if (pwrite (fd, block, blksize, offset) == -1) { -+ if (full_pwrite (fd, block, blksize, offset) == -1) { - *err = errno; - nbdkit_error ("pwrite: %m"); - return -1; -diff --git a/filters/cow/blk.c b/filters/cow/blk.c -index 4ec8d1b8..121b0dd4 100644 ---- a/filters/cow/blk.c -+++ b/filters/cow/blk.c -@@ -278,7 +278,7 @@ blk_read_multiple (nbdkit_next *next, - memset (block + n, 0, tail); - } - else if (state == BLOCK_ALLOCATED) { /* Read overlay. */ -- if (pread (fd, block, BLKSIZE * runblocks, offset) == -1) { -+ if (full_pread (fd, block, BLKSIZE * runblocks, offset) == -1) { - *err = errno; - nbdkit_error ("pread: %m"); - return -1; -@@ -353,7 +353,7 @@ blk_cache (nbdkit_next *next, - memset (block + n, 0, tail); - - if (mode == BLK_CACHE_COW) { -- if (pwrite (fd, block, BLKSIZE, offset) == -1) { -+ if (full_pwrite (fd, block, BLKSIZE, offset) == -1) { - *err = errno; - nbdkit_error ("pwrite: %m"); - return -1; -@@ -372,7 +372,7 @@ blk_write (uint64_t blknum, const uint8_t *block, int *err) - nbdkit_debug ("cow: blk_write block %" PRIu64 " (offset %" PRIu64 ")", - blknum, (uint64_t) offset); - -- if (pwrite (fd, block, BLKSIZE, offset) == -1) { -+ if (full_pwrite (fd, block, BLKSIZE, offset) == -1) { - *err = errno; - nbdkit_error ("pwrite: %m"); - return -1; --- -2.31.1 - diff --git a/SOURCES/0004-vddk-Document-troubleshooting-performance-problems.patch b/SOURCES/0004-vddk-Document-troubleshooting-performance-problems.patch new file mode 100644 index 0000000..48fe653 --- /dev/null +++ b/SOURCES/0004-vddk-Document-troubleshooting-performance-problems.patch @@ -0,0 +1,84 @@ +From 4bd9926c0e506fdb04976d348b1c7614865c8b06 Mon Sep 17 00:00:00 2001 +From: "Richard W.M. Jones" +Date: Fri, 22 Oct 2021 18:00:27 +0100 +Subject: [PATCH] vddk: Document troubleshooting performance problems + +Document how to use -D vddk.stats=1 to diagnose performance problems +with VDDK. + +(cherry picked from commit e491978c193f49010cc28ad344d0fb3c1b5ede35) +--- + plugins/vddk/nbdkit-vddk-plugin.pod | 57 +++++++++++++++++++++++++++++ + 1 file changed, 57 insertions(+) + +diff --git a/plugins/vddk/nbdkit-vddk-plugin.pod b/plugins/vddk/nbdkit-vddk-plugin.pod +index e53d3286..5a426135 100644 +--- a/plugins/vddk/nbdkit-vddk-plugin.pod ++++ b/plugins/vddk/nbdkit-vddk-plugin.pod +@@ -475,6 +475,63 @@ and restarting the C service: + + For more information see L. + ++=head2 Troubleshooting performance problems ++ ++VDDK has very uneven performance with some operations being very slow. ++This plugin has options to allow you to debug performance issues. If ++your application has a debug or diagnostic setting, add the following ++nbdkit command line options: ++ ++ -v -D nbdkit.backend.datapath=0 -D vddk.datapath=0 -D vddk.stats=1 ++ ++C<-v> enables verbose messages and the two datapath options I ++the very verbose per-read/-write messages. C<-D vddk.stats=1> enables ++a summary when nbdkit exits of the cumulative time taken in each VDDK ++function, the number of times each function was called, and (for read ++and write) the number of bytes transferred. An example of what those ++stats look like can be found here: ++L ++ ++You can interpret the stats as follows: ++ ++=over 4 ++ ++=item C ++ ++The cumulative time spent waiting for VDDK to return from ++C calls, the number of times this function was ++called, and the total bytes read. You can use this to determine the ++read bandwidth to the VMware server. ++ ++=item C ++ ++=item C ++ ++Same as above, but for writing and flushing writes. ++ ++=item C ++ ++This call is used to query information about the sparseness of the ++remote disk. It is only available in VDDK E 6.7. The call is ++notably very slow in all versions of VMware we have tested. ++ ++=item C ++ ++=item C ++ ++=item C ++ ++=item C ++ ++=item C ++ ++=item C ++ ++The cumulative time spent connecting and disconnecting from the VMware ++server, which can also be very slow. ++ ++=back ++ + =head1 SUPPORTED VERSIONS OF VDDK + + This plugin requires VDDK E 5.5.5, which in turn means that it +-- +2.31.1 + diff --git a/SOURCES/0005-cache-Implement-cache-on-read-PATH.patch b/SOURCES/0005-cache-Implement-cache-on-read-PATH.patch deleted file mode 100644 index 559a659..0000000 --- a/SOURCES/0005-cache-Implement-cache-on-read-PATH.patch +++ /dev/null @@ -1,151 +0,0 @@ -From 4db23fd29af0488aa9c7e01577a5be9565a4465e Mon Sep 17 00:00:00 2001 -From: "Richard W.M. Jones" -Date: Mon, 26 Jul 2021 16:16:15 +0100 -Subject: [PATCH] cache: Implement cache-on-read=/PATH - -For virt-v2v we will need to be able to turn cache-on-read on while -performing inspection and modification of the guest, and off when -doing the bulk copy. To do that allow the cache-on-read parameter to -refer to a path where the existence of the path toggles the feature. - -(We could restart nbdkit between these phases, but this change avoids -doing that.) - -(cherry picked from commit c8b575241b15b3bf0adaf15313e67e5ed4270b5a) ---- - filters/cache/blk.c | 2 +- - filters/cache/cache.c | 33 ++++++++++++++++++++------- - filters/cache/cache.h | 10 ++++++-- - filters/cache/nbdkit-cache-filter.pod | 11 ++++++++- - 4 files changed, 44 insertions(+), 12 deletions(-) - -diff --git a/filters/cache/blk.c b/filters/cache/blk.c -index 42bd3779..19f79605 100644 ---- a/filters/cache/blk.c -+++ b/filters/cache/blk.c -@@ -244,7 +244,7 @@ _blk_read_multiple (nbdkit_next *next, - memset (block + n, 0, tail); - - /* If cache-on-read, copy the blocks to the cache. */ -- if (cache_on_read) { -+ if (cache_on_read ()) { - if (cache_debug_verbose) - nbdkit_debug ("cache: cache-on-read block %" PRIu64 - " (offset %" PRIu64 ")", -diff --git a/filters/cache/cache.c b/filters/cache/cache.c -index 14cc03f2..44da0008 100644 ---- a/filters/cache/cache.c -+++ b/filters/cache/cache.c -@@ -74,7 +74,8 @@ unsigned blksize; - enum cache_mode cache_mode = CACHE_MODE_WRITEBACK; - int64_t max_size = -1; - unsigned hi_thresh = 95, lo_thresh = 80; --bool cache_on_read = false; -+enum cor_mode cor_mode = COR_OFF; -+const char *cor_path; - - static int cache_flush (nbdkit_next *next, void *handle, uint32_t flags, - int *err); -@@ -161,12 +162,16 @@ cache_config (nbdkit_next_config *next, nbdkit_backend *nxdata, - } - #endif /* !HAVE_CACHE_RECLAIM */ - else if (strcmp (key, "cache-on-read") == 0) { -- int r; -- -- r = nbdkit_parse_bool (value); -- if (r == -1) -- return -1; -- cache_on_read = r; -+ if (value[0] == '/') { -+ cor_path = value; -+ cor_mode = COR_PATH; -+ } -+ else { -+ int r = nbdkit_parse_bool (value); -+ if (r == -1) -+ return -1; -+ cor_mode = r ? COR_ON : COR_OFF; -+ } - return 0; - } - else { -@@ -177,7 +182,7 @@ cache_config (nbdkit_next_config *next, nbdkit_backend *nxdata, - #define cache_config_help_common \ - "cache=MODE Set cache MODE, one of writeback (default),\n" \ - " writethrough, or unsafe.\n" \ -- "cache-on-read=BOOL Set to true to cache on reads (default false).\n" -+ "cache-on-read=BOOL|/PATH Set to true to cache on reads (default false).\n" - #ifndef HAVE_CACHE_RECLAIM - #define cache_config_help cache_config_help_common - #else -@@ -187,6 +192,18 @@ cache_config (nbdkit_next_config *next, nbdkit_backend *nxdata, - "cache-low-threshold=PCT Percentage of max size where reclaim ends.\n" - #endif - -+/* Decide if cache-on-read is currently on or off. */ -+bool -+cache_on_read (void) -+{ -+ switch (cor_mode) { -+ case COR_ON: return true; -+ case COR_OFF: return false; -+ case COR_PATH: return access (cor_path, F_OK) == 0; -+ default: abort (); -+ } -+} -+ - static int - cache_config_complete (nbdkit_next_config_complete *next, - nbdkit_backend *nxdata) -diff --git a/filters/cache/cache.h b/filters/cache/cache.h -index 2b72221f..a559adef 100644 ---- a/filters/cache/cache.h -+++ b/filters/cache/cache.h -@@ -49,7 +49,13 @@ extern unsigned blksize; - extern int64_t max_size; - extern unsigned hi_thresh, lo_thresh; - --/* Cache read requests. */ --extern bool cache_on_read; -+/* Cache on read mode. */ -+extern enum cor_mode { -+ COR_OFF, -+ COR_ON, -+ COR_PATH, -+} cor_mode; -+extern const char *cor_path; -+extern bool cache_on_read (void); - - #endif /* NBDKIT_CACHE_H */ -diff --git a/filters/cache/nbdkit-cache-filter.pod b/filters/cache/nbdkit-cache-filter.pod -index ebcf1d10..f20cb9ce 100644 ---- a/filters/cache/nbdkit-cache-filter.pod -+++ b/filters/cache/nbdkit-cache-filter.pod -@@ -8,7 +8,7 @@ nbdkit-cache-filter - nbdkit caching filter - [cache-max-size=SIZE] - [cache-high-threshold=N] - [cache-low-threshold=N] -- [cache-on-read=true|false] -+ [cache-on-read=true|false|/PATH] - [plugin-args...] - - =head1 DESCRIPTION -@@ -87,6 +87,15 @@ the plugin. - - Do not cache read requests (this is the default). - -+=item B -+ -+(nbdkit E 1.28) -+ -+When F (which must be an absolute path) exists, this behaves -+like C, and when it does not exist like -+C. This allows you to control the cache-on-read -+behaviour while nbdkit is running. -+ - =back - - =head1 CACHE MAXIMUM SIZE --- -2.31.1 - diff --git a/SOURCES/0005-vddk-Include-VDDK-major-library-version-in-dump-plug.patch b/SOURCES/0005-vddk-Include-VDDK-major-library-version-in-dump-plug.patch new file mode 100644 index 0000000..373a0a5 --- /dev/null +++ b/SOURCES/0005-vddk-Include-VDDK-major-library-version-in-dump-plug.patch @@ -0,0 +1,141 @@ +From eb6ccb03d0ca12ef19e5705cd96f81824910087b Mon Sep 17 00:00:00 2001 +From: "Richard W.M. Jones" +Date: Sat, 23 Oct 2021 16:16:39 +0100 +Subject: [PATCH] vddk: Include VDDK major library version in --dump-plugin + output + +Although it doesn't seem to be possible to get the precise VDDK +version, With a relatively simple change we can at least return the +VDDK major version. Currently this can be 5, 6 or 7. + +(cherry picked from commit 8700649d147948897f3b97810a1dff37924bdd6e) +--- + plugins/vddk/nbdkit-vddk-plugin.pod | 4 ++++ + plugins/vddk/vddk.c | 29 +++++++++++++++++++---------- + tests/test-vddk-real-dump-plugin.sh | 2 ++ + 3 files changed, 25 insertions(+), 10 deletions(-) + +diff --git a/plugins/vddk/nbdkit-vddk-plugin.pod b/plugins/vddk/nbdkit-vddk-plugin.pod +index 5a426135..bc3c3c94 100644 +--- a/plugins/vddk/nbdkit-vddk-plugin.pod ++++ b/plugins/vddk/nbdkit-vddk-plugin.pod +@@ -422,6 +422,10 @@ at runtime. + If this is printed then the C parameter is supported + by this build. + ++=item C ++ ++The VDDK major library version: 5, 6, 7, ... ++ + =item C + + Prints the full path to the VDDK shared library. Since this requires +diff --git a/plugins/vddk/vddk.c b/plugins/vddk/vddk.c +index 993f2d76..d74a484d 100644 +--- a/plugins/vddk/vddk.c ++++ b/plugins/vddk/vddk.c +@@ -81,6 +81,7 @@ NBDKIT_DLL_PUBLIC int vddk_debug_stats; + static void *dl; /* dlopen handle */ + static bool init_called; /* was InitEx called */ + static __thread int error_suppression; /* threadlocal error suppression */ ++static int library_version; /* VDDK major: 5, 6, 7, ... */ + + static enum { NONE = 0, ZLIB, FASTLZ, SKIPZ } compression; /* compression */ + static char *config; /* config */ +@@ -405,7 +406,10 @@ vddk_config (const char *key, const char *value) + static void + load_library (bool load_error_is_fatal) + { +- static const char *sonames[] = { ++ static struct { ++ const char *soname; ++ int library_version; ++ } libs[] = { + /* Prefer the newest library in case multiple exist. Check two + * possible directories: the usual VDDK installation puts .so + * files in an arch-specific subdirectory of $libdir (our minimum +@@ -413,12 +417,13 @@ load_library (bool load_error_is_fatal) + * but our testsuite is easier to write if we point libdir + * directly to a stub .so. + */ +- "lib64/libvixDiskLib.so.7", +- "libvixDiskLib.so.7", +- "lib64/libvixDiskLib.so.6", +- "libvixDiskLib.so.6", +- "lib64/libvixDiskLib.so.5", +- "libvixDiskLib.so.5", ++ { "lib64/libvixDiskLib.so.7", 7 }, ++ { "libvixDiskLib.so.7", 7 }, ++ { "lib64/libvixDiskLib.so.6", 6 }, ++ { "libvixDiskLib.so.6", 6 }, ++ { "lib64/libvixDiskLib.so.5", 5 }, ++ { "libvixDiskLib.so.5", 5 }, ++ { NULL } + }; + size_t i; + CLEANUP_FREE char *orig_error = NULL; +@@ -431,19 +436,20 @@ load_library (bool load_error_is_fatal) + } + } + +- for (i = 0; i < sizeof sonames / sizeof sonames[0]; ++i) { ++ for (i = 0; libs[i].soname != NULL; ++i) { + CLEANUP_FREE char *path; + + /* Set the full path so that dlopen will preferentially load the + * system libraries from the same directory. + */ +- if (asprintf (&path, "%s/%s", libdir, sonames[i]) == -1) { ++ if (asprintf (&path, "%s/%s", libdir, libs[i].soname) == -1) { + nbdkit_error ("asprintf: %m"); + exit (EXIT_FAILURE); + } + + dl = dlopen (path, RTLD_NOW); + if (dl != NULL) { ++ library_version = libs[i].library_version; + /* Now that we found the library, ensure that LD_LIBRARY_PATH + * includes its directory for all future loads. This may modify + * path in-place and/or re-exec nbdkit, but that's okay. +@@ -464,10 +470,12 @@ load_library (bool load_error_is_fatal) + "If '%s' is located on a non-standard path you may need to\n" + "set libdir=/path/to/vmware-vix-disklib-distrib.\n\n" + "See nbdkit-vddk-plugin(1) man page section \"LIBRARY LOCATION\" for details.", +- orig_error ? : "(unknown error)", sonames[0]); ++ orig_error ? : "(unknown error)", libs[0].soname); + exit (EXIT_FAILURE); + } + ++ assert (library_version >= 5); ++ + /* Load symbols. */ + #define STUB(fn,ret,args) \ + do { \ +@@ -583,6 +591,7 @@ vddk_dump_plugin (void) + + printf ("vddk_default_libdir=%s\n", VDDK_LIBDIR); + printf ("vddk_has_nfchostport=1\n"); ++ printf ("vddk_library_version=%d\n", library_version); + + #if defined(HAVE_DLADDR) + /* It would be nice to print the version of VDDK from the shared +diff --git a/tests/test-vddk-real-dump-plugin.sh b/tests/test-vddk-real-dump-plugin.sh +index 2cb7724e..0a079c6c 100755 +--- a/tests/test-vddk-real-dump-plugin.sh ++++ b/tests/test-vddk-real-dump-plugin.sh +@@ -58,10 +58,12 @@ rm -f $files + cleanup_fn rm -f $files + + nbdkit -f -v vddk libdir="$vddkdir" --dump-plugin > $out ++cat $out + + # Check the vddk_* entries are set. + grep ^vddk_default_libdir= $out + grep ^vddk_has_nfchostport= $out ++grep ^vddk_library_version= $out + grep ^vddk_dll= $out + + dll="$(grep ^vddk_dll $out | cut -d= -f2)" +-- +2.31.1 + diff --git a/SOURCES/0006-cache-Add-cache-min-block-size-parameter.patch b/SOURCES/0006-cache-Add-cache-min-block-size-parameter.patch deleted file mode 100644 index 4ca0778..0000000 --- a/SOURCES/0006-cache-Add-cache-min-block-size-parameter.patch +++ /dev/null @@ -1,278 +0,0 @@ -From f7f4b71d559dc6950bc795742f64e8eaeeadf3ec Mon Sep 17 00:00:00 2001 -From: "Richard W.M. Jones" -Date: Mon, 26 Jul 2021 16:30:26 +0100 -Subject: [PATCH] cache: Add cache-min-block-size parameter - -This allows you to choose a larger block size. I found experimentally -that this improves performance because of locality in access patterns. -The idea came from qcow2 which implicitly does the same thing because -of the relatively large cluster size (32K). - -nbdkit + cache-filter with 4K block size + cache-on-read + curl -(to a very slow remote site): -=> virt-inspector took 22 mins - -same with 64K block size: -=> virt-inspector took 19 mins - -However compared to a qcow2 file using qemu's copy-on-read, backed -with nbdkit + curl we are still a lot slower, possibly because having -the cache inside virt-inspector greatly reduces round trip overhead: -=> virt-inspector took 13 mins - -(cherry picked from commit 4ceacb6caa64e12bd78af5f90e86ee591e055944) ---- - filters/cache/blk.c | 2 +- - filters/cache/cache.c | 36 ++++++++++---- - filters/cache/cache.h | 3 ++ - filters/cache/nbdkit-cache-filter.pod | 9 ++++ - tests/Makefile.am | 2 + - tests/test-cache-block-size.sh | 70 +++++++++++++++++++++++++++ - 6 files changed, 112 insertions(+), 10 deletions(-) - create mode 100755 tests/test-cache-block-size.sh - -diff --git a/filters/cache/blk.c b/filters/cache/blk.c -index 19f79605..6276985f 100644 ---- a/filters/cache/blk.c -+++ b/filters/cache/blk.c -@@ -149,7 +149,7 @@ blk_init (void) - nbdkit_error ("fstatvfs: %s: %m", tmpdir); - return -1; - } -- blksize = MAX (4096, statvfs.f_bsize); -+ blksize = MAX (min_block_size, statvfs.f_bsize); - nbdkit_debug ("cache: block size: %u", blksize); - - bitmap_init (&bm, blksize, 2 /* bits per block */); -diff --git a/filters/cache/cache.c b/filters/cache/cache.c -index 44da0008..109ac89e 100644 ---- a/filters/cache/cache.c -+++ b/filters/cache/cache.c -@@ -40,6 +40,7 @@ - #include - #include - #include -+#include - #include - #include - #include -@@ -62,6 +63,7 @@ - #include "blk.h" - #include "reclaim.h" - #include "isaligned.h" -+#include "ispowerof2.h" - #include "minmax.h" - #include "rounding.h" - -@@ -70,7 +72,8 @@ - */ - static pthread_mutex_t lock = PTHREAD_MUTEX_INITIALIZER; - --unsigned blksize; -+unsigned blksize; /* actual block size (picked by blk.c) */ -+unsigned min_block_size = 4096; - enum cache_mode cache_mode = CACHE_MODE_WRITEBACK; - int64_t max_size = -1; - unsigned hi_thresh = 95, lo_thresh = 80; -@@ -80,13 +83,6 @@ const char *cor_path; - static int cache_flush (nbdkit_next *next, void *handle, uint32_t flags, - int *err); - --static void --cache_load (void) --{ -- if (blk_init () == -1) -- exit (EXIT_FAILURE); --} -- - static void - cache_unload (void) - { -@@ -116,6 +112,19 @@ cache_config (nbdkit_next_config *next, nbdkit_backend *nxdata, - return -1; - } - } -+ else if (strcmp (key, "cache-min-block-size") == 0) { -+ int64_t r; -+ -+ r = nbdkit_parse_size (value); -+ if (r == -1) -+ return -1; -+ if (r < 4096 || !is_power_of_2 (r) || r > UINT_MAX) { -+ nbdkit_error ("cache-min-block-size is not a power of 2, or is too small or too large"); -+ return -1; -+ } -+ min_block_size = r; -+ return 0; -+ } - #ifdef HAVE_CACHE_RECLAIM - else if (strcmp (key, "cache-max-size") == 0) { - int64_t r; -@@ -220,6 +229,15 @@ cache_config_complete (nbdkit_next_config_complete *next, - return next (nxdata); - } - -+static int -+cache_get_ready (int thread_model) -+{ -+ if (blk_init () == -1) -+ return -1; -+ -+ return 0; -+} -+ - /* Get the file size, set the cache size. */ - static int64_t - cache_get_size (nbdkit_next *next, -@@ -691,11 +709,11 @@ cache_cache (nbdkit_next *next, - static struct nbdkit_filter filter = { - .name = "cache", - .longname = "nbdkit caching filter", -- .load = cache_load, - .unload = cache_unload, - .config = cache_config, - .config_complete = cache_config_complete, - .config_help = cache_config_help, -+ .get_ready = cache_get_ready, - .prepare = cache_prepare, - .get_size = cache_get_size, - .can_cache = cache_can_cache, -diff --git a/filters/cache/cache.h b/filters/cache/cache.h -index a559adef..5c32c37c 100644 ---- a/filters/cache/cache.h -+++ b/filters/cache/cache.h -@@ -45,6 +45,9 @@ extern enum cache_mode { - /* Size of a block in the cache. */ - extern unsigned blksize; - -+/* Minimum block size (cache-min-block-size parameter). */ -+extern unsigned min_block_size; -+ - /* Maximum size of the cache and high/low thresholds. */ - extern int64_t max_size; - extern unsigned hi_thresh, lo_thresh; -diff --git a/filters/cache/nbdkit-cache-filter.pod b/filters/cache/nbdkit-cache-filter.pod -index f20cb9ce..6cbd1c08 100644 ---- a/filters/cache/nbdkit-cache-filter.pod -+++ b/filters/cache/nbdkit-cache-filter.pod -@@ -5,6 +5,7 @@ nbdkit-cache-filter - nbdkit caching filter - =head1 SYNOPSIS - - nbdkit --filter=cache plugin [cache=writeback|writethrough|unsafe] -+ [cache-min-block-size=SIZE] - [cache-max-size=SIZE] - [cache-high-threshold=N] - [cache-low-threshold=N] -@@ -59,6 +60,14 @@ This is dangerous and can cause data loss, but this may be acceptable - if you only use it for testing or with data that you don't care about - or can cheaply reconstruct. - -+=item BSIZE -+ -+Set the minimum block size used by the cache. This must be a power of -+2 and E 4096. -+ -+The default is 4096, or the block size of the filesystem which -+contains the temporary file storing the cache (whichever is larger). -+ - =item BSIZE - - =item BN -diff --git a/tests/Makefile.am b/tests/Makefile.am -index 9630205d..a038eabc 100644 ---- a/tests/Makefile.am -+++ b/tests/Makefile.am -@@ -1371,12 +1371,14 @@ EXTRA_DIST += test-blocksize.sh test-blocksize-extents.sh - # cache filter test. - TESTS += \ - test-cache.sh \ -+ test-cache-block-size.sh \ - test-cache-on-read.sh \ - test-cache-max-size.sh \ - test-cache-unaligned.sh \ - $(NULL) - EXTRA_DIST += \ - test-cache.sh \ -+ test-cache-block-size.sh \ - test-cache-on-read.sh \ - test-cache-max-size.sh \ - test-cache-unaligned.sh \ -diff --git a/tests/test-cache-block-size.sh b/tests/test-cache-block-size.sh -new file mode 100755 -index 00000000..a2a27407 ---- /dev/null -+++ b/tests/test-cache-block-size.sh -@@ -0,0 +1,70 @@ -+#!/usr/bin/env bash -+# nbdkit -+# Copyright (C) 2018-2021 Red Hat Inc. -+# -+# Redistribution and use in source and binary forms, with or without -+# modification, are permitted provided that the following conditions are -+# met: -+# -+# * Redistributions of source code must retain the above copyright -+# notice, this list of conditions and the following disclaimer. -+# -+# * Redistributions in binary form must reproduce the above copyright -+# notice, this list of conditions and the following disclaimer in the -+# documentation and/or other materials provided with the distribution. -+# -+# * Neither the name of Red Hat nor the names of its contributors may be -+# used to endorse or promote products derived from this software without -+# specific prior written permission. -+# -+# THIS SOFTWARE IS PROVIDED BY RED HAT AND CONTRIBUTORS ''AS IS'' AND -+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, -+# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A -+# PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL RED HAT OR -+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF -+# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND -+# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -+# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT -+# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF -+# SUCH DAMAGE. -+ -+source ./functions.sh -+set -e -+set -x -+ -+requires_filter cache -+requires_nbdsh_uri -+ -+sock=$(mktemp -u /tmp/nbdkit-test-sock.XXXXXX) -+files="cache-block-size.img $sock cache-block-size.pid" -+rm -f $files -+cleanup_fn rm -f $files -+ -+# Create an empty base image. -+truncate -s 128K cache-block-size.img -+ -+# Run nbdkit with the caching filter. -+start_nbdkit -P cache-block-size.pid -U $sock --filter=cache \ -+ file cache-block-size.img cache-min-block-size=64K -+ -+nbdsh --connect "nbd+unix://?socket=$sock" \ -+ -c ' -+# Write some pattern data to the overlay and check it reads back OK. -+buf = b"abcd" * 16384 -+h.pwrite(buf, 32768) -+zero = h.pread(32768, 0) -+assert zero == bytearray(32768) -+buf2 = h.pread(65536, 32768) -+assert buf == buf2 -+ -+# Flushing should write through to the underlying file. -+h.flush() -+ -+with open("cache-block-size.img", "rb") as file: -+ zero = file.read(32768) -+ assert zero == bytearray(32768) -+ buf2 = file.read(65536) -+ assert buf == buf2 -+' --- -2.31.1 - diff --git a/SOURCES/0006-vddk-Add-logical-and-physical-sector-size-to-D-vddk..patch b/SOURCES/0006-vddk-Add-logical-and-physical-sector-size-to-D-vddk..patch new file mode 100644 index 0000000..0709f63 --- /dev/null +++ b/SOURCES/0006-vddk-Add-logical-and-physical-sector-size-to-D-vddk..patch @@ -0,0 +1,52 @@ +From 0139f1815e9259fa789d84d2f32d30ee59bd728c Mon Sep 17 00:00:00 2001 +From: "Richard W.M. Jones" +Date: Sat, 23 Oct 2021 16:24:27 +0100 +Subject: [PATCH] vddk: Add logical and physical sector size to -D + vddk.diskinfo output + +In VDDK >= 7 it is possible to display the logical and physical sector +size in debug output. + +This commit also extends the test since this flag was not tested +before. + +(cherry picked from commit 5bb8f0586e1faabcbf4f43d722a3b3cb5b352e33) +--- + plugins/vddk/vddk.c | 6 ++++++ + tests/test-vddk-real.sh | 3 ++- + 2 files changed, 8 insertions(+), 1 deletion(-) + +diff --git a/plugins/vddk/vddk.c b/plugins/vddk/vddk.c +index d74a484d..50bdde26 100644 +--- a/plugins/vddk/vddk.c ++++ b/plugins/vddk/vddk.c +@@ -822,6 +822,12 @@ vddk_get_size (void *handle) + info->parentFileNameHint ? : "NULL"); + nbdkit_debug ("disk info: uuid: %s", + info->uuid ? : "NULL"); ++ if (library_version >= 7) { ++ nbdkit_debug ("disk info: sectory size: " ++ "logical %" PRIu32 " physical %" PRIu32, ++ info->logicalSectorSize, ++ info->physicalSectorSize); ++ } + } + + VDDK_CALL_START (VixDiskLib_FreeInfo, "info") +diff --git a/tests/test-vddk-real.sh b/tests/test-vddk-real.sh +index a6aceac9..ae965245 100755 +--- a/tests/test-vddk-real.sh ++++ b/tests/test-vddk-real.sh +@@ -89,7 +89,8 @@ if grep 'cannot open shared object file' $log; then + fi + + # Now run nbdkit for the test. +-start_nbdkit -P $pid -U $sock -D vddk.stats=1 vddk libdir="$vddkdir" $vmdk ++start_nbdkit -P $pid -U $sock -D vddk.stats=1 -D vddk.diskinfo=1 \ ++ vddk libdir="$vddkdir" $vmdk + uri="nbd+unix:///?socket=$sock" + + # VDDK < 6.0 did not support flush, so disable flush test there. Also +-- +2.31.1 + diff --git a/SOURCES/0007-cache-cow-Use-a-64K-block-size-by-default.patch b/SOURCES/0007-cache-cow-Use-a-64K-block-size-by-default.patch deleted file mode 100644 index ff5870f..0000000 --- a/SOURCES/0007-cache-cow-Use-a-64K-block-size-by-default.patch +++ /dev/null @@ -1,138 +0,0 @@ -From 83e1167e1a350bd08ac6245f47a5877438408492 Mon Sep 17 00:00:00 2001 -From: "Richard W.M. Jones" -Date: Mon, 26 Jul 2021 17:39:23 +0100 -Subject: [PATCH] cache, cow: Use a 64K block size by default - -Based on the results presented in the previous commit, use a 64K block -size by default in both the cache and cow filters. For the cache -filter you could go back to a 4K block size if you wanted by using the -cache-min-block-size=4K parameter. For cow it is compiled in so -cannot be adjusted. - -(cherry picked from commit c1905b0a28677d961babdb16d6f30ae61042c825) ---- - filters/cache/cache.c | 2 +- - filters/cache/nbdkit-cache-filter.pod | 4 ++-- - filters/cow/blk.h | 2 +- - tests/test-cache-block-size.sh | 2 +- - tests/test-cow-extents1.sh | 33 +++++++++++++++------------ - 5 files changed, 23 insertions(+), 20 deletions(-) - -diff --git a/filters/cache/cache.c b/filters/cache/cache.c -index 109ac89e..c912c5fb 100644 ---- a/filters/cache/cache.c -+++ b/filters/cache/cache.c -@@ -73,7 +73,7 @@ - static pthread_mutex_t lock = PTHREAD_MUTEX_INITIALIZER; - - unsigned blksize; /* actual block size (picked by blk.c) */ --unsigned min_block_size = 4096; -+unsigned min_block_size = 65536; - enum cache_mode cache_mode = CACHE_MODE_WRITEBACK; - int64_t max_size = -1; - unsigned hi_thresh = 95, lo_thresh = 80; -diff --git a/filters/cache/nbdkit-cache-filter.pod b/filters/cache/nbdkit-cache-filter.pod -index 6cbd1c08..df9c1f99 100644 ---- a/filters/cache/nbdkit-cache-filter.pod -+++ b/filters/cache/nbdkit-cache-filter.pod -@@ -65,8 +65,8 @@ or can cheaply reconstruct. - Set the minimum block size used by the cache. This must be a power of - 2 and E 4096. - --The default is 4096, or the block size of the filesystem which --contains the temporary file storing the cache (whichever is larger). -+The default is 64K, or the block size of the filesystem which contains -+the temporary file storing the cache (whichever is larger). - - =item BSIZE - -diff --git a/filters/cow/blk.h b/filters/cow/blk.h -index b066c602..1bc85283 100644 ---- a/filters/cow/blk.h -+++ b/filters/cow/blk.h -@@ -36,7 +36,7 @@ - /* Size of a block in the overlay. A 4K block size means that we need - * 64 MB of memory to store the bitmap for a 1 TB underlying image. - */ --#define BLKSIZE 4096 -+#define BLKSIZE 65536 - - /* Initialize the overlay and bitmap. */ - extern int blk_init (void); -diff --git a/tests/test-cache-block-size.sh b/tests/test-cache-block-size.sh -index a2a27407..d20cc940 100755 ---- a/tests/test-cache-block-size.sh -+++ b/tests/test-cache-block-size.sh -@@ -47,7 +47,7 @@ truncate -s 128K cache-block-size.img - - # Run nbdkit with the caching filter. - start_nbdkit -P cache-block-size.pid -U $sock --filter=cache \ -- file cache-block-size.img cache-min-block-size=64K -+ file cache-block-size.img cache-min-block-size=4K - - nbdsh --connect "nbd+unix://?socket=$sock" \ - -c ' -diff --git a/tests/test-cow-extents1.sh b/tests/test-cow-extents1.sh -index 8e0e0383..ebfd83f6 100755 ---- a/tests/test-cow-extents1.sh -+++ b/tests/test-cow-extents1.sh -@@ -65,7 +65,7 @@ cleanup_fn rm -f $files - - # Create a base file which is half allocated, half sparse. - dd if=/dev/urandom of=$base count=128 bs=1K --truncate -s 256K $base -+truncate -s 4M $base - lastmod="$(stat -c "%y" $base)" - - # Run nbdkit with a COW overlay. -@@ -76,30 +76,33 @@ uri="nbd+unix:///?socket=$sock" - nbdinfo --map "$uri" > $out - cat $out - if [ "$(tr -s ' ' < $out | cut -d' ' -f 1-4)" != " 0 131072 0 -- 131072 131072 3" ]; then -+ 131072 4063232 3" ]; then - echo "$0: unexpected initial file map" - exit 1 - fi - - # Punch some holes. - nbdsh -u "$uri" \ -- -c 'h.trim(4096, 4096)' \ -- -c 'h.trim(4098, 16383)' \ -- -c 'h.pwrite(b"1"*4096, 65536)' \ -- -c 'h.trim(8192, 131072)' \ -- -c 'h.pwrite(b"2"*8192, 196608)' -+ -c 'bs = 65536' \ -+ -c 'h.trim(bs, bs)' \ -+ -c 'h.trim(bs+2, 4*bs-1)' \ -+ -c 'h.pwrite(b"1"*bs, 16*bs)' \ -+ -c 'h.trim(2*bs, 32*bs)' \ -+ -c 'h.pwrite(b"2"*(2*bs), 48*bs)' - - # The extents map should be fully allocated. - nbdinfo --map "$uri" > $out - cat $out --if [ "$(tr -s ' ' < $out | cut -d' ' -f 1-4)" != " 0 4096 0 -- 4096 4096 3 -- 8192 8192 0 -- 16384 4096 3 -- 20480 110592 0 -- 131072 65536 3 -- 196608 8192 0 -- 204800 57344 3" ]; then -+if [ "$(tr -s ' ' < $out | cut -d' ' -f 1-4)" != " 0 65536 0 -+ 65536 131072 3 -+ 196608 65536 0 -+ 262144 65536 3 -+ 327680 65536 0 -+ 393216 655360 3 -+ 1048576 65536 0 -+ 1114112 2031616 3 -+ 3145728 131072 0 -+ 3276800 917504 3" ]; then - echo "$0: unexpected trimmed file map" - exit 1 - fi --- -2.31.1 - diff --git a/SOURCES/0007-vddk-Fix-typo-in-debug-message.patch b/SOURCES/0007-vddk-Fix-typo-in-debug-message.patch new file mode 100644 index 0000000..734ba2e --- /dev/null +++ b/SOURCES/0007-vddk-Fix-typo-in-debug-message.patch @@ -0,0 +1,27 @@ +From a5f73cbcbb6891d2e3c2cb541d47b44a236785ce Mon Sep 17 00:00:00 2001 +From: "Richard W.M. Jones" +Date: Sat, 23 Oct 2021 19:41:07 +0100 +Subject: [PATCH] vddk: Fix typo in debug message + +Fixes: commit 5bb8f0586e1faabcbf4f43d722a3b3cb5b352e33 +(cherry picked from commit 343dadeb7340d7b8c5730e2bbab33c829b569122) +--- + plugins/vddk/vddk.c | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/plugins/vddk/vddk.c b/plugins/vddk/vddk.c +index 50bdde26..65399a91 100644 +--- a/plugins/vddk/vddk.c ++++ b/plugins/vddk/vddk.c +@@ -823,7 +823,7 @@ vddk_get_size (void *handle) + nbdkit_debug ("disk info: uuid: %s", + info->uuid ? : "NULL"); + if (library_version >= 7) { +- nbdkit_debug ("disk info: sectory size: " ++ nbdkit_debug ("disk info: sector size: " + "logical %" PRIu32 " physical %" PRIu32, + info->logicalSectorSize, + info->physicalSectorSize); +-- +2.31.1 + diff --git a/SOURCES/0008-cache-Refactor-printing-state-into-new-function.patch b/SOURCES/0008-cache-Refactor-printing-state-into-new-function.patch deleted file mode 100644 index f42a1bb..0000000 --- a/SOURCES/0008-cache-Refactor-printing-state-into-new-function.patch +++ /dev/null @@ -1,50 +0,0 @@ -From 2592bb42051b3e6d17240badc814b9b16f121c1d Mon Sep 17 00:00:00 2001 -From: "Richard W.M. Jones" -Date: Tue, 27 Jul 2021 21:16:30 +0100 -Subject: [PATCH] cache: Refactor printing state into new function - -This minor refactoring just makes the cache and cow filters' blk.c a -little bit more similar. - -(cherry picked from commit bdb86ea14c00a950f2a2d34071ac1e0799d29132) ---- - filters/cache/blk.c | 16 ++++++++++++---- - 1 file changed, 12 insertions(+), 4 deletions(-) - -diff --git a/filters/cache/blk.c b/filters/cache/blk.c -index 6276985f..e50a7f24 100644 ---- a/filters/cache/blk.c -+++ b/filters/cache/blk.c -@@ -94,6 +94,17 @@ enum bm_entry { - BLOCK_DIRTY = 3, - }; - -+static const char * -+state_to_string (enum bm_entry state) -+{ -+ switch (state) { -+ case BLOCK_NOT_CACHED: return "not cached"; -+ case BLOCK_CLEAN: return "clean"; -+ case BLOCK_DIRTY: return "dirty"; -+ default: abort (); -+ } -+} -+ - /* Extra debugging (-D cache.verbose=1). */ - NBDKIT_DLL_PUBLIC int cache_debug_verbose = 0; - -@@ -312,10 +323,7 @@ blk_cache (nbdkit_next *next, - nbdkit_debug ("cache: blk_cache block %" PRIu64 - " (offset %" PRIu64 ") is %s", - blknum, (uint64_t) offset, -- state == BLOCK_NOT_CACHED ? "not cached" : -- state == BLOCK_CLEAN ? "clean" : -- state == BLOCK_DIRTY ? "dirty" : -- "unknown"); -+ state_to_string (state)); - - if (state == BLOCK_NOT_CACHED) { - /* Read underlying plugin, copy to cache regardless of cache-on-read. */ --- -2.31.1 - diff --git a/SOURCES/0008-vddk-Only-print-vddk_library_version-when-we-managed.patch b/SOURCES/0008-vddk-Only-print-vddk_library_version-when-we-managed.patch new file mode 100644 index 0000000..0e9dd8f --- /dev/null +++ b/SOURCES/0008-vddk-Only-print-vddk_library_version-when-we-managed.patch @@ -0,0 +1,55 @@ +From 1cb810a416e1bdd78a8e5df886a3185d3cfa54d0 Mon Sep 17 00:00:00 2001 +From: "Richard W.M. Jones" +Date: Sat, 23 Oct 2021 19:50:52 +0100 +Subject: [PATCH] vddk: Only print vddk_library_version when we managed to load + the library + +Because --dump-plugin calls load_library (false) it won't fail if we +didn't manage to load the library. This results in library_version +being 0, which we printed incorrectly. + +Resolve this problem by not printing the vddk_library_version entry in +this case. + +Fixes: commit 8700649d147948897f3b97810a1dff37924bdd6e +(cherry picked from commit a3fba12c3e9c2113009f556360ae0bd04c45f6bb) +--- + plugins/vddk/nbdkit-vddk-plugin.pod | 1 + + plugins/vddk/vddk.c | 9 ++++++++- + 2 files changed, 9 insertions(+), 1 deletion(-) + +diff --git a/plugins/vddk/nbdkit-vddk-plugin.pod b/plugins/vddk/nbdkit-vddk-plugin.pod +index bc3c3c94..49e3d75d 100644 +--- a/plugins/vddk/nbdkit-vddk-plugin.pod ++++ b/plugins/vddk/nbdkit-vddk-plugin.pod +@@ -425,6 +425,7 @@ by this build. + =item C + + The VDDK major library version: 5, 6, 7, ... ++If this is omitted it means the library could not be loaded. + + =item C + +diff --git a/plugins/vddk/vddk.c b/plugins/vddk/vddk.c +index 65399a91..39a7d261 100644 +--- a/plugins/vddk/vddk.c ++++ b/plugins/vddk/vddk.c +@@ -591,7 +591,14 @@ vddk_dump_plugin (void) + + printf ("vddk_default_libdir=%s\n", VDDK_LIBDIR); + printf ("vddk_has_nfchostport=1\n"); +- printf ("vddk_library_version=%d\n", library_version); ++ ++ /* Because load_library (false) we might not have loaded VDDK, in ++ * which case we didn't set library_version. Note this cannot ++ * happen in the normal (non-debug-plugin) path because there we use ++ * load_library (true). ++ */ ++ if (library_version > 0) ++ printf ("vddk_library_version=%d\n", library_version); + + #if defined(HAVE_DLADDR) + /* It would be nice to print the version of VDDK from the shared +-- +2.31.1 + diff --git a/SOURCES/0009-tests-cache-Test-cache-on-read-option-really-caches.patch b/SOURCES/0009-tests-cache-Test-cache-on-read-option-really-caches.patch deleted file mode 100644 index f81121e..0000000 --- a/SOURCES/0009-tests-cache-Test-cache-on-read-option-really-caches.patch +++ /dev/null @@ -1,147 +0,0 @@ -From 315948e75e06d038bd8afa319a41e3fde33b4174 Mon Sep 17 00:00:00 2001 -From: "Richard W.M. Jones" -Date: Thu, 29 Jul 2021 20:16:43 +0100 -Subject: [PATCH] tests: cache: Test cache-on-read option really caches - -By making use of the delay filter to add a penalty for hitting the -plugin we can check whether or not the cache-on-read option is -working. - -(cherry picked from commit 3ae7aa533bb9322ab6dc6deecb687ded76634ab4) ---- - tests/Makefile.am | 2 + - tests/test-cache-on-read-caches.sh | 87 ++++++++++++++++++++++++++++++ - tests/test-cache-on-read.sh | 5 -- - 3 files changed, 89 insertions(+), 5 deletions(-) - create mode 100755 tests/test-cache-on-read-caches.sh - -diff --git a/tests/Makefile.am b/tests/Makefile.am -index a038eabc..51ca913a 100644 ---- a/tests/Makefile.am -+++ b/tests/Makefile.am -@@ -1373,6 +1373,7 @@ TESTS += \ - test-cache.sh \ - test-cache-block-size.sh \ - test-cache-on-read.sh \ -+ test-cache-on-read-caches.sh \ - test-cache-max-size.sh \ - test-cache-unaligned.sh \ - $(NULL) -@@ -1380,6 +1381,7 @@ EXTRA_DIST += \ - test-cache.sh \ - test-cache-block-size.sh \ - test-cache-on-read.sh \ -+ test-cache-on-read-caches.sh \ - test-cache-max-size.sh \ - test-cache-unaligned.sh \ - $(NULL) -diff --git a/tests/test-cache-on-read-caches.sh b/tests/test-cache-on-read-caches.sh -new file mode 100755 -index 00000000..80b34159 ---- /dev/null -+++ b/tests/test-cache-on-read-caches.sh -@@ -0,0 +1,87 @@ -+#!/usr/bin/env bash -+# nbdkit -+# Copyright (C) 2018-2021 Red Hat Inc. -+# -+# Redistribution and use in source and binary forms, with or without -+# modification, are permitted provided that the following conditions are -+# met: -+# -+# * Redistributions of source code must retain the above copyright -+# notice, this list of conditions and the following disclaimer. -+# -+# * Redistributions in binary form must reproduce the above copyright -+# notice, this list of conditions and the following disclaimer in the -+# documentation and/or other materials provided with the distribution. -+# -+# * Neither the name of Red Hat nor the names of its contributors may be -+# used to endorse or promote products derived from this software without -+# specific prior written permission. -+# -+# THIS SOFTWARE IS PROVIDED BY RED HAT AND CONTRIBUTORS ''AS IS'' AND -+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, -+# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A -+# PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL RED HAT OR -+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF -+# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND -+# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -+# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT -+# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF -+# SUCH DAMAGE. -+ -+source ./functions.sh -+set -e -+set -x -+ -+requires_filter cache -+requires_filter delay -+requires_nbdsh_uri -+ -+sock=$(mktemp -u /tmp/nbdkit-test-sock.XXXXXX) -+files="$sock cache-on-read-caches.pid" -+rm -f $files -+cleanup_fn rm -f $files -+ -+# Run nbdkit with the cache filter, cache-on-read and a read delay. -+start_nbdkit -P cache-on-read-caches.pid -U $sock \ -+ --filter=cache --filter=delay \ -+ memory 64K cache-on-read=true rdelay=10 -+ -+nbdsh --connect "nbd+unix://?socket=$sock" \ -+ -c ' -+from time import time -+ -+# First read should suffer a penalty. Because we are reading -+# a single 64K block (same size as the cache block), we should -+# only suffer one penalty of approx. 10 seconds. -+st = time() -+zb = h.pread(65536, 0) -+et = time() -+el = et-st -+print("elapsed time: %g" % el) -+assert et-st >= 10 -+assert zb == bytearray(65536) -+ -+# Second read should not suffer a penalty. -+st = time() -+zb = h.pread(65536, 0) -+et = time() -+el = et-st -+print("elapsed time: %g" % el) -+assert el < 10 -+assert zb == bytearray(65536) -+ -+# Write something. -+buf = b"abcd" * 16384 -+h.pwrite(buf, 0) -+ -+# Reading back should be quick since it is stored in the overlay. -+st = time() -+buf2 = h.pread(65536, 0) -+et = time() -+el = et-st -+print("elapsed time: %g" % el) -+assert el < 10 -+assert buf == buf2 -+' -diff --git a/tests/test-cache-on-read.sh b/tests/test-cache-on-read.sh -index f8584dcd..85ca83d4 100755 ---- a/tests/test-cache-on-read.sh -+++ b/tests/test-cache-on-read.sh -@@ -56,9 +56,4 @@ zero = h.pread(32768, 0) - assert zero == bytearray(32768) - buf2 = h.pread(65536, 32768) - assert buf == buf2 -- --# XXX Suggestion to improve this test: Use the delay filter below the --# cache filter, and time reads to prove that the second read is faster --# because it is not going through the delay filter and plugin. --# XXX second h.pread here ... - ' --- -2.31.1 - diff --git a/SOURCES/0009-vddk-Print-one-line-in-dump-plugin-output-for-each-V.patch b/SOURCES/0009-vddk-Print-one-line-in-dump-plugin-output-for-each-V.patch new file mode 100644 index 0000000..786076d --- /dev/null +++ b/SOURCES/0009-vddk-Print-one-line-in-dump-plugin-output-for-each-V.patch @@ -0,0 +1,72 @@ +From 8780009ec092d9cc5a408b7597d88aa54db13639 Mon Sep 17 00:00:00 2001 +From: "Richard W.M. Jones" +Date: Mon, 25 Oct 2021 08:36:53 +0100 +Subject: [PATCH] vddk: Print one line in --dump-plugin output for each VDDK + API + +Helps when detecting if certain optional features are being used, such +as flush and extents. + +(cherry picked from commit 4ee13559e46cf622410d0bdd7db29bb00908b40a) +--- + plugins/vddk/nbdkit-vddk-plugin.pod | 9 +++++++++ + plugins/vddk/vddk.c | 10 ++++++++++ + tests/test-vddk-real-dump-plugin.sh | 1 + + 3 files changed, 20 insertions(+) + +diff --git a/plugins/vddk/nbdkit-vddk-plugin.pod b/plugins/vddk/nbdkit-vddk-plugin.pod +index 49e3d75d..0702aa75 100644 +--- a/plugins/vddk/nbdkit-vddk-plugin.pod ++++ b/plugins/vddk/nbdkit-vddk-plugin.pod +@@ -432,6 +432,15 @@ If this is omitted it means the library could not be loaded. + Prints the full path to the VDDK shared library. Since this requires + a glibc extension it may not be available in all builds of the plugin. + ++=item C ++ ++For each VDDK API that the plugin uses I which is present in the ++VDDK library that was loaded, we print the name of the API ++(eg. C). This lets you see which optional APIs are ++available, such as C and ++C. If the library could not be ++loaded then these lines are not printed. ++ + =back + + =head1 NOTES +diff --git a/plugins/vddk/vddk.c b/plugins/vddk/vddk.c +index 39a7d261..096b04bf 100644 +--- a/plugins/vddk/vddk.c ++++ b/plugins/vddk/vddk.c +@@ -616,6 +616,16 @@ vddk_dump_plugin (void) + printf ("vddk_dll=%s\n", p); + } + #endif ++ ++ /* Note we print all VDDK APIs found here, not just the optional ++ * ones. That is so if we update the baseline VDDK in future and ++ * make optional into required APIs, the output doesn't change. ++ */ ++#define STUB(fn,ret,args) if (fn != NULL) printf ("%s=1\n", #fn); ++#define OPTIONAL_STUB(fn,ret,args) STUB(fn,ret,args) ++#include "vddk-stubs.h" ++#undef STUB ++#undef OPTIONAL_STUB + } + + /* The rules on threads and VDDK are here: +diff --git a/tests/test-vddk-real-dump-plugin.sh b/tests/test-vddk-real-dump-plugin.sh +index 0a079c6c..e37c8b54 100755 +--- a/tests/test-vddk-real-dump-plugin.sh ++++ b/tests/test-vddk-real-dump-plugin.sh +@@ -65,6 +65,7 @@ grep ^vddk_default_libdir= $out + grep ^vddk_has_nfchostport= $out + grep ^vddk_library_version= $out + grep ^vddk_dll= $out ++grep ^VixDiskLib_Open=1 $out + + dll="$(grep ^vddk_dll $out | cut -d= -f2)" + test -f "$dll" +-- +2.31.1 + diff --git a/SOURCES/0010-cow-Implement-cow-on-read.patch b/SOURCES/0010-cow-Implement-cow-on-read.patch deleted file mode 100644 index fd5fcfd..0000000 --- a/SOURCES/0010-cow-Implement-cow-on-read.patch +++ /dev/null @@ -1,457 +0,0 @@ -From 57f9bd29f9d7432ad5a70620c373b28db768a314 Mon Sep 17 00:00:00 2001 -From: "Richard W.M. Jones" -Date: Tue, 27 Jul 2021 23:01:52 +0100 -Subject: [PATCH] cow: Implement cow-on-read - -This is very similar to the nbdkit-cache-filter cache-on-read flag. - -(cherry picked from commit bd93b3f27246f917de48a6cc2525d9c424c07976) ---- - filters/cow/blk.c | 21 ++++++-- - filters/cow/blk.h | 10 ++-- - filters/cow/cow.c | 56 ++++++++++++++++---- - filters/cow/nbdkit-cow-filter.pod | 17 ++++++ - tests/Makefile.am | 4 ++ - tests/test-cow-on-read-caches.sh | 87 +++++++++++++++++++++++++++++++ - tests/test-cow-on-read.sh | 59 +++++++++++++++++++++ - 7 files changed, 236 insertions(+), 18 deletions(-) - create mode 100755 tests/test-cow-on-read-caches.sh - create mode 100755 tests/test-cow-on-read.sh - -diff --git a/filters/cow/blk.c b/filters/cow/blk.c -index 121b0dd4..4f84e092 100644 ---- a/filters/cow/blk.c -+++ b/filters/cow/blk.c -@@ -226,7 +226,7 @@ blk_status (uint64_t blknum, bool *present, bool *trimmed) - int - blk_read_multiple (nbdkit_next *next, - uint64_t blknum, uint64_t nrblocks, -- uint8_t *block, int *err) -+ uint8_t *block, bool cow_on_read, int *err) - { - off_t offset = blknum * BLKSIZE; - enum bm_entry state; -@@ -276,6 +276,19 @@ blk_read_multiple (nbdkit_next *next, - * zeroing the tail. - */ - memset (block + n, 0, tail); -+ -+ /* If cow-on-read is true then copy the blocks to the cache and -+ * set them as allocated. -+ */ -+ if (cow_on_read) { -+ if (full_pwrite (fd, block, BLKSIZE * runblocks, offset) == -1) { -+ *err = errno; -+ nbdkit_error ("pwrite: %m"); -+ return -1; -+ } -+ for (b = 0; b < runblocks; ++b) -+ bitmap_set_blk (&bm, blknum+b, BLOCK_ALLOCATED); -+ } - } - else if (state == BLOCK_ALLOCATED) { /* Read overlay. */ - if (full_pread (fd, block, BLKSIZE * runblocks, offset) == -1) { -@@ -297,14 +310,14 @@ blk_read_multiple (nbdkit_next *next, - blknum + runblocks, - nrblocks - runblocks, - block + BLKSIZE * runblocks, -- err); -+ cow_on_read, err); - } - - int - blk_read (nbdkit_next *next, -- uint64_t blknum, uint8_t *block, int *err) -+ uint64_t blknum, uint8_t *block, bool cow_on_read, int *err) - { -- return blk_read_multiple (next, blknum, 1, block, err); -+ return blk_read_multiple (next, blknum, 1, block, cow_on_read, err); - } - - int -diff --git a/filters/cow/blk.h b/filters/cow/blk.h -index 1bc85283..b7e6f092 100644 ---- a/filters/cow/blk.h -+++ b/filters/cow/blk.h -@@ -52,14 +52,16 @@ extern void blk_status (uint64_t blknum, bool *present, bool *trimmed); - - /* Read a single block from the overlay or plugin. */ - extern int blk_read (nbdkit_next *next, -- uint64_t blknum, uint8_t *block, int *err) -- __attribute__((__nonnull__ (1, 3, 4))); -+ uint64_t blknum, uint8_t *block, -+ bool cow_on_read, int *err) -+ __attribute__((__nonnull__ (1, 3, 5))); - - /* Read multiple blocks from the overlay or plugin. */ - extern int blk_read_multiple (nbdkit_next *next, - uint64_t blknum, uint64_t nrblocks, -- uint8_t *block, int *err) -- __attribute__((__nonnull__ (1, 4, 5))); -+ uint8_t *block, -+ bool cow_on_read, int *err) -+ __attribute__((__nonnull__ (1, 4, 6))); - - /* Cache mode for blocks not already in overlay */ - enum cache_mode { -diff --git a/filters/cow/cow.c b/filters/cow/cow.c -index 78daca22..6efb39f2 100644 ---- a/filters/cow/cow.c -+++ b/filters/cow/cow.c -@@ -38,6 +38,7 @@ - #include - #include - #include -+#include - #include - - #include -@@ -59,6 +60,15 @@ static pthread_mutex_t rmw_lock = PTHREAD_MUTEX_INITIALIZER; - - static bool cow_on_cache; - -+/* Cache on read ("cow-on-read") mode. */ -+extern enum cor_mode { -+ COR_OFF, -+ COR_ON, -+ COR_PATH, -+} cor_mode; -+enum cor_mode cor_mode = COR_OFF; -+const char *cor_path; -+ - static void - cow_load (void) - { -@@ -85,13 +95,39 @@ cow_config (nbdkit_next_config *next, nbdkit_backend *nxdata, - cow_on_cache = r; - return 0; - } -+ else if (strcmp (key, "cow-on-read") == 0) { -+ if (value[0] == '/') { -+ cor_path = value; -+ cor_mode = COR_PATH; -+ } -+ else { -+ int r = nbdkit_parse_bool (value); -+ if (r == -1) -+ return -1; -+ cor_mode = r ? COR_ON : COR_OFF; -+ } -+ return 0; -+ } - else { - return next (nxdata, key, value); - } - } - - #define cow_config_help \ -- "cow-on-cache= Set to true to treat client cache requests as writes.\n" -+ "cow-on-cache= Copy cache (prefetch) requests to the overlay.\n" \ -+ "cow-on-read=|/PATH Copy read requests to the overlay." -+ -+/* Decide if cow-on-read is currently on or off. */ -+bool -+cow_on_read (void) -+{ -+ switch (cor_mode) { -+ case COR_ON: return true; -+ case COR_OFF: return false; -+ case COR_PATH: return access (cor_path, F_OK) == 0; -+ default: abort (); -+ } -+} - - static void * - cow_open (nbdkit_next_open *next, nbdkit_context *nxdata, -@@ -230,7 +266,7 @@ cow_pread (nbdkit_next *next, - uint64_t n = MIN (BLKSIZE - blkoffs, count); - - assert (block); -- r = blk_read (next, blknum, block, err); -+ r = blk_read (next, blknum, block, cow_on_read (), err); - if (r == -1) - return -1; - -@@ -245,7 +281,7 @@ cow_pread (nbdkit_next *next, - /* Aligned body */ - nrblocks = count / BLKSIZE; - if (nrblocks > 0) { -- r = blk_read_multiple (next, blknum, nrblocks, buf, err); -+ r = blk_read_multiple (next, blknum, nrblocks, buf, cow_on_read (), err); - if (r == -1) - return -1; - -@@ -258,7 +294,7 @@ cow_pread (nbdkit_next *next, - /* Unaligned tail */ - if (count) { - assert (block); -- r = blk_read (next, blknum, block, err); -+ r = blk_read (next, blknum, block, cow_on_read (), err); - if (r == -1) - return -1; - -@@ -299,7 +335,7 @@ cow_pwrite (nbdkit_next *next, - */ - assert (block); - ACQUIRE_LOCK_FOR_CURRENT_SCOPE (&rmw_lock); -- r = blk_read (next, blknum, block, err); -+ r = blk_read (next, blknum, block, cow_on_read (), err); - if (r != -1) { - memcpy (&block[blkoffs], buf, n); - r = blk_write (blknum, block, err); -@@ -329,7 +365,7 @@ cow_pwrite (nbdkit_next *next, - if (count) { - assert (block); - ACQUIRE_LOCK_FOR_CURRENT_SCOPE (&rmw_lock); -- r = blk_read (next, blknum, block, err); -+ r = blk_read (next, blknum, block, cow_on_read (), err); - if (r != -1) { - memcpy (block, buf, count); - r = blk_write (blknum, block, err); -@@ -379,7 +415,7 @@ cow_zero (nbdkit_next *next, - * Hold the rmw_lock over the whole operation. - */ - ACQUIRE_LOCK_FOR_CURRENT_SCOPE (&rmw_lock); -- r = blk_read (next, blknum, block, err); -+ r = blk_read (next, blknum, block, cow_on_read (), err); - if (r != -1) { - memset (&block[blkoffs], 0, n); - r = blk_write (blknum, block, err); -@@ -411,7 +447,7 @@ cow_zero (nbdkit_next *next, - /* Unaligned tail */ - if (count) { - ACQUIRE_LOCK_FOR_CURRENT_SCOPE (&rmw_lock); -- r = blk_read (next, blknum, block, err); -+ r = blk_read (next, blknum, block, cow_on_read (), err); - if (r != -1) { - memset (block, 0, count); - r = blk_write (blknum, block, err); -@@ -455,7 +491,7 @@ cow_trim (nbdkit_next *next, - * Hold the lock over the whole operation. - */ - ACQUIRE_LOCK_FOR_CURRENT_SCOPE (&rmw_lock); -- r = blk_read (next, blknum, block, err); -+ r = blk_read (next, blknum, block, cow_on_read (), err); - if (r != -1) { - memset (&block[blkoffs], 0, n); - r = blk_write (blknum, block, err); -@@ -482,7 +518,7 @@ cow_trim (nbdkit_next *next, - /* Unaligned tail */ - if (count) { - ACQUIRE_LOCK_FOR_CURRENT_SCOPE (&rmw_lock); -- r = blk_read (next, blknum, block, err); -+ r = blk_read (next, blknum, block, cow_on_read (), err); - if (r != -1) { - memset (block, 0, count); - r = blk_write (blknum, block, err); -diff --git a/filters/cow/nbdkit-cow-filter.pod b/filters/cow/nbdkit-cow-filter.pod -index 571189e7..01261429 100644 ---- a/filters/cow/nbdkit-cow-filter.pod -+++ b/filters/cow/nbdkit-cow-filter.pod -@@ -62,6 +62,23 @@ the data from the plugin into the overlay. - Do not save data from cache (prefetch) requests in the overlay. This - leaves the overlay as small as possible. This is the default. - -+=item B -+ -+When the client issues a read request, copy the data into the overlay -+so that the same data can be served more quickly later. -+ -+=item B -+ -+Do not save data from read requests in the overlay. This leaves the -+overlay as small as possible. This is the default. -+ -+=item B -+ -+When F (which must be an absolute path) exists, this behaves -+like C, and when it does not exist like -+C. This allows you to control the C -+behaviour while nbdkit is running. -+ - =back - - =head1 EXAMPLES -diff --git a/tests/Makefile.am b/tests/Makefile.am -index 51ca913a..edc8d66d 100644 ---- a/tests/Makefile.am -+++ b/tests/Makefile.am -@@ -1407,6 +1407,8 @@ TESTS += \ - test-cow-extents1.sh \ - test-cow-extents2.sh \ - test-cow-extents-large.sh \ -+ test-cow-on-read.sh \ -+ test-cow-on-read-caches.sh \ - test-cow-unaligned.sh \ - $(NULL) - endif -@@ -1417,6 +1419,8 @@ EXTRA_DIST += \ - test-cow-extents2.sh \ - test-cow-extents-large.sh \ - test-cow-null.sh \ -+ test-cow-on-read.sh \ -+ test-cow-on-read-caches.sh \ - test-cow-unaligned.sh \ - $(NULL) - -diff --git a/tests/test-cow-on-read-caches.sh b/tests/test-cow-on-read-caches.sh -new file mode 100755 -index 00000000..c5b60198 ---- /dev/null -+++ b/tests/test-cow-on-read-caches.sh -@@ -0,0 +1,87 @@ -+#!/usr/bin/env bash -+# nbdkit -+# Copyright (C) 2018-2021 Red Hat Inc. -+# -+# Redistribution and use in source and binary forms, with or without -+# modification, are permitted provided that the following conditions are -+# met: -+# -+# * Redistributions of source code must retain the above copyright -+# notice, this list of conditions and the following disclaimer. -+# -+# * Redistributions in binary form must reproduce the above copyright -+# notice, this list of conditions and the following disclaimer in the -+# documentation and/or other materials provided with the distribution. -+# -+# * Neither the name of Red Hat nor the names of its contributors may be -+# used to endorse or promote products derived from this software without -+# specific prior written permission. -+# -+# THIS SOFTWARE IS PROVIDED BY RED HAT AND CONTRIBUTORS ''AS IS'' AND -+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, -+# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A -+# PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL RED HAT OR -+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF -+# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND -+# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -+# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT -+# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF -+# SUCH DAMAGE. -+ -+source ./functions.sh -+set -e -+set -x -+ -+requires_filter cow -+requires_filter delay -+requires_nbdsh_uri -+ -+sock=$(mktemp -u /tmp/nbdkit-test-sock.XXXXXX) -+files="$sock cow-on-read-caches.pid" -+rm -f $files -+cleanup_fn rm -f $files -+ -+# Run nbdkit with the cow filter, cow-on-read and a read delay. -+start_nbdkit -P cow-on-read-caches.pid -U $sock \ -+ --filter=cow --filter=delay \ -+ memory 64K cow-on-read=true rdelay=10 -+ -+nbdsh --connect "nbd+unix://?socket=$sock" \ -+ -c ' -+from time import time -+ -+# First read should suffer a penalty. Because we are reading -+# a single 64K block (same size as the COW block), we should -+# only suffer one penalty of approx. 10 seconds. -+st = time() -+zb = h.pread(65536, 0) -+et = time() -+el = et-st -+print("elapsed time: %g" % el) -+assert et-st >= 10 -+assert zb == bytearray(65536) -+ -+# Second read should not suffer a penalty. -+st = time() -+zb = h.pread(65536, 0) -+et = time() -+el = et-st -+print("elapsed time: %g" % el) -+assert el < 10 -+assert zb == bytearray(65536) -+ -+# Write something. -+buf = b"abcd" * 16384 -+h.pwrite(buf, 0) -+ -+# Reading back should be quick since it is stored in the overlay. -+st = time() -+buf2 = h.pread(65536, 0) -+et = time() -+el = et-st -+print("elapsed time: %g" % el) -+assert el < 10 -+assert buf == buf2 -+' -diff --git a/tests/test-cow-on-read.sh b/tests/test-cow-on-read.sh -new file mode 100755 -index 00000000..4f58b33b ---- /dev/null -+++ b/tests/test-cow-on-read.sh -@@ -0,0 +1,59 @@ -+#!/usr/bin/env bash -+# nbdkit -+# Copyright (C) 2018-2021 Red Hat Inc. -+# -+# Redistribution and use in source and binary forms, with or without -+# modification, are permitted provided that the following conditions are -+# met: -+# -+# * Redistributions of source code must retain the above copyright -+# notice, this list of conditions and the following disclaimer. -+# -+# * Redistributions in binary form must reproduce the above copyright -+# notice, this list of conditions and the following disclaimer in the -+# documentation and/or other materials provided with the distribution. -+# -+# * Neither the name of Red Hat nor the names of its contributors may be -+# used to endorse or promote products derived from this software without -+# specific prior written permission. -+# -+# THIS SOFTWARE IS PROVIDED BY RED HAT AND CONTRIBUTORS ''AS IS'' AND -+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, -+# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A -+# PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL RED HAT OR -+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF -+# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND -+# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -+# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT -+# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF -+# SUCH DAMAGE. -+ -+source ./functions.sh -+set -e -+set -x -+ -+requires_filter cow -+requires_nbdsh_uri -+ -+sock=$(mktemp -u /tmp/nbdkit-test-sock.XXXXXX) -+files="$sock cow-on-read.pid" -+rm -f $files -+cleanup_fn rm -f $files -+ -+# Run nbdkit with the cow filter and cow-on-read. -+start_nbdkit -P cow-on-read.pid -U $sock \ -+ --filter=cow \ -+ memory 128K cow-on-read=true -+ -+nbdsh --connect "nbd+unix://?socket=$sock" \ -+ -c ' -+# Write some pattern data to the overlay and check it reads back OK. -+buf = b"abcd" * 16384 -+h.pwrite(buf, 32768) -+zero = h.pread(32768, 0) -+assert zero == bytearray(32768) -+buf2 = h.pread(65536, 32768) -+assert buf == buf2 -+' --- -2.31.1 - diff --git a/SOURCES/0010-vddk-Move-minimum-version-to-VDDK-6.5.patch b/SOURCES/0010-vddk-Move-minimum-version-to-VDDK-6.5.patch new file mode 100644 index 0000000..913ddfc --- /dev/null +++ b/SOURCES/0010-vddk-Move-minimum-version-to-VDDK-6.5.patch @@ -0,0 +1,147 @@ +From e34016cbba4340b25f9a52c98db918aa72b38a7c Mon Sep 17 00:00:00 2001 +From: "Richard W.M. Jones" +Date: Tue, 26 Oct 2021 19:46:32 +0100 +Subject: [PATCH] vddk: Move minimum version to VDDK 6.5 + +Drop support for VDDK 5.5.5 (released in 2015) and 6.0 (released the +same year). Move minimum supported version to 6.5 (released Nov +2016). This is so we can use asynchronous operations. + +Acked-by: Laszlo Ersek +(cherry picked from commit 5ed23616762a72e039531a9a7cd81353cd4f436e) +--- + plugins/vddk/nbdkit-vddk-plugin.pod | 10 +++------- + plugins/vddk/vddk-stubs.h | 3 +-- + plugins/vddk/vddk.c | 24 ++++++++++++++++-------- + tests/dummy-vddk.c | 6 ++++++ + 4 files changed, 26 insertions(+), 17 deletions(-) + +diff --git a/plugins/vddk/nbdkit-vddk-plugin.pod b/plugins/vddk/nbdkit-vddk-plugin.pod +index 0702aa75..1c16d096 100644 +--- a/plugins/vddk/nbdkit-vddk-plugin.pod ++++ b/plugins/vddk/nbdkit-vddk-plugin.pod +@@ -424,7 +424,7 @@ by this build. + + =item C + +-The VDDK major library version: 5, 6, 7, ... ++The VDDK major library version: 6, 7, ... + If this is omitted it means the library could not be loaded. + + =item C +@@ -548,16 +548,12 @@ server, which can also be very slow. + + =head1 SUPPORTED VERSIONS OF VDDK + +-This plugin requires VDDK E 5.5.5, which in turn means that it +-is only supported on x64-64 platforms. ++This plugin requires VDDK E 6.5 (released Nov 2016). It is only ++supported on the x64-64 archtecture. + + It has been tested with all versions up to 7.0.3 (but should work with + future versions). + +-VDDK E 6.0 should be used if possible. This is the first version +-which added Flush support which is crucial for data integrity when +-writing. +- + VDDK 6.7 was the first version that supported the + C API, required to provide extent + information over NBD. +diff --git a/plugins/vddk/vddk-stubs.h b/plugins/vddk/vddk-stubs.h +index 5e70238d..a94df9cd 100644 +--- a/plugins/vddk/vddk-stubs.h ++++ b/plugins/vddk/vddk-stubs.h +@@ -40,8 +40,7 @@ + */ + + /* Required stubs, present in all versions of VDDK that we support. I +- * have checked that all these exist in at least VDDK 5.5.5 (2015) +- * which is the earliest version of VDDK that we support. ++ * have checked that all these exist in at least VDDK 5.5.5 (2015). + */ + + STUB (VixDiskLib_InitEx, +diff --git a/plugins/vddk/vddk.c b/plugins/vddk/vddk.c +index 096b04bf..babffc28 100644 +--- a/plugins/vddk/vddk.c ++++ b/plugins/vddk/vddk.c +@@ -75,13 +75,13 @@ NBDKIT_DLL_PUBLIC int vddk_debug_stats; + #undef OPTIONAL_STUB + + /* Parameters passed to InitEx. */ +-#define VDDK_MAJOR 5 ++#define VDDK_MAJOR 6 + #define VDDK_MINOR 5 + + static void *dl; /* dlopen handle */ + static bool init_called; /* was InitEx called */ + static __thread int error_suppression; /* threadlocal error suppression */ +-static int library_version; /* VDDK major: 5, 6, 7, ... */ ++static int library_version; /* VDDK major: 6, 7, ... */ + + static enum { NONE = 0, ZLIB, FASTLZ, SKIPZ } compression; /* compression */ + static char *config; /* config */ +@@ -413,16 +413,14 @@ load_library (bool load_error_is_fatal) + /* Prefer the newest library in case multiple exist. Check two + * possible directories: the usual VDDK installation puts .so + * files in an arch-specific subdirectory of $libdir (our minimum +- * supported version is VDDK 5.5.5, which only supports x64-64); +- * but our testsuite is easier to write if we point libdir +- * directly to a stub .so. ++ * supported version is VDDK 6.5, which only supports x64-64); but ++ * our testsuite is easier to write if we point libdir directly to ++ * a stub .so. + */ + { "lib64/libvixDiskLib.so.7", 7 }, + { "libvixDiskLib.so.7", 7 }, + { "lib64/libvixDiskLib.so.6", 6 }, + { "libvixDiskLib.so.6", 6 }, +- { "lib64/libvixDiskLib.so.5", 5 }, +- { "libvixDiskLib.so.5", 5 }, + { NULL } + }; + size_t i; +@@ -474,7 +472,7 @@ load_library (bool load_error_is_fatal) + exit (EXIT_FAILURE); + } + +- assert (library_version >= 5); ++ assert (library_version >= 6); + + /* Load symbols. */ + #define STUB(fn,ret,args) \ +@@ -490,6 +488,16 @@ load_library (bool load_error_is_fatal) + #include "vddk-stubs.h" + #undef STUB + #undef OPTIONAL_STUB ++ ++ /* Additionally, VDDK version must be >= 6.5. This was the first ++ * version which introduced VixDiskLib_Wait symbol so we can check ++ * for that. ++ */ ++ if (VixDiskLib_Wait == NULL) { ++ nbdkit_error ("VDDK version must be >= 6.5. " ++ "See nbdkit-vddk-plugin(1) man page section \"SUPPORTED VERSIONS OF VDDK\"."); ++ exit (EXIT_FAILURE); ++ } + } + + static int +diff --git a/tests/dummy-vddk.c b/tests/dummy-vddk.c +index 9b5ae0a2..cb88380c 100644 +--- a/tests/dummy-vddk.c ++++ b/tests/dummy-vddk.c +@@ -198,3 +198,9 @@ VixDiskLib_Write (VixDiskLibHandle handle, + memcpy (disk + offset, buf, nr_sectors * VIXDISKLIB_SECTOR_SIZE); + return VIX_OK; + } ++ ++NBDKIT_DLL_PUBLIC VixError ++VixDiskLib_Wait (VixDiskLibHandle handle) ++{ ++ return VIX_OK; ++} +-- +2.31.1 + diff --git a/SOURCES/0011-delay-Add-delay-open-and-delay-close.patch b/SOURCES/0011-delay-Add-delay-open-and-delay-close.patch deleted file mode 100644 index 0faf658..0000000 --- a/SOURCES/0011-delay-Add-delay-open-and-delay-close.patch +++ /dev/null @@ -1,170 +0,0 @@ -From a7e7af18d64164fac42581452f6dc3c07650fcae Mon Sep 17 00:00:00 2001 -From: "Richard W.M. Jones" -Date: Fri, 30 Jul 2021 10:19:57 +0100 -Subject: [PATCH] delay: Add delay-open and delay-close - -Useful for simulating VDDK which has very slow connection. - -(cherry picked from commit de8dcd3a34a38b088a0f9a6f8ca754702ad1f598) ---- - filters/delay/delay.c | 58 ++++++++++++++++++++++++++- - filters/delay/nbdkit-delay-filter.pod | 27 +++++++++++-- - 2 files changed, 80 insertions(+), 5 deletions(-) - -diff --git a/filters/delay/delay.c b/filters/delay/delay.c -index 5a925aa4..df3729a7 100644 ---- a/filters/delay/delay.c -+++ b/filters/delay/delay.c -@@ -48,6 +48,8 @@ static unsigned delay_zero_ms = 0; /* zero delay (milliseconds) */ - static unsigned delay_trim_ms = 0; /* trim delay (milliseconds) */ - static unsigned delay_extents_ms = 0;/* extents delay (milliseconds) */ - static unsigned delay_cache_ms = 0; /* cache delay (milliseconds) */ -+static unsigned delay_open_ms = 0; /* open delay (milliseconds) */ -+static unsigned delay_close_ms = 0; /* close delay (milliseconds) */ - - static int delay_fast_zero = 1; /* whether delaying zero includes fast zero */ - -@@ -126,6 +128,18 @@ cache_delay (int *err) - return delay (delay_cache_ms, err); - } - -+static int -+open_delay (int *err) -+{ -+ return delay (delay_open_ms, err); -+} -+ -+static int -+close_delay (int *err) -+{ -+ return delay (delay_close_ms, err); -+} -+ - /* Called for each key=value passed on the command line. */ - static int - delay_config (nbdkit_next_config *next, nbdkit_backend *nxdata, -@@ -182,6 +196,16 @@ delay_config (nbdkit_next_config *next, nbdkit_backend *nxdata, - return -1; - return 0; - } -+ else if (strcmp (key, "delay-open") == 0) { -+ if (parse_delay (key, value, &delay_open_ms) == -1) -+ return -1; -+ return 0; -+ } -+ else if (strcmp (key, "delay-close") == 0) { -+ if (parse_delay (key, value, &delay_close_ms) == -1) -+ return -1; -+ return 0; -+ } - else - return next (nxdata, key, value); - } -@@ -195,7 +219,9 @@ delay_config (nbdkit_next_config *next, nbdkit_backend *nxdata, - "delay-extents=[ms] Extents delay in seconds/milliseconds.\n" \ - "delay-cache=[ms] Cache delay in seconds/milliseconds.\n" \ - "wdelay=[ms] Write, zero and trim delay in secs/msecs.\n" \ -- "delay-fast-zero= Delay fast zero requests (default true).\n" -+ "delay-fast-zero= Delay fast zero requests (default true).\n" \ -+ "delay-open=[ms] Open delay in seconds/milliseconds.\n" \ -+ "delay-close=[ms] Close delay in seconds/milliseconds." - - /* Override the plugin's .can_fast_zero if needed */ - static int -@@ -208,6 +234,34 @@ delay_can_fast_zero (nbdkit_next *next, - return next->can_fast_zero (next); - } - -+/* Open connection. */ -+static void * -+delay_open (nbdkit_next_open *next, nbdkit_context *nxdata, -+ int readonly, const char *exportname, int is_tls) -+{ -+ int err; -+ -+ if (open_delay (&err) == -1) { -+ errno = err; -+ nbdkit_error ("delay: %m"); -+ return NULL; -+ } -+ -+ if (next (nxdata, readonly, exportname) == -1) -+ return NULL; -+ -+ return NBDKIT_HANDLE_NOT_NEEDED; -+} -+ -+/* Close connection. */ -+static void -+delay_close (void *handle) -+{ -+ int err; -+ -+ close_delay (&err); -+} -+ - /* Read data. */ - static int - delay_pread (nbdkit_next *next, -@@ -285,6 +339,8 @@ static struct nbdkit_filter filter = { - .config = delay_config, - .config_help = delay_config_help, - .can_fast_zero = delay_can_fast_zero, -+ .open = delay_open, -+ .close = delay_close, - .pread = delay_pread, - .pwrite = delay_pwrite, - .zero = delay_zero, -diff --git a/filters/delay/nbdkit-delay-filter.pod b/filters/delay/nbdkit-delay-filter.pod -index d6961a9e..11ae544b 100644 ---- a/filters/delay/nbdkit-delay-filter.pod -+++ b/filters/delay/nbdkit-delay-filter.pod -@@ -9,10 +9,15 @@ nbdkit-delay-filter - nbdkit delay filter - nbdkit --filter=delay plugin rdelay=NNms wdelay=NNms [plugin-args...] - - nbdkit --filter=delay plugin [plugin-args ...] -- delay-read=(SECS|NNms) delay-write=(SECS|NNms) -- delay-zero=(SECS|NNms) delay-trim=(SECS|NNms) -- delay-extents=(SECS|NNms) delay-cache=(SECS|NNms) -+ delay-read=(SECS|NNms) -+ delay-write=(SECS|NNms) -+ delay-zero=(SECS|NNms) -+ delay-trim=(SECS|NNms) -+ delay-extents=(SECS|NNms) -+ delay-cache=(SECS|NNms) - delay-fast-zero=BOOL -+ delay-open=(SECS|NNms) -+ delay-close=(SECS|NNms) - - =head1 DESCRIPTION - -@@ -108,6 +113,20 @@ delay as any other zero request; but setting this parameter to false - instantly fails a fast zero response without waiting for or consulting - the plugin. - -+=item BSECS -+ -+=item BNNB -+ -+=item BSECS -+ -+=item BNNB -+ -+(nbdkit E 1.28) -+ -+Delay open and close operations by C seconds or C -+milliseconds. Open corresponds to client connection. Close may not -+be visible to clients if they abruptly disconnect. -+ - =back - - =head1 FILES -@@ -140,4 +159,4 @@ Richard W.M. Jones - - =head1 COPYRIGHT - --Copyright (C) 2018 Red Hat Inc. -+Copyright (C) 2018-2021 Red Hat Inc. --- -2.31.1 - diff --git a/SOURCES/0011-vddk-Add-read-write-and-wait-asynchronous-functions.patch b/SOURCES/0011-vddk-Add-read-write-and-wait-asynchronous-functions.patch new file mode 100644 index 0000000..c836f82 --- /dev/null +++ b/SOURCES/0011-vddk-Add-read-write-and-wait-asynchronous-functions.patch @@ -0,0 +1,72 @@ +From 69b989b37c8e33f52d928c7202146e9e11a2a93c Mon Sep 17 00:00:00 2001 +From: "Richard W.M. Jones" +Date: Wed, 27 Oct 2021 11:57:35 +0100 +Subject: [PATCH] vddk: Add read, write and wait asynchronous functions + +These functions added in VDDK 6.0 - 6.5 implement asynchronous read +and write. + +Acked-by: Laszlo Ersek +(cherry picked from commit ad53e7becafed6ca3573795a79c534281fe9c274) +--- + plugins/vddk/vddk-structs.h | 3 +++ + plugins/vddk/vddk-stubs.h | 19 ++++++++++++++++++- + 2 files changed, 21 insertions(+), 1 deletion(-) + +diff --git a/plugins/vddk/vddk-structs.h b/plugins/vddk/vddk-structs.h +index aeb5bfd0..e97f017c 100644 +--- a/plugins/vddk/vddk-structs.h ++++ b/plugins/vddk/vddk-structs.h +@@ -43,6 +43,7 @@ + + typedef uint64_t VixError; + #define VIX_OK 0 ++#define VIX_ASYNC 25000 + + #define VIXDISKLIB_FLAG_OPEN_UNBUFFERED 1 + #define VIXDISKLIB_FLAG_OPEN_SINGLE_LINK 2 +@@ -61,6 +62,8 @@ typedef void *VixDiskLibHandle; + + typedef void VixDiskLibGenericLogFunc (const char *fmt, va_list args); + ++typedef void (*VixDiskLibCompletionCB) (void *data, VixError result); ++ + enum VixDiskLibCredType { + VIXDISKLIB_CRED_UID = 1, + VIXDISKLIB_CRED_SESSIONID = 2, +diff --git a/plugins/vddk/vddk-stubs.h b/plugins/vddk/vddk-stubs.h +index a94df9cd..66353691 100644 +--- a/plugins/vddk/vddk-stubs.h ++++ b/plugins/vddk/vddk-stubs.h +@@ -103,10 +103,27 @@ STUB (VixDiskLib_Write, + uint64_t start_sector, uint64_t nr_sectors, + const unsigned char *buf)); + +-/* Added in VDDK 6.0, this will be NULL in earlier versions. */ ++/* Added in VDDK 6.0, these will be NULL in earlier versions. */ + OPTIONAL_STUB (VixDiskLib_Flush, + VixError, + (VixDiskLibHandle handle)); ++OPTIONAL_STUB (VixDiskLib_ReadAsync, ++ VixError, ++ (VixDiskLibHandle handle, ++ uint64_t start_sector, uint64_t nr_sectors, ++ unsigned char *buf, ++ VixDiskLibCompletionCB callback, void *data)); ++OPTIONAL_STUB (VixDiskLib_WriteAsync, ++ VixError, ++ (VixDiskLibHandle handle, ++ uint64_t start_sector, uint64_t nr_sectors, ++ const unsigned char *buf, ++ VixDiskLibCompletionCB callback, void *data)); ++ ++/* Added in VDDK 6.5, this will be NULL in earlier versions. */ ++OPTIONAL_STUB (VixDiskLib_Wait, ++ VixError, ++ (VixDiskLibHandle handle)); + + /* Added in VDDK 6.7, these will be NULL for earlier versions: */ + OPTIONAL_STUB (VixDiskLib_QueryAllocatedBlocks, +-- +2.31.1 + diff --git a/SOURCES/0012-python-Implement-.cleanup-method.patch b/SOURCES/0012-python-Implement-.cleanup-method.patch deleted file mode 100644 index 321c669..0000000 --- a/SOURCES/0012-python-Implement-.cleanup-method.patch +++ /dev/null @@ -1,90 +0,0 @@ -From 17a912a449fa75b5c12ac3acab596b476699c671 Mon Sep 17 00:00:00 2001 -From: "Richard W.M. Jones" -Date: Tue, 3 Aug 2021 14:19:38 +0100 -Subject: [PATCH] python: Implement .cleanup() method - -(cherry picked from commit f2fe99e4b0f54467ab8028eaf2d039cf918b2961) ---- - plugins/python/nbdkit-python-plugin.pod | 20 +++++++++++++++++--- - plugins/python/plugin.c | 19 +++++++++++++++++++ - 2 files changed, 36 insertions(+), 3 deletions(-) - -diff --git a/plugins/python/nbdkit-python-plugin.pod b/plugins/python/nbdkit-python-plugin.pod -index 6f5f2c00..a92a557f 100644 ---- a/plugins/python/nbdkit-python-plugin.pod -+++ b/plugins/python/nbdkit-python-plugin.pod -@@ -257,6 +257,12 @@ There are no arguments or return value. - - There are no arguments or return value. - -+=item C -+ -+(Optional, nbdkit E 1.28) -+ -+There are no arguments or return value. -+ - =item C - - (Optional) -@@ -498,10 +504,18 @@ optionally using C first. - - =over 4 - --=item Missing: C and C -+=item Missing: C - --These are not needed because you can just use ordinary Python --constructs. -+This is not needed since you can use regular Python mechanisms like -+top level statements to run code when the module is loaded. -+ -+=item Missing: C -+ -+This is missing, but in nbdkit E 1.28 you can put code in the -+C function to have it run when nbdkit exits. In earlier -+versions of nbdkit, using a Python -+L handler is -+recommended. - - =item Missing: - C, -diff --git a/plugins/python/plugin.c b/plugins/python/plugin.c -index 64430a1a..f85512b4 100644 ---- a/plugins/python/plugin.c -+++ b/plugins/python/plugin.c -@@ -298,6 +298,24 @@ py_after_fork (void) - return 0; - } - -+static void -+py_cleanup (void) -+{ -+ ACQUIRE_PYTHON_GIL_FOR_CURRENT_SCOPE; -+ PyObject *fn; -+ PyObject *r; -+ -+ if (callback_defined ("cleanup", &fn)) { -+ PyErr_Clear (); -+ -+ r = PyObject_CallObject (fn, NULL); -+ Py_DECREF (fn); -+ if (check_python_failure ("cleanup") == -1) -+ return; -+ Py_DECREF (r); -+ } -+} -+ - static int - py_list_exports (int readonly, int is_tls, struct nbdkit_exports *exports) - { -@@ -1039,6 +1057,7 @@ static struct nbdkit_plugin plugin = { - .thread_model = py_thread_model, - .get_ready = py_get_ready, - .after_fork = py_after_fork, -+ .cleanup = py_cleanup, - .list_exports = py_list_exports, - .default_export = py_default_export, - --- -2.31.1 - diff --git a/SOURCES/0012-vddk-Start-to-split-VDDK-over-several-files.patch b/SOURCES/0012-vddk-Start-to-split-VDDK-over-several-files.patch new file mode 100644 index 0000000..0e49adf --- /dev/null +++ b/SOURCES/0012-vddk-Start-to-split-VDDK-over-several-files.patch @@ -0,0 +1,259 @@ +From 98a499c0e9d08f208474759012ec3ed823ce2335 Mon Sep 17 00:00:00 2001 +From: "Richard W.M. Jones" +Date: Wed, 27 Oct 2021 12:20:31 +0100 +Subject: [PATCH] vddk: Start to split VDDK over several files + +This change doesn't do anything except move some definitions into the +header file vddk.h, but it allows future commits to split up the very +large vddk.c file. + +Acked-by: Laszlo Ersek +(cherry picked from commit 117634dccf4e29394e8718a8d62e93a9edf0a39c) +--- + plugins/vddk/vddk.c | 91 +++++++++++++-------------------------------- + plugins/vddk/vddk.h | 89 +++++++++++++++++++++++++++++++++++++++++++- + 2 files changed, 112 insertions(+), 68 deletions(-) + +diff --git a/plugins/vddk/vddk.c b/plugins/vddk/vddk.c +index babffc28..041bff1a 100644 +--- a/plugins/vddk/vddk.c ++++ b/plugins/vddk/vddk.c +@@ -50,14 +50,12 @@ + #include + + #include "cleanup.h" +-#include "isaligned.h" + #include "minmax.h" + #include "rounding.h" + #include "tvdiff.h" + #include "vector.h" + + #include "vddk.h" +-#include "vddk-structs.h" + + /* Debug flags. */ + NBDKIT_DLL_PUBLIC int vddk_debug_diskinfo; +@@ -65,11 +63,11 @@ NBDKIT_DLL_PUBLIC int vddk_debug_extents; + NBDKIT_DLL_PUBLIC int vddk_debug_datapath = 1; + NBDKIT_DLL_PUBLIC int vddk_debug_stats; + +-/* For each VDDK API define a static global variable. These globals +- * are initialized when the plugin is loaded (by vddk_get_ready). ++/* For each VDDK API define a global variable. These globals are ++ * initialized when the plugin is loaded (by vddk_get_ready). + */ +-#define STUB(fn,ret,args) static ret (*fn) args +-#define OPTIONAL_STUB(fn,ret,args) static ret (*fn) args ++#define STUB(fn,ret,args) ret (*fn) args ++#define OPTIONAL_STUB(fn,ret,args) ret (*fn) args + #include "vddk-stubs.h" + #undef STUB + #undef OPTIONAL_STUB +@@ -78,28 +76,28 @@ NBDKIT_DLL_PUBLIC int vddk_debug_stats; + #define VDDK_MAJOR 6 + #define VDDK_MINOR 5 + +-static void *dl; /* dlopen handle */ +-static bool init_called; /* was InitEx called */ +-static __thread int error_suppression; /* threadlocal error suppression */ +-static int library_version; /* VDDK major: 6, 7, ... */ ++void *dl; /* dlopen handle */ ++bool init_called; /* was InitEx called */ ++__thread int error_suppression; /* threadlocal error suppression */ ++int library_version; /* VDDK major: 6, 7, ... */ ++bool is_remote; /* true if remote connection */ + +-static enum { NONE = 0, ZLIB, FASTLZ, SKIPZ } compression; /* compression */ +-static char *config; /* config */ +-static const char *cookie; /* cookie */ +-static const char *filename; /* file */ +-char *libdir; /* libdir */ +-static uint16_t nfc_host_port; /* nfchostport */ +-char *password; /* password */ +-static uint16_t port; /* port */ +-static const char *server_name; /* server */ +-static bool single_link; /* single-link */ +-static const char *snapshot_moref; /* snapshot */ +-static const char *thumb_print; /* thumbprint */ +-static const char *transport_modes; /* transports */ +-static bool unbuffered; /* unbuffered */ +-static const char *username; /* user */ +-static const char *vmx_spec; /* vm */ +-static bool is_remote; ++enum compression_type compression; /* compression */ ++char *config; /* config */ ++const char *cookie; /* cookie */ ++const char *filename; /* file */ ++char *libdir; /* libdir */ ++uint16_t nfc_host_port; /* nfchostport */ ++char *password; /* password */ ++uint16_t port; /* port */ ++const char *server_name; /* server */ ++bool single_link; /* single-link */ ++const char *snapshot_moref; /* snapshot */ ++const char *thumb_print; /* thumbprint */ ++const char *transport_modes; /* transports */ ++bool unbuffered; /* unbuffered */ ++const char *username; /* user */ ++const char *vmx_spec; /* vm */ + + /* For each VDDK API define a variable to store the time taken (used + * to implement -D vddk.stats=1). +@@ -120,45 +118,6 @@ static void display_stats (void); + #undef STUB + #undef OPTIONAL_STUB + +-/* Macros to bracket each VDDK API call, for printing debugging +- * information and collecting statistics. +- */ +-#define VDDK_CALL_START(fn, fs, ...) \ +- do { \ +- struct timeval start_t, end_t; \ +- /* GCC can optimize this away at compile time: */ \ +- const bool datapath = \ +- strcmp (#fn, "VixDiskLib_Read") == 0 || \ +- strcmp (#fn, "VixDiskLib_Write") == 0; \ +- if (vddk_debug_stats) \ +- gettimeofday (&start_t, NULL); \ +- if (!datapath || vddk_debug_datapath) \ +- nbdkit_debug ("VDDK call: %s (" fs ")", #fn, ##__VA_ARGS__); \ +- do +-#define VDDK_CALL_END(fn, bytes_) \ +- while (0); \ +- if (vddk_debug_stats) { \ +- gettimeofday (&end_t, NULL); \ +- ACQUIRE_LOCK_FOR_CURRENT_SCOPE (&stats_lock); \ +- stats_##fn.usecs += tvdiff_usec (&start_t, &end_t); \ +- stats_##fn.calls++; \ +- stats_##fn.bytes += bytes_; \ +- } \ +- } while (0) +- +-/* Print VDDK errors. */ +-#define VDDK_ERROR(err, fs, ...) \ +- do { \ +- char *vddk_err_msg; \ +- VDDK_CALL_START (VixDiskLib_GetErrorText, "%lu", err) \ +- vddk_err_msg = VixDiskLib_GetErrorText ((err), NULL); \ +- VDDK_CALL_END (VixDiskLib_GetErrorText, 0); \ +- nbdkit_error (fs ": %s", ##__VA_ARGS__, vddk_err_msg); \ +- VDDK_CALL_START (VixDiskLib_FreeErrorText, "") \ +- VixDiskLib_FreeErrorText (vddk_err_msg); \ +- VDDK_CALL_END (VixDiskLib_FreeErrorText, 0); \ +- } while (0) +- + /* Unload the plugin. */ + static void + vddk_unload (void) +diff --git a/plugins/vddk/vddk.h b/plugins/vddk/vddk.h +index 8c63b4ee..29775eb4 100644 +--- a/plugins/vddk/vddk.h ++++ b/plugins/vddk/vddk.h +@@ -1,5 +1,5 @@ + /* nbdkit +- * Copyright (C) 2013-2020 Red Hat Inc. ++ * Copyright (C) 2013-2021 Red Hat Inc. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are +@@ -33,11 +33,96 @@ + #ifndef NBDKIT_VDDK_H + #define NBDKIT_VDDK_H + ++#include ++#include ++#include ++ ++#include ++ ++#include "isaligned.h" ++#include "tvdiff.h" ++#include "vector.h" ++ ++#include "vddk-structs.h" ++ ++enum compression_type { NONE = 0, ZLIB, FASTLZ, SKIPZ }; ++ ++extern void *dl; ++extern bool init_called; ++extern __thread int error_suppression; ++extern int library_version; ++extern bool is_remote; ++ ++extern enum compression_type compression; ++extern char *config; ++extern const char *cookie; ++extern const char *filename; + extern char *libdir; ++extern uint16_t nfc_host_port; + extern char *password; ++extern uint16_t port; ++extern const char *server_name; ++extern bool single_link; ++extern const char *snapshot_moref; ++extern const char *thumb_print; ++extern const char *transport_modes; ++extern bool unbuffered; ++extern const char *username; ++extern const char *vmx_spec; ++ ++extern int vddk_debug_diskinfo; ++extern int vddk_debug_extents; ++extern int vddk_debug_datapath; ++extern int vddk_debug_stats; ++ ++#define STUB(fn,ret,args) extern ret (*fn) args ++#define OPTIONAL_STUB(fn,ret,args) extern ret (*fn) args ++#include "vddk-stubs.h" ++#undef STUB ++#undef OPTIONAL_STUB ++ ++/* Macros to bracket each VDDK API call, for printing debugging ++ * information and collecting statistics. ++ */ ++#define VDDK_CALL_START(fn, fs, ...) \ ++ do { \ ++ struct timeval start_t, end_t; \ ++ /* GCC can optimize this away at compile time: */ \ ++ const bool datapath = \ ++ strcmp (#fn, "VixDiskLib_Read") == 0 || \ ++ strcmp (#fn, "VixDiskLib_Write") == 0; \ ++ if (vddk_debug_stats) \ ++ gettimeofday (&start_t, NULL); \ ++ if (!datapath || vddk_debug_datapath) \ ++ nbdkit_debug ("VDDK call: %s (" fs ")", #fn, ##__VA_ARGS__); \ ++ do ++#define VDDK_CALL_END(fn, bytes_) \ ++ while (0); \ ++ if (vddk_debug_stats) { \ ++ gettimeofday (&end_t, NULL); \ ++ ACQUIRE_LOCK_FOR_CURRENT_SCOPE (&stats_lock); \ ++ stats_##fn.usecs += tvdiff_usec (&start_t, &end_t); \ ++ stats_##fn.calls++; \ ++ stats_##fn.bytes += bytes_; \ ++ } \ ++ } while (0) ++ ++/* Print VDDK errors. */ ++#define VDDK_ERROR(err, fs, ...) \ ++ do { \ ++ char *vddk_err_msg; \ ++ VDDK_CALL_START (VixDiskLib_GetErrorText, "%lu", err) \ ++ vddk_err_msg = VixDiskLib_GetErrorText ((err), NULL); \ ++ VDDK_CALL_END (VixDiskLib_GetErrorText, 0); \ ++ nbdkit_error (fs ": %s", ##__VA_ARGS__, vddk_err_msg); \ ++ VDDK_CALL_START (VixDiskLib_FreeErrorText, "") \ ++ VixDiskLib_FreeErrorText (vddk_err_msg); \ ++ VDDK_CALL_END (VixDiskLib_FreeErrorText, 0); \ ++ } while (0) ++ ++/* reexec.c */ + extern bool noreexec; + extern char *reexeced; +- + extern void reexec_if_needed (const char *prepend); + extern int restore_ld_library_path (void); + +-- +2.31.1 + diff --git a/SOURCES/0013-cow-General-revision-and-updates-to-the-manual.patch b/SOURCES/0013-cow-General-revision-and-updates-to-the-manual.patch deleted file mode 100644 index b3d8381..0000000 --- a/SOURCES/0013-cow-General-revision-and-updates-to-the-manual.patch +++ /dev/null @@ -1,150 +0,0 @@ -From e9abe97b40fef6f9bd9028a2520f45203bba0749 Mon Sep 17 00:00:00 2001 -From: "Richard W.M. Jones" -Date: Thu, 5 Aug 2021 18:18:34 +0100 -Subject: [PATCH] cow: General revision and updates to the manual - -(cherry picked from commit ba5517b81307c228577cf3c54a651d044ac91a25) ---- - filters/cow/nbdkit-cow-filter.pod | 74 ++++++++++++++++--------------- - 1 file changed, 39 insertions(+), 35 deletions(-) - -diff --git a/filters/cow/nbdkit-cow-filter.pod b/filters/cow/nbdkit-cow-filter.pod -index 01261429..7f861140 100644 ---- a/filters/cow/nbdkit-cow-filter.pod -+++ b/filters/cow/nbdkit-cow-filter.pod -@@ -5,33 +5,23 @@ nbdkit-cow-filter - nbdkit copy-on-write (COW) filter - =head1 SYNOPSIS - - nbdkit --filter=cow plugin [plugin-args...] -+ [cow-on-cache=false|true] -+ [cow-on-read=false|true|/PATH] - - =head1 DESCRIPTION - - C is a filter that makes a temporary writable copy --on top of a read-only plugin. It can be used to enable writes for --plugins which only implement read-only access. Note that: -+on top of a plugin. It can also be used to enable writes for plugins -+which are read-only. - --=over 4 -+The underlying plugin is opened read-only. This filter does not pass -+any writes or write-like operations (like trim and zero) through to -+the underlying plugin. - --=item * -- --B -- --=item * -- --All connections to the nbdkit instance see the same view of the disk. -- --This is different from L where each connection sees its --own copy-on-write overlay and simply disconnecting the client throws --that away. It also allows us to create diffs, see below. -- --=item * -- --The plugin is opened read-only (as if the I<-r> flag was passed), but --you should B pass the I<-r> flag to nbdkit. -- --=back -+B -+If you want to save changes, either copy out the whole disk using a -+tool like L, or use the method described in L -+below to create a diff. - - Limitations of the filter include: - -@@ -52,26 +42,26 @@ serve the same data to each client. - - =over 4 - --=item B -- --When the client issues a cache (prefetch) request, preemptively save --the data from the plugin into the overlay. -- - =item B - - Do not save data from cache (prefetch) requests in the overlay. This - leaves the overlay as small as possible. This is the default. - --=item B -+=item B - --When the client issues a read request, copy the data into the overlay --so that the same data can be served more quickly later. -+When the client issues a cache (prefetch) request, preemptively save -+the data from the plugin into the overlay. - - =item B - - Do not save data from read requests in the overlay. This leaves the - overlay as small as possible. This is the default. - -+=item B -+ -+When the client issues a read request, copy the data into the overlay -+so that the same data can be served more quickly later. -+ - =item B - - When F (which must be an absolute path) exists, this behaves -@@ -83,18 +73,23 @@ behaviour while nbdkit is running. - - =head1 EXAMPLES - -+=head2 nbdkit --filter=cow file disk.img -+ - Serve the file F, allowing writes, but do not save any --changes into the file: -+changes into the file. - -- nbdkit --filter=cow file disk.img -+=head2 nbdkit --filter=cow --filter=xz file disk.xz cow-on-read=true - - L only supports read access, but you can provide --temporary write access by doing (although this does B save --changes to the file): -+temporary write access by using the command above. Because xz -+decompression is slow, using C causes reads to be -+cached as well as writes, improving performance at the expense of -+using more temporary space. Note that writes are thrown away when -+nbdkit exits and do not get saved into the file. - -- nbdkit --filter=cow --filter=xz file disk.xz -+=head1 NOTES - --=head1 CREATING A DIFF WITH QEMU-IMG -+=head2 Creating a diff with qemu-img - - Although nbdkit-cow-filter itself cannot save the differences, it is - possible to do this using an obscure feature of L. -@@ -118,6 +113,14 @@ F now contains the differences between the base - (F) and the changes stored in nbdkit-cow-filter. C - can now be killed. - -+=head2 Compared to nbd-server -c option -+ -+All connections to the nbdkit instance see the same view of the disk. -+This is different from L I<-c> option where each -+connection sees its own copy-on-write overlay and simply disconnecting -+the client throws that away. It also allows us to create diffs as -+above. -+ - =head1 ENVIRONMENT VARIABLES - - =over 4 -@@ -154,6 +157,7 @@ L, - L, - L, - L, -+L, - L. - - =head1 AUTHORS --- -2.31.1 - diff --git a/SOURCES/0013-vddk-Refactor-D-vddk.stats-1-into-a-new-file.patch b/SOURCES/0013-vddk-Refactor-D-vddk.stats-1-into-a-new-file.patch new file mode 100644 index 0000000..5dc720a --- /dev/null +++ b/SOURCES/0013-vddk-Refactor-D-vddk.stats-1-into-a-new-file.patch @@ -0,0 +1,287 @@ +From d602150dbb5ebacea42c25a0f6c8c26c45766a49 Mon Sep 17 00:00:00 2001 +From: "Richard W.M. Jones" +Date: Wed, 27 Oct 2021 12:30:41 +0100 +Subject: [PATCH] vddk: Refactor -D vddk.stats=1 into a new file + +Acked-by: Laszlo Ersek +(cherry picked from commit dcd5bc51ed7710c32d956345ea8da14ba15ef8f5) +--- + plugins/vddk/Makefile.am | 1 + + plugins/vddk/stats.c | 118 +++++++++++++++++++++++++++++++++++++++ + plugins/vddk/vddk.c | 78 +------------------------- + plugins/vddk/vddk.h | 15 +++++ + 4 files changed, 135 insertions(+), 77 deletions(-) + create mode 100644 plugins/vddk/stats.c + +diff --git a/plugins/vddk/Makefile.am b/plugins/vddk/Makefile.am +index 232aaedd..4f470ff9 100644 +--- a/plugins/vddk/Makefile.am ++++ b/plugins/vddk/Makefile.am +@@ -46,6 +46,7 @@ nbdkit_vddk_plugin_la_SOURCES = \ + vddk.c \ + vddk.h \ + reexec.c \ ++ stats.c \ + vddk-structs.h \ + vddk-stubs.h \ + $(top_srcdir)/include/nbdkit-plugin.h \ +diff --git a/plugins/vddk/stats.c b/plugins/vddk/stats.c +new file mode 100644 +index 00000000..18a42714 +--- /dev/null ++++ b/plugins/vddk/stats.c +@@ -0,0 +1,118 @@ ++/* nbdkit ++ * Copyright (C) 2013-2021 Red Hat Inc. ++ * ++ * Redistribution and use in source and binary forms, with or without ++ * modification, are permitted provided that the following conditions are ++ * met: ++ * ++ * * Redistributions of source code must retain the above copyright ++ * notice, this list of conditions and the following disclaimer. ++ * ++ * * Redistributions in binary form must reproduce the above copyright ++ * notice, this list of conditions and the following disclaimer in the ++ * documentation and/or other materials provided with the distribution. ++ * ++ * * Neither the name of Red Hat nor the names of its contributors may be ++ * used to endorse or promote products derived from this software without ++ * specific prior written permission. ++ * ++ * THIS SOFTWARE IS PROVIDED BY RED HAT AND CONTRIBUTORS ''AS IS'' AND ++ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, ++ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A ++ * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL RED HAT OR ++ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, ++ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT ++ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF ++ * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ++ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, ++ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT ++ * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF ++ * SUCH DAMAGE. ++ */ ++ ++#include ++ ++#include ++#include ++#include ++#include ++ ++#include ++ ++#define NBDKIT_API_VERSION 2 ++#include ++ ++#include "vector.h" ++ ++#include "vddk.h" ++ ++/* Debug flags. */ ++NBDKIT_DLL_PUBLIC int vddk_debug_stats; ++ ++pthread_mutex_t stats_lock = PTHREAD_MUTEX_INITIALIZER; ++ ++/* For each VDDK API define a variable to store the time taken (used ++ * to implement -D vddk.stats=1). ++ */ ++#define STUB(fn,ret,args) struct vddk_stat stats_##fn = { .name = #fn } ++#define OPTIONAL_STUB(fn,ret,args) STUB(fn,ret,args) ++#include "vddk-stubs.h" ++#undef STUB ++#undef OPTIONAL_STUB ++ ++DEFINE_VECTOR_TYPE(statlist, struct vddk_stat) ++ ++static int ++stat_compare (const void *vp1, const void *vp2) ++{ ++ const struct vddk_stat *st1 = vp1; ++ const struct vddk_stat *st2 = vp2; ++ ++ /* Note: sorts in reverse order of time spent in each API call. */ ++ if (st1->usecs < st2->usecs) return 1; ++ else if (st1->usecs > st2->usecs) return -1; ++ else return 0; ++} ++ ++static const char * ++api_name_without_prefix (const char *name) ++{ ++ return strncmp (name, "VixDiskLib_", 11) == 0 ? name + 11 : name; ++} ++ ++void ++display_stats (void) ++{ ++ statlist stats = empty_vector; ++ size_t i; ++ ++ if (!vddk_debug_stats) return; ++ ++#define STUB(fn,ret,args) statlist_append (&stats, stats_##fn) ++#define OPTIONAL_STUB(fn,ret,args) statlist_append (&stats, stats_##fn) ++#include "vddk-stubs.h" ++#undef STUB ++#undef OPTIONAL_STUB ++ ++ qsort (stats.ptr, stats.size, sizeof stats.ptr[0], stat_compare); ++ ++ nbdkit_debug ("VDDK function stats (-D vddk.stats=1):"); ++ nbdkit_debug ("%-24s %15s %5s %15s", ++ "VixDiskLib_...", "µs", "calls", "bytes"); ++ for (i = 0; i < stats.size; ++i) { ++ if (stats.ptr[i].usecs) { ++ if (stats.ptr[i].bytes > 0) ++ nbdkit_debug (" %-22s %15" PRIi64 " %5" PRIu64 " %15" PRIu64, ++ api_name_without_prefix (stats.ptr[i].name), ++ stats.ptr[i].usecs, ++ stats.ptr[i].calls, ++ stats.ptr[i].bytes); ++ else ++ nbdkit_debug (" %-22s %15" PRIi64 " %5" PRIu64, ++ api_name_without_prefix (stats.ptr[i].name), ++ stats.ptr[i].usecs, ++ stats.ptr[i].calls); ++ } ++ } ++ statlist_reset (&stats); ++} +diff --git a/plugins/vddk/vddk.c b/plugins/vddk/vddk.c +index 041bff1a..67ac775c 100644 +--- a/plugins/vddk/vddk.c ++++ b/plugins/vddk/vddk.c +@@ -61,7 +61,6 @@ + NBDKIT_DLL_PUBLIC int vddk_debug_diskinfo; + NBDKIT_DLL_PUBLIC int vddk_debug_extents; + NBDKIT_DLL_PUBLIC int vddk_debug_datapath = 1; +-NBDKIT_DLL_PUBLIC int vddk_debug_stats; + + /* For each VDDK API define a global variable. These globals are + * initialized when the plugin is loaded (by vddk_get_ready). +@@ -99,25 +98,6 @@ bool unbuffered; /* unbuffered */ + const char *username; /* user */ + const char *vmx_spec; /* vm */ + +-/* For each VDDK API define a variable to store the time taken (used +- * to implement -D vddk.stats=1). +- */ +-struct vddk_stat { +- const char *name; /* function name */ +- int64_t usecs; /* total number of usecs consumed */ +- uint64_t calls; /* number of times called */ +- uint64_t bytes; /* bytes transferred, datapath calls only */ +-}; +-static pthread_mutex_t stats_lock = PTHREAD_MUTEX_INITIALIZER; +-static void display_stats (void); +-#define STUB(fn,ret,args) \ +- static struct vddk_stat stats_##fn = { .name = #fn } +-#define OPTIONAL_STUB(fn,ret,args) \ +- static struct vddk_stat stats_##fn = { .name = #fn } +-#include "vddk-stubs.h" +-#undef STUB +-#undef OPTIONAL_STUB +- + /* Unload the plugin. */ + static void + vddk_unload (void) +@@ -130,69 +110,13 @@ vddk_unload (void) + if (dl) + dlclose (dl); + +- if (vddk_debug_stats) +- display_stats (); ++ display_stats (); + + free (config); + free (libdir); + free (password); + } + +-DEFINE_VECTOR_TYPE(statlist, struct vddk_stat) +- +-static int +-stat_compare (const void *vp1, const void *vp2) +-{ +- const struct vddk_stat *st1 = vp1; +- const struct vddk_stat *st2 = vp2; +- +- /* Note: sorts in reverse order of time spent in each API call. */ +- if (st1->usecs < st2->usecs) return 1; +- else if (st1->usecs > st2->usecs) return -1; +- else return 0; +-} +- +-static const char * +-api_name_without_prefix (const char *name) +-{ +- return strncmp (name, "VixDiskLib_", 11) == 0 ? name + 11 : name; +-} +- +-static void +-display_stats (void) +-{ +- statlist stats = empty_vector; +- size_t i; +- +-#define STUB(fn,ret,args) statlist_append (&stats, stats_##fn) +-#define OPTIONAL_STUB(fn,ret,args) statlist_append (&stats, stats_##fn) +-#include "vddk-stubs.h" +-#undef STUB +-#undef OPTIONAL_STUB +- +- qsort (stats.ptr, stats.size, sizeof stats.ptr[0], stat_compare); +- +- nbdkit_debug ("VDDK function stats (-D vddk.stats=1):"); +- nbdkit_debug ("%-24s %15s %5s %15s", +- "VixDiskLib_...", "µs", "calls", "bytes"); +- for (i = 0; i < stats.size; ++i) { +- if (stats.ptr[i].usecs) { +- if (stats.ptr[i].bytes > 0) +- nbdkit_debug (" %-22s %15" PRIi64 " %5" PRIu64 " %15" PRIu64, +- api_name_without_prefix (stats.ptr[i].name), +- stats.ptr[i].usecs, +- stats.ptr[i].calls, +- stats.ptr[i].bytes); +- else +- nbdkit_debug (" %-22s %15" PRIi64 " %5" PRIu64, +- api_name_without_prefix (stats.ptr[i].name), +- stats.ptr[i].usecs, +- stats.ptr[i].calls); +- } +- } +- statlist_reset (&stats); +-} +- + static void + trim (char *str) + { +diff --git a/plugins/vddk/vddk.h b/plugins/vddk/vddk.h +index 29775eb4..1400589d 100644 +--- a/plugins/vddk/vddk.h ++++ b/plugins/vddk/vddk.h +@@ -126,4 +126,19 @@ extern char *reexeced; + extern void reexec_if_needed (const char *prepend); + extern int restore_ld_library_path (void); + ++/* stats.c */ ++struct vddk_stat { ++ const char *name; /* function name */ ++ int64_t usecs; /* total number of usecs consumed */ ++ uint64_t calls; /* number of times called */ ++ uint64_t bytes; /* bytes transferred, datapath calls only */ ++}; ++extern pthread_mutex_t stats_lock; ++#define STUB(fn,ret,args) extern struct vddk_stat stats_##fn; ++#define OPTIONAL_STUB(fn,ret,args) STUB(fn,ret,args) ++#include "vddk-stubs.h" ++#undef STUB ++#undef OPTIONAL_STUB ++extern void display_stats (void); ++ + #endif /* NBDKIT_VDDK_H */ +-- +2.31.1 + diff --git a/SOURCES/0014-cache-Move-plugin-args-in-synopsis-earlier.patch b/SOURCES/0014-cache-Move-plugin-args-in-synopsis-earlier.patch deleted file mode 100644 index 5efa94d..0000000 --- a/SOURCES/0014-cache-Move-plugin-args-in-synopsis-earlier.patch +++ /dev/null @@ -1,35 +0,0 @@ -From c8c1e74a8c1c112b83646ac09fe7f9bde097a52a Mon Sep 17 00:00:00 2001 -From: "Richard W.M. Jones" -Date: Thu, 5 Aug 2021 18:20:37 +0100 -Subject: [PATCH] cache: Move plugin-args in synopsis earlier - -Makes this page consistent with nbdkit-cow-filter. - -(cherry picked from commit f1ddcef468907b0321041b1c4e0a430be46920be) ---- - filters/cache/nbdkit-cache-filter.pod | 4 ++-- - 1 file changed, 2 insertions(+), 2 deletions(-) - -diff --git a/filters/cache/nbdkit-cache-filter.pod b/filters/cache/nbdkit-cache-filter.pod -index df9c1f99..d85fef09 100644 ---- a/filters/cache/nbdkit-cache-filter.pod -+++ b/filters/cache/nbdkit-cache-filter.pod -@@ -4,13 +4,13 @@ nbdkit-cache-filter - nbdkit caching filter - - =head1 SYNOPSIS - -- nbdkit --filter=cache plugin [cache=writeback|writethrough|unsafe] -+ nbdkit --filter=cache plugin [plugin-args...] -+ [cache=writeback|writethrough|unsafe] - [cache-min-block-size=SIZE] - [cache-max-size=SIZE] - [cache-high-threshold=N] - [cache-low-threshold=N] - [cache-on-read=true|false|/PATH] -- [plugin-args...] - - =head1 DESCRIPTION - --- -2.31.1 - diff --git a/SOURCES/0014-vddk-Implement-parallel-thread-model.patch b/SOURCES/0014-vddk-Implement-parallel-thread-model.patch new file mode 100644 index 0000000..4b12382 --- /dev/null +++ b/SOURCES/0014-vddk-Implement-parallel-thread-model.patch @@ -0,0 +1,1287 @@ +From 5744b0000addaa0d50b6e0ee8e4540349623be0a Mon Sep 17 00:00:00 2001 +From: "Richard W.M. Jones" +Date: Wed, 27 Oct 2021 10:17:22 +0100 +Subject: [PATCH] vddk: Implement parallel thread model + +Since VDDK 6.0, asynchronous read and write operations are available. +This commit makes use of these, allowing us to use the parallel thread +model for increased performance. + +Note that at least VDDK 6.5 is required because VDDK 6.0 had a +different and incompatible signature for VixDiskLibCompletionCB. + +Also note at least vSphere 6.7 is required for asynch calls to make +any performance difference. In older versions they work +synchronously. + +In the parallel thread model, nbdkit will be calling us in parallel +from multiple nbdkit threads. VDDK does not allow multiple threads to +simultaneously call VDDK operations on the same handle. So we create +a background thread per handle (== connection). + +Only the background thread makes VDDK calls[1]. The background thread +handles a mix of synchronous (like extents, flush) and asynchronous +(like read, write) operations, but all from one thread. + +Parallel nbdkit threads issue commands to the background thread +associated with each handle, and wait until they are retired. + +[1] All VDDK calls except for connecting and disconnecting which for +different reasons are protected by a global lock, so I did not need to +change those. + +(cherry picked from commit 1eecf15fc3d8ea253ccec4f5883fdbb9aa6f8c2b) +--- + plugins/vddk/Makefile.am | 1 + + plugins/vddk/nbdkit-vddk-plugin.pod | 11 +- + plugins/vddk/vddk.c | 380 +++++-------------- + plugins/vddk/vddk.h | 49 ++- + plugins/vddk/worker.c | 567 ++++++++++++++++++++++++++++ + tests/dummy-vddk.c | 32 ++ + 6 files changed, 745 insertions(+), 295 deletions(-) + create mode 100644 plugins/vddk/worker.c + +diff --git a/plugins/vddk/Makefile.am b/plugins/vddk/Makefile.am +index 4f470ff9..f8382fc9 100644 +--- a/plugins/vddk/Makefile.am ++++ b/plugins/vddk/Makefile.am +@@ -49,6 +49,7 @@ nbdkit_vddk_plugin_la_SOURCES = \ + stats.c \ + vddk-structs.h \ + vddk-stubs.h \ ++ worker.c \ + $(top_srcdir)/include/nbdkit-plugin.h \ + $(NULL) + +diff --git a/plugins/vddk/nbdkit-vddk-plugin.pod b/plugins/vddk/nbdkit-vddk-plugin.pod +index 1c16d096..ce82a734 100644 +--- a/plugins/vddk/nbdkit-vddk-plugin.pod ++++ b/plugins/vddk/nbdkit-vddk-plugin.pod +@@ -523,6 +523,14 @@ read bandwidth to the VMware server. + + Same as above, but for writing and flushing writes. + ++=item C ++ ++=item C ++ ++Same as above, but for asynchronous read and write calls introduced in ++nbdkit 1.30. Unfortunately at the moment the amount of time spent in ++these calls is not accounted for correctly. ++ + =item C + + This call is used to query information about the sparseness of the +@@ -580,7 +588,8 @@ Debug extents returned by C. + + =item B<-D vddk.datapath=0> + +-Suppress debugging of datapath calls (C and C). ++Suppress debugging of datapath calls (C, C, C ++and C). + + =item B<-D vddk.stats=1> + +diff --git a/plugins/vddk/vddk.c b/plugins/vddk/vddk.c +index 67ac775c..9f223db0 100644 +--- a/plugins/vddk/vddk.c ++++ b/plugins/vddk/vddk.c +@@ -50,9 +50,6 @@ + #include + + #include "cleanup.h" +-#include "minmax.h" +-#include "rounding.h" +-#include "tvdiff.h" + #include "vector.h" + + #include "vddk.h" +@@ -522,23 +519,18 @@ vddk_dump_plugin (void) + /* The rules on threads and VDDK are here: + * https://code.vmware.com/docs/11750/virtual-disk-development-kit-programming-guide/GUID-6BE903E8-DC70-46D9-98E4-E34A2002C2AD.html + * +- * Before nbdkit 1.22 we used SERIALIZE_ALL_REQUESTS. Since nbdkit +- * 1.22 we changed this to SERIALIZE_REQUESTS and added a mutex around +- * calls to VixDiskLib_Open and VixDiskLib_Close. This is not quite +- * within the letter of the rules, but is within the spirit. ++ * Before nbdkit 1.22 we used SERIALIZE_ALL_REQUESTS. In nbdkit ++ * 1.22-1.28 we changed this to SERIALIZE_REQUESTS and added a mutex ++ * around calls to VixDiskLib_Open and VixDiskLib_Close. In nbdkit ++ * 1.30 and above we assign a background thread per connection to do ++ * asynch operations and use the PARALLEL model. We still need the ++ * lock around Open and Close. + */ +-#define THREAD_MODEL NBDKIT_THREAD_MODEL_SERIALIZE_REQUESTS ++#define THREAD_MODEL NBDKIT_THREAD_MODEL_PARALLEL + + /* Lock protecting open/close calls - see above. */ + static pthread_mutex_t open_close_lock = PTHREAD_MUTEX_INITIALIZER; + +-/* The per-connection handle. */ +-struct vddk_handle { +- VixDiskLibConnectParams *params; /* connection parameters */ +- VixDiskLibConnection connection; /* connection */ +- VixDiskLibHandle handle; /* disk handle */ +-}; +- + static inline VixDiskLibConnectParams * + allocate_connect_params (void) + { +@@ -579,12 +571,16 @@ vddk_open (int readonly) + VixError err; + uint32_t flags; + const char *transport_mode; ++ int pterr; + +- h = malloc (sizeof *h); ++ h = calloc (1, sizeof *h); + if (h == NULL) { +- nbdkit_error ("malloc: %m"); ++ nbdkit_error ("calloc: %m"); + return NULL; + } ++ h->commands = (command_queue) empty_vector; ++ pthread_mutex_init (&h->commands_lock, NULL); ++ pthread_cond_init (&h->commands_cond, NULL); + + h->params = allocate_connect_params (); + if (h->params == NULL) { +@@ -661,8 +657,22 @@ vddk_open (int readonly) + VDDK_CALL_END (VixDiskLib_GetTransportMode, 0); + nbdkit_debug ("transport mode: %s", transport_mode); + ++ /* Start the background thread which actually does the asynchronous ++ * work. ++ */ ++ pterr = pthread_create (&h->thread, NULL, vddk_worker_thread, h); ++ if (pterr != 0) { ++ errno = pterr; ++ nbdkit_error ("pthread_create: %m"); ++ goto err3; ++ } ++ + return h; + ++ err3: ++ VDDK_CALL_START (VixDiskLib_Close, "handle") ++ VixDiskLib_Close (h->handle); ++ VDDK_CALL_END (VixDiskLib_Close, 0); + err2: + VDDK_CALL_START (VixDiskLib_Disconnect, "connection") + VixDiskLib_Disconnect (h->connection); +@@ -670,6 +680,8 @@ vddk_open (int readonly) + err1: + free_connect_params (h->params); + err0: ++ pthread_mutex_destroy (&h->commands_lock); ++ pthread_cond_destroy (&h->commands_cond); + free (h); + return NULL; + } +@@ -680,6 +692,10 @@ vddk_close (void *handle) + { + ACQUIRE_LOCK_FOR_CURRENT_SCOPE (&open_close_lock); + struct vddk_handle *h = handle; ++ struct command stop_cmd = { .type = STOP }; ++ ++ send_command_and_wait (h, &stop_cmd); ++ pthread_join (h->thread, NULL); + + VDDK_CALL_START (VixDiskLib_Close, "handle") + VixDiskLib_Close (h->handle); +@@ -689,6 +705,9 @@ vddk_close (void *handle) + VDDK_CALL_END (VixDiskLib_Disconnect, 0); + + free_connect_params (h->params); ++ pthread_mutex_destroy (&h->commands_lock); ++ pthread_cond_destroy (&h->commands_cond); ++ command_queue_reset (&h->commands); + free (h); + } + +@@ -697,54 +716,29 @@ static int64_t + vddk_get_size (void *handle) + { + struct vddk_handle *h = handle; +- VixDiskLibInfo *info; +- VixError err; + uint64_t size; ++ struct command get_size_cmd = { .type = GET_SIZE, .ptr = &size }; + +- VDDK_CALL_START (VixDiskLib_GetInfo, "handle, &info") +- err = VixDiskLib_GetInfo (h->handle, &info); +- VDDK_CALL_END (VixDiskLib_GetInfo, 0); +- if (err != VIX_OK) { +- VDDK_ERROR (err, "VixDiskLib_GetInfo"); ++ if (send_command_and_wait (h, &get_size_cmd) == -1) + return -1; +- } +- +- size = info->capacity * (uint64_t)VIXDISKLIB_SECTOR_SIZE; +- +- if (vddk_debug_diskinfo) { +- nbdkit_debug ("disk info: capacity: %" PRIu64 " sectors " +- "(%" PRIi64 " bytes)", +- info->capacity, size); +- nbdkit_debug ("disk info: biosGeo: C:%" PRIu32 " H:%" PRIu32 " S:%" PRIu32, +- info->biosGeo.cylinders, +- info->biosGeo.heads, +- info->biosGeo.sectors); +- nbdkit_debug ("disk info: physGeo: C:%" PRIu32 " H:%" PRIu32 " S:%" PRIu32, +- info->physGeo.cylinders, +- info->physGeo.heads, +- info->physGeo.sectors); +- nbdkit_debug ("disk info: adapter type: %d", +- (int) info->adapterType); +- nbdkit_debug ("disk info: num links: %d", info->numLinks); +- nbdkit_debug ("disk info: parent filename hint: %s", +- info->parentFileNameHint ? : "NULL"); +- nbdkit_debug ("disk info: uuid: %s", +- info->uuid ? : "NULL"); +- if (library_version >= 7) { +- nbdkit_debug ("disk info: sector size: " +- "logical %" PRIu32 " physical %" PRIu32, +- info->logicalSectorSize, +- info->physicalSectorSize); +- } +- } +- +- VDDK_CALL_START (VixDiskLib_FreeInfo, "info") +- VixDiskLib_FreeInfo (info); +- VDDK_CALL_END (VixDiskLib_FreeInfo, 0); + + return (int64_t) size; + } + ++static int ++vddk_can_fua (void *handle) ++{ ++ /* The Flush call was not available in VDDK < 6.0. */ ++ return VixDiskLib_Flush != NULL ? NBDKIT_FUA_NATIVE : NBDKIT_FUA_NONE; ++} ++ ++static int ++vddk_can_flush (void *handle) ++{ ++ /* The Flush call was not available in VDDK < 6.0. */ ++ return VixDiskLib_Flush != NULL; ++} ++ + /* Read data from the file. + * + * Note that reads have to be aligned to sectors (XXX). +@@ -754,32 +748,14 @@ vddk_pread (void *handle, void *buf, uint32_t count, uint64_t offset, + uint32_t flags) + { + struct vddk_handle *h = handle; +- VixError err; ++ struct command read_cmd = { ++ .type = READ, ++ .ptr = buf, ++ .count = count, ++ .offset = offset, ++ }; + +- /* Align to sectors. */ +- if (!IS_ALIGNED (offset, VIXDISKLIB_SECTOR_SIZE)) { +- nbdkit_error ("%s is not aligned to sectors", "read"); +- return -1; +- } +- if (!IS_ALIGNED (count, VIXDISKLIB_SECTOR_SIZE)) { +- nbdkit_error ("%s is not aligned to sectors", "read"); +- return -1; +- } +- offset /= VIXDISKLIB_SECTOR_SIZE; +- count /= VIXDISKLIB_SECTOR_SIZE; +- +- VDDK_CALL_START (VixDiskLib_Read, +- "handle, %" PRIu64 " sectors, " +- "%" PRIu32 " sectors, buffer", +- offset, count) +- err = VixDiskLib_Read (h->handle, offset, count, buf); +- VDDK_CALL_END (VixDiskLib_Read, count * VIXDISKLIB_SECTOR_SIZE); +- if (err != VIX_OK) { +- VDDK_ERROR (err, "VixDiskLib_Read"); +- return -1; +- } +- +- return 0; ++ return send_command_and_wait (h, &read_cmd); + } + + static int vddk_flush (void *handle, uint32_t flags); +@@ -792,32 +768,17 @@ static int + vddk_pwrite (void *handle, const void *buf, uint32_t count, uint64_t offset, + uint32_t flags) + { ++ struct vddk_handle *h = handle; + const bool fua = flags & NBDKIT_FLAG_FUA; +- struct vddk_handle *h = handle; +- VixError err; ++ struct command write_cmd = { ++ .type = WRITE, ++ .ptr = (void *) buf, ++ .count = count, ++ .offset = offset, ++ }; + +- /* Align to sectors. */ +- if (!IS_ALIGNED (offset, VIXDISKLIB_SECTOR_SIZE)) { +- nbdkit_error ("%s is not aligned to sectors", "write"); ++ if (send_command_and_wait (h, &write_cmd) == -1) + return -1; +- } +- if (!IS_ALIGNED (count, VIXDISKLIB_SECTOR_SIZE)) { +- nbdkit_error ("%s is not aligned to sectors", "write"); +- return -1; +- } +- offset /= VIXDISKLIB_SECTOR_SIZE; +- count /= VIXDISKLIB_SECTOR_SIZE; +- +- VDDK_CALL_START (VixDiskLib_Write, +- "handle, %" PRIu64 " sectors, " +- "%" PRIu32 " sectors, buffer", +- offset, count) +- err = VixDiskLib_Write (h->handle, offset, count, buf); +- VDDK_CALL_END (VixDiskLib_Write, count * VIXDISKLIB_SECTOR_SIZE); +- if (err != VIX_OK) { +- VDDK_ERROR (err, "VixDiskLib_Write"); +- return -1; +- } + + if (fua) { + if (vddk_flush (handle, 0) == -1) +@@ -827,126 +788,32 @@ vddk_pwrite (void *handle, const void *buf, uint32_t count, uint64_t offset, + return 0; + } + +-static int +-vddk_can_fua (void *handle) +-{ +- /* The Flush call was not available in VDDK < 6.0. */ +- return VixDiskLib_Flush != NULL ? NBDKIT_FUA_NATIVE : NBDKIT_FUA_NONE; +-} +- +-static int +-vddk_can_flush (void *handle) +-{ +- /* The Flush call was not available in VDDK < 6.0. */ +- return VixDiskLib_Flush != NULL; +-} +- + /* Flush data to the file. */ + static int + vddk_flush (void *handle, uint32_t flags) + { + struct vddk_handle *h = handle; +- VixError err; ++ struct command flush_cmd = { ++ .type = FLUSH, ++ }; + +- /* The documentation for Flush is missing, but the comment in the +- * header file seems to indicate that it waits for WriteAsync +- * commands to finish. We don't use WriteAsync, and in any case +- * there's a new function Wait to wait for those. However I +- * verified using strace that in fact Flush does call fsync on the +- * file so it appears to be the correct call to use here. +- */ +- +- VDDK_CALL_START (VixDiskLib_Flush, "handle") +- err = VixDiskLib_Flush (h->handle); +- VDDK_CALL_END (VixDiskLib_Flush, 0); +- if (err != VIX_OK) { +- VDDK_ERROR (err, "VixDiskLib_Flush"); +- return -1; +- } +- +- return 0; ++ return send_command_and_wait (h, &flush_cmd); + } + + static int + vddk_can_extents (void *handle) + { + struct vddk_handle *h = handle; +- VixError err; +- VixDiskLibBlockList *block_list; ++ int ret; ++ struct command can_extents_cmd = { ++ .type = CAN_EXTENTS, ++ .ptr = &ret, ++ }; + +- /* This call was added in VDDK 6.7. In earlier versions the +- * function pointer will be NULL and we cannot query extents. +- */ +- if (VixDiskLib_QueryAllocatedBlocks == NULL) { +- nbdkit_debug ("can_extents: VixDiskLib_QueryAllocatedBlocks == NULL, " +- "probably this is VDDK < 6.7"); +- return 0; +- } +- +- /* Suppress errors around this call. See: +- * https://bugzilla.redhat.com/show_bug.cgi?id=1709211#c7 +- */ +- error_suppression = 1; +- +- /* However even when the call is available it rarely works well so +- * the best thing we can do here is to try the call and if it's +- * non-functional return false. +- */ +- VDDK_CALL_START (VixDiskLib_QueryAllocatedBlocks, +- "handle, 0, %d sectors, %d sectors", +- VIXDISKLIB_MIN_CHUNK_SIZE, VIXDISKLIB_MIN_CHUNK_SIZE) +- err = VixDiskLib_QueryAllocatedBlocks (h->handle, +- 0, VIXDISKLIB_MIN_CHUNK_SIZE, +- VIXDISKLIB_MIN_CHUNK_SIZE, +- &block_list); +- VDDK_CALL_END (VixDiskLib_QueryAllocatedBlocks, 0); +- error_suppression = 0; +- if (err == VIX_OK) { +- VDDK_CALL_START (VixDiskLib_FreeBlockList, "block_list") +- VixDiskLib_FreeBlockList (block_list); +- VDDK_CALL_END (VixDiskLib_FreeBlockList, 0); +- } +- if (err != VIX_OK) { +- char *errmsg = VixDiskLib_GetErrorText (err, NULL); +- nbdkit_debug ("can_extents: VixDiskLib_QueryAllocatedBlocks test failed, " +- "extents support will be disabled: " +- "original error: %s", +- errmsg); +- VixDiskLib_FreeErrorText (errmsg); +- return 0; +- } +- +- return 1; +-} +- +-static int +-add_extent (struct nbdkit_extents *extents, +- uint64_t *position, uint64_t next_position, bool is_hole) +-{ +- uint32_t type = 0; +- const uint64_t length = next_position - *position; +- +- if (is_hole) { +- type = NBDKIT_EXTENT_HOLE; +- /* Images opened as single link might be backed by another file in the +- chain, so the holes are not guaranteed to be zeroes. */ +- if (!single_link) +- type |= NBDKIT_EXTENT_ZERO; +- } +- +- assert (*position <= next_position); +- if (*position == next_position) +- return 0; +- +- if (vddk_debug_extents) +- nbdkit_debug ("adding extent type %s at [%" PRIu64 "...%" PRIu64 "]", +- is_hole ? "hole" : "allocated data", +- *position, next_position-1); +- if (nbdkit_add_extent (extents, *position, length, type) == -1) ++ if (send_command_and_wait (h, &can_extents_cmd) == -1) + return -1; + +- *position = next_position; +- return 0; ++ return ret; + } + + static int +@@ -955,88 +822,15 @@ vddk_extents (void *handle, uint32_t count, uint64_t offset, uint32_t flags, + { + struct vddk_handle *h = handle; + bool req_one = flags & NBDKIT_FLAG_REQ_ONE; +- uint64_t position, end, start_sector; +- +- position = offset; +- end = offset + count; +- +- /* We can only query whole chunks. Therefore start with the first +- * chunk before offset. +- */ +- start_sector = +- ROUND_DOWN (offset, VIXDISKLIB_MIN_CHUNK_SIZE * VIXDISKLIB_SECTOR_SIZE) +- / VIXDISKLIB_SECTOR_SIZE; +- while (start_sector * VIXDISKLIB_SECTOR_SIZE < end) { +- VixError err; +- uint32_t i; +- uint64_t nr_chunks, nr_sectors; +- VixDiskLibBlockList *block_list; +- +- assert (IS_ALIGNED (start_sector, VIXDISKLIB_MIN_CHUNK_SIZE)); +- +- nr_chunks = +- ROUND_UP (end - start_sector * VIXDISKLIB_SECTOR_SIZE, +- VIXDISKLIB_MIN_CHUNK_SIZE * VIXDISKLIB_SECTOR_SIZE) +- / (VIXDISKLIB_MIN_CHUNK_SIZE * VIXDISKLIB_SECTOR_SIZE); +- nr_chunks = MIN (nr_chunks, VIXDISKLIB_MAX_CHUNK_NUMBER); +- nr_sectors = nr_chunks * VIXDISKLIB_MIN_CHUNK_SIZE; +- +- VDDK_CALL_START (VixDiskLib_QueryAllocatedBlocks, +- "handle, %" PRIu64 " sectors, %" PRIu64 " sectors, " +- "%d sectors", +- start_sector, nr_sectors, VIXDISKLIB_MIN_CHUNK_SIZE) +- err = VixDiskLib_QueryAllocatedBlocks (h->handle, +- start_sector, nr_sectors, +- VIXDISKLIB_MIN_CHUNK_SIZE, +- &block_list); +- VDDK_CALL_END (VixDiskLib_QueryAllocatedBlocks, 0); +- if (err != VIX_OK) { +- VDDK_ERROR (err, "VixDiskLib_QueryAllocatedBlocks"); +- return -1; +- } +- +- for (i = 0; i < block_list->numBlocks; ++i) { +- uint64_t blk_offset, blk_length; +- +- blk_offset = block_list->blocks[i].offset * VIXDISKLIB_SECTOR_SIZE; +- blk_length = block_list->blocks[i].length * VIXDISKLIB_SECTOR_SIZE; +- +- /* The query returns allocated blocks. We must insert holes +- * between the blocks as necessary. +- */ +- if ((position < blk_offset && +- add_extent (extents, &position, blk_offset, true) == -1) || +- (add_extent (extents, +- &position, blk_offset + blk_length, false) == -1)) { +- VDDK_CALL_START (VixDiskLib_FreeBlockList, "block_list") +- VixDiskLib_FreeBlockList (block_list); +- VDDK_CALL_END (VixDiskLib_FreeBlockList, 0); +- return -1; +- } +- } +- VDDK_CALL_START (VixDiskLib_FreeBlockList, "block_list") +- VixDiskLib_FreeBlockList (block_list); +- VDDK_CALL_END (VixDiskLib_FreeBlockList, 0); +- +- /* There's an implicit hole after the returned list of blocks, up +- * to the end of the QueryAllocatedBlocks request. +- */ +- if (add_extent (extents, +- &position, +- (start_sector + nr_sectors) * VIXDISKLIB_SECTOR_SIZE, +- true) == -1) +- return -1; +- +- start_sector += nr_sectors; +- +- /* If one extent was requested, as long as we've added an extent +- * overlapping the original offset we're done. +- */ +- if (req_one && position > offset) +- break; +- } +- +- return 0; ++ struct command extents_cmd = { ++ .type = EXTENTS, ++ .ptr = extents, ++ .count = count, ++ .offset = offset, ++ .req_one = req_one, ++ }; ++ ++ return send_command_and_wait (h, &extents_cmd); + } + + static struct nbdkit_plugin plugin = { +diff --git a/plugins/vddk/vddk.h b/plugins/vddk/vddk.h +index 1400589d..be0b3492 100644 +--- a/plugins/vddk/vddk.h ++++ b/plugins/vddk/vddk.h +@@ -90,7 +90,9 @@ extern int vddk_debug_stats; + /* GCC can optimize this away at compile time: */ \ + const bool datapath = \ + strcmp (#fn, "VixDiskLib_Read") == 0 || \ +- strcmp (#fn, "VixDiskLib_Write") == 0; \ ++ strcmp (#fn, "VixDiskLib_ReadAsync") == 0 || \ ++ strcmp (#fn, "VixDiskLib_Write") == 0 || \ ++ strcmp (#fn, "VixDiskLib_WriteAsync") == 0; \ + if (vddk_debug_stats) \ + gettimeofday (&start_t, NULL); \ + if (!datapath || vddk_debug_datapath) \ +@@ -120,6 +122,46 @@ extern int vddk_debug_stats; + VDDK_CALL_END (VixDiskLib_FreeErrorText, 0); \ + } while (0) + ++/* Queue of asynchronous commands sent to the background thread. */ ++enum command_type { GET_SIZE, READ, WRITE, FLUSH, CAN_EXTENTS, EXTENTS, STOP }; ++struct command { ++ /* These fields are set by the caller. */ ++ enum command_type type; /* command */ ++ void *ptr; /* buffer, extents list, return values */ ++ uint32_t count; /* READ, WRITE, EXTENTS */ ++ uint64_t offset; /* READ, WRITE, EXTENTS */ ++ bool req_one; /* EXTENTS NBDKIT_FLAG_REQ_ONE */ ++ ++ /* This field is set to a unique value by send_command_and_wait. */ ++ uint64_t id; /* serial number */ ++ ++ /* These fields are used by the internal implementation. */ ++ pthread_mutex_t mutex; /* completion mutex */ ++ pthread_cond_t cond; /* completion condition */ ++ enum { SUBMITTED, SUCCEEDED, FAILED } status; ++}; ++ ++DEFINE_VECTOR_TYPE(command_queue, struct command *) ++ ++/* The per-connection handle. */ ++struct vddk_handle { ++ VixDiskLibConnectParams *params; /* connection parameters */ ++ VixDiskLibConnection connection; /* connection */ ++ VixDiskLibHandle handle; /* disk handle */ ++ ++ pthread_t thread; /* background thread for asynch work */ ++ ++ /* Command queue of commands sent to the background thread. Use ++ * send_command_and_wait to add a command. Only the background ++ * thread must make VDDK API calls (apart from opening and closing). ++ * The lock protects all of these fields. ++ */ ++ pthread_mutex_t commands_lock; /* lock */ ++ command_queue commands; /* command queue */ ++ pthread_cond_t commands_cond; /* condition (queue size 0 -> 1) */ ++ uint64_t id; /* next command ID */ ++}; ++ + /* reexec.c */ + extern bool noreexec; + extern char *reexeced; +@@ -141,4 +183,9 @@ extern pthread_mutex_t stats_lock; + #undef OPTIONAL_STUB + extern void display_stats (void); + ++/* worker.c */ ++extern const char *command_type_string (enum command_type type); ++extern int send_command_and_wait (struct vddk_handle *h, struct command *cmd); ++extern void *vddk_worker_thread (void *handle); ++ + #endif /* NBDKIT_VDDK_H */ +diff --git a/plugins/vddk/worker.c b/plugins/vddk/worker.c +new file mode 100644 +index 00000000..2a1d4f26 +--- /dev/null ++++ b/plugins/vddk/worker.c +@@ -0,0 +1,567 @@ ++/* nbdkit ++ * Copyright (C) 2013-2021 Red Hat Inc. ++ * ++ * Redistribution and use in source and binary forms, with or without ++ * modification, are permitted provided that the following conditions are ++ * met: ++ * ++ * * Redistributions of source code must retain the above copyright ++ * notice, this list of conditions and the following disclaimer. ++ * ++ * * Redistributions in binary form must reproduce the above copyright ++ * notice, this list of conditions and the following disclaimer in the ++ * documentation and/or other materials provided with the distribution. ++ * ++ * * Neither the name of Red Hat nor the names of its contributors may be ++ * used to endorse or promote products derived from this software without ++ * specific prior written permission. ++ * ++ * THIS SOFTWARE IS PROVIDED BY RED HAT AND CONTRIBUTORS ''AS IS'' AND ++ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, ++ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A ++ * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL RED HAT OR ++ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, ++ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT ++ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF ++ * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ++ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, ++ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT ++ * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF ++ * SUCH DAMAGE. ++ */ ++ ++#include ++ ++#include ++#include ++#include ++#include ++ ++#include ++ ++#define NBDKIT_API_VERSION 2 ++#include ++ ++#include "cleanup.h" ++#include "minmax.h" ++#include "rounding.h" ++#include "vector.h" ++ ++#include "vddk.h" ++ ++const char * ++command_type_string (enum command_type type) ++{ ++ switch (type) { ++ case GET_SIZE: return "get_size"; ++ case READ: return "read"; ++ case WRITE: return "write"; ++ case FLUSH: return "flush"; ++ case CAN_EXTENTS: return "can_extents"; ++ case EXTENTS: return "extents"; ++ case STOP: return "stop"; ++ default: abort (); ++ } ++} ++ ++/* Send command to the background thread and wait for completion. ++ * ++ * Returns 0 for OK ++ * On error, calls nbdkit_error and returns -1. ++ */ ++int ++send_command_and_wait (struct vddk_handle *h, struct command *cmd) ++{ ++ /* Add the command to the command queue. */ ++ { ++ ACQUIRE_LOCK_FOR_CURRENT_SCOPE (&h->commands_lock); ++ cmd->id = h->id++; ++ ++ if (command_queue_append (&h->commands, cmd) == -1) ++ /* On error command_queue_append will call nbdkit_error. */ ++ return -1; ++ ++ /* Signal the caller if it could be sleeping on an empty queue. */ ++ if (h->commands.size == 1) ++ pthread_cond_signal (&h->commands_cond); ++ ++ /* This will be used to signal command completion back to us. */ ++ pthread_mutex_init (&cmd->mutex, NULL); ++ pthread_cond_init (&cmd->cond, NULL); ++ } ++ ++ /* Wait for the command to be completed by the background thread. */ ++ { ++ ACQUIRE_LOCK_FOR_CURRENT_SCOPE (&cmd->mutex); ++ while (cmd->status == SUBMITTED) ++ pthread_cond_wait (&cmd->cond, &cmd->mutex); ++ } ++ ++ pthread_mutex_destroy (&cmd->mutex); ++ pthread_cond_destroy (&cmd->cond); ++ ++ /* On error the background thread will call nbdkit_error. */ ++ switch (cmd->status) { ++ case SUCCEEDED: return 0; ++ case FAILED: return -1; ++ default: abort (); ++ } ++} ++ ++/* Asynchronous commands are completed when this function is called. */ ++static void ++complete_command (void *vp, VixError result) ++{ ++ struct command *cmd = vp; ++ ++ if (vddk_debug_datapath) ++ nbdkit_debug ("command %" PRIu64 " completed", cmd->id); ++ ++ ACQUIRE_LOCK_FOR_CURRENT_SCOPE (&cmd->mutex); ++ ++ if (result == VIX_OK) { ++ cmd->status = SUCCEEDED; ++ } else { ++ VDDK_ERROR (result, "command %" PRIu64 ": asynchronous %s failed", ++ cmd->id, command_type_string (cmd->type)); ++ cmd->status = FAILED; ++ } ++ ++ pthread_cond_signal (&cmd->cond); ++} ++ ++/* Wait for any asynchronous commands to complete. */ ++static int ++do_stop (struct command *cmd, struct vddk_handle *h) ++{ ++ VixError err; ++ ++ /* Because we assume VDDK >= 6.5, VixDiskLib_Wait must exist. */ ++ VDDK_CALL_START (VixDiskLib_Wait, "handle") ++ err = VixDiskLib_Wait (h->handle); ++ VDDK_CALL_END (VixDiskLib_Wait, 0); ++ if (err != VIX_OK) { ++ VDDK_ERROR (err, "VixDiskLib_Wait"); ++ /* In the end this error indication is ignored because it only ++ * happens on the close path when we cannot handle errors. ++ */ ++ return -1; ++ } ++ return 0; ++} ++ ++/* Get size command. */ ++static int64_t ++do_get_size (struct command *cmd, struct vddk_handle *h) ++{ ++ VixError err; ++ VixDiskLibInfo *info; ++ uint64_t size; ++ ++ VDDK_CALL_START (VixDiskLib_GetInfo, "handle, &info") ++ err = VixDiskLib_GetInfo (h->handle, &info); ++ VDDK_CALL_END (VixDiskLib_GetInfo, 0); ++ if (err != VIX_OK) { ++ VDDK_ERROR (err, "VixDiskLib_GetInfo"); ++ return -1; ++ } ++ ++ size = info->capacity * (uint64_t)VIXDISKLIB_SECTOR_SIZE; ++ ++ if (vddk_debug_diskinfo) { ++ nbdkit_debug ("disk info: capacity: %" PRIu64 " sectors " ++ "(%" PRIi64 " bytes)", ++ info->capacity, size); ++ nbdkit_debug ("disk info: biosGeo: C:%" PRIu32 " H:%" PRIu32 " S:%" PRIu32, ++ info->biosGeo.cylinders, ++ info->biosGeo.heads, ++ info->biosGeo.sectors); ++ nbdkit_debug ("disk info: physGeo: C:%" PRIu32 " H:%" PRIu32 " S:%" PRIu32, ++ info->physGeo.cylinders, ++ info->physGeo.heads, ++ info->physGeo.sectors); ++ nbdkit_debug ("disk info: adapter type: %d", ++ (int) info->adapterType); ++ nbdkit_debug ("disk info: num links: %d", info->numLinks); ++ nbdkit_debug ("disk info: parent filename hint: %s", ++ info->parentFileNameHint ? : "NULL"); ++ nbdkit_debug ("disk info: uuid: %s", ++ info->uuid ? : "NULL"); ++ if (library_version >= 7) { ++ nbdkit_debug ("disk info: sector size: " ++ "logical %" PRIu32 " physical %" PRIu32, ++ info->logicalSectorSize, ++ info->physicalSectorSize); ++ } ++ } ++ ++ VDDK_CALL_START (VixDiskLib_FreeInfo, "info") ++ VixDiskLib_FreeInfo (info); ++ VDDK_CALL_END (VixDiskLib_FreeInfo, 0); ++ ++ return (int64_t) size; ++} ++ ++static int ++do_read (struct command *cmd, struct vddk_handle *h) ++{ ++ VixError err; ++ uint32_t count = cmd->count; ++ uint64_t offset = cmd->offset; ++ void *buf = cmd->ptr; ++ ++ /* Align to sectors. */ ++ if (!IS_ALIGNED (offset, VIXDISKLIB_SECTOR_SIZE)) { ++ nbdkit_error ("%s is not aligned to sectors", "read"); ++ return -1; ++ } ++ if (!IS_ALIGNED (count, VIXDISKLIB_SECTOR_SIZE)) { ++ nbdkit_error ("%s is not aligned to sectors", "read"); ++ return -1; ++ } ++ offset /= VIXDISKLIB_SECTOR_SIZE; ++ count /= VIXDISKLIB_SECTOR_SIZE; ++ ++ VDDK_CALL_START (VixDiskLib_ReadAsync, ++ "handle, %" PRIu64 " sectors, " ++ "%" PRIu32 " sectors, buffer, callback, %" PRIu64, ++ offset, count, cmd->id) ++ err = VixDiskLib_ReadAsync (h->handle, offset, count, buf, ++ complete_command, cmd); ++ VDDK_CALL_END (VixDiskLib_ReadAsync, count * VIXDISKLIB_SECTOR_SIZE); ++ if (err != VIX_ASYNC) { ++ VDDK_ERROR (err, "VixDiskLib_ReadAsync"); ++ return -1; ++ } ++ ++ return 0; ++} ++ ++static int ++do_write (struct command *cmd, struct vddk_handle *h) ++{ ++ VixError err; ++ uint32_t count = cmd->count; ++ uint64_t offset = cmd->offset; ++ const void *buf = cmd->ptr; ++ ++ /* Align to sectors. */ ++ if (!IS_ALIGNED (offset, VIXDISKLIB_SECTOR_SIZE)) { ++ nbdkit_error ("%s is not aligned to sectors", "write"); ++ return -1; ++ } ++ if (!IS_ALIGNED (count, VIXDISKLIB_SECTOR_SIZE)) { ++ nbdkit_error ("%s is not aligned to sectors", "write"); ++ return -1; ++ } ++ offset /= VIXDISKLIB_SECTOR_SIZE; ++ count /= VIXDISKLIB_SECTOR_SIZE; ++ ++ VDDK_CALL_START (VixDiskLib_WriteAsync, ++ "handle, %" PRIu64 " sectors, " ++ "%" PRIu32 " sectors, buffer, callback, %" PRIu64, ++ offset, count, cmd->id) ++ err = VixDiskLib_WriteAsync (h->handle, offset, count, buf, ++ complete_command, cmd); ++ VDDK_CALL_END (VixDiskLib_WriteAsync, count * VIXDISKLIB_SECTOR_SIZE); ++ if (err != VIX_ASYNC) { ++ VDDK_ERROR (err, "VixDiskLib_WriteAsync"); ++ return -1; ++ } ++ ++ return 0; ++} ++ ++static int ++do_flush (struct command *cmd, struct vddk_handle *h) ++{ ++ VixError err; ++ ++ /* It seems safer to wait for outstanding asynchronous commands to ++ * complete before doing a flush, so do this but ignore errors ++ * except to print them. ++ */ ++ VDDK_CALL_START (VixDiskLib_Wait, "handle") ++ err = VixDiskLib_Wait (h->handle); ++ VDDK_CALL_END (VixDiskLib_Wait, 0); ++ if (err != VIX_OK) ++ VDDK_ERROR (err, "VixDiskLib_Wait"); ++ ++ /* The documentation for Flush is missing, but the comment in the ++ * header file seems to indicate that it waits for WriteAsync ++ * commands to finish. There's a new function Wait to wait for ++ * those. However I verified using strace that in fact Flush calls ++ * fsync on the file so it appears to be the correct call to use ++ * here. ++ */ ++ VDDK_CALL_START (VixDiskLib_Flush, "handle") ++ err = VixDiskLib_Flush (h->handle); ++ VDDK_CALL_END (VixDiskLib_Flush, 0); ++ if (err != VIX_OK) { ++ VDDK_ERROR (err, "VixDiskLib_Flush"); ++ return -1; ++ } ++ ++ return 0; ++} ++ ++static int ++do_can_extents (struct command *cmd, struct vddk_handle *h) ++{ ++ VixError err; ++ VixDiskLibBlockList *block_list; ++ ++ /* This call was added in VDDK 6.7. In earlier versions the ++ * function pointer will be NULL and we cannot query extents. ++ */ ++ if (VixDiskLib_QueryAllocatedBlocks == NULL) { ++ nbdkit_debug ("can_extents: VixDiskLib_QueryAllocatedBlocks == NULL, " ++ "probably this is VDDK < 6.7"); ++ return 0; ++ } ++ ++ /* Suppress errors around this call. See: ++ * https://bugzilla.redhat.com/show_bug.cgi?id=1709211#c7 ++ */ ++ error_suppression = 1; ++ ++ /* However even when the call is available it rarely works well so ++ * the best thing we can do here is to try the call and if it's ++ * non-functional return false. ++ */ ++ VDDK_CALL_START (VixDiskLib_QueryAllocatedBlocks, ++ "handle, 0, %d sectors, %d sectors", ++ VIXDISKLIB_MIN_CHUNK_SIZE, VIXDISKLIB_MIN_CHUNK_SIZE) ++ err = VixDiskLib_QueryAllocatedBlocks (h->handle, ++ 0, VIXDISKLIB_MIN_CHUNK_SIZE, ++ VIXDISKLIB_MIN_CHUNK_SIZE, ++ &block_list); ++ VDDK_CALL_END (VixDiskLib_QueryAllocatedBlocks, 0); ++ error_suppression = 0; ++ if (err == VIX_OK) { ++ VDDK_CALL_START (VixDiskLib_FreeBlockList, "block_list") ++ VixDiskLib_FreeBlockList (block_list); ++ VDDK_CALL_END (VixDiskLib_FreeBlockList, 0); ++ } ++ if (err != VIX_OK) { ++ char *errmsg = VixDiskLib_GetErrorText (err, NULL); ++ nbdkit_debug ("can_extents: " ++ "VixDiskLib_QueryAllocatedBlocks test failed, " ++ "extents support will be disabled: " ++ "original error: %s", ++ errmsg); ++ VixDiskLib_FreeErrorText (errmsg); ++ return 0; ++ } ++ ++ return 1; ++} ++ ++/* Add an extent to the list of extents. */ ++static int ++add_extent (struct nbdkit_extents *extents, ++ uint64_t *position, uint64_t next_position, bool is_hole) ++{ ++ uint32_t type = 0; ++ const uint64_t length = next_position - *position; ++ ++ if (is_hole) { ++ type = NBDKIT_EXTENT_HOLE; ++ /* Images opened as single link might be backed by another file in the ++ chain, so the holes are not guaranteed to be zeroes. */ ++ if (!single_link) ++ type |= NBDKIT_EXTENT_ZERO; ++ } ++ ++ assert (*position <= next_position); ++ if (*position == next_position) ++ return 0; ++ ++ if (vddk_debug_extents) ++ nbdkit_debug ("adding extent type %s at [%" PRIu64 "...%" PRIu64 "]", ++ is_hole ? "hole" : "allocated data", ++ *position, next_position-1); ++ if (nbdkit_add_extent (extents, *position, length, type) == -1) ++ return -1; ++ ++ *position = next_position; ++ return 0; ++} ++ ++static int ++do_extents (struct command *cmd, struct vddk_handle *h) ++{ ++ uint32_t count = cmd->count; ++ uint64_t offset = cmd->offset; ++ bool req_one = cmd->req_one; ++ struct nbdkit_extents *extents = cmd->ptr; ++ uint64_t position, end, start_sector; ++ ++ position = offset; ++ end = offset + count; ++ ++ /* We can only query whole chunks. Therefore start with the ++ * first chunk before offset. ++ */ ++ start_sector = ++ ROUND_DOWN (offset, VIXDISKLIB_MIN_CHUNK_SIZE * VIXDISKLIB_SECTOR_SIZE) ++ / VIXDISKLIB_SECTOR_SIZE; ++ while (start_sector * VIXDISKLIB_SECTOR_SIZE < end) { ++ VixError err; ++ uint32_t i; ++ uint64_t nr_chunks, nr_sectors; ++ VixDiskLibBlockList *block_list; ++ ++ assert (IS_ALIGNED (start_sector, VIXDISKLIB_MIN_CHUNK_SIZE)); ++ ++ nr_chunks = ++ ROUND_UP (end - start_sector * VIXDISKLIB_SECTOR_SIZE, ++ VIXDISKLIB_MIN_CHUNK_SIZE * VIXDISKLIB_SECTOR_SIZE) ++ / (VIXDISKLIB_MIN_CHUNK_SIZE * VIXDISKLIB_SECTOR_SIZE); ++ nr_chunks = MIN (nr_chunks, VIXDISKLIB_MAX_CHUNK_NUMBER); ++ nr_sectors = nr_chunks * VIXDISKLIB_MIN_CHUNK_SIZE; ++ ++ VDDK_CALL_START (VixDiskLib_QueryAllocatedBlocks, ++ "handle, %" PRIu64 " sectors, %" PRIu64 " sectors, " ++ "%d sectors", ++ start_sector, nr_sectors, VIXDISKLIB_MIN_CHUNK_SIZE) ++ err = VixDiskLib_QueryAllocatedBlocks (h->handle, ++ start_sector, nr_sectors, ++ VIXDISKLIB_MIN_CHUNK_SIZE, ++ &block_list); ++ VDDK_CALL_END (VixDiskLib_QueryAllocatedBlocks, 0); ++ if (err != VIX_OK) { ++ VDDK_ERROR (err, "VixDiskLib_QueryAllocatedBlocks"); ++ return -1; ++ } ++ ++ for (i = 0; i < block_list->numBlocks; ++i) { ++ uint64_t blk_offset, blk_length; ++ ++ blk_offset = block_list->blocks[i].offset * VIXDISKLIB_SECTOR_SIZE; ++ blk_length = block_list->blocks[i].length * VIXDISKLIB_SECTOR_SIZE; ++ ++ /* The query returns allocated blocks. We must insert holes ++ * between the blocks as necessary. ++ */ ++ if ((position < blk_offset && ++ add_extent (extents, &position, blk_offset, true) == -1) || ++ (add_extent (extents, ++ &position, blk_offset + blk_length, false) == -1)) { ++ VDDK_CALL_START (VixDiskLib_FreeBlockList, "block_list") ++ VixDiskLib_FreeBlockList (block_list); ++ VDDK_CALL_END (VixDiskLib_FreeBlockList, 0); ++ return -1; ++ } ++ } ++ VDDK_CALL_START (VixDiskLib_FreeBlockList, "block_list") ++ VixDiskLib_FreeBlockList (block_list); ++ VDDK_CALL_END (VixDiskLib_FreeBlockList, 0); ++ ++ /* There's an implicit hole after the returned list of blocks, ++ * up to the end of the QueryAllocatedBlocks request. ++ */ ++ if (add_extent (extents, ++ &position, ++ (start_sector + nr_sectors) * VIXDISKLIB_SECTOR_SIZE, ++ true) == -1) { ++ return -1; ++ } ++ ++ start_sector += nr_sectors; ++ ++ /* If one extent was requested, as long as we've added an extent ++ * overlapping the original offset we're done. ++ */ ++ if (req_one && position > offset) ++ break; ++ } ++ ++ return 0; ++} ++ ++/* Background worker thread, one per connection, which is where the ++ * VDDK commands are issued. ++ */ ++void * ++vddk_worker_thread (void *handle) ++{ ++ struct vddk_handle *h = handle; ++ bool stop = false; ++ ++ while (!stop) { ++ struct command *cmd; ++ int r; ++ bool async = false; ++ ++ /* Wait until we are sent at least one command. */ ++ { ++ ACQUIRE_LOCK_FOR_CURRENT_SCOPE (&h->commands_lock); ++ while (h->commands.size == 0) ++ pthread_cond_wait (&h->commands_cond, &h->commands_lock); ++ cmd = h->commands.ptr[0]; ++ command_queue_remove (&h->commands, 0); ++ } ++ ++ switch (cmd->type) { ++ case STOP: ++ r = do_stop (cmd, h); ++ stop = true; ++ break; ++ ++ case GET_SIZE: { ++ int64_t size = do_get_size (cmd, h); ++ if (size == -1) ++ r = -1; ++ else { ++ r = 0; ++ *(uint64_t *)cmd->ptr = size; ++ } ++ break; ++ } ++ ++ case READ: ++ r = do_read (cmd, h); ++ /* If async is true, don't retire this command now. */ ++ async = r == 0; ++ break; ++ ++ case WRITE: ++ r = do_write (cmd, h); ++ /* If async is true, don't retire this command now. */ ++ async = r == 0; ++ break; ++ ++ case FLUSH: ++ r = do_flush (cmd, h); ++ break; ++ ++ case CAN_EXTENTS: ++ r = do_can_extents (cmd, h); ++ if (r >= 0) ++ *(int *)cmd->ptr = r; ++ break; ++ ++ case EXTENTS: ++ r = do_extents (cmd, h); ++ break; ++ ++ default: abort (); /* impossible, but keeps GCC happy */ ++ } /* switch */ ++ ++ if (!async) { ++ /* Update the command status. */ ++ ACQUIRE_LOCK_FOR_CURRENT_SCOPE (&cmd->mutex); ++ cmd->status = r >= 0 ? SUCCEEDED : FAILED; ++ ++ /* For synchronous commands signal the caller thread that the ++ * command has completed. (Asynchronous commands are completed in ++ * the callback handler). ++ */ ++ pthread_cond_signal (&cmd->cond); ++ } ++ } /* while (!stop) */ ++ ++ /* Exit the worker thread. */ ++ return NULL; ++} +diff --git a/tests/dummy-vddk.c b/tests/dummy-vddk.c +index cb88380c..b6f12042 100644 +--- a/tests/dummy-vddk.c ++++ b/tests/dummy-vddk.c +@@ -188,6 +188,19 @@ VixDiskLib_Read (VixDiskLibHandle handle, + return VIX_OK; + } + ++NBDKIT_DLL_PUBLIC VixError ++VixDiskLib_ReadAsync (VixDiskLibHandle handle, ++ uint64_t start_sector, uint64_t nr_sectors, ++ unsigned char *buf, ++ VixDiskLibCompletionCB callback, void *data) ++{ ++ size_t offset = start_sector * VIXDISKLIB_SECTOR_SIZE; ++ ++ memcpy (buf, disk + offset, nr_sectors * VIXDISKLIB_SECTOR_SIZE); ++ callback (data, VIX_OK); ++ return VIX_ASYNC; ++} ++ + NBDKIT_DLL_PUBLIC VixError + VixDiskLib_Write (VixDiskLibHandle handle, + uint64_t start_sector, uint64_t nr_sectors, +@@ -199,6 +212,25 @@ VixDiskLib_Write (VixDiskLibHandle handle, + return VIX_OK; + } + ++NBDKIT_DLL_PUBLIC VixError ++VixDiskLib_WriteAsync (VixDiskLibHandle handle, ++ uint64_t start_sector, uint64_t nr_sectors, ++ const unsigned char *buf, ++ VixDiskLibCompletionCB callback, void *data) ++{ ++ size_t offset = start_sector * VIXDISKLIB_SECTOR_SIZE; ++ ++ memcpy (disk + offset, buf, nr_sectors * VIXDISKLIB_SECTOR_SIZE); ++ callback (data, VIX_OK); ++ return VIX_ASYNC; ++} ++ ++NBDKIT_DLL_PUBLIC VixError ++VixDiskLib_Flush (VixDiskLibHandle handle) ++{ ++ return VIX_OK; ++} ++ + NBDKIT_DLL_PUBLIC VixError + VixDiskLib_Wait (VixDiskLibHandle handle) + { +-- +2.31.1 + diff --git a/SOURCES/0015-data-Improve-the-example-with-a-diagram.patch b/SOURCES/0015-data-Improve-the-example-with-a-diagram.patch deleted file mode 100644 index 4ff51f6..0000000 --- a/SOURCES/0015-data-Improve-the-example-with-a-diagram.patch +++ /dev/null @@ -1,53 +0,0 @@ -From 0eae7ebf6f714fb339f4a476b65e070b528824ec Mon Sep 17 00:00:00 2001 -From: "Richard W.M. Jones" -Date: Sun, 8 Aug 2021 16:32:38 +0100 -Subject: [PATCH] data: Improve the example with a diagram - -And other improvements to readability. - -(cherry picked from commit 4e3a9bda2b7a3d141234e26250c69baa6ed5194d) ---- - plugins/data/nbdkit-data-plugin.pod | 24 +++++++++++++++--------- - 1 file changed, 15 insertions(+), 9 deletions(-) - -diff --git a/plugins/data/nbdkit-data-plugin.pod b/plugins/data/nbdkit-data-plugin.pod -index 32a450ab..b69435e9 100644 ---- a/plugins/data/nbdkit-data-plugin.pod -+++ b/plugins/data/nbdkit-data-plugin.pod -@@ -172,18 +172,24 @@ compact format. It is a string containing a list of bytes which are - written into the disk image sequentially. You can move the virtual - offset where bytes are written using C<@offset>. - --For example: -- - nbdkit data '0 1 2 3 @0x1fe 0x55 0xaa' - --creates a 0x200 = 512 byte (1 sector) image containing the four bytes --C<0 1 2 3> at the start, and the two bytes C<0x55 0xaa> at the end of --the sector, with the remaining 506 bytes in the middle being all --zeroes. In this example the size (512 bytes) is implied by the data. --But you could additionally use the C parameter to either --truncate or extend (with zeroes) the disk image. -+creates: - --Whitespace between fields in the string is ignored. -+ total size 0x200 = 512 bytes (1 sector) -+┌──────┬──────┬──────┬──────┬───────── ── ── ───┬──────┬──────┐ -+│ 0 │ 1 │ 2 │ 3 │ 0 0 ... 0 │ 0x55 │ 0xaa │ -+└──────┴──────┴──────┴──────┴───────── ── ── ───┴──────┴──────┘ -+ ↑ -+ offset 0x1fe -+ -+In this example the size is implied by the data. But you could also -+use the C parameter to either truncate or extend (with zeroes) -+the disk image. Another way to write the same disk would be this, -+where we align the offset to the end of the sector and move back 2 -+bytes to write the signature: -+ -+ nbdkit data '0 1 2 3 @^0x200 @-2 le16:0xaa55' - - Fields in the string can be: - --- -2.31.1 - diff --git a/SOURCES/0015-vddk-Assume-that-VixDiskLib_Flush-is-available.patch b/SOURCES/0015-vddk-Assume-that-VixDiskLib_Flush-is-available.patch new file mode 100644 index 0000000..5735b1d --- /dev/null +++ b/SOURCES/0015-vddk-Assume-that-VixDiskLib_Flush-is-available.patch @@ -0,0 +1,57 @@ +From eda9dd7f5e610fd4e17019813c5a045f0b3603df Mon Sep 17 00:00:00 2001 +From: "Richard W.M. Jones" +Date: Fri, 29 Oct 2021 20:56:55 +0100 +Subject: [PATCH] vddk: Assume that VixDiskLib_Flush is available + +Since we now require and check that VDDK >= 6.5, we can assume that +VixDiskLib_Flush is always available. + +(cherry picked from commit e3685e6f0d0b71ab24b96fe85430a3b75da58736) +--- + plugins/vddk/vddk.c | 15 +++++++++++---- + 1 file changed, 11 insertions(+), 4 deletions(-) + +diff --git a/plugins/vddk/vddk.c b/plugins/vddk/vddk.c +index 9f223db0..f967e2d9 100644 +--- a/plugins/vddk/vddk.c ++++ b/plugins/vddk/vddk.c +@@ -378,6 +378,12 @@ load_library (bool load_error_is_fatal) + "See nbdkit-vddk-plugin(1) man page section \"SUPPORTED VERSIONS OF VDDK\"."); + exit (EXIT_FAILURE); + } ++ ++ /* Added in VDDK 6.0 so it must always be present. Since we are ++ * going to call this function unconditionally, fail early and hard ++ * if for some reason it's not present. ++ */ ++ assert (VixDiskLib_Flush != NULL); + } + + static int +@@ -725,18 +731,19 @@ vddk_get_size (void *handle) + return (int64_t) size; + } + ++/* The Flush call was added in VDDK 6.0, since we support minimum 6.5 ++ * we are always able to do FUA / flush. ++ */ + static int + vddk_can_fua (void *handle) + { +- /* The Flush call was not available in VDDK < 6.0. */ +- return VixDiskLib_Flush != NULL ? NBDKIT_FUA_NATIVE : NBDKIT_FUA_NONE; ++ return NBDKIT_FUA_NATIVE; + } + + static int + vddk_can_flush (void *handle) + { +- /* The Flush call was not available in VDDK < 6.0. */ +- return VixDiskLib_Flush != NULL; ++ return 1; + } + + /* Read data from the file. +-- +2.31.1 + diff --git a/SOURCES/0016-cow-Add-some-more-debugging-especially-for-blk_read_.patch b/SOURCES/0016-cow-Add-some-more-debugging-especially-for-blk_read_.patch deleted file mode 100644 index e9c1228..0000000 --- a/SOURCES/0016-cow-Add-some-more-debugging-especially-for-blk_read_.patch +++ /dev/null @@ -1,45 +0,0 @@ -From a22248e3075e782d28542f8f6acd046c9dfa8998 Mon Sep 17 00:00:00 2001 -From: "Richard W.M. Jones" -Date: Mon, 9 Aug 2021 14:09:31 +0100 -Subject: [PATCH] cow: Add some more debugging especially for blk_read_multiple - and cow-on-read - -Only activated when we use -D cow.verbose=1 - -(cherry picked from commit 2da1ae0ca966af955d8fcf3feffffc80d07142fd) ---- - filters/cow/blk.c | 11 +++++++++-- - 1 file changed, 9 insertions(+), 2 deletions(-) - -diff --git a/filters/cow/blk.c b/filters/cow/blk.c -index 4f84e092..c22d5886 100644 ---- a/filters/cow/blk.c -+++ b/filters/cow/blk.c -@@ -254,8 +254,10 @@ blk_read_multiple (nbdkit_next *next, - - if (cow_debug_verbose) - nbdkit_debug ("cow: blk_read_multiple block %" PRIu64 -- " (offset %" PRIu64 ") is %s", -- blknum, (uint64_t) offset, state_to_string (state)); -+ " (offset %" PRIu64 ") run of length %" PRIu64 -+ " is %s", -+ blknum, (uint64_t) offset, runblocks, -+ state_to_string (state)); - - if (state == BLOCK_NOT_ALLOCATED) { /* Read underlying plugin. */ - unsigned n, tail = 0; -@@ -281,6 +283,11 @@ blk_read_multiple (nbdkit_next *next, - * set them as allocated. - */ - if (cow_on_read) { -+ if (cow_debug_verbose) -+ nbdkit_debug ("cow: cow-on-read saving %" PRIu64 " blocks " -+ "at offset %" PRIu64 " into the cache", -+ runblocks, offset); -+ - if (full_pwrite (fd, block, BLKSIZE * runblocks, offset) == -1) { - *err = errno; - nbdkit_error ("pwrite: %m"); --- -2.31.1 - diff --git a/SOURCES/0016-vddk-Simplify-detection-of-VDDK-symbols-and-baseline.patch b/SOURCES/0016-vddk-Simplify-detection-of-VDDK-symbols-and-baseline.patch new file mode 100644 index 0000000..85b8528 --- /dev/null +++ b/SOURCES/0016-vddk-Simplify-detection-of-VDDK-symbols-and-baseline.patch @@ -0,0 +1,186 @@ +From 1b2b386c9a254808a25fbfce3640c96bdb8cf9be Mon Sep 17 00:00:00 2001 +From: "Richard W.M. Jones" +Date: Fri, 29 Oct 2021 21:02:54 +0100 +Subject: [PATCH] vddk: Simplify detection of VDDK symbols and baseline 6.5 + +Make all symbols from VDDK 6.5 into required symbols and use a single +error message function if one of these is missing. The new error is: + + nbdkit: error: required VDDK symbol "VixDiskLib_Wait" is + missing. VDDK version must be >= 6.5. See nbdkit-vddk-plugin(1) man + page section "SUPPORTED VERSIONS OF VDDK". Original dlopen error: + vmware-vix-disklib-distrib/lib64/libvixDiskLib.so.6: undefined + symbol: VixDiskLib_Wait + +Remove the extra check and assert. + +Be more consistent about #define OPTIONAL_STUB(fn,ret,args) STUB(fn,ret,args) +when we want the optional and required stubs to do the same thing. + +(cherry picked from commit ec0d22e61881efa39a69d02ccb9e4ede8bf95e75) +--- + plugins/vddk/stats.c | 2 +- + plugins/vddk/vddk-stubs.h | 45 ++++++++++++++++++--------------------- + plugins/vddk/vddk.c | 36 ++++++++++++------------------- + plugins/vddk/vddk.h | 2 +- + 4 files changed, 37 insertions(+), 48 deletions(-) + +diff --git a/plugins/vddk/stats.c b/plugins/vddk/stats.c +index 18a42714..76e0c244 100644 +--- a/plugins/vddk/stats.c ++++ b/plugins/vddk/stats.c +@@ -89,7 +89,7 @@ display_stats (void) + if (!vddk_debug_stats) return; + + #define STUB(fn,ret,args) statlist_append (&stats, stats_##fn) +-#define OPTIONAL_STUB(fn,ret,args) statlist_append (&stats, stats_##fn) ++#define OPTIONAL_STUB(fn,ret,args) STUB(fn,ret,args) + #include "vddk-stubs.h" + #undef STUB + #undef OPTIONAL_STUB +diff --git a/plugins/vddk/vddk-stubs.h b/plugins/vddk/vddk-stubs.h +index 66353691..7d8644c3 100644 +--- a/plugins/vddk/vddk-stubs.h ++++ b/plugins/vddk/vddk-stubs.h +@@ -39,10 +39,7 @@ + * function name, return value, arguments. + */ + +-/* Required stubs, present in all versions of VDDK that we support. I +- * have checked that all these exist in at least VDDK 5.5.5 (2015). +- */ +- ++/* Required stubs, present in all versions of VDDK since 6.5 (Nov 2016). */ + STUB (VixDiskLib_InitEx, + VixError, + (uint32_t major, uint32_t minor, +@@ -103,27 +100,27 @@ STUB (VixDiskLib_Write, + uint64_t start_sector, uint64_t nr_sectors, + const unsigned char *buf)); + +-/* Added in VDDK 6.0, these will be NULL in earlier versions. */ +-OPTIONAL_STUB (VixDiskLib_Flush, +- VixError, +- (VixDiskLibHandle handle)); +-OPTIONAL_STUB (VixDiskLib_ReadAsync, +- VixError, +- (VixDiskLibHandle handle, +- uint64_t start_sector, uint64_t nr_sectors, +- unsigned char *buf, +- VixDiskLibCompletionCB callback, void *data)); +-OPTIONAL_STUB (VixDiskLib_WriteAsync, +- VixError, +- (VixDiskLibHandle handle, +- uint64_t start_sector, uint64_t nr_sectors, +- const unsigned char *buf, +- VixDiskLibCompletionCB callback, void *data)); ++/* Added in VDDK 6.0. */ ++STUB (VixDiskLib_Flush, ++ VixError, ++ (VixDiskLibHandle handle)); ++STUB (VixDiskLib_ReadAsync, ++ VixError, ++ (VixDiskLibHandle handle, ++ uint64_t start_sector, uint64_t nr_sectors, ++ unsigned char *buf, ++ VixDiskLibCompletionCB callback, void *data)); ++STUB (VixDiskLib_WriteAsync, ++ VixError, ++ (VixDiskLibHandle handle, ++ uint64_t start_sector, uint64_t nr_sectors, ++ const unsigned char *buf, ++ VixDiskLibCompletionCB callback, void *data)); + +-/* Added in VDDK 6.5, this will be NULL in earlier versions. */ +-OPTIONAL_STUB (VixDiskLib_Wait, +- VixError, +- (VixDiskLibHandle handle)); ++/* Added in VDDK 6.5. */ ++STUB (VixDiskLib_Wait, ++ VixError, ++ (VixDiskLibHandle handle)); + + /* Added in VDDK 6.7, these will be NULL for earlier versions: */ + OPTIONAL_STUB (VixDiskLib_QueryAllocatedBlocks, +diff --git a/plugins/vddk/vddk.c b/plugins/vddk/vddk.c +index f967e2d9..271b5ee0 100644 +--- a/plugins/vddk/vddk.c ++++ b/plugins/vddk/vddk.c +@@ -63,7 +63,7 @@ NBDKIT_DLL_PUBLIC int vddk_debug_datapath = 1; + * initialized when the plugin is loaded (by vddk_get_ready). + */ + #define STUB(fn,ret,args) ret (*fn) args +-#define OPTIONAL_STUB(fn,ret,args) ret (*fn) args ++#define OPTIONAL_STUB(fn,ret,args) STUB(fn,ret,args) + #include "vddk-stubs.h" + #undef STUB + #undef OPTIONAL_STUB +@@ -282,6 +282,17 @@ vddk_config (const char *key, const char *value) + return 0; + } + ++static void ++missing_required_symbol (const char *fn) ++{ ++ nbdkit_error ("required VDDK symbol \"%s\" is missing. " ++ "VDDK version must be >= 6.5. " ++ "See nbdkit-vddk-plugin(1) man page section \"SUPPORTED VERSIONS OF VDDK\". " ++ "Original dlopen error: %s\n", ++ fn, dlerror ()); ++ exit (EXIT_FAILURE); ++} ++ + /* Load the VDDK library. */ + static void + load_library (bool load_error_is_fatal) +@@ -358,32 +369,13 @@ load_library (bool load_error_is_fatal) + #define STUB(fn,ret,args) \ + do { \ + fn = dlsym (dl, #fn); \ +- if (fn == NULL) { \ +- nbdkit_error ("required VDDK symbol \"%s\" is missing: %s", \ +- #fn, dlerror ()); \ +- exit (EXIT_FAILURE); \ +- } \ ++ if (fn == NULL) \ ++ missing_required_symbol (#fn); \ + } while (0) + #define OPTIONAL_STUB(fn,ret,args) fn = dlsym (dl, #fn) + #include "vddk-stubs.h" + #undef STUB + #undef OPTIONAL_STUB +- +- /* Additionally, VDDK version must be >= 6.5. This was the first +- * version which introduced VixDiskLib_Wait symbol so we can check +- * for that. +- */ +- if (VixDiskLib_Wait == NULL) { +- nbdkit_error ("VDDK version must be >= 6.5. " +- "See nbdkit-vddk-plugin(1) man page section \"SUPPORTED VERSIONS OF VDDK\"."); +- exit (EXIT_FAILURE); +- } +- +- /* Added in VDDK 6.0 so it must always be present. Since we are +- * going to call this function unconditionally, fail early and hard +- * if for some reason it's not present. +- */ +- assert (VixDiskLib_Flush != NULL); + } + + static int +diff --git a/plugins/vddk/vddk.h b/plugins/vddk/vddk.h +index be0b3492..0e3dd79e 100644 +--- a/plugins/vddk/vddk.h ++++ b/plugins/vddk/vddk.h +@@ -76,7 +76,7 @@ extern int vddk_debug_datapath; + extern int vddk_debug_stats; + + #define STUB(fn,ret,args) extern ret (*fn) args +-#define OPTIONAL_STUB(fn,ret,args) extern ret (*fn) args ++#define OPTIONAL_STUB(fn,ret,args) STUB(fn,ret,args) + #include "vddk-stubs.h" + #undef STUB + #undef OPTIONAL_STUB +-- +2.31.1 + diff --git a/SOURCES/0017-delay-Fix-delay-close.patch b/SOURCES/0017-delay-Fix-delay-close.patch deleted file mode 100644 index 3d0f3e7..0000000 --- a/SOURCES/0017-delay-Fix-delay-close.patch +++ /dev/null @@ -1,142 +0,0 @@ -From be7252bada79ee542356dffaf5f3c568a5c7fec3 Mon Sep 17 00:00:00 2001 -From: "Richard W.M. Jones" -Date: Tue, 10 Aug 2021 08:39:15 +0100 -Subject: [PATCH] delay: Fix delay-close - -See comments in the code for how this has been fixed. - -This only delays clients which use NBD_CMD_DISC (libnbd -nbd_shutdown(3)). Clients which drop the connection obviously cannot -be delayed. For example: - -$ nbdkit --filter=delay null delay-close=3 \ - --run 'time nbdsh -u $uri -c "h.shutdown()" - time nbdsh -u $uri -c "pass"' - -real 0m3.061s # Client used shutdown, was delayed -user 0m0.028s -sys 0m0.030s - -real 0m0.058s # Client disconnected, was not delayed -user 0m0.029s -sys 0m0.027s - -Reported-by: Ming Xie -Fixes: commit de8dcd3a34a38b088a0f9a6f8ca754702ad1f598 -Fixes: https://bugzilla.redhat.com/show_bug.cgi?id=1991652 -(cherry picked from commit 0cafebdb67d0d557ba1be8ea306b8acc5d9b2203) ---- - filters/delay/delay.c | 42 +++++++++++++++++++-------- - filters/delay/nbdkit-delay-filter.pod | 14 +++++++-- - 2 files changed, 41 insertions(+), 15 deletions(-) - -diff --git a/filters/delay/delay.c b/filters/delay/delay.c -index df3729a7..9252b855 100644 ---- a/filters/delay/delay.c -+++ b/filters/delay/delay.c -@@ -39,6 +39,7 @@ - #include - #include - #include -+#include - - #include - -@@ -134,12 +135,6 @@ open_delay (int *err) - return delay (delay_open_ms, err); - } - --static int --close_delay (int *err) --{ -- return delay (delay_close_ms, err); --} -- - /* Called for each key=value passed on the command line. */ - static int - delay_config (nbdkit_next_config *next, nbdkit_backend *nxdata, -@@ -253,13 +248,36 @@ delay_open (nbdkit_next_open *next, nbdkit_context *nxdata, - return NBDKIT_HANDLE_NOT_NEEDED; - } - --/* Close connection. */ --static void --delay_close (void *handle) -+/* Close connection. -+ * -+ * We cannot call nbdkit_nanosleep here because the socket may have -+ * been closed and that function will abort and return immediately. -+ * However we want to force a sleep (even if the server is shutting -+ * down) so use regular nanosleep instead. -+ * -+ * We cannot use the .close callback because that happens after the -+ * socket has closed, thus not delaying the client. By using -+ * .finalize we can delay well-behaved clients (those that use -+ * NBD_CMD_DISC). We cannot delay clients that drop the connection. -+ */ -+static int -+delay_finalize (nbdkit_next *next, void *handle) - { -- int err; -+ const unsigned ms = delay_close_ms; - -- close_delay (&err); -+ if (ms > 0) { -+ struct timespec ts; -+ -+ ts.tv_sec = ms / 1000; -+ ts.tv_nsec = (ms % 1000) * 1000000; -+ /* If nanosleep fails we don't really want to interrupt the chain -+ * of finalize calls through the other filters, so ignore any -+ * error here. -+ */ -+ nanosleep (&ts, NULL); -+ } -+ -+ return next->finalize (next); - } - - /* Read data. */ -@@ -340,7 +358,7 @@ static struct nbdkit_filter filter = { - .config_help = delay_config_help, - .can_fast_zero = delay_can_fast_zero, - .open = delay_open, -- .close = delay_close, -+ .finalize = delay_finalize, - .pread = delay_pread, - .pwrite = delay_pwrite, - .zero = delay_zero, -diff --git a/filters/delay/nbdkit-delay-filter.pod b/filters/delay/nbdkit-delay-filter.pod -index 11ae544b..76614736 100644 ---- a/filters/delay/nbdkit-delay-filter.pod -+++ b/filters/delay/nbdkit-delay-filter.pod -@@ -117,15 +117,23 @@ the plugin. - - =item BNNB - -+(nbdkit E 1.28) -+ -+Delay open (client connection) by C seconds or C -+milliseconds. -+ - =item BSECS - - =item BNNB - - (nbdkit E 1.28) - --Delay open and close operations by C seconds or C --milliseconds. Open corresponds to client connection. Close may not --be visible to clients if they abruptly disconnect. -+Delay close (client disconnection) by C seconds or C -+milliseconds. This can also cause server shutdown to be delayed if -+clients are connected at the time. This only affects clients that -+gracefully disconnect (using C / libnbd function -+L). Clients that abruptly disconnect from the server -+cannot be delayed. - - =back - --- -2.31.1 - diff --git a/SOURCES/0017-vddk-Remove-some-whitespace-from-a-couple-of-functio.patch b/SOURCES/0017-vddk-Remove-some-whitespace-from-a-couple-of-functio.patch new file mode 100644 index 0000000..d62f431 --- /dev/null +++ b/SOURCES/0017-vddk-Remove-some-whitespace-from-a-couple-of-functio.patch @@ -0,0 +1,40 @@ +From 2363e76ab34a2e11b57970d82161f73453a4a8ec Mon Sep 17 00:00:00 2001 +From: "Richard W.M. Jones" +Date: Sat, 30 Oct 2021 08:34:28 +0100 +Subject: [PATCH] vddk: Remove some whitespace from a couple of functions + +(cherry picked from commit 974dce2c2ef84fc096ee319f340054234a29df91) +--- + plugins/vddk/vddk.c | 9 ++------- + 1 file changed, 2 insertions(+), 7 deletions(-) + +diff --git a/plugins/vddk/vddk.c b/plugins/vddk/vddk.c +index 271b5ee0..184f1a9c 100644 +--- a/plugins/vddk/vddk.c ++++ b/plugins/vddk/vddk.c +@@ -792,9 +792,7 @@ static int + vddk_flush (void *handle, uint32_t flags) + { + struct vddk_handle *h = handle; +- struct command flush_cmd = { +- .type = FLUSH, +- }; ++ struct command flush_cmd = { .type = FLUSH }; + + return send_command_and_wait (h, &flush_cmd); + } +@@ -804,10 +802,7 @@ vddk_can_extents (void *handle) + { + struct vddk_handle *h = handle; + int ret; +- struct command can_extents_cmd = { +- .type = CAN_EXTENTS, +- .ptr = &ret, +- }; ++ struct command can_extents_cmd = { .type = CAN_EXTENTS, .ptr = &ret }; + + if (send_command_and_wait (h, &can_extents_cmd) == -1) + return -1; +-- +2.31.1 + diff --git a/SOURCES/0018-delay-Test-delay-open-and-delay-close.patch b/SOURCES/0018-delay-Test-delay-open-and-delay-close.patch deleted file mode 100644 index 2acd71d..0000000 --- a/SOURCES/0018-delay-Test-delay-open-and-delay-close.patch +++ /dev/null @@ -1,155 +0,0 @@ -From 838ec052abe63056434c08ea80f4609e697dad0f Mon Sep 17 00:00:00 2001 -From: "Richard W.M. Jones" -Date: Tue, 10 Aug 2021 09:11:43 +0100 -Subject: [PATCH] delay: Test delay-open and delay-close - -(cherry picked from commit 3caabaf87ec744b863b50b5bf77a9c1b93a7c3e0) ---- - tests/Makefile.am | 12 +++++++-- - tests/test-delay-close.sh | 54 +++++++++++++++++++++++++++++++++++++++ - tests/test-delay-open.sh | 49 +++++++++++++++++++++++++++++++++++ - 3 files changed, 113 insertions(+), 2 deletions(-) - create mode 100755 tests/test-delay-close.sh - create mode 100755 tests/test-delay-open.sh - -diff --git a/tests/Makefile.am b/tests/Makefile.am -index edc8d66d..e61c5829 100644 ---- a/tests/Makefile.am -+++ b/tests/Makefile.am -@@ -1425,8 +1425,16 @@ EXTRA_DIST += \ - $(NULL) - - # delay filter tests. --TESTS += test-delay-shutdown.sh --EXTRA_DIST += test-delay-shutdown.sh -+TESTS += \ -+ test-delay-close.sh \ -+ test-delay-open.sh \ -+ test-delay-shutdown.sh \ -+ $(NULL) -+EXTRA_DIST += \ -+ test-delay-close.sh \ -+ test-delay-open.sh \ -+ test-delay-shutdown.sh \ -+ $(NULL) - LIBNBD_TESTS += test-delay - - test_delay_SOURCES = test-delay.c -diff --git a/tests/test-delay-close.sh b/tests/test-delay-close.sh -new file mode 100755 -index 00000000..1de305f5 ---- /dev/null -+++ b/tests/test-delay-close.sh -@@ -0,0 +1,54 @@ -+#!/usr/bin/env bash -+# nbdkit -+# Copyright (C) 2018-2021 Red Hat Inc. -+# -+# Redistribution and use in source and binary forms, with or without -+# modification, are permitted provided that the following conditions are -+# met: -+# -+# * Redistributions of source code must retain the above copyright -+# notice, this list of conditions and the following disclaimer. -+# -+# * Redistributions in binary form must reproduce the above copyright -+# notice, this list of conditions and the following disclaimer in the -+# documentation and/or other materials provided with the distribution. -+# -+# * Neither the name of Red Hat nor the names of its contributors may be -+# used to endorse or promote products derived from this software without -+# specific prior written permission. -+# -+# THIS SOFTWARE IS PROVIDED BY RED HAT AND CONTRIBUTORS ''AS IS'' AND -+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, -+# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A -+# PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL RED HAT OR -+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF -+# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND -+# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -+# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT -+# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF -+# SUCH DAMAGE. -+ -+source ./functions.sh -+set -e -+set -x -+ -+requires_run -+requires_plugin null -+requires_filter delay -+requires nbdsh --version -+ -+# Test delay-close with a well-behaved client. -+ -+nbdkit -U - null --filter=delay delay-close=3 \ -+ --run ' -+start_t=$SECONDS -+nbdsh -u "$uri" -c "h.shutdown()" -+end_t=$SECONDS -+ -+if [ $((end_t - start_t)) -lt 3 ]; then -+ echo "$0: delay filter failed: delay-close=3 caused delay < 3 seconds" -+ exit 1 -+fi -+' -diff --git a/tests/test-delay-open.sh b/tests/test-delay-open.sh -new file mode 100755 -index 00000000..2a74e44c ---- /dev/null -+++ b/tests/test-delay-open.sh -@@ -0,0 +1,49 @@ -+#!/usr/bin/env bash -+# nbdkit -+# Copyright (C) 2018-2021 Red Hat Inc. -+# -+# Redistribution and use in source and binary forms, with or without -+# modification, are permitted provided that the following conditions are -+# met: -+# -+# * Redistributions of source code must retain the above copyright -+# notice, this list of conditions and the following disclaimer. -+# -+# * Redistributions in binary form must reproduce the above copyright -+# notice, this list of conditions and the following disclaimer in the -+# documentation and/or other materials provided with the distribution. -+# -+# * Neither the name of Red Hat nor the names of its contributors may be -+# used to endorse or promote products derived from this software without -+# specific prior written permission. -+# -+# THIS SOFTWARE IS PROVIDED BY RED HAT AND CONTRIBUTORS ''AS IS'' AND -+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, -+# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A -+# PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL RED HAT OR -+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF -+# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND -+# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -+# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT -+# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF -+# SUCH DAMAGE. -+ -+source ./functions.sh -+set -e -+set -x -+ -+requires_run -+requires_plugin null -+requires_filter delay -+requires nbdinfo --version -+ -+start_t=$SECONDS -+nbdkit -U - null --filter=delay delay-open=3 --run 'nbdinfo "$uri"' -+end_t=$SECONDS -+ -+if [ $((end_t - start_t)) -lt 3 ]; then -+ echo "$0: delay filter failed: delay-open=3 caused delay < 3 seconds" -+ exit 1 -+fi --- -2.31.1 - diff --git a/SOURCES/0018-vddk-Move-config-debug-error-and-utility-functions-a.patch b/SOURCES/0018-vddk-Move-config-debug-error-and-utility-functions-a.patch new file mode 100644 index 0000000..fbfa113 --- /dev/null +++ b/SOURCES/0018-vddk-Move-config-debug-error-and-utility-functions-a.patch @@ -0,0 +1,338 @@ +From 6c0034cf8802d466b170135fec0d6a97d1eb2f2a Mon Sep 17 00:00:00 2001 +From: "Richard W.M. Jones" +Date: Sat, 30 Oct 2021 08:27:39 +0100 +Subject: [PATCH] vddk: Move config, debug/error and utility functions around + +Move the functions so they are nearer to where they are used. +Introduce a utils.c file for utility functions. + +This is just code rearrangement with no other effects. + +(cherry picked from commit c59be086210a06688b9195e0b91f8603a668654a) +--- + plugins/vddk/Makefile.am | 1 + + plugins/vddk/utils.c | 51 ++++++++++ + plugins/vddk/vddk.c | 201 +++++++++++++++++++-------------------- + plugins/vddk/vddk.h | 3 + + 4 files changed, 151 insertions(+), 105 deletions(-) + create mode 100644 plugins/vddk/utils.c + +diff --git a/plugins/vddk/Makefile.am b/plugins/vddk/Makefile.am +index f8382fc9..02113da0 100644 +--- a/plugins/vddk/Makefile.am ++++ b/plugins/vddk/Makefile.am +@@ -47,6 +47,7 @@ nbdkit_vddk_plugin_la_SOURCES = \ + vddk.h \ + reexec.c \ + stats.c \ ++ utils.c \ + vddk-structs.h \ + vddk-stubs.h \ + worker.c \ +diff --git a/plugins/vddk/utils.c b/plugins/vddk/utils.c +new file mode 100644 +index 00000000..f0c19950 +--- /dev/null ++++ b/plugins/vddk/utils.c +@@ -0,0 +1,51 @@ ++/* nbdkit ++ * Copyright (C) 2013-2021 Red Hat Inc. ++ * ++ * Redistribution and use in source and binary forms, with or without ++ * modification, are permitted provided that the following conditions are ++ * met: ++ * ++ * * Redistributions of source code must retain the above copyright ++ * notice, this list of conditions and the following disclaimer. ++ * ++ * * Redistributions in binary form must reproduce the above copyright ++ * notice, this list of conditions and the following disclaimer in the ++ * documentation and/or other materials provided with the distribution. ++ * ++ * * Neither the name of Red Hat nor the names of its contributors may be ++ * used to endorse or promote products derived from this software without ++ * specific prior written permission. ++ * ++ * THIS SOFTWARE IS PROVIDED BY RED HAT AND CONTRIBUTORS ''AS IS'' AND ++ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, ++ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A ++ * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL RED HAT OR ++ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, ++ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT ++ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF ++ * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ++ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, ++ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT ++ * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF ++ * SUCH DAMAGE. ++ */ ++ ++#include ++ ++#include ++#include ++#include ++ ++#define NBDKIT_API_VERSION 2 ++#include ++ ++#include "vddk.h" ++ ++void ++trim (char *str) ++{ ++ size_t len = strlen (str); ++ ++ if (len > 0 && str[len-1] == '\n') ++ str[len-1] = '\0'; ++} +diff --git a/plugins/vddk/vddk.c b/plugins/vddk/vddk.c +index 184f1a9c..31e5e23b 100644 +--- a/plugins/vddk/vddk.c ++++ b/plugins/vddk/vddk.c +@@ -114,61 +114,6 @@ vddk_unload (void) + free (password); + } + +-static void +-trim (char *str) +-{ +- size_t len = strlen (str); +- +- if (len > 0 && str[len-1] == '\n') +- str[len-1] = '\0'; +-} +- +-/* Turn log messages from the library into nbdkit_debug. */ +-static void +-debug_function (const char *fs, va_list args) +-{ +- CLEANUP_FREE char *str = NULL; +- +- if (vasprintf (&str, fs, args) == -1) { +- nbdkit_debug ("lost debug message: %s", fs); +- return; +- } +- +- trim (str); +- +- nbdkit_debug ("%s", str); +-} +- +-/* Turn error messages from the library into nbdkit_error. */ +-static void +-error_function (const char *fs, va_list args) +-{ +- CLEANUP_FREE char *str = NULL; +- +- /* If the thread-local error_suppression flag is non-zero then we +- * will suppress error messages from VDDK in this thread. +- */ +- if (error_suppression) return; +- +- if (vasprintf (&str, fs, args) == -1) { +- nbdkit_error ("lost error message: %s", fs); +- return; +- } +- +- trim (str); +- +- /* VDDK 7 added a useless error message about their "phone home" +- * system called CEIP which only panics users. Demote it to a debug +- * statement. https://bugzilla.redhat.com/show_bug.cgi?id=1834267 +- */ +- if (strstr (str, "Get CEIP status failed") != NULL) { +- nbdkit_debug ("%s", str); +- return; +- } +- +- nbdkit_error ("%s", str); +-} +- + /* Configuration. */ + static int + vddk_config (const char *key, const char *value) +@@ -282,6 +227,56 @@ vddk_config (const char *key, const char *value) + return 0; + } + ++static int ++vddk_config_complete (void) ++{ ++ if (filename == NULL) { ++ nbdkit_error ("you must supply the file= parameter " ++ "after the plugin name on the command line"); ++ return -1; ++ } ++ ++ /* For remote connections, check all the parameters have been ++ * passed. Note that VDDK will segfault if parameters that it ++ * expects are NULL (and there's no real way to tell what parameters ++ * it is expecting). This implements the same test that the VDDK ++ * sample program does. ++ */ ++ is_remote = ++ vmx_spec || ++ server_name || ++ username || ++ password || ++ cookie || ++ thumb_print || ++ port || ++ nfc_host_port; ++ ++ if (is_remote) { ++#define missing(test, param) \ ++ if (test) { \ ++ nbdkit_error ("remote connection requested, missing parameter: %s", \ ++ param); \ ++ return -1; \ ++ } ++ missing (!server_name, "server"); ++ missing (!username, "user"); ++ missing (!password, "password"); ++ missing (!vmx_spec, "vm"); ++#undef missing ++ } ++ ++ /* Restore original LD_LIBRARY_PATH after reexec. */ ++ if (restore_ld_library_path () == -1) ++ return -1; ++ ++ return 0; ++} ++ ++#define vddk_config_help \ ++ "[file=] (required) The filename (eg. VMDK file) to serve.\n" \ ++ "Many optional parameters are supported, see nbdkit-vddk-plugin(1)." ++ + static void + missing_required_symbol (const char *fn) + { +@@ -378,56 +373,6 @@ load_library (bool load_error_is_fatal) + #undef OPTIONAL_STUB + } + +-static int +-vddk_config_complete (void) +-{ +- if (filename == NULL) { +- nbdkit_error ("you must supply the file= parameter " +- "after the plugin name on the command line"); +- return -1; +- } +- +- /* For remote connections, check all the parameters have been +- * passed. Note that VDDK will segfault if parameters that it +- * expects are NULL (and there's no real way to tell what parameters +- * it is expecting). This implements the same test that the VDDK +- * sample program does. +- */ +- is_remote = +- vmx_spec || +- server_name || +- username || +- password || +- cookie || +- thumb_print || +- port || +- nfc_host_port; +- +- if (is_remote) { +-#define missing(test, param) \ +- if (test) { \ +- nbdkit_error ("remote connection requested, missing parameter: %s", \ +- param); \ +- return -1; \ +- } +- missing (!server_name, "server"); +- missing (!username, "user"); +- missing (!password, "password"); +- missing (!vmx_spec, "vm"); +-#undef missing +- } +- +- /* Restore original LD_LIBRARY_PATH after reexec. */ +- if (restore_ld_library_path () == -1) +- return -1; +- +- return 0; +-} +- +-#define vddk_config_help \ +- "[file=] (required) The filename (eg. VMDK file) to serve.\n" \ +- "Many optional parameters are supported, see nbdkit-vddk-plugin(1)." +- + static int + vddk_get_ready (void) + { +@@ -435,6 +380,52 @@ vddk_get_ready (void) + return 0; + } + ++/* Turn log messages from the library into nbdkit_debug. */ ++static void ++debug_function (const char *fs, va_list args) ++{ ++ CLEANUP_FREE char *str = NULL; ++ ++ if (vasprintf (&str, fs, args) == -1) { ++ nbdkit_debug ("lost debug message: %s", fs); ++ return; ++ } ++ ++ trim (str); ++ ++ nbdkit_debug ("%s", str); ++} ++ ++/* Turn error messages from the library into nbdkit_error. */ ++static void ++error_function (const char *fs, va_list args) ++{ ++ CLEANUP_FREE char *str = NULL; ++ ++ /* If the thread-local error_suppression flag is non-zero then we ++ * will suppress error messages from VDDK in this thread. ++ */ ++ if (error_suppression) return; ++ ++ if (vasprintf (&str, fs, args) == -1) { ++ nbdkit_error ("lost error message: %s", fs); ++ return; ++ } ++ ++ trim (str); ++ ++ /* VDDK 7 added a useless error message about their "phone home" ++ * system called CEIP which only panics users. Demote it to a debug ++ * statement. https://bugzilla.redhat.com/show_bug.cgi?id=1834267 ++ */ ++ if (strstr (str, "Get CEIP status failed") != NULL) { ++ nbdkit_debug ("%s", str); ++ return; ++ } ++ ++ nbdkit_error ("%s", str); ++} ++ + /* Defer VDDK initialization until after fork because it is known to + * create background threads from VixDiskLib_InitEx. Unfortunately + * error reporting from this callback is difficult, but we have +diff --git a/plugins/vddk/vddk.h b/plugins/vddk/vddk.h +index 0e3dd79e..d99b6f4b 100644 +--- a/plugins/vddk/vddk.h ++++ b/plugins/vddk/vddk.h +@@ -183,6 +183,9 @@ extern pthread_mutex_t stats_lock; + #undef OPTIONAL_STUB + extern void display_stats (void); + ++/* utils.c */ ++extern void trim (char *str); ++ + /* worker.c */ + extern const char *command_type_string (enum command_type type); + extern int send_command_and_wait (struct vddk_handle *h, struct command *cmd); +-- +2.31.1 + diff --git a/SOURCES/0019-common-utils-test-vector.c-Add-vector-benchmarks.patch b/SOURCES/0019-common-utils-test-vector.c-Add-vector-benchmarks.patch new file mode 100644 index 0000000..02c96c2 --- /dev/null +++ b/SOURCES/0019-common-utils-test-vector.c-Add-vector-benchmarks.patch @@ -0,0 +1,245 @@ +From 6459704cc66f5fa0a2e6fc1e199458b77327fe52 Mon Sep 17 00:00:00 2001 +From: Nir Soffer +Date: Fri, 5 Nov 2021 20:36:42 +0200 +Subject: [PATCH] common/utils/test-vector.c: Add vector benchmarks + +The generic vector reallocs on every append. Add benchmarks to measure +the cost with uint32 vector (used for copying extents) and the effect of +reserving space upfront. + +The tests show that realloc is pretty efficient, but calling reserve +before the appends speeds the appends up significantly. + + NBDKIT_BENCH=1 ./test-vector + bench_reserve: 1000000 appends in 0.004503 s + bench_append: 1000000 appends in 0.014986 s + +The new benchmarks do not run by default to avoid trouble in CI on +overloaded machines or under qemu emulation. + +A new target added to run all benchmaks: + + make bench + +Ported from libnbd: +- commit dc9ae0174ab1384081a57a8d54b10f8147ea6430 +- commit f6c06a3b4d87fe976a96ea04f8da1f22b2531dbd + +(cherry picked from commit a227af7921c9a51c4f1ab699a3b9f06a9a645126) +--- + Makefile.am | 5 +++ + README | 7 ++++ + common/utils/Makefile.am | 5 ++- + common/utils/bench.h | 72 ++++++++++++++++++++++++++++++++++++++ + common/utils/test-vector.c | 55 +++++++++++++++++++++++++++-- + 5 files changed, 141 insertions(+), 3 deletions(-) + create mode 100644 common/utils/bench.h + +diff --git a/Makefile.am b/Makefile.am +index b21d69ed..49f5d91c 100644 +--- a/Makefile.am ++++ b/Makefile.am +@@ -102,6 +102,11 @@ check-root: + check-vddk: + $(MAKE) -C tests check-vddk + ++bench: all ++ @for d in common/utils; do \ ++ $(MAKE) -C $$d bench || exit 1; \ ++ done ++ + #---------------------------------------------------------------------- + # Maintainers only! + +diff --git a/README b/README +index a04325be..b001620c 100644 +--- a/README ++++ b/README +@@ -274,6 +274,13 @@ nbdkit-vddk-plugin against the library like this: + + make check-vddk vddkdir=vmware-vix-disklib-distrib + ++Running the benchmarks ++---------------------- ++ ++To run benchmarks: ++ ++ make bench ++ + DOWNLOAD TARBALLS + ================= + +diff --git a/common/utils/Makefile.am b/common/utils/Makefile.am +index 14e9dfc4..55415535 100644 +--- a/common/utils/Makefile.am ++++ b/common/utils/Makefile.am +@@ -100,6 +100,9 @@ test_quotes_SOURCES = test-quotes.c quote.c utils.h + test_quotes_CPPFLAGS = -I$(srcdir) + test_quotes_CFLAGS = $(WARNINGS_CFLAGS) + +-test_vector_SOURCES = test-vector.c vector.c vector.h ++test_vector_SOURCES = test-vector.c vector.c vector.h bench.h + test_vector_CPPFLAGS = -I$(srcdir) + test_vector_CFLAGS = $(WARNINGS_CFLAGS) ++ ++bench: test-vector ++ NBDKIT_BENCH=1 ./test-vector +diff --git a/common/utils/bench.h b/common/utils/bench.h +new file mode 100644 +index 00000000..496a3614 +--- /dev/null ++++ b/common/utils/bench.h +@@ -0,0 +1,72 @@ ++/* libnbd ++ * Copyright (C) 2021 Red Hat Inc. ++ * ++ * Redistribution and use in source and binary forms, with or without ++ * modification, are permitted provided that the following conditions are ++ * met: ++ * ++ * * Redistributions of source code must retain the above copyright ++ * notice, this list of conditions and the following disclaimer. ++ * ++ * * Redistributions in binary form must reproduce the above copyright ++ * notice, this list of conditions and the following disclaimer in the ++ * documentation and/or other materials provided with the distribution. ++ * ++ * * Neither the name of Red Hat nor the names of its contributors may be ++ * used to endorse or promote products derived from this software without ++ * specific prior written permission. ++ * ++ * THIS SOFTWARE IS PROVIDED BY RED HAT AND CONTRIBUTORS ''AS IS'' AND ++ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, ++ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A ++ * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL RED HAT OR ++ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, ++ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT ++ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF ++ * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ++ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, ++ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT ++ * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF ++ * SUCH DAMAGE. ++ */ ++ ++#ifndef LIBNBD_BENCH_H ++#define LIBNBD_BENCH_H ++ ++#include ++ ++#define MICROSECONDS 1000000 ++ ++struct bench { ++ struct timeval start, stop; ++}; ++ ++static inline void ++bench_start(struct bench *b) ++{ ++ gettimeofday (&b->start, NULL); ++} ++ ++static inline void ++bench_stop(struct bench *b) ++{ ++ gettimeofday (&b->stop, NULL); ++} ++ ++static inline double ++bench_sec(struct bench *b) ++{ ++ struct timeval dt; ++ ++ dt.tv_sec = b->stop.tv_sec - b->start.tv_sec; ++ dt.tv_usec = b->stop.tv_usec - b->start.tv_usec; ++ ++ if (dt.tv_usec < 0) { ++ dt.tv_sec -= 1; ++ dt.tv_usec += MICROSECONDS; ++ } ++ ++ return ((double)dt.tv_sec * MICROSECONDS + dt.tv_usec) / MICROSECONDS; ++} ++ ++#endif /* LIBNBD_BENCH_H */ +diff --git a/common/utils/test-vector.c b/common/utils/test-vector.c +index 94b2aeb7..28af59b8 100644 +--- a/common/utils/test-vector.c ++++ b/common/utils/test-vector.c +@@ -38,9 +38,13 @@ + #undef NDEBUG /* Keep test strong even for nbdkit built without assertions */ + #include + ++#include "bench.h" + #include "vector.h" + ++#define APPENDS 1000000 ++ + DEFINE_VECTOR_TYPE(int64_vector, int64_t); ++DEFINE_VECTOR_TYPE(uint32_vector, uint32_t); + DEFINE_VECTOR_TYPE(string_vector, char *); + + static int +@@ -113,10 +117,57 @@ test_string_vector (void) + free (v.ptr); + } + ++static void ++bench_reserve (void) ++{ ++ uint32_vector v = empty_vector; ++ struct bench b; ++ ++ bench_start(&b); ++ ++ uint32_vector_reserve(&v, APPENDS); ++ ++ for (uint32_t i = 0; i < APPENDS; i++) { ++ uint32_vector_append (&v, i); ++ } ++ ++ bench_stop(&b); ++ ++ assert (v.ptr[APPENDS - 1] == APPENDS - 1); ++ free (v.ptr); ++ ++ printf ("bench_reserve: %d appends in %.6f s\n", APPENDS, bench_sec (&b)); ++} ++ ++static void ++bench_append (void) ++{ ++ uint32_vector v = empty_vector; ++ struct bench b; ++ ++ bench_start(&b); ++ ++ for (uint32_t i = 0; i < APPENDS; i++) { ++ uint32_vector_append (&v, i); ++ } ++ ++ bench_stop(&b); ++ ++ assert (v.ptr[APPENDS - 1] == APPENDS - 1); ++ free (v.ptr); ++ ++ printf ("bench_append: %d appends in %.6f s\n", APPENDS, bench_sec (&b)); ++} ++ + int + main (int argc, char *argv[]) + { +- test_int64_vector (); +- test_string_vector (); ++ if (getenv("NBDKIT_BENCH")) { ++ bench_reserve (); ++ bench_append (); ++ } else { ++ test_int64_vector (); ++ test_string_vector (); ++ } + return 0; + } +-- +2.31.1 + diff --git a/SOURCES/0019-vddk-Implement-can_flush-and-can_fua.patch b/SOURCES/0019-vddk-Implement-can_flush-and-can_fua.patch deleted file mode 100644 index 98d6225..0000000 --- a/SOURCES/0019-vddk-Implement-can_flush-and-can_fua.patch +++ /dev/null @@ -1,74 +0,0 @@ -From 2104686eb708bf87070c21e7af0e70e0317306b6 Mon Sep 17 00:00:00 2001 -From: "Richard W.M. Jones" -Date: Mon, 5 Jul 2021 21:36:41 +0100 -Subject: [PATCH] vddk: Implement can_flush and can_fua - -VDDK < 6.0 doesn't support flush. Previously we advertised flush and -FUA but ignored them if VDDK didn't support it. Instead, correctly -set these flags in the NBD protocol according to what VDDK supports. - -(cherry picked from commit 04b05274414a8cf4615eb2d6f46d5658814509c1) ---- - plugins/vddk/vddk.c | 28 ++++++++++++++++++++-------- - 1 file changed, 20 insertions(+), 8 deletions(-) - -diff --git a/plugins/vddk/vddk.c b/plugins/vddk/vddk.c -index 76faa768..b5bce9a0 100644 ---- a/plugins/vddk/vddk.c -+++ b/plugins/vddk/vddk.c -@@ -772,12 +772,28 @@ vddk_pwrite (void *handle, const void *buf, uint32_t count, uint64_t offset, - return -1; - } - -- if (fua && vddk_flush (handle, 0) == -1) -- return -1; -+ if (fua) { -+ if (vddk_flush (handle, 0) == -1) -+ return -1; -+ } - - return 0; - } - -+static int -+vddk_can_fua (void *handle) -+{ -+ /* The Flush call was not available in VDDK < 6.0. */ -+ return VixDiskLib_Flush != NULL ? NBDKIT_FUA_NATIVE : NBDKIT_FUA_NONE; -+} -+ -+static int -+vddk_can_flush (void *handle) -+{ -+ /* The Flush call was not available in VDDK < 6.0. */ -+ return VixDiskLib_Flush != NULL; -+} -+ - /* Flush data to the file. */ - static int - vddk_flush (void *handle, uint32_t flags) -@@ -785,12 +801,6 @@ vddk_flush (void *handle, uint32_t flags) - struct vddk_handle *h = handle; - VixError err; - -- /* The Flush call was not available in VDDK < 6.0 so this is simply -- * ignored on earlier versions. -- */ -- if (VixDiskLib_Flush == NULL) -- return 0; -- - DEBUG_CALL ("VixDiskLib_Flush", "handle"); - err = VixDiskLib_Flush (h->handle); - if (err != VIX_OK) { -@@ -985,6 +995,8 @@ static struct nbdkit_plugin plugin = { - .get_size = vddk_get_size, - .pread = vddk_pread, - .pwrite = vddk_pwrite, -+ .can_fua = vddk_can_fua, -+ .can_flush = vddk_can_flush, - .flush = vddk_flush, - .can_extents = vddk_can_extents, - .extents = vddk_extents, --- -2.31.1 - diff --git a/SOURCES/0020-common-urils-vector.c-Optimize-vector-append.patch b/SOURCES/0020-common-urils-vector.c-Optimize-vector-append.patch new file mode 100644 index 0000000..99da6d7 --- /dev/null +++ b/SOURCES/0020-common-urils-vector.c-Optimize-vector-append.patch @@ -0,0 +1,54 @@ +From 5454ced7c8cfc2ba278c2635eecb9a5e4841e613 Mon Sep 17 00:00:00 2001 +From: Nir Soffer +Date: Fri, 5 Nov 2021 22:16:26 +0200 +Subject: [PATCH] common/urils/vector.c: Optimize vector append + +Minimize reallocs by growing the backing array by factor of 1.5. + +Testing show that now append() is fast without calling reserve() +upfront, simplifying code using vector. + + NBDKIT_BENCH=1 ./test-vector + bench_reserve: 1000000 appends in 0.004496 s + bench_append: 1000000 appends in 0.004180 s + +This can make a difference in code appending millions of items. + +Ported from libnbd commit 985dfa72ae2e41901f0af21e7205ef85428cd4bd. + +(cherry picked from commit 12356fa97a840de19bb61e0abedd6e7c7e578e5a) +--- + common/utils/vector.c | 14 ++++++++++++-- + 1 file changed, 12 insertions(+), 2 deletions(-) + +diff --git a/common/utils/vector.c b/common/utils/vector.c +index 00cd2546..7df17e1b 100644 +--- a/common/utils/vector.c ++++ b/common/utils/vector.c +@@ -41,11 +41,21 @@ int + generic_vector_reserve (struct generic_vector *v, size_t n, size_t itemsize) + { + void *newptr; ++ size_t reqalloc, newalloc; + +- newptr = realloc (v->ptr, (n + v->alloc) * itemsize); ++ reqalloc = v->alloc + n; ++ if (reqalloc < v->alloc) ++ return -1; /* overflow */ ++ ++ newalloc = (v->alloc * 3 + 1) / 2; ++ ++ if (newalloc < reqalloc) ++ newalloc = reqalloc; ++ ++ newptr = realloc (v->ptr, newalloc * itemsize); + if (newptr == NULL) + return -1; + v->ptr = newptr; +- v->alloc += n; ++ v->alloc = newalloc; + return 0; + } +-- +2.31.1 + diff --git a/SOURCES/0020-vddk-Replace-DEBUG_CALL-with-bracketed-VDDK_CALL_STA.patch b/SOURCES/0020-vddk-Replace-DEBUG_CALL-with-bracketed-VDDK_CALL_STA.patch deleted file mode 100644 index 7a39936..0000000 --- a/SOURCES/0020-vddk-Replace-DEBUG_CALL-with-bracketed-VDDK_CALL_STA.patch +++ /dev/null @@ -1,319 +0,0 @@ -From 51713e7702d389fd55d5721c4773fca40e3e89f6 Mon Sep 17 00:00:00 2001 -From: "Richard W.M. Jones" -Date: Wed, 18 Aug 2021 14:26:30 +0100 -Subject: [PATCH] vddk: Replace DEBUG_CALL with bracketed VDDK_CALL_START/END - macros - -This is neutral refactoring, but allows us in the next commit to -collect statistics about the amount of time spent in these calls. - -(cherry picked from commit 1335ebfb5637bf5a44403d0b152da7272fdd3e54) ---- - plugins/vddk/vddk.c | 175 +++++++++++++++++++++++++------------------- - 1 file changed, 99 insertions(+), 76 deletions(-) - -diff --git a/plugins/vddk/vddk.c b/plugins/vddk/vddk.c -index b5bce9a0..888009ab 100644 ---- a/plugins/vddk/vddk.c -+++ b/plugins/vddk/vddk.c -@@ -104,19 +104,23 @@ static bool is_remote; - VixDiskLib_FreeErrorText (vddk_err_msg); \ - } while (0) - --#define DEBUG_CALL(fn, fs, ...) \ -- nbdkit_debug ("VDDK call: %s (" fs ")", fn, ##__VA_ARGS__) --#define DEBUG_CALL_DATAPATH(fn, fs, ...) \ -- if (vddk_debug_datapath) \ -- nbdkit_debug ("VDDK call: %s (" fs ")", fn, ##__VA_ARGS__) -+#define VDDK_CALL_START(fn, fs, ...) \ -+ nbdkit_debug ("VDDK call: %s (" fs ")", #fn, ##__VA_ARGS__); \ -+ do -+#define VDDK_CALL_START_DATAPATH(fn, fs, ...) \ -+ if (vddk_debug_datapath) \ -+ nbdkit_debug ("VDDK call: %s (" fs ")", #fn, ##__VA_ARGS__); \ -+ do -+#define VDDK_CALL_END(fn) while (0) - - /* Unload the plugin. */ - static void - vddk_unload (void) - { - if (init_called) { -- DEBUG_CALL ("VixDiskLib_Exit", ""); -- VixDiskLib_Exit (); -+ VDDK_CALL_START (VixDiskLib_Exit, "") { -+ VixDiskLib_Exit (); -+ } VDDK_CALL_END (VixDiskLib_Exit); - } - if (dl) - dlclose (dl); -@@ -449,15 +453,16 @@ vddk_after_fork (void) - VixError err; - - /* Initialize VDDK library. */ -- DEBUG_CALL ("VixDiskLib_InitEx", -- "%d, %d, &debug_fn, &error_fn, &error_fn, %s, %s", -- VDDK_MAJOR, VDDK_MINOR, -- libdir, config ? : "NULL"); -- err = VixDiskLib_InitEx (VDDK_MAJOR, VDDK_MINOR, -- &debug_function, /* log function */ -- &error_function, /* warn function */ -- &error_function, /* panic function */ -- libdir, config); -+ VDDK_CALL_START (VixDiskLib_InitEx, -+ "%d, %d, &debug_fn, &error_fn, &error_fn, %s, %s", -+ VDDK_MAJOR, VDDK_MINOR, -+ libdir, config ? : "NULL") { -+ err = VixDiskLib_InitEx (VDDK_MAJOR, VDDK_MINOR, -+ &debug_function, /* log function */ -+ &error_function, /* warn function */ -+ &error_function, /* panic function */ -+ libdir, config); -+ } VDDK_CALL_END (VixDiskLib_InitEx); - if (err != VIX_OK) { - VDDK_ERROR (err, "VixDiskLib_InitEx"); - exit (EXIT_FAILURE); -@@ -519,8 +524,9 @@ allocate_connect_params (void) - VixDiskLibConnectParams *ret; - - if (VixDiskLib_AllocateConnectParams != NULL) { -- DEBUG_CALL ("VixDiskLib_AllocateConnectParams", ""); -- ret = VixDiskLib_AllocateConnectParams (); -+ VDDK_CALL_START (VixDiskLib_AllocateConnectParams, "") { -+ ret = VixDiskLib_AllocateConnectParams (); -+ } VDDK_CALL_END (VixDiskLib_AllocateConnectParams); - } - else - ret = calloc (1, sizeof (VixDiskLibConnectParams)); -@@ -535,8 +541,9 @@ free_connect_params (VixDiskLibConnectParams *params) - * originally called. Otherwise use free. - */ - if (VixDiskLib_AllocateConnectParams != NULL) { -- DEBUG_CALL ("VixDiskLib_FreeConnectParams", "params"); -- VixDiskLib_FreeConnectParams (params); -+ VDDK_CALL_START (VixDiskLib_FreeConnectParams, "params") { -+ VixDiskLib_FreeConnectParams (params); -+ } VDDK_CALL_END (VixDiskLib_FreeConnectParams); - } - else - free (params); -@@ -589,16 +596,17 @@ vddk_open (int readonly) - * either ESXi or vCenter servers. - */ - -- DEBUG_CALL ("VixDiskLib_ConnectEx", -- "h->params, %d, %s, %s, &connection", -- readonly, -- snapshot_moref ? : "NULL", -- transport_modes ? : "NULL"); -- err = VixDiskLib_ConnectEx (h->params, -- readonly, -- snapshot_moref, -- transport_modes, -- &h->connection); -+ VDDK_CALL_START (VixDiskLib_ConnectEx, -+ "h->params, %d, %s, %s, &connection", -+ readonly, -+ snapshot_moref ? : "NULL", -+ transport_modes ? : "NULL") { -+ err = VixDiskLib_ConnectEx (h->params, -+ readonly, -+ snapshot_moref, -+ transport_modes, -+ &h->connection); -+ } VDDK_CALL_END (VixDiskLib_ConnectEx); - if (err != VIX_OK) { - VDDK_ERROR (err, "VixDiskLib_ConnectEx"); - goto err1; -@@ -618,9 +626,10 @@ vddk_open (int readonly) - case NONE: break; - } - -- DEBUG_CALL ("VixDiskLib_Open", -- "connection, %s, %d, &handle", filename, flags); -- err = VixDiskLib_Open (h->connection, filename, flags, &h->handle); -+ VDDK_CALL_START (VixDiskLib_Open, -+ "connection, %s, %d, &handle", filename, flags) { -+ err = VixDiskLib_Open (h->connection, filename, flags, &h->handle); -+ } VDDK_CALL_END (VixDiskLib_Open); - if (err != VIX_OK) { - VDDK_ERROR (err, "VixDiskLib_Open: %s", filename); - goto err2; -@@ -632,8 +641,9 @@ vddk_open (int readonly) - return h; - - err2: -- DEBUG_CALL ("VixDiskLib_Disconnect", "connection"); -- VixDiskLib_Disconnect (h->connection); -+ VDDK_CALL_START (VixDiskLib_Disconnect, "connection") { -+ VixDiskLib_Disconnect (h->connection); -+ } VDDK_CALL_END (VixDiskLib_Disconnect); - err1: - free_connect_params (h->params); - err0: -@@ -648,10 +658,13 @@ vddk_close (void *handle) - ACQUIRE_LOCK_FOR_CURRENT_SCOPE (&open_close_lock); - struct vddk_handle *h = handle; - -- DEBUG_CALL ("VixDiskLib_Close", "handle"); -- VixDiskLib_Close (h->handle); -- DEBUG_CALL ("VixDiskLib_Disconnect", "connection"); -- VixDiskLib_Disconnect (h->connection); -+ VDDK_CALL_START (VixDiskLib_Close, "handle") { -+ VixDiskLib_Close (h->handle); -+ } VDDK_CALL_END (VixDiskLib_Close); -+ VDDK_CALL_START (VixDiskLib_Disconnect, "connection") { -+ VixDiskLib_Disconnect (h->connection); -+ } VDDK_CALL_END (VixDiskLib_Disconnect); -+ - free_connect_params (h->params); - free (h); - } -@@ -665,8 +678,9 @@ vddk_get_size (void *handle) - VixError err; - uint64_t size; - -- DEBUG_CALL ("VixDiskLib_GetInfo", "handle, &info"); -- err = VixDiskLib_GetInfo (h->handle, &info); -+ VDDK_CALL_START (VixDiskLib_GetInfo, "handle, &info") { -+ err = VixDiskLib_GetInfo (h->handle, &info); -+ } VDDK_CALL_END (VixDiskLib_GetInfo); - if (err != VIX_OK) { - VDDK_ERROR (err, "VixDiskLib_GetInfo"); - return -1; -@@ -694,8 +708,9 @@ vddk_get_size (void *handle) - info->uuid ? : "NULL"); - } - -- DEBUG_CALL ("VixDiskLib_FreeInfo", "info"); -- VixDiskLib_FreeInfo (info); -+ VDDK_CALL_START (VixDiskLib_FreeInfo, "info") { -+ VixDiskLib_FreeInfo (info); -+ } VDDK_CALL_END (VixDiskLib_FreeInfo); - - return (int64_t) size; - } -@@ -723,11 +738,12 @@ vddk_pread (void *handle, void *buf, uint32_t count, uint64_t offset, - offset /= VIXDISKLIB_SECTOR_SIZE; - count /= VIXDISKLIB_SECTOR_SIZE; - -- DEBUG_CALL_DATAPATH ("VixDiskLib_Read", -- "handle, %" PRIu64 " sectors, " -- "%" PRIu32 " sectors, buffer", -- offset, count); -- err = VixDiskLib_Read (h->handle, offset, count, buf); -+ VDDK_CALL_START_DATAPATH (VixDiskLib_Read, -+ "handle, %" PRIu64 " sectors, " -+ "%" PRIu32 " sectors, buffer", -+ offset, count) { -+ err = VixDiskLib_Read (h->handle, offset, count, buf); -+ } VDDK_CALL_END (VixDiskLib_Read); - if (err != VIX_OK) { - VDDK_ERROR (err, "VixDiskLib_Read"); - return -1; -@@ -762,11 +778,12 @@ vddk_pwrite (void *handle, const void *buf, uint32_t count, uint64_t offset, - offset /= VIXDISKLIB_SECTOR_SIZE; - count /= VIXDISKLIB_SECTOR_SIZE; - -- DEBUG_CALL_DATAPATH ("VixDiskLib_Write", -- "handle, %" PRIu64 " sectors, " -- "%" PRIu32 " sectors, buffer", -- offset, count); -- err = VixDiskLib_Write (h->handle, offset, count, buf); -+ VDDK_CALL_START_DATAPATH (VixDiskLib_Write, -+ "handle, %" PRIu64 " sectors, " -+ "%" PRIu32 " sectors, buffer", -+ offset, count) { -+ err = VixDiskLib_Write (h->handle, offset, count, buf); -+ } VDDK_CALL_END (VixDiskLib_Write); - if (err != VIX_OK) { - VDDK_ERROR (err, "VixDiskLib_Write"); - return -1; -@@ -801,8 +818,9 @@ vddk_flush (void *handle, uint32_t flags) - struct vddk_handle *h = handle; - VixError err; - -- DEBUG_CALL ("VixDiskLib_Flush", "handle"); -- err = VixDiskLib_Flush (h->handle); -+ VDDK_CALL_START (VixDiskLib_Flush, "handle") { -+ err = VixDiskLib_Flush (h->handle); -+ } VDDK_CALL_END (VixDiskLib_Flush); - if (err != VIX_OK) { - VDDK_ERROR (err, "VixDiskLib_Flush"); - return -1; -@@ -836,17 +854,19 @@ vddk_can_extents (void *handle) - * the best thing we can do here is to try the call and if it's - * non-functional return false. - */ -- DEBUG_CALL ("VixDiskLib_QueryAllocatedBlocks", -- "handle, 0, %d sectors, %d sectors", -- VIXDISKLIB_MIN_CHUNK_SIZE, VIXDISKLIB_MIN_CHUNK_SIZE); -- err = VixDiskLib_QueryAllocatedBlocks (h->handle, -- 0, VIXDISKLIB_MIN_CHUNK_SIZE, -- VIXDISKLIB_MIN_CHUNK_SIZE, -- &block_list); -+ VDDK_CALL_START (VixDiskLib_QueryAllocatedBlocks, -+ "handle, 0, %d sectors, %d sectors", -+ VIXDISKLIB_MIN_CHUNK_SIZE, VIXDISKLIB_MIN_CHUNK_SIZE) { -+ err = VixDiskLib_QueryAllocatedBlocks (h->handle, -+ 0, VIXDISKLIB_MIN_CHUNK_SIZE, -+ VIXDISKLIB_MIN_CHUNK_SIZE, -+ &block_list); -+ } VDDK_CALL_END (VixDiskLib_QueryAllocatedBlocks); - error_suppression = 0; - if (err == VIX_OK) { -- DEBUG_CALL ("VixDiskLib_FreeBlockList", "block_list"); -- VixDiskLib_FreeBlockList (block_list); -+ VDDK_CALL_START (VixDiskLib_FreeBlockList, "block_list") { -+ VixDiskLib_FreeBlockList (block_list); -+ } VDDK_CALL_END (VixDiskLib_FreeBlockList); - } - if (err != VIX_OK) { - char *errmsg = VixDiskLib_GetErrorText (err, NULL); -@@ -923,14 +943,15 @@ vddk_extents (void *handle, uint32_t count, uint64_t offset, uint32_t flags, - nr_chunks = MIN (nr_chunks, VIXDISKLIB_MAX_CHUNK_NUMBER); - nr_sectors = nr_chunks * VIXDISKLIB_MIN_CHUNK_SIZE; - -- DEBUG_CALL ("VixDiskLib_QueryAllocatedBlocks", -- "handle, %" PRIu64 " sectors, %" PRIu64 " sectors, " -- "%d sectors", -- start_sector, nr_sectors, VIXDISKLIB_MIN_CHUNK_SIZE); -- err = VixDiskLib_QueryAllocatedBlocks (h->handle, -- start_sector, nr_sectors, -- VIXDISKLIB_MIN_CHUNK_SIZE, -- &block_list); -+ VDDK_CALL_START (VixDiskLib_QueryAllocatedBlocks, -+ "handle, %" PRIu64 " sectors, %" PRIu64 " sectors, " -+ "%d sectors", -+ start_sector, nr_sectors, VIXDISKLIB_MIN_CHUNK_SIZE) { -+ err = VixDiskLib_QueryAllocatedBlocks (h->handle, -+ start_sector, nr_sectors, -+ VIXDISKLIB_MIN_CHUNK_SIZE, -+ &block_list); -+ } VDDK_CALL_END (VixDiskLib_QueryAllocatedBlocks); - if (err != VIX_OK) { - VDDK_ERROR (err, "VixDiskLib_QueryAllocatedBlocks"); - return -1; -@@ -949,13 +970,15 @@ vddk_extents (void *handle, uint32_t count, uint64_t offset, uint32_t flags, - add_extent (extents, &position, blk_offset, true) == -1) || - (add_extent (extents, - &position, blk_offset + blk_length, false) == -1)) { -- DEBUG_CALL ("VixDiskLib_FreeBlockList", "block_list"); -- VixDiskLib_FreeBlockList (block_list); -+ VDDK_CALL_START (VixDiskLib_FreeBlockList, "block_list") { -+ VixDiskLib_FreeBlockList (block_list); -+ } VDDK_CALL_END (VixDiskLib_FreeBlockList); - return -1; - } - } -- DEBUG_CALL ("VixDiskLib_FreeBlockList", "block_list"); -- VixDiskLib_FreeBlockList (block_list); -+ VDDK_CALL_START (VixDiskLib_FreeBlockList, "block_list") { -+ VixDiskLib_FreeBlockList (block_list); -+ } VDDK_CALL_END (VixDiskLib_FreeBlockList); - - /* There's an implicit hole after the returned list of blocks, up - * to the end of the QueryAllocatedBlocks request. --- -2.31.1 - diff --git a/SOURCES/0021-common-utils-vector-Rename-alloc-to-cap.patch b/SOURCES/0021-common-utils-vector-Rename-alloc-to-cap.patch new file mode 100644 index 0000000..0cf18b9 --- /dev/null +++ b/SOURCES/0021-common-utils-vector-Rename-alloc-to-cap.patch @@ -0,0 +1,188 @@ +From 304f180b61fa28421b9901d2173a280e633b55c2 Mon Sep 17 00:00:00 2001 +From: Nir Soffer +Date: Fri, 5 Nov 2021 22:59:38 +0200 +Subject: [PATCH] common/utils/vector: Rename `alloc` to `cap` + +The `alloc` field is the maximum number of items you can append to a +vector before it need to be resized. This may confuse users with the +size of the `ptr` array which is `alloc * itemsize`. Rename to "cap", +common term for this property in many languages (e.g C++, Rust, Go). + +Tested with "make check". Tests requiring root or external libraries +(vddk) not tested. + +Ported from libnbd commit e3c7f02a2a844295564c832108d36c939c4e4ecf. + +(cherry picked from commit 75a44237c4463524dbf7951bb62b59c373c85865) +--- + common/allocators/malloc.c | 24 ++++++++++++------------ + common/utils/vector.c | 16 ++++++++-------- + common/utils/vector.h | 12 ++++++------ + plugins/vddk/reexec.c | 2 +- + 4 files changed, 27 insertions(+), 27 deletions(-) + +diff --git a/common/allocators/malloc.c b/common/allocators/malloc.c +index 59409c24..f7474465 100644 +--- a/common/allocators/malloc.c ++++ b/common/allocators/malloc.c +@@ -88,16 +88,16 @@ extend (struct m_alloc *ma, uint64_t new_size) + ACQUIRE_WRLOCK_FOR_CURRENT_SCOPE (&ma->lock); + size_t old_size, n; + +- if (ma->ba.alloc < new_size) { +- old_size = ma->ba.alloc; +- n = new_size - ma->ba.alloc; ++ if (ma->ba.cap < new_size) { ++ old_size = ma->ba.cap; ++ n = new_size - ma->ba.cap; + + #ifdef HAVE_MUNLOCK + /* Since the memory might be moved by realloc, we must unlock the + * original array. + */ + if (ma->use_mlock) +- munlock (ma->ba.ptr, ma->ba.alloc); ++ munlock (ma->ba.ptr, ma->ba.cap); + #endif + + if (bytearray_reserve (&ma->ba, n) == -1) { +@@ -110,7 +110,7 @@ extend (struct m_alloc *ma, uint64_t new_size) + + #ifdef HAVE_MLOCK + if (ma->use_mlock) { +- if (mlock (ma->ba.ptr, ma->ba.alloc) == -1) { ++ if (mlock (ma->ba.ptr, ma->ba.cap) == -1) { + nbdkit_error ("allocator=malloc: mlock: %m"); + return -1; + } +@@ -138,11 +138,11 @@ m_alloc_read (struct allocator *a, void *buf, + /* Avoid reading beyond the end of the allocated array. Return + * zeroes for that part. + */ +- if (offset >= ma->ba.alloc) ++ if (offset >= ma->ba.cap) + memset (buf, 0, count); +- else if (offset + count > ma->ba.alloc) { +- memcpy (buf, ma->ba.ptr + offset, ma->ba.alloc - offset); +- memset (buf + ma->ba.alloc - offset, 0, offset + count - ma->ba.alloc); ++ else if (offset + count > ma->ba.cap) { ++ memcpy (buf, ma->ba.ptr + offset, ma->ba.cap - offset); ++ memset (buf + ma->ba.cap - offset, 0, offset + count - ma->ba.cap); + } + else + memcpy (buf, ma->ba.ptr + offset, count); +@@ -191,9 +191,9 @@ m_alloc_zero (struct allocator *a, uint64_t count, uint64_t offset) + /* Try to avoid extending the array, since the unallocated part + * always reads as zero. + */ +- if (offset < ma->ba.alloc) { +- if (offset + count > ma->ba.alloc) +- memset (ma->ba.ptr + offset, 0, ma->ba.alloc - offset); ++ if (offset < ma->ba.cap) { ++ if (offset + count > ma->ba.cap) ++ memset (ma->ba.ptr + offset, 0, ma->ba.cap - offset); + else + memset (ma->ba.ptr + offset, 0, count); + } +diff --git a/common/utils/vector.c b/common/utils/vector.c +index 7df17e1b..a4b43ce7 100644 +--- a/common/utils/vector.c ++++ b/common/utils/vector.c +@@ -41,21 +41,21 @@ int + generic_vector_reserve (struct generic_vector *v, size_t n, size_t itemsize) + { + void *newptr; +- size_t reqalloc, newalloc; ++ size_t reqcap, newcap; + +- reqalloc = v->alloc + n; +- if (reqalloc < v->alloc) ++ reqcap = v->cap + n; ++ if (reqcap < v->cap) + return -1; /* overflow */ + +- newalloc = (v->alloc * 3 + 1) / 2; ++ newcap = (v->cap * 3 + 1) / 2; + +- if (newalloc < reqalloc) +- newalloc = reqalloc; ++ if (newcap < reqcap) ++ newcap = reqcap; + +- newptr = realloc (v->ptr, newalloc * itemsize); ++ newptr = realloc (v->ptr, newcap * itemsize); + if (newptr == NULL) + return -1; + v->ptr = newptr; +- v->alloc = newalloc; ++ v->cap = newcap; + return 0; + } +diff --git a/common/utils/vector.h b/common/utils/vector.h +index f6a0af78..782dcba6 100644 +--- a/common/utils/vector.h ++++ b/common/utils/vector.h +@@ -86,7 +86,7 @@ + struct name { \ + type *ptr; /* Pointer to array of items. */ \ + size_t size; /* Number of valid items in the array. */ \ +- size_t alloc; /* Number of items allocated. */ \ ++ size_t cap; /* Maximum number of items. */ \ + }; \ + typedef struct name name; \ + \ +@@ -106,7 +106,7 @@ + name##_insert (name *v, type elem, size_t i) \ + { \ + assert (i <= v->size); \ +- if (v->size >= v->alloc) { \ ++ if (v->size >= v->cap) { \ + if (name##_reserve (v, 1) == -1) return -1; \ + } \ + memmove (&v->ptr[i+1], &v->ptr[i], (v->size-i) * sizeof (elem)); \ +@@ -137,7 +137,7 @@ + { \ + free (v->ptr); \ + v->ptr = NULL; \ +- v->size = v->alloc = 0; \ ++ v->size = v->cap = 0; \ + } \ + \ + /* Iterate over the vector, calling f() on each element. */ \ +@@ -181,17 +181,17 @@ + if (newptr == NULL) return -1; \ + memcpy (newptr, vptr, len); \ + copy->ptr = newptr; \ +- copy->size = copy->alloc = v->size; \ ++ copy->size = copy->cap = v->size; \ + return 0; \ + } \ + \ + +-#define empty_vector { .ptr = NULL, .size = 0, .alloc = 0 } ++#define empty_vector { .ptr = NULL, .size = 0, .cap = 0 } + + struct generic_vector { + void *ptr; + size_t size; +- size_t alloc; ++ size_t cap; + }; + + extern int generic_vector_reserve (struct generic_vector *v, +diff --git a/plugins/vddk/reexec.c b/plugins/vddk/reexec.c +index 46acdb62..9e87025e 100644 +--- a/plugins/vddk/reexec.c ++++ b/plugins/vddk/reexec.c +@@ -116,7 +116,7 @@ perform_reexec (const char *env, const char *prepend) + nbdkit_error ("realloc: %m"); + exit (EXIT_FAILURE); + } +- r = read (fd, buf.ptr + buf.size, buf.alloc - buf.size); ++ r = read (fd, buf.ptr + buf.size, buf.cap - buf.size); + if (r == -1) { + nbdkit_error ("read: %s: %m", cmdline_file); + exit (EXIT_FAILURE); +-- +2.31.1 + diff --git a/SOURCES/0021-tests-Add-a-better-test-of-real-VDDK.patch b/SOURCES/0021-tests-Add-a-better-test-of-real-VDDK.patch deleted file mode 100644 index c9dc143..0000000 --- a/SOURCES/0021-tests-Add-a-better-test-of-real-VDDK.patch +++ /dev/null @@ -1,103 +0,0 @@ -From bd181ea739ebfafbf7239b5fa89e98becdb8cb72 Mon Sep 17 00:00:00 2001 -From: "Richard W.M. Jones" -Date: Mon, 5 Jul 2021 22:03:10 +0100 -Subject: [PATCH] tests: Add a better test of real VDDK - -The previous test only tested reading and maybe extents, and used an -all-zero disk. I'm fairly convinced the test only worked accidentally -since you must use an absolute path when opening a local file and the -test did not do that. - -Add a more comprehensive test that tests writing and flush too. - -(cherry picked from commit a6ca4f24593008bb2d8efb177e7f424cff51dfbf) ---- - tests/test-vddk-real.sh | 55 ++++++++++++++++++++++++++++------------- - 1 file changed, 38 insertions(+), 17 deletions(-) - -diff --git a/tests/test-vddk-real.sh b/tests/test-vddk-real.sh -index df486ba1..f848db44 100755 ---- a/tests/test-vddk-real.sh -+++ b/tests/test-vddk-real.sh -@@ -37,8 +37,12 @@ set -x - requires test "x$vddkdir" != "x" - requires test -d "$vddkdir" - requires test -f "$vddkdir/lib64/libvixDiskLib.so" -+requires test -r /dev/urandom -+requires cmp --version -+requires dd --version - requires qemu-img --version - requires nbdcopy --version -+requires nbdinfo --version - requires stat --version - - # VDDK > 5.1.1 only supports x86_64. -@@ -47,31 +51,48 @@ if [ `uname -m` != "x86_64" ]; then - exit 77 - fi - --files="test-vddk-real.vmdk test-vddk-real.out test-vddk-real.log" --rm -f $files --cleanup_fn rm -f $files -- --qemu-img create -f vmdk test-vddk-real.vmdk 100M -- - # Since we are comparing error messages below, let's make sure we're - # not translating errors. - export LANG=C - --fail=0 --nbdkit -f -v -U - \ -- --filter=readahead \ -- vddk libdir="$vddkdir" test-vddk-real.vmdk \ -- --run 'nbdcopy "$uri" test-vddk-real.out' \ -- > test-vddk-real.log 2>&1 || fail=1 -+pid=test-vddk-real.pid -+sock=$(mktemp -u /tmp/nbdkit-test-sock.XXXXXX) -+vmdk=$PWD/test-vddk-real.vmdk ;# note must be an absolute path -+raw=test-vddk-real.raw -+raw2=test-vddk-real.raw2 -+log=test-vddk-real.log -+files="$pid $sock $vmdk $raw $raw2 $log" -+rm -f $files -+cleanup_fn rm -f $files -+ -+qemu-img create -f vmdk $vmdk 10M -+ -+# Check first that the VDDK library can be fully loaded. We have to -+# check the log file for missing modules since they may not show up as -+# errors. -+nbdkit -fv -U - vddk libdir="$vddkdir" $vmdk --run 'nbdinfo "$uri"' >$log 2>&1 - - # Check the log for missing modules --cat test-vddk-real.log -+cat $log - if grep 'cannot open shared object file' test-vddk-real.log; then - exit 1 - fi - --# Check the raw output file has exactly the right size. --size="$(stat -c '%s' test-vddk-real.out)" --test "$size" -eq $((100 * 1024 * 1024)) -+# Now run nbdkit for the test. -+start_nbdkit -P $pid -U $sock vddk libdir="$vddkdir" $vmdk -+uri="nbd+unix:///?socket=$sock" - --exit $fail -+# VDDK < 6.0 did not support flush, so disable flush test there. Also -+# if nbdinfo doesn't support the --can flush syntax (added in libnbd -+# 1.10) then this is disabled. -+if nbdinfo --can flush "$uri"; then flush="--flush"; else flush=""; fi -+ -+# Copy in and out some data. This should exercise read, write, -+# extents and flushing. -+dd if=/dev/urandom of=$raw count=5 bs=$((1024*1024)) -+truncate -s 10M $raw -+ -+nbdcopy $flush $raw "$uri" -+nbdcopy "$uri" $raw2 -+ -+cmp $raw $raw2 --- -2.31.1 - diff --git a/SOURCES/0022-common-utils-vector-Rename-size-to-len.patch b/SOURCES/0022-common-utils-vector-Rename-size-to-len.patch new file mode 100644 index 0000000..822f84e --- /dev/null +++ b/SOURCES/0022-common-utils-vector-Rename-size-to-len.patch @@ -0,0 +1,1747 @@ +From a53a2234147543b04ee483aff7b9895c0d5082b5 Mon Sep 17 00:00:00 2001 +From: Nir Soffer +Date: Sat, 6 Nov 2021 00:03:11 +0200 +Subject: [PATCH] common/utils/vector: Rename `size` to `len` + +The field `size` may be confusing with the size of the underlying array. +Rename to `len`, a common term for this concept. + +Tested with "make check" as regular user, tests that need root or +external libraries (vddk) are not tested. + +Ported from libnbd commit cc0567e9aed7e6b40a44bf8eac0a262ac7314fec. + +(cherry picked from commit 0b0eece73f04963a66b9adc507e5cbaba608660b) +--- + common/allocators/allocator.c | 2 +- + common/allocators/malloc.c | 2 +- + common/allocators/sparse.c | 6 +- + common/allocators/zstd.c | 6 +- + common/regions/regions.h | 6 +- + common/utils/environ.c | 2 +- + common/utils/test-vector.c | 2 +- + common/utils/vector.h | 52 ++++++------ + filters/ddrescue/ddrescue.c | 2 +- + filters/exitwhen/exitwhen.c | 2 +- + filters/extentlist/extentlist.c | 14 ++-- + filters/multi-conn/multi-conn.c | 16 ++-- + plugins/cc/cc.c | 4 +- + plugins/data/data.c | 4 +- + plugins/data/format.c | 114 +++++++++++++-------------- + plugins/eval/eval.c | 2 +- + plugins/floppy/directory-lfn.c | 24 +++--- + plugins/floppy/floppy.c | 2 +- + plugins/floppy/virtual-floppy.c | 32 ++++---- + plugins/iso/iso.c | 4 +- + plugins/nbd/nbd.c | 6 +- + plugins/partitioning/partition-mbr.c | 8 +- + plugins/partitioning/partitioning.c | 18 ++--- + plugins/partitioning/virtual-disk.c | 12 +-- + plugins/partitioning/virtual-disk.h | 2 +- + plugins/split/split.c | 14 ++-- + plugins/ssh/ssh.c | 2 +- + plugins/vddk/reexec.c | 8 +- + plugins/vddk/stats.c | 4 +- + plugins/vddk/worker.c | 4 +- + server/exports.c | 6 +- + server/extents.c | 22 +++--- + server/main.c | 2 +- + server/sockets.c | 8 +- + wrapper.c | 4 +- + 35 files changed, 209 insertions(+), 209 deletions(-) + +diff --git a/common/allocators/allocator.c b/common/allocators/allocator.c +index d306a842..019c68cd 100644 +--- a/common/allocators/allocator.c ++++ b/common/allocators/allocator.c +@@ -137,7 +137,7 @@ create_allocator (const char *type, bool debug) + return NULL; + + /* See if we can find the allocator. */ +- for (i = 0; i < allocators.size; ++i) { ++ for (i = 0; i < allocators.len; ++i) { + if (strncmp (type, allocators.ptr[i]->type, type_len) == 0) { + ret = allocators.ptr[i]->create (¶ms); + break; +diff --git a/common/allocators/malloc.c b/common/allocators/malloc.c +index f7474465..eea44432 100644 +--- a/common/allocators/malloc.c ++++ b/common/allocators/malloc.c +@@ -241,7 +241,7 @@ m_alloc_create (const void *paramsv) + size_t i; + + /* Parse the optional mlock=true|false parameter. */ +- for (i = 0; i < params->size; ++i) { ++ for (i = 0; i < params->len; ++i) { + if (strcmp (params->ptr[i].key, "mlock") == 0) { + int r = nbdkit_parse_bool (params->ptr[i].value); + if (r == -1) return NULL; +diff --git a/common/allocators/sparse.c b/common/allocators/sparse.c +index ca508c35..7c6d1636 100644 +--- a/common/allocators/sparse.c ++++ b/common/allocators/sparse.c +@@ -150,7 +150,7 @@ sparse_array_free (struct allocator *a) + size_t i; + + if (sa) { +- for (i = 0; i < sa->l1_dir.size; ++i) ++ for (i = 0; i < sa->l1_dir.len; ++i) + free_l2_dir (sa->l1_dir.ptr[i].l2_dir); + free (sa->l1_dir.ptr); + pthread_mutex_destroy (&sa->lock); +@@ -184,7 +184,7 @@ insert_l1_entry (struct sparse_array *sa, const struct l1_entry *entry) + { + size_t i; + +- for (i = 0; i < sa->l1_dir.size; ++i) { ++ for (i = 0; i < sa->l1_dir.len; ++i) { + if (entry->offset < sa->l1_dir.ptr[i].offset) { + /* Insert new entry before i'th directory entry. */ + if (l1_dir_insert (&sa->l1_dir, *entry, i) == -1) { +@@ -508,7 +508,7 @@ sparse_array_create (const void *paramsv) + const allocator_parameters *params = paramsv; + struct sparse_array *sa; + +- if (params->size > 0) { ++ if (params->len > 0) { + nbdkit_error ("allocator=sparse does not take extra parameters"); + return NULL; + } +diff --git a/common/allocators/zstd.c b/common/allocators/zstd.c +index 81fe4ed0..1675d21c 100644 +--- a/common/allocators/zstd.c ++++ b/common/allocators/zstd.c +@@ -136,7 +136,7 @@ zstd_array_free (struct allocator *a) + + ZSTD_freeCCtx (za->zcctx); + ZSTD_freeDStream (za->zdstrm); +- for (i = 0; i < za->l1_dir.size; ++i) ++ for (i = 0; i < za->l1_dir.len; ++i) + free_l2_dir (za->l1_dir.ptr[i].l2_dir); + free (za->l1_dir.ptr); + pthread_mutex_destroy (&za->lock); +@@ -170,7 +170,7 @@ insert_l1_entry (struct zstd_array *za, const struct l1_entry *entry) + { + size_t i; + +- for (i = 0; i < za->l1_dir.size; ++i) { ++ for (i = 0; i < za->l1_dir.len; ++i) { + if (entry->offset < za->l1_dir.ptr[i].offset) { + /* Insert new entry before i'th directory entry. */ + if (l1_dir_insert (&za->l1_dir, *entry, i) == -1) { +@@ -600,7 +600,7 @@ zstd_array_create (const void *paramsv) + const allocator_parameters *params = paramsv; + struct zstd_array *za; + +- if (params->size > 0) { ++ if (params->len > 0) { + nbdkit_error ("allocator=zstd does not take extra parameters"); + return NULL; + } +diff --git a/common/regions/regions.h b/common/regions/regions.h +index 13fc41e2..34a398cd 100644 +--- a/common/regions/regions.h ++++ b/common/regions/regions.h +@@ -84,17 +84,17 @@ extern void free_regions (regions *regions) + static inline size_t __attribute__((__nonnull__ (1))) + nr_regions (regions *rs) + { +- return rs->size; ++ return rs->len; + } + + /* Return the virtual size of the disk. */ + static inline int64_t __attribute__((__nonnull__ (1))) + virtual_size (regions *rs) + { +- if (rs->size == 0) ++ if (rs->len == 0) + return 0; + else +- return rs->ptr[rs->size-1].end + 1; ++ return rs->ptr[rs->len-1].end + 1; + } + + /* Look up the region corresponding to the given offset. If the +diff --git a/common/utils/environ.c b/common/utils/environ.c +index e70976cb..2ad996eb 100644 +--- a/common/utils/environ.c ++++ b/common/utils/environ.c +@@ -82,7 +82,7 @@ copy_environ (char **env, ...) + + /* Search for key in the existing environment. It's O(n^2) ... */ + len = strlen (key); +- for (i = 0; i < ret.size; ++i) { ++ for (i = 0; i < ret.len; ++i) { + if (strncmp (key, ret.ptr[i], len) == 0 && ret.ptr[i][len] == '=') { + /* Replace the existing key. */ + free (ret.ptr[i]); +diff --git a/common/utils/test-vector.c b/common/utils/test-vector.c +index 28af59b8..6d89a281 100644 +--- a/common/utils/test-vector.c ++++ b/common/utils/test-vector.c +@@ -73,7 +73,7 @@ test_int64_vector (void) + assert (v.ptr[i] == i); + + int64_vector_remove (&v, 1); +- assert (v.size == 9); ++ assert (v.len == 9); + assert (v.ptr[1] == 2); + + tmp = 10; +diff --git a/common/utils/vector.h b/common/utils/vector.h +index 782dcba6..1d04f812 100644 +--- a/common/utils/vector.h ++++ b/common/utils/vector.h +@@ -59,14 +59,14 @@ + * + * string_vector names = empty_vector; + * +- * where ‘names.ptr[]’ will be an array of strings and ‘names.size’ ++ * where ‘names.ptr[]’ will be an array of strings and ‘names.len’ + * will be the number of strings. There are no get/set accessors. To + * iterate over the strings you can use the ‘.ptr’ field directly: + * +- * for (size_t i = 0; i < names.size; ++i) ++ * for (size_t i = 0; i < names.len; ++i) + * printf ("%s\n", names.ptr[i]); + * +- * Initializing with ‘empty_vector’ sets ‘.ptr = NULL’ and ‘.size = 0’. ++ * Initializing with ‘empty_vector’ sets ‘.ptr = NULL’ and ‘.len = 0’. + * + * DEFINE_VECTOR_TYPE also defines utility functions. For the full + * list see the definition below, but useful functions include: +@@ -84,15 +84,15 @@ + */ + #define DEFINE_VECTOR_TYPE(name, type) \ + struct name { \ +- type *ptr; /* Pointer to array of items. */ \ +- size_t size; /* Number of valid items in the array. */ \ +- size_t cap; /* Maximum number of items. */ \ ++ type *ptr; /* Pointer to array of items. */ \ ++ size_t len; /* Number of valid items in the array. */ \ ++ size_t cap; /* Maximum number of items. */ \ + }; \ + typedef struct name name; \ + \ + /* Reserve n elements at the end of the vector. Note space is \ +- * allocated but the vector size is not increased and the new \ +- * elements are not initialized. \ ++ * allocated and capacity is increased, but the vector length \ ++ * is not increased and the new elements are not initialized. \ + */ \ + static inline int \ + name##_reserve (name *v, size_t n) \ +@@ -101,17 +101,17 @@ + sizeof (type)); \ + } \ + \ +- /* Insert at i'th element. i=0 => beginning i=size => append */ \ ++ /* Insert at i'th element. i=0 => beginning i=len => append */ \ + static inline int \ + name##_insert (name *v, type elem, size_t i) \ + { \ +- assert (i <= v->size); \ +- if (v->size >= v->cap) { \ ++ assert (i <= v->len); \ ++ if (v->len >= v->cap) { \ + if (name##_reserve (v, 1) == -1) return -1; \ + } \ +- memmove (&v->ptr[i+1], &v->ptr[i], (v->size-i) * sizeof (elem)); \ ++ memmove (&v->ptr[i+1], &v->ptr[i], (v->len-i) * sizeof (elem)); \ + v->ptr[i] = elem; \ +- v->size++; \ ++ v->len++; \ + return 0; \ + } \ + \ +@@ -119,16 +119,16 @@ + static inline int \ + name##_append (name *v, type elem) \ + { \ +- return name##_insert (v, elem, v->size); \ ++ return name##_insert (v, elem, v->len); \ + } \ + \ +- /* Remove i'th element. i=0 => beginning i=size-1 => end */ \ ++ /* Remove i'th element. i=0 => beginning i=len-1 => end */ \ + static inline void \ + name##_remove (name *v, size_t i) \ + { \ +- assert (i < v->size); \ +- memmove (&v->ptr[i], &v->ptr[i+1], (v->size-i-1) * sizeof (type)); \ +- v->size--; \ ++ assert (i < v->len); \ ++ memmove (&v->ptr[i], &v->ptr[i+1], (v->len-i-1) * sizeof (type)); \ ++ v->len--; \ + } \ + \ + /* Remove all elements and deallocate the vector. */ \ +@@ -137,7 +137,7 @@ + { \ + free (v->ptr); \ + v->ptr = NULL; \ +- v->size = v->cap = 0; \ ++ v->len = v->cap = 0; \ + } \ + \ + /* Iterate over the vector, calling f() on each element. */ \ +@@ -145,7 +145,7 @@ + name##_iter (name *v, void (*f) (type elem)) \ + { \ + size_t i; \ +- for (i = 0; i < v->size; ++i) \ ++ for (i = 0; i < v->len; ++i) \ + f (v->ptr[i]); \ + } \ + \ +@@ -154,7 +154,7 @@ + name##_sort (name *v, \ + int (*compare) (const type *p1, const type *p2)) \ + { \ +- qsort (v->ptr, v->size, sizeof (type), (void *) compare); \ ++ qsort (v->ptr, v->len, sizeof (type), (void *) compare); \ + } \ + \ + /* Search for an exactly matching element in the vector using a \ +@@ -164,7 +164,7 @@ + name##_search (const name *v, const void *key, \ + int (*compare) (const void *key, const type *v)) \ + { \ +- return bsearch (key, v->ptr, v->size, sizeof (type), \ ++ return bsearch (key, v->ptr, v->len, sizeof (type), \ + (void *) compare); \ + } \ + \ +@@ -175,22 +175,22 @@ + /* Note it's allowed for v and copy to be the same pointer. */ \ + type *vptr = v->ptr; \ + type *newptr; \ +- size_t len = v->size * sizeof (type); \ ++ size_t len = v->len * sizeof (type); \ + \ + newptr = malloc (len); \ + if (newptr == NULL) return -1; \ + memcpy (newptr, vptr, len); \ + copy->ptr = newptr; \ +- copy->size = copy->cap = v->size; \ ++ copy->len = copy->cap = v->len; \ + return 0; \ + } \ + \ + +-#define empty_vector { .ptr = NULL, .size = 0, .cap = 0 } ++#define empty_vector { .ptr = NULL, .len = 0, .cap = 0 } + + struct generic_vector { + void *ptr; +- size_t size; ++ size_t len; + size_t cap; + }; + +diff --git a/filters/ddrescue/ddrescue.c b/filters/ddrescue/ddrescue.c +index 7b1c9c1e..218c8ee5 100644 +--- a/filters/ddrescue/ddrescue.c ++++ b/filters/ddrescue/ddrescue.c +@@ -180,7 +180,7 @@ ddrescue_pread (nbdkit_next *next, + { + size_t i; + +- for (i = 0; i < map.ranges.size; i++) { ++ for (i = 0; i < map.ranges.len; i++) { + if (map.ranges.ptr[i].status != '+') + continue; + if (offset >= map.ranges.ptr[i].start && offset <= map.ranges.ptr[i].end) { +diff --git a/filters/exitwhen/exitwhen.c b/filters/exitwhen/exitwhen.c +index 543af058..83e99953 100644 +--- a/filters/exitwhen/exitwhen.c ++++ b/filters/exitwhen/exitwhen.c +@@ -143,7 +143,7 @@ check_for_event (void) + size_t i; + + if (!exiting) { +- for (i = 0; i < events.size; ++i) { ++ for (i = 0; i < events.len; ++i) { + const struct event *event = &events.ptr[i]; + + switch (event->type) { +diff --git a/filters/extentlist/extentlist.c b/filters/extentlist/extentlist.c +index 7e6f1b78..c91fbfea 100644 +--- a/filters/extentlist/extentlist.c ++++ b/filters/extentlist/extentlist.c +@@ -134,7 +134,7 @@ parse_extentlist (void) + + assert (extentlist != NULL); + assert (extents.ptr == NULL); +- assert (extents.size == 0); ++ assert (extents.len == 0); + + fp = fopen (extentlist, "r"); + if (!fp) { +@@ -200,7 +200,7 @@ parse_extentlist (void) + + /* There must not be overlaps at this point. */ + end = 0; +- for (i = 0; i < extents.size; ++i) { ++ for (i = 0; i < extents.len; ++i) { + if (extents.ptr[i].offset < end || + extents.ptr[i].offset + extents.ptr[i].length < extents.ptr[i].offset) { + nbdkit_error ("extents in the extent list are overlapping"); +@@ -210,8 +210,8 @@ parse_extentlist (void) + } + + /* If there's a gap at the beginning, insert a hole|zero extent. */ +- if (extents.size == 0 || extents.ptr[0].offset > 0) { +- end = extents.size == 0 ? UINT64_MAX : extents.ptr[0].offset; ++ if (extents.len == 0 || extents.ptr[0].offset > 0) { ++ end = extents.len == 0 ? UINT64_MAX : extents.ptr[0].offset; + if (extent_list_insert (&extents, + (struct extent){.offset = 0, .length = end, + .type = HOLE}, +@@ -224,7 +224,7 @@ parse_extentlist (void) + /* Now insert hole|zero extents after every extent where there + * is a gap between that extent and the next one. + */ +- for (i = 0; i < extents.size-1; ++i) { ++ for (i = 0; i < extents.len-1; ++i) { + end = extents.ptr[i].offset + extents.ptr[i].length; + if (end < extents.ptr[i+1].offset) + if (extent_list_insert (&extents, +@@ -238,7 +238,7 @@ parse_extentlist (void) + } + + /* If there's a gap at the end, insert a hole|zero extent. */ +- end = extents.ptr[extents.size-1].offset + extents.ptr[extents.size-1].length; ++ end = extents.ptr[extents.len-1].offset + extents.ptr[extents.len-1].length; + if (end < UINT64_MAX) { + if (extent_list_append (&extents, + (struct extent){.offset = end, +@@ -250,7 +250,7 @@ parse_extentlist (void) + } + + /* Debug the final list. */ +- for (i = 0; i < extents.size; ++i) { ++ for (i = 0; i < extents.len; ++i) { + nbdkit_debug ("extentlist: " + "extent[%zu] = %" PRIu64 "-%" PRIu64 " (length %" PRIu64 ")" + " type %" PRIu32, +diff --git a/filters/multi-conn/multi-conn.c b/filters/multi-conn/multi-conn.c +index a6a25ef9..c7421a39 100644 +--- a/filters/multi-conn/multi-conn.c ++++ b/filters/multi-conn/multi-conn.c +@@ -207,14 +207,14 @@ multi_conn_prepare (nbdkit_next *next, void *handle, int readonly) + ACQUIRE_LOCK_FOR_CURRENT_SCOPE (&lock); + if (byname) { + g = NULL; +- for (i = 0; i < groups.size; i++) ++ for (i = 0; i < groups.len; i++) + if (strcmp (groups.ptr[i]->name, h->name) == 0) { + g = groups.ptr[i]; + break; + } + } + else +- g = groups.size ? groups.ptr[0] : NULL; ++ g = groups.len ? groups.ptr[0] : NULL; + + if (!g) { + g = calloc (1, sizeof *g); +@@ -230,7 +230,7 @@ multi_conn_prepare (nbdkit_next *next, void *handle, int readonly) + } + if (conns_vector_append (&g->conns, h) == -1) { + if (new_group) { +- group_vector_remove (&groups, groups.size - 1); ++ group_vector_remove (&groups, groups.len - 1); + free (g->name); + free (g); + } +@@ -251,14 +251,14 @@ multi_conn_finalize (nbdkit_next *next, void *handle) + assert (h->group); + + /* XXX should we add a config param to flush if the client forgot? */ +- for (i = 0; i < h->group->conns.size; i++) { ++ for (i = 0; i < h->group->conns.len; i++) { + if (h->group->conns.ptr[i] == h) { + conns_vector_remove (&h->group->conns, i); + break; + } + } +- if (h->group->conns.size == 0) { +- for (i = 0; i < groups.size; i++) ++ if (h->group->conns.len == 0) { ++ for (i = 0; i < groups.len; i++) + if (groups.ptr[i] == h->group) { + group_vector_remove (&groups, i); + free (h->group->name); +@@ -451,7 +451,7 @@ multi_conn_flush (nbdkit_next *next, + assert (h->group); + if (h->mode == EMULATE) { + ACQUIRE_LOCK_FOR_CURRENT_SCOPE (&lock); +- for (i = 0; i < h->group->conns.size; i++) { ++ for (i = 0; i < h->group->conns.len; i++) { + h2 = h->group->conns.ptr[i]; + if (track == OFF || (h->group->dirty && + (track == FAST || h2->dirty & READ)) || +@@ -474,7 +474,7 @@ multi_conn_flush (nbdkit_next *next, + case CONN: + if (next->can_multi_conn (next) == 1) { + ACQUIRE_LOCK_FOR_CURRENT_SCOPE (&lock); +- for (i = 0; i < h->group->conns.size; i++) ++ for (i = 0; i < h->group->conns.len; i++) + h->group->conns.ptr[i]->dirty = 0; + h->group->dirty = 0; + } +diff --git a/plugins/cc/cc.c b/plugins/cc/cc.c +index 3251f312..c8c3c86b 100644 +--- a/plugins/cc/cc.c ++++ b/plugins/cc/cc.c +@@ -292,12 +292,12 @@ cc_config_complete (void) + if (subplugin.load) + subplugin.load (); + if (subplugin.config) { +- for (i = 0; i < params.size; ++i) { ++ for (i = 0; i < params.len; ++i) { + if (subplugin.config (params.ptr[i].key, params.ptr[i].value) == -1) + return -1; + } + } +- else if (params.size > 0) { ++ else if (params.len > 0) { + /* Just print the first one in the error message. */ + nbdkit_error ("unknown parameter: %s", params.ptr[0].key); + return -1; +diff --git a/plugins/data/data.c b/plugins/data/data.c +index 03bcc8a5..960cf97d 100644 +--- a/plugins/data/data.c ++++ b/plugins/data/data.c +@@ -158,7 +158,7 @@ get_extra_param (const char *name) + { + size_t i; + +- for (i = 0; i < params.size; ++i) { ++ for (i = 0; i < params.len; ++i) { + if (strcmp (params.ptr[i].key, name) == 0) + return params.ptr[i].value; + } +@@ -176,7 +176,7 @@ data_config_complete (void) + return -1; + } + +- if (data_seen != DATA && params.size != 0) { ++ if (data_seen != DATA && params.len != 0) { + nbdkit_error ("extra parameters passed and not using data='...'"); + return -1; + } +diff --git a/plugins/data/format.c b/plugins/data/format.c +index d351f79a..986a0f6e 100644 +--- a/plugins/data/format.c ++++ b/plugins/data/format.c +@@ -83,7 +83,7 @@ substring (string s, size_t offset, size_t len) + string r = empty_vector; + + for (i = 0; i < len; ++i) { +- assert (offset+i < s.size); ++ assert (offset+i < s.len); + if (string_append (&r, s.ptr[offset+i]) == -1) { + nbdkit_error ("realloc: %m"); + exit (EXIT_FAILURE); +@@ -167,7 +167,7 @@ static expr_list expr_table; + static node_id + new_node (const expr_t e) + { +- if (expr_table.size == 0) { ++ if (expr_table.len == 0) { + static const expr_t enull = { .t = EXPR_NULL }; + if (expr_list_append (&expr_table, enull) == -1) + goto out_of_memory; +@@ -179,14 +179,14 @@ new_node (const expr_t e) + nbdkit_error ("realloc"); + exit (EXIT_FAILURE); + } +- return expr_table.size-1; ++ return expr_table.len-1; + } + + /* Get an expression by node_id. */ + static expr_t + get_node (node_id id) + { +- assert (id < expr_table.size); ++ assert (id < expr_table.len); + return expr_table.ptr[id]; + } + +@@ -196,7 +196,7 @@ free_expr_table (void) + size_t i; + expr_t e; + +- for (i = 0; i < expr_table.size; ++i) { ++ for (i = 0; i < expr_table.len; ++i) { + e = get_node (i); + switch (e.t) { + case EXPR_LIST: free (e.list.ptr); break; +@@ -344,7 +344,7 @@ debug_expr (node_id id, int level) + break; + case EXPR_LIST: + nbdkit_debug ("%s(", debug_indent (level)); +- for (i = 0; i < e.list.size; ++i) ++ for (i = 0; i < e.list.len; ++i) + debug_expr (e.list.ptr[i], level+1); + nbdkit_debug ("%s)", debug_indent (level)); + break; +@@ -370,7 +370,7 @@ debug_expr (node_id id, int level) + CLEANUP_FREE_STRING string s = empty_vector; + static const char hex[] = "0123456789abcdef"; + +- for (i = 0; i < e.string.size; ++i) { ++ for (i = 0; i < e.string.len; ++i) { + char c = e.string.ptr[i]; + if (ascii_isprint ((char) c)) + string_append (&s, e.string.ptr[i]); +@@ -445,7 +445,7 @@ read_data_format (const char *value, struct allocator *a, uint64_t *size_rtn) + uint64_t offset = 0; + int r = -1; + +- assert (expr_table.size == 0); ++ assert (expr_table.len == 0); + + /* Run the parser across the entire string, returning the top level + * expression. +@@ -600,11 +600,11 @@ parser (int level, const char *value, size_t *start, size_t len, + + case '*': /* expr*N */ + i++; +- if (list.size == 0) { ++ if (list.len == 0) { + nbdkit_error ("*N must follow an expression"); + return -1; + } +- if (! is_data_expr (get_node (list.ptr[list.size-1]))) { ++ if (! is_data_expr (get_node (list.ptr[list.len-1]))) { + nbdkit_error ("*N cannot be applied to this type of expression"); + return -1; + } +@@ -619,18 +619,18 @@ parser (int level, const char *value, size_t *start, size_t len, + nbdkit_error ("*N not numeric"); + return -1; + } +- id = list.ptr[list.size-1]; +- list.size--; ++ id = list.ptr[list.len-1]; ++ list.len--; + APPEND_EXPR (new_node (expr (EXPR_REPEAT, id, (uint64_t) i64))); + break; + + case '[': /* expr[k:m] */ + i++; +- if (list.size == 0) { ++ if (list.len == 0) { + nbdkit_error ("[N:M] must follow an expression"); + return -1; + } +- if (! is_data_expr (get_node (list.ptr[list.size-1]))) { ++ if (! is_data_expr (get_node (list.ptr[list.len-1]))) { + nbdkit_error ("[N:M] cannot be applied to this type of expression"); + return -1; + } +@@ -647,8 +647,8 @@ parser (int level, const char *value, size_t *start, size_t len, + nbdkit_error ("enclosed pattern (...)[N:M] not numeric"); + return -1; + } +- id = list.ptr[list.size-1]; +- list.size--; ++ id = list.ptr[list.len-1]; ++ list.len--; + APPEND_EXPR (new_node (expr (EXPR_SLICE, id, i64, m))); + break; + +@@ -720,11 +720,11 @@ parser (int level, const char *value, size_t *start, size_t len, + i++; + if (value[i] != '>') goto parse_error; + i++; +- if (list.size == 0) { ++ if (list.len == 0) { + nbdkit_error ("-> must follow an expression"); + return -1; + } +- if (! is_data_expr (get_node (list.ptr[list.size-1]))) { ++ if (! is_data_expr (get_node (list.ptr[list.len-1]))) { + nbdkit_error ("-> cannot be applied to this type of expression"); + return -1; + } +@@ -735,9 +735,9 @@ parser (int level, const char *value, size_t *start, size_t len, + nbdkit_error ("strndup: %m"); + return -1; + } +- id = list.ptr[list.size-1]; ++ id = list.ptr[list.len-1]; + i += flen; +- list.size--; ++ list.len--; + APPEND_EXPR (new_node (expr (EXPR_ASSIGN, name, id))); + break; + } +@@ -1023,7 +1023,7 @@ parse_word (const char *value, size_t *start, size_t len, string *rtn) + } + memcpy (copy.ptr, &value[*start], n); + copy.ptr[n] = '\0'; +- copy.size = n + 1; ++ copy.len = n + 1; + *start = i; + + /* Reserve enough space in the return buffer for the longest +@@ -1036,22 +1036,22 @@ parse_word (const char *value, size_t *start, size_t len, string *rtn) + + /* Parse the rest of {le|be}{16|32|64}: */ + if (strncmp (copy.ptr, "le16:", 5) == 0) { +- endian = little; rtn->size = 2; ++ endian = little; rtn->len = 2; + } + else if (strncmp (copy.ptr, "le32:", 5) == 0) { +- endian = little; rtn->size = 4; ++ endian = little; rtn->len = 4; + } + else if (strncmp (copy.ptr, "le64:", 5) == 0) { +- endian = little; rtn->size = 8; ++ endian = little; rtn->len = 8; + } + else if (strncmp (copy.ptr, "be16:", 5) == 0) { +- endian = big; rtn->size = 2; ++ endian = big; rtn->len = 2; + } + else if (strncmp (copy.ptr, "be32:", 5) == 0) { +- endian = big; rtn->size = 4; ++ endian = big; rtn->len = 4; + } + else if (strncmp (copy.ptr, "be64:", 5) == 0) { +- endian = big; rtn->size = 8; ++ endian = big; rtn->len = 8; + } + else { + nbdkit_error ("data parameter: expected \"le16/32/64:\" " +@@ -1060,7 +1060,7 @@ parse_word (const char *value, size_t *start, size_t len, string *rtn) + } + + /* Parse the word field into a host-order unsigned int. */ +- switch (rtn->size) { ++ switch (rtn->len) { + case 2: + if (nbdkit_parse_uint16_t ("data", ©.ptr[5], &u16) == -1) + return -1; +@@ -1081,7 +1081,7 @@ parse_word (const char *value, size_t *start, size_t len, string *rtn) + */ + switch (endian) { + case little: +- switch (rtn->size) { ++ switch (rtn->len) { + case 2: /* le16: */ + *((uint16_t *) rtn->ptr) = htole16 (u16); + break; +@@ -1096,7 +1096,7 @@ parse_word (const char *value, size_t *start, size_t len, string *rtn) + break; + + case big: +- switch (rtn->size) { ++ switch (rtn->len) { + case 2: /* be16: */ + *((uint16_t *) rtn->ptr) = htobe16 (u16); + break; +@@ -1134,7 +1134,7 @@ optimize_ast (node_id root, node_id *root_rtn) + /* For convenience this makes a new list node. */ + + /* Optimize each element of the list. */ +- for (i = 0; i < get_node (root).list.size; ++i) { ++ for (i = 0; i < get_node (root).list.len; ++i) { + id = get_node (root).list.ptr[i]; + if (optimize_ast (id, &id) == -1) + return -1; +@@ -1148,7 +1148,7 @@ optimize_ast (node_id root, node_id *root_rtn) + * because flattening the list changes the scope. + */ + if (list_safe_to_inline (get_node (id).list)) { +- for (j = 0; j < get_node (id).list.size; ++j) { ++ for (j = 0; j < get_node (id).list.len; ++j) { + if (node_ids_append (&list, get_node (id).list.ptr[j]) == -1) + goto list_append_error; + } +@@ -1165,7 +1165,7 @@ optimize_ast (node_id root, node_id *root_rtn) + } + + /* Combine adjacent pairs of elements if possible. */ +- for (i = 1; i < list.size; ++i) { ++ for (i = 1; i < list.len; ++i) { + node_id id0, id1; + + id0 = list.ptr[i-1]; +@@ -1178,7 +1178,7 @@ optimize_ast (node_id root, node_id *root_rtn) + } + + /* List of length 0 is replaced with null. */ +- if (list.size == 0) { ++ if (list.len == 0) { + free (list.ptr); + *root_rtn = new_node (expr (EXPR_NULL)); + return 0; +@@ -1187,7 +1187,7 @@ optimize_ast (node_id root, node_id *root_rtn) + /* List of length 1 is replaced with the first element, but as + * above avoid inlining if it is not a safe expression. + */ +- if (list.size == 1 && expr_safe_to_inline (get_node (list.ptr[0]))) { ++ if (list.len == 1 && expr_safe_to_inline (get_node (list.ptr[0]))) { + id = list.ptr[0]; + free (list.ptr); + *root_rtn = id; +@@ -1242,13 +1242,13 @@ optimize_ast (node_id root, node_id *root_rtn) + */ + if (get_node (id).t == EXPR_STRING && + get_node (root).r.n <= 4 && +- get_node (id).string.size <= 512) { ++ get_node (id).string.len <= 512) { + string s = empty_vector; + size_t n = get_node (root).r.n; + const string sub = get_node (id).string; + + for (i = 0; i < n; ++i) { +- for (j = 0; j < sub.size; ++j) { ++ for (j = 0; j < sub.len; ++j) { + if (string_append (&s, sub.ptr[j]) == -1) { + nbdkit_error ("realloc: %m"); + return -1; +@@ -1307,7 +1307,7 @@ optimize_ast (node_id root, node_id *root_rtn) + } + break; + case EXPR_STRING: /* substring */ +- len = get_node (id).string.size; ++ len = get_node (id).string.len; + if (m >= 0 && n <= m && m <= len) { + if (m-n == 1) + *root_rtn = new_node (expr (EXPR_BYTE, get_node (id).string.ptr[n])); +@@ -1355,23 +1355,23 @@ optimize_ast (node_id root, node_id *root_rtn) + + case EXPR_STRING: + /* A zero length string can be replaced with null. */ +- if (get_node (root).string.size == 0) { ++ if (get_node (root).string.len == 0) { + *root_rtn = new_node (expr (EXPR_NULL)); + return 0; + } + /* Strings containing the same character can be replaced by a + * fill. These can be produced by other optimizations. + */ +- if (get_node (root).string.size > 1) { ++ if (get_node (root).string.len > 1) { + const string s = get_node (root).string; + uint8_t b = s.ptr[0]; + +- for (i = 1; i < s.size; ++i) ++ for (i = 1; i < s.len; ++i) + if (s.ptr[i] != b) + break; + +- if (i == s.size) { +- *root_rtn = new_node (expr (EXPR_FILL, b, (uint64_t) s.size)); ++ if (i == s.len) { ++ *root_rtn = new_node (expr (EXPR_FILL, b, (uint64_t) s.len)); + return 0; + } + } +@@ -1442,7 +1442,7 @@ list_safe_to_inline (const node_ids list) + { + size_t i; + +- for (i = 0; i < list.size; ++i) { ++ for (i = 0; i < list.len; ++i) { + if (!expr_safe_to_inline (get_node (list.ptr[i]))) + return false; + } +@@ -1461,11 +1461,11 @@ expr_is_single_byte (const expr_t e, uint8_t *b) + if (b) *b = e.b; + return true; + case EXPR_LIST: /* A single element list if it is single byte */ +- if (e.list.size != 1) ++ if (e.list.len != 1) + return false; + return expr_is_single_byte (get_node (e.list.ptr[0]), b); + case EXPR_STRING: /* A length-1 string. */ +- if (e.string.size != 1) ++ if (e.string.len != 1) + return false; + if (b) *b = e.string.ptr[0]; + return true; +@@ -1511,10 +1511,10 @@ exprs_can_combine (expr_t e0, expr_t e1, node_id *id_rtn) + } + return true; + case EXPR_STRING: /* byte string => string */ +- len = e1.string.size; ++ len = e1.string.len; + if (string_reserve (&s, len+1) == -1) + goto out_of_memory; +- s.size = len+1; ++ s.len = len+1; + s.ptr[0] = e0.b; + memcpy (&s.ptr[1], e1.string.ptr, len); + *id_rtn = new_node (expr (EXPR_STRING, s)); +@@ -1533,20 +1533,20 @@ exprs_can_combine (expr_t e0, expr_t e1, node_id *id_rtn) + case EXPR_STRING: + switch (e1.t) { + case EXPR_BYTE: /* string byte => string */ +- len = e0.string.size; ++ len = e0.string.len; + if (string_reserve (&s, len+1) == -1) + goto out_of_memory; +- s.size = len+1; ++ s.len = len+1; + memcpy (s.ptr, e0.string.ptr, len); + s.ptr[len] = e1.b; + *id_rtn = new_node (expr (EXPR_STRING, s)); + return true; + case EXPR_STRING: /* string string => string */ +- len = e0.string.size; +- len1 = e1.string.size; ++ len = e0.string.len; ++ len1 = e1.string.len; + if (string_reserve (&s, len+len1) == -1) + goto out_of_memory; +- s.size = len+len1; ++ s.len = len+len1; + memcpy (s.ptr, e0.string.ptr, len); + memcpy (&s.ptr[len], e1.string.ptr, len1); + *id_rtn = new_node (expr (EXPR_STRING, s)); +@@ -1618,11 +1618,11 @@ evaluate (const dict_t *dict, node_id root, + list = get_node (root).list; + } + else { +- list.size = 1; ++ list.len = 1; + list.ptr = &root; + } + +- for (i = 0; i < list.size; ++i) { ++ for (i = 0; i < list.len; ++i) { + const expr_t e = get_node (list.ptr[i]); + + switch (e.t) { +@@ -1667,9 +1667,9 @@ evaluate (const dict_t *dict, node_id root, + + case EXPR_STRING: + /* Copy the string into the allocator. */ +- if (a->f->write (a, e.string.ptr, e.string.size, *offset) == -1) ++ if (a->f->write (a, e.string.ptr, e.string.len, *offset) == -1) + return -1; +- *offset += e.string.size; ++ *offset += e.string.len; + break; + + case EXPR_FILL: +diff --git a/plugins/eval/eval.c b/plugins/eval/eval.c +index fa1c23ff..b312a59c 100644 +--- a/plugins/eval/eval.c ++++ b/plugins/eval/eval.c +@@ -114,7 +114,7 @@ insert_method_script (const char *method, char *script) + size_t i; + struct method_script new_entry = { .method = method, .script = script }; + +- for (i = 0; i < method_scripts.size; ++i) { ++ for (i = 0; i < method_scripts.len; ++i) { + r = compare_script (method, &method_scripts.ptr[i]); + /* This shouldn't happen. insert_method_script() must not be + * called if the method has already been added. Call get_script() +diff --git a/plugins/floppy/directory-lfn.c b/plugins/floppy/directory-lfn.c +index a87d376a..fe47e0b6 100644 +--- a/plugins/floppy/directory-lfn.c ++++ b/plugins/floppy/directory-lfn.c +@@ -79,8 +79,8 @@ create_directory (size_t di, const char *label, + struct virtual_floppy *floppy) + { + size_t i; +- const size_t nr_subdirs = floppy->dirs.ptr[di].subdirs.size; +- const size_t nr_files = floppy->dirs.ptr[di].fileidxs.size; ++ const size_t nr_subdirs = floppy->dirs.ptr[di].subdirs.len; ++ const size_t nr_files = floppy->dirs.ptr[di].fileidxs.len; + struct lfn *lfns, *lfn; + const char *name; + uint8_t attributes; +@@ -109,14 +109,14 @@ create_directory (size_t di, const char *label, + } + for (i = 0; i < nr_subdirs; ++i) { + const size_t sdi = floppy->dirs.ptr[di].subdirs.ptr[i]; +- assert (sdi < floppy->dirs.size); ++ assert (sdi < floppy->dirs.len); + + name = floppy->dirs.ptr[sdi].name; + lfns[i].name = name; + } + for (i = 0; i < nr_files; ++i) { + const size_t fi = floppy->dirs.ptr[di].fileidxs.ptr[i]; +- assert (fi < floppy->files.size); ++ assert (fi < floppy->files.len); + + name = floppy->files.ptr[fi].name; + lfns[nr_subdirs+i].name = name; +@@ -132,7 +132,7 @@ create_directory (size_t di, const char *label, + file_size = 0; + for (i = 0; i < nr_subdirs; ++i) { + const size_t sdi = floppy->dirs.ptr[di].subdirs.ptr[i]; +- assert (sdi < floppy->dirs.size); ++ assert (sdi < floppy->dirs.len); + + lfn = &lfns[i]; + statbuf = &floppy->dirs.ptr[sdi].statbuf; +@@ -148,7 +148,7 @@ create_directory (size_t di, const char *label, + attributes = DIR_ENTRY_ARCHIVE; /* Same as set by Linux kernel. */ + for (i = 0; i < nr_files; ++i) { + const size_t fi = floppy->dirs.ptr[di].fileidxs.ptr[i]; +- assert (fi < floppy->files.size); ++ assert (fi < floppy->files.len); + + lfn = &lfns[nr_subdirs+i]; + statbuf = &floppy->files.ptr[fi].statbuf; +@@ -532,7 +532,7 @@ append_dir_table (size_t di, const struct dir_entry *entry, + { + size_t i; + +- i = floppy->dirs.ptr[di].table.size; ++ i = floppy->dirs.ptr[di].table.len; + if (dir_entries_append (&floppy->dirs.ptr[di].table, *entry) == -1) { + nbdkit_error ("realloc: %m"); + return -1; +@@ -550,8 +550,8 @@ int + update_directory_first_cluster (size_t di, struct virtual_floppy *floppy) + { + size_t i, j, pdi; +- const size_t nr_subdirs = floppy->dirs.ptr[di].subdirs.size; +- const size_t nr_files = floppy->dirs.ptr[di].fileidxs.size; ++ const size_t nr_subdirs = floppy->dirs.ptr[di].subdirs.len; ++ const size_t nr_files = floppy->dirs.ptr[di].fileidxs.len; + uint32_t first_cluster; + struct dir_entry *entry; + +@@ -561,7 +561,7 @@ update_directory_first_cluster (size_t di, struct virtual_floppy *floppy) + * table entries. + */ + i = 0; +- for (j = 0; j < floppy->dirs.ptr[di].table.size; ++j) { ++ for (j = 0; j < floppy->dirs.ptr[di].table.len; ++j) { + entry = &floppy->dirs.ptr[di].table.ptr[j]; + + /* Skip LFN entries. */ +@@ -596,12 +596,12 @@ update_directory_first_cluster (size_t di, struct virtual_floppy *floppy) + */ + if (i < nr_subdirs) { + const size_t sdi = floppy->dirs.ptr[di].subdirs.ptr[i]; +- assert (sdi < floppy->dirs.size); ++ assert (sdi < floppy->dirs.len); + first_cluster = floppy->dirs.ptr[sdi].first_cluster; + } + else if (i < nr_subdirs + nr_files) { + const size_t fi = floppy->dirs.ptr[di].fileidxs.ptr[i-nr_subdirs]; +- assert (fi < floppy->files.size); ++ assert (fi < floppy->files.len); + first_cluster = floppy->files.ptr[fi].first_cluster; + } + else +diff --git a/plugins/floppy/floppy.c b/plugins/floppy/floppy.c +index 80f350af..938f5bec 100644 +--- a/plugins/floppy/floppy.c ++++ b/plugins/floppy/floppy.c +@@ -172,7 +172,7 @@ floppy_pread (void *handle, void *buf, uint32_t count, uint64_t offset) + switch (region->type) { + case region_file: + i = region->u.i; +- assert (i < floppy.files.size); ++ assert (i < floppy.files.len); + host_path = floppy.files.ptr[i].host_path; + fd = open (host_path, O_RDONLY|O_CLOEXEC); + if (fd == -1) { +diff --git a/plugins/floppy/virtual-floppy.c b/plugins/floppy/virtual-floppy.c +index 6eae5600..b1546bd5 100644 +--- a/plugins/floppy/virtual-floppy.c ++++ b/plugins/floppy/virtual-floppy.c +@@ -97,10 +97,10 @@ create_virtual_floppy (const char *dir, const char *label, uint64_t size, + return -1; + + nbdkit_debug ("floppy: %zu directories and %zu files", +- floppy->dirs.size, floppy->files.size); ++ floppy->dirs.len, floppy->files.len); + + /* Create the on disk directory tables. */ +- for (i = 0; i < floppy->dirs.size; ++i) { ++ for (i = 0; i < floppy->dirs.len; ++i) { + if (create_directory (i, label, floppy) == -1) + return -1; + } +@@ -115,10 +115,10 @@ create_virtual_floppy (const char *dir, const char *label, uint64_t size, + */ + data_used_size = 0; + cluster = 2; +- for (i = 0; i < floppy->dirs.size; ++i) { ++ for (i = 0; i < floppy->dirs.len; ++i) { + floppy->dirs.ptr[i].first_cluster = cluster; + nr_bytes = +- ROUND_UP (floppy->dirs.ptr[i].table.size * sizeof (struct dir_entry), ++ ROUND_UP (floppy->dirs.ptr[i].table.len * sizeof (struct dir_entry), + CLUSTER_SIZE); + data_used_size += nr_bytes; + nr_clusters = nr_bytes / CLUSTER_SIZE; +@@ -127,7 +127,7 @@ create_virtual_floppy (const char *dir, const char *label, uint64_t size, + floppy->dirs.ptr[i].nr_clusters = nr_clusters; + cluster += nr_clusters; + } +- for (i = 0; i < floppy->files.size; ++i) { ++ for (i = 0; i < floppy->files.len; ++i) { + floppy->files.ptr[i].first_cluster = cluster; + nr_bytes = ROUND_UP (floppy->files.ptr[i].statbuf.st_size, CLUSTER_SIZE); + data_used_size += nr_bytes; +@@ -187,7 +187,7 @@ create_virtual_floppy (const char *dir, const char *label, uint64_t size, + * directory entries (which we didn't have available during + * create_directory above). + */ +- for (i = 0; i < floppy->dirs.size; ++i) { ++ for (i = 0; i < floppy->dirs.len; ++i) { + if (update_directory_first_cluster (i, floppy) == -1) + return -1; + } +@@ -230,13 +230,13 @@ free_virtual_floppy (struct virtual_floppy *floppy) + + free (floppy->fat); + +- for (i = 0; i < floppy->files.size; ++i) { ++ for (i = 0; i < floppy->files.len; ++i) { + free (floppy->files.ptr[i].name); + free (floppy->files.ptr[i].host_path); + } + free (floppy->files.ptr); + +- for (i = 0; i < floppy->dirs.size; ++i) { ++ for (i = 0; i < floppy->dirs.len; ++i) { + free (floppy->dirs.ptr[i].name); + free (floppy->dirs.ptr[i].subdirs.ptr); + free (floppy->dirs.ptr[i].fileidxs.ptr); +@@ -266,7 +266,7 @@ visit (const char *dir, struct virtual_floppy *floppy) + * directory will always be at dirs[0]. + */ + memset (&null_dir, 0, sizeof null_dir); +- di = floppy->dirs.size; ++ di = floppy->dirs.len; + if (dirs_append (&floppy->dirs, null_dir) == -1) { + nbdkit_error ("realloc: %m"); + goto error0; +@@ -423,7 +423,7 @@ visit_file (const char *dir, const char *name, + } + new_file.host_path = host_path; + new_file.statbuf = *statbuf; +- fi = floppy->files.size; ++ fi = floppy->files.len; + if (files_append (&floppy->files, new_file) == -1) { + nbdkit_error ("realloc: %m"); + free (host_path); +@@ -574,11 +574,11 @@ create_fat (struct virtual_floppy *floppy) + floppy->fat[0] = htole32 (0x0ffffff8); + floppy->fat[1] = htole32 (0x0fffffff); + +- for (i = 0; i < floppy->dirs.size; ++i) { ++ for (i = 0; i < floppy->dirs.len; ++i) { + write_fat_file (floppy->dirs.ptr[i].first_cluster, + floppy->dirs.ptr[i].nr_clusters, floppy); + } +- for (i = 0; i < floppy->files.size; ++i) { ++ for (i = 0; i < floppy->files.len; ++i) { + write_fat_file (floppy->files.ptr[i].first_cluster, + floppy->files.ptr[i].nr_clusters, floppy); + } +@@ -676,15 +676,15 @@ create_regions (struct virtual_floppy *floppy) + /* Now we're into the data region. We add all directory tables + * first. + */ +- for (i = 0; i < floppy->dirs.size; ++i) { ++ for (i = 0; i < floppy->dirs.len; ++i) { + /* Directories can never be completely empty because of the volume + * label (root) or "." and ".." entries (non-root). + */ +- assert (floppy->dirs.ptr[i].table.size > 0); ++ assert (floppy->dirs.ptr[i].table.len > 0); + + if (append_region_len (&floppy->regions, + i == 0 ? "root directory" : floppy->dirs.ptr[i].name, +- floppy->dirs.ptr[i].table.size * ++ floppy->dirs.ptr[i].table.len * + sizeof (struct dir_entry), + 0, CLUSTER_SIZE, + region_data, +@@ -693,7 +693,7 @@ create_regions (struct virtual_floppy *floppy) + } + + /* Add all files. */ +- for (i = 0; i < floppy->files.size; ++i) { ++ for (i = 0; i < floppy->files.len; ++i) { + /* It's possible for a file to have zero size, in which case it + * doesn't occupy a region or cluster. + */ +diff --git a/plugins/iso/iso.c b/plugins/iso/iso.c +index cb621f41..e232175f 100644 +--- a/plugins/iso/iso.c ++++ b/plugins/iso/iso.c +@@ -105,7 +105,7 @@ make_iso (void) + fprintf (fp, " -quiet"); + if (params) + fprintf (fp, " %s", params); +- for (i = 0; i < dirs.size; ++i) { ++ for (i = 0; i < dirs.len; ++i) { + fputc (' ', fp); + shell_quote (dirs.ptr[i], fp); + } +@@ -169,7 +169,7 @@ iso_config (const char *key, const char *value) + static int + iso_config_complete (void) + { +- if (dirs.size == 0) { ++ if (dirs.len == 0) { + nbdkit_error ("you must supply the dir= parameter " + "after the plugin name on the command line"); + return -1; +diff --git a/plugins/nbd/nbd.c b/plugins/nbd/nbd.c +index 488eadb2..ae595ea7 100644 +--- a/plugins/nbd/nbd.c ++++ b/plugins/nbd/nbd.c +@@ -244,7 +244,7 @@ static int + nbdplug_config_complete (void) + { + int c = !!sockname + !!hostname + !!uri + +- (command.size > 0) + (socket_fd >= 0) + !!raw_cid; ++ (command.len > 0) + (socket_fd >= 0) + !!raw_cid; + + /* Check the user passed exactly one connection parameter. */ + if (c > 1) { +@@ -303,7 +303,7 @@ nbdplug_config_complete (void) + return -1; + #endif + } +- else if (command.size > 0) { ++ else if (command.len > 0) { + /* Add NULL sentinel to the command. */ + if (string_vector_append (&command, NULL) == -1) { + nbdkit_error ("realloc: %m"); +@@ -574,7 +574,7 @@ nbdplug_connect (struct nbd_handle *nbd) + #else + return nbd_connect_vsock (nbd, cid, vport); + #endif +- else if (command.size > 0) ++ else if (command.len > 0) + return nbd_connect_systemd_socket_activation (nbd, (char **) command.ptr); + else if (socket_fd >= 0) + return nbd_connect_socket (nbd, socket_fd); +diff --git a/plugins/partitioning/partition-mbr.c b/plugins/partitioning/partition-mbr.c +index 9a1a043c..1f178dcb 100644 +--- a/plugins/partitioning/partition-mbr.c ++++ b/plugins/partitioning/partition-mbr.c +@@ -62,9 +62,9 @@ create_mbr_layout (void) + primary[0x1fe] = 0x55; + primary[0x1ff] = 0xaa; + +- if (the_files.size <= 4) { ++ if (the_files.len <= 4) { + /* Basic MBR with no extended partition. */ +- for (i = 0; i < the_files.size; ++i) { ++ for (i = 0; i < the_files.len; ++i) { + const struct region *region = find_file_region (i, &j); + + create_mbr_partition_table_entry (region, i == 0, the_files.ptr[i].mbr_id, +@@ -97,7 +97,7 @@ create_mbr_layout (void) + /* The remaining files are mapped to logical partitions living in + * the fourth extended partition. + */ +- for (i = 3; i < the_files.size; ++i) { ++ for (i = 3; i < the_files.len; ++i) { + if (i == 3) + eptr = eptr0; + else +@@ -117,7 +117,7 @@ create_mbr_layout (void) + create_mbr_partition_table_entry (®ion, false, the_files.ptr[i].mbr_id, + &ebr[i-3][0x1be]); + +- if (i < the_files.size-1) { ++ if (i < the_files.len-1) { + size_t j2 = j; + const struct region *enext = find_ebr_region (i+1, &j2); + const struct region *rnext = find_file_region (i+1, &j2); +diff --git a/plugins/partitioning/partitioning.c b/plugins/partitioning/partitioning.c +index 231b2d77..2301ba4e 100644 +--- a/plugins/partitioning/partitioning.c ++++ b/plugins/partitioning/partitioning.c +@@ -104,7 +104,7 @@ partitioning_unload (void) + { + size_t i; + +- for (i = 0; i < the_files.size; ++i) ++ for (i = 0; i < the_files.len; ++i) + close (the_files.ptr[i].fd); + free (the_files.ptr); + +@@ -116,7 +116,7 @@ partitioning_unload (void) + free (primary); + free (secondary); + if (ebr) { +- for (i = 0; i < the_files.size-3; ++i) ++ for (i = 0; i < the_files.len-3; ++i) + free (ebr[i]); + free (ebr); + } +@@ -235,19 +235,19 @@ partitioning_config_complete (void) + bool needs_gpt; + + /* Not enough / too many files? */ +- if (the_files.size == 0) { ++ if (the_files.len == 0) { + nbdkit_error ("at least one file= parameter must be supplied"); + return -1; + } + + total_size = 0; +- for (i = 0; i < the_files.size; ++i) ++ for (i = 0; i < the_files.len; ++i) + total_size += the_files.ptr[i].statbuf.st_size; + needs_gpt = total_size > MAX_MBR_DISK_SIZE; + + /* Choose default parttype if not set. */ + if (parttype == PARTTYPE_UNSET) { +- if (needs_gpt || the_files.size > 4) { ++ if (needs_gpt || the_files.len > 4) { + parttype = PARTTYPE_GPT; + nbdkit_debug ("picking partition type GPT"); + } +@@ -262,7 +262,7 @@ partitioning_config_complete (void) + "but you requested %zu partition(s) " + "and a total size of %" PRIu64 " bytes (> %" PRIu64 "). " + "Try using: partition-type=gpt", +- the_files.size, total_size, (uint64_t) MAX_MBR_DISK_SIZE); ++ the_files.len, total_size, (uint64_t) MAX_MBR_DISK_SIZE); + return -1; + } + +@@ -327,7 +327,7 @@ partitioning_pread (void *handle, void *buf, uint32_t count, uint64_t offset) + switch (region->type) { + case region_file: + i = region->u.i; +- assert (i < the_files.size); ++ assert (i < the_files.len); + r = pread (the_files.ptr[i].fd, buf, len, offset - region->start); + if (r == -1) { + nbdkit_error ("pread: %s: %m", the_files.ptr[i].filename); +@@ -376,7 +376,7 @@ partitioning_pwrite (void *handle, + switch (region->type) { + case region_file: + i = region->u.i; +- assert (i < the_files.size); ++ assert (i < the_files.len); + r = pwrite (the_files.ptr[i].fd, buf, len, offset - region->start); + if (r == -1) { + nbdkit_error ("pwrite: %s: %m", the_files.ptr[i].filename); +@@ -418,7 +418,7 @@ partitioning_flush (void *handle) + { + size_t i; + +- for (i = 0; i < the_files.size; ++i) { ++ for (i = 0; i < the_files.len; ++i) { + if (fdatasync (the_files.ptr[i].fd) == -1) { + nbdkit_error ("fdatasync: %m"); + return -1; +diff --git a/plugins/partitioning/virtual-disk.c b/plugins/partitioning/virtual-disk.c +index 389a17b6..d46ca46a 100644 +--- a/plugins/partitioning/virtual-disk.c ++++ b/plugins/partitioning/virtual-disk.c +@@ -56,7 +56,7 @@ create_virtual_disk_layout (void) + size_t i; + + assert (nr_regions (&the_regions) == 0); +- assert (the_files.size > 0); ++ assert (the_files.len > 0); + assert (primary == NULL); + assert (secondary == NULL); + +@@ -68,17 +68,17 @@ create_virtual_disk_layout (void) + return -1; + } + +- if (the_files.size > 4) { ++ if (the_files.len > 4) { + /* The first 3 primary partitions will be real partitions, the + * 4th will be an extended partition, and so we need to store + * EBRs for the_files.size-3 logical partitions. + */ +- ebr = malloc (sizeof (unsigned char *) * (the_files.size-3)); ++ ebr = malloc (sizeof (unsigned char *) * (the_files.len-3)); + if (ebr == NULL) { + nbdkit_error ("malloc: %m"); + return -1; + } +- for (i = 0; i < the_files.size-3; ++i) { ++ for (i = 0; i < the_files.len-3; ++i) { + ebr[i] = calloc (1, SECTOR_SIZE); + if (ebr[i] == NULL) { + nbdkit_error ("malloc: %m"); +@@ -117,7 +117,7 @@ create_virtual_disk_layout (void) + } + + /* The partitions. */ +- for (i = 0; i < the_files.size; ++i) { ++ for (i = 0; i < the_files.len; ++i) { + uint64_t offset; + + offset = virtual_size (&the_regions); +@@ -127,7 +127,7 @@ create_virtual_disk_layout (void) + assert (IS_ALIGNED (offset, SECTOR_SIZE)); + + /* Logical partitions are preceeded by an EBR. */ +- if (parttype == PARTTYPE_MBR && the_files.size > 4 && i >= 3) { ++ if (parttype == PARTTYPE_MBR && the_files.len > 4 && i >= 3) { + if (append_region_len (&the_regions, "EBR", + SECTOR_SIZE, 0, 0, + region_data, ebr[i-3]) == -1) +diff --git a/plugins/partitioning/virtual-disk.h b/plugins/partitioning/virtual-disk.h +index 7032dfc8..d56c2b86 100644 +--- a/plugins/partitioning/virtual-disk.h ++++ b/plugins/partitioning/virtual-disk.h +@@ -55,7 +55,7 @@ + * 32 if the number of files is <= GPT_MIN_PARTITIONS, which is the + * normal case. + */ +-#define GPT_PTA_SIZE ROUND_UP (the_files.size, GPT_MIN_PARTITIONS) ++#define GPT_PTA_SIZE ROUND_UP (the_files.len, GPT_MIN_PARTITIONS) + #define GPT_PTA_LBAs (GPT_PTA_SIZE * GPT_PT_ENTRY_SIZE / SECTOR_SIZE) + + /* Maximum possible and default alignment between partitions. */ +diff --git a/plugins/split/split.c b/plugins/split/split.c +index c559a0cd..4c9790a6 100644 +--- a/plugins/split/split.c ++++ b/plugins/split/split.c +@@ -121,13 +121,13 @@ split_open (int readonly) + return NULL; + } + +- h->files = malloc (filenames.size * sizeof (struct file)); ++ h->files = malloc (filenames.len * sizeof (struct file)); + if (h->files == NULL) { + nbdkit_error ("malloc: %m"); + free (h); + return NULL; + } +- for (i = 0; i < filenames.size; ++i) ++ for (i = 0; i < filenames.len; ++i) + h->files[i].fd = -1; + + /* Open the files. */ +@@ -137,7 +137,7 @@ split_open (int readonly) + else + flags |= O_RDWR; + +- for (i = 0; i < filenames.size; ++i) { ++ for (i = 0; i < filenames.len; ++i) { + h->files[i].fd = open (filenames.ptr[i], flags); + if (h->files[i].fd == -1) { + nbdkit_error ("open: %s: %m", filenames.ptr[i]); +@@ -146,7 +146,7 @@ split_open (int readonly) + } + + offset = 0; +- for (i = 0; i < filenames.size; ++i) { ++ for (i = 0; i < filenames.len; ++i) { + h->files[i].offset = offset; + + if (fstat (h->files[i].fd, &statbuf) == -1) { +@@ -179,7 +179,7 @@ split_open (int readonly) + return h; + + err: +- for (i = 0; i < filenames.size; ++i) { ++ for (i = 0; i < filenames.len; ++i) { + if (h->files[i].fd >= 0) + close (h->files[i].fd); + } +@@ -195,7 +195,7 @@ split_close (void *handle) + struct handle *h = handle; + size_t i; + +- for (i = 0; i < filenames.size; ++i) ++ for (i = 0; i < filenames.len; ++i) + close (h->files[i].fd); + free (h->files); + free (h); +@@ -242,7 +242,7 @@ static struct file * + get_file (struct handle *h, uint64_t offset) + { + return bsearch (&offset, h->files, +- filenames.size, sizeof (struct file), ++ filenames.len, sizeof (struct file), + compare_offset); + } + +diff --git a/plugins/ssh/ssh.c b/plugins/ssh/ssh.c +index 535caf1a..80623525 100644 +--- a/plugins/ssh/ssh.c ++++ b/plugins/ssh/ssh.c +@@ -397,7 +397,7 @@ ssh_open (int readonly) + * as this file is rarely present. + */ + } +- for (i = 0; i < identities.size; ++i) { ++ for (i = 0; i < identities.len; ++i) { + r = ssh_options_set (h->session, + SSH_OPTIONS_ADD_IDENTITY, identities.ptr[i]); + if (r != SSH_OK) { +diff --git a/plugins/vddk/reexec.c b/plugins/vddk/reexec.c +index 9e87025e..4eae2221 100644 +--- a/plugins/vddk/reexec.c ++++ b/plugins/vddk/reexec.c +@@ -116,20 +116,20 @@ perform_reexec (const char *env, const char *prepend) + nbdkit_error ("realloc: %m"); + exit (EXIT_FAILURE); + } +- r = read (fd, buf.ptr + buf.size, buf.cap - buf.size); ++ r = read (fd, buf.ptr + buf.len, buf.cap - buf.len); + if (r == -1) { + nbdkit_error ("read: %s: %m", cmdline_file); + exit (EXIT_FAILURE); + } + if (r == 0) + break; +- buf.size += r; ++ buf.len += r; + } + close (fd); +- nbdkit_debug ("original command line occupies %zu bytes", buf.size); ++ nbdkit_debug ("original command line occupies %zu bytes", buf.len); + + /* Split cmdline into argv, then append one more arg. */ +- for (len = 0; len < buf.size; len += strlen (buf.ptr + len) + 1) { ++ for (len = 0; len < buf.len; len += strlen (buf.ptr + len) + 1) { + char *arg = buf.ptr + len; /* Next \0-terminated argument. */ + + /* See below for why we eat password parameter(s). */ +diff --git a/plugins/vddk/stats.c b/plugins/vddk/stats.c +index 76e0c244..bb5401b1 100644 +--- a/plugins/vddk/stats.c ++++ b/plugins/vddk/stats.c +@@ -94,12 +94,12 @@ display_stats (void) + #undef STUB + #undef OPTIONAL_STUB + +- qsort (stats.ptr, stats.size, sizeof stats.ptr[0], stat_compare); ++ qsort (stats.ptr, stats.len, sizeof stats.ptr[0], stat_compare); + + nbdkit_debug ("VDDK function stats (-D vddk.stats=1):"); + nbdkit_debug ("%-24s %15s %5s %15s", + "VixDiskLib_...", "µs", "calls", "bytes"); +- for (i = 0; i < stats.size; ++i) { ++ for (i = 0; i < stats.len; ++i) { + if (stats.ptr[i].usecs) { + if (stats.ptr[i].bytes > 0) + nbdkit_debug (" %-22s %15" PRIi64 " %5" PRIu64 " %15" PRIu64, +diff --git a/plugins/vddk/worker.c b/plugins/vddk/worker.c +index 2a1d4f26..c6e2fd22 100644 +--- a/plugins/vddk/worker.c ++++ b/plugins/vddk/worker.c +@@ -82,7 +82,7 @@ send_command_and_wait (struct vddk_handle *h, struct command *cmd) + return -1; + + /* Signal the caller if it could be sleeping on an empty queue. */ +- if (h->commands.size == 1) ++ if (h->commands.len == 1) + pthread_cond_signal (&h->commands_cond); + + /* This will be used to signal command completion back to us. */ +@@ -497,7 +497,7 @@ vddk_worker_thread (void *handle) + /* Wait until we are sent at least one command. */ + { + ACQUIRE_LOCK_FOR_CURRENT_SCOPE (&h->commands_lock); +- while (h->commands.size == 0) ++ while (h->commands.len == 0) + pthread_cond_wait (&h->commands_cond, &h->commands_lock); + cmd = h->commands.ptr[0]; + command_queue_remove (&h->commands, 0); +diff --git a/server/exports.c b/server/exports.c +index 7ce1eda9..12c8a879 100644 +--- a/server/exports.c ++++ b/server/exports.c +@@ -90,13 +90,13 @@ nbdkit_exports_free (struct nbdkit_exports *exps) + NBDKIT_DLL_PUBLIC size_t + nbdkit_exports_count (const struct nbdkit_exports *exps) + { +- return exps->exports.size; ++ return exps->exports.len; + } + + NBDKIT_DLL_PUBLIC const struct nbdkit_export + nbdkit_get_export (const struct nbdkit_exports *exps, size_t i) + { +- assert (i < exps->exports.size); ++ assert (i < exps->exports.len); + return exps->exports.ptr[i]; + } + +@@ -106,7 +106,7 @@ nbdkit_add_export (struct nbdkit_exports *exps, + { + struct nbdkit_export e = { NULL, NULL }; + +- if (exps->exports.size == MAX_EXPORTS) { ++ if (exps->exports.len == MAX_EXPORTS) { + nbdkit_error ("nbdkit_add_export: too many exports"); + errno = EINVAL; + return -1; +diff --git a/server/extents.c b/server/extents.c +index 8da82cf1..e180e313 100644 +--- a/server/extents.c ++++ b/server/extents.c +@@ -117,13 +117,13 @@ nbdkit_extents_free (struct nbdkit_extents *exts) + NBDKIT_DLL_PUBLIC size_t + nbdkit_extents_count (const struct nbdkit_extents *exts) + { +- return exts->extents.size; ++ return exts->extents.len; + } + + NBDKIT_DLL_PUBLIC struct nbdkit_extent + nbdkit_get_extent (const struct nbdkit_extents *exts, size_t i) + { +- assert (i < exts->extents.size); ++ assert (i < exts->extents.len); + return exts->extents.ptr[i]; + } + +@@ -160,7 +160,7 @@ nbdkit_add_extent (struct nbdkit_extents *exts, + return 0; + + /* Ignore extents beyond the end of the range, or if list is full. */ +- if (offset >= exts->end || exts->extents.size >= MAX_EXTENTS) ++ if (offset >= exts->end || exts->extents.len >= MAX_EXTENTS) + return 0; + + /* Shorten extents that overlap the end of the range. */ +@@ -169,7 +169,7 @@ nbdkit_add_extent (struct nbdkit_extents *exts, + length -= overlap; + } + +- if (exts->extents.size == 0) { ++ if (exts->extents.len == 0) { + /* If there are no existing extents, and the new extent is + * entirely before start, ignore it. + */ +@@ -196,10 +196,10 @@ nbdkit_add_extent (struct nbdkit_extents *exts, + } + + /* If we get here we are going to either add or extend. */ +- if (exts->extents.size > 0 && +- exts->extents.ptr[exts->extents.size-1].type == type) { ++ if (exts->extents.len > 0 && ++ exts->extents.ptr[exts->extents.len-1].type == type) { + /* Coalesce with the last extent. */ +- exts->extents.ptr[exts->extents.size-1].length += length; ++ exts->extents.ptr[exts->extents.len-1].length += length; + return 0; + } + else { +@@ -226,13 +226,13 @@ nbdkit_extents_aligned (struct context *next_c, + /* Perform an initial query, then scan for the first unaligned extent. */ + if (next->extents (next_c, count, offset, flags, exts, err) == -1) + return -1; +- for (i = 0; i < exts->extents.size; ++i) { ++ for (i = 0; i < exts->extents.len; ++i) { + e = &exts->extents.ptr[i]; + if (!IS_ALIGNED(e->length, align)) { + /* If the unalignment is past align, just truncate and return early */ + if (e->offset + e->length > offset + align) { + e->length = ROUND_DOWN (e->length, align); +- exts->extents.size = i + !!e->length; ++ exts->extents.len = i + !!e->length; + exts->next = e->offset + e->length; + break; + } +@@ -249,7 +249,7 @@ nbdkit_extents_aligned (struct context *next_c, + */ + assert (i == 0); + while (e->length < align) { +- if (exts->extents.size > 1) { ++ if (exts->extents.len > 1) { + e->length += exts->extents.ptr[1].length; + e->type &= exts->extents.ptr[1].type; + extents_remove (&exts->extents, 1); +@@ -284,7 +284,7 @@ nbdkit_extents_aligned (struct context *next_c, + } + } + e->length = align; +- exts->extents.size = 1; ++ exts->extents.len = 1; + exts->next = e->offset + e->length; + break; + } +diff --git a/server/main.c b/server/main.c +index 5fd8308f..225258de 100644 +--- a/server/main.c ++++ b/server/main.c +@@ -940,7 +940,7 @@ start_serving (void) + r = sockets_append (&socks, s); + assert (r == 0); + } +- debug ("using socket activation, nr_socks = %zu", socks.size); ++ debug ("using socket activation, nr_socks = %zu", socks.len); + change_user (); + write_pidfile (); + top->after_fork (top); +diff --git a/server/sockets.c b/server/sockets.c +index 95fce484..f13f8600 100644 +--- a/server/sockets.c ++++ b/server/sockets.c +@@ -246,14 +246,14 @@ bind_tcpip_socket (sockets *socks) + + freeaddrinfo (ai); + +- if (socks->size == 0 && addr_in_use) { ++ if (socks->len == 0 && addr_in_use) { + fprintf (stderr, "%s: unable to bind to any sockets: %s\n", + program_name, strerror (EADDRINUSE)); + exit (EXIT_FAILURE); + } + + debug ("bound to IP address %s:%s (%zu socket(s))", +- ipaddr ? ipaddr : "", port, socks->size); ++ ipaddr ? ipaddr : "", port, socks->len); + } + + void +@@ -443,7 +443,7 @@ accept_connection (int listen_sock) + static void + check_sockets_and_quit_fd (const sockets *socks) + { +- const size_t nr_socks = socks->size; ++ const size_t nr_socks = socks->len; + size_t i; + int r; + +@@ -552,7 +552,7 @@ accept_incoming_connections (const sockets *socks) + } + pthread_mutex_unlock (&count_mutex); + +- for (i = 0; i < socks->size; ++i) ++ for (i = 0; i < socks->len; ++i) + closesocket (socks->ptr[i]); + free (socks->ptr); + } +diff --git a/wrapper.c b/wrapper.c +index 3bab2074..87e5a033 100644 +--- a/wrapper.c ++++ b/wrapper.c +@@ -130,9 +130,9 @@ print_command (void) + { + size_t i; + +- if (cmd.size > 0) ++ if (cmd.len > 0) + shell_quote (cmd.ptr[0], stderr); +- for (i = 1; i < cmd.size && cmd.ptr[i] != NULL; ++i) { ++ for (i = 1; i < cmd.len && cmd.ptr[i] != NULL; ++i) { + fputc (' ', stderr); + shell_quote (cmd.ptr[i], stderr); + } +-- +2.31.1 + diff --git a/SOURCES/0022-vddk-Add-stats-about-the-amount-of-time-spent-in-VDD.patch b/SOURCES/0022-vddk-Add-stats-about-the-amount-of-time-spent-in-VDD.patch deleted file mode 100644 index dd18103..0000000 --- a/SOURCES/0022-vddk-Add-stats-about-the-amount-of-time-spent-in-VDD.patch +++ /dev/null @@ -1,245 +0,0 @@ -From 45db64d72bf03fece8a7fb994887360954905a3b Mon Sep 17 00:00:00 2001 -From: "Richard W.M. Jones" -Date: Wed, 18 Aug 2021 14:47:58 +0100 -Subject: [PATCH] vddk: Add stats about the amount of time spent in VDDK calls -MIME-Version: 1.0 -Content-Type: text/plain; charset=UTF-8 -Content-Transfer-Encoding: 8bit - -If you use -D vddk.stats=1 then when the plugin is unloaded it will -display the amount of time spent in each VDDK call. The output will -look something like this: - -nbdkit: debug: VDDK function stats (-D vddk.stats=1): -nbdkit: debug: µs -nbdkit: debug: VixDiskLib_Exit 1001154 -nbdkit: debug: VixDiskLib_InitEx 96008 -nbdkit: debug: VixDiskLib_Flush 15722 -nbdkit: debug: VixDiskLib_Write 12081 -nbdkit: debug: VixDiskLib_Open 6029 -nbdkit: debug: VixDiskLib_Read 1364 -nbdkit: debug: VixDiskLib_Close 605 -nbdkit: debug: VixDiskLib_QueryAllocatedBlocks 191 -nbdkit: debug: VixDiskLib_ConnectEx 134 -nbdkit: debug: VixDiskLib_Disconnect 76 -nbdkit: debug: VixDiskLib_FreeConnectParams 57 -nbdkit: debug: VixDiskLib_GetInfo 56 -nbdkit: debug: VixDiskLib_GetTransportMode 43 -nbdkit: debug: VixDiskLib_FreeInfo 42 -nbdkit: debug: VixDiskLib_FreeBlockList 32 -nbdkit: debug: VixDiskLib_AllocateConnectParams 28 - -VDDK APIs which are never called are not printed. - -(cherry picked from commit f2dfc7d74ee650bdf2cc930a07b1c5bcb509976c) ---- - plugins/vddk/nbdkit-vddk-plugin.pod | 5 ++ - plugins/vddk/vddk.c | 107 +++++++++++++++++++++++++--- - tests/test-vddk-real.sh | 2 +- - 3 files changed, 103 insertions(+), 11 deletions(-) - -diff --git a/plugins/vddk/nbdkit-vddk-plugin.pod b/plugins/vddk/nbdkit-vddk-plugin.pod -index b783f13a..2a1b17dc 100644 ---- a/plugins/vddk/nbdkit-vddk-plugin.pod -+++ b/plugins/vddk/nbdkit-vddk-plugin.pod -@@ -510,6 +510,11 @@ Debug extents returned by C. - - Suppress debugging of datapath calls (C and C). - -+=item B<-D vddk.stats=1> -+ -+When the plugin exits print some statistics about the amount of time -+spent waiting on each VDDK call. -+ - =back - - =head1 FILES -diff --git a/plugins/vddk/vddk.c b/plugins/vddk/vddk.c -index 888009ab..fce96d9a 100644 ---- a/plugins/vddk/vddk.c -+++ b/plugins/vddk/vddk.c -@@ -42,6 +42,7 @@ - #include - #include - #include -+#include - - #include - -@@ -52,6 +53,8 @@ - #include "isaligned.h" - #include "minmax.h" - #include "rounding.h" -+#include "tvdiff.h" -+#include "vector.h" - - #include "vddk.h" - #include "vddk-structs.h" -@@ -60,6 +63,7 @@ - NBDKIT_DLL_PUBLIC int vddk_debug_diskinfo; - NBDKIT_DLL_PUBLIC int vddk_debug_extents; - NBDKIT_DLL_PUBLIC int vddk_debug_datapath = 1; -+NBDKIT_DLL_PUBLIC int vddk_debug_stats; - - /* For each VDDK API define a static global variable. These globals - * are initialized when the plugin is loaded (by vddk_get_ready). -@@ -96,22 +100,52 @@ static const char *username; /* user */ - static const char *vmx_spec; /* vm */ - static bool is_remote; - --#define VDDK_ERROR(err, fs, ...) \ -- do { \ -- char *vddk_err_msg; \ -- vddk_err_msg = VixDiskLib_GetErrorText ((err), NULL); \ -- nbdkit_error (fs ": %s", ##__VA_ARGS__, vddk_err_msg); \ -- VixDiskLib_FreeErrorText (vddk_err_msg); \ -- } while (0) -+/* For each VDDK API define a variable to store the time taken (used -+ * to implement -D vddk.stats=1). -+ */ -+static pthread_mutex_t stats_lock = PTHREAD_MUTEX_INITIALIZER; -+static void display_stats (void); -+#define STUB(fn,ret,args) static int64_t stats_##fn; -+#define OPTIONAL_STUB(fn,ret,args) static int64_t stats_##fn; -+#include "vddk-stubs.h" -+#undef STUB -+#undef OPTIONAL_STUB - - #define VDDK_CALL_START(fn, fs, ...) \ -+ do { \ -+ struct timeval start_t, end_t; \ -+ if (vddk_debug_stats) \ -+ gettimeofday (&start_t, NULL); \ - nbdkit_debug ("VDDK call: %s (" fs ")", #fn, ##__VA_ARGS__); \ - do - #define VDDK_CALL_START_DATAPATH(fn, fs, ...) \ -+ do { \ -+ struct timeval start_t, end_t; \ -+ if (vddk_debug_stats) \ -+ gettimeofday (&start_t, NULL); \ - if (vddk_debug_datapath) \ - nbdkit_debug ("VDDK call: %s (" fs ")", #fn, ##__VA_ARGS__); \ - do --#define VDDK_CALL_END(fn) while (0) -+#define VDDK_CALL_END(fn) \ -+ while (0); \ -+ if (vddk_debug_stats) { \ -+ gettimeofday (&end_t, NULL); \ -+ ACQUIRE_LOCK_FOR_CURRENT_SCOPE (&stats_lock); \ -+ stats_##fn += tvdiff_usec (&start_t, &end_t); \ -+ } \ -+ } while (0) -+ -+#define VDDK_ERROR(err, fs, ...) \ -+ do { \ -+ char *vddk_err_msg; \ -+ VDDK_CALL_START (VixDiskLib_GetErrorText, "%lu", err) { \ -+ vddk_err_msg = VixDiskLib_GetErrorText ((err), NULL); \ -+ } VDDK_CALL_END (VixDiskLib_GetErrorText); \ -+ nbdkit_error (fs ": %s", ##__VA_ARGS__, vddk_err_msg); \ -+ VDDK_CALL_START (VixDiskLib_FreeErrorText, "") { \ -+ VixDiskLib_FreeErrorText (vddk_err_msg); \ -+ } VDDK_CALL_END (VixDiskLib_FreeErrorText); \ -+ } while (0) - - /* Unload the plugin. */ - static void -@@ -124,11 +158,61 @@ vddk_unload (void) - } - if (dl) - dlclose (dl); -+ -+ if (vddk_debug_stats) -+ display_stats (); -+ - free (config); - free (libdir); - free (password); - } - -+struct vddk_stat { -+ const char *fn; -+ int64_t usecs; -+}; -+DEFINE_VECTOR_TYPE(statlist, struct vddk_stat) -+ -+static int -+stat_compare (const void *vp1, const void *vp2) -+{ -+ const struct vddk_stat *st1 = vp1; -+ const struct vddk_stat *st2 = vp2; -+ -+ /* Note: sorts in reverse order. */ -+ if (st1->usecs < st2->usecs) return 1; -+ else if (st1->usecs > st2->usecs) return -1; -+ else return 0; -+} -+ -+static void -+display_stats (void) -+{ -+ statlist stats = empty_vector; -+ struct vddk_stat st; -+ size_t i; -+ -+#define ADD_ONE_STAT(fn_, usecs_) \ -+ st.fn = fn_; \ -+ st.usecs = usecs_; \ -+ statlist_append (&stats, st) -+#define STUB(fn,ret,args) ADD_ONE_STAT (#fn, stats_##fn); -+#define OPTIONAL_STUB(fn,ret,args) ADD_ONE_STAT (#fn, stats_##fn); -+#include "vddk-stubs.h" -+#undef STUB -+#undef OPTIONAL_STUB -+#undef ADD_ONE_STAT -+ -+ qsort (stats.ptr, stats.size, sizeof stats.ptr[0], stat_compare); -+ -+ nbdkit_debug ("VDDK function stats (-D vddk.stats=1):"); -+ nbdkit_debug ("%-40s %9s", "", "µs"); -+ for (i = 0; i < stats.size; ++i) { -+ if (stats.ptr[i].usecs) -+ nbdkit_debug ("%-40s %9" PRIi64, stats.ptr[i].fn, stats.ptr[i].usecs); -+ } -+} -+ - static void - trim (char *str) - { -@@ -557,6 +641,7 @@ vddk_open (int readonly) - struct vddk_handle *h; - VixError err; - uint32_t flags; -+ const char *transport_mode; - - h = malloc (sizeof *h); - if (h == NULL) { -@@ -635,8 +720,10 @@ vddk_open (int readonly) - goto err2; - } - -- nbdkit_debug ("transport mode: %s", -- VixDiskLib_GetTransportMode (h->handle)); -+ VDDK_CALL_START (VixDiskLib_GetTransportMode, "handle") { -+ transport_mode = VixDiskLib_GetTransportMode (h->handle); -+ } VDDK_CALL_END (VixDiskLib_GetTransportMode); -+ nbdkit_debug ("transport mode: %s", transport_mode); - - return h; - -diff --git a/tests/test-vddk-real.sh b/tests/test-vddk-real.sh -index f848db44..3c8b4262 100755 ---- a/tests/test-vddk-real.sh -+++ b/tests/test-vddk-real.sh -@@ -79,7 +79,7 @@ if grep 'cannot open shared object file' test-vddk-real.log; then - fi - - # Now run nbdkit for the test. --start_nbdkit -P $pid -U $sock vddk libdir="$vddkdir" $vmdk -+start_nbdkit -P $pid -U $sock -D vddk.stats=1 vddk libdir="$vddkdir" $vmdk - uri="nbd+unix:///?socket=$sock" - - # VDDK < 6.0 did not support flush, so disable flush test there. Also --- -2.31.1 - diff --git a/SOURCES/0023-cow-Make-the-block-size-configurable.patch b/SOURCES/0023-cow-Make-the-block-size-configurable.patch deleted file mode 100644 index a879abc..0000000 --- a/SOURCES/0023-cow-Make-the-block-size-configurable.patch +++ /dev/null @@ -1,772 +0,0 @@ -From 0be4847cdec9effd6128da03ea42a4953e5a6343 Mon Sep 17 00:00:00 2001 -From: "Richard W.M. Jones" -Date: Tue, 17 Aug 2021 22:03:11 +0100 -Subject: [PATCH] cow: Make the block size configurable - -Commit c1905b0a28 ("cache, cow: Use a 64K block size by default") -changed the nbdkit-cow-filter block size to 64K, but it was still a -fixed size. In contrast the cache filter allows the block size to be -adjusted. - -Allow the block size in this filter to be adjusted up or down with a -new cow-block-size=N parameter. - -When using the VDDK plugin, adjusting this setting can make a -difference. The following timings come from a modified virt-v2v which -sets cow-block-size and was used to convert from a VMware server to --o null (this is also using cow-on-read=true): - - cow-block-size=64K: 18m18 - cow-block-size=256K: 14m13 - cow-block-size=1M: 14m19 - cow-block-size=4M: 37m33 - -As you can see it's not obvious how to choose a good block size, but -at least by allowing adjustment we can tune things. - -(cherry picked from commit 7182c47d04d2b68005fceadefc0c14bfaa61a533) ---- - filters/cow/blk.c | 35 +++---- - filters/cow/blk.h | 5 - - filters/cow/cow.c | 150 +++++++++++++++++------------- - filters/cow/cow.h | 39 ++++++++ - filters/cow/nbdkit-cow-filter.pod | 5 + - tests/Makefile.am | 2 + - tests/test-cow-block-size.sh | 72 ++++++++++++++ - 7 files changed, 221 insertions(+), 87 deletions(-) - create mode 100644 filters/cow/cow.h - create mode 100755 tests/test-cow-block-size.sh - -diff --git a/filters/cow/blk.c b/filters/cow/blk.c -index c22d5886..f9341dc1 100644 ---- a/filters/cow/blk.c -+++ b/filters/cow/blk.c -@@ -99,6 +99,7 @@ - #include "pwrite.h" - #include "utils.h" - -+#include "cow.h" - #include "blk.h" - - /* The temporary overlay. */ -@@ -137,7 +138,7 @@ blk_init (void) - size_t len; - char *template; - -- bitmap_init (&bm, BLKSIZE, 2 /* bits per block */); -+ bitmap_init (&bm, blksize, 2 /* bits per block */); - - tmpdir = getenv ("TMPDIR"); - if (!tmpdir) -@@ -199,7 +200,7 @@ blk_set_size (uint64_t new_size) - if (bitmap_resize (&bm, size) == -1) - return -1; - -- if (ftruncate (fd, ROUND_UP (size, BLKSIZE)) == -1) { -+ if (ftruncate (fd, ROUND_UP (size, blksize)) == -1) { - nbdkit_error ("ftruncate: %m"); - return -1; - } -@@ -228,7 +229,7 @@ blk_read_multiple (nbdkit_next *next, - uint64_t blknum, uint64_t nrblocks, - uint8_t *block, bool cow_on_read, int *err) - { -- off_t offset = blknum * BLKSIZE; -+ off_t offset = blknum * blksize; - enum bm_entry state; - uint64_t b, runblocks; - -@@ -262,8 +263,8 @@ blk_read_multiple (nbdkit_next *next, - if (state == BLOCK_NOT_ALLOCATED) { /* Read underlying plugin. */ - unsigned n, tail = 0; - -- assert (BLKSIZE * runblocks <= UINT_MAX); -- n = BLKSIZE * runblocks; -+ assert (blksize * runblocks <= UINT_MAX); -+ n = blksize * runblocks; - - if (offset + n > size) { - tail = offset + n - size; -@@ -288,7 +289,7 @@ blk_read_multiple (nbdkit_next *next, - "at offset %" PRIu64 " into the cache", - runblocks, offset); - -- if (full_pwrite (fd, block, BLKSIZE * runblocks, offset) == -1) { -+ if (full_pwrite (fd, block, blksize * runblocks, offset) == -1) { - *err = errno; - nbdkit_error ("pwrite: %m"); - return -1; -@@ -298,14 +299,14 @@ blk_read_multiple (nbdkit_next *next, - } - } - else if (state == BLOCK_ALLOCATED) { /* Read overlay. */ -- if (full_pread (fd, block, BLKSIZE * runblocks, offset) == -1) { -+ if (full_pread (fd, block, blksize * runblocks, offset) == -1) { - *err = errno; - nbdkit_error ("pread: %m"); - return -1; - } - } - else /* state == BLOCK_TRIMMED */ { -- memset (block, 0, BLKSIZE * runblocks); -+ memset (block, 0, blksize * runblocks); - } - - /* If all done, return. */ -@@ -316,7 +317,7 @@ blk_read_multiple (nbdkit_next *next, - return blk_read_multiple (next, - blknum + runblocks, - nrblocks - runblocks, -- block + BLKSIZE * runblocks, -+ block + blksize * runblocks, - cow_on_read, err); - } - -@@ -333,9 +334,9 @@ blk_cache (nbdkit_next *next, - { - /* XXX Could make this lock more fine-grained with some thought. */ - ACQUIRE_LOCK_FOR_CURRENT_SCOPE (&lock); -- off_t offset = blknum * BLKSIZE; -+ off_t offset = blknum * blksize; - enum bm_entry state = bitmap_get_blk (&bm, blknum, BLOCK_NOT_ALLOCATED); -- unsigned n = BLKSIZE, tail = 0; -+ unsigned n = blksize, tail = 0; - - if (offset + n > size) { - tail = offset + n - size; -@@ -348,7 +349,7 @@ blk_cache (nbdkit_next *next, - - if (state == BLOCK_ALLOCATED) { - #if HAVE_POSIX_FADVISE -- int r = posix_fadvise (fd, offset, BLKSIZE, POSIX_FADV_WILLNEED); -+ int r = posix_fadvise (fd, offset, blksize, POSIX_FADV_WILLNEED); - if (r) { - errno = r; - nbdkit_error ("posix_fadvise: %m"); -@@ -373,7 +374,7 @@ blk_cache (nbdkit_next *next, - memset (block + n, 0, tail); - - if (mode == BLK_CACHE_COW) { -- if (full_pwrite (fd, block, BLKSIZE, offset) == -1) { -+ if (full_pwrite (fd, block, blksize, offset) == -1) { - *err = errno; - nbdkit_error ("pwrite: %m"); - return -1; -@@ -386,13 +387,13 @@ blk_cache (nbdkit_next *next, - int - blk_write (uint64_t blknum, const uint8_t *block, int *err) - { -- off_t offset = blknum * BLKSIZE; -+ off_t offset = blknum * blksize; - - if (cow_debug_verbose) - nbdkit_debug ("cow: blk_write block %" PRIu64 " (offset %" PRIu64 ")", - blknum, (uint64_t) offset); - -- if (full_pwrite (fd, block, BLKSIZE, offset) == -1) { -+ if (full_pwrite (fd, block, blksize, offset) == -1) { - *err = errno; - nbdkit_error ("pwrite: %m"); - return -1; -@@ -407,14 +408,14 @@ blk_write (uint64_t blknum, const uint8_t *block, int *err) - int - blk_trim (uint64_t blknum, int *err) - { -- off_t offset = blknum * BLKSIZE; -+ off_t offset = blknum * blksize; - - if (cow_debug_verbose) - nbdkit_debug ("cow: blk_trim block %" PRIu64 " (offset %" PRIu64 ")", - blknum, (uint64_t) offset); - - /* XXX As an optimization we could punch a whole in the overlay -- * here. However it's not trivial since BLKSIZE is unrelated to the -+ * here. However it's not trivial since blksize is unrelated to the - * overlay filesystem block size. - */ - ACQUIRE_LOCK_FOR_CURRENT_SCOPE (&lock); -diff --git a/filters/cow/blk.h b/filters/cow/blk.h -index b7e6f092..62fb5416 100644 ---- a/filters/cow/blk.h -+++ b/filters/cow/blk.h -@@ -33,11 +33,6 @@ - #ifndef NBDKIT_BLK_H - #define NBDKIT_BLK_H - --/* Size of a block in the overlay. A 4K block size means that we need -- * 64 MB of memory to store the bitmap for a 1 TB underlying image. -- */ --#define BLKSIZE 65536 -- - /* Initialize the overlay and bitmap. */ - extern int blk_init (void); - -diff --git a/filters/cow/cow.c b/filters/cow/cow.c -index 6efb39f2..1c62c857 100644 ---- a/filters/cow/cow.c -+++ b/filters/cow/cow.c -@@ -40,6 +40,7 @@ - #include - #include - #include -+#include - - #include - -@@ -47,9 +48,11 @@ - - #include "cleanup.h" - #include "isaligned.h" -+#include "ispowerof2.h" - #include "minmax.h" - #include "rounding.h" - -+#include "cow.h" - #include "blk.h" - - /* Read-modify-write requests are serialized through this global lock. -@@ -58,6 +61,8 @@ - */ - static pthread_mutex_t rmw_lock = PTHREAD_MUTEX_INITIALIZER; - -+unsigned blksize = 65536; /* block size */ -+ - static bool cow_on_cache; - - /* Cache on read ("cow-on-read") mode. */ -@@ -69,13 +74,6 @@ extern enum cor_mode { - enum cor_mode cor_mode = COR_OFF; - const char *cor_path; - --static void --cow_load (void) --{ -- if (blk_init () == -1) -- exit (EXIT_FAILURE); --} -- - static void - cow_unload (void) - { -@@ -86,7 +84,19 @@ static int - cow_config (nbdkit_next_config *next, nbdkit_backend *nxdata, - const char *key, const char *value) - { -- if (strcmp (key, "cow-on-cache") == 0) { -+ if (strcmp (key, "cow-block-size") == 0) { -+ int64_t r = nbdkit_parse_size (value); -+ if (r == -1) -+ return -1; -+ if (r <= 4096 || r > UINT_MAX || !is_power_of_2 (r)) { -+ nbdkit_error ("cow-block-size is out of range (4096..2G) " -+ "or not a power of 2"); -+ return -1; -+ } -+ blksize = r; -+ return 0; -+ } -+ else if (strcmp (key, "cow-on-cache") == 0) { - int r; - - r = nbdkit_parse_bool (value); -@@ -114,9 +124,19 @@ cow_config (nbdkit_next_config *next, nbdkit_backend *nxdata, - } - - #define cow_config_help \ -+ "cow-block-size= Set COW block size.\n" \ - "cow-on-cache= Copy cache (prefetch) requests to the overlay.\n" \ - "cow-on-read=|/PATH Copy read requests to the overlay." - -+static int -+cow_get_ready (int thread_model) -+{ -+ if (blk_init () == -1) -+ return -1; -+ -+ return 0; -+} -+ - /* Decide if cow-on-read is currently on or off. */ - bool - cow_on_read (void) -@@ -249,8 +269,8 @@ cow_pread (nbdkit_next *next, - uint64_t blknum, blkoffs, nrblocks; - int r; - -- if (!IS_ALIGNED (count | offset, BLKSIZE)) { -- block = malloc (BLKSIZE); -+ if (!IS_ALIGNED (count | offset, blksize)) { -+ block = malloc (blksize); - if (block == NULL) { - *err = errno; - nbdkit_error ("malloc: %m"); -@@ -258,12 +278,12 @@ cow_pread (nbdkit_next *next, - } - } - -- blknum = offset / BLKSIZE; /* block number */ -- blkoffs = offset % BLKSIZE; /* offset within the block */ -+ blknum = offset / blksize; /* block number */ -+ blkoffs = offset % blksize; /* offset within the block */ - - /* Unaligned head */ - if (blkoffs) { -- uint64_t n = MIN (BLKSIZE - blkoffs, count); -+ uint64_t n = MIN (blksize - blkoffs, count); - - assert (block); - r = blk_read (next, blknum, block, cow_on_read (), err); -@@ -279,15 +299,15 @@ cow_pread (nbdkit_next *next, - } - - /* Aligned body */ -- nrblocks = count / BLKSIZE; -+ nrblocks = count / blksize; - if (nrblocks > 0) { - r = blk_read_multiple (next, blknum, nrblocks, buf, cow_on_read (), err); - if (r == -1) - return -1; - -- buf += nrblocks * BLKSIZE; -- count -= nrblocks * BLKSIZE; -- offset += nrblocks * BLKSIZE; -+ buf += nrblocks * blksize; -+ count -= nrblocks * blksize; -+ offset += nrblocks * blksize; - blknum += nrblocks; - } - -@@ -314,8 +334,8 @@ cow_pwrite (nbdkit_next *next, - uint64_t blknum, blkoffs; - int r; - -- if (!IS_ALIGNED (count | offset, BLKSIZE)) { -- block = malloc (BLKSIZE); -+ if (!IS_ALIGNED (count | offset, blksize)) { -+ block = malloc (blksize); - if (block == NULL) { - *err = errno; - nbdkit_error ("malloc: %m"); -@@ -323,12 +343,12 @@ cow_pwrite (nbdkit_next *next, - } - } - -- blknum = offset / BLKSIZE; /* block number */ -- blkoffs = offset % BLKSIZE; /* offset within the block */ -+ blknum = offset / blksize; /* block number */ -+ blkoffs = offset % blksize; /* offset within the block */ - - /* Unaligned head */ - if (blkoffs) { -- uint64_t n = MIN (BLKSIZE - blkoffs, count); -+ uint64_t n = MIN (blksize - blkoffs, count); - - /* Do a read-modify-write operation on the current block. - * Hold the rmw_lock over the whole operation. -@@ -350,14 +370,14 @@ cow_pwrite (nbdkit_next *next, - } - - /* Aligned body */ -- while (count >= BLKSIZE) { -+ while (count >= blksize) { - r = blk_write (blknum, buf, err); - if (r == -1) - return -1; - -- buf += BLKSIZE; -- count -= BLKSIZE; -- offset += BLKSIZE; -+ buf += blksize; -+ count -= blksize; -+ offset += blksize; - blknum++; - } - -@@ -397,19 +417,19 @@ cow_zero (nbdkit_next *next, - return -1; - } - -- block = malloc (BLKSIZE); -+ block = malloc (blksize); - if (block == NULL) { - *err = errno; - nbdkit_error ("malloc: %m"); - return -1; - } - -- blknum = offset / BLKSIZE; /* block number */ -- blkoffs = offset % BLKSIZE; /* offset within the block */ -+ blknum = offset / blksize; /* block number */ -+ blkoffs = offset % blksize; /* offset within the block */ - - /* Unaligned head */ - if (blkoffs) { -- uint64_t n = MIN (BLKSIZE - blkoffs, count); -+ uint64_t n = MIN (blksize - blkoffs, count); - - /* Do a read-modify-write operation on the current block. - * Hold the rmw_lock over the whole operation. -@@ -429,9 +449,9 @@ cow_zero (nbdkit_next *next, - } - - /* Aligned body */ -- if (count >= BLKSIZE) -- memset (block, 0, BLKSIZE); -- while (count >= BLKSIZE) { -+ if (count >= blksize) -+ memset (block, 0, blksize); -+ while (count >= blksize) { - /* XXX There is the possibility of optimizing this: since this loop is - * writing a whole, aligned block, we should use FALLOC_FL_ZERO_RANGE. - */ -@@ -439,8 +459,8 @@ cow_zero (nbdkit_next *next, - if (r == -1) - return -1; - -- count -= BLKSIZE; -- offset += BLKSIZE; -+ count -= blksize; -+ offset += blksize; - blknum++; - } - -@@ -471,8 +491,8 @@ cow_trim (nbdkit_next *next, - uint64_t blknum, blkoffs; - int r; - -- if (!IS_ALIGNED (count | offset, BLKSIZE)) { -- block = malloc (BLKSIZE); -+ if (!IS_ALIGNED (count | offset, blksize)) { -+ block = malloc (blksize); - if (block == NULL) { - *err = errno; - nbdkit_error ("malloc: %m"); -@@ -480,12 +500,12 @@ cow_trim (nbdkit_next *next, - } - } - -- blknum = offset / BLKSIZE; /* block number */ -- blkoffs = offset % BLKSIZE; /* offset within the block */ -+ blknum = offset / blksize; /* block number */ -+ blkoffs = offset % blksize; /* offset within the block */ - - /* Unaligned head */ - if (blkoffs) { -- uint64_t n = MIN (BLKSIZE - blkoffs, count); -+ uint64_t n = MIN (blksize - blkoffs, count); - - /* Do a read-modify-write operation on the current block. - * Hold the lock over the whole operation. -@@ -505,13 +525,13 @@ cow_trim (nbdkit_next *next, - } - - /* Aligned body */ -- while (count >= BLKSIZE) { -+ while (count >= blksize) { - r = blk_trim (blknum, err); - if (r == -1) - return -1; - -- count -= BLKSIZE; -- offset += BLKSIZE; -+ count -= blksize; -+ offset += blksize; - blknum++; - } - -@@ -568,22 +588,22 @@ cow_cache (nbdkit_next *next, - mode = BLK_CACHE_COW; - - assert (!flags); -- block = malloc (BLKSIZE); -+ block = malloc (blksize); - if (block == NULL) { - *err = errno; - nbdkit_error ("malloc: %m"); - return -1; - } - -- blknum = offset / BLKSIZE; /* block number */ -- blkoffs = offset % BLKSIZE; /* offset within the block */ -+ blknum = offset / blksize; /* block number */ -+ blkoffs = offset % blksize; /* offset within the block */ - - /* Unaligned head */ - remaining += blkoffs; - offset -= blkoffs; - - /* Unaligned tail */ -- remaining = ROUND_UP (remaining, BLKSIZE); -+ remaining = ROUND_UP (remaining, blksize); - - /* Aligned body */ - while (remaining) { -@@ -591,8 +611,8 @@ cow_cache (nbdkit_next *next, - if (r == -1) - return -1; - -- remaining -= BLKSIZE; -- offset += BLKSIZE; -+ remaining -= blksize; -+ offset += blksize; - blknum++; - } - -@@ -616,13 +636,13 @@ cow_extents (nbdkit_next *next, - * value so rounding up is safe here. - */ - end = offset + count; -- offset = ROUND_DOWN (offset, BLKSIZE); -- end = ROUND_UP (end, BLKSIZE); -+ offset = ROUND_DOWN (offset, blksize); -+ end = ROUND_UP (end, blksize); - count = end - offset; -- blknum = offset / BLKSIZE; -+ blknum = offset / blksize; - -- assert (IS_ALIGNED (offset, BLKSIZE)); -- assert (IS_ALIGNED (count, BLKSIZE)); -+ assert (IS_ALIGNED (offset, blksize)); -+ assert (IS_ALIGNED (count, blksize)); - assert (count > 0); /* We must make forward progress. */ - - while (count > 0) { -@@ -634,7 +654,7 @@ cow_extents (nbdkit_next *next, - /* Present in the overlay. */ - if (present) { - e.offset = offset; -- e.length = BLKSIZE; -+ e.length = blksize; - - if (trimmed) - e.type = NBDKIT_EXTENT_HOLE|NBDKIT_EXTENT_ZERO; -@@ -647,8 +667,8 @@ cow_extents (nbdkit_next *next, - } - - blknum++; -- offset += BLKSIZE; -- count -= BLKSIZE; -+ offset += blksize; -+ count -= blksize; - } - - /* Not present in the overlay, but we can ask the plugin. */ -@@ -667,12 +687,12 @@ cow_extents (nbdkit_next *next, - * (range_count), but count is a 64 bit quantity, so don't - * overflow range_count here. - */ -- if (range_count >= UINT32_MAX - BLKSIZE + 1) break; -+ if (range_count >= UINT32_MAX - blksize + 1) break; - - blknum++; -- offset += BLKSIZE; -- count -= BLKSIZE; -- range_count += BLKSIZE; -+ offset += blksize; -+ count -= blksize; -+ range_count += blksize; - - if (count == 0) break; - blk_status (blknum, &present, &trimmed); -@@ -706,7 +726,7 @@ cow_extents (nbdkit_next *next, - /* Otherwise assume the block is non-sparse. */ - else { - e.offset = offset; -- e.length = BLKSIZE; -+ e.length = blksize; - e.type = 0; - - if (nbdkit_add_extent (extents, e.offset, e.length, e.type) == -1) { -@@ -715,8 +735,8 @@ cow_extents (nbdkit_next *next, - } - - blknum++; -- offset += BLKSIZE; -- count -= BLKSIZE; -+ offset += blksize; -+ count -= blksize; - } - - /* If the caller only wanted the first extent, and we've managed -@@ -734,11 +754,11 @@ cow_extents (nbdkit_next *next, - static struct nbdkit_filter filter = { - .name = "cow", - .longname = "nbdkit copy-on-write (COW) filter", -- .load = cow_load, - .unload = cow_unload, - .open = cow_open, - .config = cow_config, - .config_help = cow_config_help, -+ .get_ready = cow_get_ready, - .prepare = cow_prepare, - .get_size = cow_get_size, - .can_write = cow_can_write, -diff --git a/filters/cow/cow.h b/filters/cow/cow.h -new file mode 100644 -index 00000000..d46dbe91 ---- /dev/null -+++ b/filters/cow/cow.h -@@ -0,0 +1,39 @@ -+/* nbdkit -+ * Copyright (C) 2018-2021 Red Hat Inc. -+ * -+ * Redistribution and use in source and binary forms, with or without -+ * modification, are permitted provided that the following conditions are -+ * met: -+ * -+ * * Redistributions of source code must retain the above copyright -+ * notice, this list of conditions and the following disclaimer. -+ * -+ * * Redistributions in binary form must reproduce the above copyright -+ * notice, this list of conditions and the following disclaimer in the -+ * documentation and/or other materials provided with the distribution. -+ * -+ * * Neither the name of Red Hat nor the names of its contributors may be -+ * used to endorse or promote products derived from this software without -+ * specific prior written permission. -+ * -+ * THIS SOFTWARE IS PROVIDED BY RED HAT AND CONTRIBUTORS ''AS IS'' AND -+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, -+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A -+ * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL RED HAT OR -+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF -+ * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND -+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT -+ * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF -+ * SUCH DAMAGE. -+ */ -+ -+#ifndef NBDKIT_COW_H -+#define NBDKIT_COW_H -+ -+/* Size of a block in the cache. */ -+extern unsigned blksize; -+ -+#endif /* NBDKIT_COW_H */ -diff --git a/filters/cow/nbdkit-cow-filter.pod b/filters/cow/nbdkit-cow-filter.pod -index 7f861140..997c9097 100644 ---- a/filters/cow/nbdkit-cow-filter.pod -+++ b/filters/cow/nbdkit-cow-filter.pod -@@ -5,6 +5,7 @@ nbdkit-cow-filter - nbdkit copy-on-write (COW) filter - =head1 SYNOPSIS - - nbdkit --filter=cow plugin [plugin-args...] -+ [cow-block-size=N] - [cow-on-cache=false|true] - [cow-on-read=false|true|/PATH] - -@@ -42,6 +43,10 @@ serve the same data to each client. - - =over 4 - -+=item BN -+ -+Set the block size used by the filter. The default is 64K. -+ - =item B - - Do not save data from cache (prefetch) requests in the overlay. This -diff --git a/tests/Makefile.am b/tests/Makefile.am -index e61c5829..d93f848f 100644 ---- a/tests/Makefile.am -+++ b/tests/Makefile.am -@@ -1404,6 +1404,7 @@ EXTRA_DIST += \ - if HAVE_MKE2FS_WITH_D - TESTS += \ - test-cow.sh \ -+ test-cow-block-size.sh \ - test-cow-extents1.sh \ - test-cow-extents2.sh \ - test-cow-extents-large.sh \ -@@ -1415,6 +1416,7 @@ endif - TESTS += test-cow-null.sh - EXTRA_DIST += \ - test-cow.sh \ -+ test-cow-block-size.sh \ - test-cow-extents1.sh \ - test-cow-extents2.sh \ - test-cow-extents-large.sh \ -diff --git a/tests/test-cow-block-size.sh b/tests/test-cow-block-size.sh -new file mode 100755 -index 00000000..6de1c068 ---- /dev/null -+++ b/tests/test-cow-block-size.sh -@@ -0,0 +1,72 @@ -+#!/usr/bin/env bash -+# nbdkit -+# Copyright (C) 2018-2021 Red Hat Inc. -+# -+# Redistribution and use in source and binary forms, with or without -+# modification, are permitted provided that the following conditions are -+# met: -+# -+# * Redistributions of source code must retain the above copyright -+# notice, this list of conditions and the following disclaimer. -+# -+# * Redistributions in binary form must reproduce the above copyright -+# notice, this list of conditions and the following disclaimer in the -+# documentation and/or other materials provided with the distribution. -+# -+# * Neither the name of Red Hat nor the names of its contributors may be -+# used to endorse or promote products derived from this software without -+# specific prior written permission. -+# -+# THIS SOFTWARE IS PROVIDED BY RED HAT AND CONTRIBUTORS ''AS IS'' AND -+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, -+# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A -+# PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL RED HAT OR -+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF -+# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND -+# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -+# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT -+# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF -+# SUCH DAMAGE. -+ -+source ./functions.sh -+set -e -+set -x -+ -+requires_plugin linuxdisk -+requires guestfish --version -+requires nbdcopy --version -+requires qemu-img --version -+ -+sock=$(mktemp -u /tmp/nbdkit-test-sock.XXXXXX) -+files="cow-block-size-base.img $sock cow-block-size.pid" -+rm -f $files -+cleanup_fn rm -f $files -+ -+# Create a base image which is partitioned with an empty filesystem. -+rm -rf cow-block-size.d -+mkdir cow-block-size.d -+cleanup_fn rm -rf cow-block-size.d -+nbdkit -fv -U - linuxdisk cow-block-size.d size=100M \ -+ --run 'nbdcopy "$uri" cow-block-size-base.img' -+lastmod="$(stat -c "%y" cow-block-size-base.img)" -+ -+# Run nbdkit with a COW overlay, 4M block size and copy on read. -+start_nbdkit -P cow-block-size.pid -U $sock \ -+ --filter=cow file cow-block-size-base.img \ -+ cow-block-size=4M cow-on-read=true -+ -+# Write some data into the overlay. -+guestfish --format=raw -a "nbd://?socket=$sock" -m /dev/sda1 < +Date: Mon, 8 Nov 2021 19:47:57 +0200 +Subject: [PATCH] podwrapper.pl.in: Use short commit date + +We can use git short commit date format $cs. Maybe it was not available +when podwrapper.pl was created. + +Signed-off-by: Nir Soffer +(cherry picked from libnbd commit 0306fdcb08e8dc5957a9e344b54200711fca1220) +(cherry picked from commit 7a1e79c6b5ca4adcef47fc0929d25d54610fc417) +--- + podwrapper.pl.in | 3 +-- + 1 file changed, 1 insertion(+), 2 deletions(-) + +diff --git a/podwrapper.pl.in b/podwrapper.pl.in +index abad578d..63c1025a 100755 +--- a/podwrapper.pl.in ++++ b/podwrapper.pl.in +@@ -233,8 +233,7 @@ my $date; + my $filename = "$abs_top_srcdir/.git"; + if (!$date && -d $filename) { + local $ENV{GIT_DIR} = $filename; +- $_ = `git show -O/dev/null -s --format=%ci`; +- $date = $1 if /^(\d+-\d+-\d+)\s/; ++ $date = `git show -O/dev/null -s --format=%cs`; + } + if (!$date) { + my ($day, $month, $year) = (gmtime($ENV{SOURCE_DATE_EPOCH} || time))[3,4,5]; +-- +2.31.1 + diff --git a/SOURCES/0024-cow-Ship-cow.h-header.patch b/SOURCES/0024-cow-Ship-cow.h-header.patch deleted file mode 100644 index 3a6e8e0..0000000 --- a/SOURCES/0024-cow-Ship-cow.h-header.patch +++ /dev/null @@ -1,26 +0,0 @@ -From 8d2ef02bd4de988e20ad1efba8038d311cd59665 Mon Sep 17 00:00:00 2001 -From: Eric Blake -Date: Wed, 18 Aug 2021 19:16:43 -0500 -Subject: [PATCH] cow: Ship cow.h header - -Fixes: 7182c47d0 (cow: Make the block size configurable) -(cherry picked from commit 75ff1b8b1afb3744b21a306c62e4973c90d386be) ---- - filters/cow/Makefile.am | 1 + - 1 file changed, 1 insertion(+) - -diff --git a/filters/cow/Makefile.am b/filters/cow/Makefile.am -index a80ccd8f..88cda497 100644 ---- a/filters/cow/Makefile.am -+++ b/filters/cow/Makefile.am -@@ -39,6 +39,7 @@ nbdkit_cow_filter_la_SOURCES = \ - blk.c \ - blk.h \ - cow.c \ -+ cow.h \ - $(top_srcdir)/include/nbdkit-filter.h \ - $(NULL) - --- -2.31.1 - diff --git a/SOURCES/0024-ocaml-Replace-noalloc-with-noalloc-annotation.patch b/SOURCES/0024-ocaml-Replace-noalloc-with-noalloc-annotation.patch new file mode 100644 index 0000000..983c610 --- /dev/null +++ b/SOURCES/0024-ocaml-Replace-noalloc-with-noalloc-annotation.patch @@ -0,0 +1,92 @@ +From 2955179919fc6233427b82d27ae61755b2b5e3d7 Mon Sep 17 00:00:00 2001 +From: "Richard W.M. Jones" +Date: Tue, 9 Nov 2021 09:07:42 +0000 +Subject: [PATCH] ocaml: Replace "noalloc" with [@@noalloc] annotation + +This requires OCaml >= 4.03 (released April 2016). The previous +minimum version was 4.02.2. + +(cherry picked from commit d15dd73845065cc9ca04aa785e2be994f76bf832) +--- + README | 2 +- + plugins/ocaml/NBDKit.ml | 18 +++++++++--------- + plugins/ocaml/nbdkit-ocaml-plugin.pod | 5 +---- + 3 files changed, 11 insertions(+), 14 deletions(-) + +diff --git a/README b/README +index b001620c..160856b6 100644 +--- a/README ++++ b/README +@@ -155,7 +155,7 @@ For the Python plugin: + + For the OCaml plugin: + +- - OCaml >= 4.02.2 ++ - OCaml >= 4.03 + + For the Tcl plugin: + +diff --git a/plugins/ocaml/NBDKit.ml b/plugins/ocaml/NBDKit.ml +index ecdc00c1..c9ce31b5 100644 +--- a/plugins/ocaml/NBDKit.ml ++++ b/plugins/ocaml/NBDKit.ml +@@ -152,13 +152,13 @@ let default_callbacks = { + export_description = None; + } + +-external set_name : string -> unit = "ocaml_nbdkit_set_name" "noalloc" +-external set_longname : string -> unit = "ocaml_nbdkit_set_longname" "noalloc" +-external set_version : string -> unit = "ocaml_nbdkit_set_version" "noalloc" +-external set_description : string -> unit = "ocaml_nbdkit_set_description" "noalloc" +-external set_config_help : string -> unit = "ocaml_nbdkit_set_config_help" "noalloc" ++external set_name : string -> unit = "ocaml_nbdkit_set_name" [@@noalloc] ++external set_longname : string -> unit = "ocaml_nbdkit_set_longname" [@@noalloc] ++external set_version : string -> unit = "ocaml_nbdkit_set_version" [@@noalloc] ++external set_description : string -> unit = "ocaml_nbdkit_set_description" [@@noalloc] ++external set_config_help : string -> unit = "ocaml_nbdkit_set_config_help" [@@noalloc] + +-external set_field : string -> 'a -> unit = "ocaml_nbdkit_set_field" "noalloc" ++external set_field : string -> 'a -> unit = "ocaml_nbdkit_set_field" [@@noalloc] + + let register_plugin plugin = + (* Check the required fields have been set by the caller. *) +@@ -220,7 +220,7 @@ let register_plugin plugin = + + (* Bindings to nbdkit server functions. *) + +-external _set_error : int -> unit = "ocaml_nbdkit_set_error" "noalloc" ++external _set_error : int -> unit = "ocaml_nbdkit_set_error" [@@noalloc] + + let set_error unix_error = + (* There's an awkward triple translation going on here, because +@@ -250,9 +250,9 @@ external read_password : string -> string = "ocaml_nbdkit_read_password" + external realpath : string -> string = "ocaml_nbdkit_realpath" + external nanosleep : int -> int -> unit = "ocaml_nbdkit_nanosleep" + external export_name : unit -> string = "ocaml_nbdkit_export_name" +-external shutdown : unit -> unit = "ocaml_nbdkit_shutdown" "noalloc" ++external shutdown : unit -> unit = "ocaml_nbdkit_shutdown" [@@noalloc] + +-external _debug : string -> unit = "ocaml_nbdkit_debug" "noalloc" ++external _debug : string -> unit = "ocaml_nbdkit_debug" [@@noalloc] + + let debug fs = + ksprintf _debug fs +diff --git a/plugins/ocaml/nbdkit-ocaml-plugin.pod b/plugins/ocaml/nbdkit-ocaml-plugin.pod +index 2bd0af25..293f8143 100644 +--- a/plugins/ocaml/nbdkit-ocaml-plugin.pod ++++ b/plugins/ocaml/nbdkit-ocaml-plugin.pod +@@ -11,10 +11,7 @@ nbdkit-ocaml-plugin - writing nbdkit plugins in OCaml + =head1 DESCRIPTION + + This manual page describes how to write nbdkit plugins in natively +-compiled OCaml code. +- +-Note this requires OCaml E 4.02.2, which has support for shared +-libraries. See L ++compiled OCaml code. This requires OCaml E 4.03. + + =head1 WRITING AN OCAML NBDKIT PLUGIN + +-- +2.31.1 + diff --git a/SOURCES/nbdkit-1.26.5.tar.gz.sig b/SOURCES/nbdkit-1.26.5.tar.gz.sig deleted file mode 100644 index 93ce749..0000000 --- a/SOURCES/nbdkit-1.26.5.tar.gz.sig +++ /dev/null @@ -1,11 +0,0 @@ ------BEGIN PGP SIGNATURE----- - -iQEzBAABCAAdFiEEccLMIrHEYCkn0vOqp6FrSiUnQ2oFAmEdoO0ACgkQp6FrSiUn -Q2r9AAf/VN10pDdAapr8W5bPNbgMIJ4dGtGl3sbMKg5mUVNkWwReUpiiUInZwnQW -I+TUjMkvB2jJGkruldgipuRNbAAGROZV3JugE2fMl8qQC1TvL/RMAOHcfKCswzfv -pVik1FmvpI88litCw05csH30TEA1BtFM0TlOR9xoeDkV9e2IUtWcxFJYP6RN5COr -NgTMfouxHWuR+FlVpXkvPl4aOuCavpplobcS0OaKNrqFXMhN+qcKjYgKazUKPB9C -TEExW8/CKTBVdaNpMbcLW/VBEaE85c3mv0xU26YKcEkj+OPAl4AzJ4Z+0MSDXk7t -3nrWSxPX67gBU5XDtKZ1IMUttydvzA== -=NyyV ------END PGP SIGNATURE----- diff --git a/SOURCES/nbdkit-1.28.2.tar.gz.sig b/SOURCES/nbdkit-1.28.2.tar.gz.sig new file mode 100644 index 0000000..ffd05bc --- /dev/null +++ b/SOURCES/nbdkit-1.28.2.tar.gz.sig @@ -0,0 +1,17 @@ +-----BEGIN PGP SIGNATURE----- + +iQJFBAABCAAvFiEE93dPsa0HSn6Mh2fqkXOPc+G3aKAFAmGKYx0RHHJpY2hAYW5u +ZXhpYS5vcmcACgkQkXOPc+G3aKAw7g//Q/42avSvu3DZQXPCl772S9K7erYzw1LK +PzffbDKWwsR4YN+m4w94gFjb6n2NZOnSBVpDYofkt5mzHM9dwYTlCJdS+J4SxV8e +s23+RQjgX0fQMDv4qfGE7CUNQXX6CLRWUlXSu4dMMbsxbF8y18VHl+eLIN7U15Ec +8E2lwcthQ8ipwz0xuxKE1GUNXzWsJ1SHBaCmo8HH1x1Rs+mWu0QjwFllhTc4+gq9 +5yrdZuViv96DxIsk3u7RByqrTJYZi2tK9dU75dVeXI6o74b1WUrXkwqW8uqIR4m5 +hZry1+vXPzWIO4G5o1gika+C4xvYuyIShNl/c/aWJsA0ku4fDtUxAkjC/k78D4yR +0FQbNfztiCFgvQNS3UH6M0d63S4GE76m7HgwI7SkUtczlnPl2AhzSjjfXZ5DrHH9 +S8SgmgwP4qfwszY6Xs7ISFofRV5YBgMhSIiJc/5yYRLtZZCcRWNvv/OWYNTGdVBp +rRYQRvnhe2VbaaVexOn1Fkzcv/FNeD+96bsTp1B1SailUUFrTOpRZ0yIOxfZZZO/ +JcroGuFvq0BywP1U6Rq2PI1IHD4I6x3QLYIfNBP076Rc17BhcXyn2Ei18XkRo+xd +iFTCDpe0zlSCl3tLL7Hvp2EtetYkpiQKHA59g7RoSVQDX/RAml3VFDvwZ2rNBeCL +yegMoVPEmc0= +=c3GR +-----END PGP SIGNATURE----- diff --git a/SPECS/nbdkit.spec b/SPECS/nbdkit.spec index b5e3d52..815a753 100644 --- a/SPECS/nbdkit.spec +++ b/SPECS/nbdkit.spec @@ -47,10 +47,10 @@ ExclusiveArch: x86_64 %global patches_touch_autotools 1 # The source directory. -%global source_directory 1.26-stable +%global source_directory 1.28-stable Name: nbdkit -Version: 1.26.5 +Version: 1.28.2 Release: 1%{?dist} Summary: NBD server @@ -76,30 +76,30 @@ Source3: copy-patches.sh # https://gitlab.com/nbdkit/nbdkit/-/commits/rhel-9.0/ # Patches. -Patch0001: 0001-server-reset-meta-context-replies-on-starttls.patch -Patch0002: 0002-cache-Reduce-verbosity-of-debugging.patch -Patch0003: 0003-cache-cow-Add-blk_read_multiple-function.patch -Patch0004: 0004-cache-cow-Use-full-pread-pwrite-operations.patch -Patch0005: 0005-cache-Implement-cache-on-read-PATH.patch -Patch0006: 0006-cache-Add-cache-min-block-size-parameter.patch -Patch0007: 0007-cache-cow-Use-a-64K-block-size-by-default.patch -Patch0008: 0008-cache-Refactor-printing-state-into-new-function.patch -Patch0009: 0009-tests-cache-Test-cache-on-read-option-really-caches.patch -Patch0010: 0010-cow-Implement-cow-on-read.patch -Patch0011: 0011-delay-Add-delay-open-and-delay-close.patch -Patch0012: 0012-python-Implement-.cleanup-method.patch -Patch0013: 0013-cow-General-revision-and-updates-to-the-manual.patch -Patch0014: 0014-cache-Move-plugin-args-in-synopsis-earlier.patch -Patch0015: 0015-data-Improve-the-example-with-a-diagram.patch -Patch0016: 0016-cow-Add-some-more-debugging-especially-for-blk_read_.patch -Patch0017: 0017-delay-Fix-delay-close.patch -Patch0018: 0018-delay-Test-delay-open-and-delay-close.patch -Patch0019: 0019-vddk-Implement-can_flush-and-can_fua.patch -Patch0020: 0020-vddk-Replace-DEBUG_CALL-with-bracketed-VDDK_CALL_STA.patch -Patch0021: 0021-tests-Add-a-better-test-of-real-VDDK.patch -Patch0022: 0022-vddk-Add-stats-about-the-amount-of-time-spent-in-VDD.patch -Patch0023: 0023-cow-Make-the-block-size-configurable.patch -Patch0024: 0024-cow-Ship-cow.h-header.patch +Patch0001: 0001-vddk-Refactor-how-D-vddk.stats-1-is-collected.patch +Patch0002: 0002-vddk-Extend-D-vddk.stats-1-to-show-number-of-calls-a.patch +Patch0003: 0003-vddk-Simplify-and-consolidate-VDDK_CALL_START-END-ma.patch +Patch0004: 0004-vddk-Document-troubleshooting-performance-problems.patch +Patch0005: 0005-vddk-Include-VDDK-major-library-version-in-dump-plug.patch +Patch0006: 0006-vddk-Add-logical-and-physical-sector-size-to-D-vddk..patch +Patch0007: 0007-vddk-Fix-typo-in-debug-message.patch +Patch0008: 0008-vddk-Only-print-vddk_library_version-when-we-managed.patch +Patch0009: 0009-vddk-Print-one-line-in-dump-plugin-output-for-each-V.patch +Patch0010: 0010-vddk-Move-minimum-version-to-VDDK-6.5.patch +Patch0011: 0011-vddk-Add-read-write-and-wait-asynchronous-functions.patch +Patch0012: 0012-vddk-Start-to-split-VDDK-over-several-files.patch +Patch0013: 0013-vddk-Refactor-D-vddk.stats-1-into-a-new-file.patch +Patch0014: 0014-vddk-Implement-parallel-thread-model.patch +Patch0015: 0015-vddk-Assume-that-VixDiskLib_Flush-is-available.patch +Patch0016: 0016-vddk-Simplify-detection-of-VDDK-symbols-and-baseline.patch +Patch0017: 0017-vddk-Remove-some-whitespace-from-a-couple-of-functio.patch +Patch0018: 0018-vddk-Move-config-debug-error-and-utility-functions-a.patch +Patch0019: 0019-common-utils-test-vector.c-Add-vector-benchmarks.patch +Patch0020: 0020-common-urils-vector.c-Optimize-vector-append.patch +Patch0021: 0021-common-utils-vector-Rename-alloc-to-cap.patch +Patch0022: 0022-common-utils-vector-Rename-size-to-len.patch +Patch0023: 0023-podwrapper.pl.in-Use-short-commit-date.patch +Patch0024: 0024-ocaml-Replace-noalloc-with-noalloc-annotation.patch BuildRequires: make %if 0%{patches_touch_autotools} @@ -125,7 +125,7 @@ BuildRequires: libnbd-devel >= 1.3.11 BuildRequires: libssh-devel BuildRequires: e2fsprogs, e2fsprogs-devel %if !0%{?rhel} -BuildRequires: genisoimage +BuildRequires: xorriso BuildRequires: rb_libtorrent-devel %endif BuildRequires: bash-completion @@ -217,16 +217,18 @@ reading the nbdkit(1) and nbdkit-plugin(3) manual pages. %package server Summary: The %{name} server License: BSD - +Provides: %{name}-null-plugin = %{version}-%{release} %description server -This package contains the %{name} server with no plugins or filters. +This package contains the %{name} server with only the null plugin +and no filters. To install a basic set of plugins and filters you +need to install "nbdkit-basic-plugins", "nbdkit-basic-filters" or +the metapackage "nbdkit". %package basic-plugins Summary: Basic plugins for %{name} License: BSD - Requires: %{name}-server%{?_isa} = %{version}-%{release} Provides: %{name}-data-plugin = %{version}-%{release} Provides: %{name}-eval-plugin = %{version}-%{release} @@ -235,7 +237,6 @@ Provides: %{name}-floppy-plugin = %{version}-%{release} Provides: %{name}-full-plugin = %{version}-%{release} Provides: %{name}-info-plugin = %{version}-%{release} Provides: %{name}-memory-plugin = %{version}-%{release} -Provides: %{name}-null-plugin = %{version}-%{release} Provides: %{name}-ondemand-plugin = %{version}-%{release} Provides: %{name}-pattern-plugin = %{version}-%{release} Provides: %{name}-partitioning-plugin = %{version}-%{release} @@ -265,8 +266,6 @@ nbdkit-info-plugin Serve client and server information. nbdkit-memory-plugin A virtual memory plugin. -nbdkit-null-plugin A null (bitbucket) plugin. - nbdkit-ondemand-plugin Create filesystems on demand. nbdkit-pattern-plugin Fixed test pattern. @@ -287,14 +286,12 @@ nbdkit-zero-plugin Zero-length plugin for testing. %package example-plugins Summary: Example plugins for %{name} License: BSD - Requires: %{name}-server%{?_isa} = %{version}-%{release} %if !0%{?rhel} # example4 is written in Perl. Requires: %{name}-perl-plugin %endif - %description example-plugins This package contains example plugins for %{name}. @@ -306,12 +303,10 @@ This package contains example plugins for %{name}. %package cc-plugin Summary: Write small inline C plugins and scripts for %{name} License: BSD - Requires: %{name}-server%{?_isa} = %{version}-%{release} Requires: gcc Requires: %{_bindir}/cat - %description cc-plugin This package contains support for writing inline C plugins and scripts for %{name}. NOTE this is NOT the right package for writing plugins @@ -323,12 +318,10 @@ in C, install %{name}-devel for that. %package cdi-plugin Summary: Containerized Data Import plugin for %{name} License: BSD - Requires: %{name}-server%{?_isa} = %{version}-%{release} Requires: jq Requires: podman - %description cdi-plugin This package contains Containerized Data Import support for %{name}. %endif @@ -337,10 +330,8 @@ This package contains Containerized Data Import support for %{name}. %package curl-plugin Summary: HTTP/FTP (cURL) plugin for %{name} License: BSD - Requires: %{name}-server%{?_isa} = %{version}-%{release} - %description curl-plugin This package contains cURL (HTTP/FTP) support for %{name}. @@ -349,10 +340,8 @@ This package contains cURL (HTTP/FTP) support for %{name}. %package guestfs-plugin Summary: libguestfs plugin for %{name} License: BSD - Requires: %{name}-server%{?_isa} = %{version}-%{release} - %description guestfs-plugin This package is a libguestfs plugin for %{name}. %endif @@ -362,10 +351,8 @@ This package is a libguestfs plugin for %{name}. %package iso-plugin Summary: Virtual ISO 9660 plugin for %{name} License: BSD - Requires: %{name}-server%{?_isa} = %{version}-%{release} -Requires: genisoimage - +Requires: xorriso %description iso-plugin This package is a virtual ISO 9660 (CD-ROM) plugin for %{name}. @@ -376,10 +363,8 @@ This package is a virtual ISO 9660 (CD-ROM) plugin for %{name}. %package libvirt-plugin Summary: Libvirt plugin for %{name} License: BSD - Requires: %{name}-server%{?_isa} = %{version}-%{release} - %description libvirt-plugin This package is a libvirt plugin for %{name}. It lets you access libvirt guest disks readonly. It is implemented using the libvirt @@ -390,12 +375,10 @@ virDomainBlockPeek API. %package linuxdisk-plugin Summary: Virtual Linux disk plugin for %{name} License: BSD - Requires: %{name}-server%{?_isa} = %{version}-%{release} # for mke2fs Requires: e2fsprogs - %description linuxdisk-plugin This package is a virtual Linux disk plugin for %{name}. @@ -404,10 +387,8 @@ This package is a virtual Linux disk plugin for %{name}. %package lua-plugin Summary: Lua plugin for %{name} License: BSD - Requires: %{name}-server%{?_isa} = %{version}-%{release} - %description lua-plugin This package lets you write Lua plugins for %{name}. %endif @@ -416,10 +397,8 @@ This package lets you write Lua plugins for %{name}. %package nbd-plugin Summary: NBD proxy / forward plugin for %{name} License: BSD - Requires: %{name}-server%{?_isa} = %{version}-%{release} - %description nbd-plugin This package lets you forward NBD connections from %{name} to another NBD server. @@ -429,10 +408,8 @@ to another NBD server. %package ocaml-plugin Summary: OCaml plugin for %{name} License: BSD - Requires: %{name}-server%{?_isa} = %{version}-%{release} - %description ocaml-plugin This package lets you run OCaml plugins for %{name}. @@ -443,11 +420,9 @@ To compile OCaml plugins you will also need to install %package ocaml-plugin-devel Summary: OCaml development environment for %{name} License: BSD - Requires: %{name}-server%{?_isa} = %{version}-%{release} Requires: %{name}-ocaml-plugin%{?_isa} = %{version}-%{release} - %description ocaml-plugin-devel This package lets you write OCaml plugins for %{name}. %endif @@ -457,10 +432,8 @@ This package lets you write OCaml plugins for %{name}. %package perl-plugin Summary: Perl plugin for %{name} License: BSD - Requires: %{name}-server%{?_isa} = %{version}-%{release} - %description perl-plugin This package lets you write Perl plugins for %{name}. %endif @@ -469,10 +442,8 @@ This package lets you write Perl plugins for %{name}. %package python-plugin Summary: Python 3 plugin for %{name} License: BSD - Requires: %{name}-server%{?_isa} = %{version}-%{release} - %description python-plugin This package lets you write Python 3 plugins for %{name}. @@ -481,10 +452,8 @@ This package lets you write Python 3 plugins for %{name}. %package ruby-plugin Summary: Ruby plugin for %{name} License: BSD - Requires: %{name}-server%{?_isa} = %{version}-%{release} - %description ruby-plugin This package lets you write Ruby plugins for %{name}. %endif @@ -496,12 +465,10 @@ This package lets you write Ruby plugins for %{name}. %package S3-plugin Summary: Amazon S3 and Ceph plugin for %{name} License: BSD - Requires: %{name}-python-plugin >= 1.22 # XXX Should not need to add this. Requires: python3-boto3 - %description S3-plugin This package lets you open disk images stored in Amazon S3 or Ceph using %{name}. @@ -511,10 +478,8 @@ or Ceph using %{name}. %package ssh-plugin Summary: SSH plugin for %{name} License: BSD - Requires: %{name}-server%{?_isa} = %{version}-%{release} - %description ssh-plugin This package contains SSH support for %{name}. @@ -523,10 +488,8 @@ This package contains SSH support for %{name}. %package tcl-plugin Summary: Tcl plugin for %{name} License: BSD - Requires: %{name}-server%{?_isa} = %{version}-%{release} - %description tcl-plugin This package lets you write Tcl plugins for %{name}. %endif @@ -535,7 +498,6 @@ This package lets you write Tcl plugins for %{name}. %package tmpdisk-plugin Summary: Remote temporary filesystem disk plugin for %{name} License: BSD - Requires: %{name}-server%{?_isa} = %{version}-%{release} # For mkfs and mke2fs (defaults). Requires: util-linux, e2fsprogs @@ -545,7 +507,6 @@ Suggests: xfsprogs Suggests: ntfsprogs, dosfstools %endif - %description tmpdisk-plugin This package is a remote temporary filesystem disk plugin for %{name}. @@ -554,10 +515,8 @@ This package is a remote temporary filesystem disk plugin for %{name}. %package torrent-plugin Summary: BitTorrent plugin for %{name} License: BSD - Requires: %{name}-server%{?_isa} = %{version}-%{release} - %description torrent-plugin This package is a BitTorrent plugin for %{name}. %endif @@ -567,12 +526,10 @@ This package is a BitTorrent plugin for %{name}. %package vddk-plugin Summary: VMware VDDK plugin for %{name} License: BSD - Requires: %{name}-server%{?_isa} = %{version}-%{release} # https://bugzilla.redhat.com/show_bug.cgi?id=1931818 Requires: libxcrypt-compat - %description vddk-plugin This package is a plugin for %{name} which connects to VMware VDDK for accessing VMware disks and servers. @@ -582,7 +539,6 @@ VMware VDDK for accessing VMware disks and servers. %package basic-filters Summary: Basic filters for %{name} License: BSD - Requires: %{name}-server%{?_isa} = %{version}-%{release} Provides: %{name}-blocksize-filter = %{version}-%{release} Provides: %{name}-cache-filter = %{version}-%{release} @@ -617,7 +573,6 @@ Provides: %{name}-swab-filter = %{version}-%{release} Provides: %{name}-tls-fallback-filter = %{version}-%{release} Provides: %{name}-truncate-filter = %{version}-%{release} - %description basic-filters This package contains filters for %{name} which only depend on simple C libraries: glibc, gnutls. Other filters for nbdkit with more @@ -692,10 +647,8 @@ nbdkit-truncate-filter Truncate, expand, round up or round down size. %package ext2-filter Summary: ext2, ext3 and ext4 filesystem support for %{name} License: BSD - Requires: %{name}-server%{?_isa} = %{version}-%{release} - %description ext2-filter This package contains ext2, ext3 and ext4 filesystem support for %{name}. @@ -705,10 +658,8 @@ This package contains ext2, ext3 and ext4 filesystem support for %package gzip-filter Summary: GZip filter for %{name} License: BSD - Requires: %{name}-server%{?_isa} = %{version}-%{release} - %description gzip-filter This package is a gzip filter for %{name}. @@ -716,13 +667,10 @@ This package is a gzip filter for %{name}. %package tar-filter Summary: Tar archive filter for %{name} License: BSD - Requires: %{name}-server%{?_isa} = %{version}-%{release} Requires: tar - Obsoletes: %{name}-tar-plugin < 1.23.9-3 - %description tar-filter This package is a tar archive filter for %{name}. @@ -730,10 +678,8 @@ This package is a tar archive filter for %{name}. %package xz-filter Summary: XZ filter for %{name} License: BSD - Requires: %{name}-server%{?_isa} = %{version}-%{release} - %description xz-filter This package is the xz filter for %{name}. @@ -741,11 +687,9 @@ This package is the xz filter for %{name}. %package devel Summary: Development files and documentation for %{name} License: BSD - Requires: %{name}-server%{?_isa} = %{version}-%{release} Requires: pkgconfig - %description devel This package contains development files and documentation for %{name}. Install this package if you want to develop @@ -758,7 +702,6 @@ BuildArch: noarch Requires: bash-completion >= 2.0 Requires: %{name}-server = %{version}-%{release} - %description bash-completion Install this package if you want intelligent bash tab-completion for %{name}. @@ -907,11 +850,13 @@ export LIBGUESTFS_TRACE=1 %{_sbindir}/nbdkit %dir %{_libdir}/%{name} %dir %{_libdir}/%{name}/plugins +%{_libdir}/%{name}/plugins/nbdkit-null-plugin.so %dir %{_libdir}/%{name}/filters %{_mandir}/man1/nbdkit.1* %{_mandir}/man1/nbdkit-captive.1* %{_mandir}/man1/nbdkit-client.1* %{_mandir}/man1/nbdkit-loop.1* +%{_mandir}/man1/nbdkit-null-plugin.1* %{_mandir}/man1/nbdkit-probing.1* %{_mandir}/man1/nbdkit-protocol.1* %{_mandir}/man1/nbdkit-service.1* @@ -929,7 +874,6 @@ export LIBGUESTFS_TRACE=1 %{_libdir}/%{name}/plugins/nbdkit-full-plugin.so %{_libdir}/%{name}/plugins/nbdkit-info-plugin.so %{_libdir}/%{name}/plugins/nbdkit-memory-plugin.so -%{_libdir}/%{name}/plugins/nbdkit-null-plugin.so %{_libdir}/%{name}/plugins/nbdkit-ondemand-plugin.so %{_libdir}/%{name}/plugins/nbdkit-partitioning-plugin.so %{_libdir}/%{name}/plugins/nbdkit-pattern-plugin.so @@ -945,7 +889,6 @@ export LIBGUESTFS_TRACE=1 %{_mandir}/man1/nbdkit-full-plugin.1* %{_mandir}/man1/nbdkit-info-plugin.1* %{_mandir}/man1/nbdkit-memory-plugin.1* -%{_mandir}/man1/nbdkit-null-plugin.1* %{_mandir}/man1/nbdkit-ondemand-plugin.1* %{_mandir}/man1/nbdkit-partitioning-plugin.1* %{_mandir}/man1/nbdkit-pattern-plugin.1* @@ -1123,7 +1066,7 @@ export LIBGUESTFS_TRACE=1 %ifarch x86_64 %files vddk-plugin -%doc README +%doc README plugins/vddk/README.VDDK %license LICENSE %{_libdir}/%{name}/plugins/nbdkit-vddk-plugin.so %{_mandir}/man1/nbdkit-vddk-plugin.1* @@ -1270,6 +1213,16 @@ export LIBGUESTFS_TRACE=1 %changelog +* Tue Nov 09 2021 Richard W.M. Jones - 1.28.2-1 +- Move nbdkit-null-plugin to nbdkit-server package + resolves: rhbz#2021154 +- Add asynchronous support in nbdkit-vddk-plugin + resolves: rhbz#2018463 +- Rebase to new stable branch version 1.28.2 + resolves: rhbz#2011709 +- Switch to xorriso (instead of genisoimage) +- Distribute README.VDDK in nbdkit-vddk-plugin subpackage + * Thu Aug 19 2021 Richard W.M. Jones - 1.26.5-1 - Rebase along stable branch to 1.26.5 resolves: rhbz#1995327