diff --git a/0001-server-reset-meta-context-replies-on-starttls.patch b/0001-server-reset-meta-context-replies-on-starttls.patch new file mode 100644 index 0000000..fb51c55 --- /dev/null +++ b/0001-server-reset-meta-context-replies-on-starttls.patch @@ -0,0 +1,39 @@ +From 89ef17c90996c0e212e3a17c8d26ff930ab464ea Mon Sep 17 00:00:00 2001 +From: Eric Blake +Date: Mon, 16 Aug 2021 13:43:29 -0500 +Subject: [PATCH] server: reset meta context replies on starttls + +Related to CVE-2021-3716, but not as severe. No compliant client will +send NBD_CMD_BLOCK_STATUS unless it first negotiates +NBD_OPT_SET_META_CONTEXT. If an attacker injects a premature +SET_META_CONTEXT, either the client will never notice (because it +never uses BLOCK_STATUS), or the client will overwrite the attacker's +attempt with the client's own SET_META_CONTEXT request after +encryption is enabled. So I don't class this as having the potential +to trigger denial-of-service due to any protocol mismatch between +compliant client and server (I don't care what happens with +non-compliant clients). + +Fixes: 26455d45 (server: protocol: Implement Block Status "base:allocation".) +(cherry picked from commit 6c5faac6a37077cf2366388a80862bb00616d0d8) +--- + server/protocol-handshake-newstyle.c | 3 +++ + 1 file changed, 3 insertions(+) + +diff --git a/server/protocol-handshake-newstyle.c b/server/protocol-handshake-newstyle.c +index 7e6b7b1b..79b2c8ce 100644 +--- a/server/protocol-handshake-newstyle.c ++++ b/server/protocol-handshake-newstyle.c +@@ -497,6 +497,9 @@ negotiate_handshake_newstyle_options (void) + debug ("using TLS on this connection"); + /* Wipe out any cached state. */ + conn->structured_replies = false; ++ free (conn->exportname_from_set_meta_context); ++ conn->exportname_from_set_meta_context = NULL; ++ conn->meta_context_base_allocation = false; + for_each_backend (b) { + free (conn->default_exportname[b->i]); + conn->default_exportname[b->i] = NULL; +-- +2.31.1 + diff --git a/0001-cache-Reduce-verbosity-of-debugging.patch b/0002-cache-Reduce-verbosity-of-debugging.patch similarity index 98% rename from 0001-cache-Reduce-verbosity-of-debugging.patch rename to 0002-cache-Reduce-verbosity-of-debugging.patch index 4d6f23c..5ddcebc 100644 --- a/0001-cache-Reduce-verbosity-of-debugging.patch +++ b/0002-cache-Reduce-verbosity-of-debugging.patch @@ -1,4 +1,4 @@ -From 8785f90134fa912e31e72190d217db9c39754fcf Mon Sep 17 00:00:00 2001 +From 4b576a8e0eb99ec1a79ca432350fb7ac27a5c089 Mon Sep 17 00:00:00 2001 From: "Richard W.M. Jones" Date: Mon, 26 Jul 2021 11:59:43 +0100 Subject: [PATCH] cache: Reduce verbosity of debugging diff --git a/0002-cache-cow-Add-blk_read_multiple-function.patch b/0003-cache-cow-Add-blk_read_multiple-function.patch similarity index 98% rename from 0002-cache-cow-Add-blk_read_multiple-function.patch rename to 0003-cache-cow-Add-blk_read_multiple-function.patch index b222302..5b56637 100644 --- a/0002-cache-cow-Add-blk_read_multiple-function.patch +++ b/0003-cache-cow-Add-blk_read_multiple-function.patch @@ -1,4 +1,4 @@ -From f6d831cd517851157e822ee96de5894e0b37c22d Mon Sep 17 00:00:00 2001 +From b5dc8577c5c6d1205e2106b629fad327c3a409ea Mon Sep 17 00:00:00 2001 From: "Richard W.M. Jones" Date: Mon, 26 Jul 2021 13:55:21 +0100 Subject: [PATCH] cache, cow: Add blk_read_multiple function @@ -224,7 +224,7 @@ index 745f552d..14cc03f2 100644 /* Unaligned tail */ diff --git a/filters/cow/blk.c b/filters/cow/blk.c -index 0f12d510..9e6c8879 100644 +index b7c4d7f1..4ec8d1b8 100644 --- a/filters/cow/blk.c +++ b/filters/cow/blk.c @@ -79,6 +79,7 @@ @@ -235,7 +235,7 @@ index 0f12d510..9e6c8879 100644 #include #include -@@ -223,33 +224,48 @@ blk_status (uint64_t blknum, bool *present, bool *trimmed) +@@ -219,33 +220,48 @@ blk_status (uint64_t blknum, bool *present, bool *trimmed) *trimmed = state == BLOCK_TRIMMED; } @@ -295,7 +295,7 @@ index 0f12d510..9e6c8879 100644 if (offset + n > size) { tail = offset + n - size; -@@ -264,20 +280,35 @@ blk_read (nbdkit_next *next, +@@ -260,20 +276,35 @@ blk_read (nbdkit_next *next, * zeroing the tail. */ memset (block + n, 0, tail); @@ -354,7 +354,7 @@ index e6fd7417..b066c602 100644 enum cache_mode { BLK_CACHE_IGNORE, /* Do nothing */ diff --git a/filters/cow/cow.c b/filters/cow/cow.c -index 6cefee36..e939f23f 100644 +index f30b7505..78daca22 100644 --- a/filters/cow/cow.c +++ b/filters/cow/cow.c @@ -210,7 +210,7 @@ cow_pread (nbdkit_next *next, diff --git a/0003-cache-cow-Use-full-pread-pwrite-operations.patch b/0004-cache-cow-Use-full-pread-pwrite-operations.patch similarity index 96% rename from 0003-cache-cow-Use-full-pread-pwrite-operations.patch rename to 0004-cache-cow-Use-full-pread-pwrite-operations.patch index f6fdd51..300aeb2 100644 --- a/0003-cache-cow-Use-full-pread-pwrite-operations.patch +++ b/0004-cache-cow-Use-full-pread-pwrite-operations.patch @@ -1,4 +1,4 @@ -From 1b21e41dfd69b0a5f51657bc6f8b1cbc49663ee2 Mon Sep 17 00:00:00 2001 +From 5bd332a683811586039f99f31c01d4f2f7181334 Mon Sep 17 00:00:00 2001 From: "Richard W.M. Jones" Date: Mon, 26 Jul 2021 15:21:18 +0100 Subject: [PATCH] cache, cow: Use full pread/pwrite operations @@ -180,10 +180,10 @@ index f85ada35..42bd3779 100644 nbdkit_error ("pwrite: %m"); return -1; diff --git a/filters/cow/blk.c b/filters/cow/blk.c -index 9e6c8879..cebd9454 100644 +index 4ec8d1b8..121b0dd4 100644 --- a/filters/cow/blk.c +++ b/filters/cow/blk.c -@@ -282,7 +282,7 @@ blk_read_multiple (nbdkit_next *next, +@@ -278,7 +278,7 @@ blk_read_multiple (nbdkit_next *next, memset (block + n, 0, tail); } else if (state == BLOCK_ALLOCATED) { /* Read overlay. */ @@ -192,7 +192,7 @@ index 9e6c8879..cebd9454 100644 *err = errno; nbdkit_error ("pread: %m"); return -1; -@@ -357,7 +357,7 @@ blk_cache (nbdkit_next *next, +@@ -353,7 +353,7 @@ blk_cache (nbdkit_next *next, memset (block + n, 0, tail); if (mode == BLK_CACHE_COW) { @@ -201,7 +201,7 @@ index 9e6c8879..cebd9454 100644 *err = errno; nbdkit_error ("pwrite: %m"); return -1; -@@ -376,7 +376,7 @@ blk_write (uint64_t blknum, const uint8_t *block, int *err) +@@ -372,7 +372,7 @@ blk_write (uint64_t blknum, const uint8_t *block, int *err) nbdkit_debug ("cow: blk_write block %" PRIu64 " (offset %" PRIu64 ")", blknum, (uint64_t) offset); diff --git a/0004-cache-Implement-cache-on-read-PATH.patch b/0005-cache-Implement-cache-on-read-PATH.patch similarity index 98% rename from 0004-cache-Implement-cache-on-read-PATH.patch rename to 0005-cache-Implement-cache-on-read-PATH.patch index d2c5b9e..559a659 100644 --- a/0004-cache-Implement-cache-on-read-PATH.patch +++ b/0005-cache-Implement-cache-on-read-PATH.patch @@ -1,4 +1,4 @@ -From dac1245e839d64cc5ee22ae4f804a950e487ff5d Mon Sep 17 00:00:00 2001 +From 4db23fd29af0488aa9c7e01577a5be9565a4465e Mon Sep 17 00:00:00 2001 From: "Richard W.M. Jones" Date: Mon, 26 Jul 2021 16:16:15 +0100 Subject: [PATCH] cache: Implement cache-on-read=/PATH diff --git a/0005-cache-Add-cache-min-block-size-parameter.patch b/0006-cache-Add-cache-min-block-size-parameter.patch similarity index 99% rename from 0005-cache-Add-cache-min-block-size-parameter.patch rename to 0006-cache-Add-cache-min-block-size-parameter.patch index f97cba1..4ca0778 100644 --- a/0005-cache-Add-cache-min-block-size-parameter.patch +++ b/0006-cache-Add-cache-min-block-size-parameter.patch @@ -1,4 +1,4 @@ -From 9d1d3bb689a6a88bcc376b699d9668a3a623faee Mon Sep 17 00:00:00 2001 +From f7f4b71d559dc6950bc795742f64e8eaeeadf3ec Mon Sep 17 00:00:00 2001 From: "Richard W.M. Jones" Date: Mon, 26 Jul 2021 16:30:26 +0100 Subject: [PATCH] cache: Add cache-min-block-size parameter diff --git a/0006-cache-cow-Use-a-64K-block-size-by-default.patch b/0007-cache-cow-Use-a-64K-block-size-by-default.patch similarity index 98% rename from 0006-cache-cow-Use-a-64K-block-size-by-default.patch rename to 0007-cache-cow-Use-a-64K-block-size-by-default.patch index ba02c9e..ff5870f 100644 --- a/0006-cache-cow-Use-a-64K-block-size-by-default.patch +++ b/0007-cache-cow-Use-a-64K-block-size-by-default.patch @@ -1,4 +1,4 @@ -From e0f76fe019449c81297de39ddd6a12006d94481a Mon Sep 17 00:00:00 2001 +From 83e1167e1a350bd08ac6245f47a5877438408492 Mon Sep 17 00:00:00 2001 From: "Richard W.M. Jones" Date: Mon, 26 Jul 2021 17:39:23 +0100 Subject: [PATCH] cache, cow: Use a 64K block size by default diff --git a/0007-cache-Refactor-printing-state-into-new-function.patch b/0008-cache-Refactor-printing-state-into-new-function.patch similarity index 95% rename from 0007-cache-Refactor-printing-state-into-new-function.patch rename to 0008-cache-Refactor-printing-state-into-new-function.patch index 80de09f..f42a1bb 100644 --- a/0007-cache-Refactor-printing-state-into-new-function.patch +++ b/0008-cache-Refactor-printing-state-into-new-function.patch @@ -1,4 +1,4 @@ -From 9b4f5045c92f6a666c8d4b08379c34fb8e677dcd Mon Sep 17 00:00:00 2001 +From 2592bb42051b3e6d17240badc814b9b16f121c1d Mon Sep 17 00:00:00 2001 From: "Richard W.M. Jones" Date: Tue, 27 Jul 2021 21:16:30 +0100 Subject: [PATCH] cache: Refactor printing state into new function diff --git a/0008-tests-cache-Test-cache-on-read-option-really-caches.patch b/0009-tests-cache-Test-cache-on-read-option-really-caches.patch similarity index 98% rename from 0008-tests-cache-Test-cache-on-read-option-really-caches.patch rename to 0009-tests-cache-Test-cache-on-read-option-really-caches.patch index c851232..f81121e 100644 --- a/0008-tests-cache-Test-cache-on-read-option-really-caches.patch +++ b/0009-tests-cache-Test-cache-on-read-option-really-caches.patch @@ -1,4 +1,4 @@ -From fcd113cf7bf5587f8cac6d331ec1e427be0a830f Mon Sep 17 00:00:00 2001 +From 315948e75e06d038bd8afa319a41e3fde33b4174 Mon Sep 17 00:00:00 2001 From: "Richard W.M. Jones" Date: Thu, 29 Jul 2021 20:16:43 +0100 Subject: [PATCH] tests: cache: Test cache-on-read option really caches diff --git a/0009-cow-Implement-cow-on-read.patch b/0010-cow-Implement-cow-on-read.patch similarity index 97% rename from 0009-cow-Implement-cow-on-read.patch rename to 0010-cow-Implement-cow-on-read.patch index 041364c..fd5fcfd 100644 --- a/0009-cow-Implement-cow-on-read.patch +++ b/0010-cow-Implement-cow-on-read.patch @@ -1,4 +1,4 @@ -From e7ff9f833cc11e47db90868a02fee95900f62a84 Mon Sep 17 00:00:00 2001 +From 57f9bd29f9d7432ad5a70620c373b28db768a314 Mon Sep 17 00:00:00 2001 From: "Richard W.M. Jones" Date: Tue, 27 Jul 2021 23:01:52 +0100 Subject: [PATCH] cow: Implement cow-on-read @@ -19,10 +19,10 @@ This is very similar to the nbdkit-cache-filter cache-on-read flag. create mode 100755 tests/test-cow-on-read.sh diff --git a/filters/cow/blk.c b/filters/cow/blk.c -index cebd9454..9d42b5fc 100644 +index 121b0dd4..4f84e092 100644 --- a/filters/cow/blk.c +++ b/filters/cow/blk.c -@@ -230,7 +230,7 @@ blk_status (uint64_t blknum, bool *present, bool *trimmed) +@@ -226,7 +226,7 @@ blk_status (uint64_t blknum, bool *present, bool *trimmed) int blk_read_multiple (nbdkit_next *next, uint64_t blknum, uint64_t nrblocks, @@ -31,7 +31,7 @@ index cebd9454..9d42b5fc 100644 { off_t offset = blknum * BLKSIZE; enum bm_entry state; -@@ -280,6 +280,19 @@ blk_read_multiple (nbdkit_next *next, +@@ -276,6 +276,19 @@ blk_read_multiple (nbdkit_next *next, * zeroing the tail. */ memset (block + n, 0, tail); @@ -51,7 +51,7 @@ index cebd9454..9d42b5fc 100644 } else if (state == BLOCK_ALLOCATED) { /* Read overlay. */ if (full_pread (fd, block, BLKSIZE * runblocks, offset) == -1) { -@@ -301,14 +314,14 @@ blk_read_multiple (nbdkit_next *next, +@@ -297,14 +310,14 @@ blk_read_multiple (nbdkit_next *next, blknum + runblocks, nrblocks - runblocks, block + BLKSIZE * runblocks, @@ -95,7 +95,7 @@ index 1bc85283..b7e6f092 100644 /* Cache mode for blocks not already in overlay */ enum cache_mode { diff --git a/filters/cow/cow.c b/filters/cow/cow.c -index e939f23f..6ad42eec 100644 +index 78daca22..6efb39f2 100644 --- a/filters/cow/cow.c +++ b/filters/cow/cow.c @@ -38,6 +38,7 @@ @@ -108,7 +108,7 @@ index e939f23f..6ad42eec 100644 #include @@ -59,6 +60,15 @@ static pthread_mutex_t rmw_lock = PTHREAD_MUTEX_INITIALIZER; - bool cow_on_cache; + static bool cow_on_cache; +/* Cache on read ("cow-on-read") mode. */ +extern enum cor_mode { diff --git a/0010-delay-Add-delay-open-and-delay-close.patch b/0011-delay-Add-delay-open-and-delay-close.patch similarity index 79% rename from 0010-delay-Add-delay-open-and-delay-close.patch rename to 0011-delay-Add-delay-open-and-delay-close.patch index f0e34de..0faf658 100644 --- a/0010-delay-Add-delay-open-and-delay-close.patch +++ b/0011-delay-Add-delay-open-and-delay-close.patch @@ -1,4 +1,4 @@ -From a37d5a2176af0362ad3e4234fe1fc14a47978261 Mon Sep 17 00:00:00 2001 +From a7e7af18d64164fac42581452f6dc3c07650fcae Mon Sep 17 00:00:00 2001 From: "Richard W.M. Jones" Date: Fri, 30 Jul 2021 10:19:57 +0100 Subject: [PATCH] delay: Add delay-open and delay-close @@ -7,24 +7,24 @@ Useful for simulating VDDK which has very slow connection. (cherry picked from commit de8dcd3a34a38b088a0f9a6f8ca754702ad1f598) --- - filters/delay/delay.c | 60 ++++++++++++++++++++++++++- - filters/delay/nbdkit-delay-filter.pod | 27 ++++++++++-- - 2 files changed, 82 insertions(+), 5 deletions(-) + filters/delay/delay.c | 58 ++++++++++++++++++++++++++- + filters/delay/nbdkit-delay-filter.pod | 27 +++++++++++-- + 2 files changed, 80 insertions(+), 5 deletions(-) diff --git a/filters/delay/delay.c b/filters/delay/delay.c -index 7e7fe195..5bd21321 100644 +index 5a925aa4..df3729a7 100644 --- a/filters/delay/delay.c +++ b/filters/delay/delay.c -@@ -49,6 +49,8 @@ static int delay_trim_ms = 0; /* trim delay (milliseconds) */ - static int delay_extents_ms = 0;/* extents delay (milliseconds) */ - static int delay_cache_ms = 0; /* cache delay (milliseconds) */ - static int delay_fast_zero = 1; /* whether delaying zero includes fast zero */ -+static int delay_open_ms = 0; /* open delay (milliseconds) */ -+static int delay_close_ms = 0; /* close delay (milliseconds) */ +@@ -48,6 +48,8 @@ static unsigned delay_zero_ms = 0; /* zero delay (milliseconds) */ + static unsigned delay_trim_ms = 0; /* trim delay (milliseconds) */ + static unsigned delay_extents_ms = 0;/* extents delay (milliseconds) */ + static unsigned delay_cache_ms = 0; /* cache delay (milliseconds) */ ++static unsigned delay_open_ms = 0; /* open delay (milliseconds) */ ++static unsigned delay_close_ms = 0; /* close delay (milliseconds) */ - static int - parse_delay (const char *key, const char *value) -@@ -128,6 +130,18 @@ cache_delay (int *err) + static int delay_fast_zero = 1; /* whether delaying zero includes fast zero */ + +@@ -126,6 +128,18 @@ cache_delay (int *err) return delay (delay_cache_ms, err); } @@ -43,26 +43,24 @@ index 7e7fe195..5bd21321 100644 /* Called for each key=value passed on the command line. */ static int delay_config (nbdkit_next_config *next, nbdkit_backend *nxdata, -@@ -191,6 +205,18 @@ delay_config (nbdkit_next_config *next, nbdkit_backend *nxdata, +@@ -182,6 +196,16 @@ delay_config (nbdkit_next_config *next, nbdkit_backend *nxdata, return -1; return 0; } + else if (strcmp (key, "delay-open") == 0) { -+ delay_open_ms = parse_delay (key, value); -+ if (delay_open_ms == -1) ++ if (parse_delay (key, value, &delay_open_ms) == -1) + return -1; + return 0; + } + else if (strcmp (key, "delay-close") == 0) { -+ delay_close_ms = parse_delay (key, value); -+ if (delay_close_ms == -1) ++ if (parse_delay (key, value, &delay_close_ms) == -1) + return -1; + return 0; + } else return next (nxdata, key, value); } -@@ -204,7 +230,9 @@ delay_config (nbdkit_next_config *next, nbdkit_backend *nxdata, +@@ -195,7 +219,9 @@ delay_config (nbdkit_next_config *next, nbdkit_backend *nxdata, "delay-extents=[ms] Extents delay in seconds/milliseconds.\n" \ "delay-cache=[ms] Cache delay in seconds/milliseconds.\n" \ "wdelay=[ms] Write, zero and trim delay in secs/msecs.\n" \ @@ -73,7 +71,7 @@ index 7e7fe195..5bd21321 100644 /* Override the plugin's .can_fast_zero if needed */ static int -@@ -217,6 +245,34 @@ delay_can_fast_zero (nbdkit_next *next, +@@ -208,6 +234,34 @@ delay_can_fast_zero (nbdkit_next *next, return next->can_fast_zero (next); } @@ -108,7 +106,7 @@ index 7e7fe195..5bd21321 100644 /* Read data. */ static int delay_pread (nbdkit_next *next, -@@ -294,6 +350,8 @@ static struct nbdkit_filter filter = { +@@ -285,6 +339,8 @@ static struct nbdkit_filter filter = { .config = delay_config, .config_help = delay_config_help, .can_fast_zero = delay_can_fast_zero, diff --git a/0011-python-Implement-.cleanup-method.patch b/0012-python-Implement-.cleanup-method.patch similarity index 97% rename from 0011-python-Implement-.cleanup-method.patch rename to 0012-python-Implement-.cleanup-method.patch index 1d37926..321c669 100644 --- a/0011-python-Implement-.cleanup-method.patch +++ b/0012-python-Implement-.cleanup-method.patch @@ -1,4 +1,4 @@ -From 77310049efc37daaf7bd993298000f8f3638497e Mon Sep 17 00:00:00 2001 +From 17a912a449fa75b5c12ac3acab596b476699c671 Mon Sep 17 00:00:00 2001 From: "Richard W.M. Jones" Date: Tue, 3 Aug 2021 14:19:38 +0100 Subject: [PATCH] python: Implement .cleanup() method diff --git a/0012-server-Make-debug-messages-atomic.patch b/0012-server-Make-debug-messages-atomic.patch deleted file mode 100644 index 0e97b71..0000000 --- a/0012-server-Make-debug-messages-atomic.patch +++ /dev/null @@ -1,157 +0,0 @@ -From 453a7611b625fc6f306a47ccc61bf3b83d75e522 Mon Sep 17 00:00:00 2001 -From: "Richard W.M. Jones" -Date: Thu, 5 Aug 2021 10:13:53 +0100 -Subject: [PATCH] server: Make debug messages atomic - -Previously we used flockfile around the separate fprintf statements -used to emit debug messages. However this does not make debug -messages atomic (only atomic with respect to other nbdkit threads). -It's easy to see this in strace: - -1915655 write(2, "nbdkit: ", 8) = 8 -1915655 write(2, "debug: ", 7) = 7 -1915655 write(2, "nbdkit 1.27.3 (nbdkit-1.27.3-1.f"..., 36) = 36 -1915655 write(2, "\n", 1) = 1 - -We require open_memstream already, so use this to print messages to a -string buffer which we can emit in a single write. - -This fixes various tests which grep the log file looking for sentinel -messages. In particular a recent Fedora build failure in -test-nbdkit-backend-debug.sh was caused by this, but there are other -tests that fail occasionally for the same reason. - -This change also tries harder to set errno correctly before expanding -the format string (for debug messages that contain %m). - -Note that libnbd already emits atomic messages. - -(cherry picked from commit 30eef3bf2d93b12072f91f95987bae33f9e3fe1a) ---- - server/debug.c | 72 ++++++++++++++++++++++++++++++++------------------ - 1 file changed, 46 insertions(+), 26 deletions(-) - -diff --git a/server/debug.c b/server/debug.c -index eede5e16..07710581 100644 ---- a/server/debug.c -+++ b/server/debug.c -@@ -40,23 +40,22 @@ - - #include "internal.h" - --/* Called with flockfile (stderr) taken. */ - static void --prologue (void) -+prologue (FILE *fp) - { - const char *name = threadlocal_get_name (); - size_t instance_num = threadlocal_get_instance_num (); - -- fprintf (stderr, "%s: ", program_name); -+ fprintf (fp, "%s: ", program_name); - - if (name) { -- fprintf (stderr, "%s", name); -+ fprintf (fp, "%s", name); - if (instance_num > 0) -- fprintf (stderr, "[%zu]", instance_num); -- fprintf (stderr, ": "); -+ fprintf (fp, "[%zu]", instance_num); -+ fprintf (fp, ": "); - } - -- fprintf (stderr, "debug: "); -+ fprintf (fp, "debug: "); - } - - /* Note: preserves the previous value of errno. */ -@@ -64,20 +63,30 @@ NBDKIT_DLL_PUBLIC void - nbdkit_vdebug (const char *fs, va_list args) - { - int err = errno; -+ CLEANUP_FREE char *str = NULL; -+ size_t len = 0; -+ FILE *fp; - - if (!verbose) - return; --#ifdef HAVE_FLOCKFILE -- flockfile (stderr); --#endif -- prologue (); - -- vfprintf (stderr, fs, args); -+ fp = open_memstream (&str, &len); -+ if (fp == NULL) { -+ /* Try to emit what we can. */ -+ errno = err; -+ vfprintf (stderr, fs, args); -+ return; -+ } - -- fprintf (stderr, "\n"); --#ifdef HAVE_FUNLOCKFILE -- funlockfile (stderr); --#endif -+ prologue (fp); -+ -+ errno = err; -+ vfprintf (fp, fs, args); -+ -+ fprintf (fp, "\n"); -+ fclose (fp); -+ -+ fputs (str, stderr); - - errno = err; - } -@@ -86,25 +95,36 @@ nbdkit_vdebug (const char *fs, va_list args) - NBDKIT_DLL_PUBLIC void - nbdkit_debug (const char *fs, ...) - { -+ int err = errno; - va_list args; -- int err = errno; -+ CLEANUP_FREE char *str = NULL; -+ size_t len = 0; -+ FILE *fp; - - if (!verbose) - return; - --#ifdef HAVE_FLOCKFILE -- flockfile (stderr); --#endif -- prologue (); -+ fp = open_memstream (&str, &len); -+ if (fp == NULL) { -+ /* Try to emit what we can. */ -+ va_start (args, fs); -+ errno = err; -+ vfprintf (stderr, fs, args); -+ va_end (args); -+ return; -+ } -+ -+ prologue (fp); - - va_start (args, fs); -- vfprintf (stderr, fs, args); -+ errno = err; -+ vfprintf (fp, fs, args); - va_end (args); - -- fprintf (stderr, "\n"); --#ifdef HAVE_FUNLOCKFILE -- funlockfile (stderr); --#endif -+ fprintf (fp, "\n"); -+ fclose (fp); -+ -+ fputs (str, stderr); - - errno = err; - } --- -2.31.1 - diff --git a/0013-cow-General-revision-and-updates-to-the-manual.patch b/0013-cow-General-revision-and-updates-to-the-manual.patch index 1ae7c56..b3d8381 100644 --- a/0013-cow-General-revision-and-updates-to-the-manual.patch +++ b/0013-cow-General-revision-and-updates-to-the-manual.patch @@ -1,4 +1,4 @@ -From d15e320d20260c14973ef84172ae8cbe337a2b48 Mon Sep 17 00:00:00 2001 +From e9abe97b40fef6f9bd9028a2520f45203bba0749 Mon Sep 17 00:00:00 2001 From: "Richard W.M. Jones" Date: Thu, 5 Aug 2021 18:18:34 +0100 Subject: [PATCH] cow: General revision and updates to the manual diff --git a/0014-cache-Move-plugin-args-in-synopsis-earlier.patch b/0014-cache-Move-plugin-args-in-synopsis-earlier.patch index b30531d..5efa94d 100644 --- a/0014-cache-Move-plugin-args-in-synopsis-earlier.patch +++ b/0014-cache-Move-plugin-args-in-synopsis-earlier.patch @@ -1,4 +1,4 @@ -From 583d5308ea8d26248e521b76afb380432d2084bc Mon Sep 17 00:00:00 2001 +From c8c1e74a8c1c112b83646ac09fe7f9bde097a52a Mon Sep 17 00:00:00 2001 From: "Richard W.M. Jones" Date: Thu, 5 Aug 2021 18:20:37 +0100 Subject: [PATCH] cache: Move plugin-args in synopsis earlier diff --git a/0015-data-Improve-the-example-with-a-diagram.patch b/0015-data-Improve-the-example-with-a-diagram.patch index 075cdca..4ff51f6 100644 --- a/0015-data-Improve-the-example-with-a-diagram.patch +++ b/0015-data-Improve-the-example-with-a-diagram.patch @@ -1,4 +1,4 @@ -From 72b87c985dc9324d896333f9ddaf317cece8a812 Mon Sep 17 00:00:00 2001 +From 0eae7ebf6f714fb339f4a476b65e070b528824ec Mon Sep 17 00:00:00 2001 From: "Richard W.M. Jones" Date: Sun, 8 Aug 2021 16:32:38 +0100 Subject: [PATCH] data: Improve the example with a diagram diff --git a/0016-cow-Add-some-more-debugging-especially-for-blk_read_.patch b/0016-cow-Add-some-more-debugging-especially-for-blk_read_.patch index 68175ca..e9c1228 100644 --- a/0016-cow-Add-some-more-debugging-especially-for-blk_read_.patch +++ b/0016-cow-Add-some-more-debugging-especially-for-blk_read_.patch @@ -1,4 +1,4 @@ -From d2ed77d1b8fedcba3aedadeed883553886f4bb56 Mon Sep 17 00:00:00 2001 +From a22248e3075e782d28542f8f6acd046c9dfa8998 Mon Sep 17 00:00:00 2001 From: "Richard W.M. Jones" Date: Mon, 9 Aug 2021 14:09:31 +0100 Subject: [PATCH] cow: Add some more debugging especially for blk_read_multiple @@ -12,10 +12,10 @@ Only activated when we use -D cow.verbose=1 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/filters/cow/blk.c b/filters/cow/blk.c -index 9d42b5fc..879f471a 100644 +index 4f84e092..c22d5886 100644 --- a/filters/cow/blk.c +++ b/filters/cow/blk.c -@@ -258,8 +258,10 @@ blk_read_multiple (nbdkit_next *next, +@@ -254,8 +254,10 @@ blk_read_multiple (nbdkit_next *next, if (cow_debug_verbose) nbdkit_debug ("cow: blk_read_multiple block %" PRIu64 @@ -28,7 +28,7 @@ index 9d42b5fc..879f471a 100644 if (state == BLOCK_NOT_ALLOCATED) { /* Read underlying plugin. */ unsigned n, tail = 0; -@@ -285,6 +287,11 @@ blk_read_multiple (nbdkit_next *next, +@@ -281,6 +283,11 @@ blk_read_multiple (nbdkit_next *next, * set them as allocated. */ if (cow_on_read) { diff --git a/0021-delay-Fix-delay-close.patch b/0017-delay-Fix-delay-close.patch similarity index 98% rename from 0021-delay-Fix-delay-close.patch rename to 0017-delay-Fix-delay-close.patch index d6d2091..3d0f3e7 100644 --- a/0021-delay-Fix-delay-close.patch +++ b/0017-delay-Fix-delay-close.patch @@ -1,4 +1,4 @@ -From a8ea42d2aa3738c680a932e2c42257ce4a880a47 Mon Sep 17 00:00:00 2001 +From be7252bada79ee542356dffaf5f3c568a5c7fec3 Mon Sep 17 00:00:00 2001 From: "Richard W.M. Jones" Date: Tue, 10 Aug 2021 08:39:15 +0100 Subject: [PATCH] delay: Fix delay-close diff --git a/0017-tests-test-debug-flags.sh-Don-t-use-port-10809-durin.patch b/0017-tests-test-debug-flags.sh-Don-t-use-port-10809-durin.patch deleted file mode 100644 index 1462493..0000000 --- a/0017-tests-test-debug-flags.sh-Don-t-use-port-10809-durin.patch +++ /dev/null @@ -1,74 +0,0 @@ -From ad2b4d2c07def233b2192c2a7ff925d1b6b823e7 Mon Sep 17 00:00:00 2001 -From: "Richard W.M. Jones" -Date: Mon, 9 Aug 2021 14:48:15 +0100 -Subject: [PATCH] tests/test-debug-flags.sh: Don't use port 10809 during test - -Use a temporary Unix domain socket (-U -) instead. - -Reported-by: Ming Xie -Fixes: commit 0e3a54f78f4ab0cbe4bee2b965ec0e610c399a6e -(cherry picked from commit 31f4e5c31825485e7bb39d170a7102ddbc4043c2) ---- - tests/test-debug-flags.sh | 22 +++++++++++----------- - 1 file changed, 11 insertions(+), 11 deletions(-) - -diff --git a/tests/test-debug-flags.sh b/tests/test-debug-flags.sh -index 73be9904..2be799a1 100755 ---- a/tests/test-debug-flags.sh -+++ b/tests/test-debug-flags.sh -@@ -46,7 +46,7 @@ rm -f $files - cleanup_fn rm -f $files - - # This should work and show the "extra debugging" line in debug output. --nbdkit -f -v -D example2.extra=1 example2 file=disk \ -+nbdkit -U - -f -v -D example2.extra=1 example2 file=disk \ - --run 'nbdinfo "$uri"' 2>debug-flags.out - cat debug-flags.out - if ! grep -sq 'extra debugging:' debug-flags.out ; then -@@ -82,33 +82,33 @@ check_warning () - - # This is expected to fail because we didn't set the file= parameter, - # but it should not fail because of the debug flag. --nbdkit -f -D example2.extra=1 example2 2>debug-flags.out && expected_failure -+nbdkit -U - -f -D example2.extra=1 example2 2>debug-flags.out && expected_failure - check_error "you must supply the file=" - - # This should fail because we didn't set the file= parameter, but it - # should also print a warning about the unknown -D flag. --nbdkit -f -D example2.unknown=1 example2 2>debug-flags.out && expected_failure -+nbdkit -U - -f -D example2.unknown=1 example2 2>debug-flags.out && expected_failure - check_error "you must supply the file=" - check_warning "does not contain a global variable called example2_debug_unknown" - - # This should fail because we didn't set the file= parameter, but it - # should also print a warning because the -D flag is unused. --nbdkit -f -D example1.foo=1 example2 2>debug-flags.out && expected_failure -+nbdkit -U - -f -D example1.foo=1 example2 2>debug-flags.out && expected_failure - check_error "you must supply the file=" - check_warning "was not used" - - # These should fail because the -D flag has a bad format. --nbdkit -f -D = example2 2>debug-flags.out && expected_failure -+nbdkit -U - -f -D = example2 2>debug-flags.out && expected_failure - check_error "must have the format" --nbdkit -f -D . example2 2>debug-flags.out && expected_failure -+nbdkit -U - -f -D . example2 2>debug-flags.out && expected_failure - check_error "must have the format" --nbdkit -f -D =. example2 2>debug-flags.out && expected_failure -+nbdkit -U - -f -D =. example2 2>debug-flags.out && expected_failure - check_error "must have the format" --nbdkit -f -D .= example2 2>debug-flags.out && expected_failure -+nbdkit -U - -f -D .= example2 2>debug-flags.out && expected_failure - check_error "must have the format" --nbdkit -f -D .extra=1 example2 2>debug-flags.out && expected_failure -+nbdkit -U - -f -D .extra=1 example2 2>debug-flags.out && expected_failure - check_error "must have the format" --nbdkit -f -D example2.=1 example2 2>debug-flags.out && expected_failure -+nbdkit -U - -f -D example2.=1 example2 2>debug-flags.out && expected_failure - check_error "must have the format" --nbdkit -f -D example2.extra= example2 2>debug-flags.out && expected_failure -+nbdkit -U - -f -D example2.extra= example2 2>debug-flags.out && expected_failure - check_error "must have the format" --- -2.31.1 - diff --git a/0018-delay-Improve-parsing-and-representation-of-delay-ti.patch b/0018-delay-Improve-parsing-and-representation-of-delay-ti.patch deleted file mode 100644 index 09d2141..0000000 --- a/0018-delay-Improve-parsing-and-representation-of-delay-ti.patch +++ /dev/null @@ -1,182 +0,0 @@ -From 810c2449cb519100bf9ea50d743162a391eac873 Mon Sep 17 00:00:00 2001 -From: "Richard W.M. Jones" -Date: Mon, 9 Aug 2021 16:16:36 +0100 -Subject: [PATCH] delay: Improve parsing and representation of delay times - -We used sscanf to parse the delay settings for both NNms and NN -(seconds). In the first case we need to use sscanf otherwise the code -is awkward. In the second case using sscanf meant that bogus suffices -were not ignored, eg: - - $ nbdkit null --filter=delay delay-open=10SECS --run 'nbdinfo $uri' - -Use nbdkit_parse_unsigned instead. This now results in an error: - - nbdkit: error: delay-open: could not parse number: "10SECS": trailing garbage - -Also, previously we stored delay times as an int. This makes no sense -since they cannot be negative, so use an unsigned instead. - -Reported-by: Ming Xie -(cherry picked from commit 58187831a4346b44e398f105163abac8f3dfb7f0) ---- - filters/delay/delay.c | 73 ++++++++++++++++++------------------------- - 1 file changed, 31 insertions(+), 42 deletions(-) - -diff --git a/filters/delay/delay.c b/filters/delay/delay.c -index 5bd21321..df3729a7 100644 ---- a/filters/delay/delay.c -+++ b/filters/delay/delay.c -@@ -42,25 +42,28 @@ - - #include - --static int delay_read_ms = 0; /* read delay (milliseconds) */ --static int delay_write_ms = 0; /* write delay (milliseconds) */ --static int delay_zero_ms = 0; /* zero delay (milliseconds) */ --static int delay_trim_ms = 0; /* trim delay (milliseconds) */ --static int delay_extents_ms = 0;/* extents delay (milliseconds) */ --static int delay_cache_ms = 0; /* cache delay (milliseconds) */ -+static unsigned delay_read_ms = 0; /* read delay (milliseconds) */ -+static unsigned delay_write_ms = 0; /* write delay (milliseconds) */ -+static unsigned delay_zero_ms = 0; /* zero delay (milliseconds) */ -+static unsigned delay_trim_ms = 0; /* trim delay (milliseconds) */ -+static unsigned delay_extents_ms = 0;/* extents delay (milliseconds) */ -+static unsigned delay_cache_ms = 0; /* cache delay (milliseconds) */ -+static unsigned delay_open_ms = 0; /* open delay (milliseconds) */ -+static unsigned delay_close_ms = 0; /* close delay (milliseconds) */ -+ - static int delay_fast_zero = 1; /* whether delaying zero includes fast zero */ --static int delay_open_ms = 0; /* open delay (milliseconds) */ --static int delay_close_ms = 0; /* close delay (milliseconds) */ - - static int --parse_delay (const char *key, const char *value) -+parse_delay (const char *key, const char *value, unsigned *r) - { - size_t len = strlen (value); -- int r; - - if (len > 2 && strcmp (&value[len-2], "ms") == 0) { -- if (sscanf (value, "%d", &r) == 1 && r >= 0) -- return r; -+ /* We have to use sscanf here instead of nbdkit_parse_unsigned -+ * because that function will reject the "ms" suffix. -+ */ -+ if (sscanf (value, "%u", r) == 1) -+ return 0; - else { - nbdkit_error ("cannot parse %s in milliseconds parameter: %s", - key, value); -@@ -68,24 +71,19 @@ parse_delay (const char *key, const char *value) - } - } - else { -- if (sscanf (value, "%d", &r) == 1 && r >= 0) { -- if (r * 1000LL > INT_MAX) { -- nbdkit_error ("seconds parameter %s is too large: %s", -- key, value); -- return -1; -- } -- return r * 1000; -- } -- else { -- nbdkit_error ("cannot parse %s in seconds parameter: %s", -- key, value); -+ if (nbdkit_parse_unsigned (key, value, r) == -1) -+ return -1; -+ if (*r * 1000U > UINT_MAX) { -+ nbdkit_error ("seconds parameter %s is too large: %s", key, value); - return -1; - } -+ *r *= 1000; -+ return 0; - } - } - - static int --delay (int ms, int *err) -+delay (unsigned ms, int *err) - { - if (ms > 0 && nbdkit_nanosleep (ms / 1000, (ms % 1000) * 1000000) == -1) { - *err = errno; -@@ -150,14 +148,12 @@ delay_config (nbdkit_next_config *next, nbdkit_backend *nxdata, - if (strcmp (key, "rdelay") == 0 || - strcmp (key, "delay-read") == 0 || - strcmp (key, "delay-reads") == 0) { -- delay_read_ms = parse_delay (key, value); -- if (delay_read_ms == -1) -+ if (parse_delay (key, value, &delay_read_ms) == -1) - return -1; - return 0; - } - else if (strcmp (key, "wdelay") == 0) { -- delay_write_ms = parse_delay (key, value); -- if (delay_write_ms == -1) -+ if (parse_delay (key, value, &delay_write_ms) == -1) - return -1; - /* Historically wdelay set all write-related delays. */ - delay_zero_ms = delay_trim_ms = delay_write_ms; -@@ -165,15 +161,13 @@ delay_config (nbdkit_next_config *next, nbdkit_backend *nxdata, - } - else if (strcmp (key, "delay-write") == 0 || - strcmp (key, "delay-writes") == 0) { -- delay_write_ms = parse_delay (key, value); -- if (delay_write_ms == -1) -+ if (parse_delay (key, value, &delay_write_ms) == -1) - return -1; - return 0; - } - else if (strcmp (key, "delay-zero") == 0 || - strcmp (key, "delay-zeroes") == 0) { -- delay_zero_ms = parse_delay (key, value); -- if (delay_zero_ms == -1) -+ if (parse_delay (key, value, &delay_zero_ms) == -1) - return -1; - return 0; - } -@@ -181,21 +175,18 @@ delay_config (nbdkit_next_config *next, nbdkit_backend *nxdata, - strcmp (key, "delay-trims") == 0 || - strcmp (key, "delay-discard") == 0 || - strcmp (key, "delay-discards") == 0) { -- delay_trim_ms = parse_delay (key, value); -- if (delay_trim_ms == -1) -+ if (parse_delay (key, value, &delay_trim_ms) == -1) - return -1; - return 0; - } - else if (strcmp (key, "delay-extent") == 0 || - strcmp (key, "delay-extents") == 0) { -- delay_extents_ms = parse_delay (key, value); -- if (delay_extents_ms == -1) -+ if (parse_delay (key, value, &delay_extents_ms) == -1) - return -1; - return 0; - } - else if (strcmp (key, "delay-cache") == 0) { -- delay_cache_ms = parse_delay (key, value); -- if (delay_cache_ms == -1) -+ if (parse_delay (key, value, &delay_cache_ms) == -1) - return -1; - return 0; - } -@@ -206,14 +197,12 @@ delay_config (nbdkit_next_config *next, nbdkit_backend *nxdata, - return 0; - } - else if (strcmp (key, "delay-open") == 0) { -- delay_open_ms = parse_delay (key, value); -- if (delay_open_ms == -1) -+ if (parse_delay (key, value, &delay_open_ms) == -1) - return -1; - return 0; - } - else if (strcmp (key, "delay-close") == 0) { -- delay_close_ms = parse_delay (key, value); -- if (delay_close_ms == -1) -+ if (parse_delay (key, value, &delay_close_ms) == -1) - return -1; - return 0; - } --- -2.31.1 - diff --git a/0022-delay-Test-delay-open-and-delay-close.patch b/0018-delay-Test-delay-open-and-delay-close.patch similarity index 98% rename from 0022-delay-Test-delay-open-and-delay-close.patch rename to 0018-delay-Test-delay-open-and-delay-close.patch index fe3936e..2acd71d 100644 --- a/0022-delay-Test-delay-open-and-delay-close.patch +++ b/0018-delay-Test-delay-open-and-delay-close.patch @@ -1,4 +1,4 @@ -From 1f2acea1e6bd0a1120bf6b48854202ec8680f5c0 Mon Sep 17 00:00:00 2001 +From 838ec052abe63056434c08ea80f4609e697dad0f Mon Sep 17 00:00:00 2001 From: "Richard W.M. Jones" Date: Tue, 10 Aug 2021 09:11:43 +0100 Subject: [PATCH] delay: Test delay-open and delay-close diff --git a/0019-server-Return-from-nbdkit_nanosleep-early-if-the-soc.patch b/0019-server-Return-from-nbdkit_nanosleep-early-if-the-soc.patch deleted file mode 100644 index 724feb6..0000000 --- a/0019-server-Return-from-nbdkit_nanosleep-early-if-the-soc.patch +++ /dev/null @@ -1,41 +0,0 @@ -From 33318699bf1255aef0c6ee4863236d26d7b326ec Mon Sep 17 00:00:00 2001 -From: "Richard W.M. Jones" -Date: Mon, 9 Aug 2021 20:11:41 +0100 -Subject: [PATCH] server: Return from nbdkit_nanosleep early if the socket - closes - -https://bugzilla.redhat.com/show_bug.cgi?id=1991652#c2 - -Reported-by: Ming Xie -Fixes: https://bugzilla.redhat.com/show_bug.cgi?id=1991652 -(cherry picked from commit 87a88f8c52a0d2fd392c35d37f8b048bcede1382) ---- - server/public.c | 5 ++++- - 1 file changed, 4 insertions(+), 1 deletion(-) - -diff --git a/server/public.c b/server/public.c -index 3362f1ab..4870e2d3 100644 ---- a/server/public.c -+++ b/server/public.c -@@ -693,6 +693,8 @@ nbdkit_nanosleep (unsigned sec, unsigned nsec) - * - the current connection is multi-threaded and another thread detects - * NBD_CMD_DISC or a problem with the connection - * - the input socket detects POLLRDHUP/POLLHUP/POLLERR -+ * - the input socket is invalid (POLLNVAL, probably closed by -+ * another thread) - */ - struct connection *conn = threadlocal_get_conn (); - struct pollfd fds[] = { -@@ -724,7 +726,8 @@ nbdkit_nanosleep (unsigned sec, unsigned nsec) - */ - assert (quit || - (conn && conn->nworkers > 0 && connection_get_status () < 1) || -- (conn && (fds[2].revents & (POLLRDHUP | POLLHUP | POLLERR)))); -+ (conn && (fds[2].revents & (POLLRDHUP | POLLHUP | POLLERR | -+ POLLNVAL)))); - nbdkit_error ("aborting sleep to shut down"); - errno = ESHUTDOWN; - return -1; --- -2.31.1 - diff --git a/0019-vddk-Implement-can_flush-and-can_fua.patch b/0019-vddk-Implement-can_flush-and-can_fua.patch new file mode 100644 index 0000000..98d6225 --- /dev/null +++ b/0019-vddk-Implement-can_flush-and-can_fua.patch @@ -0,0 +1,74 @@ +From 2104686eb708bf87070c21e7af0e70e0317306b6 Mon Sep 17 00:00:00 2001 +From: "Richard W.M. Jones" +Date: Mon, 5 Jul 2021 21:36:41 +0100 +Subject: [PATCH] vddk: Implement can_flush and can_fua + +VDDK < 6.0 doesn't support flush. Previously we advertised flush and +FUA but ignored them if VDDK didn't support it. Instead, correctly +set these flags in the NBD protocol according to what VDDK supports. + +(cherry picked from commit 04b05274414a8cf4615eb2d6f46d5658814509c1) +--- + plugins/vddk/vddk.c | 28 ++++++++++++++++++++-------- + 1 file changed, 20 insertions(+), 8 deletions(-) + +diff --git a/plugins/vddk/vddk.c b/plugins/vddk/vddk.c +index 76faa768..b5bce9a0 100644 +--- a/plugins/vddk/vddk.c ++++ b/plugins/vddk/vddk.c +@@ -772,12 +772,28 @@ vddk_pwrite (void *handle, const void *buf, uint32_t count, uint64_t offset, + return -1; + } + +- if (fua && vddk_flush (handle, 0) == -1) +- return -1; ++ if (fua) { ++ if (vddk_flush (handle, 0) == -1) ++ return -1; ++ } + + return 0; + } + ++static int ++vddk_can_fua (void *handle) ++{ ++ /* The Flush call was not available in VDDK < 6.0. */ ++ return VixDiskLib_Flush != NULL ? NBDKIT_FUA_NATIVE : NBDKIT_FUA_NONE; ++} ++ ++static int ++vddk_can_flush (void *handle) ++{ ++ /* The Flush call was not available in VDDK < 6.0. */ ++ return VixDiskLib_Flush != NULL; ++} ++ + /* Flush data to the file. */ + static int + vddk_flush (void *handle, uint32_t flags) +@@ -785,12 +801,6 @@ vddk_flush (void *handle, uint32_t flags) + struct vddk_handle *h = handle; + VixError err; + +- /* The Flush call was not available in VDDK < 6.0 so this is simply +- * ignored on earlier versions. +- */ +- if (VixDiskLib_Flush == NULL) +- return 0; +- + DEBUG_CALL ("VixDiskLib_Flush", "handle"); + err = VixDiskLib_Flush (h->handle); + if (err != VIX_OK) { +@@ -985,6 +995,8 @@ static struct nbdkit_plugin plugin = { + .get_size = vddk_get_size, + .pread = vddk_pread, + .pwrite = vddk_pwrite, ++ .can_fua = vddk_can_fua, ++ .can_flush = vddk_can_flush, + .flush = vddk_flush, + .can_extents = vddk_can_extents, + .extents = vddk_extents, +-- +2.31.1 + diff --git a/0020-server-nanosleep-Change-error-for-early-end-of-sleep.patch b/0020-server-nanosleep-Change-error-for-early-end-of-sleep.patch deleted file mode 100644 index edc212e..0000000 --- a/0020-server-nanosleep-Change-error-for-early-end-of-sleep.patch +++ /dev/null @@ -1,43 +0,0 @@ -From 032531cd5d402119a81efbaf07d781123c5b02af Mon Sep 17 00:00:00 2001 -From: "Richard W.M. Jones" -Date: Tue, 10 Aug 2021 08:30:43 +0100 -Subject: [PATCH] server: nanosleep: Change error for early end of sleep - -At the moment nbdkit_nanosleep gives an incorrect error message if it -aborts early. Even in the case when the server is not actually -shutting down it will say: - - $ nbdkit --filter=delay null delay-close=3 --run 'nbdinfo --size $uri; nbdinfo --size $uri' - 0 - nbdkit: null[1]: error: aborting sleep to shut down - 0 - nbdkit: null[2]: error: aborting sleep to shut down - -This commit changes the error so we only talk about shut down when the -server is actually shutting down, and use a different message in other -cases. - -(cherry picked from commit cd24d9c418992e6f2c721c7deec70e564c23ab83) ---- - server/public.c | 5 ++++- - 1 file changed, 4 insertions(+), 1 deletion(-) - -diff --git a/server/public.c b/server/public.c -index 4870e2d3..d9ed0d9c 100644 ---- a/server/public.c -+++ b/server/public.c -@@ -728,7 +728,10 @@ nbdkit_nanosleep (unsigned sec, unsigned nsec) - (conn && conn->nworkers > 0 && connection_get_status () < 1) || - (conn && (fds[2].revents & (POLLRDHUP | POLLHUP | POLLERR | - POLLNVAL)))); -- nbdkit_error ("aborting sleep to shut down"); -+ if (quit) -+ nbdkit_error ("aborting sleep because of server shut down"); -+ else -+ nbdkit_error ("aborting sleep because of connection close or error"); - errno = ESHUTDOWN; - return -1; - --- -2.31.1 - diff --git a/0020-vddk-Replace-DEBUG_CALL-with-bracketed-VDDK_CALL_STA.patch b/0020-vddk-Replace-DEBUG_CALL-with-bracketed-VDDK_CALL_STA.patch new file mode 100644 index 0000000..7a39936 --- /dev/null +++ b/0020-vddk-Replace-DEBUG_CALL-with-bracketed-VDDK_CALL_STA.patch @@ -0,0 +1,319 @@ +From 51713e7702d389fd55d5721c4773fca40e3e89f6 Mon Sep 17 00:00:00 2001 +From: "Richard W.M. Jones" +Date: Wed, 18 Aug 2021 14:26:30 +0100 +Subject: [PATCH] vddk: Replace DEBUG_CALL with bracketed VDDK_CALL_START/END + macros + +This is neutral refactoring, but allows us in the next commit to +collect statistics about the amount of time spent in these calls. + +(cherry picked from commit 1335ebfb5637bf5a44403d0b152da7272fdd3e54) +--- + plugins/vddk/vddk.c | 175 +++++++++++++++++++++++++------------------- + 1 file changed, 99 insertions(+), 76 deletions(-) + +diff --git a/plugins/vddk/vddk.c b/plugins/vddk/vddk.c +index b5bce9a0..888009ab 100644 +--- a/plugins/vddk/vddk.c ++++ b/plugins/vddk/vddk.c +@@ -104,19 +104,23 @@ static bool is_remote; + VixDiskLib_FreeErrorText (vddk_err_msg); \ + } while (0) + +-#define DEBUG_CALL(fn, fs, ...) \ +- nbdkit_debug ("VDDK call: %s (" fs ")", fn, ##__VA_ARGS__) +-#define DEBUG_CALL_DATAPATH(fn, fs, ...) \ +- if (vddk_debug_datapath) \ +- nbdkit_debug ("VDDK call: %s (" fs ")", fn, ##__VA_ARGS__) ++#define VDDK_CALL_START(fn, fs, ...) \ ++ nbdkit_debug ("VDDK call: %s (" fs ")", #fn, ##__VA_ARGS__); \ ++ do ++#define VDDK_CALL_START_DATAPATH(fn, fs, ...) \ ++ if (vddk_debug_datapath) \ ++ nbdkit_debug ("VDDK call: %s (" fs ")", #fn, ##__VA_ARGS__); \ ++ do ++#define VDDK_CALL_END(fn) while (0) + + /* Unload the plugin. */ + static void + vddk_unload (void) + { + if (init_called) { +- DEBUG_CALL ("VixDiskLib_Exit", ""); +- VixDiskLib_Exit (); ++ VDDK_CALL_START (VixDiskLib_Exit, "") { ++ VixDiskLib_Exit (); ++ } VDDK_CALL_END (VixDiskLib_Exit); + } + if (dl) + dlclose (dl); +@@ -449,15 +453,16 @@ vddk_after_fork (void) + VixError err; + + /* Initialize VDDK library. */ +- DEBUG_CALL ("VixDiskLib_InitEx", +- "%d, %d, &debug_fn, &error_fn, &error_fn, %s, %s", +- VDDK_MAJOR, VDDK_MINOR, +- libdir, config ? : "NULL"); +- err = VixDiskLib_InitEx (VDDK_MAJOR, VDDK_MINOR, +- &debug_function, /* log function */ +- &error_function, /* warn function */ +- &error_function, /* panic function */ +- libdir, config); ++ VDDK_CALL_START (VixDiskLib_InitEx, ++ "%d, %d, &debug_fn, &error_fn, &error_fn, %s, %s", ++ VDDK_MAJOR, VDDK_MINOR, ++ libdir, config ? : "NULL") { ++ err = VixDiskLib_InitEx (VDDK_MAJOR, VDDK_MINOR, ++ &debug_function, /* log function */ ++ &error_function, /* warn function */ ++ &error_function, /* panic function */ ++ libdir, config); ++ } VDDK_CALL_END (VixDiskLib_InitEx); + if (err != VIX_OK) { + VDDK_ERROR (err, "VixDiskLib_InitEx"); + exit (EXIT_FAILURE); +@@ -519,8 +524,9 @@ allocate_connect_params (void) + VixDiskLibConnectParams *ret; + + if (VixDiskLib_AllocateConnectParams != NULL) { +- DEBUG_CALL ("VixDiskLib_AllocateConnectParams", ""); +- ret = VixDiskLib_AllocateConnectParams (); ++ VDDK_CALL_START (VixDiskLib_AllocateConnectParams, "") { ++ ret = VixDiskLib_AllocateConnectParams (); ++ } VDDK_CALL_END (VixDiskLib_AllocateConnectParams); + } + else + ret = calloc (1, sizeof (VixDiskLibConnectParams)); +@@ -535,8 +541,9 @@ free_connect_params (VixDiskLibConnectParams *params) + * originally called. Otherwise use free. + */ + if (VixDiskLib_AllocateConnectParams != NULL) { +- DEBUG_CALL ("VixDiskLib_FreeConnectParams", "params"); +- VixDiskLib_FreeConnectParams (params); ++ VDDK_CALL_START (VixDiskLib_FreeConnectParams, "params") { ++ VixDiskLib_FreeConnectParams (params); ++ } VDDK_CALL_END (VixDiskLib_FreeConnectParams); + } + else + free (params); +@@ -589,16 +596,17 @@ vddk_open (int readonly) + * either ESXi or vCenter servers. + */ + +- DEBUG_CALL ("VixDiskLib_ConnectEx", +- "h->params, %d, %s, %s, &connection", +- readonly, +- snapshot_moref ? : "NULL", +- transport_modes ? : "NULL"); +- err = VixDiskLib_ConnectEx (h->params, +- readonly, +- snapshot_moref, +- transport_modes, +- &h->connection); ++ VDDK_CALL_START (VixDiskLib_ConnectEx, ++ "h->params, %d, %s, %s, &connection", ++ readonly, ++ snapshot_moref ? : "NULL", ++ transport_modes ? : "NULL") { ++ err = VixDiskLib_ConnectEx (h->params, ++ readonly, ++ snapshot_moref, ++ transport_modes, ++ &h->connection); ++ } VDDK_CALL_END (VixDiskLib_ConnectEx); + if (err != VIX_OK) { + VDDK_ERROR (err, "VixDiskLib_ConnectEx"); + goto err1; +@@ -618,9 +626,10 @@ vddk_open (int readonly) + case NONE: break; + } + +- DEBUG_CALL ("VixDiskLib_Open", +- "connection, %s, %d, &handle", filename, flags); +- err = VixDiskLib_Open (h->connection, filename, flags, &h->handle); ++ VDDK_CALL_START (VixDiskLib_Open, ++ "connection, %s, %d, &handle", filename, flags) { ++ err = VixDiskLib_Open (h->connection, filename, flags, &h->handle); ++ } VDDK_CALL_END (VixDiskLib_Open); + if (err != VIX_OK) { + VDDK_ERROR (err, "VixDiskLib_Open: %s", filename); + goto err2; +@@ -632,8 +641,9 @@ vddk_open (int readonly) + return h; + + err2: +- DEBUG_CALL ("VixDiskLib_Disconnect", "connection"); +- VixDiskLib_Disconnect (h->connection); ++ VDDK_CALL_START (VixDiskLib_Disconnect, "connection") { ++ VixDiskLib_Disconnect (h->connection); ++ } VDDK_CALL_END (VixDiskLib_Disconnect); + err1: + free_connect_params (h->params); + err0: +@@ -648,10 +658,13 @@ vddk_close (void *handle) + ACQUIRE_LOCK_FOR_CURRENT_SCOPE (&open_close_lock); + struct vddk_handle *h = handle; + +- DEBUG_CALL ("VixDiskLib_Close", "handle"); +- VixDiskLib_Close (h->handle); +- DEBUG_CALL ("VixDiskLib_Disconnect", "connection"); +- VixDiskLib_Disconnect (h->connection); ++ VDDK_CALL_START (VixDiskLib_Close, "handle") { ++ VixDiskLib_Close (h->handle); ++ } VDDK_CALL_END (VixDiskLib_Close); ++ VDDK_CALL_START (VixDiskLib_Disconnect, "connection") { ++ VixDiskLib_Disconnect (h->connection); ++ } VDDK_CALL_END (VixDiskLib_Disconnect); ++ + free_connect_params (h->params); + free (h); + } +@@ -665,8 +678,9 @@ vddk_get_size (void *handle) + VixError err; + uint64_t size; + +- DEBUG_CALL ("VixDiskLib_GetInfo", "handle, &info"); +- err = VixDiskLib_GetInfo (h->handle, &info); ++ VDDK_CALL_START (VixDiskLib_GetInfo, "handle, &info") { ++ err = VixDiskLib_GetInfo (h->handle, &info); ++ } VDDK_CALL_END (VixDiskLib_GetInfo); + if (err != VIX_OK) { + VDDK_ERROR (err, "VixDiskLib_GetInfo"); + return -1; +@@ -694,8 +708,9 @@ vddk_get_size (void *handle) + info->uuid ? : "NULL"); + } + +- DEBUG_CALL ("VixDiskLib_FreeInfo", "info"); +- VixDiskLib_FreeInfo (info); ++ VDDK_CALL_START (VixDiskLib_FreeInfo, "info") { ++ VixDiskLib_FreeInfo (info); ++ } VDDK_CALL_END (VixDiskLib_FreeInfo); + + return (int64_t) size; + } +@@ -723,11 +738,12 @@ vddk_pread (void *handle, void *buf, uint32_t count, uint64_t offset, + offset /= VIXDISKLIB_SECTOR_SIZE; + count /= VIXDISKLIB_SECTOR_SIZE; + +- DEBUG_CALL_DATAPATH ("VixDiskLib_Read", +- "handle, %" PRIu64 " sectors, " +- "%" PRIu32 " sectors, buffer", +- offset, count); +- err = VixDiskLib_Read (h->handle, offset, count, buf); ++ VDDK_CALL_START_DATAPATH (VixDiskLib_Read, ++ "handle, %" PRIu64 " sectors, " ++ "%" PRIu32 " sectors, buffer", ++ offset, count) { ++ err = VixDiskLib_Read (h->handle, offset, count, buf); ++ } VDDK_CALL_END (VixDiskLib_Read); + if (err != VIX_OK) { + VDDK_ERROR (err, "VixDiskLib_Read"); + return -1; +@@ -762,11 +778,12 @@ vddk_pwrite (void *handle, const void *buf, uint32_t count, uint64_t offset, + offset /= VIXDISKLIB_SECTOR_SIZE; + count /= VIXDISKLIB_SECTOR_SIZE; + +- DEBUG_CALL_DATAPATH ("VixDiskLib_Write", +- "handle, %" PRIu64 " sectors, " +- "%" PRIu32 " sectors, buffer", +- offset, count); +- err = VixDiskLib_Write (h->handle, offset, count, buf); ++ VDDK_CALL_START_DATAPATH (VixDiskLib_Write, ++ "handle, %" PRIu64 " sectors, " ++ "%" PRIu32 " sectors, buffer", ++ offset, count) { ++ err = VixDiskLib_Write (h->handle, offset, count, buf); ++ } VDDK_CALL_END (VixDiskLib_Write); + if (err != VIX_OK) { + VDDK_ERROR (err, "VixDiskLib_Write"); + return -1; +@@ -801,8 +818,9 @@ vddk_flush (void *handle, uint32_t flags) + struct vddk_handle *h = handle; + VixError err; + +- DEBUG_CALL ("VixDiskLib_Flush", "handle"); +- err = VixDiskLib_Flush (h->handle); ++ VDDK_CALL_START (VixDiskLib_Flush, "handle") { ++ err = VixDiskLib_Flush (h->handle); ++ } VDDK_CALL_END (VixDiskLib_Flush); + if (err != VIX_OK) { + VDDK_ERROR (err, "VixDiskLib_Flush"); + return -1; +@@ -836,17 +854,19 @@ vddk_can_extents (void *handle) + * the best thing we can do here is to try the call and if it's + * non-functional return false. + */ +- DEBUG_CALL ("VixDiskLib_QueryAllocatedBlocks", +- "handle, 0, %d sectors, %d sectors", +- VIXDISKLIB_MIN_CHUNK_SIZE, VIXDISKLIB_MIN_CHUNK_SIZE); +- err = VixDiskLib_QueryAllocatedBlocks (h->handle, +- 0, VIXDISKLIB_MIN_CHUNK_SIZE, +- VIXDISKLIB_MIN_CHUNK_SIZE, +- &block_list); ++ VDDK_CALL_START (VixDiskLib_QueryAllocatedBlocks, ++ "handle, 0, %d sectors, %d sectors", ++ VIXDISKLIB_MIN_CHUNK_SIZE, VIXDISKLIB_MIN_CHUNK_SIZE) { ++ err = VixDiskLib_QueryAllocatedBlocks (h->handle, ++ 0, VIXDISKLIB_MIN_CHUNK_SIZE, ++ VIXDISKLIB_MIN_CHUNK_SIZE, ++ &block_list); ++ } VDDK_CALL_END (VixDiskLib_QueryAllocatedBlocks); + error_suppression = 0; + if (err == VIX_OK) { +- DEBUG_CALL ("VixDiskLib_FreeBlockList", "block_list"); +- VixDiskLib_FreeBlockList (block_list); ++ VDDK_CALL_START (VixDiskLib_FreeBlockList, "block_list") { ++ VixDiskLib_FreeBlockList (block_list); ++ } VDDK_CALL_END (VixDiskLib_FreeBlockList); + } + if (err != VIX_OK) { + char *errmsg = VixDiskLib_GetErrorText (err, NULL); +@@ -923,14 +943,15 @@ vddk_extents (void *handle, uint32_t count, uint64_t offset, uint32_t flags, + nr_chunks = MIN (nr_chunks, VIXDISKLIB_MAX_CHUNK_NUMBER); + nr_sectors = nr_chunks * VIXDISKLIB_MIN_CHUNK_SIZE; + +- DEBUG_CALL ("VixDiskLib_QueryAllocatedBlocks", +- "handle, %" PRIu64 " sectors, %" PRIu64 " sectors, " +- "%d sectors", +- start_sector, nr_sectors, VIXDISKLIB_MIN_CHUNK_SIZE); +- err = VixDiskLib_QueryAllocatedBlocks (h->handle, +- start_sector, nr_sectors, +- VIXDISKLIB_MIN_CHUNK_SIZE, +- &block_list); ++ VDDK_CALL_START (VixDiskLib_QueryAllocatedBlocks, ++ "handle, %" PRIu64 " sectors, %" PRIu64 " sectors, " ++ "%d sectors", ++ start_sector, nr_sectors, VIXDISKLIB_MIN_CHUNK_SIZE) { ++ err = VixDiskLib_QueryAllocatedBlocks (h->handle, ++ start_sector, nr_sectors, ++ VIXDISKLIB_MIN_CHUNK_SIZE, ++ &block_list); ++ } VDDK_CALL_END (VixDiskLib_QueryAllocatedBlocks); + if (err != VIX_OK) { + VDDK_ERROR (err, "VixDiskLib_QueryAllocatedBlocks"); + return -1; +@@ -949,13 +970,15 @@ vddk_extents (void *handle, uint32_t count, uint64_t offset, uint32_t flags, + add_extent (extents, &position, blk_offset, true) == -1) || + (add_extent (extents, + &position, blk_offset + blk_length, false) == -1)) { +- DEBUG_CALL ("VixDiskLib_FreeBlockList", "block_list"); +- VixDiskLib_FreeBlockList (block_list); ++ VDDK_CALL_START (VixDiskLib_FreeBlockList, "block_list") { ++ VixDiskLib_FreeBlockList (block_list); ++ } VDDK_CALL_END (VixDiskLib_FreeBlockList); + return -1; + } + } +- DEBUG_CALL ("VixDiskLib_FreeBlockList", "block_list"); +- VixDiskLib_FreeBlockList (block_list); ++ VDDK_CALL_START (VixDiskLib_FreeBlockList, "block_list") { ++ VixDiskLib_FreeBlockList (block_list); ++ } VDDK_CALL_END (VixDiskLib_FreeBlockList); + + /* There's an implicit hole after the returned list of blocks, up + * to the end of the QueryAllocatedBlocks request. +-- +2.31.1 + diff --git a/0021-tests-Add-a-better-test-of-real-VDDK.patch b/0021-tests-Add-a-better-test-of-real-VDDK.patch new file mode 100644 index 0000000..c9dc143 --- /dev/null +++ b/0021-tests-Add-a-better-test-of-real-VDDK.patch @@ -0,0 +1,103 @@ +From bd181ea739ebfafbf7239b5fa89e98becdb8cb72 Mon Sep 17 00:00:00 2001 +From: "Richard W.M. Jones" +Date: Mon, 5 Jul 2021 22:03:10 +0100 +Subject: [PATCH] tests: Add a better test of real VDDK + +The previous test only tested reading and maybe extents, and used an +all-zero disk. I'm fairly convinced the test only worked accidentally +since you must use an absolute path when opening a local file and the +test did not do that. + +Add a more comprehensive test that tests writing and flush too. + +(cherry picked from commit a6ca4f24593008bb2d8efb177e7f424cff51dfbf) +--- + tests/test-vddk-real.sh | 55 ++++++++++++++++++++++++++++------------- + 1 file changed, 38 insertions(+), 17 deletions(-) + +diff --git a/tests/test-vddk-real.sh b/tests/test-vddk-real.sh +index df486ba1..f848db44 100755 +--- a/tests/test-vddk-real.sh ++++ b/tests/test-vddk-real.sh +@@ -37,8 +37,12 @@ set -x + requires test "x$vddkdir" != "x" + requires test -d "$vddkdir" + requires test -f "$vddkdir/lib64/libvixDiskLib.so" ++requires test -r /dev/urandom ++requires cmp --version ++requires dd --version + requires qemu-img --version + requires nbdcopy --version ++requires nbdinfo --version + requires stat --version + + # VDDK > 5.1.1 only supports x86_64. +@@ -47,31 +51,48 @@ if [ `uname -m` != "x86_64" ]; then + exit 77 + fi + +-files="test-vddk-real.vmdk test-vddk-real.out test-vddk-real.log" +-rm -f $files +-cleanup_fn rm -f $files +- +-qemu-img create -f vmdk test-vddk-real.vmdk 100M +- + # Since we are comparing error messages below, let's make sure we're + # not translating errors. + export LANG=C + +-fail=0 +-nbdkit -f -v -U - \ +- --filter=readahead \ +- vddk libdir="$vddkdir" test-vddk-real.vmdk \ +- --run 'nbdcopy "$uri" test-vddk-real.out' \ +- > test-vddk-real.log 2>&1 || fail=1 ++pid=test-vddk-real.pid ++sock=$(mktemp -u /tmp/nbdkit-test-sock.XXXXXX) ++vmdk=$PWD/test-vddk-real.vmdk ;# note must be an absolute path ++raw=test-vddk-real.raw ++raw2=test-vddk-real.raw2 ++log=test-vddk-real.log ++files="$pid $sock $vmdk $raw $raw2 $log" ++rm -f $files ++cleanup_fn rm -f $files ++ ++qemu-img create -f vmdk $vmdk 10M ++ ++# Check first that the VDDK library can be fully loaded. We have to ++# check the log file for missing modules since they may not show up as ++# errors. ++nbdkit -fv -U - vddk libdir="$vddkdir" $vmdk --run 'nbdinfo "$uri"' >$log 2>&1 + + # Check the log for missing modules +-cat test-vddk-real.log ++cat $log + if grep 'cannot open shared object file' test-vddk-real.log; then + exit 1 + fi + +-# Check the raw output file has exactly the right size. +-size="$(stat -c '%s' test-vddk-real.out)" +-test "$size" -eq $((100 * 1024 * 1024)) ++# Now run nbdkit for the test. ++start_nbdkit -P $pid -U $sock vddk libdir="$vddkdir" $vmdk ++uri="nbd+unix:///?socket=$sock" + +-exit $fail ++# VDDK < 6.0 did not support flush, so disable flush test there. Also ++# if nbdinfo doesn't support the --can flush syntax (added in libnbd ++# 1.10) then this is disabled. ++if nbdinfo --can flush "$uri"; then flush="--flush"; else flush=""; fi ++ ++# Copy in and out some data. This should exercise read, write, ++# extents and flushing. ++dd if=/dev/urandom of=$raw count=5 bs=$((1024*1024)) ++truncate -s 10M $raw ++ ++nbdcopy $flush $raw "$uri" ++nbdcopy "$uri" $raw2 ++ ++cmp $raw $raw2 +-- +2.31.1 + diff --git a/0022-vddk-Add-stats-about-the-amount-of-time-spent-in-VDD.patch b/0022-vddk-Add-stats-about-the-amount-of-time-spent-in-VDD.patch new file mode 100644 index 0000000..dd18103 --- /dev/null +++ b/0022-vddk-Add-stats-about-the-amount-of-time-spent-in-VDD.patch @@ -0,0 +1,245 @@ +From 45db64d72bf03fece8a7fb994887360954905a3b Mon Sep 17 00:00:00 2001 +From: "Richard W.M. Jones" +Date: Wed, 18 Aug 2021 14:47:58 +0100 +Subject: [PATCH] vddk: Add stats about the amount of time spent in VDDK calls +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +If you use -D vddk.stats=1 then when the plugin is unloaded it will +display the amount of time spent in each VDDK call. The output will +look something like this: + +nbdkit: debug: VDDK function stats (-D vddk.stats=1): +nbdkit: debug: µs +nbdkit: debug: VixDiskLib_Exit 1001154 +nbdkit: debug: VixDiskLib_InitEx 96008 +nbdkit: debug: VixDiskLib_Flush 15722 +nbdkit: debug: VixDiskLib_Write 12081 +nbdkit: debug: VixDiskLib_Open 6029 +nbdkit: debug: VixDiskLib_Read 1364 +nbdkit: debug: VixDiskLib_Close 605 +nbdkit: debug: VixDiskLib_QueryAllocatedBlocks 191 +nbdkit: debug: VixDiskLib_ConnectEx 134 +nbdkit: debug: VixDiskLib_Disconnect 76 +nbdkit: debug: VixDiskLib_FreeConnectParams 57 +nbdkit: debug: VixDiskLib_GetInfo 56 +nbdkit: debug: VixDiskLib_GetTransportMode 43 +nbdkit: debug: VixDiskLib_FreeInfo 42 +nbdkit: debug: VixDiskLib_FreeBlockList 32 +nbdkit: debug: VixDiskLib_AllocateConnectParams 28 + +VDDK APIs which are never called are not printed. + +(cherry picked from commit f2dfc7d74ee650bdf2cc930a07b1c5bcb509976c) +--- + plugins/vddk/nbdkit-vddk-plugin.pod | 5 ++ + plugins/vddk/vddk.c | 107 +++++++++++++++++++++++++--- + tests/test-vddk-real.sh | 2 +- + 3 files changed, 103 insertions(+), 11 deletions(-) + +diff --git a/plugins/vddk/nbdkit-vddk-plugin.pod b/plugins/vddk/nbdkit-vddk-plugin.pod +index b783f13a..2a1b17dc 100644 +--- a/plugins/vddk/nbdkit-vddk-plugin.pod ++++ b/plugins/vddk/nbdkit-vddk-plugin.pod +@@ -510,6 +510,11 @@ Debug extents returned by C. + + Suppress debugging of datapath calls (C and C). + ++=item B<-D vddk.stats=1> ++ ++When the plugin exits print some statistics about the amount of time ++spent waiting on each VDDK call. ++ + =back + + =head1 FILES +diff --git a/plugins/vddk/vddk.c b/plugins/vddk/vddk.c +index 888009ab..fce96d9a 100644 +--- a/plugins/vddk/vddk.c ++++ b/plugins/vddk/vddk.c +@@ -42,6 +42,7 @@ + #include + #include + #include ++#include + + #include + +@@ -52,6 +53,8 @@ + #include "isaligned.h" + #include "minmax.h" + #include "rounding.h" ++#include "tvdiff.h" ++#include "vector.h" + + #include "vddk.h" + #include "vddk-structs.h" +@@ -60,6 +63,7 @@ + NBDKIT_DLL_PUBLIC int vddk_debug_diskinfo; + NBDKIT_DLL_PUBLIC int vddk_debug_extents; + NBDKIT_DLL_PUBLIC int vddk_debug_datapath = 1; ++NBDKIT_DLL_PUBLIC int vddk_debug_stats; + + /* For each VDDK API define a static global variable. These globals + * are initialized when the plugin is loaded (by vddk_get_ready). +@@ -96,22 +100,52 @@ static const char *username; /* user */ + static const char *vmx_spec; /* vm */ + static bool is_remote; + +-#define VDDK_ERROR(err, fs, ...) \ +- do { \ +- char *vddk_err_msg; \ +- vddk_err_msg = VixDiskLib_GetErrorText ((err), NULL); \ +- nbdkit_error (fs ": %s", ##__VA_ARGS__, vddk_err_msg); \ +- VixDiskLib_FreeErrorText (vddk_err_msg); \ +- } while (0) ++/* For each VDDK API define a variable to store the time taken (used ++ * to implement -D vddk.stats=1). ++ */ ++static pthread_mutex_t stats_lock = PTHREAD_MUTEX_INITIALIZER; ++static void display_stats (void); ++#define STUB(fn,ret,args) static int64_t stats_##fn; ++#define OPTIONAL_STUB(fn,ret,args) static int64_t stats_##fn; ++#include "vddk-stubs.h" ++#undef STUB ++#undef OPTIONAL_STUB + + #define VDDK_CALL_START(fn, fs, ...) \ ++ do { \ ++ struct timeval start_t, end_t; \ ++ if (vddk_debug_stats) \ ++ gettimeofday (&start_t, NULL); \ + nbdkit_debug ("VDDK call: %s (" fs ")", #fn, ##__VA_ARGS__); \ + do + #define VDDK_CALL_START_DATAPATH(fn, fs, ...) \ ++ do { \ ++ struct timeval start_t, end_t; \ ++ if (vddk_debug_stats) \ ++ gettimeofday (&start_t, NULL); \ + if (vddk_debug_datapath) \ + nbdkit_debug ("VDDK call: %s (" fs ")", #fn, ##__VA_ARGS__); \ + do +-#define VDDK_CALL_END(fn) while (0) ++#define VDDK_CALL_END(fn) \ ++ while (0); \ ++ if (vddk_debug_stats) { \ ++ gettimeofday (&end_t, NULL); \ ++ ACQUIRE_LOCK_FOR_CURRENT_SCOPE (&stats_lock); \ ++ stats_##fn += tvdiff_usec (&start_t, &end_t); \ ++ } \ ++ } while (0) ++ ++#define VDDK_ERROR(err, fs, ...) \ ++ do { \ ++ char *vddk_err_msg; \ ++ VDDK_CALL_START (VixDiskLib_GetErrorText, "%lu", err) { \ ++ vddk_err_msg = VixDiskLib_GetErrorText ((err), NULL); \ ++ } VDDK_CALL_END (VixDiskLib_GetErrorText); \ ++ nbdkit_error (fs ": %s", ##__VA_ARGS__, vddk_err_msg); \ ++ VDDK_CALL_START (VixDiskLib_FreeErrorText, "") { \ ++ VixDiskLib_FreeErrorText (vddk_err_msg); \ ++ } VDDK_CALL_END (VixDiskLib_FreeErrorText); \ ++ } while (0) + + /* Unload the plugin. */ + static void +@@ -124,11 +158,61 @@ vddk_unload (void) + } + if (dl) + dlclose (dl); ++ ++ if (vddk_debug_stats) ++ display_stats (); ++ + free (config); + free (libdir); + free (password); + } + ++struct vddk_stat { ++ const char *fn; ++ int64_t usecs; ++}; ++DEFINE_VECTOR_TYPE(statlist, struct vddk_stat) ++ ++static int ++stat_compare (const void *vp1, const void *vp2) ++{ ++ const struct vddk_stat *st1 = vp1; ++ const struct vddk_stat *st2 = vp2; ++ ++ /* Note: sorts in reverse order. */ ++ if (st1->usecs < st2->usecs) return 1; ++ else if (st1->usecs > st2->usecs) return -1; ++ else return 0; ++} ++ ++static void ++display_stats (void) ++{ ++ statlist stats = empty_vector; ++ struct vddk_stat st; ++ size_t i; ++ ++#define ADD_ONE_STAT(fn_, usecs_) \ ++ st.fn = fn_; \ ++ st.usecs = usecs_; \ ++ statlist_append (&stats, st) ++#define STUB(fn,ret,args) ADD_ONE_STAT (#fn, stats_##fn); ++#define OPTIONAL_STUB(fn,ret,args) ADD_ONE_STAT (#fn, stats_##fn); ++#include "vddk-stubs.h" ++#undef STUB ++#undef OPTIONAL_STUB ++#undef ADD_ONE_STAT ++ ++ qsort (stats.ptr, stats.size, sizeof stats.ptr[0], stat_compare); ++ ++ nbdkit_debug ("VDDK function stats (-D vddk.stats=1):"); ++ nbdkit_debug ("%-40s %9s", "", "µs"); ++ for (i = 0; i < stats.size; ++i) { ++ if (stats.ptr[i].usecs) ++ nbdkit_debug ("%-40s %9" PRIi64, stats.ptr[i].fn, stats.ptr[i].usecs); ++ } ++} ++ + static void + trim (char *str) + { +@@ -557,6 +641,7 @@ vddk_open (int readonly) + struct vddk_handle *h; + VixError err; + uint32_t flags; ++ const char *transport_mode; + + h = malloc (sizeof *h); + if (h == NULL) { +@@ -635,8 +720,10 @@ vddk_open (int readonly) + goto err2; + } + +- nbdkit_debug ("transport mode: %s", +- VixDiskLib_GetTransportMode (h->handle)); ++ VDDK_CALL_START (VixDiskLib_GetTransportMode, "handle") { ++ transport_mode = VixDiskLib_GetTransportMode (h->handle); ++ } VDDK_CALL_END (VixDiskLib_GetTransportMode); ++ nbdkit_debug ("transport mode: %s", transport_mode); + + return h; + +diff --git a/tests/test-vddk-real.sh b/tests/test-vddk-real.sh +index f848db44..3c8b4262 100755 +--- a/tests/test-vddk-real.sh ++++ b/tests/test-vddk-real.sh +@@ -79,7 +79,7 @@ if grep 'cannot open shared object file' test-vddk-real.log; then + fi + + # Now run nbdkit for the test. +-start_nbdkit -P $pid -U $sock vddk libdir="$vddkdir" $vmdk ++start_nbdkit -P $pid -U $sock -D vddk.stats=1 vddk libdir="$vddkdir" $vmdk + uri="nbd+unix:///?socket=$sock" + + # VDDK < 6.0 did not support flush, so disable flush test there. Also +-- +2.31.1 + diff --git a/0023-common-allocators-malloc.c-Remove-bogus-kernel-hints.patch b/0023-common-allocators-malloc.c-Remove-bogus-kernel-hints.patch deleted file mode 100644 index d6e4ef6..0000000 --- a/0023-common-allocators-malloc.c-Remove-bogus-kernel-hints.patch +++ /dev/null @@ -1,62 +0,0 @@ -From 85aea60685b493eac5e7664581c1887ede987461 Mon Sep 17 00:00:00 2001 -From: "Richard W.M. Jones" -Date: Wed, 11 Aug 2021 05:54:15 -0400 -Subject: [PATCH] common/allocators/malloc.c: Remove bogus kernel hints - -These kernel hints are wrong in several ways. - -MADV_DONTFORK should not be used because when we use captive nbdkit -(the --run option) we do actually fork and run nbdkit as the child. -However the kernel does not have to provide the mallocd memory to this -child process so it disappears. - -Even if the hints were not wrong, setting them using -madvise (ma->ba.ptr ...) would be wrong because the allocator buffer -can be extended at any time using realloc and could move in memory. -The hints would then apply to unrelated glibc allocations. I believe -this is what caused the crash I observed. - -For some reason the bug was only seen on s390x where it caused memory -corruption in glibc followed by a crash, but I don't believe this bug -is specific to s390x, it's just something about that architecture that -made it more likely to happen. - -Fixes: https://bugzilla.redhat.com/show_bug.cgi?id=1992542 -(cherry picked from commit 557a7a85c944dcd247feb0d670b0deca8da46576) ---- - common/allocators/malloc.c | 19 ------------------- - 1 file changed, 19 deletions(-) - -diff --git a/common/allocators/malloc.c b/common/allocators/malloc.c -index 9dc25a8e..59409c24 100644 ---- a/common/allocators/malloc.c -+++ b/common/allocators/malloc.c -@@ -105,25 +105,6 @@ extend (struct m_alloc *ma, uint64_t new_size) - return -1; - } - -- /* Hints to the kernel. Doesn't matter if these fail. -- * XXX Consider in future: MADV_MERGEABLE (tunable) -- */ --#ifdef MADV_RANDOM -- madvise (ma->ba.ptr, ma->ba.alloc, MADV_RANDOM); --#endif --#ifdef MADV_WILLNEED -- madvise (ma->ba.ptr, ma->ba.alloc, MADV_WILLNEED); --#endif --#ifdef MADV_DONTFORK -- madvise (ma->ba.ptr, ma->ba.alloc, MADV_DONTFORK); --#endif --#ifdef MADV_HUGEPAGE -- madvise (ma->ba.ptr, ma->ba.alloc, MADV_HUGEPAGE); --#endif --#ifdef MADV_DONTDUMP -- madvise (ma->ba.ptr, ma->ba.alloc, MADV_DONTDUMP); --#endif -- - /* Initialize the newly allocated memory to 0. */ - memset (ma->ba.ptr + old_size, 0, n); - --- -2.31.1 - diff --git a/0023-cow-Make-the-block-size-configurable.patch b/0023-cow-Make-the-block-size-configurable.patch new file mode 100644 index 0000000..a879abc --- /dev/null +++ b/0023-cow-Make-the-block-size-configurable.patch @@ -0,0 +1,772 @@ +From 0be4847cdec9effd6128da03ea42a4953e5a6343 Mon Sep 17 00:00:00 2001 +From: "Richard W.M. Jones" +Date: Tue, 17 Aug 2021 22:03:11 +0100 +Subject: [PATCH] cow: Make the block size configurable + +Commit c1905b0a28 ("cache, cow: Use a 64K block size by default") +changed the nbdkit-cow-filter block size to 64K, but it was still a +fixed size. In contrast the cache filter allows the block size to be +adjusted. + +Allow the block size in this filter to be adjusted up or down with a +new cow-block-size=N parameter. + +When using the VDDK plugin, adjusting this setting can make a +difference. The following timings come from a modified virt-v2v which +sets cow-block-size and was used to convert from a VMware server to +-o null (this is also using cow-on-read=true): + + cow-block-size=64K: 18m18 + cow-block-size=256K: 14m13 + cow-block-size=1M: 14m19 + cow-block-size=4M: 37m33 + +As you can see it's not obvious how to choose a good block size, but +at least by allowing adjustment we can tune things. + +(cherry picked from commit 7182c47d04d2b68005fceadefc0c14bfaa61a533) +--- + filters/cow/blk.c | 35 +++---- + filters/cow/blk.h | 5 - + filters/cow/cow.c | 150 +++++++++++++++++------------- + filters/cow/cow.h | 39 ++++++++ + filters/cow/nbdkit-cow-filter.pod | 5 + + tests/Makefile.am | 2 + + tests/test-cow-block-size.sh | 72 ++++++++++++++ + 7 files changed, 221 insertions(+), 87 deletions(-) + create mode 100644 filters/cow/cow.h + create mode 100755 tests/test-cow-block-size.sh + +diff --git a/filters/cow/blk.c b/filters/cow/blk.c +index c22d5886..f9341dc1 100644 +--- a/filters/cow/blk.c ++++ b/filters/cow/blk.c +@@ -99,6 +99,7 @@ + #include "pwrite.h" + #include "utils.h" + ++#include "cow.h" + #include "blk.h" + + /* The temporary overlay. */ +@@ -137,7 +138,7 @@ blk_init (void) + size_t len; + char *template; + +- bitmap_init (&bm, BLKSIZE, 2 /* bits per block */); ++ bitmap_init (&bm, blksize, 2 /* bits per block */); + + tmpdir = getenv ("TMPDIR"); + if (!tmpdir) +@@ -199,7 +200,7 @@ blk_set_size (uint64_t new_size) + if (bitmap_resize (&bm, size) == -1) + return -1; + +- if (ftruncate (fd, ROUND_UP (size, BLKSIZE)) == -1) { ++ if (ftruncate (fd, ROUND_UP (size, blksize)) == -1) { + nbdkit_error ("ftruncate: %m"); + return -1; + } +@@ -228,7 +229,7 @@ blk_read_multiple (nbdkit_next *next, + uint64_t blknum, uint64_t nrblocks, + uint8_t *block, bool cow_on_read, int *err) + { +- off_t offset = blknum * BLKSIZE; ++ off_t offset = blknum * blksize; + enum bm_entry state; + uint64_t b, runblocks; + +@@ -262,8 +263,8 @@ blk_read_multiple (nbdkit_next *next, + if (state == BLOCK_NOT_ALLOCATED) { /* Read underlying plugin. */ + unsigned n, tail = 0; + +- assert (BLKSIZE * runblocks <= UINT_MAX); +- n = BLKSIZE * runblocks; ++ assert (blksize * runblocks <= UINT_MAX); ++ n = blksize * runblocks; + + if (offset + n > size) { + tail = offset + n - size; +@@ -288,7 +289,7 @@ blk_read_multiple (nbdkit_next *next, + "at offset %" PRIu64 " into the cache", + runblocks, offset); + +- if (full_pwrite (fd, block, BLKSIZE * runblocks, offset) == -1) { ++ if (full_pwrite (fd, block, blksize * runblocks, offset) == -1) { + *err = errno; + nbdkit_error ("pwrite: %m"); + return -1; +@@ -298,14 +299,14 @@ blk_read_multiple (nbdkit_next *next, + } + } + else if (state == BLOCK_ALLOCATED) { /* Read overlay. */ +- if (full_pread (fd, block, BLKSIZE * runblocks, offset) == -1) { ++ if (full_pread (fd, block, blksize * runblocks, offset) == -1) { + *err = errno; + nbdkit_error ("pread: %m"); + return -1; + } + } + else /* state == BLOCK_TRIMMED */ { +- memset (block, 0, BLKSIZE * runblocks); ++ memset (block, 0, blksize * runblocks); + } + + /* If all done, return. */ +@@ -316,7 +317,7 @@ blk_read_multiple (nbdkit_next *next, + return blk_read_multiple (next, + blknum + runblocks, + nrblocks - runblocks, +- block + BLKSIZE * runblocks, ++ block + blksize * runblocks, + cow_on_read, err); + } + +@@ -333,9 +334,9 @@ blk_cache (nbdkit_next *next, + { + /* XXX Could make this lock more fine-grained with some thought. */ + ACQUIRE_LOCK_FOR_CURRENT_SCOPE (&lock); +- off_t offset = blknum * BLKSIZE; ++ off_t offset = blknum * blksize; + enum bm_entry state = bitmap_get_blk (&bm, blknum, BLOCK_NOT_ALLOCATED); +- unsigned n = BLKSIZE, tail = 0; ++ unsigned n = blksize, tail = 0; + + if (offset + n > size) { + tail = offset + n - size; +@@ -348,7 +349,7 @@ blk_cache (nbdkit_next *next, + + if (state == BLOCK_ALLOCATED) { + #if HAVE_POSIX_FADVISE +- int r = posix_fadvise (fd, offset, BLKSIZE, POSIX_FADV_WILLNEED); ++ int r = posix_fadvise (fd, offset, blksize, POSIX_FADV_WILLNEED); + if (r) { + errno = r; + nbdkit_error ("posix_fadvise: %m"); +@@ -373,7 +374,7 @@ blk_cache (nbdkit_next *next, + memset (block + n, 0, tail); + + if (mode == BLK_CACHE_COW) { +- if (full_pwrite (fd, block, BLKSIZE, offset) == -1) { ++ if (full_pwrite (fd, block, blksize, offset) == -1) { + *err = errno; + nbdkit_error ("pwrite: %m"); + return -1; +@@ -386,13 +387,13 @@ blk_cache (nbdkit_next *next, + int + blk_write (uint64_t blknum, const uint8_t *block, int *err) + { +- off_t offset = blknum * BLKSIZE; ++ off_t offset = blknum * blksize; + + if (cow_debug_verbose) + nbdkit_debug ("cow: blk_write block %" PRIu64 " (offset %" PRIu64 ")", + blknum, (uint64_t) offset); + +- if (full_pwrite (fd, block, BLKSIZE, offset) == -1) { ++ if (full_pwrite (fd, block, blksize, offset) == -1) { + *err = errno; + nbdkit_error ("pwrite: %m"); + return -1; +@@ -407,14 +408,14 @@ blk_write (uint64_t blknum, const uint8_t *block, int *err) + int + blk_trim (uint64_t blknum, int *err) + { +- off_t offset = blknum * BLKSIZE; ++ off_t offset = blknum * blksize; + + if (cow_debug_verbose) + nbdkit_debug ("cow: blk_trim block %" PRIu64 " (offset %" PRIu64 ")", + blknum, (uint64_t) offset); + + /* XXX As an optimization we could punch a whole in the overlay +- * here. However it's not trivial since BLKSIZE is unrelated to the ++ * here. However it's not trivial since blksize is unrelated to the + * overlay filesystem block size. + */ + ACQUIRE_LOCK_FOR_CURRENT_SCOPE (&lock); +diff --git a/filters/cow/blk.h b/filters/cow/blk.h +index b7e6f092..62fb5416 100644 +--- a/filters/cow/blk.h ++++ b/filters/cow/blk.h +@@ -33,11 +33,6 @@ + #ifndef NBDKIT_BLK_H + #define NBDKIT_BLK_H + +-/* Size of a block in the overlay. A 4K block size means that we need +- * 64 MB of memory to store the bitmap for a 1 TB underlying image. +- */ +-#define BLKSIZE 65536 +- + /* Initialize the overlay and bitmap. */ + extern int blk_init (void); + +diff --git a/filters/cow/cow.c b/filters/cow/cow.c +index 6efb39f2..1c62c857 100644 +--- a/filters/cow/cow.c ++++ b/filters/cow/cow.c +@@ -40,6 +40,7 @@ + #include + #include + #include ++#include + + #include + +@@ -47,9 +48,11 @@ + + #include "cleanup.h" + #include "isaligned.h" ++#include "ispowerof2.h" + #include "minmax.h" + #include "rounding.h" + ++#include "cow.h" + #include "blk.h" + + /* Read-modify-write requests are serialized through this global lock. +@@ -58,6 +61,8 @@ + */ + static pthread_mutex_t rmw_lock = PTHREAD_MUTEX_INITIALIZER; + ++unsigned blksize = 65536; /* block size */ ++ + static bool cow_on_cache; + + /* Cache on read ("cow-on-read") mode. */ +@@ -69,13 +74,6 @@ extern enum cor_mode { + enum cor_mode cor_mode = COR_OFF; + const char *cor_path; + +-static void +-cow_load (void) +-{ +- if (blk_init () == -1) +- exit (EXIT_FAILURE); +-} +- + static void + cow_unload (void) + { +@@ -86,7 +84,19 @@ static int + cow_config (nbdkit_next_config *next, nbdkit_backend *nxdata, + const char *key, const char *value) + { +- if (strcmp (key, "cow-on-cache") == 0) { ++ if (strcmp (key, "cow-block-size") == 0) { ++ int64_t r = nbdkit_parse_size (value); ++ if (r == -1) ++ return -1; ++ if (r <= 4096 || r > UINT_MAX || !is_power_of_2 (r)) { ++ nbdkit_error ("cow-block-size is out of range (4096..2G) " ++ "or not a power of 2"); ++ return -1; ++ } ++ blksize = r; ++ return 0; ++ } ++ else if (strcmp (key, "cow-on-cache") == 0) { + int r; + + r = nbdkit_parse_bool (value); +@@ -114,9 +124,19 @@ cow_config (nbdkit_next_config *next, nbdkit_backend *nxdata, + } + + #define cow_config_help \ ++ "cow-block-size= Set COW block size.\n" \ + "cow-on-cache= Copy cache (prefetch) requests to the overlay.\n" \ + "cow-on-read=|/PATH Copy read requests to the overlay." + ++static int ++cow_get_ready (int thread_model) ++{ ++ if (blk_init () == -1) ++ return -1; ++ ++ return 0; ++} ++ + /* Decide if cow-on-read is currently on or off. */ + bool + cow_on_read (void) +@@ -249,8 +269,8 @@ cow_pread (nbdkit_next *next, + uint64_t blknum, blkoffs, nrblocks; + int r; + +- if (!IS_ALIGNED (count | offset, BLKSIZE)) { +- block = malloc (BLKSIZE); ++ if (!IS_ALIGNED (count | offset, blksize)) { ++ block = malloc (blksize); + if (block == NULL) { + *err = errno; + nbdkit_error ("malloc: %m"); +@@ -258,12 +278,12 @@ cow_pread (nbdkit_next *next, + } + } + +- blknum = offset / BLKSIZE; /* block number */ +- blkoffs = offset % BLKSIZE; /* offset within the block */ ++ blknum = offset / blksize; /* block number */ ++ blkoffs = offset % blksize; /* offset within the block */ + + /* Unaligned head */ + if (blkoffs) { +- uint64_t n = MIN (BLKSIZE - blkoffs, count); ++ uint64_t n = MIN (blksize - blkoffs, count); + + assert (block); + r = blk_read (next, blknum, block, cow_on_read (), err); +@@ -279,15 +299,15 @@ cow_pread (nbdkit_next *next, + } + + /* Aligned body */ +- nrblocks = count / BLKSIZE; ++ nrblocks = count / blksize; + if (nrblocks > 0) { + r = blk_read_multiple (next, blknum, nrblocks, buf, cow_on_read (), err); + if (r == -1) + return -1; + +- buf += nrblocks * BLKSIZE; +- count -= nrblocks * BLKSIZE; +- offset += nrblocks * BLKSIZE; ++ buf += nrblocks * blksize; ++ count -= nrblocks * blksize; ++ offset += nrblocks * blksize; + blknum += nrblocks; + } + +@@ -314,8 +334,8 @@ cow_pwrite (nbdkit_next *next, + uint64_t blknum, blkoffs; + int r; + +- if (!IS_ALIGNED (count | offset, BLKSIZE)) { +- block = malloc (BLKSIZE); ++ if (!IS_ALIGNED (count | offset, blksize)) { ++ block = malloc (blksize); + if (block == NULL) { + *err = errno; + nbdkit_error ("malloc: %m"); +@@ -323,12 +343,12 @@ cow_pwrite (nbdkit_next *next, + } + } + +- blknum = offset / BLKSIZE; /* block number */ +- blkoffs = offset % BLKSIZE; /* offset within the block */ ++ blknum = offset / blksize; /* block number */ ++ blkoffs = offset % blksize; /* offset within the block */ + + /* Unaligned head */ + if (blkoffs) { +- uint64_t n = MIN (BLKSIZE - blkoffs, count); ++ uint64_t n = MIN (blksize - blkoffs, count); + + /* Do a read-modify-write operation on the current block. + * Hold the rmw_lock over the whole operation. +@@ -350,14 +370,14 @@ cow_pwrite (nbdkit_next *next, + } + + /* Aligned body */ +- while (count >= BLKSIZE) { ++ while (count >= blksize) { + r = blk_write (blknum, buf, err); + if (r == -1) + return -1; + +- buf += BLKSIZE; +- count -= BLKSIZE; +- offset += BLKSIZE; ++ buf += blksize; ++ count -= blksize; ++ offset += blksize; + blknum++; + } + +@@ -397,19 +417,19 @@ cow_zero (nbdkit_next *next, + return -1; + } + +- block = malloc (BLKSIZE); ++ block = malloc (blksize); + if (block == NULL) { + *err = errno; + nbdkit_error ("malloc: %m"); + return -1; + } + +- blknum = offset / BLKSIZE; /* block number */ +- blkoffs = offset % BLKSIZE; /* offset within the block */ ++ blknum = offset / blksize; /* block number */ ++ blkoffs = offset % blksize; /* offset within the block */ + + /* Unaligned head */ + if (blkoffs) { +- uint64_t n = MIN (BLKSIZE - blkoffs, count); ++ uint64_t n = MIN (blksize - blkoffs, count); + + /* Do a read-modify-write operation on the current block. + * Hold the rmw_lock over the whole operation. +@@ -429,9 +449,9 @@ cow_zero (nbdkit_next *next, + } + + /* Aligned body */ +- if (count >= BLKSIZE) +- memset (block, 0, BLKSIZE); +- while (count >= BLKSIZE) { ++ if (count >= blksize) ++ memset (block, 0, blksize); ++ while (count >= blksize) { + /* XXX There is the possibility of optimizing this: since this loop is + * writing a whole, aligned block, we should use FALLOC_FL_ZERO_RANGE. + */ +@@ -439,8 +459,8 @@ cow_zero (nbdkit_next *next, + if (r == -1) + return -1; + +- count -= BLKSIZE; +- offset += BLKSIZE; ++ count -= blksize; ++ offset += blksize; + blknum++; + } + +@@ -471,8 +491,8 @@ cow_trim (nbdkit_next *next, + uint64_t blknum, blkoffs; + int r; + +- if (!IS_ALIGNED (count | offset, BLKSIZE)) { +- block = malloc (BLKSIZE); ++ if (!IS_ALIGNED (count | offset, blksize)) { ++ block = malloc (blksize); + if (block == NULL) { + *err = errno; + nbdkit_error ("malloc: %m"); +@@ -480,12 +500,12 @@ cow_trim (nbdkit_next *next, + } + } + +- blknum = offset / BLKSIZE; /* block number */ +- blkoffs = offset % BLKSIZE; /* offset within the block */ ++ blknum = offset / blksize; /* block number */ ++ blkoffs = offset % blksize; /* offset within the block */ + + /* Unaligned head */ + if (blkoffs) { +- uint64_t n = MIN (BLKSIZE - blkoffs, count); ++ uint64_t n = MIN (blksize - blkoffs, count); + + /* Do a read-modify-write operation on the current block. + * Hold the lock over the whole operation. +@@ -505,13 +525,13 @@ cow_trim (nbdkit_next *next, + } + + /* Aligned body */ +- while (count >= BLKSIZE) { ++ while (count >= blksize) { + r = blk_trim (blknum, err); + if (r == -1) + return -1; + +- count -= BLKSIZE; +- offset += BLKSIZE; ++ count -= blksize; ++ offset += blksize; + blknum++; + } + +@@ -568,22 +588,22 @@ cow_cache (nbdkit_next *next, + mode = BLK_CACHE_COW; + + assert (!flags); +- block = malloc (BLKSIZE); ++ block = malloc (blksize); + if (block == NULL) { + *err = errno; + nbdkit_error ("malloc: %m"); + return -1; + } + +- blknum = offset / BLKSIZE; /* block number */ +- blkoffs = offset % BLKSIZE; /* offset within the block */ ++ blknum = offset / blksize; /* block number */ ++ blkoffs = offset % blksize; /* offset within the block */ + + /* Unaligned head */ + remaining += blkoffs; + offset -= blkoffs; + + /* Unaligned tail */ +- remaining = ROUND_UP (remaining, BLKSIZE); ++ remaining = ROUND_UP (remaining, blksize); + + /* Aligned body */ + while (remaining) { +@@ -591,8 +611,8 @@ cow_cache (nbdkit_next *next, + if (r == -1) + return -1; + +- remaining -= BLKSIZE; +- offset += BLKSIZE; ++ remaining -= blksize; ++ offset += blksize; + blknum++; + } + +@@ -616,13 +636,13 @@ cow_extents (nbdkit_next *next, + * value so rounding up is safe here. + */ + end = offset + count; +- offset = ROUND_DOWN (offset, BLKSIZE); +- end = ROUND_UP (end, BLKSIZE); ++ offset = ROUND_DOWN (offset, blksize); ++ end = ROUND_UP (end, blksize); + count = end - offset; +- blknum = offset / BLKSIZE; ++ blknum = offset / blksize; + +- assert (IS_ALIGNED (offset, BLKSIZE)); +- assert (IS_ALIGNED (count, BLKSIZE)); ++ assert (IS_ALIGNED (offset, blksize)); ++ assert (IS_ALIGNED (count, blksize)); + assert (count > 0); /* We must make forward progress. */ + + while (count > 0) { +@@ -634,7 +654,7 @@ cow_extents (nbdkit_next *next, + /* Present in the overlay. */ + if (present) { + e.offset = offset; +- e.length = BLKSIZE; ++ e.length = blksize; + + if (trimmed) + e.type = NBDKIT_EXTENT_HOLE|NBDKIT_EXTENT_ZERO; +@@ -647,8 +667,8 @@ cow_extents (nbdkit_next *next, + } + + blknum++; +- offset += BLKSIZE; +- count -= BLKSIZE; ++ offset += blksize; ++ count -= blksize; + } + + /* Not present in the overlay, but we can ask the plugin. */ +@@ -667,12 +687,12 @@ cow_extents (nbdkit_next *next, + * (range_count), but count is a 64 bit quantity, so don't + * overflow range_count here. + */ +- if (range_count >= UINT32_MAX - BLKSIZE + 1) break; ++ if (range_count >= UINT32_MAX - blksize + 1) break; + + blknum++; +- offset += BLKSIZE; +- count -= BLKSIZE; +- range_count += BLKSIZE; ++ offset += blksize; ++ count -= blksize; ++ range_count += blksize; + + if (count == 0) break; + blk_status (blknum, &present, &trimmed); +@@ -706,7 +726,7 @@ cow_extents (nbdkit_next *next, + /* Otherwise assume the block is non-sparse. */ + else { + e.offset = offset; +- e.length = BLKSIZE; ++ e.length = blksize; + e.type = 0; + + if (nbdkit_add_extent (extents, e.offset, e.length, e.type) == -1) { +@@ -715,8 +735,8 @@ cow_extents (nbdkit_next *next, + } + + blknum++; +- offset += BLKSIZE; +- count -= BLKSIZE; ++ offset += blksize; ++ count -= blksize; + } + + /* If the caller only wanted the first extent, and we've managed +@@ -734,11 +754,11 @@ cow_extents (nbdkit_next *next, + static struct nbdkit_filter filter = { + .name = "cow", + .longname = "nbdkit copy-on-write (COW) filter", +- .load = cow_load, + .unload = cow_unload, + .open = cow_open, + .config = cow_config, + .config_help = cow_config_help, ++ .get_ready = cow_get_ready, + .prepare = cow_prepare, + .get_size = cow_get_size, + .can_write = cow_can_write, +diff --git a/filters/cow/cow.h b/filters/cow/cow.h +new file mode 100644 +index 00000000..d46dbe91 +--- /dev/null ++++ b/filters/cow/cow.h +@@ -0,0 +1,39 @@ ++/* nbdkit ++ * Copyright (C) 2018-2021 Red Hat Inc. ++ * ++ * Redistribution and use in source and binary forms, with or without ++ * modification, are permitted provided that the following conditions are ++ * met: ++ * ++ * * Redistributions of source code must retain the above copyright ++ * notice, this list of conditions and the following disclaimer. ++ * ++ * * Redistributions in binary form must reproduce the above copyright ++ * notice, this list of conditions and the following disclaimer in the ++ * documentation and/or other materials provided with the distribution. ++ * ++ * * Neither the name of Red Hat nor the names of its contributors may be ++ * used to endorse or promote products derived from this software without ++ * specific prior written permission. ++ * ++ * THIS SOFTWARE IS PROVIDED BY RED HAT AND CONTRIBUTORS ''AS IS'' AND ++ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, ++ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A ++ * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL RED HAT OR ++ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, ++ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT ++ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF ++ * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ++ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, ++ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT ++ * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF ++ * SUCH DAMAGE. ++ */ ++ ++#ifndef NBDKIT_COW_H ++#define NBDKIT_COW_H ++ ++/* Size of a block in the cache. */ ++extern unsigned blksize; ++ ++#endif /* NBDKIT_COW_H */ +diff --git a/filters/cow/nbdkit-cow-filter.pod b/filters/cow/nbdkit-cow-filter.pod +index 7f861140..997c9097 100644 +--- a/filters/cow/nbdkit-cow-filter.pod ++++ b/filters/cow/nbdkit-cow-filter.pod +@@ -5,6 +5,7 @@ nbdkit-cow-filter - nbdkit copy-on-write (COW) filter + =head1 SYNOPSIS + + nbdkit --filter=cow plugin [plugin-args...] ++ [cow-block-size=N] + [cow-on-cache=false|true] + [cow-on-read=false|true|/PATH] + +@@ -42,6 +43,10 @@ serve the same data to each client. + + =over 4 + ++=item BN ++ ++Set the block size used by the filter. The default is 64K. ++ + =item B + + Do not save data from cache (prefetch) requests in the overlay. This +diff --git a/tests/Makefile.am b/tests/Makefile.am +index e61c5829..d93f848f 100644 +--- a/tests/Makefile.am ++++ b/tests/Makefile.am +@@ -1404,6 +1404,7 @@ EXTRA_DIST += \ + if HAVE_MKE2FS_WITH_D + TESTS += \ + test-cow.sh \ ++ test-cow-block-size.sh \ + test-cow-extents1.sh \ + test-cow-extents2.sh \ + test-cow-extents-large.sh \ +@@ -1415,6 +1416,7 @@ endif + TESTS += test-cow-null.sh + EXTRA_DIST += \ + test-cow.sh \ ++ test-cow-block-size.sh \ + test-cow-extents1.sh \ + test-cow-extents2.sh \ + test-cow-extents-large.sh \ +diff --git a/tests/test-cow-block-size.sh b/tests/test-cow-block-size.sh +new file mode 100755 +index 00000000..6de1c068 +--- /dev/null ++++ b/tests/test-cow-block-size.sh +@@ -0,0 +1,72 @@ ++#!/usr/bin/env bash ++# nbdkit ++# Copyright (C) 2018-2021 Red Hat Inc. ++# ++# Redistribution and use in source and binary forms, with or without ++# modification, are permitted provided that the following conditions are ++# met: ++# ++# * Redistributions of source code must retain the above copyright ++# notice, this list of conditions and the following disclaimer. ++# ++# * Redistributions in binary form must reproduce the above copyright ++# notice, this list of conditions and the following disclaimer in the ++# documentation and/or other materials provided with the distribution. ++# ++# * Neither the name of Red Hat nor the names of its contributors may be ++# used to endorse or promote products derived from this software without ++# specific prior written permission. ++# ++# THIS SOFTWARE IS PROVIDED BY RED HAT AND CONTRIBUTORS ''AS IS'' AND ++# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, ++# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A ++# PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL RED HAT OR ++# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, ++# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT ++# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF ++# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ++# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, ++# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT ++# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF ++# SUCH DAMAGE. ++ ++source ./functions.sh ++set -e ++set -x ++ ++requires_plugin linuxdisk ++requires guestfish --version ++requires nbdcopy --version ++requires qemu-img --version ++ ++sock=$(mktemp -u /tmp/nbdkit-test-sock.XXXXXX) ++files="cow-block-size-base.img $sock cow-block-size.pid" ++rm -f $files ++cleanup_fn rm -f $files ++ ++# Create a base image which is partitioned with an empty filesystem. ++rm -rf cow-block-size.d ++mkdir cow-block-size.d ++cleanup_fn rm -rf cow-block-size.d ++nbdkit -fv -U - linuxdisk cow-block-size.d size=100M \ ++ --run 'nbdcopy "$uri" cow-block-size-base.img' ++lastmod="$(stat -c "%y" cow-block-size-base.img)" ++ ++# Run nbdkit with a COW overlay, 4M block size and copy on read. ++start_nbdkit -P cow-block-size.pid -U $sock \ ++ --filter=cow file cow-block-size-base.img \ ++ cow-block-size=4M cow-on-read=true ++ ++# Write some data into the overlay. ++guestfish --format=raw -a "nbd://?socket=$sock" -m /dev/sda1 < +Date: Wed, 18 Aug 2021 19:16:43 -0500 +Subject: [PATCH] cow: Ship cow.h header + +Fixes: 7182c47d0 (cow: Make the block size configurable) +(cherry picked from commit 75ff1b8b1afb3744b21a306c62e4973c90d386be) +--- + filters/cow/Makefile.am | 1 + + 1 file changed, 1 insertion(+) + +diff --git a/filters/cow/Makefile.am b/filters/cow/Makefile.am +index a80ccd8f..88cda497 100644 +--- a/filters/cow/Makefile.am ++++ b/filters/cow/Makefile.am +@@ -39,6 +39,7 @@ nbdkit_cow_filter_la_SOURCES = \ + blk.c \ + blk.h \ + cow.c \ ++ cow.h \ + $(top_srcdir)/include/nbdkit-filter.h \ + $(NULL) + +-- +2.31.1 + diff --git a/libguestfs.keyring b/libguestfs.keyring index bb3eb55..f0508c8 100644 Binary files a/libguestfs.keyring and b/libguestfs.keyring differ diff --git a/nbdkit.spec b/nbdkit.spec index 0867673..b5e3d52 100644 --- a/nbdkit.spec +++ b/nbdkit.spec @@ -50,8 +50,8 @@ ExclusiveArch: x86_64 %global source_directory 1.26-stable Name: nbdkit -Version: 1.26.3 -Release: 4%{?dist} +Version: 1.26.5 +Release: 1%{?dist} Summary: NBD server License: BSD @@ -76,29 +76,30 @@ Source3: copy-patches.sh # https://gitlab.com/nbdkit/nbdkit/-/commits/rhel-9.0/ # Patches. -Patch0001: 0001-cache-Reduce-verbosity-of-debugging.patch -Patch0002: 0002-cache-cow-Add-blk_read_multiple-function.patch -Patch0003: 0003-cache-cow-Use-full-pread-pwrite-operations.patch -Patch0004: 0004-cache-Implement-cache-on-read-PATH.patch -Patch0005: 0005-cache-Add-cache-min-block-size-parameter.patch -Patch0006: 0006-cache-cow-Use-a-64K-block-size-by-default.patch -Patch0007: 0007-cache-Refactor-printing-state-into-new-function.patch -Patch0008: 0008-tests-cache-Test-cache-on-read-option-really-caches.patch -Patch0009: 0009-cow-Implement-cow-on-read.patch -Patch0010: 0010-delay-Add-delay-open-and-delay-close.patch -Patch0011: 0011-python-Implement-.cleanup-method.patch -Patch0012: 0012-server-Make-debug-messages-atomic.patch +Patch0001: 0001-server-reset-meta-context-replies-on-starttls.patch +Patch0002: 0002-cache-Reduce-verbosity-of-debugging.patch +Patch0003: 0003-cache-cow-Add-blk_read_multiple-function.patch +Patch0004: 0004-cache-cow-Use-full-pread-pwrite-operations.patch +Patch0005: 0005-cache-Implement-cache-on-read-PATH.patch +Patch0006: 0006-cache-Add-cache-min-block-size-parameter.patch +Patch0007: 0007-cache-cow-Use-a-64K-block-size-by-default.patch +Patch0008: 0008-cache-Refactor-printing-state-into-new-function.patch +Patch0009: 0009-tests-cache-Test-cache-on-read-option-really-caches.patch +Patch0010: 0010-cow-Implement-cow-on-read.patch +Patch0011: 0011-delay-Add-delay-open-and-delay-close.patch +Patch0012: 0012-python-Implement-.cleanup-method.patch Patch0013: 0013-cow-General-revision-and-updates-to-the-manual.patch Patch0014: 0014-cache-Move-plugin-args-in-synopsis-earlier.patch Patch0015: 0015-data-Improve-the-example-with-a-diagram.patch Patch0016: 0016-cow-Add-some-more-debugging-especially-for-blk_read_.patch -Patch0017: 0017-tests-test-debug-flags.sh-Don-t-use-port-10809-durin.patch -Patch0018: 0018-delay-Improve-parsing-and-representation-of-delay-ti.patch -Patch0019: 0019-server-Return-from-nbdkit_nanosleep-early-if-the-soc.patch -Patch0020: 0020-server-nanosleep-Change-error-for-early-end-of-sleep.patch -Patch0021: 0021-delay-Fix-delay-close.patch -Patch0022: 0022-delay-Test-delay-open-and-delay-close.patch -Patch0023: 0023-common-allocators-malloc.c-Remove-bogus-kernel-hints.patch +Patch0017: 0017-delay-Fix-delay-close.patch +Patch0018: 0018-delay-Test-delay-open-and-delay-close.patch +Patch0019: 0019-vddk-Implement-can_flush-and-can_fua.patch +Patch0020: 0020-vddk-Replace-DEBUG_CALL-with-bracketed-VDDK_CALL_STA.patch +Patch0021: 0021-tests-Add-a-better-test-of-real-VDDK.patch +Patch0022: 0022-vddk-Add-stats-about-the-amount-of-time-spent-in-VDD.patch +Patch0023: 0023-cow-Make-the-block-size-configurable.patch +Patch0024: 0024-cow-Ship-cow.h-header.patch BuildRequires: make %if 0%{patches_touch_autotools} @@ -1269,6 +1270,16 @@ export LIBGUESTFS_TRACE=1 %changelog +* Thu Aug 19 2021 Richard W.M. Jones - 1.26.5-1 +- Rebase along stable branch to 1.26.5 + resolves: rhbz#1995327 +- Add nbdkit-vddk-plugin -D vddk.stats=1 flag + resolves: rhbz#1995329 +- Add nbdkit-cow-filter cow-block-size parameter + resolves: rhbz#1995332 +- Fix CVE-2021-3716 nbdkit: NBD_OPT_STRUCTURED_REPLY injection on STARTTLS +- Update keyring + * Wed Aug 11 2021 Richard W.M. Jones - 1.26.3-4 - Remove bogus kernel hints in allocator=malloc resolves: rhbz#1992542 diff --git a/sources b/sources index 6b4b5c9..2872f06 100644 --- a/sources +++ b/sources @@ -1,2 +1,2 @@ -SHA512 (nbdkit-1.26.3.tar.gz) = 7a85b1f35aabbe9cf9f5b45f2cd6458df1e7acf25b4753cfeae13196cd16d83c8d91b491d3f4b4ee24f567f8432a82e1ddda0c3c4a145429eefe43e68aa9f0c7 -SHA512 (nbdkit-1.26.3.tar.gz.sig) = 34c61897a8f36a53f88c062de5da42ef12c21f43e6fe39b874da3c862be2e86ef075b02bd079195d91da43bd7aced126ae4e4d7763152329ff38ceee3906423c +SHA512 (nbdkit-1.26.5.tar.gz) = 966bee315e7e9535eadf5ee392fd3c7b8ee33e0bc910ac903cb4f1e2748e6bcff3c459594665534693a24f64a2ff440ac30c2c673a2ce39ccddcb557b7afc4ce +SHA512 (nbdkit-1.26.5.tar.gz.sig) = 8bd9279896e747c417a2417bae89cfae8eccf9a0bf445a510b69c682fa9342329996cff5e0ed6394741edd2a842deef05456c33642a4b48c44a0848b08e1f97f