From e7309e9d45884bd23e2b6792de7b6be39b20b5b5 Mon Sep 17 00:00:00 2001 From: AlmaLinux RelEng Bot Date: Mon, 6 Apr 2026 13:24:09 -0400 Subject: [PATCH] import Oracle_OSS nbdkit-1.44.1-6.el10_1 --- ...fix-ftruncate-error-with-filter-name.patch | 56 ++++ ...k-blk-lock-and-blk-bm-on-error-paths.patch | 40 +++ ...-to-store-the-overlay-file-descripto.patch | 191 ++++++++++++ ...t-function-to-create-temporary-files.patch | 97 ++++++ ...f-the-temporary-file-to-debug-output.patch | 30 ++ ...ort-overlays-larger-than-16T-on-ext4.patch | 292 ++++++++++++++++++ ...che-cow-Add-prefix-before-more-calls.patch | 133 ++++++++ 0019-cow-Fix-block-fd-calculation.patch | 30 ++ ...ow-Fix-offsets-when-overlay-is-split.patch | 112 +++++++ ...cow-Make-some-offset-variables-const.patch | 55 ++++ copy-patches.sh | 0 nbdkit-find-provides | 0 nbdkit.spec | 16 +- 13 files changed, 1051 insertions(+), 1 deletion(-) create mode 100644 0012-cache-cow-Prefix-ftruncate-error-with-filter-name.patch create mode 100644 0013-cow-Don-t-leak-blk-lock-and-blk-bm-on-error-paths.patch create mode 100644 0014-cow-Use-a-vector-to-store-the-overlay-file-descripto.patch create mode 100644 0015-cow-Split-out-function-to-create-temporary-files.patch create mode 100644 0016-cow-Add-name-of-the-temporary-file-to-debug-output.patch create mode 100644 0017-cow-Support-overlays-larger-than-16T-on-ext4.patch create mode 100644 0018-cache-cow-Add-prefix-before-more-calls.patch create mode 100644 0019-cow-Fix-block-fd-calculation.patch create mode 100644 0020-cow-Fix-offsets-when-overlay-is-split.patch create mode 100644 0021-cow-Make-some-offset-variables-const.patch mode change 100644 => 100755 copy-patches.sh mode change 100644 => 100755 nbdkit-find-provides diff --git a/0012-cache-cow-Prefix-ftruncate-error-with-filter-name.patch b/0012-cache-cow-Prefix-ftruncate-error-with-filter-name.patch new file mode 100644 index 0000000..8ed78a1 --- /dev/null +++ b/0012-cache-cow-Prefix-ftruncate-error-with-filter-name.patch @@ -0,0 +1,56 @@ +From 34ee72b180f2a35e35df7d763881e7cfc205c665 Mon Sep 17 00:00:00 2001 +From: "Richard W.M. Jones" +Date: Wed, 1 Apr 2026 14:34:39 +0100 +Subject: [PATCH] cache, cow: Prefix ftruncate error with filter name + +We had a peculiar qemu error reported: + + server reported: ftruncate: File too large + +It was unclear at first where this message came from. (Originally it +was thought to come from qemu). Prefix the filter name on the error +message to hopefully make this a little easier to debug in future. + +(cherry picked from commit 63416a8347f9865cc69e162a1c1e42015d394b24) +--- + filters/cache/blk.c | 2 +- + filters/cow/blk.c | 3 +-- + 2 files changed, 2 insertions(+), 3 deletions(-) + +diff --git a/filters/cache/blk.c b/filters/cache/blk.c +index ba2c41b4..9a200dde 100644 +--- a/filters/cache/blk.c ++++ b/filters/cache/blk.c +@@ -195,7 +195,7 @@ blk_set_size (uint64_t new_size) + return -1; + + if (ftruncate (fd, ROUND_UP (size, blksize)) == -1) { +- nbdkit_error ("ftruncate: %m"); ++ nbdkit_error ("cache: ftruncate: %m"); + return -1; + } + +diff --git a/filters/cow/blk.c b/filters/cow/blk.c +index ec51eea1..cc2f536b 100644 +--- a/filters/cow/blk.c ++++ b/filters/cow/blk.c +@@ -225,7 +225,6 @@ blk_free (struct blk_overlay *blk) + } + } + +- + /* Allocate or resize the overlay file and bitmap. */ + int + blk_set_size (struct blk_overlay *blk, uint64_t new_size) +@@ -238,7 +237,7 @@ blk_set_size (struct blk_overlay *blk, uint64_t new_size) + return -1; + + if (ftruncate (blk->fd, ROUND_UP (blk->size, blksize)) == -1) { +- nbdkit_error ("ftruncate: %m"); ++ nbdkit_error ("cow: ftruncate: %m"); + return -1; + } + +-- +2.47.3 + diff --git a/0013-cow-Don-t-leak-blk-lock-and-blk-bm-on-error-paths.patch b/0013-cow-Don-t-leak-blk-lock-and-blk-bm-on-error-paths.patch new file mode 100644 index 0000000..3342cc6 --- /dev/null +++ b/0013-cow-Don-t-leak-blk-lock-and-blk-bm-on-error-paths.patch @@ -0,0 +1,40 @@ +From e60280f405be972d6ca585494825642817d37629 Mon Sep 17 00:00:00 2001 +From: "Richard W.M. Jones" +Date: Wed, 1 Apr 2026 15:30:15 +0100 +Subject: [PATCH] cow: Don't leak blk->lock and blk->bm on error paths + +Theoretically these were leaked on error paths, although in reality +nbdkit immediately exits on failure here so there wasn't a real issue. +Still it makes the code slightly cleaner. + +(cherry picked from commit 04d2b3e30e980f7da787d4cd2a4cde839be756fb) +--- + filters/cow/blk.c | 5 +++-- + 1 file changed, 3 insertions(+), 2 deletions(-) + +diff --git a/filters/cow/blk.c b/filters/cow/blk.c +index cc2f536b..c3173fdd 100644 +--- a/filters/cow/blk.c ++++ b/filters/cow/blk.c +@@ -174,8 +174,6 @@ blk_create (void) + } + + blk->fd = -1; +- pthread_mutex_init (&blk->lock, NULL); +- bitmap_init (&blk->bm, blksize, 2 /* bits per block */); + + filename = strdup (template); + if (filename == NULL) { +@@ -210,6 +208,9 @@ blk_create (void) + unlink (filename); + free (filename); + ++ pthread_mutex_init (&blk->lock, NULL); ++ bitmap_init (&blk->bm, blksize, 2 /* bits per block */); ++ + return blk; + } + +-- +2.47.3 + diff --git a/0014-cow-Use-a-vector-to-store-the-overlay-file-descripto.patch b/0014-cow-Use-a-vector-to-store-the-overlay-file-descripto.patch new file mode 100644 index 0000000..4a254bb --- /dev/null +++ b/0014-cow-Use-a-vector-to-store-the-overlay-file-descripto.patch @@ -0,0 +1,191 @@ +From d0f211ae730f5cea4610b1e5455f2ca649c9ae69 Mon Sep 17 00:00:00 2001 +From: "Richard W.M. Jones" +Date: Wed, 1 Apr 2026 15:33:30 +0100 +Subject: [PATCH] cow: Use a vector to store the overlay file descriptor + +Instead of storing a single blk->fd field, replace this with a vector +of file descriptors. In this change it is only used to store a single +file descriptor, so there is no functional change so far. + +(cherry picked from commit bd01870d6e29a83a95dee6c3ab36afff9d96b651) +--- + filters/cow/blk.c | 48 ++++++++++++++++++++++++++++++----------------- + 1 file changed, 31 insertions(+), 17 deletions(-) + +diff --git a/filters/cow/blk.c b/filters/cow/blk.c +index c3173fdd..4957c871 100644 +--- a/filters/cow/blk.c ++++ b/filters/cow/blk.c +@@ -98,6 +98,7 @@ + #include "pread.h" + #include "pwrite.h" + #include "utils.h" ++#include "vector.h" + + #include "cow.h" + #include "blk.h" +@@ -145,9 +146,11 @@ blk_unload (void) + free (template); + } + ++DEFINE_VECTOR_TYPE (fd_vector, int); ++ + struct blk_overlay { +- /* The temporary overlay. */ +- int fd; ++ /* The temporary overlays. Each covers 8T virtual size. */ ++ fd_vector fds; + + /* This lock protects the bitmap from parallel access. */ + pthread_mutex_t lock; +@@ -165,6 +168,7 @@ struct blk_overlay * + blk_create (void) + { + struct blk_overlay *blk; ++ int fd; + char *filename; + + blk = calloc (1, sizeof *blk); +@@ -173,8 +177,9 @@ blk_create (void) + return NULL; + } + +- blk->fd = -1; ++ blk->fds = (fd_vector)empty_vector; + ++ /* We always create the first file, even if the disk size is 0. */ + filename = strdup (template); + if (filename == NULL) { + nbdkit_error ("strdup: %m"); +@@ -183,22 +188,22 @@ blk_create (void) + } + + #ifdef HAVE_MKOSTEMP +- blk->fd = mkostemp (filename, O_CLOEXEC); ++ fd = mkostemp (filename, O_CLOEXEC); + #else + /* Not atomic, but this is only invoked during .load, so the race + * won't affect any plugin actions trying to fork + */ +- blk->fd = mkstemp (filename); +- if (blk->fd >= 0) { +- blk->fd = set_cloexec (blk->fd); +- if (blk->fd < 0) { ++ fd = mkstemp (filename); ++ if (fd >= 0) { ++ fd = set_cloexec (fd); ++ if (fd < 0) { + int e = errno; + unlink (template); + errno = e; + } + } + #endif +- if (blk->fd == -1) { ++ if (fd == -1) { + nbdkit_error ("mkostemp: %s: %m", template); + free (blk); + free (filename); +@@ -208,6 +213,13 @@ blk_create (void) + unlink (filename); + free (filename); + ++ if (fd_vector_append (&blk->fds, fd) == -1) { ++ nbdkit_error ("realloc: %m"); ++ close (fd); ++ free (blk); ++ return NULL; ++ } ++ + pthread_mutex_init (&blk->lock, NULL); + bitmap_init (&blk->bm, blksize, 2 /* bits per block */); + +@@ -218,8 +230,7 @@ void + blk_free (struct blk_overlay *blk) + { + if (blk) { +- if (blk->fd >= 0) +- close (blk->fd); ++ fd_vector_iter (&blk->fds, (void*)close); + bitmap_free (&blk->bm); + pthread_mutex_destroy (&blk->lock); + free (blk); +@@ -237,7 +248,7 @@ blk_set_size (struct blk_overlay *blk, uint64_t new_size) + if (bitmap_resize (&blk->bm, blk->size) == -1) + return -1; + +- if (ftruncate (blk->fd, ROUND_UP (blk->size, blksize)) == -1) { ++ if (ftruncate (blk->fds.ptr[0], ROUND_UP (blk->size, blksize)) == -1) { + nbdkit_error ("cow: ftruncate: %m"); + return -1; + } +@@ -271,6 +282,7 @@ blk_read_multiple (struct blk_overlay *blk, + off_t offset = blknum * blksize; + enum bm_entry state; + uint64_t b, runblocks; ++ const int fd = blk->fds.ptr[0]; + + /* Find out how many of the following blocks form a "run" with the + * same state. We can process that many blocks in one go. +@@ -329,7 +341,7 @@ blk_read_multiple (struct blk_overlay *blk, + "at offset %" PRIu64 " into the cache", + runblocks, offset); + +- if (full_pwrite (blk->fd, block, blksize * runblocks, offset) == -1) { ++ if (full_pwrite (fd, block, blksize * runblocks, offset) == -1) { + *err = errno; + nbdkit_error ("pwrite: %m"); + return -1; +@@ -339,7 +351,7 @@ blk_read_multiple (struct blk_overlay *blk, + } + } + else if (state == BLOCK_ALLOCATED) { /* Read overlay. */ +- if (full_pread (blk->fd, block, blksize * runblocks, offset) == -1) { ++ if (full_pread (fd, block, blksize * runblocks, offset) == -1) { + *err = errno; + nbdkit_error ("pread: %m"); + return -1; +@@ -379,6 +391,7 @@ blk_cache (struct blk_overlay *blk, + off_t offset = blknum * blksize; + enum bm_entry state = bitmap_get_blk (&blk->bm, blknum, BLOCK_NOT_ALLOCATED); + unsigned n = blksize, tail = 0; ++ const int fd = blk->fds.ptr[0]; + + if (offset + n > blk->size) { + tail = offset + n - blk->size; +@@ -391,7 +404,7 @@ blk_cache (struct blk_overlay *blk, + + if (state == BLOCK_ALLOCATED) { + #if HAVE_POSIX_FADVISE +- int r = posix_fadvise (blk->fd, offset, blksize, POSIX_FADV_WILLNEED); ++ int r = posix_fadvise (fd, offset, blksize, POSIX_FADV_WILLNEED); + if (r) { + errno = r; + nbdkit_error ("posix_fadvise: %m"); +@@ -416,7 +429,7 @@ blk_cache (struct blk_overlay *blk, + memset (block + n, 0, tail); + + if (mode == BLK_CACHE_COW) { +- if (full_pwrite (blk->fd, block, blksize, offset) == -1) { ++ if (full_pwrite (fd, block, blksize, offset) == -1) { + *err = errno; + nbdkit_error ("pwrite: %m"); + return -1; +@@ -431,12 +444,13 @@ blk_write (struct blk_overlay *blk, + uint64_t blknum, const uint8_t *block, int *err) + { + off_t offset = blknum * blksize; ++ const int fd = blk->fds.ptr[0]; + + if (cow_debug_verbose) + nbdkit_debug ("cow: blk_write block %" PRIu64 " (offset %" PRIu64 ")", + blknum, (uint64_t) offset); + +- if (full_pwrite (blk->fd, block, blksize, offset) == -1) { ++ if (full_pwrite (fd, block, blksize, offset) == -1) { + *err = errno; + nbdkit_error ("pwrite: %m"); + return -1; +-- +2.47.3 + diff --git a/0015-cow-Split-out-function-to-create-temporary-files.patch b/0015-cow-Split-out-function-to-create-temporary-files.patch new file mode 100644 index 0000000..642990a --- /dev/null +++ b/0015-cow-Split-out-function-to-create-temporary-files.patch @@ -0,0 +1,97 @@ +From 0fc4138b253d3803ea2239df2f268c005ee3b3f4 Mon Sep 17 00:00:00 2001 +From: "Richard W.M. Jones" +Date: Wed, 1 Apr 2026 16:03:21 +0100 +Subject: [PATCH] cow: Split out function to create temporary files + +Just code refactoring. + +(cherry picked from commit e8e409311b2c28b89da78b538196b9be018dd47d) +--- + filters/cow/blk.c | 45 ++++++++++++++++++++++++++++----------------- + 1 file changed, 28 insertions(+), 17 deletions(-) + +diff --git a/filters/cow/blk.c b/filters/cow/blk.c +index 4957c871..96d613e2 100644 +--- a/filters/cow/blk.c ++++ b/filters/cow/blk.c +@@ -164,27 +164,16 @@ struct blk_overlay { + uint64_t size; + }; + +-struct blk_overlay * +-blk_create (void) ++static int ++create_temporary_file (void) + { +- struct blk_overlay *blk; + int fd; + char *filename; + +- blk = calloc (1, sizeof *blk); +- if (blk == NULL) { +- nbdkit_error ("calloc: %m"); +- return NULL; +- } +- +- blk->fds = (fd_vector)empty_vector; +- +- /* We always create the first file, even if the disk size is 0. */ + filename = strdup (template); + if (filename == NULL) { + nbdkit_error ("strdup: %m"); +- free (blk); +- return NULL; ++ return -1; + } + + #ifdef HAVE_MKOSTEMP +@@ -198,21 +187,43 @@ blk_create (void) + fd = set_cloexec (fd); + if (fd < 0) { + int e = errno; +- unlink (template); ++ unlink (filename); + errno = e; + } + } + #endif + if (fd == -1) { + nbdkit_error ("mkostemp: %s: %m", template); +- free (blk); + free (filename); +- return NULL; ++ return -1; + } + + unlink (filename); + free (filename); + ++ return fd; ++} ++ ++struct blk_overlay * ++blk_create (void) ++{ ++ struct blk_overlay *blk; ++ int fd; ++ ++ blk = calloc (1, sizeof *blk); ++ if (blk == NULL) { ++ nbdkit_error ("calloc: %m"); ++ return NULL; ++ } ++ ++ blk->fds = (fd_vector)empty_vector; ++ ++ /* We always create the first file, even if the disk size is 0. */ ++ fd = create_temporary_file (); ++ if (fd == -1) { ++ free (blk); ++ return NULL; ++ } + if (fd_vector_append (&blk->fds, fd) == -1) { + nbdkit_error ("realloc: %m"); + close (fd); +-- +2.47.3 + diff --git a/0016-cow-Add-name-of-the-temporary-file-to-debug-output.patch b/0016-cow-Add-name-of-the-temporary-file-to-debug-output.patch new file mode 100644 index 0000000..72632f0 --- /dev/null +++ b/0016-cow-Add-name-of-the-temporary-file-to-debug-output.patch @@ -0,0 +1,30 @@ +From 021b9fab2ba58dde338658052b45618fcbb8ebc7 Mon Sep 17 00:00:00 2001 +From: "Richard W.M. Jones" +Date: Wed, 1 Apr 2026 16:43:45 +0100 +Subject: [PATCH] cow: Add name of the temporary file to debug output + +If -D cow.verbose=1 display the name of the temporary file. The file +is deleted so it does not appear in the filesystem. + +(cherry picked from commit d415227d30ce7056ded227bb9490e937de230ae1) +--- + filters/cow/blk.c | 3 +++ + 1 file changed, 3 insertions(+) + +diff --git a/filters/cow/blk.c b/filters/cow/blk.c +index 96d613e2..1012fb5e 100644 +--- a/filters/cow/blk.c ++++ b/filters/cow/blk.c +@@ -198,6 +198,9 @@ create_temporary_file (void) + return -1; + } + ++ if (cow_debug_verbose) ++ nbdkit_debug ("cow: created temporary file for overlay: %s", filename); ++ + unlink (filename); + free (filename); + +-- +2.47.3 + diff --git a/0017-cow-Support-overlays-larger-than-16T-on-ext4.patch b/0017-cow-Support-overlays-larger-than-16T-on-ext4.patch new file mode 100644 index 0000000..6b765b1 --- /dev/null +++ b/0017-cow-Support-overlays-larger-than-16T-on-ext4.patch @@ -0,0 +1,292 @@ +From c02b8506469203e537fdf6fe37d4447cc4d358a3 Mon Sep 17 00:00:00 2001 +From: "Richard W.M. Jones" +Date: Wed, 1 Apr 2026 16:37:01 +0100 +Subject: [PATCH] cow: Support overlays larger than 16T on ext4 + +nbdkit-cow-filter writes the copy-on-write overlay into a temporary +file under /var/tmp which has the same size as the underlying virtual +disk. If the temporary directory being used is on ext4, this +effectively limits the size of the overlay to just under 16T, since +that is the maximum file size supported. (Note that the file is +sparse so this does not mean all the space is used.). For example: + + $ nbdkit --filter=cow null 17T --run 'nbdinfo "$uri"' + nbdkit: null[1]: error: ftruncate: File too large + nbdkit: null[1]: error: ftruncate: File too large + nbdkit: null[1]: error: ftruncate: File too large + nbdinfo: nbd_opt_go: the server has no export named '': ftruncate: File too large: No such file or directory for the default export + nbdinfo: suggestion: to list all exports on the server, use --list + protocol: newstyle-fixed without TLS, using structured packets + +This change splits the overlay into as many 8T chunks as needed, so +the ext4 maximum file size is no longer a limit. + +Fixes: https://issues.redhat.com/browse/RHEL-163983 +(cherry picked from commit eb79a0e0c63ba1884bde02dc884a26aea2fc4324) +--- + filters/cow/blk.c | 69 ++++++++++++++++++++++++++++++++-- + tests/Makefile.am | 6 ++- + tests/test-cow-huge.sh | 85 ++++++++++++++++++++++++++++++++++++++++++ + 3 files changed, 155 insertions(+), 5 deletions(-) + create mode 100755 tests/test-cow-huge.sh + +diff --git a/filters/cow/blk.c b/filters/cow/blk.c +index 1012fb5e..defd0d94 100644 +--- a/filters/cow/blk.c ++++ b/filters/cow/blk.c +@@ -67,6 +67,10 @@ + * + * Since the overlay is a deleted temporary file, we can ignore FUA + * and flush commands. ++ * ++ * For disk sizes larger than 8T, the temporary file is split into ++ * multiple files, at most 8T in size. This ensures we can work on ++ * filesystems with file size limits like ext4 (max file size < 16T). + */ + + #include +@@ -81,6 +85,7 @@ + #include + #include + #include ++#include + #include + + #ifdef HAVE_ALLOCA_H +@@ -109,6 +114,8 @@ enum bm_entry { + BLOCK_TRIMMED = 3, + }; + ++#define MAX_FILE_SIZE (UINT64_C (8) * 1024 * 1024 * 1024 * 1024) /* 8T */ ++ + static const char * + state_to_string (enum bm_entry state) + { +@@ -256,13 +263,56 @@ int + blk_set_size (struct blk_overlay *blk, uint64_t new_size) + { + ACQUIRE_LOCK_FOR_CURRENT_SCOPE (&blk->lock); ++ const uint64_t new_size_rounded_up = ROUND_UP (new_size, blksize); ++ uint64_t remainder; ++ size_t i, n; ++ int fd; + + blk->size = new_size; + + if (bitmap_resize (&blk->bm, blk->size) == -1) + return -1; + +- if (ftruncate (blk->fds.ptr[0], ROUND_UP (blk->size, blksize)) == -1) { ++ /* How many temporary files do we need? Each temp file is limited to 8T. */ ++ n = DIV_ROUND_UP (new_size_rounded_up, MAX_FILE_SIZE); ++ if (n == 0) ++ n = 1; ++ nbdkit_debug ("cow: using %zu temporary file(s)", n); ++ ++ /* Create new temporary files in case it has expanded. */ ++ while (blk->fds.len < n) { ++ fd = create_temporary_file (); ++ if (fd == -1) ++ return -1; ++ if (fd_vector_append (&blk->fds, fd) == -1) { ++ nbdkit_error ("realloc: %m"); ++ /* The other fds are stored in the handle and will be closed by ++ * blk_free, so we only need to close this one. ++ */ ++ close (fd); ++ return -1; ++ } ++ } ++ ++ /* Remove any extra temporary files in case it has shrunk. */ ++ while (blk->fds.len > n) { ++ close (blk->fds.ptr[n]); ++ fd_vector_remove (&blk->fds, n); ++ } ++ ++ /* The first 0..n-1 temp files have to be truncated to the full ++ * size, with the last temp file being truncated to the remainder. ++ */ ++ remainder = new_size_rounded_up; ++ for (i = 0; i < n-1; ++i) { ++ if (ftruncate (blk->fds.ptr[i], MAX_FILE_SIZE) == -1) { ++ nbdkit_error ("cow: ftruncate: %m"); ++ return -1; ++ } ++ assert (remainder >= MAX_FILE_SIZE); ++ remainder -= MAX_FILE_SIZE; ++ } ++ if (ftruncate (blk->fds.ptr[n-1], remainder) == -1) { + nbdkit_error ("cow: ftruncate: %m"); + return -1; + } +@@ -270,6 +320,15 @@ blk_set_size (struct blk_overlay *blk, uint64_t new_size) + return 0; + } + ++/* Get the temporary file corresponding to a block. */ ++static int ++get_fd_for_blknum (struct blk_overlay *blk, uint64_t blknum) ++{ ++ size_t i = blknum / MAX_FILE_SIZE / blksize; ++ assert (i < blk->fds.len); ++ return blk->fds.ptr[i]; ++} ++ + /* This is a bit of a hack since usually this information is hidden in + * the blk module. However it is needed when calculating extents. + */ +@@ -296,7 +355,7 @@ blk_read_multiple (struct blk_overlay *blk, + off_t offset = blknum * blksize; + enum bm_entry state; + uint64_t b, runblocks; +- const int fd = blk->fds.ptr[0]; ++ const int fd = get_fd_for_blknum (blk, blknum); + + /* Find out how many of the following blocks form a "run" with the + * same state. We can process that many blocks in one go. +@@ -316,6 +375,8 @@ blk_read_multiple (struct blk_overlay *blk, + bitmap_get_blk (&blk->bm, blknum + b, BLOCK_NOT_ALLOCATED); + if (state != s) + break; ++ if (get_fd_for_blknum (blk, blknum + b) != fd) ++ break; + } + } + +@@ -405,7 +466,7 @@ blk_cache (struct blk_overlay *blk, + off_t offset = blknum * blksize; + enum bm_entry state = bitmap_get_blk (&blk->bm, blknum, BLOCK_NOT_ALLOCATED); + unsigned n = blksize, tail = 0; +- const int fd = blk->fds.ptr[0]; ++ const int fd = get_fd_for_blknum (blk, blknum); + + if (offset + n > blk->size) { + tail = offset + n - blk->size; +@@ -458,7 +519,7 @@ blk_write (struct blk_overlay *blk, + uint64_t blknum, const uint8_t *block, int *err) + { + off_t offset = blknum * blksize; +- const int fd = blk->fds.ptr[0]; ++ const int fd = get_fd_for_blknum (blk, blknum); + + if (cow_debug_verbose) + nbdkit_debug ("cow: blk_write block %" PRIu64 " (offset %" PRIu64 ")", +diff --git a/tests/Makefile.am b/tests/Makefile.am +index d7053ba2..bc66b61c 100644 +--- a/tests/Makefile.am ++++ b/tests/Makefile.am +@@ -1724,7 +1724,10 @@ TESTS += \ + test-cow-unaligned.sh \ + $(NULL) + endif +-TESTS += test-cow-null.sh ++TESTS += \ ++ test-cow-huge.sh \ ++ test-cow-null.sh \ ++ $(NULL) + EXTRA_DIST += \ + test-cow.sh \ + test-cow-block-size.sh \ +@@ -1732,6 +1735,7 @@ EXTRA_DIST += \ + test-cow-extents1.sh \ + test-cow-extents2.sh \ + test-cow-extents-large.sh \ ++ test-cow-huge.sh \ + test-cow-null.sh \ + test-cow-on-read.sh \ + test-cow-on-read-caches.sh \ +diff --git a/tests/test-cow-huge.sh b/tests/test-cow-huge.sh +new file mode 100755 +index 00000000..cad1f6cc +--- /dev/null ++++ b/tests/test-cow-huge.sh +@@ -0,0 +1,85 @@ ++#!/usr/bin/env bash ++# nbdkit ++# Copyright Red Hat ++# ++# Redistribution and use in source and binary forms, with or without ++# modification, are permitted provided that the following conditions are ++# met: ++# ++# * Redistributions of source code must retain the above copyright ++# notice, this list of conditions and the following disclaimer. ++# ++# * Redistributions in binary form must reproduce the above copyright ++# notice, this list of conditions and the following disclaimer in the ++# documentation and/or other materials provided with the distribution. ++# ++# * Neither the name of Red Hat nor the names of its contributors may be ++# used to endorse or promote products derived from this software without ++# specific prior written permission. ++# ++# THIS SOFTWARE IS PROVIDED BY RED HAT AND CONTRIBUTORS ''AS IS'' AND ++# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, ++# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A ++# PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL RED HAT OR ++# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, ++# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT ++# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF ++# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ++# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, ++# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT ++# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF ++# SUCH DAMAGE. ++ ++# Test cow filter with huge overlays. This used to break because of ++# file size limits on /var/tmp. ++ ++source ./functions.sh ++set -e ++set -x ++set -u ++ ++requires_run ++requires_nbdsh_uri ++ ++# ext4 has a 16T file size limit. ++size=17T ++ ++define script <<'EOF' ++ ++import random ++ ++random.seed(None) ++ ++M = 1024*1024 ++T = 1024*1024*1024*1024 ++ ++# Test a few megabytes either side of the 8T boundary. ++overlay = bytearray(10 * M) ++overlay_offset = int(8 * T - len(overlay)/2) ++assert h.get_size() > overlay_offset + len(overlay) ++ ++for iter in range(1, 200): ++ offset = random.randint(0, len(overlay) - 10) ++ count = min(len(overlay) - offset, random.randint(1, 1 * M)) ++ buf = h.pread(count, overlay_offset + offset) ++ assert buf == overlay[offset:offset+count] ++ c = random.randint(1, 255) ++ buf = bytearray([c]) * count ++ h.pwrite(buf, overlay_offset + offset) ++ overlay[offset:offset+count] = buf ++ buf = h.pread(count, overlay_offset + offset) ++ assert buf == overlay[offset:offset+count] ++ ++ if random.randint(0, 10) >= 7: ++ offset = random.randint(0, len(overlay) - 10) ++ count = min(len(overlay) - offset, random.randint(1, 1 * M)) ++ zbuf = bytearray(count) ++ h.trim(count, overlay_offset + offset) ++ overlay[offset:offset+count] = zbuf ++ buf = h.pread(count, overlay_offset + offset) ++ assert buf == zbuf ++ ++EOF ++export script ++ ++nbdkit -fv --filter=cow null $size --run ' nbdsh -u "$uri" -c "$script" ' +-- +2.47.3 + diff --git a/0018-cache-cow-Add-prefix-before-more-calls.patch b/0018-cache-cow-Add-prefix-before-more-calls.patch new file mode 100644 index 0000000..8aaa94a --- /dev/null +++ b/0018-cache-cow-Add-prefix-before-more-calls.patch @@ -0,0 +1,133 @@ +From 5261fbf8b7579bd6592fbfd51b6a99ce09a8b3c1 Mon Sep 17 00:00:00 2001 +From: "Richard W.M. Jones" +Date: Fri, 3 Apr 2026 13:41:36 +0100 +Subject: [PATCH] cache, cow: Add prefix before more calls + +Try to diagnose the origin of more error messages. + +Related: commit 63416a8347f9865cc69e162a1c1e42015d394b24 +(cherry picked from commit 25e1d1fdaedfd3c0cdce0600585f42478c93eda7) +--- + filters/cache/blk.c | 14 +++++++------- + filters/cow/blk.c | 10 +++++----- + 2 files changed, 12 insertions(+), 12 deletions(-) + +diff --git a/filters/cache/blk.c b/filters/cache/blk.c +index 9a200dde..02c33633 100644 +--- a/filters/cache/blk.c ++++ b/filters/cache/blk.c +@@ -157,7 +157,7 @@ blk_init (void) + * least as large as the filesystem block size. + */ + if (fstatvfs (fd, &statvfs) == -1) { +- nbdkit_error ("fstatvfs: %s: %m", tmpdir); ++ nbdkit_error ("cache: fstatvfs: %s: %m", tmpdir); + return -1; + } + blksize = MAX (min_block_size, statvfs.f_bsize); +@@ -263,7 +263,7 @@ _blk_read_multiple (nbdkit_next *next, + + if (full_pwrite (fd, block, blksize * runblocks, offset) == -1) { + *err = errno; +- nbdkit_error ("pwrite: %m"); ++ nbdkit_error ("cache: pwrite: %m"); + return -1; + } + for (b = 0; b < runblocks; ++b) { +@@ -275,7 +275,7 @@ _blk_read_multiple (nbdkit_next *next, + else { /* Read cache. */ + if (full_pread (fd, block, blksize * runblocks, offset) == -1) { + *err = errno; +- nbdkit_error ("pread: %m"); ++ nbdkit_error ("cache: pread: %m"); + return -1; + } + for (b = 0; b < runblocks; ++b) +@@ -349,7 +349,7 @@ blk_cache (nbdkit_next *next, + + if (full_pwrite (fd, block, blksize, offset) == -1) { + *err = errno; +- nbdkit_error ("pwrite: %m"); ++ nbdkit_error ("cache: pwrite: %m"); + return -1; + } + bitmap_set_blk (&bm, blknum, BLOCK_CLEAN); +@@ -360,7 +360,7 @@ blk_cache (nbdkit_next *next, + int r = posix_fadvise (fd, offset, blksize, POSIX_FADV_WILLNEED); + if (r) { + errno = r; +- nbdkit_error ("posix_fadvise: %m"); ++ nbdkit_error ("cache: posix_fadvise: %m"); + return -1; + } + #endif +@@ -390,7 +390,7 @@ blk_writethrough (nbdkit_next *next, + + if (full_pwrite (fd, block, blksize, offset) == -1) { + *err = errno; +- nbdkit_error ("pwrite: %m"); ++ nbdkit_error ("cache: pwrite: %m"); + return -1; + } + +@@ -424,7 +424,7 @@ blk_write (nbdkit_next *next, + + if (full_pwrite (fd, block, blksize, offset) == -1) { + *err = errno; +- nbdkit_error ("pwrite: %m"); ++ nbdkit_error ("cache: pwrite: %m"); + return -1; + } + bitmap_set_blk (&bm, blknum, BLOCK_DIRTY); +diff --git a/filters/cow/blk.c b/filters/cow/blk.c +index defd0d94..afb48af9 100644 +--- a/filters/cow/blk.c ++++ b/filters/cow/blk.c +@@ -418,7 +418,7 @@ blk_read_multiple (struct blk_overlay *blk, + + if (full_pwrite (fd, block, blksize * runblocks, offset) == -1) { + *err = errno; +- nbdkit_error ("pwrite: %m"); ++ nbdkit_error ("cow: pwrite: %m"); + return -1; + } + for (b = 0; b < runblocks; ++b) +@@ -428,7 +428,7 @@ blk_read_multiple (struct blk_overlay *blk, + else if (state == BLOCK_ALLOCATED) { /* Read overlay. */ + if (full_pread (fd, block, blksize * runblocks, offset) == -1) { + *err = errno; +- nbdkit_error ("pread: %m"); ++ nbdkit_error ("cow: pread: %m"); + return -1; + } + } +@@ -482,7 +482,7 @@ blk_cache (struct blk_overlay *blk, + int r = posix_fadvise (fd, offset, blksize, POSIX_FADV_WILLNEED); + if (r) { + errno = r; +- nbdkit_error ("posix_fadvise: %m"); ++ nbdkit_error ("cow: posix_fadvise: %m"); + return -1; + } + #endif +@@ -506,7 +506,7 @@ blk_cache (struct blk_overlay *blk, + if (mode == BLK_CACHE_COW) { + if (full_pwrite (fd, block, blksize, offset) == -1) { + *err = errno; +- nbdkit_error ("pwrite: %m"); ++ nbdkit_error ("cow: pwrite: %m"); + return -1; + } + bitmap_set_blk (&blk->bm, blknum, BLOCK_ALLOCATED); +@@ -527,7 +527,7 @@ blk_write (struct blk_overlay *blk, + + if (full_pwrite (fd, block, blksize, offset) == -1) { + *err = errno; +- nbdkit_error ("pwrite: %m"); ++ nbdkit_error ("cow: pwrite: %m"); + return -1; + } + +-- +2.47.3 + diff --git a/0019-cow-Fix-block-fd-calculation.patch b/0019-cow-Fix-block-fd-calculation.patch new file mode 100644 index 0000000..c11a073 --- /dev/null +++ b/0019-cow-Fix-block-fd-calculation.patch @@ -0,0 +1,30 @@ +From ed6aca2c0a58a8b96531f015cf4d629f5208348b Mon Sep 17 00:00:00 2001 +From: "Richard W.M. Jones" +Date: Fri, 3 Apr 2026 14:06:31 +0100 +Subject: [PATCH] cow: Fix block -> fd calculation + +This calculation was plainly wrong, so the other overlay files were +never used. + +Fixes: commit eb79a0e0c63ba1884bde02dc884a26aea2fc4324 +(cherry picked from commit 7180bbf0f3d7b29e7eccf30207980849f3586583) +--- + filters/cow/blk.c | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/filters/cow/blk.c b/filters/cow/blk.c +index afb48af9..3a416d55 100644 +--- a/filters/cow/blk.c ++++ b/filters/cow/blk.c +@@ -324,7 +324,7 @@ blk_set_size (struct blk_overlay *blk, uint64_t new_size) + static int + get_fd_for_blknum (struct blk_overlay *blk, uint64_t blknum) + { +- size_t i = blknum / MAX_FILE_SIZE / blksize; ++ size_t i = blknum * blksize / MAX_FILE_SIZE; + assert (i < blk->fds.len); + return blk->fds.ptr[i]; + } +-- +2.47.3 + diff --git a/0020-cow-Fix-offsets-when-overlay-is-split.patch b/0020-cow-Fix-offsets-when-overlay-is-split.patch new file mode 100644 index 0000000..a1050a2 --- /dev/null +++ b/0020-cow-Fix-offsets-when-overlay-is-split.patch @@ -0,0 +1,112 @@ +From 24d4313cddced9920a262e43e7dde70bb4aeae90 Mon Sep 17 00:00:00 2001 +From: "Richard W.M. Jones" +Date: Fri, 3 Apr 2026 13:50:37 +0100 +Subject: [PATCH] cow: Fix offsets when overlay is split + +Commit eb79a0e0c6 ("cow: Support overlays larger than 16T on ext4") +splits the overlay into 8T chunks. However we neglected to change the +offset we are writing to with the result that it still tried to write +beyond the end of files > 16T in size. Fix the offsets so they are +correct. + +This is not a data corruptor since this and the previous commit +together were causing only the first overlay to be used with full +offsets. However it completely broke the intent of commit eb79a0e0c6. + +The test that was added only tests around the 8T mark, so does not hit +this case even if TMPDIR is ext4. Adjust the test also. + +The error seen is: + nbdkit: file.0: error: pwrite: File too large + +Reported-by: Ming Xie +Fixes: commit eb79a0e0c63ba1884bde02dc884a26aea2fc4324 +(cherry picked from commit c495b7f46b60a8e549327b2444591fe9e3173da0) +--- + filters/cow/blk.c | 18 ++++++++++++------ + tests/test-cow-huge.sh | 4 ++-- + 2 files changed, 14 insertions(+), 8 deletions(-) + +diff --git a/filters/cow/blk.c b/filters/cow/blk.c +index 3a416d55..7a3f40c1 100644 +--- a/filters/cow/blk.c ++++ b/filters/cow/blk.c +@@ -114,7 +114,8 @@ enum bm_entry { + BLOCK_TRIMMED = 3, + }; + +-#define MAX_FILE_SIZE (UINT64_C (8) * 1024 * 1024 * 1024 * 1024) /* 8T */ ++/* Maximum overlay file size, 8T. This must be a power of 2. */ ++#define MAX_FILE_SIZE (UINT64_C (8) * 1024 * 1024 * 1024 * 1024) + + static const char * + state_to_string (enum bm_entry state) +@@ -416,7 +417,8 @@ blk_read_multiple (struct blk_overlay *blk, + "at offset %" PRIu64 " into the cache", + runblocks, offset); + +- if (full_pwrite (fd, block, blksize * runblocks, offset) == -1) { ++ if (full_pwrite (fd, block, blksize * runblocks, ++ offset & (MAX_FILE_SIZE - 1)) == -1) { + *err = errno; + nbdkit_error ("cow: pwrite: %m"); + return -1; +@@ -426,7 +428,8 @@ blk_read_multiple (struct blk_overlay *blk, + } + } + else if (state == BLOCK_ALLOCATED) { /* Read overlay. */ +- if (full_pread (fd, block, blksize * runblocks, offset) == -1) { ++ if (full_pread (fd, block, blksize * runblocks, ++ offset & (MAX_FILE_SIZE - 1)) == -1) { + *err = errno; + nbdkit_error ("cow: pread: %m"); + return -1; +@@ -479,7 +482,8 @@ blk_cache (struct blk_overlay *blk, + + if (state == BLOCK_ALLOCATED) { + #if HAVE_POSIX_FADVISE +- int r = posix_fadvise (fd, offset, blksize, POSIX_FADV_WILLNEED); ++ int r = posix_fadvise (fd, offset & (MAX_FILE_SIZE - 1), ++ blksize, POSIX_FADV_WILLNEED); + if (r) { + errno = r; + nbdkit_error ("cow: posix_fadvise: %m"); +@@ -504,7 +508,8 @@ blk_cache (struct blk_overlay *blk, + memset (block + n, 0, tail); + + if (mode == BLK_CACHE_COW) { +- if (full_pwrite (fd, block, blksize, offset) == -1) { ++ if (full_pwrite (fd, block, blksize, ++ offset & (MAX_FILE_SIZE - 1)) == -1) { + *err = errno; + nbdkit_error ("cow: pwrite: %m"); + return -1; +@@ -525,7 +530,8 @@ blk_write (struct blk_overlay *blk, + nbdkit_debug ("cow: blk_write block %" PRIu64 " (offset %" PRIu64 ")", + blknum, (uint64_t) offset); + +- if (full_pwrite (fd, block, blksize, offset) == -1) { ++ if (full_pwrite (fd, block, blksize, ++ offset & (MAX_FILE_SIZE - 1)) == -1) { + *err = errno; + nbdkit_error ("cow: pwrite: %m"); + return -1; +diff --git a/tests/test-cow-huge.sh b/tests/test-cow-huge.sh +index cad1f6cc..b459b1b8 100755 +--- a/tests/test-cow-huge.sh ++++ b/tests/test-cow-huge.sh +@@ -53,9 +53,9 @@ random.seed(None) + M = 1024*1024 + T = 1024*1024*1024*1024 + +-# Test a few megabytes either side of the 8T boundary. ++# Test a few megabytes either side of the 16T boundary. + overlay = bytearray(10 * M) +-overlay_offset = int(8 * T - len(overlay)/2) ++overlay_offset = int(16 * T - len(overlay)/2) + assert h.get_size() > overlay_offset + len(overlay) + + for iter in range(1, 200): +-- +2.47.3 + diff --git a/0021-cow-Make-some-offset-variables-const.patch b/0021-cow-Make-some-offset-variables-const.patch new file mode 100644 index 0000000..a31aa8b --- /dev/null +++ b/0021-cow-Make-some-offset-variables-const.patch @@ -0,0 +1,55 @@ +From aba2a885ab8dc8796fdc8f100cd35ad621bb307a Mon Sep 17 00:00:00 2001 +From: "Richard W.M. Jones" +Date: Fri, 3 Apr 2026 14:10:51 +0100 +Subject: [PATCH] cow: Make some offset variables const + +These variables are not modified so set them to const. + +(cherry picked from commit 6ed71d8709cd03dd9a3c53cba9dd03e5b884031d) +--- + filters/cow/blk.c | 8 ++++---- + 1 file changed, 4 insertions(+), 4 deletions(-) + +diff --git a/filters/cow/blk.c b/filters/cow/blk.c +index 7a3f40c1..4c1ee9d5 100644 +--- a/filters/cow/blk.c ++++ b/filters/cow/blk.c +@@ -353,7 +353,7 @@ blk_read_multiple (struct blk_overlay *blk, + uint64_t blknum, uint64_t nrblocks, + uint8_t *block, bool cow_on_read, int *err) + { +- off_t offset = blknum * blksize; ++ const off_t offset = blknum * blksize; + enum bm_entry state; + uint64_t b, runblocks; + const int fd = get_fd_for_blknum (blk, blknum); +@@ -466,7 +466,7 @@ blk_cache (struct blk_overlay *blk, + { + /* XXX Could make this lock more fine-grained with some thought. */ + ACQUIRE_LOCK_FOR_CURRENT_SCOPE (&blk->lock); +- off_t offset = blknum * blksize; ++ const off_t offset = blknum * blksize; + enum bm_entry state = bitmap_get_blk (&blk->bm, blknum, BLOCK_NOT_ALLOCATED); + unsigned n = blksize, tail = 0; + const int fd = get_fd_for_blknum (blk, blknum); +@@ -523,7 +523,7 @@ int + blk_write (struct blk_overlay *blk, + uint64_t blknum, const uint8_t *block, int *err) + { +- off_t offset = blknum * blksize; ++ const off_t offset = blknum * blksize; + const int fd = get_fd_for_blknum (blk, blknum); + + if (cow_debug_verbose) +@@ -547,7 +547,7 @@ int + blk_trim (struct blk_overlay *blk, + uint64_t blknum, int *err) + { +- off_t offset = blknum * blksize; ++ const off_t offset = blknum * blksize; + + if (cow_debug_verbose) + nbdkit_debug ("cow: blk_trim block %" PRIu64 " (offset %" PRIu64 ")", +-- +2.47.3 + diff --git a/copy-patches.sh b/copy-patches.sh old mode 100644 new mode 100755 diff --git a/nbdkit-find-provides b/nbdkit-find-provides old mode 100644 new mode 100755 diff --git a/nbdkit.spec b/nbdkit.spec index df87f67..dbb44d4 100644 --- a/nbdkit.spec +++ b/nbdkit.spec @@ -55,7 +55,7 @@ Name: nbdkit Version: 1.44.1 -Release: 4%{?dist} +Release: 6%{?dist} Summary: NBD server License: BSD-3-Clause @@ -91,6 +91,16 @@ Patch0008: 0008-count-Clarify-documentation.patch Patch0009: 0009-vddk-Don-t-use-FNM_PATHNAME-when-matching-export-par.patch Patch0010: 0010-file-Don-t-advertise-minimum_io_size-64K-the-max-sup.patch Patch0011: 0011-file-Change-calculations-of-block-size-hints-for-blo.patch +Patch0012: 0012-cache-cow-Prefix-ftruncate-error-with-filter-name.patch +Patch0013: 0013-cow-Don-t-leak-blk-lock-and-blk-bm-on-error-paths.patch +Patch0014: 0014-cow-Use-a-vector-to-store-the-overlay-file-descripto.patch +Patch0015: 0015-cow-Split-out-function-to-create-temporary-files.patch +Patch0016: 0016-cow-Add-name-of-the-temporary-file-to-debug-output.patch +Patch0017: 0017-cow-Support-overlays-larger-than-16T-on-ext4.patch +Patch0018: 0018-cache-cow-Add-prefix-before-more-calls.patch +Patch0019: 0019-cow-Fix-block-fd-calculation.patch +Patch0020: 0020-cow-Fix-offsets-when-overlay-is-split.patch +Patch0021: 0021-cow-Make-some-offset-variables-const.patch # For automatic RPM Provides generation. # See: https://rpm-software-management.github.io/rpm/manual/dependency_generators.html @@ -1561,6 +1571,10 @@ fi %changelog +* Fri Apr 03 2026 Richard W.M. Jones - 1.44.1-6 +- cow: Support overlays larger than 16T on ext4 + further fixes + resolves: RHEL-164552 + * Mon Jan 12 2026 Richard W.M. Jones - 1.44.1-4 - Fix v2v conversion failure when minimum_io_size > 64K resolves: RHEL-140707