From d48364b33bea7997c38461ba2d85015a8901f2f3 Mon Sep 17 00:00:00 2001 From: "Richard W.M. Jones" Date: Sun, 1 Feb 2026 19:01:54 +0000 Subject: [PATCH] sparse-random: Make block size configurable For trying to reproduce a virt-v2v corruption case, we would like to use this plugin to simulate quite closely a VDDK source. VDDK uses a 64K block size for extents, which is different from the default (4K) for this plugin. This change allows us to use 'blocksize=64K' to simulate this. (cherry picked from commit 5612598a49aaf4ac49f1b3e096dc4945ea7df640) --- .../nbdkit-sparse-random-plugin.pod | 11 ++- plugins/sparse-random/sparse-random.c | 99 +++++++++++-------- tests/Makefile.am | 2 + tests/test-sparse-random-blocksize.sh | 49 +++++++++ 4 files changed, 117 insertions(+), 44 deletions(-) create mode 100755 tests/test-sparse-random-blocksize.sh diff --git a/plugins/sparse-random/nbdkit-sparse-random-plugin.pod b/plugins/sparse-random/nbdkit-sparse-random-plugin.pod index 0d64633f..d0472e00 100644 --- a/plugins/sparse-random/nbdkit-sparse-random-plugin.pod +++ b/plugins/sparse-random/nbdkit-sparse-random-plugin.pod @@ -4,8 +4,8 @@ nbdkit-sparse-random-plugin - make sparse random disks =head1 SYNOPSIS - nbdkit sparse-random [size=]SIZE [seed=SEED] - [percent=N] [runlength=N] + nbdkit sparse-random [size=]SIZE [blocksize=N] + [seed=SEED] [percent=N] [runlength=N] [random-content=true] =head1 DESCRIPTION @@ -52,6 +52,13 @@ See also L. =over 4 +=item BN + +Set the block size. This is the granularity that this plugin operates +at. Sparse extents will be aligned to the block size. + +It must be a power of 2 and E 1024. The default is 4096. + =item BN Specify the approximate percentage of the disk which contains random diff --git a/plugins/sparse-random/sparse-random.c b/plugins/sparse-random/sparse-random.c index 823f85a1..8d1c2944 100644 --- a/plugins/sparse-random/sparse-random.c +++ b/plugins/sparse-random/sparse-random.c @@ -36,6 +36,7 @@ #include #include #include +#include #include #include #include @@ -46,6 +47,7 @@ #include "bitmap.h" #include "cleanup.h" #include "isaligned.h" +#include "ispowerof2.h" #include "iszero.h" #include "minmax.h" #include "random.h" @@ -60,7 +62,7 @@ static int random_content; /* false: Repeat same byte true: Random bytes*/ /* We need to store 1 bit per block. Using a 4K block size means we * need 32M to map each 1T of virtual disk. */ -#define BLOCKSIZE 4096 +static unsigned blocksize = 4096; static struct bitmap bm; /* Bitmap of data blocks. */ @@ -72,8 +74,6 @@ sparse_random_load (void) * parameter. */ seed = time (NULL); - - bitmap_init (&bm, BLOCKSIZE, 1 /* bits per block */); } static void @@ -97,6 +97,18 @@ sparse_random_config (const char *key, const char *value) return -1; size = r; } + else if (strcmp (key, "blocksize") == 0 || + strcmp (key, "block-size") == 0) { + r = nbdkit_parse_size (value); + if (r == -1) + return -1; + if (r < 1024 || r > 0x10000000 || !is_power_of_2 (r)) { + nbdkit_error ("block size must be a power of 2, " + "and between 1024 and 2^28"); + return -1; + } + blocksize = r; + } else if (strcmp (key, "percent") == 0) { if (sscanf (value, "%lf", &percent) != 1 || percent < 0 || percent > 100) { @@ -129,6 +141,7 @@ sparse_random_config (const char *key, const char *value) #define sparse_random_config_help \ "size= (required) Size of the backing disk\n" \ + "blocksize= Set block size (default: 4K)\n" \ "seed= Random number generator seed\n" \ "percent= Percentage of data\n" \ "runlength= Expected average run length of data\n" \ @@ -182,6 +195,8 @@ sparse_random_get_ready (void) uint64_t data_run_length = 0; uint64_t avg_data_run_length = 0; + bitmap_init (&bm, blocksize, 1 /* bits per block */); + if (bitmap_resize (&bm, size) == -1) return -1; @@ -196,7 +211,7 @@ sparse_random_get_ready (void) } /* Otherwise calculate the probability parameters as above. */ - P_dh = 1. / ((double) runlength / BLOCKSIZE); + P_dh = 1. / ((double) runlength / blocksize); P_hd = (percent / 100.) * P_dh / (1. - (percent / 100.)); nbdkit_debug ("percent requested = %g%%, " @@ -242,8 +257,8 @@ sparse_random_get_ready (void) avg_data_run_length = 0; nbdkit_debug ("percent actual = %g%%, " "average run length = %" PRIu64, - 100. * BLOCKSIZE * nr_data_blocks / size, - avg_data_run_length * BLOCKSIZE); + 100. * blocksize * nr_data_blocks / size, + avg_data_run_length * blocksize); return 0; } @@ -270,7 +285,7 @@ sparse_random_block_size (void *handle, uint32_t *maximum) { *minimum = 1; - *preferred = BLOCKSIZE; + *preferred = blocksize; *maximum = 0xffffffff; return 0; } @@ -301,20 +316,20 @@ read_block (uint64_t blknum, uint64_t offset, void *buf) struct random_state state; if (bitmap_get_blk (&bm, blknum, 0) == 0) /* hole */ - memset (buf, 0, BLOCKSIZE); + memset (buf, 0, blocksize); else if (!random_content) { /* data when random-content=false */ xsrandom (seed + offset, &state); s = xrandom (&state); s &= 255; if (s == 0) s = 1; - memset (buf, (int)s, BLOCKSIZE); + memset (buf, (int)s, blocksize); } else { /* data when random-content=true */ /* This produces repeatable data for the same offset. Note it * works because we are called on whole blocks only. */ xsrandom (seed + offset, &state); - for (i = 0; i < BLOCKSIZE; ++i) { + for (i = 0; i < blocksize; ++i) { s = xrandom (&state); s &= 255; b[i] = s; @@ -330,20 +345,20 @@ sparse_random_pread (void *handle, void *buf, uint32_t count, uint64_t offset, CLEANUP_FREE uint8_t *block = NULL; uint64_t blknum, blkoffs; - if (!IS_ALIGNED (count | offset, BLOCKSIZE)) { - block = malloc (BLOCKSIZE); + if (!IS_ALIGNED (count | offset, blocksize)) { + block = malloc (blocksize); if (block == NULL) { nbdkit_error ("malloc: %m"); return -1; } } - blknum = offset / BLOCKSIZE; /* block number */ - blkoffs = offset % BLOCKSIZE; /* offset within the block */ + blknum = offset / blocksize; /* block number */ + blkoffs = offset % blocksize; /* offset within the block */ /* Unaligned head */ if (blkoffs) { - uint64_t n = MIN (BLOCKSIZE - blkoffs, count); + uint64_t n = MIN (blocksize - blkoffs, count); read_block (blknum, offset, block); memcpy (buf, &block[blkoffs], n); @@ -355,12 +370,12 @@ sparse_random_pread (void *handle, void *buf, uint32_t count, uint64_t offset, } /* Aligned body */ - while (count >= BLOCKSIZE) { + while (count >= blocksize) { read_block (blknum, offset, buf); - buf += BLOCKSIZE; - count -= BLOCKSIZE; - offset += BLOCKSIZE; + buf += blocksize; + count -= blocksize; + offset += blocksize; blknum++; } @@ -386,18 +401,18 @@ sparse_random_pwrite (void *handle, const void *buf, CLEANUP_FREE uint8_t *block; uint64_t blknum, blkoffs; - block = malloc (BLOCKSIZE); + block = malloc (blocksize); if (block == NULL) { nbdkit_error ("malloc: %m"); return -1; } - blknum = offset / BLOCKSIZE; /* block number */ - blkoffs = offset % BLOCKSIZE; /* offset within the block */ + blknum = offset / blocksize; /* block number */ + blkoffs = offset % blocksize; /* offset within the block */ /* Unaligned head */ if (blkoffs) { - uint64_t n = MIN (BLOCKSIZE - blkoffs, count); + uint64_t n = MIN (blocksize - blkoffs, count); read_block (blknum, offset, block); if (memcmp (buf, &block[blkoffs], n) != 0) { @@ -414,23 +429,23 @@ sparse_random_pwrite (void *handle, const void *buf, } /* Aligned body */ - while (count >= BLOCKSIZE) { + while (count >= blocksize) { /* As an optimization, skip calling read_block if we know this is * a hole. Call is_zero instead which should be faster. */ if (bitmap_get_blk (&bm, blknum, 0) == 0) { - if (! is_zero (buf, BLOCKSIZE)) + if (! is_zero (buf, blocksize)) goto unexpected_data; } else { read_block (blknum, offset, block); - if (memcmp (buf, block, BLOCKSIZE) != 0) + if (memcmp (buf, block, blocksize) != 0) goto unexpected_data; } - buf += BLOCKSIZE; - count -= BLOCKSIZE; - offset += BLOCKSIZE; + buf += blocksize; + count -= blocksize; + offset += blocksize; blknum++; } @@ -465,12 +480,12 @@ sparse_random_trim_zero (void *handle, uint32_t count, uint64_t offset, { uint64_t blknum, blkoffs; - blknum = offset / BLOCKSIZE; /* block number */ - blkoffs = offset % BLOCKSIZE; /* offset within the block */ + blknum = offset / blocksize; /* block number */ + blkoffs = offset % blocksize; /* offset within the block */ /* Unaligned head */ if (blkoffs) { - uint64_t n = MIN (BLOCKSIZE - blkoffs, count); + uint64_t n = MIN (blocksize - blkoffs, count); if (bitmap_get_blk (&bm, blknum, 0) != 0) { unexpected_trim: @@ -485,12 +500,12 @@ sparse_random_trim_zero (void *handle, uint32_t count, uint64_t offset, } /* Aligned body */ - while (count >= BLOCKSIZE) { + while (count >= blocksize) { if (bitmap_get_blk (&bm, blknum, 0) != 0) goto unexpected_trim; - count -= BLOCKSIZE; - offset += BLOCKSIZE; + count -= blocksize; + offset += blocksize; blknum++; } @@ -510,12 +525,12 @@ sparse_random_extents (void *handle, uint32_t count, uint64_t offset, uint64_t blknum, blkoffs; uint32_t type; - blknum = offset / BLOCKSIZE; /* block number */ - blkoffs = offset % BLOCKSIZE; /* offset within the block */ + blknum = offset / blocksize; /* block number */ + blkoffs = offset % blocksize; /* offset within the block */ /* Unaligned head */ if (blkoffs) { - uint64_t n = MIN (BLOCKSIZE - blkoffs, count); + uint64_t n = MIN (blocksize - blkoffs, count); if (bitmap_get_blk (&bm, blknum, 0) == 0) type = NBDKIT_EXTENT_HOLE | NBDKIT_EXTENT_ZERO; @@ -530,16 +545,16 @@ sparse_random_extents (void *handle, uint32_t count, uint64_t offset, } /* Aligned body */ - while (count >= BLOCKSIZE) { + while (count >= blocksize) { if (bitmap_get_blk (&bm, blknum, 0) == 0) type = NBDKIT_EXTENT_HOLE | NBDKIT_EXTENT_ZERO; else type = 0; /* data */ - if (nbdkit_add_extent (extents, offset, BLOCKSIZE, type) == -1) + if (nbdkit_add_extent (extents, offset, blocksize, type) == -1) return -1; - count -= BLOCKSIZE; - offset += BLOCKSIZE; + count -= blocksize; + offset += blocksize; blknum++; } diff --git a/tests/Makefile.am b/tests/Makefile.am index e2bc640d..2ae0c3c2 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -1293,10 +1293,12 @@ EXTRA_DIST += \ # sparse-random plugin test. TESTS += \ + test-sparse-random-blocksize.sh \ test-sparse-random-copy.sh \ test-sparse-random-info.sh \ $(NULL) EXTRA_DIST += \ + test-sparse-random-blocksize.sh \ test-sparse-random-copy.sh \ test-sparse-random-info.sh \ $(NULL) diff --git a/tests/test-sparse-random-blocksize.sh b/tests/test-sparse-random-blocksize.sh new file mode 100755 index 00000000..c1230809 --- /dev/null +++ b/tests/test-sparse-random-blocksize.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env bash +# nbdkit +# Copyright Red Hat +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# * Neither the name of Red Hat nor the names of its contributors may be +# used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY RED HAT AND CONTRIBUTORS ''AS IS'' AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A +# PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL RED HAT OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF +# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT +# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +# SUCH DAMAGE. + +# Test the sparse-random plugin with non-standard blocksize. + +source ./functions.sh +set -e +set -x +set -u + +requires_run + +# nbdcopy >= 1.5.9 required for this test. +requires_nbdcopy +requires_libnbd_version 1.5.9 + +nbdkit -v \ + sparse-random \ + size=10G blocksize=64K \ + --run 'nbdcopy "$uri" "$uri"' -- 2.47.3