commit dd7a8c39f594c5976f32862fb1e41f5f4f8f979c Author: James Antill Date: Mon Aug 8 14:10:51 2022 -0400 Import rpm: 64a15868f7a8724e8cee61f214e98c8ea3fa59b4 diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..e138377 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +SOURCES/libguestfs.keyring +SOURCES/nbdkit-1.24.0.tar.gz diff --git a/0001-cache-cow-Fix-data-corruption-in-zero-and-trim-on-un.patch b/0001-cache-cow-Fix-data-corruption-in-zero-and-trim-on-un.patch new file mode 100644 index 0000000..151fc17 --- /dev/null +++ b/0001-cache-cow-Fix-data-corruption-in-zero-and-trim-on-un.patch @@ -0,0 +1,82 @@ +From 99788909d9ec36e3210cf85976fe5b18da690ddd Mon Sep 17 00:00:00 2001 +From: "Richard W.M. Jones" +Date: Wed, 4 Aug 2021 20:24:59 +0100 +Subject: [PATCH] cache, cow: Fix data corruption in zero and trim on unaligned + tail + +Commit eb6009b092 ("cache, cow: Reduce use of bounce-buffer") first +introduced in nbdkit 1.14 added an optimization of the +read-modify-write mechanism used for unaligned heads and tails when +zeroing in the cache layer. + +Unfortunately the part applied to the tail contained a mistake: It +zeroes the end of the buffer rather than the beginning. This causes +data corruption when you use the zero or trim function with an offset +and count which is not aligned to the block size. + +Although the bug has been around for years, a recent change made it +more likely to happen. Commit c1905b0a28 ("cache, cow: Use a 64K +block size by default") increased the default block size from 4K to +64K. Most filesystems use a 4K block size so operations like fstrim +will make 4K-aligned requests, and with a 4K block size also in the +cache or cow filter the unaligned case would never have been hit +before. + +We can demonstrate the bug simply by filling a buffer with data +(100000 bytes in the example), and then trimming that data, which +ought to zero it out. + +Before this commit there is data visible after the trim: + +$ nbdkit --filter=cow data "0x21 * 100000" --run 'nbdsh -u $uri -c "h.trim(100000, 0)" ; nbdcopy $uri - | hexdump -C' +00000000 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| +* +00018000 21 21 21 21 21 21 21 21 21 21 21 21 21 21 21 21 |!!!!!!!!!!!!!!!!| +* +000186a0 + +After this commit the trim completely clears the data: + +$ nbdkit --filter=cow data "0x21 * 100000" --run 'nbdsh -u $uri -c "h.trim(100000, 0)" ; nbdcopy $uri - | hexdump -C' +00000000 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| +* +000186a0 + +Thanks: Ming Xie for finding the bug +Fixes: commit eb6009b092ae642ed25f133d487dd40ef7bf70f8 +(cherry picked from commit a0ae7b2158598ce48ac31706319007f716d01c87) +(cherry picked from commit c0b15574647672cb5c48178333acdd07424692ef) +--- + filters/cache/cache.c | 2 +- + filters/cow/cow.c | 2 +- + 2 files changed, 2 insertions(+), 2 deletions(-) + +diff --git a/filters/cache/cache.c b/filters/cache/cache.c +index 91dcc43d..0616cc7b 100644 +--- a/filters/cache/cache.c ++++ b/filters/cache/cache.c +@@ -493,7 +493,7 @@ cache_zero (struct nbdkit_next_ops *next_ops, void *nxdata, + ACQUIRE_LOCK_FOR_CURRENT_SCOPE (&lock); + r = blk_read (next_ops, nxdata, blknum, block, err); + if (r != -1) { +- memset (&block[count], 0, blksize - count); ++ memset (block, 0, count); + r = blk_write (next_ops, nxdata, blknum, block, flags, err); + } + if (r == -1) +diff --git a/filters/cow/cow.c b/filters/cow/cow.c +index 51ca64a4..1cfcc4e7 100644 +--- a/filters/cow/cow.c ++++ b/filters/cow/cow.c +@@ -419,7 +419,7 @@ cow_zero (struct nbdkit_next_ops *next_ops, void *nxdata, + ACQUIRE_LOCK_FOR_CURRENT_SCOPE (&lock); + r = blk_read (next_ops, nxdata, blknum, block, err); + if (r != -1) { +- memset (&block[count], 0, BLKSIZE - count); ++ memset (block, 0, count); + r = blk_write (blknum, block, err); + } + if (r == -1) +-- +2.31.1 + diff --git a/0001-ssh-Allow-the-remote-file-to-be-created.patch b/0001-ssh-Allow-the-remote-file-to-be-created.patch new file mode 100644 index 0000000..f3e07e4 --- /dev/null +++ b/0001-ssh-Allow-the-remote-file-to-be-created.patch @@ -0,0 +1,293 @@ +From 6a2b0aac8be655524ea223e32cac0395fcc9f975 Mon Sep 17 00:00:00 2001 +From: "Richard W.M. Jones" +Date: Fri, 15 Apr 2022 12:08:37 +0100 +Subject: [PATCH] ssh: Allow the remote file to be created + +This adds new parameters, create=(true|false), create-size=SIZE and +create-mode=MODE to create and truncate the remote file. + +Reviewed-by: Laszlo Ersek +(cherry picked from commit 0793f30b1071753532362b2ebf9cb8156a88c3c3) +--- + plugins/ssh/nbdkit-ssh-plugin.pod | 34 ++++++++- + plugins/ssh/ssh.c | 112 +++++++++++++++++++++++++++--- + tests/test-ssh.sh | 13 +++- + 3 files changed, 146 insertions(+), 13 deletions(-) + +diff --git a/plugins/ssh/nbdkit-ssh-plugin.pod b/plugins/ssh/nbdkit-ssh-plugin.pod +index 3f401c15..2bc2c4a7 100644 +--- a/plugins/ssh/nbdkit-ssh-plugin.pod ++++ b/plugins/ssh/nbdkit-ssh-plugin.pod +@@ -5,8 +5,10 @@ nbdkit-ssh-plugin - access disk images over the SSH protocol + =head1 SYNOPSIS + + nbdkit ssh host=HOST [path=]PATH +- [compression=true] [config=CONFIG_FILE] [identity=FILENAME] +- [known-hosts=FILENAME] [password=PASSWORD|-|+FILENAME] ++ [compression=true] [config=CONFIG_FILE] ++ [create=true] [create-mode=MODE] [create-size=SIZE] ++ [identity=FILENAME] [known-hosts=FILENAME] ++ [password=PASSWORD|-|+FILENAME] + [port=PORT] [timeout=SECS] [user=USER] + [verify-remote-host=false] + +@@ -62,6 +64,34 @@ The C parameter is optional. If it is I specified at all + then F<~/.ssh/config> and F are both read. + Missing or unreadable files are ignored. + ++=item B ++ ++(nbdkit E 1.32) ++ ++If set, the remote file will be created. The remote file is created ++on the first NBD connection to nbdkit, not when nbdkit starts up. If ++the file already exists, it will be replaced and any existing content ++lost. ++ ++If using this option, you must use C. C can ++be used to control the permissions of the new file. ++ ++=item BMODE ++ ++(nbdkit E 1.32) ++ ++If using C specify the default permissions of the new ++remote file. You can use octal modes like C or ++C. The default is C<0600>, ie. only readable and ++writable by the remote user. ++ ++=item BSIZE ++ ++(nbdkit E 1.32) ++ ++If using C, specify the virtual size of the new disk. ++C can use modifiers like C<100M> etc. ++ + =item BHOST + + Specify the name or IP address of the remote host. +diff --git a/plugins/ssh/ssh.c b/plugins/ssh/ssh.c +index 39d77e44..5e314cd7 100644 +--- a/plugins/ssh/ssh.c ++++ b/plugins/ssh/ssh.c +@@ -44,6 +44,8 @@ + #include + #include + ++#include ++ + #include + #include + #include +@@ -51,6 +53,7 @@ + #include + + #include "array-size.h" ++#include "cleanup.h" + #include "const-string-vector.h" + #include "minmax.h" + +@@ -64,6 +67,9 @@ static const char *known_hosts = NULL; + static const_string_vector identities = empty_vector; + static uint32_t timeout = 0; + static bool compression = false; ++static bool create = false; ++static int64_t create_size = -1; ++static unsigned create_mode = S_IRUSR | S_IWUSR /* 0600 */; + + /* config can be: + * NULL => parse options from default file +@@ -167,6 +173,27 @@ ssh_config (const char *key, const char *value) + return -1; + compression = r; + } ++ else if (strcmp (key, "create") == 0) { ++ r = nbdkit_parse_bool (value); ++ if (r == -1) ++ return -1; ++ create = r; ++ } ++ else if (strcmp (key, "create-size") == 0) { ++ create_size = nbdkit_parse_size (value); ++ if (create_size == -1) ++ return -1; ++ } ++ else if (strcmp (key, "create-mode") == 0) { ++ r = nbdkit_parse_unsigned (key, value, &create_mode); ++ if (r == -1) ++ return -1; ++ /* OpenSSH checks this too. */ ++ if (create_mode > 0777) { ++ nbdkit_error ("create-mode must be <= 0777"); ++ return -1; ++ } ++ } + + else { + nbdkit_error ("unknown parameter '%s'", key); +@@ -186,6 +213,13 @@ ssh_config_complete (void) + return -1; + } + ++ /* If create=true, create-size must be supplied. */ ++ if (create && create_size == -1) { ++ nbdkit_error ("if using create=true, you must specify the size " ++ "of the new remote file using create-size=SIZE"); ++ return -1; ++ } ++ + return 0; + } + +@@ -200,7 +234,10 @@ ssh_config_complete (void) + "identity= Prepend private key (identity) file.\n" \ + "timeout=SECS Set SSH connection timeout.\n" \ + "verify-remote-host=false Ignore known_hosts.\n" \ +- "compression=true Enable compression." ++ "compression=true Enable compression.\n" \ ++ "create=true Create the remote file.\n" \ ++ "create-mode=MODE Set the permissions of the remote file.\n" \ ++ "create-size=SIZE Set the size of the remote file." + + /* Since we must simulate atomic pread and pwrite using seek + + * read/write, calls on each handle must be serialized. +@@ -329,6 +366,65 @@ authenticate (struct ssh_handle *h) + return -1; + } + ++/* This function opens or creates the remote file (depending on ++ * create=false|true). Parallel connections might call this function ++ * at the same time, and so we must hold a lock to ensure that the ++ * file is created at most once. ++ */ ++static pthread_mutex_t create_lock = PTHREAD_MUTEX_INITIALIZER; ++ ++static sftp_file ++open_or_create_path (ssh_session session, sftp_session sftp, int readonly) ++{ ++ ACQUIRE_LOCK_FOR_CURRENT_SCOPE (&create_lock); ++ int access_type; ++ int r; ++ sftp_file file; ++ ++ access_type = readonly ? O_RDONLY : O_RDWR; ++ if (create) access_type |= O_CREAT | O_TRUNC; ++ ++ file = sftp_open (sftp, path, access_type, S_IRWXU); ++ if (!file) { ++ nbdkit_error ("cannot %s file for %s: %s", ++ create ? "create" : "open", ++ readonly ? "reading" : "writing", ++ ssh_get_error (session)); ++ return NULL; ++ } ++ ++ if (create) { ++ /* There's no sftp_truncate call. However OpenSSH lets you call ++ * SSH_FXP_SETSTAT + SSH_FILEXFER_ATTR_SIZE which invokes ++ * truncate(2) on the server. Libssh doesn't provide a binding ++ * for SSH_FXP_FSETSTAT so we have to pass the session + path. ++ */ ++ struct sftp_attributes_struct attrs = { ++ .flags = SSH_FILEXFER_ATTR_SIZE | ++ SSH_FILEXFER_ATTR_PERMISSIONS, ++ .size = create_size, ++ .permissions = create_mode, ++ }; ++ ++ r = sftp_setstat (sftp, path, &attrs); ++ if (r != SSH_OK) { ++ nbdkit_error ("setstat failed: %s", ssh_get_error (session)); ++ ++ /* Best-effort attempt to delete the remote file on failure. */ ++ r = sftp_unlink (sftp, path); ++ if (r != SSH_OK) ++ nbdkit_debug ("unlink failed: %s", ssh_get_error (session)); ++ ++ return NULL; ++ } ++ } ++ ++ /* On the next connection, don't create or truncate the file. */ ++ create = false; ++ ++ return file; ++} ++ + /* Create the per-connection handle. */ + static void * + ssh_open (int readonly) +@@ -337,7 +433,6 @@ ssh_open (int readonly) + const int set = 1; + size_t i; + int r; +- int access_type; + + h = calloc (1, sizeof *h); + if (h == NULL) { +@@ -471,7 +566,7 @@ ssh_open (int readonly) + if (authenticate (h) == -1) + goto err; + +- /* Open the SFTP connection and file. */ ++ /* Open the SFTP connection. */ + h->sftp = sftp_new (h->session); + if (!h->sftp) { + nbdkit_error ("failed to allocate sftp session: %s", +@@ -484,14 +579,11 @@ ssh_open (int readonly) + ssh_get_error (h->session)); + goto err; + } +- access_type = readonly ? O_RDONLY : O_RDWR; +- h->file = sftp_open (h->sftp, path, access_type, S_IRWXU); +- if (!h->file) { +- nbdkit_error ("cannot open file for %s: %s", +- readonly ? "reading" : "writing", +- ssh_get_error (h->session)); ++ ++ /* Open or create the remote file. */ ++ h->file = open_or_create_path (h->session, h->sftp, readonly); ++ if (!h->file) + goto err; +- } + + nbdkit_debug ("opened libssh handle"); + +diff --git a/tests/test-ssh.sh b/tests/test-ssh.sh +index 6c0ce410..f04b4488 100755 +--- a/tests/test-ssh.sh ++++ b/tests/test-ssh.sh +@@ -36,6 +36,7 @@ set -x + + requires test -f disk + requires nbdcopy --version ++requires stat --version + + # Check that ssh to localhost will work without any passwords or phrases. + # +@@ -48,7 +49,7 @@ then + exit 77 + fi + +-files="ssh.img" ++files="ssh.img ssh2.img" + rm -f $files + cleanup_fn rm -f $files + +@@ -59,3 +60,13 @@ nbdkit -v -D ssh.log=2 -U - \ + + # The output should be identical. + cmp disk ssh.img ++ ++# Copy local file 'ssh.img' to newly created "remote" 'ssh2.img' ++size="$(stat -c %s disk)" ++nbdkit -v -D ssh.log=2 -U - \ ++ ssh host=localhost $PWD/ssh2.img \ ++ create=true create-size=$size \ ++ --run 'nbdcopy ssh.img "$uri"' ++ ++# The output should be identical. ++cmp disk ssh2.img +-- +2.31.1 + diff --git a/0002-readahead-Rewrite-this-filter-so-it-prefetches-using.patch b/0002-readahead-Rewrite-this-filter-so-it-prefetches-using.patch new file mode 100644 index 0000000..69d5efa --- /dev/null +++ b/0002-readahead-Rewrite-this-filter-so-it-prefetches-using.patch @@ -0,0 +1,794 @@ +From ac40ae11bc9983e11185749b23e793568cb366cc Mon Sep 17 00:00:00 2001 +From: "Richard W.M. Jones" +Date: Sat, 16 Apr 2022 18:39:13 +0100 +Subject: [PATCH] readahead: Rewrite this filter so it prefetches using .cache + +The previous readahead filter did not work well and we have stopped +using it in virt-v2v. However the concept is a good one if we can +make it work. This commit completely rethinks the filter. + +Now, in parallel with the ordinary pread command, we issue a prefetch +(ie. .cache) to the underlying plugin for the data immediately +following the pread. For example a simple sequence of operations: + + t=1 pread (offset=0, count=65536) + t=2 pread (offset=65536, count=65536) + t=3 pread (offset=131072, count=65536) + +would become: + + t=1 pread (offset=0, count=65536) <--\ issued + cache (offset=65536, count=65536) <--/ in parallel + t=2 pread (offset=65536, count=65536) + cache (offset=131072, count=65536) + t=3 pread (offset=131072, count=65536) + cache (offset=196608, count=65536) + +This requires that the underlying filter(s) and plugin chain can +actually do something with the .cache request. If this is not the +case then the filter does nothing (but it will print a warning). For +plugins which don't have native support for prefetching, it is +sufficient to insert nbdkit-cache-filter after this filter. +(nbdkit-cow-filter can also be used with cow-on-cache=true, but that +is more useful for advanced users who are already using the cow +filter). + +The implementation creates a background thread per connection to issue +the parallel .cache requests. This is safer than the alternative (one +background thread in total) since we don't have to deal with the +problem of cache requests being issued with the wrong export name or +stale next pointer. The background thread is controlled by a queue of +commands, with the only possible commands being "cache" or "quit". + +Because the background thread issues parallel requests on the same +connection, the underlying plugin must support the parallel thread +model, otherwise we would be violating the plugin's thread model. It +may be possible in future to open a new connection from the background +thread (but with the same exportname), which would lift this +restriction to at least serialize_requests. Because of the current +limitation, nbdkit-curl-plugin cannot use prefetch. + +(cherry picked from commit 2ff548d66ad3eae87868402ec5b3319edd12090f) +--- + TODO | 22 +- + filters/readahead/Makefile.am | 2 + + filters/readahead/bgthread.c | 76 ++++ + filters/readahead/nbdkit-readahead-filter.pod | 70 +++- + filters/readahead/readahead.c | 338 +++++++++--------- + filters/readahead/readahead.h | 60 ++++ + tests/test-readahead.sh | 2 +- + 7 files changed, 367 insertions(+), 203 deletions(-) + create mode 100644 filters/readahead/bgthread.c + create mode 100644 filters/readahead/readahead.h + +diff --git a/TODO b/TODO +index 5ae21db5..4d2a9796 100644 +--- a/TODO ++++ b/TODO +@@ -62,16 +62,6 @@ General ideas for improvements + continue to keep their non-standard handshake while utilizing nbdkit + to prototype new behaviors in serving the kernel. + +-* Background thread for filters. Some filters (readahead, cache and +- proposed scan filter - see below) could be more effective if they +- were able to defer work to a background thread. We finally have +- nbdkit_next_context_open and friends for allowing a background +- thread to have access into the plugin, but still need to worry about +- thread-safety (how much must the filter do vs. nbdkit, to avoid +- calling into the plugin too many times at once) and cleanup +- (spawning the thread during .after_fork is viable, but cleaning it +- up during .unload is too late). +- + * "nbdkit.so": nbdkit as a loadable shared library. The aim of nbdkit + is to make it reusable from other programs (see nbdkit-captive(1)). + If it was a loadable shared library it would be even more reusable. +@@ -228,6 +218,8 @@ Suggestions for filters + * nbdkit-cache-filter should handle ENOSPC errors automatically by + reclaiming blocks from the cache + ++* nbdkit-cache-filter could use a background thread for reclaiming. ++ + * zstd filter was requested as a way to do what we currently do with + xz but saving many hours on compression (at the cost of hundreds of + MBs of extra data) +@@ -240,6 +232,16 @@ Suggestions for filters + could inject a flush after pausing. However this requires that + filter background threads have access to the plugin (see above). + ++nbdkit-readahead-filter: ++ ++* The filter should open a new connection to the plugin per background ++ thread so it is able to work with plugins that use the ++ serialize_requests thread model (like curl). At the moment it makes ++ requests on the same connection, so it requires plugins to use the ++ parallel thread model. ++ ++* It should combine (or avoid) overlapping cache requests. ++ + nbdkit-rate-filter: + + * allow other kinds of traffic shaping such as VBR +diff --git a/filters/readahead/Makefile.am b/filters/readahead/Makefile.am +index ee5bb3fb..187993ae 100644 +--- a/filters/readahead/Makefile.am ++++ b/filters/readahead/Makefile.am +@@ -37,6 +37,8 @@ filter_LTLIBRARIES = nbdkit-readahead-filter.la + + nbdkit_readahead_filter_la_SOURCES = \ + readahead.c \ ++ readahead.h \ ++ bgthread.c \ + $(top_srcdir)/include/nbdkit-filter.h \ + $(NULL) + +diff --git a/filters/readahead/bgthread.c b/filters/readahead/bgthread.c +new file mode 100644 +index 00000000..5894bb5f +--- /dev/null ++++ b/filters/readahead/bgthread.c +@@ -0,0 +1,76 @@ ++/* nbdkit ++ * Copyright (C) 2019-2022 Red Hat Inc. ++ * ++ * Redistribution and use in source and binary forms, with or without ++ * modification, are permitted provided that the following conditions are ++ * met: ++ * ++ * * Redistributions of source code must retain the above copyright ++ * notice, this list of conditions and the following disclaimer. ++ * ++ * * Redistributions in binary form must reproduce the above copyright ++ * notice, this list of conditions and the following disclaimer in the ++ * documentation and/or other materials provided with the distribution. ++ * ++ * * Neither the name of Red Hat nor the names of its contributors may be ++ * used to endorse or promote products derived from this software without ++ * specific prior written permission. ++ * ++ * THIS SOFTWARE IS PROVIDED BY RED HAT AND CONTRIBUTORS ''AS IS'' AND ++ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, ++ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A ++ * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL RED HAT OR ++ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, ++ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT ++ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF ++ * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ++ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, ++ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT ++ * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF ++ * SUCH DAMAGE. ++ */ ++ ++#include ++ ++#include ++#include ++#include ++#include ++ ++#include ++ ++#include "readahead.h" ++ ++#include "cleanup.h" ++ ++void * ++readahead_thread (void *vp) ++{ ++ struct bgthread_ctrl *ctrl = vp; ++ ++ for (;;) { ++ struct command cmd; ++ ++ /* Wait until we are sent at least one command. */ ++ { ++ ACQUIRE_LOCK_FOR_CURRENT_SCOPE (&ctrl->lock); ++ while (ctrl->cmds.len == 0) ++ pthread_cond_wait (&ctrl->cond, &ctrl->lock); ++ cmd = ctrl->cmds.ptr[0]; ++ command_queue_remove (&ctrl->cmds, 0); ++ } ++ ++ switch (cmd.type) { ++ case CMD_QUIT: ++ /* Finish processing and exit the thread. */ ++ return NULL; ++ ++ case CMD_CACHE: ++ /* Issue .cache (readahead) to underlying plugin. We ignore any ++ * errors because there's no way to communicate that back to the ++ * client, and readahead is only advisory. ++ */ ++ cmd.next->cache (cmd.next, cmd.count, cmd.offset, 0, NULL); ++ } ++ } ++} +diff --git a/filters/readahead/nbdkit-readahead-filter.pod b/filters/readahead/nbdkit-readahead-filter.pod +index c220d379..630e5924 100644 +--- a/filters/readahead/nbdkit-readahead-filter.pod ++++ b/filters/readahead/nbdkit-readahead-filter.pod +@@ -1,28 +1,66 @@ + =head1 NAME + +-nbdkit-readahead-filter - prefetch data when reading sequentially ++nbdkit-readahead-filter - prefetch data ahead of sequential reads + + =head1 SYNOPSIS + +- nbdkit --filter=readahead plugin ++ nbdkit --filter=readahead PLUGIN ++ ++ nbdkit --filter=readahead --filter=cache PLUGIN ++ ++ nbdkit --filter=readahead --filter=cow PLUGIN cow-on-cache=true + + =head1 DESCRIPTION + + C is a filter that prefetches data when the +-client is reading sequentially. ++client is reading. + +-A common use for this filter is to accelerate sequential copy +-operations (like S>) when plugin requests have a +-high overhead (like L). For example: +- +- nbdkit -U - --filter=readahead curl https://example.com/disk.img \ +- --run 'qemu-img convert $nbd disk.img' ++When the client issues a read, this filter issues a parallel prefetch ++(C<.cache>) for subsequent data. Plugins which support this command ++will prefetch the data, making sequential reads faster. For plugins ++which do not support this command, you can inject ++L below (after) this filter, giving ++approximately the same effect. L can be used ++instead of nbdkit-cache-filter, if you add the C ++option. + + The filter uses a simple adaptive algorithm which accelerates +-sequential reads, but has a small penalty if the client does random +-reads. If the client mixes reads with writes or write-like operations +-(trimming, zeroing) then it will work but there can be a large +-performance penalty. ++sequential reads and requires no further configuration. ++ ++=head2 Limitations ++ ++In a number of significant cases this filter will do nothing. The ++filter will print a warning message if this happens. ++ ++=over 4 ++ ++=item Thread model must be parallel ++ ++For example L only supports ++C, and so this filter cannot perform prefetches in ++parallel with the read requests. ++ ++We may be able to lift this restriction in future. ++ ++=item Underlying filters or plugin must support C<.cache> (prefetch) ++ ++Very many plugins do not have the concept of prefetching and/or ++do not implement the C<.cache> callback, and so there is no ++way for this filter to issue prefetches. ++ ++You can usually get around this by adding I<--filter=cache> after this ++filter as explained above. It may be necessary to limit the total ++size of the cache (see L). ++ ++=item Clients and kernels may do readahead already ++ ++It may be the case that NBD clients are already issuing ++C (NBD prefetch) commands. It may also be the case ++that your plugin is using local file functions where the kernel is ++doing readahead. In such cases this filter is not necessary and may ++be pessimal. ++ ++=back + + =head1 PARAMETERS + +@@ -50,9 +88,9 @@ C first appeared in nbdkit 1.12. + + L, + L, +-L, ++L, ++L, + L, +-L, + L, + L, + L, +@@ -64,4 +102,4 @@ Richard W.M. Jones + + =head1 COPYRIGHT + +-Copyright (C) 2019 Red Hat Inc. ++Copyright (C) 2019-2022 Red Hat Inc. +diff --git a/filters/readahead/readahead.c b/filters/readahead/readahead.c +index f5552d4c..1d7ae111 100644 +--- a/filters/readahead/readahead.c ++++ b/filters/readahead/readahead.c +@@ -1,5 +1,5 @@ + /* nbdkit +- * Copyright (C) 2019-2021 Red Hat Inc. ++ * Copyright (C) 2019-2022 Red Hat Inc. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are +@@ -34,232 +34,218 @@ + + #include + #include ++#include + #include + #include + #include +- + #include + + #include + ++#include "readahead.h" ++ + #include "cleanup.h" + #include "minmax.h" +- +-/* Copied from server/plugins.c. */ +-#define MAX_REQUEST_SIZE (64 * 1024 * 1024) ++#include "vector.h" + + /* These could be made configurable in future. */ +-#define READAHEAD_MIN 65536 +-#define READAHEAD_MAX MAX_REQUEST_SIZE +- +-/* This lock protects the global state. */ +-static pthread_mutex_t lock = PTHREAD_MUTEX_INITIALIZER; +- +-/* The real size of the underlying plugin. */ +-static uint64_t size; ++#define READAHEAD_MIN 32768 ++#define READAHEAD_MAX (4*1024*1024) + + /* Size of the readahead window. */ ++static pthread_mutex_t window_lock = PTHREAD_MUTEX_INITIALIZER; + static uint64_t window = READAHEAD_MIN; ++static uint64_t last_offset = 0, last_readahead = 0; + +-/* The single prefetch buffer shared by all threads, and its virtual +- * location in the virtual disk. The prefetch buffer grows +- * dynamically as required, but never shrinks. ++static int thread_model = -1; /* Thread model of the underlying plugin. */ ++ ++/* Per-connection data. */ ++struct readahead_handle { ++ int can_cache; /* Can the underlying plugin cache? */ ++ pthread_t thread; /* The background thread, one per connection. */ ++ struct bgthread_ctrl ctrl; ++}; ++ ++/* We have various requirements of the underlying filter(s) + plugin: ++ * - They must support NBDKIT_CACHE_NATIVE (otherwise our requests ++ * would not do anything useful). ++ * - They must use the PARALLEL thread model (otherwise we could ++ * violate their thread model). ++ */ ++static bool ++filter_working (struct readahead_handle *h) ++{ ++ return ++ h->can_cache == NBDKIT_CACHE_NATIVE && ++ thread_model == NBDKIT_THREAD_MODEL_PARALLEL; ++} ++ ++static bool ++suggest_cache_filter (struct readahead_handle *h) ++{ ++ return ++ h->can_cache != NBDKIT_CACHE_NATIVE && ++ thread_model == NBDKIT_THREAD_MODEL_PARALLEL; ++} ++ ++/* We need to hook into .get_ready() so we can read the final thread ++ * model (of the whole server). + */ +-static char *buffer = NULL; +-static size_t bufsize = 0; +-static uint64_t position; +-static uint32_t length = 0; ++static int ++readahead_get_ready (int final_thread_model) ++{ ++ thread_model = final_thread_model; ++ return 0; ++} ++ ++static int ++send_command_to_background_thread (struct bgthread_ctrl *ctrl, ++ const struct command cmd) ++{ ++ ACQUIRE_LOCK_FOR_CURRENT_SCOPE (&ctrl->lock); ++ if (command_queue_append (&ctrl->cmds, cmd) == -1) ++ return -1; ++ /* Signal the thread if it could be sleeping on an empty queue. */ ++ if (ctrl->cmds.len == 1) ++ pthread_cond_signal (&ctrl->cond); ++ return 0; ++} ++ ++static void * ++readahead_open (nbdkit_next_open *next, nbdkit_context *nxdata, ++ int readonly, const char *exportname, int is_tls) ++{ ++ struct readahead_handle *h; ++ int err; ++ ++ if (next (nxdata, readonly, exportname) == -1) ++ return NULL; ++ ++ h = malloc (sizeof *h); ++ if (h == NULL) { ++ nbdkit_error ("malloc: %m"); ++ return NULL; ++ } ++ ++ h->ctrl.cmds = (command_queue) empty_vector; ++ pthread_mutex_init (&h->ctrl.lock, NULL); ++ pthread_cond_init (&h->ctrl.cond, NULL); ++ ++ /* Create the background thread. */ ++ err = pthread_create (&h->thread, NULL, readahead_thread, &h->ctrl); ++ if (err != 0) { ++ errno = err; ++ nbdkit_error ("pthread_create: %m"); ++ pthread_cond_destroy (&h->ctrl.cond); ++ pthread_mutex_destroy (&h->ctrl.lock); ++ free (h); ++ return NULL; ++ } ++ ++ return h; ++} + + static void +-readahead_unload (void) ++readahead_close (void *handle) + { +- free (buffer); ++ struct readahead_handle *h = handle; ++ const struct command quit_cmd = { .type = CMD_QUIT }; ++ ++ send_command_to_background_thread (&h->ctrl, quit_cmd); ++ pthread_join (h->thread, NULL); ++ pthread_cond_destroy (&h->ctrl.cond); ++ pthread_mutex_destroy (&h->ctrl.lock); ++ command_queue_reset (&h->ctrl.cmds); ++ free (h); + } + +-static int64_t readahead_get_size (nbdkit_next *next, void *handle); +- +-/* In prepare, force a call to get_size which sets the size global. */ + static int +-readahead_prepare (nbdkit_next *next, void *handle, int readonly) ++readahead_can_cache (nbdkit_next *next, void *handle) + { +- int64_t r; ++ struct readahead_handle *h = handle; ++ int r; + +- r = readahead_get_size (next, handle); +- return r >= 0 ? 0 : -1; +-} +- +-/* Get the size. */ +-static int64_t +-readahead_get_size (nbdkit_next *next, void *handle) +-{ +- int64_t r; +- +- r = next->get_size (next); ++ /* Call next->can_cache to read the underlying 'can_cache'. */ ++ r = next->can_cache (next); + if (r == -1) + return -1; ++ h->can_cache = r; + +- ACQUIRE_LOCK_FOR_CURRENT_SCOPE (&lock); +- size = r; ++ if (!filter_working (h)) { ++ nbdkit_error ("readahead: warning: underlying plugin does not support " ++ "NBD_CMD_CACHE or PARALLEL thread model, so the filter " ++ "won't do anything"); ++ if (suggest_cache_filter (h)) ++ nbdkit_error ("readahead: try adding --filter=cache " ++ "after this filter"); ++ /* This is an error, but that's just to ensure that the warning ++ * above is seen. We don't need to return -1 here. ++ */ ++ } + + return r; + } + +-/* Cache */ +-static int +-readahead_can_cache (nbdkit_next *next, void *handle) +-{ +- /* We are already operating as a cache regardless of the plugin's +- * underlying .can_cache, but it's easiest to just rely on nbdkit's +- * behavior of calling .pread for caching. +- */ +- return NBDKIT_CACHE_EMULATE; +-} +- + /* Read data. */ +- +-static int +-fill_readahead (nbdkit_next *next, +- uint32_t count, uint64_t offset, uint32_t flags, int *err) +-{ +- position = offset; +- +- /* Read at least window bytes, but if count is larger read that. +- * Note that the count cannot be bigger than the buffer size. +- */ +- length = MAX (count, window); +- +- /* Don't go beyond the end of the underlying file. */ +- length = MIN (length, size - position); +- +- /* Grow the buffer if necessary. */ +- if (bufsize < length) { +- char *new_buffer = realloc (buffer, length); +- if (new_buffer == NULL) { +- *err = errno; +- nbdkit_error ("realloc: %m"); +- return -1; +- } +- buffer = new_buffer; +- bufsize = length; +- } +- +- if (next->pread (next, buffer, length, offset, flags, err) == -1) { +- length = 0; /* failed to fill the prefetch buffer */ +- return -1; +- } +- +- return 0; +-} +- + static int + readahead_pread (nbdkit_next *next, + void *handle, void *buf, uint32_t count, uint64_t offset, + uint32_t flags, int *err) + { +- ACQUIRE_LOCK_FOR_CURRENT_SCOPE (&lock); ++ struct readahead_handle *h = handle; + +- while (count > 0) { +- if (length == 0) { +- /* We don't have a prefetch buffer at all. This could be the +- * first request or reset after a miss. +- */ +- window = READAHEAD_MIN; +- if (fill_readahead (next, count, offset, flags, err) == -1) +- return -1; +- } ++ /* If the underlying plugin doesn't support caching then skip that ++ * step completely. The filter will do nothing. ++ */ ++ if (filter_working (h)) { ++ struct command ra_cmd = { .type = CMD_CACHE, .next = NULL }; ++ int64_t size; + +- /* Can we satisfy this request partly or entirely from the prefetch +- * buffer? +- */ +- else if (position <= offset && offset < position + length) { +- uint32_t n = MIN (position - offset + length, count); +- memcpy (buf, &buffer[offset-position], n); +- buf += n; +- offset += n; +- count -= n; +- } ++ size = next->get_size (next); ++ if (size >= 0) { ++ ACQUIRE_LOCK_FOR_CURRENT_SCOPE (&window_lock); + +- /* Does the request start immediately after the prefetch buffer? +- * This is a “hit” allowing us to double the window size. +- */ +- else if (offset == position + length) { +- window = MIN (window * 2, READAHEAD_MAX); +- if (fill_readahead (next, count, offset, flags, err) == -1) +- return -1; ++ /* Generate the asynchronous (background) cache command for ++ * the readahead window. ++ */ ++ ra_cmd.offset = offset + count; ++ if (ra_cmd.offset < size) { ++ ra_cmd.count = MIN (window, size - ra_cmd.offset); ++ ra_cmd.next = next; /* If .next is non-NULL, we'll send it below. */ ++ } ++ ++ /* Should we change the window size? ++ * If the last readahead < current offset, double the window. ++ * If not, but we're still making forward progress, keep the window. ++ * If we're not making forward progress, reduce the window to minimum. ++ */ ++ if (last_readahead < offset) ++ window = MIN (window * 2, READAHEAD_MAX); ++ else if (last_offset < offset) ++ /* leave window unchanged */ ; ++ else ++ window = READAHEAD_MIN; ++ last_offset = offset; ++ last_readahead = ra_cmd.offset + ra_cmd.count; + } + +- /* Else it's a “miss”. Reset everything and start again. */ +- else +- length = 0; ++ if (ra_cmd.next && ++ send_command_to_background_thread (&h->ctrl, ra_cmd) == -1) ++ return -1; + } + +- return 0; +-} +- +-/* Any writes or write-like operations kill the prefetch buffer. +- * +- * We could do better here, but for the current use case of this +- * filter it doesn't matter. XXX +- */ +- +-static void +-kill_readahead (void) +-{ +- ACQUIRE_LOCK_FOR_CURRENT_SCOPE (&lock); +- window = READAHEAD_MIN; +- length = 0; +-} +- +-static int +-readahead_pwrite (nbdkit_next *next, +- void *handle, +- const void *buf, uint32_t count, uint64_t offset, +- uint32_t flags, int *err) +-{ +- kill_readahead (); +- return next->pwrite (next, buf, count, offset, flags, err); +-} +- +-static int +-readahead_trim (nbdkit_next *next, +- void *handle, +- uint32_t count, uint64_t offset, uint32_t flags, +- int *err) +-{ +- kill_readahead (); +- return next->trim (next, count, offset, flags, err); +-} +- +-static int +-readahead_zero (nbdkit_next *next, +- void *handle, +- uint32_t count, uint64_t offset, uint32_t flags, +- int *err) +-{ +- kill_readahead (); +- return next->zero (next, count, offset, flags, err); +-} +- +-static int +-readahead_flush (nbdkit_next *next, +- void *handle, uint32_t flags, int *err) +-{ +- kill_readahead (); +- return next->flush (next, flags, err); ++ /* Issue the synchronous read. */ ++ return next->pread (next, buf, count, offset, flags, err); + } + + static struct nbdkit_filter filter = { + .name = "readahead", + .longname = "nbdkit readahead filter", +- .unload = readahead_unload, +- .prepare = readahead_prepare, +- .get_size = readahead_get_size, ++ .get_ready = readahead_get_ready, ++ .open = readahead_open, ++ .close = readahead_close, + .can_cache = readahead_can_cache, + .pread = readahead_pread, +- .pwrite = readahead_pwrite, +- .trim = readahead_trim, +- .zero = readahead_zero, +- .flush = readahead_flush, + }; + + NBDKIT_REGISTER_FILTER(filter) +diff --git a/filters/readahead/readahead.h b/filters/readahead/readahead.h +new file mode 100644 +index 00000000..a68204d5 +--- /dev/null ++++ b/filters/readahead/readahead.h +@@ -0,0 +1,60 @@ ++/* nbdkit ++ * Copyright (C) 2019-2022 Red Hat Inc. ++ * ++ * Redistribution and use in source and binary forms, with or without ++ * modification, are permitted provided that the following conditions are ++ * met: ++ * ++ * * Redistributions of source code must retain the above copyright ++ * notice, this list of conditions and the following disclaimer. ++ * ++ * * Redistributions in binary form must reproduce the above copyright ++ * notice, this list of conditions and the following disclaimer in the ++ * documentation and/or other materials provided with the distribution. ++ * ++ * * Neither the name of Red Hat nor the names of its contributors may be ++ * used to endorse or promote products derived from this software without ++ * specific prior written permission. ++ * ++ * THIS SOFTWARE IS PROVIDED BY RED HAT AND CONTRIBUTORS ''AS IS'' AND ++ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, ++ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A ++ * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL RED HAT OR ++ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, ++ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT ++ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF ++ * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ++ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, ++ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT ++ * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF ++ * SUCH DAMAGE. ++ */ ++ ++#ifndef NBDKIT_READAHEAD_H ++#define NBDKIT_READAHEAD_H ++ ++#include ++ ++#include ++ ++#include "vector.h" ++ ++/* List of commands issued to the background thread. */ ++struct command { ++ enum { CMD_QUIT, CMD_CACHE } type; ++ nbdkit_next *next; ++ uint64_t offset; ++ uint32_t count; ++}; ++DEFINE_VECTOR_TYPE(command_queue, struct command); ++ ++struct bgthread_ctrl { ++ command_queue cmds; /* Command queue. */ ++ pthread_mutex_t lock; /* Lock for queue. */ ++ pthread_cond_t cond; /* Condition queue size 0 -> 1. */ ++}; ++ ++/* Start background thread (one per connection). */ ++extern void *readahead_thread (void *vp); ++ ++#endif /* NBDKIT_READAHEAD_H */ +diff --git a/tests/test-readahead.sh b/tests/test-readahead.sh +index 7ec7f8e9..17126e5a 100755 +--- a/tests/test-readahead.sh ++++ b/tests/test-readahead.sh +@@ -59,7 +59,7 @@ for i in range(0, 512*10, 512): + echo $((end_t - start_t)) + } + +-t1=$(test --filter=readahead) ++t1=$(test --filter=readahead --filter=cache) + t2=$(test) + + # In the t1 case we should make only 1 request into the plugin, +-- +2.31.1 + diff --git a/0002-server-CVE-2021-3716-reset-structured-replies-on-sta.patch b/0002-server-CVE-2021-3716-reset-structured-replies-on-sta.patch new file mode 100644 index 0000000..ce6938e --- /dev/null +++ b/0002-server-CVE-2021-3716-reset-structured-replies-on-sta.patch @@ -0,0 +1,94 @@ +From 6b9d4380df9bd0be91f49aad8c4f47b4e672adde Mon Sep 17 00:00:00 2001 +From: Eric Blake +Date: Mon, 16 Aug 2021 13:43:29 -0500 +Subject: [PATCH] server: CVE-2021-3716 reset structured replies on starttls + +https://nostarttls.secvuln.info/ pointed out a series of CVEs in +common implementation flaw in various SMTP and IMAP clients and +servers, all with a common thread of improperly caching plaintext +state across the STARTTLS encryption boundary; and recommended that +other protocols with a STARTTLS operation perform a similar audit. + +It turns out that nbdkit has the same vulnerability in regards to the +NBD protocol: when nbdkit is run in opportunistic TLS mode, an +attacker is able to inject a plaintext NBD_OPT_STRUCTURED_REPLY before +proxying everything else a client sends to the server; if the server +then acts on that plaintext request (as nbdkit did before this patch), +then the server ends up sending structured replies to at least +NBD_CMD_READ, even though the client was assuming that the transition +to TLS has ruled out a MitM attack. + +On the bright side, nbdkit's behavior on a second +NBD_OPT_STRUCTURED_REPLY was to still reply with success, so a client +that always requests structured replies after starting TLS sees no +difference in behavior (that is, qemu 2.12 and later are immune) (had +nbdkit given an error to the second request, that may have caused +confusion to more clients). And there is always the mitigation of +using --tls=require, which lets nbdkit reject the MitM message +pre-encryption. However, nbd-client 3.15 to the present do not +understand structured replies, and I have confirmed that a MitM +attacker can thus cause a denial-of-service attack that does not +trigger until the client does its first encrypted NBD_CMD_READ. + +The NBD spec has been recently tightened to declare the nbdkit +behavior to be a security hole: +https://github.com/NetworkBlockDevice/nbd/commit/77e55378096aa +Fixes: eaa4c6e9a2c4bd (server: Minimal implementation of NBD Structured Replies.) + +(cherry picked from commit 09a13dafb7bb3a38ab52eb5501cba786365ba7fd) +(cherry picked from commit 6185b15a81e6915734d678f0781e31d45a7941a1) +--- + docs/nbdkit-security.pod | 11 +++++++++-- + server/protocol-handshake-newstyle.c | 3 ++- + 2 files changed, 11 insertions(+), 3 deletions(-) + +diff --git a/docs/nbdkit-security.pod b/docs/nbdkit-security.pod +index 3a28e54d..5a4e6da8 100644 +--- a/docs/nbdkit-security.pod ++++ b/docs/nbdkit-security.pod +@@ -10,7 +10,7 @@ For how to report new security issues, see the C file in the + top level source directory, also available online here: + L + +-=head2 CVE-2019-14850 ++=head2 CVE-2019-14850 + denial of service due to premature opening of back-end connection + + See the full announcement and links to mitigation, tests and fixes +@@ -26,6 +26,13 @@ See the full announcement and links to mitigation, tests and fixes + here: + https://www.redhat.com/archives/libguestfs/2019-September/msg00272.html + ++=head2 CVE-2021-3716 ++structured read denial of service attack against starttls ++ ++See the full announcement and links to mitigation, tests and fixes ++here: ++https://www.redhat.com/archives/libguestfs/2021-August/msg00083.html ++ + =head1 SEE ALSO + + L. +@@ -38,4 +45,4 @@ Richard W.M. Jones + + =head1 COPYRIGHT + +-Copyright (C) 2013-2020 Red Hat Inc. ++Copyright (C) 2013-2021 Red Hat Inc. +diff --git a/server/protocol-handshake-newstyle.c b/server/protocol-handshake-newstyle.c +index 0a76a814..b94950e2 100644 +--- a/server/protocol-handshake-newstyle.c ++++ b/server/protocol-handshake-newstyle.c +@@ -495,7 +495,8 @@ negotiate_handshake_newstyle_options (void) + return -1; + conn->using_tls = true; + debug ("using TLS on this connection"); +- /* Wipe out any cached default export name. */ ++ /* Wipe out any cached state. */ ++ conn->structured_replies = false; + for_each_backend (b) { + struct handle *h = get_handle (conn, b->i); + free (h->default_exportname); +-- +2.31.1 + diff --git a/0003-readahead-Fix-test.patch b/0003-readahead-Fix-test.patch new file mode 100644 index 0000000..52962b7 --- /dev/null +++ b/0003-readahead-Fix-test.patch @@ -0,0 +1,120 @@ +From b41b7d7ddf6d3fba23ac7978c8b272f2ff84265d Mon Sep 17 00:00:00 2001 +From: "Richard W.M. Jones" +Date: Thu, 21 Apr 2022 16:14:46 +0100 +Subject: [PATCH] readahead: Fix test + +The previous test turned out to be pretty bad at testing the new +filter. A specific problem is that the filter starts a background +thread which issues .cache requests, while on the main connection +.pread requests are being passed through. The test used +--filter=readahead --filter=cache with the cache filter only caching +on .cache requests (since cache-on-read defaults to false), so only +caching requests made by the background thread. + + main thread + client ---- .pread ----- delay-filter -------> plugin + \ + \ background thread + .cache --- cache-filter + +Under very high load, the background thread could be starved. This +means no requests were being cached at all, and all requests were +passing through the delay filter. It would appear that readahead was +failing (which it was, in a way). + +It's not very easy to fix this since readahead is best-effort, but we +can go back to using a simpler plugin that logs reads and caches and +check that they look valid. + +Update: commit 2ff548d66ad3eae87868402ec5b3319edd12090f +(cherry picked from commit db1e3311727c6ecab3264a1811d33db1aa45a4d0) +--- + tests/test-readahead.sh | 61 +++++++++++++++++++++++------------------ + 1 file changed, 35 insertions(+), 26 deletions(-) + +diff --git a/tests/test-readahead.sh b/tests/test-readahead.sh +index 17126e5a..37f4a06f 100755 +--- a/tests/test-readahead.sh ++++ b/tests/test-readahead.sh +@@ -30,43 +30,52 @@ + # OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + # SUCH DAMAGE. + +-# Is the readahead filter faster? Copy a blank disk with a custom +-# plugin that sleeps on every request. Because the readahead filter +-# should result in fewer requests it should run faster. +- + source ./functions.sh + set -e + set -x + +-requires_filter delay ++requires_plugin sh + requires nbdsh --version + requires dd iflag=count_bytes > readahead.out ++ ;; ++ pread) ++ echo "$@" >> readahead.out ++ dd if=/dev/zero count=$3 iflag=count_bytes ++ ;; ++ *) ++ exit 2 ++ ;; ++esac ++EOF + +- end_t=$SECONDS +- echo $((end_t - start_t)) +-} ++cat readahead.out + +-t1=$(test --filter=readahead --filter=cache) +-t2=$(test) +- +-# In the t1 case we should make only 1 request into the plugin, +-# resulting in around 1 sleep period (5 seconds). In the t2 case we +-# make 10 requests so sleep for around 50 seconds. t1 should be < t2 +-# is every reasonable scenario. +-if [ $t1 -ge $t2 ]; then +- echo "$0: readahead filter took longer, should be shorter" +- exit 1 +-fi ++# We should see the pread requests, and additional cache requests for ++# the 32K region following each pread request. ++for i in `seq 0 512 $((512*10 - 512))` ; do ++ grep "pread 512 $i" readahead.out ++ grep "cache 32768 $((i+512))" readahead.out ++done +-- +2.31.1 + diff --git a/0003-server-reset-meta-context-replies-on-starttls.patch b/0003-server-reset-meta-context-replies-on-starttls.patch new file mode 100644 index 0000000..4ab0de0 --- /dev/null +++ b/0003-server-reset-meta-context-replies-on-starttls.patch @@ -0,0 +1,40 @@ +From add9b794b9dc697a1b52115c997fcfb6e06bf64c Mon Sep 17 00:00:00 2001 +From: Eric Blake +Date: Mon, 16 Aug 2021 13:43:29 -0500 +Subject: [PATCH] server: reset meta context replies on starttls + +Related to CVE-2021-3716, but not as severe. No compliant client will +send NBD_CMD_BLOCK_STATUS unless it first negotiates +NBD_OPT_SET_META_CONTEXT. If an attacker injects a premature +SET_META_CONTEXT, either the client will never notice (because it +never uses BLOCK_STATUS), or the client will overwrite the attacker's +attempt with the client's own SET_META_CONTEXT request after +encryption is enabled. So I don't class this as having the potential +to trigger denial-of-service due to any protocol mismatch between +compliant client and server (I don't care what happens with +non-compliant clients). + +Fixes: 26455d45 (server: protocol: Implement Block Status "base:allocation".) +(cherry picked from commit 6c5faac6a37077cf2366388a80862bb00616d0d8) +(cherry picked from commit 814d8103fb4b581dc01dfd25d2cd81596576f211) +--- + server/protocol-handshake-newstyle.c | 3 +++ + 1 file changed, 3 insertions(+) + +diff --git a/server/protocol-handshake-newstyle.c b/server/protocol-handshake-newstyle.c +index b94950e2..eb0f3961 100644 +--- a/server/protocol-handshake-newstyle.c ++++ b/server/protocol-handshake-newstyle.c +@@ -497,6 +497,9 @@ negotiate_handshake_newstyle_options (void) + debug ("using TLS on this connection"); + /* Wipe out any cached state. */ + conn->structured_replies = false; ++ free (conn->exportname_from_set_meta_context); ++ conn->exportname_from_set_meta_context = NULL; ++ conn->meta_context_base_allocation = false; + for_each_backend (b) { + struct handle *h = get_handle (conn, b->i); + free (h->default_exportname); +-- +2.31.1 + diff --git a/0004-New-filter-luks.patch b/0004-New-filter-luks.patch new file mode 100644 index 0000000..5d14e25 --- /dev/null +++ b/0004-New-filter-luks.patch @@ -0,0 +1,1816 @@ +From c19936170cf8b385687cf40f5a9507d87ae08267 Mon Sep 17 00:00:00 2001 +From: "Richard W.M. Jones" +Date: Sat, 30 Apr 2022 12:35:07 +0100 +Subject: [PATCH] New filter: luks + +This filter allows you to open, read and write LUKSv1 disk images, +compatible with the ones used by dm-crypt and qemu. + +(cherry picked from commit 468919dce6c5eb57503eacac0f67e5dd87c58e6c) +--- + TODO | 11 +- + configure.ac | 6 +- + docs/nbdkit-tls.pod | 1 + + filters/luks/Makefile.am | 77 ++ + filters/luks/luks.c | 1263 +++++++++++++++++++++++++++ + filters/luks/nbdkit-luks-filter.pod | 120 +++ + plugins/file/nbdkit-file-plugin.pod | 1 + + tests/Makefile.am | 12 + + tests/test-luks-copy.sh | 125 +++ + tests/test-luks-info.sh | 56 ++ + 10 files changed, 1668 insertions(+), 4 deletions(-) + create mode 100644 filters/luks/Makefile.am + create mode 100644 filters/luks/luks.c + create mode 100644 filters/luks/nbdkit-luks-filter.pod + create mode 100755 tests/test-luks-copy.sh + create mode 100755 tests/test-luks-info.sh + +diff --git a/TODO b/TODO +index 4d2a9796..0f5dc41d 100644 +--- a/TODO ++++ b/TODO +@@ -195,9 +195,6 @@ Suggestions for filters + connections. This may even allow a filter to offer a more parallel + threading model than the underlying plugin. + +-* LUKS encrypt/decrypt filter, bonus points if compatible with qemu +- LUKS-encrypted disk images +- + * CBT filter to track dirty blocks. See these links for inspiration: + https://www.cloudandheat.com/block-level-data-tracking-using-davice-mappers-dm-era/ + https://github.com/qemu/qemu/blob/master/docs/interop/bitmaps.rst +@@ -232,6 +229,14 @@ Suggestions for filters + could inject a flush after pausing. However this requires that + filter background threads have access to the plugin (see above). + ++nbdkit-luks-filter: ++ ++* This filter should also support LUKSv2 (and so should qemu). ++ ++* There are some missing features: ESSIV, more ciphers. ++ ++* Implement trim and zero if possible. ++ + nbdkit-readahead-filter: + + * The filter should open a new connection to the plugin per background +diff --git a/configure.ac b/configure.ac +index a402921b..de85b4da 100644 +--- a/configure.ac ++++ b/configure.ac +@@ -127,6 +127,7 @@ filters="\ + ip \ + limit \ + log \ ++ luks \ + multi-conn \ + nocache \ + noextents \ +@@ -614,8 +615,9 @@ PKG_CHECK_MODULES([GNUTLS], [gnutls >= 3.3.0], [ + ], [ + AC_MSG_WARN([gnutls not found or < 3.3.0, TLS support will be disabled.]) + ]) ++AM_CONDITIONAL([HAVE_GNUTLS], [test "x$GNUTLS_LIBS" != "x"]) + +-AS_IF([test "$GNUTLS_LIBS" != ""],[ ++AS_IF([test "x$GNUTLS_LIBS" != "x"],[ + AC_MSG_CHECKING([for default TLS session priority string]) + AC_ARG_WITH([tls-priority], + [AS_HELP_STRING([--with-tls-priority=...], +@@ -1383,6 +1385,7 @@ AC_CONFIG_FILES([Makefile + filters/ip/Makefile + filters/limit/Makefile + filters/log/Makefile ++ filters/luks/Makefile + filters/multi-conn/Makefile + filters/nocache/Makefile + filters/noextents/Makefile +@@ -1481,6 +1484,7 @@ echo "Optional filters:" + echo + feature "ext2" test "x$HAVE_EXT2_TRUE" = "x" + feature "gzip" test "x$HAVE_ZLIB_TRUE" = "x" ++feature "LUKS" test "x$HAVE_GNUTLS_TRUE" != "x" + feature "xz" test "x$HAVE_LIBLZMA_TRUE" = "x" + + echo +diff --git a/docs/nbdkit-tls.pod b/docs/nbdkit-tls.pod +index 86f5f984..4d0dc14c 100644 +--- a/docs/nbdkit-tls.pod ++++ b/docs/nbdkit-tls.pod +@@ -364,6 +364,7 @@ More information can be found in L. + =head1 SEE ALSO + + L, ++L, + L, + L, + L, +diff --git a/filters/luks/Makefile.am b/filters/luks/Makefile.am +new file mode 100644 +index 00000000..30089621 +--- /dev/null ++++ b/filters/luks/Makefile.am +@@ -0,0 +1,77 @@ ++# nbdkit ++# Copyright (C) 2019-2022 Red Hat Inc. ++# ++# Redistribution and use in source and binary forms, with or without ++# modification, are permitted provided that the following conditions are ++# met: ++# ++# * Redistributions of source code must retain the above copyright ++# notice, this list of conditions and the following disclaimer. ++# ++# * Redistributions in binary form must reproduce the above copyright ++# notice, this list of conditions and the following disclaimer in the ++# documentation and/or other materials provided with the distribution. ++# ++# * Neither the name of Red Hat nor the names of its contributors may be ++# used to endorse or promote products derived from this software without ++# specific prior written permission. ++# ++# THIS SOFTWARE IS PROVIDED BY RED HAT AND CONTRIBUTORS ''AS IS'' AND ++# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, ++# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A ++# PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL RED HAT OR ++# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, ++# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT ++# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF ++# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ++# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, ++# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT ++# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF ++# SUCH DAMAGE. ++ ++include $(top_srcdir)/common-rules.mk ++ ++EXTRA_DIST = nbdkit-luks-filter.pod ++ ++if HAVE_GNUTLS ++ ++filter_LTLIBRARIES = nbdkit-luks-filter.la ++ ++nbdkit_luks_filter_la_SOURCES = \ ++ luks.c \ ++ $(top_srcdir)/include/nbdkit-filter.h \ ++ $(NULL) ++ ++nbdkit_luks_filter_la_CPPFLAGS = \ ++ -I$(top_srcdir)/include \ ++ -I$(top_srcdir)/common/include \ ++ -I$(top_srcdir)/common/utils \ ++ $(NULL) ++nbdkit_luks_filter_la_CFLAGS = \ ++ $(WARNINGS_CFLAGS) \ ++ $(GNUTLS_CFLAGS) \ ++ $(NULL) ++nbdkit_luks_filter_la_LIBADD = \ ++ $(top_builddir)/common/utils/libutils.la \ ++ $(IMPORT_LIBRARY_ON_WINDOWS) \ ++ $(GNUTLS_LIBS) \ ++ $(NULL) ++nbdkit_luks_filter_la_LDFLAGS = \ ++ -module -avoid-version -shared $(NO_UNDEFINED_ON_WINDOWS) \ ++ -Wl,--version-script=$(top_srcdir)/filters/filters.syms \ ++ $(NULL) ++ ++if HAVE_POD ++ ++man_MANS = nbdkit-luks-filter.1 ++CLEANFILES += $(man_MANS) ++ ++nbdkit-luks-filter.1: nbdkit-luks-filter.pod \ ++ $(top_builddir)/podwrapper.pl ++ $(PODWRAPPER) --section=1 --man $@ \ ++ --html $(top_builddir)/html/$@.html \ ++ $< ++ ++endif HAVE_POD ++ ++endif +diff --git a/filters/luks/luks.c b/filters/luks/luks.c +new file mode 100644 +index 00000000..706a9bd2 +--- /dev/null ++++ b/filters/luks/luks.c +@@ -0,0 +1,1263 @@ ++/* nbdkit ++ * Copyright (C) 2018-2022 Red Hat Inc. ++ * ++ * Redistribution and use in source and binary forms, with or without ++ * modification, are permitted provided that the following conditions are ++ * met: ++ * ++ * * Redistributions of source code must retain the above copyright ++ * notice, this list of conditions and the following disclaimer. ++ * ++ * * Redistributions in binary form must reproduce the above copyright ++ * notice, this list of conditions and the following disclaimer in the ++ * documentation and/or other materials provided with the distribution. ++ * ++ * * Neither the name of Red Hat nor the names of its contributors may be ++ * used to endorse or promote products derived from this software without ++ * specific prior written permission. ++ * ++ * THIS SOFTWARE IS PROVIDED BY RED HAT AND CONTRIBUTORS ''AS IS'' AND ++ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, ++ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A ++ * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL RED HAT OR ++ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, ++ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT ++ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF ++ * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ++ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, ++ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT ++ * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF ++ * SUCH DAMAGE. ++ */ ++ ++#include ++ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#include ++ ++#include ++ ++#include "byte-swapping.h" ++#include "cleanup.h" ++#include "isaligned.h" ++#include "minmax.h" ++#include "rounding.h" ++ ++/* LUKSv1 constants. */ ++#define LUKS_MAGIC { 'L', 'U', 'K', 'S', 0xBA, 0xBE } ++#define LUKS_MAGIC_LEN 6 ++#define LUKS_DIGESTSIZE 20 ++#define LUKS_SALTSIZE 32 ++#define LUKS_NUMKEYS 8 ++#define LUKS_KEY_DISABLED 0x0000DEAD ++#define LUKS_KEY_ENABLED 0x00AC71F3 ++#define LUKS_STRIPES 4000 ++#define LUKS_ALIGN_KEYSLOTS 4096 ++#define LUKS_SECTOR_SIZE 512 ++ ++/* Key slot. */ ++struct luks_keyslot { ++ uint32_t active; /* LUKS_KEY_DISABLED|LUKS_KEY_ENABLED */ ++ uint32_t password_iterations; ++ char password_salt[LUKS_SALTSIZE]; ++ uint32_t key_material_offset; ++ uint32_t stripes; ++} __attribute__((__packed__)); ++ ++/* LUKS superblock. */ ++struct luks_phdr { ++ char magic[LUKS_MAGIC_LEN]; /* LUKS_MAGIC */ ++ uint16_t version; /* Only 1 is supported. */ ++ char cipher_name[32]; ++ char cipher_mode[32]; ++ char hash_spec[32]; ++ uint32_t payload_offset; ++ uint32_t master_key_len; ++ uint8_t master_key_digest[LUKS_DIGESTSIZE]; ++ uint8_t master_key_salt[LUKS_SALTSIZE]; ++ uint32_t master_key_digest_iterations; ++ uint8_t uuid[40]; ++ ++ struct luks_keyslot keyslot[LUKS_NUMKEYS]; /* Key slots. */ ++} __attribute__((__packed__)); ++ ++static char *passphrase = NULL; ++ ++static void ++luks_unload (void) ++{ ++ /* XXX We should really store the passphrase (and master key) ++ * in mlock-ed memory. ++ */ ++ if (passphrase) { ++ memset (passphrase, 0, strlen (passphrase)); ++ free (passphrase); ++ } ++} ++ ++static int ++luks_thread_model (void) ++{ ++ return NBDKIT_THREAD_MODEL_PARALLEL; ++} ++ ++static int ++luks_config (nbdkit_next_config *next, nbdkit_backend *nxdata, ++ const char *key, const char *value) ++{ ++ if (strcmp (key, "passphrase") == 0) { ++ if (nbdkit_read_password (value, &passphrase) == -1) ++ return -1; ++ return 0; ++ } ++ ++ return next (nxdata, key, value); ++} ++ ++static int ++luks_config_complete (nbdkit_next_config_complete *next, nbdkit_backend *nxdata) ++{ ++ if (passphrase == NULL) { ++ nbdkit_error ("LUKS \"passphrase\" parameter is missing"); ++ return -1; ++ } ++ return next (nxdata); ++} ++ ++#define luks_config_help \ ++ "passphrase= Secret passphrase." ++ ++enum cipher_mode { ++ CIPHER_MODE_ECB, CIPHER_MODE_CBC, CIPHER_MODE_XTS, CIPHER_MODE_CTR, ++}; ++ ++static enum cipher_mode ++lookup_cipher_mode (const char *str) ++{ ++ if (strcmp (str, "ecb") == 0) ++ return CIPHER_MODE_ECB; ++ if (strcmp (str, "cbc") == 0) ++ return CIPHER_MODE_CBC; ++ if (strcmp (str, "xts") == 0) ++ return CIPHER_MODE_XTS; ++ if (strcmp (str, "ctr") == 0) ++ return CIPHER_MODE_CTR; ++ nbdkit_error ("unknown cipher mode: %s " ++ "(expecting \"ecb\", \"cbc\", \"xts\" or \"ctr\")", str); ++ return -1; ++} ++ ++static const char * ++cipher_mode_to_string (enum cipher_mode v) ++{ ++ switch (v) { ++ case CIPHER_MODE_ECB: return "ecb"; ++ case CIPHER_MODE_CBC: return "cbc"; ++ case CIPHER_MODE_XTS: return "xts"; ++ case CIPHER_MODE_CTR: return "ctr"; ++ default: abort (); ++ } ++} ++ ++enum ivgen { ++ IVGEN_PLAIN, IVGEN_PLAIN64, /* IVGEN_ESSIV, */ ++}; ++ ++static enum ivgen ++lookup_ivgen (const char *str) ++{ ++ if (strcmp (str, "plain") == 0) ++ return IVGEN_PLAIN; ++ if (strcmp (str, "plain64") == 0) ++ return IVGEN_PLAIN64; ++/* ++ if (strcmp (str, "essiv") == 0) ++ return IVGEN_ESSIV; ++*/ ++ nbdkit_error ("unknown IV generation algorithm: %s " ++ "(expecting \"plain\", \"plain64\" etc)", str); ++ return -1; ++} ++ ++static const char * ++ivgen_to_string (enum ivgen v) ++{ ++ switch (v) { ++ case IVGEN_PLAIN: return "plain"; ++ case IVGEN_PLAIN64: return "plain64"; ++ /*case IVGEN_ESSIV: return "essiv";*/ ++ default: abort (); ++ } ++} ++ ++static void ++calculate_iv (enum ivgen v, uint8_t *iv, size_t ivlen, uint64_t sector) ++{ ++ size_t prefixlen; ++ uint32_t sector32; ++ ++ switch (v) { ++ case IVGEN_PLAIN: ++ prefixlen = 4; /* 32 bits */ ++ if (prefixlen > ivlen) ++ prefixlen = ivlen; ++ sector32 = (uint32_t) sector; /* truncate to only lower bits */ ++ sector32 = htole32 (sector32); ++ memcpy (iv, §or32, prefixlen); ++ memset (iv + prefixlen, 0, ivlen - prefixlen); ++ break; ++ ++ case IVGEN_PLAIN64: ++ prefixlen = 8; /* 64 bits */ ++ if (prefixlen > ivlen) ++ prefixlen = ivlen; ++ sector = htole64 (sector); ++ memcpy (iv, §or, prefixlen); ++ memset (iv + prefixlen, 0, ivlen - prefixlen); ++ break; ++ ++ /*case IVGEN_ESSIV:*/ ++ default: abort (); ++ } ++} ++ ++enum cipher_alg { ++ CIPHER_ALG_AES_128, CIPHER_ALG_AES_192, CIPHER_ALG_AES_256, ++}; ++ ++static enum cipher_alg ++lookup_cipher_alg (const char *str, enum cipher_mode mode, int key_bytes) ++{ ++ if (mode == CIPHER_MODE_XTS) ++ key_bytes /= 2; ++ ++ if (strcmp (str, "aes") == 0) { ++ if (key_bytes == 16) ++ return CIPHER_ALG_AES_128; ++ if (key_bytes == 24) ++ return CIPHER_ALG_AES_192; ++ if (key_bytes == 32) ++ return CIPHER_ALG_AES_256; ++ } ++ nbdkit_error ("unknown cipher algorithm: %s (expecting \"aes\", etc)", str); ++ return -1; ++} ++ ++static const char * ++cipher_alg_to_string (enum cipher_alg v) ++{ ++ switch (v) { ++ case CIPHER_ALG_AES_128: return "aes-128"; ++ case CIPHER_ALG_AES_192: return "aes-192"; ++ case CIPHER_ALG_AES_256: return "aes-256"; ++ default: abort (); ++ } ++} ++ ++#if 0 ++static int ++cipher_alg_key_bytes (enum cipher_alg v) ++{ ++ switch (v) { ++ case CIPHER_ALG_AES_128: return 16; ++ case CIPHER_ALG_AES_192: return 24; ++ case CIPHER_ALG_AES_256: return 32; ++ default: abort (); ++ } ++} ++#endif ++ ++static int ++cipher_alg_iv_len (enum cipher_alg v, enum cipher_mode mode) ++{ ++ if (CIPHER_MODE_ECB) ++ return 0; /* Don't need an IV in this mode. */ ++ ++ switch (v) { ++ case CIPHER_ALG_AES_128: ++ case CIPHER_ALG_AES_192: ++ case CIPHER_ALG_AES_256: ++ return 16; ++ default: abort (); ++ } ++} ++ ++static gnutls_digest_algorithm_t ++lookup_hash (const char *str) ++{ ++ if (strcmp (str, "md5") == 0) ++ return GNUTLS_DIG_MD5; ++ if (strcmp (str, "sha1") == 0) ++ return GNUTLS_DIG_SHA1; ++ if (strcmp (str, "sha224") == 0) ++ return GNUTLS_DIG_SHA224; ++ if (strcmp (str, "sha256") == 0) ++ return GNUTLS_DIG_SHA256; ++ if (strcmp (str, "sha384") == 0) ++ return GNUTLS_DIG_SHA384; ++ if (strcmp (str, "sha512") == 0) ++ return GNUTLS_DIG_SHA512; ++ if (strcmp (str, "ripemd160") == 0) ++ return GNUTLS_DIG_RMD160; ++ nbdkit_error ("unknown hash algorithm: %s " ++ "(expecting \"md5\", \"sha1\", \"sha224\", etc)", str); ++ return -1; ++} ++ ++static const char * ++hash_to_string (gnutls_digest_algorithm_t v) ++{ ++ switch (v) { ++ case GNUTLS_DIG_UNKNOWN: return "unknown"; ++ case GNUTLS_DIG_MD5: return "md5"; ++ case GNUTLS_DIG_SHA1: return "sha1"; ++ case GNUTLS_DIG_SHA224: return "sha224"; ++ case GNUTLS_DIG_SHA256: return "sha256"; ++ case GNUTLS_DIG_SHA384: return "sha384"; ++ case GNUTLS_DIG_SHA512: return "sha512"; ++ case GNUTLS_DIG_RMD160: return "ripemd160"; ++ default: abort (); ++ } ++} ++ ++#if 0 ++/* See qemu & dm-crypt implementations for an explanation of what's ++ * going on here. ++ */ ++static enum cipher_alg ++lookup_essiv_cipher (enum cipher_alg cipher_alg, ++ gnutls_digest_algorithm_t ivgen_hash_alg) ++{ ++ int digest_bytes = gnutls_hash_get_len (ivgen_hash_alg); ++ int key_bytes = cipher_alg_key_bytes (cipher_alg); ++ ++ if (digest_bytes == key_bytes) ++ return cipher_alg; ++ ++ switch (cipher_alg) { ++ case CIPHER_ALG_AES_128: ++ case CIPHER_ALG_AES_192: ++ case CIPHER_ALG_AES_256: ++ if (digest_bytes == 16) return CIPHER_ALG_AES_128; ++ if (digest_bytes == 24) return CIPHER_ALG_AES_192; ++ if (digest_bytes == 32) return CIPHER_ALG_AES_256; ++ nbdkit_error ("no %s cipher available with key size %d", ++ "AES", digest_bytes); ++ return -1; ++ default: ++ nbdkit_error ("ESSIV does not support cipher %s", ++ cipher_alg_to_string (cipher_alg)); ++ return -1; ++ } ++} ++#endif ++ ++/* Per-connection handle. */ ++struct handle { ++ /* LUKS header, if necessary byte-swapped into host order. */ ++ struct luks_phdr phdr; ++ ++ /* Decoded algorithm etc. */ ++ enum cipher_alg cipher_alg; ++ enum cipher_mode cipher_mode; ++ gnutls_digest_algorithm_t hash_alg; ++ enum ivgen ivgen_alg; ++ gnutls_digest_algorithm_t ivgen_hash_alg; ++ enum cipher_alg ivgen_cipher_alg; ++ ++ /* GnuTLS algorithm. */ ++ gnutls_cipher_algorithm_t gnutls_cipher; ++ ++ /* If we managed to decrypt one of the keyslots using the passphrase ++ * then this contains the master key, otherwise NULL. ++ */ ++ uint8_t *masterkey; ++}; ++ ++static void * ++luks_open (nbdkit_next_open *next, nbdkit_context *nxdata, ++ int readonly, const char *exportname, int is_tls) ++{ ++ struct handle *h; ++ ++ if (next (nxdata, readonly, exportname) == -1) ++ return NULL; ++ ++ h = calloc (1, sizeof *h); ++ if (h == NULL) { ++ nbdkit_error ("calloc: %m"); ++ return NULL; ++ } ++ ++ return h; ++} ++ ++static void ++luks_close (void *handle) ++{ ++ struct handle *h = handle; ++ ++ if (h->masterkey) { ++ memset (h->masterkey, 0, h->phdr.master_key_len); ++ free (h->masterkey); ++ } ++ free (h); ++} ++ ++/* Perform decryption of a block of data in memory. */ ++static int ++do_decrypt (struct handle *h, gnutls_cipher_hd_t cipher, ++ uint64_t offset, uint8_t *buf, size_t len) ++{ ++ const size_t ivlen = cipher_alg_iv_len (h->cipher_alg, h->cipher_mode); ++ uint64_t sector = offset / LUKS_SECTOR_SIZE; ++ CLEANUP_FREE uint8_t *iv = malloc (ivlen); ++ int r; ++ ++ assert (IS_ALIGNED (offset, LUKS_SECTOR_SIZE)); ++ assert (IS_ALIGNED (len, LUKS_SECTOR_SIZE)); ++ ++ while (len) { ++ calculate_iv (h->ivgen_alg, iv, ivlen, sector); ++ gnutls_cipher_set_iv (cipher, iv, ivlen); ++ r = gnutls_cipher_decrypt2 (cipher, ++ buf, LUKS_SECTOR_SIZE, /* ciphertext */ ++ buf, LUKS_SECTOR_SIZE /* plaintext */); ++ if (r != 0) { ++ nbdkit_error ("gnutls_cipher_decrypt2: %s", gnutls_strerror (r)); ++ return -1; ++ } ++ ++ buf += LUKS_SECTOR_SIZE; ++ offset += LUKS_SECTOR_SIZE; ++ len -= LUKS_SECTOR_SIZE; ++ sector++; ++ } ++ ++ return 0; ++} ++ ++/* Perform encryption of a block of data in memory. */ ++static int ++do_encrypt (struct handle *h, gnutls_cipher_hd_t cipher, ++ uint64_t offset, uint8_t *buf, size_t len) ++{ ++ const size_t ivlen = cipher_alg_iv_len (h->cipher_alg, h->cipher_mode); ++ uint64_t sector = offset / LUKS_SECTOR_SIZE; ++ CLEANUP_FREE uint8_t *iv = malloc (ivlen); ++ int r; ++ ++ assert (IS_ALIGNED (offset, LUKS_SECTOR_SIZE)); ++ assert (IS_ALIGNED (len, LUKS_SECTOR_SIZE)); ++ ++ while (len) { ++ calculate_iv (h->ivgen_alg, iv, ivlen, sector); ++ gnutls_cipher_set_iv (cipher, iv, ivlen); ++ r = gnutls_cipher_encrypt2 (cipher, ++ buf, LUKS_SECTOR_SIZE, /* plaintext */ ++ buf, LUKS_SECTOR_SIZE /* ciphertext */); ++ if (r != 0) { ++ nbdkit_error ("gnutls_cipher_decrypt2: %s", gnutls_strerror (r)); ++ return -1; ++ } ++ ++ buf += LUKS_SECTOR_SIZE; ++ offset += LUKS_SECTOR_SIZE; ++ len -= LUKS_SECTOR_SIZE; ++ sector++; ++ } ++ ++ return 0; ++} ++ ++/* Parse the header fields containing cipher algorithm, mode, etc. */ ++static int ++parse_cipher_strings (struct handle *h) ++{ ++ char cipher_name[33], cipher_mode[33], hash_spec[33]; ++ char *ivgen, *ivhash; ++ ++ /* Copy the header fields locally and ensure they are \0 terminated. */ ++ memcpy (cipher_name, h->phdr.cipher_name, 32); ++ cipher_name[32] = 0; ++ memcpy (cipher_mode, h->phdr.cipher_mode, 32); ++ cipher_mode[32] = 0; ++ memcpy (hash_spec, h->phdr.hash_spec, 32); ++ hash_spec[32] = 0; ++ ++ nbdkit_debug ("LUKS v%" PRIu16 " cipher: %s mode: %s hash: %s " ++ "master key: %" PRIu32 " bits", ++ h->phdr.version, cipher_name, cipher_mode, hash_spec, ++ h->phdr.master_key_len * 8); ++ ++ /* The cipher_mode header has the form: "ciphermode-ivgen[:ivhash]" ++ * QEmu writes: "xts-plain64" ++ */ ++ ivgen = strchr (cipher_mode, '-'); ++ if (!ivgen) { ++ nbdkit_error ("incorrect cipher_mode header, " ++ "expecting mode-ivgenerator but got \"%s\"", cipher_mode); ++ return -1; ++ } ++ *ivgen = '\0'; ++ ivgen++; ++ ++ ivhash = strchr (ivgen, ':'); ++ if (!ivhash) ++ h->ivgen_hash_alg = GNUTLS_DIG_UNKNOWN; ++ else { ++ *ivhash = '\0'; ++ ivhash++; ++ ++ h->ivgen_hash_alg = lookup_hash (ivhash); ++ if (h->ivgen_hash_alg == -1) ++ return -1; ++ } ++ ++ h->cipher_mode = lookup_cipher_mode (cipher_mode); ++ if (h->cipher_mode == -1) ++ return -1; ++ ++ h->cipher_alg = lookup_cipher_alg (cipher_name, h->cipher_mode, ++ h->phdr.master_key_len); ++ if (h->cipher_alg == -1) ++ return -1; ++ ++ h->hash_alg = lookup_hash (hash_spec); ++ if (h->hash_alg == -1) ++ return -1; ++ ++ h->ivgen_alg = lookup_ivgen (ivgen); ++ if (h->ivgen_alg == -1) ++ return -1; ++ ++#if 0 ++ if (h->ivgen_alg == IVGEN_ESSIV) { ++ if (!ivhash) { ++ nbdkit_error ("incorrect IV generator hash specification"); ++ return -1; ++ } ++ h->ivgen_cipher_alg = lookup_essiv_cipher (h->cipher_alg, ++ h->ivgen_hash_alg); ++ if (h->ivgen_cipher_alg == -1) ++ return -1; ++ } ++ else ++#endif ++ h->ivgen_cipher_alg = h->cipher_alg; ++ ++ nbdkit_debug ("LUKS parsed ciphers: %s %s %s %s %s %s", ++ cipher_alg_to_string (h->cipher_alg), ++ cipher_mode_to_string (h->cipher_mode), ++ hash_to_string (h->hash_alg), ++ ivgen_to_string (h->ivgen_alg), ++ hash_to_string (h->ivgen_hash_alg), ++ cipher_alg_to_string (h->ivgen_cipher_alg)); ++ ++ /* GnuTLS combines cipher and block mode into a single value. Not ++ * all possible combinations are available in GnuTLS. See: ++ * https://www.gnutls.org/manual/html_node/Supported-ciphersuites.html ++ */ ++ h->gnutls_cipher = GNUTLS_CIPHER_NULL; ++ switch (h->cipher_mode) { ++ case CIPHER_MODE_XTS: ++ switch (h->cipher_alg) { ++ case CIPHER_ALG_AES_128: ++ h->gnutls_cipher = GNUTLS_CIPHER_AES_128_XTS; ++ break; ++ case CIPHER_ALG_AES_256: ++ h->gnutls_cipher = GNUTLS_CIPHER_AES_256_XTS; ++ break; ++ default: break; ++ } ++ break; ++ case CIPHER_MODE_CBC: ++ switch (h->cipher_alg) { ++ case CIPHER_ALG_AES_128: ++ h->gnutls_cipher = GNUTLS_CIPHER_AES_128_CBC; ++ break; ++ case CIPHER_ALG_AES_192: ++ h->gnutls_cipher = GNUTLS_CIPHER_AES_192_CBC; ++ break; ++ case CIPHER_ALG_AES_256: ++ h->gnutls_cipher = GNUTLS_CIPHER_AES_256_CBC; ++ break; ++ default: break; ++ } ++ default: break; ++ } ++ if (h->gnutls_cipher == GNUTLS_CIPHER_NULL) { ++ nbdkit_error ("cipher algorithm %s in mode %s is not supported by GnuTLS", ++ cipher_alg_to_string (h->cipher_alg), ++ cipher_mode_to_string (h->cipher_mode)); ++ return -1; ++ } ++ ++ return 0; ++} ++ ++/* Anti-Forensic merge operation. */ ++static void ++xor (const uint8_t *in1, const uint8_t *in2, uint8_t *out, size_t len) ++{ ++ size_t i; ++ ++ for (i = 0; i < len; ++i) ++ out[i] = in1[i] ^ in2[i]; ++} ++ ++static int ++af_hash (gnutls_digest_algorithm_t hash_alg, uint8_t *block, size_t len) ++{ ++ size_t digest_bytes = gnutls_hash_get_len (hash_alg); ++ size_t nr_blocks, last_block_len; ++ size_t i; ++ CLEANUP_FREE uint8_t *temp = malloc (digest_bytes); ++ int r; ++ gnutls_hash_hd_t hash; ++ ++ nr_blocks = len / digest_bytes; ++ last_block_len = len % digest_bytes; ++ if (last_block_len != 0) ++ nr_blocks++; ++ else ++ last_block_len = digest_bytes; ++ ++ for (i = 0; i < nr_blocks; ++i) { ++ const uint32_t iv = htobe32 (i); ++ const size_t blen = i < nr_blocks - 1 ? digest_bytes : last_block_len; ++ ++ /* Hash iv + i'th block into temp. */ ++ r = gnutls_hash_init (&hash, hash_alg); ++ if (r != 0) { ++ nbdkit_error ("gnutls_hash_init: %s", gnutls_strerror (r)); ++ return -1; ++ } ++ gnutls_hash (hash, &iv, sizeof iv); ++ gnutls_hash (hash, &block[i*digest_bytes], blen); ++ gnutls_hash_deinit (hash, temp); ++ ++ memcpy (&block[i*digest_bytes], temp, blen); ++ } ++ ++ return 0; ++} ++ ++static int ++afmerge (gnutls_digest_algorithm_t hash_alg, uint32_t stripes, ++ const uint8_t *in, uint8_t *out, size_t outlen) ++{ ++ CLEANUP_FREE uint8_t *block = calloc (1, outlen); ++ size_t i; ++ ++ /* NB: input size is stripes * master_key_len where ++ * master_key_len == outlen ++ */ ++ for (i = 0; i < stripes-1; ++i) { ++ xor (&in[i*outlen], block, block, outlen); ++ if (af_hash (hash_alg, block, outlen) == -1) ++ return -1; ++ } ++ xor (&in[i*outlen], block, out, outlen); ++ return 0; ++} ++ ++/* Length of key material in key slot i (sectors). ++ * ++ * This is basically copied from qemu because the spec description is ++ * unintelligible and apparently doesn't match reality. ++ */ ++static uint64_t ++key_material_length_in_sectors (struct handle *h, size_t i) ++{ ++ uint64_t len, r; ++ ++ len = h->phdr.master_key_len * h->phdr.keyslot[i].stripes; ++ r = DIV_ROUND_UP (len, LUKS_SECTOR_SIZE); ++ r = ROUND_UP (r, LUKS_ALIGN_KEYSLOTS / LUKS_SECTOR_SIZE); ++ return r; ++} ++ ++/* Try the passphrase in key slot i. If this returns true then the ++ * passphrase was able to decrypt the master key, and the master key ++ * has been stored in h->masterkey. ++ */ ++static int ++try_passphrase_in_keyslot (nbdkit_next *next, struct handle *h, size_t i) ++{ ++ struct luks_keyslot *ks = &h->phdr.keyslot[i]; ++ size_t split_key_len; ++ CLEANUP_FREE uint8_t *split_key = NULL; ++ CLEANUP_FREE uint8_t *masterkey = NULL; ++ const gnutls_datum_t key = ++ { (unsigned char *) passphrase, strlen (passphrase) }; ++ const gnutls_datum_t salt = ++ { (unsigned char *) ks->password_salt, LUKS_SALTSIZE }; ++ const gnutls_datum_t msalt = ++ { (unsigned char *) h->phdr.master_key_salt, LUKS_SALTSIZE }; ++ gnutls_datum_t mkey; ++ gnutls_cipher_hd_t cipher; ++ int r, err = 0; ++ uint64_t start; ++ uint8_t key_digest[LUKS_DIGESTSIZE]; ++ ++ if (ks->active != LUKS_KEY_ENABLED) ++ return 0; ++ ++ split_key_len = h->phdr.master_key_len * ks->stripes; ++ split_key = malloc (split_key_len); ++ if (split_key == NULL) { ++ nbdkit_error ("malloc: %m"); ++ return -1; ++ } ++ masterkey = malloc (h->phdr.master_key_len); ++ if (masterkey == NULL) { ++ nbdkit_error ("malloc: %m"); ++ return -1; ++ } ++ ++ /* Hash the passphrase to make a possible masterkey. */ ++ r = gnutls_pbkdf2 (h->hash_alg, &key, &salt, ks->password_iterations, ++ masterkey, h->phdr.master_key_len); ++ if (r != 0) { ++ nbdkit_error ("gnutls_pbkdf2: %s", gnutls_strerror (r)); ++ return -1; ++ } ++ ++ /* Read master key material from plugin. */ ++ start = ks->key_material_offset * LUKS_SECTOR_SIZE; ++ if (next->pread (next, split_key, split_key_len, start, 0, &err) == -1) { ++ errno = err; ++ return -1; ++ } ++ ++ /* Decrypt the (still AFsplit) master key material. */ ++ mkey.data = (unsigned char *) masterkey; ++ mkey.size = h->phdr.master_key_len; ++ r = gnutls_cipher_init (&cipher, h->gnutls_cipher, &mkey, NULL); ++ if (r != 0) { ++ nbdkit_error ("gnutls_cipher_init: %s", gnutls_strerror (r)); ++ return -1; ++ } ++ ++ r = do_decrypt (h, cipher, 0, split_key, split_key_len); ++ gnutls_cipher_deinit (cipher); ++ if (r == -1) ++ return -1; ++ ++ /* Decode AFsplit key to a possible masterkey. */ ++ if (afmerge (h->hash_alg, ks->stripes, split_key, ++ masterkey, h->phdr.master_key_len) == -1) ++ return -1; ++ ++ /* Check if the masterkey is correct by comparing hash of the ++ * masterkey with LUKS header. ++ */ ++ r = gnutls_pbkdf2 (h->hash_alg, &mkey, &msalt, ++ h->phdr.master_key_digest_iterations, ++ key_digest, LUKS_DIGESTSIZE); ++ if (r != 0) { ++ nbdkit_error ("gnutls_pbkdf2: %s", gnutls_strerror (r)); ++ return -1; ++ } ++ ++ if (memcmp (key_digest, h->phdr.master_key_digest, LUKS_DIGESTSIZE) == 0) { ++ /* The passphrase is correct so save the master key in the handle. */ ++ h->masterkey = malloc (h->phdr.master_key_len); ++ if (h->masterkey == NULL) { ++ nbdkit_error ("malloc: %m"); ++ return -1; ++ } ++ memcpy (h->masterkey, masterkey, h->phdr.master_key_len); ++ return 1; ++ } ++ ++ return 0; ++} ++ ++static int ++luks_prepare (nbdkit_next *next, void *handle, int readonly) ++{ ++ static const char expected_magic[] = LUKS_MAGIC; ++ struct handle *h = handle; ++ int64_t size; ++ int err = 0, r; ++ size_t i; ++ struct luks_keyslot *ks; ++ char uuid[41]; ++ ++ /* Check we haven't been called before, this should never happen. */ ++ assert (h->phdr.version == 0); ++ ++ /* Check the struct size matches the documentation. */ ++ assert (sizeof (struct luks_phdr) == 592); ++ ++ /* Check this is a LUKSv1 disk. */ ++ size = next->get_size (next); ++ if (size == -1) ++ return -1; ++ if (size < 16384) { ++ nbdkit_error ("disk is too small to be LUKS-encrypted"); ++ return -1; ++ } ++ ++ /* Read the phdr. */ ++ if (next->pread (next, &h->phdr, sizeof h->phdr, 0, 0, &err) == -1) { ++ errno = err; ++ return -1; ++ } ++ ++ if (memcmp (h->phdr.magic, expected_magic, LUKS_MAGIC_LEN) != 0) { ++ nbdkit_error ("this disk does not contain a LUKS header"); ++ return -1; ++ } ++ h->phdr.version = be16toh (h->phdr.version); ++ if (h->phdr.version != 1) { ++ nbdkit_error ("this disk contains a LUKS version %" PRIu16 " header, " ++ "but this filter only supports LUKSv1", ++ h->phdr.version); ++ return -1; ++ } ++ ++ /* Byte-swap the rest of the header. */ ++ h->phdr.payload_offset = be32toh (h->phdr.payload_offset); ++ h->phdr.master_key_len = be32toh (h->phdr.master_key_len); ++ h->phdr.master_key_digest_iterations = ++ be32toh (h->phdr.master_key_digest_iterations); ++ ++ for (i = 0; i < LUKS_NUMKEYS; ++i) { ++ ks = &h->phdr.keyslot[i]; ++ ks->active = be32toh (ks->active); ++ ks->password_iterations = be32toh (ks->password_iterations); ++ ks->key_material_offset = be32toh (ks->key_material_offset); ++ ks->stripes = be32toh (ks->stripes); ++ } ++ ++ /* Sanity check some fields. */ ++ if (h->phdr.payload_offset >= size / LUKS_SECTOR_SIZE) { ++ nbdkit_error ("bad LUKSv1 header: payload offset points beyond " ++ "the end of the disk"); ++ return -1; ++ } ++ ++ /* We derive several allocations from master_key_len so make sure ++ * it's not insane. ++ */ ++ if (h->phdr.master_key_len > 1024) { ++ nbdkit_error ("bad LUKSv1 header: master key is too long"); ++ return -1; ++ } ++ ++ for (i = 0; i < LUKS_NUMKEYS; ++i) { ++ uint64_t start, len; ++ ++ ks = &h->phdr.keyslot[i]; ++ switch (ks->active) { ++ case LUKS_KEY_ENABLED: ++ if (!ks->stripes) { ++ nbdkit_error ("bad LUKSv1 header: key slot %zu is corrupted", i); ++ return -1; ++ } ++ if (ks->stripes >= 10000) { ++ nbdkit_error ("bad LUKSv1 header: key slot %zu stripes too large", i); ++ return -1; ++ } ++ start = ks->key_material_offset; ++ len = key_material_length_in_sectors (h, i); ++ if (len > 4096) /* bound it at something reasonable */ { ++ nbdkit_error ("bad LUKSv1 header: key slot %zu key material length " ++ "is too large", i); ++ return -1; ++ } ++ if (start * LUKS_SECTOR_SIZE >= size || ++ (start + len) * LUKS_SECTOR_SIZE >= size) { ++ nbdkit_error ("bad LUKSv1 header: key slot %zu key material offset " ++ "points beyond the end of the disk", i); ++ return -1; ++ } ++ if (ks->password_iterations > ULONG_MAX) { ++ nbdkit_error ("bad LUKSv1 header: key slot %zu " ++ "iterations too large", i); ++ return -1; ++ } ++ /*FALLTHROUGH*/ ++ case LUKS_KEY_DISABLED: ++ break; ++ ++ default: ++ nbdkit_error ("bad LUKSv1 header: key slot %zu has " ++ "an invalid active flag", i); ++ return -1; ++ } ++ } ++ ++ /* Decode the ciphers. */ ++ if (parse_cipher_strings (h) == -1) ++ return -1; ++ ++ /* Dump some information about the header. */ ++ memcpy (uuid, h->phdr.uuid, 40); ++ uuid[40] = 0; ++ nbdkit_debug ("LUKS UUID: %s", uuid); ++ ++ for (i = 0; i < LUKS_NUMKEYS; ++i) { ++ uint64_t start, len; ++ ++ ks = &h->phdr.keyslot[i]; ++ if (ks->active == LUKS_KEY_ENABLED) { ++ start = ks->key_material_offset; ++ len = key_material_length_in_sectors (h, i); ++ nbdkit_debug ("LUKS key slot %zu: key material in sectors %" PRIu64 ++ "..%" PRIu64, ++ i, start, start+len-1); ++ } ++ } ++ ++ /* Now try to unlock the master key. */ ++ for (i = 0; i < LUKS_NUMKEYS; ++i) { ++ r = try_passphrase_in_keyslot (next, h, i); ++ if (r == -1) ++ return -1; ++ if (r > 0) ++ goto unlocked; ++ } ++ nbdkit_error ("LUKS passphrase is not correct, " ++ "no key slot could be unlocked"); ++ return -1; ++ ++ unlocked: ++ assert (h->masterkey != NULL); ++ nbdkit_debug ("LUKS unlocked block device with passphrase"); ++ ++ return 0; ++} ++ ++static int64_t ++luks_get_size (nbdkit_next *next, void *handle) ++{ ++ struct handle *h = handle; ++ int64_t size; ++ ++ /* Check that prepare has been called already. */ ++ assert (h->phdr.version > 0); ++ ++ size = next->get_size (next); ++ if (size == -1) ++ return -1; ++ ++ if (size < h->phdr.payload_offset * LUKS_SECTOR_SIZE) { ++ nbdkit_error ("disk too small, or contains an incomplete LUKS partition"); ++ return -1; ++ } ++ ++ size -= h->phdr.payload_offset * LUKS_SECTOR_SIZE; ++ return size; ++} ++ ++/* Whatever the plugin says, several operations are not supported by ++ * this filter: ++ * ++ * - extents ++ * - trim ++ * - zero ++ */ ++static int ++luks_can_extents (nbdkit_next *next, void *handle) ++{ ++ return 0; ++} ++ ++static int ++luks_can_trim (nbdkit_next *next, void *handle) ++{ ++ return 0; ++} ++ ++static int ++luks_can_zero (nbdkit_next *next, void *handle) ++{ ++ return NBDKIT_ZERO_EMULATE; ++} ++ ++static int ++luks_can_fast_zero (nbdkit_next *next, void *handle) ++{ ++ return 0; ++} ++ ++/* Rely on nbdkit to call .pread to emulate .cache calls. We will ++ * respond by decrypting the block which could be stored by the cache ++ * filter or similar on top. ++ */ ++static int ++luks_can_cache (nbdkit_next *next, void *handle) ++{ ++ return NBDKIT_CACHE_EMULATE; ++} ++ ++/* Advertise minimum/preferred sector-sized blocks, although we can in ++ * fact handle any read or write. ++ */ ++static int ++luks_block_size (nbdkit_next *next, void *handle, ++ uint32_t *minimum, uint32_t *preferred, uint32_t *maximum) ++{ ++ if (next->block_size (next, minimum, preferred, maximum) == -1) ++ return -1; ++ ++ if (*minimum == 0) { /* No constraints set by the plugin. */ ++ *minimum = LUKS_SECTOR_SIZE; ++ *preferred = LUKS_SECTOR_SIZE; ++ *maximum = 0xffffffff; ++ } ++ else { ++ *minimum = MAX (*minimum, LUKS_SECTOR_SIZE); ++ *preferred = MAX (*minimum, MAX (*preferred, LUKS_SECTOR_SIZE)); ++ } ++ return 0; ++} ++ ++/* Decrypt data. */ ++static int ++luks_pread (nbdkit_next *next, void *handle, ++ void *buf, uint32_t count, uint64_t offset, ++ uint32_t flags, int *err) ++{ ++ struct handle *h = handle; ++ const uint64_t payload_offset = h->phdr.payload_offset * LUKS_SECTOR_SIZE; ++ CLEANUP_FREE uint8_t *sector = NULL; ++ uint64_t sectnum, sectoffs; ++ const gnutls_datum_t mkey = ++ { (unsigned char *) h->masterkey, h->phdr.master_key_len }; ++ gnutls_cipher_hd_t cipher; ++ int r; ++ ++ if (!h->masterkey) { ++ *err = EIO; ++ return -1; ++ } ++ ++ if (!IS_ALIGNED (count | offset, LUKS_SECTOR_SIZE)) { ++ sector = malloc (LUKS_SECTOR_SIZE); ++ if (sector == NULL) { ++ *err = errno; ++ nbdkit_error ("malloc: %m"); ++ return -1; ++ } ++ } ++ ++ r = gnutls_cipher_init (&cipher, h->gnutls_cipher, &mkey, NULL); ++ if (r != 0) { ++ nbdkit_error ("gnutls_cipher_init: %s", gnutls_strerror (r)); ++ *err = EIO; ++ return -1; ++ } ++ ++ sectnum = offset / LUKS_SECTOR_SIZE; /* sector number */ ++ sectoffs = offset % LUKS_SECTOR_SIZE; /* offset within the sector */ ++ ++ /* Unaligned head */ ++ if (sectoffs) { ++ uint64_t n = MIN (LUKS_SECTOR_SIZE - sectoffs, count); ++ ++ assert (sector); ++ if (next->pread (next, sector, LUKS_SECTOR_SIZE, ++ sectnum * LUKS_SECTOR_SIZE + payload_offset, ++ flags, err) == -1) ++ goto err; ++ ++ if (do_decrypt (h, cipher, offset & ~LUKS_SECTOR_SIZE, ++ sector, LUKS_SECTOR_SIZE) == -1) ++ goto err; ++ ++ memcpy (buf, §or[sectoffs], n); ++ ++ buf += n; ++ count -= n; ++ offset += n; ++ sectnum++; ++ } ++ ++ /* Aligned body */ ++ while (count >= LUKS_SECTOR_SIZE) { ++ if (next->pread (next, buf, LUKS_SECTOR_SIZE, ++ sectnum * LUKS_SECTOR_SIZE + payload_offset, ++ flags, err) == -1) ++ goto err; ++ ++ if (do_decrypt (h, cipher, offset, buf, LUKS_SECTOR_SIZE) == -1) ++ goto err; ++ ++ buf += LUKS_SECTOR_SIZE; ++ count -= LUKS_SECTOR_SIZE; ++ offset += LUKS_SECTOR_SIZE; ++ sectnum++; ++ } ++ ++ /* Unaligned tail */ ++ if (count) { ++ assert (sector); ++ if (next->pread (next, sector, LUKS_SECTOR_SIZE, ++ sectnum * LUKS_SECTOR_SIZE + payload_offset, ++ flags, err) == -1) ++ goto err; ++ ++ if (do_decrypt (h, cipher, offset, sector, LUKS_SECTOR_SIZE) == -1) ++ goto err; ++ ++ memcpy (buf, sector, count); ++ } ++ ++ gnutls_cipher_deinit (cipher); ++ return 0; ++ ++ err: ++ gnutls_cipher_deinit (cipher); ++ return -1; ++} ++ ++/* Lock preventing read-modify-write cycles from overlapping. */ ++static pthread_mutex_t read_modify_write_lock = PTHREAD_MUTEX_INITIALIZER; ++ ++/* Encrypt data. */ ++static int ++luks_pwrite (nbdkit_next *next, void *handle, ++ const void *buf, uint32_t count, uint64_t offset, ++ uint32_t flags, int *err) ++{ ++ struct handle *h = handle; ++ const uint64_t payload_offset = h->phdr.payload_offset * LUKS_SECTOR_SIZE; ++ CLEANUP_FREE uint8_t *sector = NULL; ++ uint64_t sectnum, sectoffs; ++ const gnutls_datum_t mkey = ++ { (unsigned char *) h->masterkey, h->phdr.master_key_len }; ++ gnutls_cipher_hd_t cipher; ++ int r; ++ ++ if (!h->masterkey) { ++ *err = EIO; ++ return -1; ++ } ++ ++ sector = malloc (LUKS_SECTOR_SIZE); ++ if (sector == NULL) { ++ *err = errno; ++ nbdkit_error ("malloc: %m"); ++ return -1; ++ } ++ ++ r = gnutls_cipher_init (&cipher, h->gnutls_cipher, &mkey, NULL); ++ if (r != 0) { ++ nbdkit_error ("gnutls_cipher_init: %s", gnutls_strerror (r)); ++ *err = EIO; ++ return -1; ++ } ++ ++ sectnum = offset / LUKS_SECTOR_SIZE; /* sector number */ ++ sectoffs = offset % LUKS_SECTOR_SIZE; /* offset within the sector */ ++ ++ /* Unaligned head */ ++ if (sectoffs) { ++ ACQUIRE_LOCK_FOR_CURRENT_SCOPE (&read_modify_write_lock); ++ ++ uint64_t n = MIN (LUKS_SECTOR_SIZE - sectoffs, count); ++ ++ if (next->pread (next, sector, LUKS_SECTOR_SIZE, ++ sectnum * LUKS_SECTOR_SIZE + payload_offset, ++ flags, err) == -1) ++ goto err; ++ ++ memcpy (§or[sectoffs], buf, n); ++ ++ if (do_encrypt (h, cipher, offset & ~LUKS_SECTOR_SIZE, ++ sector, LUKS_SECTOR_SIZE) == -1) ++ goto err; ++ ++ if (next->pwrite (next, sector, LUKS_SECTOR_SIZE, ++ sectnum * LUKS_SECTOR_SIZE + payload_offset, ++ flags, err) == -1) ++ goto err; ++ ++ buf += n; ++ count -= n; ++ offset += n; ++ sectnum++; ++ } ++ ++ /* Aligned body */ ++ while (count >= LUKS_SECTOR_SIZE) { ++ memcpy (sector, buf, LUKS_SECTOR_SIZE); ++ ++ if (do_encrypt (h, cipher, offset, sector, LUKS_SECTOR_SIZE) == -1) ++ goto err; ++ ++ if (next->pwrite (next, sector, LUKS_SECTOR_SIZE, ++ sectnum * LUKS_SECTOR_SIZE + payload_offset, ++ flags, err) == -1) ++ goto err; ++ ++ buf += LUKS_SECTOR_SIZE; ++ count -= LUKS_SECTOR_SIZE; ++ offset += LUKS_SECTOR_SIZE; ++ sectnum++; ++ } ++ ++ /* Unaligned tail */ ++ if (count) { ++ ACQUIRE_LOCK_FOR_CURRENT_SCOPE (&read_modify_write_lock); ++ ++ if (next->pread (next, sector, LUKS_SECTOR_SIZE, ++ sectnum * LUKS_SECTOR_SIZE + payload_offset, ++ flags, err) == -1) ++ goto err; ++ ++ memcpy (sector, buf, count); ++ ++ if (do_encrypt (h, cipher, offset, sector, LUKS_SECTOR_SIZE) == -1) ++ goto err; ++ ++ if (next->pwrite (next, sector, LUKS_SECTOR_SIZE, ++ sectnum * LUKS_SECTOR_SIZE + payload_offset, ++ flags, err) == -1) ++ goto err; ++ } ++ ++ gnutls_cipher_deinit (cipher); ++ return 0; ++ ++ err: ++ gnutls_cipher_deinit (cipher); ++ return -1; ++} ++ ++static struct nbdkit_filter filter = { ++ .name = "luks", ++ .longname = "nbdkit luks filter", ++ .unload = luks_unload, ++ .thread_model = luks_thread_model, ++ .config = luks_config, ++ .config_complete = luks_config_complete, ++ .config_help = luks_config_help, ++ .open = luks_open, ++ .close = luks_close, ++ .prepare = luks_prepare, ++ .get_size = luks_get_size, ++ .can_extents = luks_can_extents, ++ .can_trim = luks_can_trim, ++ .can_zero = luks_can_zero, ++ .can_fast_zero = luks_can_fast_zero, ++ .can_cache = luks_can_cache, ++ .block_size = luks_block_size, ++ .pread = luks_pread, ++ .pwrite = luks_pwrite, ++}; ++ ++NBDKIT_REGISTER_FILTER(filter) +diff --git a/filters/luks/nbdkit-luks-filter.pod b/filters/luks/nbdkit-luks-filter.pod +new file mode 100644 +index 00000000..56e51561 +--- /dev/null ++++ b/filters/luks/nbdkit-luks-filter.pod +@@ -0,0 +1,120 @@ ++=head1 NAME ++ ++nbdkit-luks-filter - read and write LUKS-encrypted disks and partitions ++ ++=head1 SYNOPSIS ++ ++ nbdkit file encrypted-disk.img --filter=luks passphrase=+/tmp/secret ++ ++=head1 DESCRIPTION ++ ++C is a filter for L which transparently ++opens a LUKS-encrypted disk image. LUKS ("Linux Unified Key Setup") ++is the Full Disk Encryption (FDE) system commonly used by Linux ++systems. This filter is compatible with LUKSv1 as implemented by the ++Linux kernel (dm_crypt), and by qemu. ++ ++You can place this filter on top of L to ++decrypt a local file: ++ ++ nbdkit file encrypted-disk.img --filter=luks passphrase=+/tmp/secret ++ ++If LUKS is present inside a partition in the disk image then you will ++have to combine this filter with L. The ++order of the filters is important: ++ ++ nbdkit file encrypted-disk.img \ ++ --filter=luks passphrase=+/tmp/secret \ ++ --filter=partition partition=1 ++ ++This filter also works on top of other plugins such as ++L: ++ ++ nbdkit curl https://example.com/encrypted-disk.img \ ++ --filter=luks passphrase=+/tmp/secret ++ ++The web server sees only the encrypted data. Without knowing the ++passphrase, the web server cannot access the decrypted disk. Only ++encrypted data is sent over the HTTP connection. nbdkit itself will ++serve I disk data over the NBD connection (if this is a ++problem see L, or use a Unix domain socket I<-U>). ++ ++The passphrase can be stored in a file (as shown), passed directly on ++the command line (insecure), entered interactively, or passed to ++nbdkit over a file descriptor. ++ ++This filter can read and write LUKSv1. It cannot create disks, change ++passphrases, add keyslots, etc. To do that, you can use ordinary ++Linux tools like L. Note you must force LUKSv1 ++(eg. using cryptsetup I<--type luks1>). L can also ++create compatible disk images: ++ ++ qemu-img create -f luks \ ++ --object secret,data=SECRET,id=sec0 \ ++ -o key-secret=sec0 \ ++ encrypted-disk.img 1G ++ ++=head1 PARAMETERS ++ ++=over 4 ++ ++=item BSECRET ++ ++Use the secret passphrase when decrypting the disk. ++ ++Note that passing this on the command line is not secure on shared ++machines. ++ ++=item B ++ ++Ask for the passphrase (interactively) when nbdkit starts up. ++ ++=item BFILENAME ++ ++Read the passphrase from the named file. This is a secure method to ++supply a passphrase, as long as you set the permissions on the file ++appropriately. ++ ++=item BFD ++ ++Read the passphrase from file descriptor number C, inherited from ++the parent process when nbdkit starts up. This is also a secure ++method to supply a passphrase. ++ ++=back ++ ++=head1 FILES ++ ++=over 4 ++ ++=item F<$filterdir/nbdkit-luks-filter.so> ++ ++The plugin. ++ ++Use C to find the location of C<$filterdir>. ++ ++=back ++ ++=head1 VERSION ++ ++C first appeared in nbdkit 1.32. ++ ++=head1 SEE ALSO ++ ++L, ++L, ++L, ++L, ++L, ++L, ++L, ++L, ++L. ++ ++=head1 AUTHORS ++ ++Richard W.M. Jones ++ ++=head1 COPYRIGHT ++ ++Copyright (C) 2013-2022 Red Hat Inc. +diff --git a/plugins/file/nbdkit-file-plugin.pod b/plugins/file/nbdkit-file-plugin.pod +index f8f0e198..b95e7349 100644 +--- a/plugins/file/nbdkit-file-plugin.pod ++++ b/plugins/file/nbdkit-file-plugin.pod +@@ -223,6 +223,7 @@ L, + L, + L, + L, ++L, + L. + + =head1 AUTHORS +diff --git a/tests/Makefile.am b/tests/Makefile.am +index b310e8a2..c29453ba 100644 +--- a/tests/Makefile.am ++++ b/tests/Makefile.am +@@ -1596,6 +1596,18 @@ EXTRA_DIST += \ + test-log-script-info.sh \ + $(NULL) + ++# luks filter test. ++if HAVE_GNUTLS ++TESTS += \ ++ test-luks-info.sh \ ++ test-luks-copy.sh \ ++ $(NULL) ++endif ++EXTRA_DIST += \ ++ test-luks-info.sh \ ++ test-luks-copy.sh \ ++ $(NULL) ++ + # multi-conn filter test. + TESTS += \ + test-multi-conn.sh \ +diff --git a/tests/test-luks-copy.sh b/tests/test-luks-copy.sh +new file mode 100755 +index 00000000..99f300d0 +--- /dev/null ++++ b/tests/test-luks-copy.sh +@@ -0,0 +1,125 @@ ++#!/usr/bin/env bash ++# nbdkit ++# Copyright (C) 2018-2022 Red Hat Inc. ++# ++# Redistribution and use in source and binary forms, with or without ++# modification, are permitted provided that the following conditions are ++# met: ++# ++# * Redistributions of source code must retain the above copyright ++# notice, this list of conditions and the following disclaimer. ++# ++# * Redistributions in binary form must reproduce the above copyright ++# notice, this list of conditions and the following disclaimer in the ++# documentation and/or other materials provided with the distribution. ++# ++# * Neither the name of Red Hat nor the names of its contributors may be ++# used to endorse or promote products derived from this software without ++# specific prior written permission. ++# ++# THIS SOFTWARE IS PROVIDED BY RED HAT AND CONTRIBUTORS ''AS IS'' AND ++# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, ++# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A ++# PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL RED HAT OR ++# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, ++# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT ++# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF ++# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ++# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, ++# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT ++# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF ++# SUCH DAMAGE. ++ ++source ./functions.sh ++set -e ++set -x ++ ++requires nbdcopy --version ++requires nbdsh --version ++requires_nbdsh_uri ++requires qemu-img --version ++requires bash -c 'qemu-img --help | grep -- --target-image-opts' ++requires hexdump --version ++requires truncate --version ++requires_filter luks ++ ++encrypt_disk=luks-copy1.img ++plain_disk=luks-copy2.img ++pid=luks-copy.pid ++sock=$(mktemp -u /tmp/nbdkit-test-sock.XXXXXX) ++cleanup_fn rm -f $encrypt_disk $plain_disk $pid $sock ++rm -f $encrypt_disk $plain_disk $pid $sock ++ ++# Create an empty encrypted disk container. ++# ++# NB: This is complicated because qemu doesn't create an all-zeroes ++# plaintext disk for some reason when you use create -f luks. It ++# starts with random plaintext. ++# ++# https://stackoverflow.com/a/44669936 ++qemu-img create -f luks \ ++ --object secret,data=123456,id=sec0 \ ++ -o key-secret=sec0 \ ++ $encrypt_disk 10M ++truncate -s 10M $plain_disk ++qemu-img convert --target-image-opts -n \ ++ --object secret,data=123456,id=sec0 \ ++ $plain_disk \ ++ driver=luks,file.filename=$encrypt_disk,key-secret=sec0 ++rm $plain_disk ++ ++# Start nbdkit on the encrypted disk. ++start_nbdkit -P $pid -U $sock \ ++ file $encrypt_disk --filter=luks passphrase=123456 ++uri="nbd+unix:///?socket=$sock" ++ ++# Copy the whole disk out. It should be empty. ++nbdcopy "$uri" $plain_disk ++ ++if [ "$(hexdump -C $plain_disk)" != '00000000 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| ++* ++00a00000' ]; then ++ echo "$0: expected plaintext disk to be empty" ++ exit 1 ++fi ++ ++# Use nbdsh to overwrite with some known data and check we can read ++# back what we wrote. ++nbdsh -u "$uri" \ ++ -c 'h.pwrite(b"1"*65536, 0)' \ ++ -c 'h.pwrite(b"2"*65536, 128*1024)' \ ++ -c 'h.pwrite(b"3"*65536, 9*1024*1024)' \ ++ -c 'buf = h.pread(65536, 0)' \ ++ -c 'assert buf == b"1"*65536' \ ++ -c 'buf = h.pread(65536, 65536)' \ ++ -c 'assert buf == bytearray(65536)' \ ++ -c 'buf = h.pread(65536, 128*1024)' \ ++ -c 'assert buf == b"2"*65536' \ ++ -c 'buf = h.pread(65536, 9*1024*1024)' \ ++ -c 'assert buf == b"3"*65536' \ ++ -c 'h.flush()' ++ ++# Use qemu to copy out the whole disk. Note we called flush() above ++# so the disk should be synchronised. ++qemu-img convert --image-opts \ ++ --object secret,data=123456,id=sec0 \ ++ driver=luks,file.filename=$encrypt_disk,key-secret=sec0 \ ++ $plain_disk ++ ++# Check the contents are expected. ++if [ "$(hexdump -C $plain_disk)" != '00000000 31 31 31 31 31 31 31 31 31 31 31 31 31 31 31 31 |1111111111111111| ++* ++00010000 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| ++* ++00020000 32 32 32 32 32 32 32 32 32 32 32 32 32 32 32 32 |2222222222222222| ++* ++00030000 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| ++* ++00900000 33 33 33 33 33 33 33 33 33 33 33 33 33 33 33 33 |3333333333333333| ++* ++00910000 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| ++* ++00a00000' ]; then ++ echo "$0: unexpected content" ++ exit 1 ++fi +diff --git a/tests/test-luks-info.sh b/tests/test-luks-info.sh +new file mode 100755 +index 00000000..3eff657b +--- /dev/null ++++ b/tests/test-luks-info.sh +@@ -0,0 +1,56 @@ ++#!/usr/bin/env bash ++# nbdkit ++# Copyright (C) 2018-2022 Red Hat Inc. ++# ++# Redistribution and use in source and binary forms, with or without ++# modification, are permitted provided that the following conditions are ++# met: ++# ++# * Redistributions of source code must retain the above copyright ++# notice, this list of conditions and the following disclaimer. ++# ++# * Redistributions in binary form must reproduce the above copyright ++# notice, this list of conditions and the following disclaimer in the ++# documentation and/or other materials provided with the distribution. ++# ++# * Neither the name of Red Hat nor the names of its contributors may be ++# used to endorse or promote products derived from this software without ++# specific prior written permission. ++# ++# THIS SOFTWARE IS PROVIDED BY RED HAT AND CONTRIBUTORS ''AS IS'' AND ++# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, ++# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A ++# PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL RED HAT OR ++# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, ++# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT ++# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF ++# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ++# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, ++# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT ++# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF ++# SUCH DAMAGE. ++ ++source ./functions.sh ++set -e ++set -x ++ ++requires nbdinfo --version ++requires qemu-img --version ++requires_filter luks ++ ++disk=luks-info.img ++info=luks-info.log ++cleanup_fn rm -f $disk $info ++rm -f $disk $info ++ ++qemu-img create -f luks \ ++ --object secret,data=123456,id=sec0 \ ++ -o key-secret=sec0 \ ++ $disk 10M ++ ++nbdkit -U - file $disk --filter=luks passphrase=123456 \ ++ --run 'nbdinfo $uri' > $info ++cat $info ++ ++# Check the size is 10M (so it doesn't include the LUKS header). ++grep "10485760" $info +-- +2.31.1 + diff --git a/0004-cow-Fix-for-qemu-6.1-which-requires-backing-format.patch b/0004-cow-Fix-for-qemu-6.1-which-requires-backing-format.patch new file mode 100644 index 0000000..eb7e88e --- /dev/null +++ b/0004-cow-Fix-for-qemu-6.1-which-requires-backing-format.patch @@ -0,0 +1,59 @@ +From 3c2879a38c299b725091cea45329879e3f46fc99 Mon Sep 17 00:00:00 2001 +From: "Richard W.M. Jones" +Date: Tue, 31 Aug 2021 11:23:27 +0100 +Subject: [PATCH] cow: Fix for qemu 6.1 which requires backing format + +The diffing example in the manual created a qcow2 file with a backing +file but did not specify the backing format. However qemu 6.1 now +requires this and fails with: + + qemu-img: cow-diff.qcow2: Backing file specified without backing format + +or: + + qemu-img: Could not change the backing file to 'cow-base.img': backing format must be specified + +Fix the example by adding the -F option to the command line. + +Also there was a test of this rebasing sequence which failed, so this +commit updates the test too. + +(cherry picked from commit 618290ef33ce13b75c1a79fea1f1ffb327b5ba07) +--- + filters/cow/nbdkit-cow-filter.pod | 4 ++-- + tests/test-cow.sh | 4 ++-- + 2 files changed, 4 insertions(+), 4 deletions(-) + +diff --git a/filters/cow/nbdkit-cow-filter.pod b/filters/cow/nbdkit-cow-filter.pod +index 4d5ae856..510bdd40 100644 +--- a/filters/cow/nbdkit-cow-filter.pod ++++ b/filters/cow/nbdkit-cow-filter.pod +@@ -101,8 +101,8 @@ At the end, disconnect the client. + Run these C commands to construct a qcow2 file containing + the differences: + +- qemu-img create -f qcow2 -b nbd:localhost diff.qcow2 +- qemu-img rebase -b disk.img diff.qcow2 ++ qemu-img create -F raw -b nbd:localhost -f qcow2 diff.qcow2 ++ qemu-img rebase -F raw -b disk.img -f qcow2 diff.qcow2 + + F now contains the differences between the base + (F) and the changes stored in nbdkit-cow-filter. C +diff --git a/tests/test-cow.sh b/tests/test-cow.sh +index 8772afd7..edc4c223 100755 +--- a/tests/test-cow.sh ++++ b/tests/test-cow.sh +@@ -72,8 +72,8 @@ fi + # If we have qemu-img, try the hairy rebase operation documented + # in the nbdkit-cow-filter manual. + if qemu-img --version >/dev/null 2>&1; then +- qemu-img create -f qcow2 -b nbd:unix:$sock cow-diff.qcow2 +- time qemu-img rebase -b cow-base.img cow-diff.qcow2 ++ qemu-img create -F raw -b nbd:unix:$sock -f qcow2 cow-diff.qcow2 ++ time qemu-img rebase -F raw -b cow-base.img -f qcow2 cow-diff.qcow2 + qemu-img info cow-diff.qcow2 + + # This checks the file we created exists. +-- +2.31.1 + diff --git a/0005-luks-Disable-filter-with-old-GnuTLS-in-Debian-10.patch b/0005-luks-Disable-filter-with-old-GnuTLS-in-Debian-10.patch new file mode 100644 index 0000000..2ff4e07 --- /dev/null +++ b/0005-luks-Disable-filter-with-old-GnuTLS-in-Debian-10.patch @@ -0,0 +1,95 @@ +From 66daae1a7daf680e06f884e9af6a14830263c932 Mon Sep 17 00:00:00 2001 +From: "Richard W.M. Jones" +Date: Sun, 8 May 2022 12:13:39 +0100 +Subject: [PATCH] luks: Disable filter with old GnuTLS in Debian 10 +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +On Debian 10: + +luks.c: In function ‘parse_cipher_strings’: +luks.c:574:26: error: ‘GNUTLS_CIPHER_AES_128_XTS’ undeclared (first use in this function); did you mean ‘GNUTLS_CIPHER_AES_128_CCM’? + h->gnutls_cipher = GNUTLS_CIPHER_AES_128_XTS; + ^~~~~~~~~~~~~~~~~~~~~~~~~ + GNUTLS_CIPHER_AES_128_CCM +luks.c:574:26: note: each undeclared identifier is reported only once for each function it appears in +luks.c:577:26: error: ‘GNUTLS_CIPHER_AES_256_XTS’ undeclared (first use in this function); did you mean ‘GNUTLS_CIPHER_AES_256_CCM’? + h->gnutls_cipher = GNUTLS_CIPHER_AES_256_XTS; + ^~~~~~~~~~~~~~~~~~~~~~~~~ + GNUTLS_CIPHER_AES_256_CCM +luks.c: In function ‘try_passphrase_in_keyslot’: +luks.c:728:7: error: implicit declaration of function ‘gnutls_pbkdf2’; did you mean ‘gnutls_prf’? [-Werror=implicit-function-declaration] + r = gnutls_pbkdf2 (h->hash_alg, &key, &salt, ks->password_iterations, + ^~~~~~~~~~~~~ + gnutls_prf + +Because gnutls_pbkdf2 is missing there's no chance of making this +filter work on this platform so it's best to compile it out. + +Fixes: commit 468919dce6c5eb57503eacac0f67e5dd87c58e6c +(cherry picked from commit f9f67e483f4aad19ad6101163d32562f13504ca7) +--- + configure.ac | 5 ++++- + filters/luks/Makefile.am | 2 +- + tests/Makefile.am | 2 +- + 3 files changed, 6 insertions(+), 3 deletions(-) + +diff --git a/configure.ac b/configure.ac +index de85b4da..1d209f67 100644 +--- a/configure.ac ++++ b/configure.ac +@@ -636,12 +636,15 @@ AS_IF([test "x$GNUTLS_LIBS" != "x"],[ + gnutls_certificate_set_known_dh_params \ + gnutls_group_get \ + gnutls_group_get_name \ ++ gnutls_pbkdf2 \ + gnutls_session_set_verify_cert \ + gnutls_srp_server_get_username \ + gnutls_transport_is_ktls_enabled \ + ]) + LIBS="$old_LIBS" + ]) ++AM_CONDITIONAL([HAVE_GNUTLS_PBKDF2], ++ [test "x$GNUTLS_LIBS" != "x" && test "x$ac_cv_func_gnutls_pbkdf2" = xyes]) + + AC_ARG_ENABLE([linuxdisk], + [AS_HELP_STRING([--disable-linuxdisk], +@@ -1484,7 +1487,7 @@ echo "Optional filters:" + echo + feature "ext2" test "x$HAVE_EXT2_TRUE" = "x" + feature "gzip" test "x$HAVE_ZLIB_TRUE" = "x" +-feature "LUKS" test "x$HAVE_GNUTLS_TRUE" != "x" ++feature "luks" test "x$HAVE_GNUTLS_PBKDF2_TRUE" = "x" + feature "xz" test "x$HAVE_LIBLZMA_TRUE" = "x" + + echo +diff --git a/filters/luks/Makefile.am b/filters/luks/Makefile.am +index 30089621..622e5c3d 100644 +--- a/filters/luks/Makefile.am ++++ b/filters/luks/Makefile.am +@@ -33,7 +33,7 @@ include $(top_srcdir)/common-rules.mk + + EXTRA_DIST = nbdkit-luks-filter.pod + +-if HAVE_GNUTLS ++if HAVE_GNUTLS_PBKDF2 + + filter_LTLIBRARIES = nbdkit-luks-filter.la + +diff --git a/tests/Makefile.am b/tests/Makefile.am +index c29453ba..5585b3b7 100644 +--- a/tests/Makefile.am ++++ b/tests/Makefile.am +@@ -1597,7 +1597,7 @@ EXTRA_DIST += \ + $(NULL) + + # luks filter test. +-if HAVE_GNUTLS ++if HAVE_GNUTLS_PBKDF2 + TESTS += \ + test-luks-info.sh \ + test-luks-copy.sh \ +-- +2.31.1 + diff --git a/0006-luks-Various-fixes-for-Clang.patch b/0006-luks-Various-fixes-for-Clang.patch new file mode 100644 index 0000000..c0f7f2f --- /dev/null +++ b/0006-luks-Various-fixes-for-Clang.patch @@ -0,0 +1,71 @@ +From b3c05065801c723966a3e8d93c9b84e808ff38b9 Mon Sep 17 00:00:00 2001 +From: "Richard W.M. Jones" +Date: Sun, 8 May 2022 12:30:09 +0100 +Subject: [PATCH] luks: Various fixes for Clang + +With Clang: + +luks.c:728:25: error: implicit conversion from enumeration type 'gnutls_digest_algorithm_t' to different enumeration type 'gnutls_mac_algorithm_t' [-Werror,-Wenum-conversion] + r = gnutls_pbkdf2 (h->hash_alg, &key, &salt, ks->password_iterations, + ~~~~~~~~~~~~~ ~~~^~~~~~~~ +luks.c:764:25: error: implicit conversion from enumeration type 'gnutls_digest_algorithm_t' to different enumeration type 'gnutls_mac_algorithm_t' [-Werror,-Wenum-conversion] + r = gnutls_pbkdf2 (h->hash_alg, &mkey, &msalt, + ~~~~~~~~~~~~~ ~~~^~~~~~~~ +luks.c:886:35: error: result of comparison of constant 18446744073709551615 with expression of type 'uint32_t' (aka 'unsigned int') is always false [-Werror,-Wtautological-constant-out-of-range-compare] + if (ks->password_iterations > ULONG_MAX) { + ~~~~~~~~~~~~~~~~~~~~~~~ ^ ~~~~~~~~~ + +Fixes: commit 468919dce6c5eb57503eacac0f67e5dd87c58e6c +(cherry picked from commit 87d488ede9101a2effc71cd1851bf4a4caa521d2) +--- + filters/luks/luks.c | 13 ++++++------- + 1 file changed, 6 insertions(+), 7 deletions(-) + +diff --git a/filters/luks/luks.c b/filters/luks/luks.c +index 706a9bd2..cc619698 100644 +--- a/filters/luks/luks.c ++++ b/filters/luks/luks.c +@@ -693,6 +693,10 @@ key_material_length_in_sectors (struct handle *h, size_t i) + static int + try_passphrase_in_keyslot (nbdkit_next *next, struct handle *h, size_t i) + { ++ /* I believe this is supposed to be safe, looking at the GnuTLS ++ * header file. ++ */ ++ const gnutls_mac_algorithm_t mac = (gnutls_mac_algorithm_t) h->hash_alg; + struct luks_keyslot *ks = &h->phdr.keyslot[i]; + size_t split_key_len; + CLEANUP_FREE uint8_t *split_key = NULL; +@@ -725,7 +729,7 @@ try_passphrase_in_keyslot (nbdkit_next *next, struct handle *h, size_t i) + } + + /* Hash the passphrase to make a possible masterkey. */ +- r = gnutls_pbkdf2 (h->hash_alg, &key, &salt, ks->password_iterations, ++ r = gnutls_pbkdf2 (mac, &key, &salt, ks->password_iterations, + masterkey, h->phdr.master_key_len); + if (r != 0) { + nbdkit_error ("gnutls_pbkdf2: %s", gnutls_strerror (r)); +@@ -761,7 +765,7 @@ try_passphrase_in_keyslot (nbdkit_next *next, struct handle *h, size_t i) + /* Check if the masterkey is correct by comparing hash of the + * masterkey with LUKS header. + */ +- r = gnutls_pbkdf2 (h->hash_alg, &mkey, &msalt, ++ r = gnutls_pbkdf2 (mac, &mkey, &msalt, + h->phdr.master_key_digest_iterations, + key_digest, LUKS_DIGESTSIZE); + if (r != 0) { +@@ -883,11 +887,6 @@ luks_prepare (nbdkit_next *next, void *handle, int readonly) + "points beyond the end of the disk", i); + return -1; + } +- if (ks->password_iterations > ULONG_MAX) { +- nbdkit_error ("bad LUKSv1 header: key slot %zu " +- "iterations too large", i); +- return -1; +- } + /*FALLTHROUGH*/ + case LUKS_KEY_DISABLED: + break; +-- +2.31.1 + diff --git a/0007-luks-Link-with-libcompat-on-Windows.patch b/0007-luks-Link-with-libcompat-on-Windows.patch new file mode 100644 index 0000000..f93465b --- /dev/null +++ b/0007-luks-Link-with-libcompat-on-Windows.patch @@ -0,0 +1,43 @@ +From 9416effd73a5cb2e1c929449fca88fd7152aa1be Mon Sep 17 00:00:00 2001 +From: "Richard W.M. Jones" +Date: Sun, 8 May 2022 12:38:00 +0100 +Subject: [PATCH] luks: Link with libcompat on Windows + +/usr/lib/gcc/x86_64-w64-mingw32/11.2.1/../../../../x86_64-w64-mingw32/bin/ld: ../../common/utils/.libs/libutils.a(libutils_la-full-rw.o): in function `full_pread': +/builds/nbdkit/nbdkit/common/utils/full-rw.c:53: undefined reference to `pread' +/usr/lib/gcc/x86_64-w64-mingw32/11.2.1/../../../../x86_64-w64-mingw32/bin/ld: ../../common/utils/.libs/libutils.a(libutils_la-full-rw.o): in function `full_pwrite': +/builds/nbdkit/nbdkit/common/utils/full-rw.c:76: undefined reference to `pwrite' +/usr/lib/gcc/x86_64-w64-mingw32/11.2.1/../../../../x86_64-w64-mingw32/bin/ld: ../../common/utils/.libs/libutils.a(libutils_la-vector.o): in function `generic_vector_reserve_page_aligned': +/builds/nbdkit/nbdkit/common/utils/vector.c:112: undefined reference to `sysconf' +/usr/lib/gcc/x86_64-w64-mingw32/11.2.1/../../../../x86_64-w64-mingw32/bin/ld: /builds/nbdkit/nbdkit/common/utils/vector.c:134: undefined reference to `posix_memalign' +collect2: error: ld returned 1 exit status + +Fixes: commit 468919dce6c5eb57503eacac0f67e5dd87c58e6c +(cherry picked from commit 4a28c4c46aedf270929a62a1c5ecf2c1129cd456) +--- + filters/luks/Makefile.am | 2 ++ + 1 file changed, 2 insertions(+) + +diff --git a/filters/luks/Makefile.am b/filters/luks/Makefile.am +index 622e5c3d..2688f696 100644 +--- a/filters/luks/Makefile.am ++++ b/filters/luks/Makefile.am +@@ -45,6 +45,7 @@ nbdkit_luks_filter_la_SOURCES = \ + nbdkit_luks_filter_la_CPPFLAGS = \ + -I$(top_srcdir)/include \ + -I$(top_srcdir)/common/include \ ++ -I$(top_srcdir)/common/replacements \ + -I$(top_srcdir)/common/utils \ + $(NULL) + nbdkit_luks_filter_la_CFLAGS = \ +@@ -53,6 +54,7 @@ nbdkit_luks_filter_la_CFLAGS = \ + $(NULL) + nbdkit_luks_filter_la_LIBADD = \ + $(top_builddir)/common/utils/libutils.la \ ++ $(top_builddir)/common/replacements/libcompat.la \ + $(IMPORT_LIBRARY_ON_WINDOWS) \ + $(GNUTLS_LIBS) \ + $(NULL) +-- +2.31.1 + diff --git a/0008-luks-Refactor-the-filter.patch b/0008-luks-Refactor-the-filter.patch new file mode 100644 index 0000000..feaf986 --- /dev/null +++ b/0008-luks-Refactor-the-filter.patch @@ -0,0 +1,2096 @@ +From e8279107801bb93303b22e1b927929ce18279dc5 Mon Sep 17 00:00:00 2001 +From: "Richard W.M. Jones" +Date: Sun, 8 May 2022 16:13:13 +0100 +Subject: [PATCH] luks: Refactor the filter + +Move the LUKS-specific code out into a separate file. This is mostly +pure refactoring to tidy things up. + +(cherry picked from commit 2159d85e0bed1943542da58b43c91f2caa096d6c) +--- + filters/luks/Makefile.am | 2 + + filters/luks/luks-encryption.c | 926 +++++++++++++++++++++++++++++++++ + filters/luks/luks-encryption.h | 78 +++ + filters/luks/luks.c | 874 ++----------------------------- + 4 files changed, 1037 insertions(+), 843 deletions(-) + create mode 100644 filters/luks/luks-encryption.c + create mode 100644 filters/luks/luks-encryption.h + +diff --git a/filters/luks/Makefile.am b/filters/luks/Makefile.am +index 2688f696..0894ac8b 100644 +--- a/filters/luks/Makefile.am ++++ b/filters/luks/Makefile.am +@@ -38,6 +38,8 @@ if HAVE_GNUTLS_PBKDF2 + filter_LTLIBRARIES = nbdkit-luks-filter.la + + nbdkit_luks_filter_la_SOURCES = \ ++ luks-encryption.c \ ++ luks-encryption.h \ + luks.c \ + $(top_srcdir)/include/nbdkit-filter.h \ + $(NULL) +diff --git a/filters/luks/luks-encryption.c b/filters/luks/luks-encryption.c +new file mode 100644 +index 00000000..8ee0eb35 +--- /dev/null ++++ b/filters/luks/luks-encryption.c +@@ -0,0 +1,926 @@ ++/* nbdkit ++ * Copyright (C) 2018-2022 Red Hat Inc. ++ * ++ * Redistribution and use in source and binary forms, with or without ++ * modification, are permitted provided that the following conditions are ++ * met: ++ * ++ * * Redistributions of source code must retain the above copyright ++ * notice, this list of conditions and the following disclaimer. ++ * ++ * * Redistributions in binary form must reproduce the above copyright ++ * notice, this list of conditions and the following disclaimer in the ++ * documentation and/or other materials provided with the distribution. ++ * ++ * * Neither the name of Red Hat nor the names of its contributors may be ++ * used to endorse or promote products derived from this software without ++ * specific prior written permission. ++ * ++ * THIS SOFTWARE IS PROVIDED BY RED HAT AND CONTRIBUTORS ''AS IS'' AND ++ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, ++ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A ++ * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL RED HAT OR ++ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, ++ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT ++ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF ++ * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ++ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, ++ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT ++ * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF ++ * SUCH DAMAGE. ++ */ ++ ++#include ++ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#include ++ ++#include ++ ++#include "luks-encryption.h" ++ ++#include "byte-swapping.h" ++#include "cleanup.h" ++#include "isaligned.h" ++#include "rounding.h" ++ ++/* LUKSv1 constants. */ ++#define LUKS_MAGIC { 'L', 'U', 'K', 'S', 0xBA, 0xBE } ++#define LUKS_MAGIC_LEN 6 ++#define LUKS_DIGESTSIZE 20 ++#define LUKS_SALTSIZE 32 ++#define LUKS_NUMKEYS 8 ++#define LUKS_KEY_DISABLED 0x0000DEAD ++#define LUKS_KEY_ENABLED 0x00AC71F3 ++#define LUKS_STRIPES 4000 ++#define LUKS_ALIGN_KEYSLOTS 4096 ++ ++/* Key slot. */ ++struct luks_keyslot { ++ uint32_t active; /* LUKS_KEY_DISABLED|LUKS_KEY_ENABLED */ ++ uint32_t password_iterations; ++ char password_salt[LUKS_SALTSIZE]; ++ uint32_t key_material_offset; ++ uint32_t stripes; ++} __attribute__((__packed__)); ++ ++/* LUKS superblock. */ ++struct luks_phdr { ++ char magic[LUKS_MAGIC_LEN]; /* LUKS_MAGIC */ ++ uint16_t version; /* Only 1 is supported. */ ++ char cipher_name[32]; ++ char cipher_mode[32]; ++ char hash_spec[32]; ++ uint32_t payload_offset; ++ uint32_t master_key_len; ++ uint8_t master_key_digest[LUKS_DIGESTSIZE]; ++ uint8_t master_key_salt[LUKS_SALTSIZE]; ++ uint32_t master_key_digest_iterations; ++ uint8_t uuid[40]; ++ ++ struct luks_keyslot keyslot[LUKS_NUMKEYS]; /* Key slots. */ ++} __attribute__((__packed__)); ++ ++/* Block cipher mode of operation. ++ * https://en.wikipedia.org/wiki/Block_cipher_mode_of_operation ++ */ ++enum cipher_mode { ++ CIPHER_MODE_ECB, CIPHER_MODE_CBC, CIPHER_MODE_XTS, CIPHER_MODE_CTR, ++}; ++ ++static enum cipher_mode ++lookup_cipher_mode (const char *str) ++{ ++ if (strcmp (str, "ecb") == 0) ++ return CIPHER_MODE_ECB; ++ if (strcmp (str, "cbc") == 0) ++ return CIPHER_MODE_CBC; ++ if (strcmp (str, "xts") == 0) ++ return CIPHER_MODE_XTS; ++ if (strcmp (str, "ctr") == 0) ++ return CIPHER_MODE_CTR; ++ nbdkit_error ("unknown cipher mode: %s " ++ "(expecting \"ecb\", \"cbc\", \"xts\" or \"ctr\")", str); ++ return -1; ++} ++ ++static const char * ++cipher_mode_to_string (enum cipher_mode v) ++{ ++ switch (v) { ++ case CIPHER_MODE_ECB: return "ecb"; ++ case CIPHER_MODE_CBC: return "cbc"; ++ case CIPHER_MODE_XTS: return "xts"; ++ case CIPHER_MODE_CTR: return "ctr"; ++ default: abort (); ++ } ++} ++ ++/* Methods used by LUKS to generate initial vectors. ++ * ++ * ESSIV is a bit more complicated to implement. It is supported by ++ * qemu but not by us. ++ */ ++enum ivgen { ++ IVGEN_PLAIN, IVGEN_PLAIN64, /* IVGEN_ESSIV, */ ++}; ++ ++static enum ivgen ++lookup_ivgen (const char *str) ++{ ++ if (strcmp (str, "plain") == 0) ++ return IVGEN_PLAIN; ++ if (strcmp (str, "plain64") == 0) ++ return IVGEN_PLAIN64; ++/* ++ if (strcmp (str, "essiv") == 0) ++ return IVGEN_ESSIV; ++*/ ++ nbdkit_error ("unknown IV generation algorithm: %s " ++ "(expecting \"plain\", \"plain64\" etc)", str); ++ return -1; ++} ++ ++static const char * ++ivgen_to_string (enum ivgen v) ++{ ++ switch (v) { ++ case IVGEN_PLAIN: return "plain"; ++ case IVGEN_PLAIN64: return "plain64"; ++ /*case IVGEN_ESSIV: return "essiv";*/ ++ default: abort (); ++ } ++} ++ ++static void ++calculate_iv (enum ivgen v, uint8_t *iv, size_t ivlen, uint64_t sector) ++{ ++ size_t prefixlen; ++ uint32_t sector32; ++ ++ switch (v) { ++ case IVGEN_PLAIN: ++ prefixlen = 4; /* 32 bits */ ++ if (prefixlen > ivlen) ++ prefixlen = ivlen; ++ sector32 = (uint32_t) sector; /* truncate to only lower bits */ ++ sector32 = htole32 (sector32); ++ memcpy (iv, §or32, prefixlen); ++ memset (iv + prefixlen, 0, ivlen - prefixlen); ++ break; ++ ++ case IVGEN_PLAIN64: ++ prefixlen = 8; /* 64 bits */ ++ if (prefixlen > ivlen) ++ prefixlen = ivlen; ++ sector = htole64 (sector); ++ memcpy (iv, §or, prefixlen); ++ memset (iv + prefixlen, 0, ivlen - prefixlen); ++ break; ++ ++ /*case IVGEN_ESSIV:*/ ++ default: abort (); ++ } ++} ++ ++/* Cipher algorithm. ++ * ++ * qemu in theory supports many more, but with the GnuTLS backend only ++ * AES is supported. The kernel seems to only support AES for LUKSv1. ++ */ ++enum cipher_alg { ++ CIPHER_ALG_AES_128, CIPHER_ALG_AES_192, CIPHER_ALG_AES_256, ++}; ++ ++static enum cipher_alg ++lookup_cipher_alg (const char *str, enum cipher_mode mode, int key_bytes) ++{ ++ if (mode == CIPHER_MODE_XTS) ++ key_bytes /= 2; ++ ++ if (strcmp (str, "aes") == 0) { ++ if (key_bytes == 16) ++ return CIPHER_ALG_AES_128; ++ if (key_bytes == 24) ++ return CIPHER_ALG_AES_192; ++ if (key_bytes == 32) ++ return CIPHER_ALG_AES_256; ++ } ++ nbdkit_error ("unknown cipher algorithm: %s (expecting \"aes\", etc)", str); ++ return -1; ++} ++ ++static const char * ++cipher_alg_to_string (enum cipher_alg v) ++{ ++ switch (v) { ++ case CIPHER_ALG_AES_128: return "aes-128"; ++ case CIPHER_ALG_AES_192: return "aes-192"; ++ case CIPHER_ALG_AES_256: return "aes-256"; ++ default: abort (); ++ } ++} ++ ++#if 0 ++static int ++cipher_alg_key_bytes (enum cipher_alg v) ++{ ++ switch (v) { ++ case CIPHER_ALG_AES_128: return 16; ++ case CIPHER_ALG_AES_192: return 24; ++ case CIPHER_ALG_AES_256: return 32; ++ default: abort (); ++ } ++} ++#endif ++ ++static int ++cipher_alg_iv_len (enum cipher_alg v, enum cipher_mode mode) ++{ ++ if (CIPHER_MODE_ECB) ++ return 0; /* Don't need an IV in this mode. */ ++ ++ switch (v) { ++ case CIPHER_ALG_AES_128: ++ case CIPHER_ALG_AES_192: ++ case CIPHER_ALG_AES_256: ++ return 16; ++ default: abort (); ++ } ++} ++ ++/* Hash, eg.MD5, SHA1 etc. ++ * ++ * We reuse the GnuTLS digest algorithm enum here since it supports at ++ * least all the ones that LUKSv1 does. ++ */ ++static gnutls_digest_algorithm_t ++lookup_hash (const char *str) ++{ ++ if (strcmp (str, "md5") == 0) ++ return GNUTLS_DIG_MD5; ++ if (strcmp (str, "sha1") == 0) ++ return GNUTLS_DIG_SHA1; ++ if (strcmp (str, "sha224") == 0) ++ return GNUTLS_DIG_SHA224; ++ if (strcmp (str, "sha256") == 0) ++ return GNUTLS_DIG_SHA256; ++ if (strcmp (str, "sha384") == 0) ++ return GNUTLS_DIG_SHA384; ++ if (strcmp (str, "sha512") == 0) ++ return GNUTLS_DIG_SHA512; ++ if (strcmp (str, "ripemd160") == 0) ++ return GNUTLS_DIG_RMD160; ++ nbdkit_error ("unknown hash algorithm: %s " ++ "(expecting \"md5\", \"sha1\", \"sha224\", etc)", str); ++ return -1; ++} ++ ++static const char * ++hash_to_string (gnutls_digest_algorithm_t v) ++{ ++ switch (v) { ++ case GNUTLS_DIG_UNKNOWN: return "unknown"; ++ case GNUTLS_DIG_MD5: return "md5"; ++ case GNUTLS_DIG_SHA1: return "sha1"; ++ case GNUTLS_DIG_SHA224: return "sha224"; ++ case GNUTLS_DIG_SHA256: return "sha256"; ++ case GNUTLS_DIG_SHA384: return "sha384"; ++ case GNUTLS_DIG_SHA512: return "sha512"; ++ case GNUTLS_DIG_RMD160: return "ripemd160"; ++ default: abort (); ++ } ++} ++ ++#if 0 ++/* See qemu & dm-crypt implementations for an explanation of what's ++ * going on here. ++ */ ++enum cipher_alg ++lookup_essiv_cipher (enum cipher_alg cipher_alg, ++ gnutls_digest_algorithm_t ivgen_hash_alg) ++{ ++ int digest_bytes = gnutls_hash_get_len (ivgen_hash_alg); ++ int key_bytes = cipher_alg_key_bytes (cipher_alg); ++ ++ if (digest_bytes == key_bytes) ++ return cipher_alg; ++ ++ switch (cipher_alg) { ++ case CIPHER_ALG_AES_128: ++ case CIPHER_ALG_AES_192: ++ case CIPHER_ALG_AES_256: ++ if (digest_bytes == 16) return CIPHER_ALG_AES_128; ++ if (digest_bytes == 24) return CIPHER_ALG_AES_192; ++ if (digest_bytes == 32) return CIPHER_ALG_AES_256; ++ nbdkit_error ("no %s cipher available with key size %d", ++ "AES", digest_bytes); ++ return -1; ++ default: ++ nbdkit_error ("ESSIV does not support cipher %s", ++ cipher_alg_to_string (cipher_alg)); ++ return -1; ++ } ++} ++#endif ++ ++/* Per-connection data. */ ++struct luks_data { ++ /* LUKS header, if necessary byte-swapped into host order. */ ++ struct luks_phdr phdr; ++ ++ /* Decoded algorithm etc. */ ++ enum cipher_alg cipher_alg; ++ enum cipher_mode cipher_mode; ++ gnutls_digest_algorithm_t hash_alg; ++ enum ivgen ivgen_alg; ++ gnutls_digest_algorithm_t ivgen_hash_alg; ++ enum cipher_alg ivgen_cipher_alg; ++ ++ /* GnuTLS algorithm. */ ++ gnutls_cipher_algorithm_t gnutls_cipher; ++ ++ /* If we managed to decrypt one of the keyslots using the passphrase ++ * then this contains the master key, otherwise NULL. ++ */ ++ uint8_t *masterkey; ++}; ++ ++/* Parse the header fields containing cipher algorithm, mode, etc. */ ++static int ++parse_cipher_strings (struct luks_data *h) ++{ ++ char cipher_name[33], cipher_mode[33], hash_spec[33]; ++ char *ivgen, *ivhash; ++ ++ /* Copy the header fields locally and ensure they are \0 terminated. */ ++ memcpy (cipher_name, h->phdr.cipher_name, 32); ++ cipher_name[32] = 0; ++ memcpy (cipher_mode, h->phdr.cipher_mode, 32); ++ cipher_mode[32] = 0; ++ memcpy (hash_spec, h->phdr.hash_spec, 32); ++ hash_spec[32] = 0; ++ ++ nbdkit_debug ("LUKS v%" PRIu16 " cipher: %s mode: %s hash: %s " ++ "master key: %" PRIu32 " bits", ++ h->phdr.version, cipher_name, cipher_mode, hash_spec, ++ h->phdr.master_key_len * 8); ++ ++ /* The cipher_mode header has the form: "ciphermode-ivgen[:ivhash]" ++ * QEmu writes: "xts-plain64" ++ */ ++ ivgen = strchr (cipher_mode, '-'); ++ if (!ivgen) { ++ nbdkit_error ("incorrect cipher_mode header, " ++ "expecting mode-ivgenerator but got \"%s\"", cipher_mode); ++ return -1; ++ } ++ *ivgen = '\0'; ++ ivgen++; ++ ++ ivhash = strchr (ivgen, ':'); ++ if (!ivhash) ++ h->ivgen_hash_alg = GNUTLS_DIG_UNKNOWN; ++ else { ++ *ivhash = '\0'; ++ ivhash++; ++ ++ h->ivgen_hash_alg = lookup_hash (ivhash); ++ if (h->ivgen_hash_alg == -1) ++ return -1; ++ } ++ ++ h->cipher_mode = lookup_cipher_mode (cipher_mode); ++ if (h->cipher_mode == -1) ++ return -1; ++ ++ h->cipher_alg = lookup_cipher_alg (cipher_name, h->cipher_mode, ++ h->phdr.master_key_len); ++ if (h->cipher_alg == -1) ++ return -1; ++ ++ h->hash_alg = lookup_hash (hash_spec); ++ if (h->hash_alg == -1) ++ return -1; ++ ++ h->ivgen_alg = lookup_ivgen (ivgen); ++ if (h->ivgen_alg == -1) ++ return -1; ++ ++#if 0 ++ if (h->ivgen_alg == IVGEN_ESSIV) { ++ if (!ivhash) { ++ nbdkit_error ("incorrect IV generator hash specification"); ++ return -1; ++ } ++ h->ivgen_cipher_alg = lookup_essiv_cipher (h->cipher_alg, ++ h->ivgen_hash_alg); ++ if (h->ivgen_cipher_alg == -1) ++ return -1; ++ } ++ else ++#endif ++ h->ivgen_cipher_alg = h->cipher_alg; ++ ++ nbdkit_debug ("LUKS parsed ciphers: %s %s %s %s %s %s", ++ cipher_alg_to_string (h->cipher_alg), ++ cipher_mode_to_string (h->cipher_mode), ++ hash_to_string (h->hash_alg), ++ ivgen_to_string (h->ivgen_alg), ++ hash_to_string (h->ivgen_hash_alg), ++ cipher_alg_to_string (h->ivgen_cipher_alg)); ++ ++ /* GnuTLS combines cipher and block mode into a single value. Not ++ * all possible combinations are available in GnuTLS. See: ++ * https://www.gnutls.org/manual/html_node/Supported-ciphersuites.html ++ */ ++ h->gnutls_cipher = GNUTLS_CIPHER_NULL; ++ switch (h->cipher_mode) { ++ case CIPHER_MODE_XTS: ++ switch (h->cipher_alg) { ++ case CIPHER_ALG_AES_128: ++ h->gnutls_cipher = GNUTLS_CIPHER_AES_128_XTS; ++ break; ++ case CIPHER_ALG_AES_256: ++ h->gnutls_cipher = GNUTLS_CIPHER_AES_256_XTS; ++ break; ++ default: break; ++ } ++ break; ++ case CIPHER_MODE_CBC: ++ switch (h->cipher_alg) { ++ case CIPHER_ALG_AES_128: ++ h->gnutls_cipher = GNUTLS_CIPHER_AES_128_CBC; ++ break; ++ case CIPHER_ALG_AES_192: ++ h->gnutls_cipher = GNUTLS_CIPHER_AES_192_CBC; ++ break; ++ case CIPHER_ALG_AES_256: ++ h->gnutls_cipher = GNUTLS_CIPHER_AES_256_CBC; ++ break; ++ default: break; ++ } ++ default: break; ++ } ++ if (h->gnutls_cipher == GNUTLS_CIPHER_NULL) { ++ nbdkit_error ("cipher algorithm %s in mode %s is not supported by GnuTLS", ++ cipher_alg_to_string (h->cipher_alg), ++ cipher_mode_to_string (h->cipher_mode)); ++ return -1; ++ } ++ ++ return 0; ++} ++ ++/* Anti-Forensic merge operation. */ ++static void ++xor (const uint8_t *in1, const uint8_t *in2, uint8_t *out, size_t len) ++{ ++ size_t i; ++ ++ for (i = 0; i < len; ++i) ++ out[i] = in1[i] ^ in2[i]; ++} ++ ++static int ++af_hash (gnutls_digest_algorithm_t hash_alg, uint8_t *block, size_t len) ++{ ++ size_t digest_bytes = gnutls_hash_get_len (hash_alg); ++ size_t nr_blocks, last_block_len; ++ size_t i; ++ CLEANUP_FREE uint8_t *temp = malloc (digest_bytes); ++ int r; ++ gnutls_hash_hd_t hash; ++ ++ nr_blocks = len / digest_bytes; ++ last_block_len = len % digest_bytes; ++ if (last_block_len != 0) ++ nr_blocks++; ++ else ++ last_block_len = digest_bytes; ++ ++ for (i = 0; i < nr_blocks; ++i) { ++ const uint32_t iv = htobe32 (i); ++ const size_t blen = i < nr_blocks - 1 ? digest_bytes : last_block_len; ++ ++ /* Hash iv + i'th block into temp. */ ++ r = gnutls_hash_init (&hash, hash_alg); ++ if (r != 0) { ++ nbdkit_error ("gnutls_hash_init: %s", gnutls_strerror (r)); ++ return -1; ++ } ++ gnutls_hash (hash, &iv, sizeof iv); ++ gnutls_hash (hash, &block[i*digest_bytes], blen); ++ gnutls_hash_deinit (hash, temp); ++ ++ memcpy (&block[i*digest_bytes], temp, blen); ++ } ++ ++ return 0; ++} ++ ++static int ++afmerge (gnutls_digest_algorithm_t hash_alg, uint32_t stripes, ++ const uint8_t *in, uint8_t *out, size_t outlen) ++{ ++ CLEANUP_FREE uint8_t *block = calloc (1, outlen); ++ size_t i; ++ ++ /* NB: input size is stripes * master_key_len where ++ * master_key_len == outlen ++ */ ++ for (i = 0; i < stripes-1; ++i) { ++ xor (&in[i*outlen], block, block, outlen); ++ if (af_hash (hash_alg, block, outlen) == -1) ++ return -1; ++ } ++ xor (&in[i*outlen], block, out, outlen); ++ return 0; ++} ++ ++/* Length of key material in key slot i (sectors). ++ * ++ * This is basically copied from qemu because the spec description is ++ * unintelligible and apparently doesn't match reality. ++ */ ++static uint64_t ++key_material_length_in_sectors (struct luks_data *h, size_t i) ++{ ++ uint64_t len, r; ++ ++ len = h->phdr.master_key_len * h->phdr.keyslot[i].stripes; ++ r = DIV_ROUND_UP (len, LUKS_SECTOR_SIZE); ++ r = ROUND_UP (r, LUKS_ALIGN_KEYSLOTS / LUKS_SECTOR_SIZE); ++ return r; ++} ++ ++/* Try the passphrase in key slot i. If this returns true then the ++ * passphrase was able to decrypt the master key, and the master key ++ * has been stored in h->masterkey. ++ */ ++static int ++try_passphrase_in_keyslot (nbdkit_next *next, struct luks_data *h, ++ size_t i, const char *passphrase) ++{ ++ /* I believe this is supposed to be safe, looking at the GnuTLS ++ * header file. ++ */ ++ const gnutls_mac_algorithm_t mac = (gnutls_mac_algorithm_t) h->hash_alg; ++ struct luks_keyslot *ks = &h->phdr.keyslot[i]; ++ size_t split_key_len; ++ CLEANUP_FREE uint8_t *split_key = NULL; ++ CLEANUP_FREE uint8_t *masterkey = NULL; ++ const gnutls_datum_t key = ++ { (unsigned char *) passphrase, strlen (passphrase) }; ++ const gnutls_datum_t salt = ++ { (unsigned char *) ks->password_salt, LUKS_SALTSIZE }; ++ const gnutls_datum_t msalt = ++ { (unsigned char *) h->phdr.master_key_salt, LUKS_SALTSIZE }; ++ gnutls_datum_t mkey; ++ gnutls_cipher_hd_t cipher; ++ int r, err = 0; ++ uint64_t start; ++ uint8_t key_digest[LUKS_DIGESTSIZE]; ++ ++ if (ks->active != LUKS_KEY_ENABLED) ++ return 0; ++ ++ split_key_len = h->phdr.master_key_len * ks->stripes; ++ split_key = malloc (split_key_len); ++ if (split_key == NULL) { ++ nbdkit_error ("malloc: %m"); ++ return -1; ++ } ++ masterkey = malloc (h->phdr.master_key_len); ++ if (masterkey == NULL) { ++ nbdkit_error ("malloc: %m"); ++ return -1; ++ } ++ ++ /* Hash the passphrase to make a possible masterkey. */ ++ r = gnutls_pbkdf2 (mac, &key, &salt, ks->password_iterations, ++ masterkey, h->phdr.master_key_len); ++ if (r != 0) { ++ nbdkit_error ("gnutls_pbkdf2: %s", gnutls_strerror (r)); ++ return -1; ++ } ++ ++ /* Read master key material from plugin. */ ++ start = ks->key_material_offset * LUKS_SECTOR_SIZE; ++ if (next->pread (next, split_key, split_key_len, start, 0, &err) == -1) { ++ errno = err; ++ return -1; ++ } ++ ++ /* Decrypt the (still AFsplit) master key material. */ ++ mkey.data = (unsigned char *) masterkey; ++ mkey.size = h->phdr.master_key_len; ++ r = gnutls_cipher_init (&cipher, h->gnutls_cipher, &mkey, NULL); ++ if (r != 0) { ++ nbdkit_error ("gnutls_cipher_init: %s", gnutls_strerror (r)); ++ return -1; ++ } ++ ++ r = do_decrypt (h, cipher, 0, split_key, split_key_len / LUKS_SECTOR_SIZE); ++ gnutls_cipher_deinit (cipher); ++ if (r == -1) ++ return -1; ++ ++ /* Decode AFsplit key to a possible masterkey. */ ++ if (afmerge (h->hash_alg, ks->stripes, split_key, ++ masterkey, h->phdr.master_key_len) == -1) ++ return -1; ++ ++ /* Check if the masterkey is correct by comparing hash of the ++ * masterkey with LUKS header. ++ */ ++ r = gnutls_pbkdf2 (mac, &mkey, &msalt, ++ h->phdr.master_key_digest_iterations, ++ key_digest, LUKS_DIGESTSIZE); ++ if (r != 0) { ++ nbdkit_error ("gnutls_pbkdf2: %s", gnutls_strerror (r)); ++ return -1; ++ } ++ ++ if (memcmp (key_digest, h->phdr.master_key_digest, LUKS_DIGESTSIZE) == 0) { ++ /* The passphrase is correct so save the master key in the handle. */ ++ h->masterkey = malloc (h->phdr.master_key_len); ++ if (h->masterkey == NULL) { ++ nbdkit_error ("malloc: %m"); ++ return -1; ++ } ++ memcpy (h->masterkey, masterkey, h->phdr.master_key_len); ++ return 1; ++ } ++ ++ return 0; ++} ++ ++struct luks_data * ++load_header (nbdkit_next *next, const char *passphrase) ++{ ++ static const char expected_magic[] = LUKS_MAGIC; ++ struct luks_data *h; ++ int64_t size; ++ int err = 0, r; ++ size_t i; ++ struct luks_keyslot *ks; ++ char uuid[41]; ++ ++ h = calloc (1, sizeof *h); ++ if (h == NULL) { ++ nbdkit_error ("calloc: %m"); ++ return NULL; ++ } ++ ++ /* Check the struct size matches the documentation. */ ++ assert (sizeof (struct luks_phdr) == 592); ++ ++ /* Check this is a LUKSv1 disk. */ ++ size = next->get_size (next); ++ if (size == -1) { ++ free (h); ++ return NULL; ++ } ++ if (size < 16384) { ++ nbdkit_error ("disk is too small to be LUKS-encrypted"); ++ free (h); ++ return NULL; ++ } ++ ++ /* Read the phdr. */ ++ if (next->pread (next, &h->phdr, sizeof h->phdr, 0, 0, &err) == -1) { ++ free (h); ++ errno = err; ++ return NULL; ++ } ++ ++ if (memcmp (h->phdr.magic, expected_magic, LUKS_MAGIC_LEN) != 0) { ++ nbdkit_error ("this disk does not contain a LUKS header"); ++ return NULL; ++ } ++ h->phdr.version = be16toh (h->phdr.version); ++ if (h->phdr.version != 1) { ++ nbdkit_error ("this disk contains a LUKS version %" PRIu16 " header, " ++ "but this filter only supports LUKSv1", ++ h->phdr.version); ++ free (h); ++ return NULL; ++ } ++ ++ /* Byte-swap the rest of the header. */ ++ h->phdr.payload_offset = be32toh (h->phdr.payload_offset); ++ h->phdr.master_key_len = be32toh (h->phdr.master_key_len); ++ h->phdr.master_key_digest_iterations = ++ be32toh (h->phdr.master_key_digest_iterations); ++ ++ for (i = 0; i < LUKS_NUMKEYS; ++i) { ++ ks = &h->phdr.keyslot[i]; ++ ks->active = be32toh (ks->active); ++ ks->password_iterations = be32toh (ks->password_iterations); ++ ks->key_material_offset = be32toh (ks->key_material_offset); ++ ks->stripes = be32toh (ks->stripes); ++ } ++ ++ /* Sanity check some fields. */ ++ if (h->phdr.payload_offset >= size / LUKS_SECTOR_SIZE) { ++ nbdkit_error ("bad LUKSv1 header: payload offset points beyond " ++ "the end of the disk"); ++ free (h); ++ return NULL; ++ } ++ ++ /* We derive several allocations from master_key_len so make sure ++ * it's not insane. ++ */ ++ if (h->phdr.master_key_len > 1024) { ++ nbdkit_error ("bad LUKSv1 header: master key is too long"); ++ free (h); ++ return NULL; ++ } ++ ++ for (i = 0; i < LUKS_NUMKEYS; ++i) { ++ uint64_t start, len; ++ ++ ks = &h->phdr.keyslot[i]; ++ switch (ks->active) { ++ case LUKS_KEY_ENABLED: ++ if (!ks->stripes) { ++ nbdkit_error ("bad LUKSv1 header: key slot %zu is corrupted", i); ++ free (h); ++ return NULL; ++ } ++ if (ks->stripes >= 10000) { ++ nbdkit_error ("bad LUKSv1 header: key slot %zu stripes too large", i); ++ return NULL; ++ } ++ start = ks->key_material_offset; ++ len = key_material_length_in_sectors (h, i); ++ if (len > 4096) /* bound it at something reasonable */ { ++ nbdkit_error ("bad LUKSv1 header: key slot %zu key material length " ++ "is too large", i); ++ free (h); ++ return NULL; ++ } ++ if (start * LUKS_SECTOR_SIZE >= size || ++ (start + len) * LUKS_SECTOR_SIZE >= size) { ++ nbdkit_error ("bad LUKSv1 header: key slot %zu key material offset " ++ "points beyond the end of the disk", i); ++ free (h); ++ return NULL; ++ } ++ /*FALLTHROUGH*/ ++ case LUKS_KEY_DISABLED: ++ break; ++ ++ default: ++ nbdkit_error ("bad LUKSv1 header: key slot %zu has " ++ "an invalid active flag", i); ++ return NULL; ++ } ++ } ++ ++ /* Decode the ciphers. */ ++ if (parse_cipher_strings (h) == -1) { ++ free (h); ++ return NULL; ++ } ++ ++ /* Dump some information about the header. */ ++ memcpy (uuid, h->phdr.uuid, 40); ++ uuid[40] = 0; ++ nbdkit_debug ("LUKS UUID: %s", uuid); ++ ++ for (i = 0; i < LUKS_NUMKEYS; ++i) { ++ uint64_t start, len; ++ ++ ks = &h->phdr.keyslot[i]; ++ if (ks->active == LUKS_KEY_ENABLED) { ++ start = ks->key_material_offset; ++ len = key_material_length_in_sectors (h, i); ++ nbdkit_debug ("LUKS key slot %zu: key material in sectors %" PRIu64 ++ "..%" PRIu64, ++ i, start, start+len-1); ++ } ++ } ++ ++ /* Now try to unlock the master key. */ ++ for (i = 0; i < LUKS_NUMKEYS; ++i) { ++ r = try_passphrase_in_keyslot (next, h, i, passphrase); ++ if (r == -1) { ++ free (h); ++ return NULL; ++ } ++ if (r > 0) ++ goto unlocked; ++ } ++ nbdkit_error ("LUKS passphrase is not correct, " ++ "no key slot could be unlocked"); ++ free (h); ++ return NULL; ++ ++ unlocked: ++ assert (h->masterkey != NULL); ++ nbdkit_debug ("LUKS unlocked block device with passphrase"); ++ ++ return h; ++} ++ ++/* Free fields in the handle that might have been set by load_header. */ ++void ++free_luks_data (struct luks_data *h) ++{ ++ if (h->masterkey) { ++ memset (h->masterkey, 0, h->phdr.master_key_len); ++ free (h->masterkey); ++ } ++ free (h); ++} ++ ++uint64_t ++get_payload_offset (struct luks_data *h) ++{ ++ return h->phdr.payload_offset; ++} ++ ++gnutls_cipher_hd_t ++create_cipher (struct luks_data *h) ++{ ++ gnutls_datum_t mkey; ++ gnutls_cipher_hd_t cipher; ++ int r; ++ ++ assert (h->masterkey != NULL); ++ ++ mkey.data = (unsigned char *) h->masterkey; ++ mkey.size = h->phdr.master_key_len; ++ r = gnutls_cipher_init (&cipher, h->gnutls_cipher, &mkey, NULL); ++ if (r != 0) { ++ nbdkit_error ("gnutls_cipher_init: %s", gnutls_strerror (r)); ++ return NULL; ++ } ++ return cipher; ++} ++ ++/* Perform decryption of a block of data in memory. */ ++int ++do_decrypt (struct luks_data *h, gnutls_cipher_hd_t cipher, ++ uint64_t sector, uint8_t *buf, size_t nr_sectors) ++{ ++ const size_t ivlen = cipher_alg_iv_len (h->cipher_alg, h->cipher_mode); ++ CLEANUP_FREE uint8_t *iv = malloc (ivlen); ++ int r; ++ ++ while (nr_sectors) { ++ calculate_iv (h->ivgen_alg, iv, ivlen, sector); ++ gnutls_cipher_set_iv (cipher, iv, ivlen); ++ r = gnutls_cipher_decrypt2 (cipher, ++ buf, LUKS_SECTOR_SIZE, /* ciphertext */ ++ buf, LUKS_SECTOR_SIZE /* plaintext */); ++ if (r != 0) { ++ nbdkit_error ("gnutls_cipher_decrypt2: %s", gnutls_strerror (r)); ++ return -1; ++ } ++ ++ buf += LUKS_SECTOR_SIZE; ++ nr_sectors--; ++ sector++; ++ } ++ ++ return 0; ++} ++ ++/* Perform encryption of a block of data in memory. */ ++int ++do_encrypt (struct luks_data *h, gnutls_cipher_hd_t cipher, ++ uint64_t sector, uint8_t *buf, size_t nr_sectors) ++{ ++ const size_t ivlen = cipher_alg_iv_len (h->cipher_alg, h->cipher_mode); ++ CLEANUP_FREE uint8_t *iv = malloc (ivlen); ++ int r; ++ ++ while (nr_sectors) { ++ calculate_iv (h->ivgen_alg, iv, ivlen, sector); ++ gnutls_cipher_set_iv (cipher, iv, ivlen); ++ r = gnutls_cipher_encrypt2 (cipher, ++ buf, LUKS_SECTOR_SIZE, /* plaintext */ ++ buf, LUKS_SECTOR_SIZE /* ciphertext */); ++ if (r != 0) { ++ nbdkit_error ("gnutls_cipher_decrypt2: %s", gnutls_strerror (r)); ++ return -1; ++ } ++ ++ buf += LUKS_SECTOR_SIZE; ++ nr_sectors--; ++ sector++; ++ } ++ ++ return 0; ++} +diff --git a/filters/luks/luks-encryption.h b/filters/luks/luks-encryption.h +new file mode 100644 +index 00000000..3f8b9c9e +--- /dev/null ++++ b/filters/luks/luks-encryption.h +@@ -0,0 +1,78 @@ ++/* nbdkit ++ * Copyright (C) 2013-2022 Red Hat Inc. ++ * ++ * Redistribution and use in source and binary forms, with or without ++ * modification, are permitted provided that the following conditions are ++ * met: ++ * ++ * * Redistributions of source code must retain the above copyright ++ * notice, this list of conditions and the following disclaimer. ++ * ++ * * Redistributions in binary form must reproduce the above copyright ++ * notice, this list of conditions and the following disclaimer in the ++ * documentation and/or other materials provided with the distribution. ++ * ++ * * Neither the name of Red Hat nor the names of its contributors may be ++ * used to endorse or promote products derived from this software without ++ * specific prior written permission. ++ * ++ * THIS SOFTWARE IS PROVIDED BY RED HAT AND CONTRIBUTORS ''AS IS'' AND ++ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, ++ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A ++ * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL RED HAT OR ++ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, ++ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT ++ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF ++ * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ++ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, ++ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT ++ * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF ++ * SUCH DAMAGE. ++ */ ++ ++/* This header file defines the file format used by LUKSv1. See also: ++ * https://gitlab.com/cryptsetup/cryptsetup/-/wikis/LUKS-standard/on-disk-format.pdf ++ * Note we do not yet support LUKSv2. ++ */ ++ ++#ifndef NBDKIT_LUKS_ENCRYPTION_H ++#define NBDKIT_LUKS_ENCRYPTION_H ++ ++#include ++#include ++ ++#define LUKS_SECTOR_SIZE 512 ++ ++/* Per-connection data. */ ++struct luks_data; ++ ++/* Load the LUKS header, parse the algorithms, unlock the masterkey ++ * using the passphrase, initialize all the fields in the handle. ++ * ++ * This function may call next->pread (many times). ++ */ ++extern struct luks_data *load_header (nbdkit_next *next, ++ const char *passphrase); ++ ++/* Free the handle and all fields inside it. */ ++extern void free_luks_data (struct luks_data *h); ++ ++/* Get the offset where the encrypted data starts (in sectors). */ ++extern uint64_t get_payload_offset (struct luks_data *h); ++ ++/* Create an GnuTLS cipher, initialized with the master key. Must be ++ * freed by the caller using gnutls_cipher_deinit. ++ */ ++extern gnutls_cipher_hd_t create_cipher (struct luks_data *h); ++ ++/* Perform decryption/encryption of a block of memory in-place. ++ * ++ * 'sector' is the sector number on disk, used to calculate IVs. (The ++ * keyslots also use these functions, but sector must be 0). ++ */ ++extern int do_decrypt (struct luks_data *h, gnutls_cipher_hd_t cipher, ++ uint64_t sector, uint8_t *buf, size_t nr_sectors); ++extern int do_encrypt (struct luks_data *h, gnutls_cipher_hd_t cipher, ++ uint64_t sector, uint8_t *buf, size_t nr_sectors); ++ ++#endif /* NBDKIT_LUKS_ENCRYPTION_H */ +diff --git a/filters/luks/luks.c b/filters/luks/luks.c +index cc619698..8ad3f4ec 100644 +--- a/filters/luks/luks.c ++++ b/filters/luks/luks.c +@@ -45,49 +45,11 @@ + + #include + +-#include "byte-swapping.h" ++#include "luks-encryption.h" ++ + #include "cleanup.h" + #include "isaligned.h" + #include "minmax.h" +-#include "rounding.h" +- +-/* LUKSv1 constants. */ +-#define LUKS_MAGIC { 'L', 'U', 'K', 'S', 0xBA, 0xBE } +-#define LUKS_MAGIC_LEN 6 +-#define LUKS_DIGESTSIZE 20 +-#define LUKS_SALTSIZE 32 +-#define LUKS_NUMKEYS 8 +-#define LUKS_KEY_DISABLED 0x0000DEAD +-#define LUKS_KEY_ENABLED 0x00AC71F3 +-#define LUKS_STRIPES 4000 +-#define LUKS_ALIGN_KEYSLOTS 4096 +-#define LUKS_SECTOR_SIZE 512 +- +-/* Key slot. */ +-struct luks_keyslot { +- uint32_t active; /* LUKS_KEY_DISABLED|LUKS_KEY_ENABLED */ +- uint32_t password_iterations; +- char password_salt[LUKS_SALTSIZE]; +- uint32_t key_material_offset; +- uint32_t stripes; +-} __attribute__((__packed__)); +- +-/* LUKS superblock. */ +-struct luks_phdr { +- char magic[LUKS_MAGIC_LEN]; /* LUKS_MAGIC */ +- uint16_t version; /* Only 1 is supported. */ +- char cipher_name[32]; +- char cipher_mode[32]; +- char hash_spec[32]; +- uint32_t payload_offset; +- uint32_t master_key_len; +- uint8_t master_key_digest[LUKS_DIGESTSIZE]; +- uint8_t master_key_salt[LUKS_SALTSIZE]; +- uint32_t master_key_digest_iterations; +- uint8_t uuid[40]; +- +- struct luks_keyslot keyslot[LUKS_NUMKEYS]; /* Key slots. */ +-} __attribute__((__packed__)); + + static char *passphrase = NULL; + +@@ -135,251 +97,9 @@ luks_config_complete (nbdkit_next_config_complete *next, nbdkit_backend *nxdata) + #define luks_config_help \ + "passphrase= Secret passphrase." + +-enum cipher_mode { +- CIPHER_MODE_ECB, CIPHER_MODE_CBC, CIPHER_MODE_XTS, CIPHER_MODE_CTR, +-}; +- +-static enum cipher_mode +-lookup_cipher_mode (const char *str) +-{ +- if (strcmp (str, "ecb") == 0) +- return CIPHER_MODE_ECB; +- if (strcmp (str, "cbc") == 0) +- return CIPHER_MODE_CBC; +- if (strcmp (str, "xts") == 0) +- return CIPHER_MODE_XTS; +- if (strcmp (str, "ctr") == 0) +- return CIPHER_MODE_CTR; +- nbdkit_error ("unknown cipher mode: %s " +- "(expecting \"ecb\", \"cbc\", \"xts\" or \"ctr\")", str); +- return -1; +-} +- +-static const char * +-cipher_mode_to_string (enum cipher_mode v) +-{ +- switch (v) { +- case CIPHER_MODE_ECB: return "ecb"; +- case CIPHER_MODE_CBC: return "cbc"; +- case CIPHER_MODE_XTS: return "xts"; +- case CIPHER_MODE_CTR: return "ctr"; +- default: abort (); +- } +-} +- +-enum ivgen { +- IVGEN_PLAIN, IVGEN_PLAIN64, /* IVGEN_ESSIV, */ +-}; +- +-static enum ivgen +-lookup_ivgen (const char *str) +-{ +- if (strcmp (str, "plain") == 0) +- return IVGEN_PLAIN; +- if (strcmp (str, "plain64") == 0) +- return IVGEN_PLAIN64; +-/* +- if (strcmp (str, "essiv") == 0) +- return IVGEN_ESSIV; +-*/ +- nbdkit_error ("unknown IV generation algorithm: %s " +- "(expecting \"plain\", \"plain64\" etc)", str); +- return -1; +-} +- +-static const char * +-ivgen_to_string (enum ivgen v) +-{ +- switch (v) { +- case IVGEN_PLAIN: return "plain"; +- case IVGEN_PLAIN64: return "plain64"; +- /*case IVGEN_ESSIV: return "essiv";*/ +- default: abort (); +- } +-} +- +-static void +-calculate_iv (enum ivgen v, uint8_t *iv, size_t ivlen, uint64_t sector) +-{ +- size_t prefixlen; +- uint32_t sector32; +- +- switch (v) { +- case IVGEN_PLAIN: +- prefixlen = 4; /* 32 bits */ +- if (prefixlen > ivlen) +- prefixlen = ivlen; +- sector32 = (uint32_t) sector; /* truncate to only lower bits */ +- sector32 = htole32 (sector32); +- memcpy (iv, §or32, prefixlen); +- memset (iv + prefixlen, 0, ivlen - prefixlen); +- break; +- +- case IVGEN_PLAIN64: +- prefixlen = 8; /* 64 bits */ +- if (prefixlen > ivlen) +- prefixlen = ivlen; +- sector = htole64 (sector); +- memcpy (iv, §or, prefixlen); +- memset (iv + prefixlen, 0, ivlen - prefixlen); +- break; +- +- /*case IVGEN_ESSIV:*/ +- default: abort (); +- } +-} +- +-enum cipher_alg { +- CIPHER_ALG_AES_128, CIPHER_ALG_AES_192, CIPHER_ALG_AES_256, +-}; +- +-static enum cipher_alg +-lookup_cipher_alg (const char *str, enum cipher_mode mode, int key_bytes) +-{ +- if (mode == CIPHER_MODE_XTS) +- key_bytes /= 2; +- +- if (strcmp (str, "aes") == 0) { +- if (key_bytes == 16) +- return CIPHER_ALG_AES_128; +- if (key_bytes == 24) +- return CIPHER_ALG_AES_192; +- if (key_bytes == 32) +- return CIPHER_ALG_AES_256; +- } +- nbdkit_error ("unknown cipher algorithm: %s (expecting \"aes\", etc)", str); +- return -1; +-} +- +-static const char * +-cipher_alg_to_string (enum cipher_alg v) +-{ +- switch (v) { +- case CIPHER_ALG_AES_128: return "aes-128"; +- case CIPHER_ALG_AES_192: return "aes-192"; +- case CIPHER_ALG_AES_256: return "aes-256"; +- default: abort (); +- } +-} +- +-#if 0 +-static int +-cipher_alg_key_bytes (enum cipher_alg v) +-{ +- switch (v) { +- case CIPHER_ALG_AES_128: return 16; +- case CIPHER_ALG_AES_192: return 24; +- case CIPHER_ALG_AES_256: return 32; +- default: abort (); +- } +-} +-#endif +- +-static int +-cipher_alg_iv_len (enum cipher_alg v, enum cipher_mode mode) +-{ +- if (CIPHER_MODE_ECB) +- return 0; /* Don't need an IV in this mode. */ +- +- switch (v) { +- case CIPHER_ALG_AES_128: +- case CIPHER_ALG_AES_192: +- case CIPHER_ALG_AES_256: +- return 16; +- default: abort (); +- } +-} +- +-static gnutls_digest_algorithm_t +-lookup_hash (const char *str) +-{ +- if (strcmp (str, "md5") == 0) +- return GNUTLS_DIG_MD5; +- if (strcmp (str, "sha1") == 0) +- return GNUTLS_DIG_SHA1; +- if (strcmp (str, "sha224") == 0) +- return GNUTLS_DIG_SHA224; +- if (strcmp (str, "sha256") == 0) +- return GNUTLS_DIG_SHA256; +- if (strcmp (str, "sha384") == 0) +- return GNUTLS_DIG_SHA384; +- if (strcmp (str, "sha512") == 0) +- return GNUTLS_DIG_SHA512; +- if (strcmp (str, "ripemd160") == 0) +- return GNUTLS_DIG_RMD160; +- nbdkit_error ("unknown hash algorithm: %s " +- "(expecting \"md5\", \"sha1\", \"sha224\", etc)", str); +- return -1; +-} +- +-static const char * +-hash_to_string (gnutls_digest_algorithm_t v) +-{ +- switch (v) { +- case GNUTLS_DIG_UNKNOWN: return "unknown"; +- case GNUTLS_DIG_MD5: return "md5"; +- case GNUTLS_DIG_SHA1: return "sha1"; +- case GNUTLS_DIG_SHA224: return "sha224"; +- case GNUTLS_DIG_SHA256: return "sha256"; +- case GNUTLS_DIG_SHA384: return "sha384"; +- case GNUTLS_DIG_SHA512: return "sha512"; +- case GNUTLS_DIG_RMD160: return "ripemd160"; +- default: abort (); +- } +-} +- +-#if 0 +-/* See qemu & dm-crypt implementations for an explanation of what's +- * going on here. +- */ +-static enum cipher_alg +-lookup_essiv_cipher (enum cipher_alg cipher_alg, +- gnutls_digest_algorithm_t ivgen_hash_alg) +-{ +- int digest_bytes = gnutls_hash_get_len (ivgen_hash_alg); +- int key_bytes = cipher_alg_key_bytes (cipher_alg); +- +- if (digest_bytes == key_bytes) +- return cipher_alg; +- +- switch (cipher_alg) { +- case CIPHER_ALG_AES_128: +- case CIPHER_ALG_AES_192: +- case CIPHER_ALG_AES_256: +- if (digest_bytes == 16) return CIPHER_ALG_AES_128; +- if (digest_bytes == 24) return CIPHER_ALG_AES_192; +- if (digest_bytes == 32) return CIPHER_ALG_AES_256; +- nbdkit_error ("no %s cipher available with key size %d", +- "AES", digest_bytes); +- return -1; +- default: +- nbdkit_error ("ESSIV does not support cipher %s", +- cipher_alg_to_string (cipher_alg)); +- return -1; +- } +-} +-#endif +- + /* Per-connection handle. */ + struct handle { +- /* LUKS header, if necessary byte-swapped into host order. */ +- struct luks_phdr phdr; +- +- /* Decoded algorithm etc. */ +- enum cipher_alg cipher_alg; +- enum cipher_mode cipher_mode; +- gnutls_digest_algorithm_t hash_alg; +- enum ivgen ivgen_alg; +- gnutls_digest_algorithm_t ivgen_hash_alg; +- enum cipher_alg ivgen_cipher_alg; +- +- /* GnuTLS algorithm. */ +- gnutls_cipher_algorithm_t gnutls_cipher; +- +- /* If we managed to decrypt one of the keyslots using the passphrase +- * then this contains the master key, otherwise NULL. +- */ +- uint8_t *masterkey; ++ struct luks_data *h; + }; + + static void * +@@ -405,536 +125,21 @@ luks_close (void *handle) + { + struct handle *h = handle; + +- if (h->masterkey) { +- memset (h->masterkey, 0, h->phdr.master_key_len); +- free (h->masterkey); +- } ++ free_luks_data (h->h); + free (h); + } + +-/* Perform decryption of a block of data in memory. */ +-static int +-do_decrypt (struct handle *h, gnutls_cipher_hd_t cipher, +- uint64_t offset, uint8_t *buf, size_t len) +-{ +- const size_t ivlen = cipher_alg_iv_len (h->cipher_alg, h->cipher_mode); +- uint64_t sector = offset / LUKS_SECTOR_SIZE; +- CLEANUP_FREE uint8_t *iv = malloc (ivlen); +- int r; +- +- assert (IS_ALIGNED (offset, LUKS_SECTOR_SIZE)); +- assert (IS_ALIGNED (len, LUKS_SECTOR_SIZE)); +- +- while (len) { +- calculate_iv (h->ivgen_alg, iv, ivlen, sector); +- gnutls_cipher_set_iv (cipher, iv, ivlen); +- r = gnutls_cipher_decrypt2 (cipher, +- buf, LUKS_SECTOR_SIZE, /* ciphertext */ +- buf, LUKS_SECTOR_SIZE /* plaintext */); +- if (r != 0) { +- nbdkit_error ("gnutls_cipher_decrypt2: %s", gnutls_strerror (r)); +- return -1; +- } +- +- buf += LUKS_SECTOR_SIZE; +- offset += LUKS_SECTOR_SIZE; +- len -= LUKS_SECTOR_SIZE; +- sector++; +- } +- +- return 0; +-} +- +-/* Perform encryption of a block of data in memory. */ +-static int +-do_encrypt (struct handle *h, gnutls_cipher_hd_t cipher, +- uint64_t offset, uint8_t *buf, size_t len) +-{ +- const size_t ivlen = cipher_alg_iv_len (h->cipher_alg, h->cipher_mode); +- uint64_t sector = offset / LUKS_SECTOR_SIZE; +- CLEANUP_FREE uint8_t *iv = malloc (ivlen); +- int r; +- +- assert (IS_ALIGNED (offset, LUKS_SECTOR_SIZE)); +- assert (IS_ALIGNED (len, LUKS_SECTOR_SIZE)); +- +- while (len) { +- calculate_iv (h->ivgen_alg, iv, ivlen, sector); +- gnutls_cipher_set_iv (cipher, iv, ivlen); +- r = gnutls_cipher_encrypt2 (cipher, +- buf, LUKS_SECTOR_SIZE, /* plaintext */ +- buf, LUKS_SECTOR_SIZE /* ciphertext */); +- if (r != 0) { +- nbdkit_error ("gnutls_cipher_decrypt2: %s", gnutls_strerror (r)); +- return -1; +- } +- +- buf += LUKS_SECTOR_SIZE; +- offset += LUKS_SECTOR_SIZE; +- len -= LUKS_SECTOR_SIZE; +- sector++; +- } +- +- return 0; +-} +- +-/* Parse the header fields containing cipher algorithm, mode, etc. */ +-static int +-parse_cipher_strings (struct handle *h) +-{ +- char cipher_name[33], cipher_mode[33], hash_spec[33]; +- char *ivgen, *ivhash; +- +- /* Copy the header fields locally and ensure they are \0 terminated. */ +- memcpy (cipher_name, h->phdr.cipher_name, 32); +- cipher_name[32] = 0; +- memcpy (cipher_mode, h->phdr.cipher_mode, 32); +- cipher_mode[32] = 0; +- memcpy (hash_spec, h->phdr.hash_spec, 32); +- hash_spec[32] = 0; +- +- nbdkit_debug ("LUKS v%" PRIu16 " cipher: %s mode: %s hash: %s " +- "master key: %" PRIu32 " bits", +- h->phdr.version, cipher_name, cipher_mode, hash_spec, +- h->phdr.master_key_len * 8); +- +- /* The cipher_mode header has the form: "ciphermode-ivgen[:ivhash]" +- * QEmu writes: "xts-plain64" +- */ +- ivgen = strchr (cipher_mode, '-'); +- if (!ivgen) { +- nbdkit_error ("incorrect cipher_mode header, " +- "expecting mode-ivgenerator but got \"%s\"", cipher_mode); +- return -1; +- } +- *ivgen = '\0'; +- ivgen++; +- +- ivhash = strchr (ivgen, ':'); +- if (!ivhash) +- h->ivgen_hash_alg = GNUTLS_DIG_UNKNOWN; +- else { +- *ivhash = '\0'; +- ivhash++; +- +- h->ivgen_hash_alg = lookup_hash (ivhash); +- if (h->ivgen_hash_alg == -1) +- return -1; +- } +- +- h->cipher_mode = lookup_cipher_mode (cipher_mode); +- if (h->cipher_mode == -1) +- return -1; +- +- h->cipher_alg = lookup_cipher_alg (cipher_name, h->cipher_mode, +- h->phdr.master_key_len); +- if (h->cipher_alg == -1) +- return -1; +- +- h->hash_alg = lookup_hash (hash_spec); +- if (h->hash_alg == -1) +- return -1; +- +- h->ivgen_alg = lookup_ivgen (ivgen); +- if (h->ivgen_alg == -1) +- return -1; +- +-#if 0 +- if (h->ivgen_alg == IVGEN_ESSIV) { +- if (!ivhash) { +- nbdkit_error ("incorrect IV generator hash specification"); +- return -1; +- } +- h->ivgen_cipher_alg = lookup_essiv_cipher (h->cipher_alg, +- h->ivgen_hash_alg); +- if (h->ivgen_cipher_alg == -1) +- return -1; +- } +- else +-#endif +- h->ivgen_cipher_alg = h->cipher_alg; +- +- nbdkit_debug ("LUKS parsed ciphers: %s %s %s %s %s %s", +- cipher_alg_to_string (h->cipher_alg), +- cipher_mode_to_string (h->cipher_mode), +- hash_to_string (h->hash_alg), +- ivgen_to_string (h->ivgen_alg), +- hash_to_string (h->ivgen_hash_alg), +- cipher_alg_to_string (h->ivgen_cipher_alg)); +- +- /* GnuTLS combines cipher and block mode into a single value. Not +- * all possible combinations are available in GnuTLS. See: +- * https://www.gnutls.org/manual/html_node/Supported-ciphersuites.html +- */ +- h->gnutls_cipher = GNUTLS_CIPHER_NULL; +- switch (h->cipher_mode) { +- case CIPHER_MODE_XTS: +- switch (h->cipher_alg) { +- case CIPHER_ALG_AES_128: +- h->gnutls_cipher = GNUTLS_CIPHER_AES_128_XTS; +- break; +- case CIPHER_ALG_AES_256: +- h->gnutls_cipher = GNUTLS_CIPHER_AES_256_XTS; +- break; +- default: break; +- } +- break; +- case CIPHER_MODE_CBC: +- switch (h->cipher_alg) { +- case CIPHER_ALG_AES_128: +- h->gnutls_cipher = GNUTLS_CIPHER_AES_128_CBC; +- break; +- case CIPHER_ALG_AES_192: +- h->gnutls_cipher = GNUTLS_CIPHER_AES_192_CBC; +- break; +- case CIPHER_ALG_AES_256: +- h->gnutls_cipher = GNUTLS_CIPHER_AES_256_CBC; +- break; +- default: break; +- } +- default: break; +- } +- if (h->gnutls_cipher == GNUTLS_CIPHER_NULL) { +- nbdkit_error ("cipher algorithm %s in mode %s is not supported by GnuTLS", +- cipher_alg_to_string (h->cipher_alg), +- cipher_mode_to_string (h->cipher_mode)); +- return -1; +- } +- +- return 0; +-} +- +-/* Anti-Forensic merge operation. */ +-static void +-xor (const uint8_t *in1, const uint8_t *in2, uint8_t *out, size_t len) +-{ +- size_t i; +- +- for (i = 0; i < len; ++i) +- out[i] = in1[i] ^ in2[i]; +-} +- +-static int +-af_hash (gnutls_digest_algorithm_t hash_alg, uint8_t *block, size_t len) +-{ +- size_t digest_bytes = gnutls_hash_get_len (hash_alg); +- size_t nr_blocks, last_block_len; +- size_t i; +- CLEANUP_FREE uint8_t *temp = malloc (digest_bytes); +- int r; +- gnutls_hash_hd_t hash; +- +- nr_blocks = len / digest_bytes; +- last_block_len = len % digest_bytes; +- if (last_block_len != 0) +- nr_blocks++; +- else +- last_block_len = digest_bytes; +- +- for (i = 0; i < nr_blocks; ++i) { +- const uint32_t iv = htobe32 (i); +- const size_t blen = i < nr_blocks - 1 ? digest_bytes : last_block_len; +- +- /* Hash iv + i'th block into temp. */ +- r = gnutls_hash_init (&hash, hash_alg); +- if (r != 0) { +- nbdkit_error ("gnutls_hash_init: %s", gnutls_strerror (r)); +- return -1; +- } +- gnutls_hash (hash, &iv, sizeof iv); +- gnutls_hash (hash, &block[i*digest_bytes], blen); +- gnutls_hash_deinit (hash, temp); +- +- memcpy (&block[i*digest_bytes], temp, blen); +- } +- +- return 0; +-} +- +-static int +-afmerge (gnutls_digest_algorithm_t hash_alg, uint32_t stripes, +- const uint8_t *in, uint8_t *out, size_t outlen) +-{ +- CLEANUP_FREE uint8_t *block = calloc (1, outlen); +- size_t i; +- +- /* NB: input size is stripes * master_key_len where +- * master_key_len == outlen +- */ +- for (i = 0; i < stripes-1; ++i) { +- xor (&in[i*outlen], block, block, outlen); +- if (af_hash (hash_alg, block, outlen) == -1) +- return -1; +- } +- xor (&in[i*outlen], block, out, outlen); +- return 0; +-} +- +-/* Length of key material in key slot i (sectors). +- * +- * This is basically copied from qemu because the spec description is +- * unintelligible and apparently doesn't match reality. +- */ +-static uint64_t +-key_material_length_in_sectors (struct handle *h, size_t i) +-{ +- uint64_t len, r; +- +- len = h->phdr.master_key_len * h->phdr.keyslot[i].stripes; +- r = DIV_ROUND_UP (len, LUKS_SECTOR_SIZE); +- r = ROUND_UP (r, LUKS_ALIGN_KEYSLOTS / LUKS_SECTOR_SIZE); +- return r; +-} +- +-/* Try the passphrase in key slot i. If this returns true then the +- * passphrase was able to decrypt the master key, and the master key +- * has been stored in h->masterkey. +- */ +-static int +-try_passphrase_in_keyslot (nbdkit_next *next, struct handle *h, size_t i) +-{ +- /* I believe this is supposed to be safe, looking at the GnuTLS +- * header file. +- */ +- const gnutls_mac_algorithm_t mac = (gnutls_mac_algorithm_t) h->hash_alg; +- struct luks_keyslot *ks = &h->phdr.keyslot[i]; +- size_t split_key_len; +- CLEANUP_FREE uint8_t *split_key = NULL; +- CLEANUP_FREE uint8_t *masterkey = NULL; +- const gnutls_datum_t key = +- { (unsigned char *) passphrase, strlen (passphrase) }; +- const gnutls_datum_t salt = +- { (unsigned char *) ks->password_salt, LUKS_SALTSIZE }; +- const gnutls_datum_t msalt = +- { (unsigned char *) h->phdr.master_key_salt, LUKS_SALTSIZE }; +- gnutls_datum_t mkey; +- gnutls_cipher_hd_t cipher; +- int r, err = 0; +- uint64_t start; +- uint8_t key_digest[LUKS_DIGESTSIZE]; +- +- if (ks->active != LUKS_KEY_ENABLED) +- return 0; +- +- split_key_len = h->phdr.master_key_len * ks->stripes; +- split_key = malloc (split_key_len); +- if (split_key == NULL) { +- nbdkit_error ("malloc: %m"); +- return -1; +- } +- masterkey = malloc (h->phdr.master_key_len); +- if (masterkey == NULL) { +- nbdkit_error ("malloc: %m"); +- return -1; +- } +- +- /* Hash the passphrase to make a possible masterkey. */ +- r = gnutls_pbkdf2 (mac, &key, &salt, ks->password_iterations, +- masterkey, h->phdr.master_key_len); +- if (r != 0) { +- nbdkit_error ("gnutls_pbkdf2: %s", gnutls_strerror (r)); +- return -1; +- } +- +- /* Read master key material from plugin. */ +- start = ks->key_material_offset * LUKS_SECTOR_SIZE; +- if (next->pread (next, split_key, split_key_len, start, 0, &err) == -1) { +- errno = err; +- return -1; +- } +- +- /* Decrypt the (still AFsplit) master key material. */ +- mkey.data = (unsigned char *) masterkey; +- mkey.size = h->phdr.master_key_len; +- r = gnutls_cipher_init (&cipher, h->gnutls_cipher, &mkey, NULL); +- if (r != 0) { +- nbdkit_error ("gnutls_cipher_init: %s", gnutls_strerror (r)); +- return -1; +- } +- +- r = do_decrypt (h, cipher, 0, split_key, split_key_len); +- gnutls_cipher_deinit (cipher); +- if (r == -1) +- return -1; +- +- /* Decode AFsplit key to a possible masterkey. */ +- if (afmerge (h->hash_alg, ks->stripes, split_key, +- masterkey, h->phdr.master_key_len) == -1) +- return -1; +- +- /* Check if the masterkey is correct by comparing hash of the +- * masterkey with LUKS header. +- */ +- r = gnutls_pbkdf2 (mac, &mkey, &msalt, +- h->phdr.master_key_digest_iterations, +- key_digest, LUKS_DIGESTSIZE); +- if (r != 0) { +- nbdkit_error ("gnutls_pbkdf2: %s", gnutls_strerror (r)); +- return -1; +- } +- +- if (memcmp (key_digest, h->phdr.master_key_digest, LUKS_DIGESTSIZE) == 0) { +- /* The passphrase is correct so save the master key in the handle. */ +- h->masterkey = malloc (h->phdr.master_key_len); +- if (h->masterkey == NULL) { +- nbdkit_error ("malloc: %m"); +- return -1; +- } +- memcpy (h->masterkey, masterkey, h->phdr.master_key_len); +- return 1; +- } +- +- return 0; +-} +- + static int + luks_prepare (nbdkit_next *next, void *handle, int readonly) + { +- static const char expected_magic[] = LUKS_MAGIC; + struct handle *h = handle; +- int64_t size; +- int err = 0, r; +- size_t i; +- struct luks_keyslot *ks; +- char uuid[41]; + + /* Check we haven't been called before, this should never happen. */ +- assert (h->phdr.version == 0); ++ assert (h->h == NULL); + +- /* Check the struct size matches the documentation. */ +- assert (sizeof (struct luks_phdr) == 592); +- +- /* Check this is a LUKSv1 disk. */ +- size = next->get_size (next); +- if (size == -1) +- return -1; +- if (size < 16384) { +- nbdkit_error ("disk is too small to be LUKS-encrypted"); +- return -1; +- } +- +- /* Read the phdr. */ +- if (next->pread (next, &h->phdr, sizeof h->phdr, 0, 0, &err) == -1) { +- errno = err; +- return -1; +- } +- +- if (memcmp (h->phdr.magic, expected_magic, LUKS_MAGIC_LEN) != 0) { +- nbdkit_error ("this disk does not contain a LUKS header"); ++ h->h = load_header (next, passphrase); ++ if (h->h == NULL) + return -1; +- } +- h->phdr.version = be16toh (h->phdr.version); +- if (h->phdr.version != 1) { +- nbdkit_error ("this disk contains a LUKS version %" PRIu16 " header, " +- "but this filter only supports LUKSv1", +- h->phdr.version); +- return -1; +- } +- +- /* Byte-swap the rest of the header. */ +- h->phdr.payload_offset = be32toh (h->phdr.payload_offset); +- h->phdr.master_key_len = be32toh (h->phdr.master_key_len); +- h->phdr.master_key_digest_iterations = +- be32toh (h->phdr.master_key_digest_iterations); +- +- for (i = 0; i < LUKS_NUMKEYS; ++i) { +- ks = &h->phdr.keyslot[i]; +- ks->active = be32toh (ks->active); +- ks->password_iterations = be32toh (ks->password_iterations); +- ks->key_material_offset = be32toh (ks->key_material_offset); +- ks->stripes = be32toh (ks->stripes); +- } +- +- /* Sanity check some fields. */ +- if (h->phdr.payload_offset >= size / LUKS_SECTOR_SIZE) { +- nbdkit_error ("bad LUKSv1 header: payload offset points beyond " +- "the end of the disk"); +- return -1; +- } +- +- /* We derive several allocations from master_key_len so make sure +- * it's not insane. +- */ +- if (h->phdr.master_key_len > 1024) { +- nbdkit_error ("bad LUKSv1 header: master key is too long"); +- return -1; +- } +- +- for (i = 0; i < LUKS_NUMKEYS; ++i) { +- uint64_t start, len; +- +- ks = &h->phdr.keyslot[i]; +- switch (ks->active) { +- case LUKS_KEY_ENABLED: +- if (!ks->stripes) { +- nbdkit_error ("bad LUKSv1 header: key slot %zu is corrupted", i); +- return -1; +- } +- if (ks->stripes >= 10000) { +- nbdkit_error ("bad LUKSv1 header: key slot %zu stripes too large", i); +- return -1; +- } +- start = ks->key_material_offset; +- len = key_material_length_in_sectors (h, i); +- if (len > 4096) /* bound it at something reasonable */ { +- nbdkit_error ("bad LUKSv1 header: key slot %zu key material length " +- "is too large", i); +- return -1; +- } +- if (start * LUKS_SECTOR_SIZE >= size || +- (start + len) * LUKS_SECTOR_SIZE >= size) { +- nbdkit_error ("bad LUKSv1 header: key slot %zu key material offset " +- "points beyond the end of the disk", i); +- return -1; +- } +- /*FALLTHROUGH*/ +- case LUKS_KEY_DISABLED: +- break; +- +- default: +- nbdkit_error ("bad LUKSv1 header: key slot %zu has " +- "an invalid active flag", i); +- return -1; +- } +- } +- +- /* Decode the ciphers. */ +- if (parse_cipher_strings (h) == -1) +- return -1; +- +- /* Dump some information about the header. */ +- memcpy (uuid, h->phdr.uuid, 40); +- uuid[40] = 0; +- nbdkit_debug ("LUKS UUID: %s", uuid); +- +- for (i = 0; i < LUKS_NUMKEYS; ++i) { +- uint64_t start, len; +- +- ks = &h->phdr.keyslot[i]; +- if (ks->active == LUKS_KEY_ENABLED) { +- start = ks->key_material_offset; +- len = key_material_length_in_sectors (h, i); +- nbdkit_debug ("LUKS key slot %zu: key material in sectors %" PRIu64 +- "..%" PRIu64, +- i, start, start+len-1); +- } +- } +- +- /* Now try to unlock the master key. */ +- for (i = 0; i < LUKS_NUMKEYS; ++i) { +- r = try_passphrase_in_keyslot (next, h, i); +- if (r == -1) +- return -1; +- if (r > 0) +- goto unlocked; +- } +- nbdkit_error ("LUKS passphrase is not correct, " +- "no key slot could be unlocked"); +- return -1; +- +- unlocked: +- assert (h->masterkey != NULL); +- nbdkit_debug ("LUKS unlocked block device with passphrase"); + + return 0; + } +@@ -946,19 +151,20 @@ luks_get_size (nbdkit_next *next, void *handle) + int64_t size; + + /* Check that prepare has been called already. */ +- assert (h->phdr.version > 0); ++ assert (h->h != NULL); ++ ++ const uint64_t payload_offset = get_payload_offset (h->h) * LUKS_SECTOR_SIZE; + + size = next->get_size (next); + if (size == -1) + return -1; + +- if (size < h->phdr.payload_offset * LUKS_SECTOR_SIZE) { ++ if (size < payload_offset) { + nbdkit_error ("disk too small, or contains an incomplete LUKS partition"); + return -1; + } + +- size -= h->phdr.payload_offset * LUKS_SECTOR_SIZE; +- return size; ++ return size - payload_offset; + } + + /* Whatever the plugin says, several operations are not supported by +@@ -1031,15 +237,12 @@ luks_pread (nbdkit_next *next, void *handle, + uint32_t flags, int *err) + { + struct handle *h = handle; +- const uint64_t payload_offset = h->phdr.payload_offset * LUKS_SECTOR_SIZE; ++ const uint64_t payload_offset = get_payload_offset (h->h) * LUKS_SECTOR_SIZE; + CLEANUP_FREE uint8_t *sector = NULL; + uint64_t sectnum, sectoffs; +- const gnutls_datum_t mkey = +- { (unsigned char *) h->masterkey, h->phdr.master_key_len }; + gnutls_cipher_hd_t cipher; +- int r; + +- if (!h->masterkey) { ++ if (!h->h) { + *err = EIO; + return -1; + } +@@ -1053,16 +256,13 @@ luks_pread (nbdkit_next *next, void *handle, + } + } + +- r = gnutls_cipher_init (&cipher, h->gnutls_cipher, &mkey, NULL); +- if (r != 0) { +- nbdkit_error ("gnutls_cipher_init: %s", gnutls_strerror (r)); +- *err = EIO; +- return -1; +- } +- + sectnum = offset / LUKS_SECTOR_SIZE; /* sector number */ + sectoffs = offset % LUKS_SECTOR_SIZE; /* offset within the sector */ + ++ cipher = create_cipher (h->h); ++ if (!cipher) ++ return -1; ++ + /* Unaligned head */ + if (sectoffs) { + uint64_t n = MIN (LUKS_SECTOR_SIZE - sectoffs, count); +@@ -1073,15 +273,13 @@ luks_pread (nbdkit_next *next, void *handle, + flags, err) == -1) + goto err; + +- if (do_decrypt (h, cipher, offset & ~LUKS_SECTOR_SIZE, +- sector, LUKS_SECTOR_SIZE) == -1) ++ if (do_decrypt (h->h, cipher, sectnum, sector, 1) == -1) + goto err; + + memcpy (buf, §or[sectoffs], n); + + buf += n; + count -= n; +- offset += n; + sectnum++; + } + +@@ -1092,12 +290,11 @@ luks_pread (nbdkit_next *next, void *handle, + flags, err) == -1) + goto err; + +- if (do_decrypt (h, cipher, offset, buf, LUKS_SECTOR_SIZE) == -1) ++ if (do_decrypt (h->h, cipher, sectnum, buf, 1) == -1) + goto err; + + buf += LUKS_SECTOR_SIZE; + count -= LUKS_SECTOR_SIZE; +- offset += LUKS_SECTOR_SIZE; + sectnum++; + } + +@@ -1109,7 +306,7 @@ luks_pread (nbdkit_next *next, void *handle, + flags, err) == -1) + goto err; + +- if (do_decrypt (h, cipher, offset, sector, LUKS_SECTOR_SIZE) == -1) ++ if (do_decrypt (h->h, cipher, sectnum, sector, 1) == -1) + goto err; + + memcpy (buf, sector, count); +@@ -1120,7 +317,7 @@ luks_pread (nbdkit_next *next, void *handle, + + err: + gnutls_cipher_deinit (cipher); +- return -1; ++ goto err; + } + + /* Lock preventing read-modify-write cycles from overlapping. */ +@@ -1133,15 +330,12 @@ luks_pwrite (nbdkit_next *next, void *handle, + uint32_t flags, int *err) + { + struct handle *h = handle; +- const uint64_t payload_offset = h->phdr.payload_offset * LUKS_SECTOR_SIZE; ++ const uint64_t payload_offset = get_payload_offset (h->h) * LUKS_SECTOR_SIZE; + CLEANUP_FREE uint8_t *sector = NULL; + uint64_t sectnum, sectoffs; +- const gnutls_datum_t mkey = +- { (unsigned char *) h->masterkey, h->phdr.master_key_len }; + gnutls_cipher_hd_t cipher; +- int r; + +- if (!h->masterkey) { ++ if (!h->h) { + *err = EIO; + return -1; + } +@@ -1153,16 +347,13 @@ luks_pwrite (nbdkit_next *next, void *handle, + return -1; + } + +- r = gnutls_cipher_init (&cipher, h->gnutls_cipher, &mkey, NULL); +- if (r != 0) { +- nbdkit_error ("gnutls_cipher_init: %s", gnutls_strerror (r)); +- *err = EIO; +- return -1; +- } +- + sectnum = offset / LUKS_SECTOR_SIZE; /* sector number */ + sectoffs = offset % LUKS_SECTOR_SIZE; /* offset within the sector */ + ++ cipher = create_cipher (h->h); ++ if (!cipher) ++ return -1; ++ + /* Unaligned head */ + if (sectoffs) { + ACQUIRE_LOCK_FOR_CURRENT_SCOPE (&read_modify_write_lock); +@@ -1176,8 +367,7 @@ luks_pwrite (nbdkit_next *next, void *handle, + + memcpy (§or[sectoffs], buf, n); + +- if (do_encrypt (h, cipher, offset & ~LUKS_SECTOR_SIZE, +- sector, LUKS_SECTOR_SIZE) == -1) ++ if (do_encrypt (h->h, cipher, sectnum, sector, 1) == -1) + goto err; + + if (next->pwrite (next, sector, LUKS_SECTOR_SIZE, +@@ -1187,7 +377,6 @@ luks_pwrite (nbdkit_next *next, void *handle, + + buf += n; + count -= n; +- offset += n; + sectnum++; + } + +@@ -1195,7 +384,7 @@ luks_pwrite (nbdkit_next *next, void *handle, + while (count >= LUKS_SECTOR_SIZE) { + memcpy (sector, buf, LUKS_SECTOR_SIZE); + +- if (do_encrypt (h, cipher, offset, sector, LUKS_SECTOR_SIZE) == -1) ++ if (do_encrypt (h->h, cipher, sectnum, sector, 1) == -1) + goto err; + + if (next->pwrite (next, sector, LUKS_SECTOR_SIZE, +@@ -1205,7 +394,6 @@ luks_pwrite (nbdkit_next *next, void *handle, + + buf += LUKS_SECTOR_SIZE; + count -= LUKS_SECTOR_SIZE; +- offset += LUKS_SECTOR_SIZE; + sectnum++; + } + +@@ -1220,7 +408,7 @@ luks_pwrite (nbdkit_next *next, void *handle, + + memcpy (sector, buf, count); + +- if (do_encrypt (h, cipher, offset, sector, LUKS_SECTOR_SIZE) == -1) ++ if (do_encrypt (h->h, cipher, sectnum, sector, 1) == -1) + goto err; + + if (next->pwrite (next, sector, LUKS_SECTOR_SIZE, +-- +2.31.1 + diff --git a/0009-tests-luks-Reduce-time-taken-to-run-these-tests.patch b/0009-tests-luks-Reduce-time-taken-to-run-these-tests.patch new file mode 100644 index 0000000..34419d8 --- /dev/null +++ b/0009-tests-luks-Reduce-time-taken-to-run-these-tests.patch @@ -0,0 +1,101 @@ +From 387bd4c6fee8ab339fd04e0b841b0c67e6020c8a Mon Sep 17 00:00:00 2001 +From: "Richard W.M. Jones" +Date: Sun, 8 May 2022 18:05:45 +0100 +Subject: [PATCH] tests: luks: Reduce time taken to run these tests + +Under valgrind they ran very slowly. Turns out valgrinding over +GnuTLS hashing code is not pretty. About half the time seems to be +taken opening the keyslot, and the rest copying the data. + +This change reduces the time (under valgrind) from 15 minutes 45 seconds +to about 6 mins 30 seconds. + +(cherry picked from commit 7320ae5dba476171a024ca44b889b3474302dc40) +--- + tests/test-luks-copy.sh | 18 +++++++++--------- + tests/test-luks-info.sh | 6 +++--- + 2 files changed, 12 insertions(+), 12 deletions(-) + +diff --git a/tests/test-luks-copy.sh b/tests/test-luks-copy.sh +index 99f300d0..01801811 100755 +--- a/tests/test-luks-copy.sh ++++ b/tests/test-luks-copy.sh +@@ -60,8 +60,8 @@ rm -f $encrypt_disk $plain_disk $pid $sock + qemu-img create -f luks \ + --object secret,data=123456,id=sec0 \ + -o key-secret=sec0 \ +- $encrypt_disk 10M +-truncate -s 10M $plain_disk ++ $encrypt_disk 1M ++truncate -s 1M $plain_disk + qemu-img convert --target-image-opts -n \ + --object secret,data=123456,id=sec0 \ + $plain_disk \ +@@ -74,11 +74,11 @@ start_nbdkit -P $pid -U $sock \ + uri="nbd+unix:///?socket=$sock" + + # Copy the whole disk out. It should be empty. +-nbdcopy "$uri" $plain_disk ++nbdcopy -C 1 "$uri" $plain_disk + + if [ "$(hexdump -C $plain_disk)" != '00000000 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| + * +-00a00000' ]; then ++00100000' ]; then + echo "$0: expected plaintext disk to be empty" + exit 1 + fi +@@ -88,14 +88,14 @@ fi + nbdsh -u "$uri" \ + -c 'h.pwrite(b"1"*65536, 0)' \ + -c 'h.pwrite(b"2"*65536, 128*1024)' \ +- -c 'h.pwrite(b"3"*65536, 9*1024*1024)' \ ++ -c 'h.pwrite(b"3"*65536, 900*1024)' \ + -c 'buf = h.pread(65536, 0)' \ + -c 'assert buf == b"1"*65536' \ + -c 'buf = h.pread(65536, 65536)' \ + -c 'assert buf == bytearray(65536)' \ + -c 'buf = h.pread(65536, 128*1024)' \ + -c 'assert buf == b"2"*65536' \ +- -c 'buf = h.pread(65536, 9*1024*1024)' \ ++ -c 'buf = h.pread(65536, 900*1024)' \ + -c 'assert buf == b"3"*65536' \ + -c 'h.flush()' + +@@ -115,11 +115,11 @@ if [ "$(hexdump -C $plain_disk)" != '00000000 31 31 31 31 31 31 31 31 31 31 31 + * + 00030000 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| + * +-00900000 33 33 33 33 33 33 33 33 33 33 33 33 33 33 33 33 |3333333333333333| ++000e1000 33 33 33 33 33 33 33 33 33 33 33 33 33 33 33 33 |3333333333333333| + * +-00910000 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| ++000f1000 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| + * +-00a00000' ]; then ++00100000' ]; then + echo "$0: unexpected content" + exit 1 + fi +diff --git a/tests/test-luks-info.sh b/tests/test-luks-info.sh +index 3eff657b..ef141ecd 100755 +--- a/tests/test-luks-info.sh ++++ b/tests/test-luks-info.sh +@@ -46,11 +46,11 @@ rm -f $disk $info + qemu-img create -f luks \ + --object secret,data=123456,id=sec0 \ + -o key-secret=sec0 \ +- $disk 10M ++ $disk 1M + + nbdkit -U - file $disk --filter=luks passphrase=123456 \ + --run 'nbdinfo $uri' > $info + cat $info + +-# Check the size is 10M (so it doesn't include the LUKS header). +-grep "10485760" $info ++# Check the size is 1M (so it doesn't include the LUKS header). ++grep "1048576" $info +-- +2.31.1 + diff --git a/0010-Add-nbdkit.parse_size-Python-function.patch b/0010-Add-nbdkit.parse_size-Python-function.patch new file mode 100644 index 0000000..5cb340b --- /dev/null +++ b/0010-Add-nbdkit.parse_size-Python-function.patch @@ -0,0 +1,112 @@ +From 52ee1dab95436128b44c37cc495022ff90108b2e Mon Sep 17 00:00:00 2001 +From: Nikolaus Rath +Date: Mon, 9 May 2022 10:04:30 +0100 +Subject: [PATCH] Add nbdkit.parse_size() Python function. + +This enables Python plugins to parse sizes the same way as C plugins. + +I'm not sure about the best way to test this - input is appreciated. + +I'm not too happy with the way this code is tested. It workes, but putting the tests into +test-python-plugin.py feels misplaced: this file is intended to support the unit tests in +test_python.py, not run its own unit tests. + +(cherry picked from commit 1b7d72542be68e254c1ef86ecb1a82b05c78ff63) +--- + plugins/python/modfunctions.c | 21 +++++++++++++++++++++ + plugins/python/nbdkit-python-plugin.pod | 5 +++++ + tests/test-python-plugin.py | 19 +++++++++++++++++++ + 3 files changed, 45 insertions(+) + +diff --git a/plugins/python/modfunctions.c b/plugins/python/modfunctions.c +index fffbaab2..46b0c904 100644 +--- a/plugins/python/modfunctions.c ++++ b/plugins/python/modfunctions.c +@@ -93,11 +93,32 @@ do_shutdown (PyObject *self, PyObject *args) + Py_RETURN_NONE; + } + ++/* nbdkit.parse_size */ ++static PyObject * ++parse_size (PyObject *self, PyObject *args) ++{ ++ const char *s; ++ if (!PyArg_ParseTuple (args, "s", &s)) { ++ PyErr_SetString (PyExc_TypeError, "Expected string, got something else"); ++ return NULL; ++ } ++ ++ int64_t size = nbdkit_parse_size(s); ++ if (size == -1) { ++ PyErr_SetString (PyExc_ValueError, "Unable to parse string as size"); ++ return NULL; ++ } ++ ++ return PyLong_FromSize_t((size_t)size); ++} ++ + static PyMethodDef NbdkitMethods[] = { + { "debug", debug, METH_VARARGS, + "Print a debug message" }, + { "export_name", export_name, METH_VARARGS, + "Return the optional export name negotiated with the client" }, ++ { "parse_size", parse_size, METH_VARARGS, ++ "Parse human-readable size strings into bytes" }, + { "set_error", set_error, METH_VARARGS, + "Store an errno value prior to throwing an exception" }, + { "shutdown", do_shutdown, METH_VARARGS, +diff --git a/plugins/python/nbdkit-python-plugin.pod b/plugins/python/nbdkit-python-plugin.pod +index 051b0237..ccc9406f 100644 +--- a/plugins/python/nbdkit-python-plugin.pod ++++ b/plugins/python/nbdkit-python-plugin.pod +@@ -131,6 +131,11 @@ Record C as the reason you are about to throw an exception. C + should correspond to usual errno values, where it may help to + C. + ++=head3 C ++ ++Parse a string (such as "100M") into a size in bytes. Wraps the ++C C function. ++ + =head3 C + + Request asynchronous server shutdown. +diff --git a/tests/test-python-plugin.py b/tests/test-python-plugin.py +index 0b34d532..d4f379fc 100644 +--- a/tests/test-python-plugin.py ++++ b/tests/test-python-plugin.py +@@ -34,12 +34,31 @@ + import nbdkit + import pickle + import base64 ++import unittest + + API_VERSION = 2 + + cfg = {} + + ++# Not nice, but there doesn't seem to be a better way of putting this ++class TestAPI(unittest.TestCase): ++ ++ def test_parse_size(self): ++ self.assertEqual(nbdkit.parse_size('511'), 511) ++ self.assertEqual(nbdkit.parse_size('7k'), 7*1024) ++ self.assertEqual(nbdkit.parse_size('17M'), 17*1024*1024) ++ ++ with self.assertRaises(TypeError): ++ nbdkit.parse_size(17) ++ ++ with self.assertRaises(ValueError): ++ nbdkit.parse_size('foo') ++ ++ ++TestAPI().test_parse_size() ++ ++ + def config(k, v): + global cfg + if k == "cfg": +-- +2.31.1 + diff --git a/0011-cache-Fix-cross-reference-nbdkit-readahead-filter.patch b/0011-cache-Fix-cross-reference-nbdkit-readahead-filter.patch new file mode 100644 index 0000000..ed4a4e3 --- /dev/null +++ b/0011-cache-Fix-cross-reference-nbdkit-readahead-filter.patch @@ -0,0 +1,34 @@ +From 644e0ed6333cf5fe2c1e39da157e8f1ce97267b9 Mon Sep 17 00:00:00 2001 +From: "Richard W.M. Jones" +Date: Sat, 14 May 2022 13:47:19 +0100 +Subject: [PATCH] cache: Fix cross-reference nbdkit-readahead-filter + +After the readahead filter was reimplemented so that it only issues +cache requests, the two filters should be used together, not as +alternatives. Update the documentation of the cache filter to make +this clear. + +Fixes: commit 2ff548d66ad3eae87868402ec5b3319edd12090f +(cherry picked from commit 894771f39a8fd2632caad00e497146d69cac4bac) +--- + filters/cache/nbdkit-cache-filter.pod | 4 ++-- + 1 file changed, 2 insertions(+), 2 deletions(-) + +diff --git a/filters/cache/nbdkit-cache-filter.pod b/filters/cache/nbdkit-cache-filter.pod +index d85fef09..f4234e1a 100644 +--- a/filters/cache/nbdkit-cache-filter.pod ++++ b/filters/cache/nbdkit-cache-filter.pod +@@ -28,8 +28,8 @@ loss, as the name suggests). + + This filter only caches image contents. To cache image metadata, use + L between this filter and the plugin. +-To accelerate sequential reads, use L +-instead. ++To accelerate sequential reads, use L on ++top of this filter. + + =head1 PARAMETERS + +-- +2.31.1 + diff --git a/0012-curl-Don-t-document-curl-plugin-readahead-filter.patch b/0012-curl-Don-t-document-curl-plugin-readahead-filter.patch new file mode 100644 index 0000000..4c8288e --- /dev/null +++ b/0012-curl-Don-t-document-curl-plugin-readahead-filter.patch @@ -0,0 +1,48 @@ +From 4a7e5169935c8850fddcea8da79639ded907c549 Mon Sep 17 00:00:00 2001 +From: "Richard W.M. Jones" +Date: Sat, 14 May 2022 14:00:16 +0100 +Subject: [PATCH] curl: Don't document curl plugin + readahead filter + +nbdkit readahead filter does not support plugins which do not use the +parallel thread model. + +Fixes: commit 2ff548d66ad3eae87868402ec5b3319edd12090f +(cherry picked from commit 92fbb76d11b9f17c527debd803aa2505f3642783) +--- + docs/nbdkit-captive.pod | 7 ------- + plugins/curl/nbdkit-curl-plugin.pod | 1 - + 2 files changed, 8 deletions(-) + +diff --git a/docs/nbdkit-captive.pod b/docs/nbdkit-captive.pod +index eafe36d8..d41a824d 100644 +--- a/docs/nbdkit-captive.pod ++++ b/docs/nbdkit-captive.pod +@@ -110,13 +110,6 @@ an embedded disk image. To copy it out: + + nbdkit -U - example1 --run 'qemu-img convert $nbd disk.img' + +-If plugin requests have a high overhead (for example making HTTP +-requests to a remote server), adding L may +-help performance: +- +- nbdkit -U - --filter=readahead curl https://example.com/disk.img \ +- --run 'qemu-img convert $nbd disk.img' +- + If the source suffers from temporary network failures + L or L may + help. +diff --git a/plugins/curl/nbdkit-curl-plugin.pod b/plugins/curl/nbdkit-curl-plugin.pod +index 54fce66c..fc422ca2 100644 +--- a/plugins/curl/nbdkit-curl-plugin.pod ++++ b/plugins/curl/nbdkit-curl-plugin.pod +@@ -509,7 +509,6 @@ L, + L, + L, + L, +-L, + L, + L, + L, +-- +2.31.1 + diff --git a/0013-New-filter-scan.patch b/0013-New-filter-scan.patch new file mode 100644 index 0000000..527a597 --- /dev/null +++ b/0013-New-filter-scan.patch @@ -0,0 +1,1021 @@ +From 8bfe6512d07caf778fd001425435b048c45513eb Mon Sep 17 00:00:00 2001 +From: "Richard W.M. Jones" +Date: Sat, 14 May 2022 13:46:56 +0100 +Subject: [PATCH] New filter: scan + +This filter will simply scan across the disk issuing a series of cache +requests to the underlying plugin. It is similar in scope and usage +to the new nbdkit-readahead-filter. + +(cherry picked from commit 65c20a09ceacb4431986a2982f2c2e746df63fcb) +--- + TODO | 8 - + configure.ac | 2 + + filters/cache/nbdkit-cache-filter.pod | 4 +- + .../nbdkit-cacheextents-filter.pod | 1 + + filters/readahead/nbdkit-readahead-filter.pod | 5 + + filters/scan/Makefile.am | 72 +++++ + filters/scan/bgthread.c | 131 ++++++++ + filters/scan/nbdkit-scan-filter.pod | 159 ++++++++++ + filters/scan/scan.c | 280 ++++++++++++++++++ + filters/scan/scan.h | 64 ++++ + plugins/ssh/nbdkit-ssh-plugin.pod | 1 + + plugins/torrent/nbdkit-torrent-plugin.pod | 1 + + plugins/vddk/nbdkit-vddk-plugin.pod | 1 + + tests/Makefile.am | 10 + + tests/test-scan-copy.sh | 42 +++ + tests/test-scan-info.sh | 46 +++ + 16 files changed, 817 insertions(+), 10 deletions(-) + create mode 100644 filters/scan/Makefile.am + create mode 100644 filters/scan/bgthread.c + create mode 100644 filters/scan/nbdkit-scan-filter.pod + create mode 100644 filters/scan/scan.c + create mode 100644 filters/scan/scan.h + create mode 100755 tests/test-scan-copy.sh + create mode 100755 tests/test-scan-info.sh + +diff --git a/TODO b/TODO +index 0f5dc41d..8600d9e4 100644 +--- a/TODO ++++ b/TODO +@@ -182,14 +182,6 @@ Python: + Suggestions for filters + ----------------------- + +-* Add scan filter. This would be placed on top of cache filters and +- would scan (read) the whole disk in the background, ensuring it is +- copied into the cache. Useful if you have a slow plugin, limited +- size device, and lots of local disk space, especially if you know +- that the NBD clients will eventually read all of the device. RWMJ +- wrote an implementation of this but it doesn't work well without a +- background thread. +- + * Add shared filter. Take advantage of filter context APIs to open a + single context into the backend shared among multiple client + connections. This may even allow a filter to offer a more parallel +diff --git a/configure.ac b/configure.ac +index 1d209f67..466dbd9b 100644 +--- a/configure.ac ++++ b/configure.ac +@@ -142,6 +142,7 @@ filters="\ + readahead \ + retry \ + retry-request \ ++ scan \ + stats \ + swab \ + tar \ +@@ -1403,6 +1404,7 @@ AC_CONFIG_FILES([Makefile + filters/readahead/Makefile + filters/retry/Makefile + filters/retry-request/Makefile ++ filters/scan/Makefile + filters/stats/Makefile + filters/swab/Makefile + filters/tar/Makefile +diff --git a/filters/cache/nbdkit-cache-filter.pod b/filters/cache/nbdkit-cache-filter.pod +index f4234e1a..935804b5 100644 +--- a/filters/cache/nbdkit-cache-filter.pod ++++ b/filters/cache/nbdkit-cache-filter.pod +@@ -28,8 +28,8 @@ loss, as the name suggests). + + This filter only caches image contents. To cache image metadata, use + L between this filter and the plugin. +-To accelerate sequential reads, use L on +-top of this filter. ++To accelerate sequential reads, use L or ++L on top of this filter. + + =head1 PARAMETERS + +diff --git a/filters/cacheextents/nbdkit-cacheextents-filter.pod b/filters/cacheextents/nbdkit-cacheextents-filter.pod +index bb2514a4..6464eac2 100644 +--- a/filters/cacheextents/nbdkit-cacheextents-filter.pod ++++ b/filters/cacheextents/nbdkit-cacheextents-filter.pod +@@ -54,6 +54,7 @@ L, + L, + L, + L, ++L, + L, + L, + L. +diff --git a/filters/readahead/nbdkit-readahead-filter.pod b/filters/readahead/nbdkit-readahead-filter.pod +index 630e5924..99d64dfb 100644 +--- a/filters/readahead/nbdkit-readahead-filter.pod ++++ b/filters/readahead/nbdkit-readahead-filter.pod +@@ -27,6 +27,10 @@ option. + The filter uses a simple adaptive algorithm which accelerates + sequential reads and requires no further configuration. + ++A similar filter is L which reads ahead over ++the whole disk, useful if you know that the client will be reading ++sequentially across most or all of the disk. ++ + =head2 Limitations + + In a number of significant cases this filter will do nothing. The +@@ -91,6 +95,7 @@ L, + L, + L, + L, ++L, + L, + L, + L, +diff --git a/filters/scan/Makefile.am b/filters/scan/Makefile.am +new file mode 100644 +index 00000000..d4aabfc6 +--- /dev/null ++++ b/filters/scan/Makefile.am +@@ -0,0 +1,72 @@ ++# nbdkit ++# Copyright (C) 2019-2021 Red Hat Inc. ++# ++# Redistribution and use in source and binary forms, with or without ++# modification, are permitted provided that the following conditions are ++# met: ++# ++# * Redistributions of source code must retain the above copyright ++# notice, this list of conditions and the following disclaimer. ++# ++# * Redistributions in binary form must reproduce the above copyright ++# notice, this list of conditions and the following disclaimer in the ++# documentation and/or other materials provided with the distribution. ++# ++# * Neither the name of Red Hat nor the names of its contributors may be ++# used to endorse or promote products derived from this software without ++# specific prior written permission. ++# ++# THIS SOFTWARE IS PROVIDED BY RED HAT AND CONTRIBUTORS ''AS IS'' AND ++# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, ++# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A ++# PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL RED HAT OR ++# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, ++# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT ++# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF ++# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ++# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, ++# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT ++# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF ++# SUCH DAMAGE. ++ ++include $(top_srcdir)/common-rules.mk ++ ++EXTRA_DIST = nbdkit-scan-filter.pod ++ ++filter_LTLIBRARIES = nbdkit-scan-filter.la ++ ++nbdkit_scan_filter_la_SOURCES = \ ++ scan.c \ ++ scan.h \ ++ bgthread.c \ ++ $(top_srcdir)/include/nbdkit-filter.h \ ++ $(NULL) ++ ++nbdkit_scan_filter_la_CPPFLAGS = \ ++ -I$(top_srcdir)/include \ ++ -I$(top_srcdir)/common/include \ ++ -I$(top_srcdir)/common/utils \ ++ $(NULL) ++nbdkit_scan_filter_la_CFLAGS = $(WARNINGS_CFLAGS) ++nbdkit_scan_filter_la_LDFLAGS = \ ++ -module -avoid-version -shared $(NO_UNDEFINED_ON_WINDOWS) \ ++ -Wl,--version-script=$(top_srcdir)/filters/filters.syms \ ++ $(NULL) ++nbdkit_scan_filter_la_LIBADD = \ ++ $(top_builddir)/common/utils/libutils.la \ ++ $(top_builddir)/common/replacements/libcompat.la \ ++ $(IMPORT_LIBRARY_ON_WINDOWS) \ ++ $(NULL) ++ ++if HAVE_POD ++ ++man_MANS = nbdkit-scan-filter.1 ++CLEANFILES += $(man_MANS) ++ ++nbdkit-scan-filter.1: nbdkit-scan-filter.pod \ ++ $(top_builddir)/podwrapper.pl ++ $(PODWRAPPER) --section=1 --man $@ \ ++ --html $(top_builddir)/html/$@.html \ ++ $< ++ ++endif HAVE_POD +diff --git a/filters/scan/bgthread.c b/filters/scan/bgthread.c +new file mode 100644 +index 00000000..384e79b6 +--- /dev/null ++++ b/filters/scan/bgthread.c +@@ -0,0 +1,131 @@ ++/* nbdkit ++ * Copyright (C) 2019-2022 Red Hat Inc. ++ * ++ * Redistribution and use in source and binary forms, with or without ++ * modification, are permitted provided that the following conditions are ++ * met: ++ * ++ * * Redistributions of source code must retain the above copyright ++ * notice, this list of conditions and the following disclaimer. ++ * ++ * * Redistributions in binary form must reproduce the above copyright ++ * notice, this list of conditions and the following disclaimer in the ++ * documentation and/or other materials provided with the distribution. ++ * ++ * * Neither the name of Red Hat nor the names of its contributors may be ++ * used to endorse or promote products derived from this software without ++ * specific prior written permission. ++ * ++ * THIS SOFTWARE IS PROVIDED BY RED HAT AND CONTRIBUTORS ''AS IS'' AND ++ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, ++ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A ++ * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL RED HAT OR ++ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, ++ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT ++ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF ++ * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ++ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, ++ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT ++ * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF ++ * SUCH DAMAGE. ++ */ ++ ++#include ++ ++#include ++#include ++#include ++#include ++ ++#include ++ ++#include "scan.h" ++ ++#include "cleanup.h" ++#include "minmax.h" ++ ++static pthread_mutex_t clock_lock; ++static uint64_t clock_ = 0; ++ ++static void ++adjust_clock (uint64_t offset) ++{ ++ ACQUIRE_LOCK_FOR_CURRENT_SCOPE (&clock_lock); ++ if (clock_ < offset) ++ clock_ = offset; ++} ++ ++static void ++reset_clock (uint64_t offset) ++{ ++ ACQUIRE_LOCK_FOR_CURRENT_SCOPE (&clock_lock); ++ clock_ = 0; ++} ++ ++static uint64_t ++get_starting_offset (void) ++{ ++ ACQUIRE_LOCK_FOR_CURRENT_SCOPE (&clock_lock); ++ return scan_clock ? clock_ : 0; ++} ++ ++void * ++scan_thread (void *vp) ++{ ++ struct bgthread_ctrl *ctrl = vp; ++ uint64_t offset, size; ++ int64_t r; ++ ++ assert (ctrl->next != NULL); ++ ++ /* Get the size of the underlying plugin. Exit the thread on error ++ * because there's not much we can do without knowing the size. ++ */ ++ r = ctrl->next->get_size (ctrl->next); ++ if (r == -1) ++ return NULL; ++ size = r; ++ ++ /* Start scanning. */ ++ start: ++ for (offset = get_starting_offset (); offset < size; offset += scan_size) { ++ uint64_t n; ++ ++ /* Execute any commands in the queue. */ ++ { ++ ACQUIRE_LOCK_FOR_CURRENT_SCOPE (&ctrl->lock); ++ struct command cmd; ++ ++ while (ctrl->cmds.len) { ++ cmd = ctrl->cmds.ptr[0]; ++ command_queue_remove (&ctrl->cmds, 0); ++ ++ switch (cmd.type) { ++ case CMD_QUIT: ++ nbdkit_debug ("scan: exiting background thread on connection close"); ++ return NULL; ++ ++ case CMD_NOTIFY_PREAD: ++ if (offset < cmd.offset) ++ offset = cmd.offset; ++ } ++ } ++ } ++ ++ adjust_clock (offset); ++ if (offset > size) ++ continue; ++ ++ /* Issue the next prefetch. */ ++ n = MIN (scan_size, size - offset); ++ ctrl->next->cache (ctrl->next, n, offset, 0, NULL); ++ } ++ ++ if (scan_forever) { ++ reset_clock (offset); ++ goto start; ++ } ++ ++ nbdkit_debug ("scan: finished scanning the plugin"); ++ return NULL; ++} +diff --git a/filters/scan/nbdkit-scan-filter.pod b/filters/scan/nbdkit-scan-filter.pod +new file mode 100644 +index 00000000..4a8d0ef9 +--- /dev/null ++++ b/filters/scan/nbdkit-scan-filter.pod +@@ -0,0 +1,159 @@ ++=head1 NAME ++ ++nbdkit-scan-filter - scan disk prefetching data ahead of sequential reads ++ ++=head1 SYNOPSIS ++ ++ nbdkit --filter=scan PLUGIN [scan-ahead=false] [scan-clock=false] ++ [scan-forever=true] [scan-size=]NN ++ ++ nbdkit --filter=scan --filter=cache PLUGIN ++ ++ nbdkit --filter=scan --filter=cow PLUGIN cow-on-cache=true ++ ++=head1 DESCRIPTION ++ ++C is a filter that scans the disk prefetching ++data. It is sometimes useful if you expect that the client will read ++the disk sequentially. ++ ++The basic operation of the filter is that when a client connects, the ++filter will start issuing C<.cache> (prefetch) requests to the plugin ++across the whole disk. Plugins which support this command will ++prefetch the data, making subsequent reads faster. For plugins which ++do not support this command, you can inject L ++below (after) this filter, giving approximately the same effect. ++L can be used instead of nbdkit-cache-filter, if ++you add the C option. ++ ++Various C parameters can be used to tune scanning, although ++the defaults should be suitable in most cases. ++ ++A similar filter is L. ++ ++=head2 Limitations ++ ++In a number of significant cases this filter will do nothing. The ++filter will print a warning message if this happens. ++ ++=over 4 ++ ++=item Thread model must be parallel * ++ ++For example L only supports ++C, and so this filter cannot perform prefetches in ++parallel with the read requests. ++ ++=item Only scans while clients are connected * ++ ++The current filter only scans while there is at least one client ++connected. ++ ++=item Only scans the default export * ++ ++The current filter only scans the default export and ignores all ++clients connecting to the non-default export name. ++ ++* We may be able to lift these restrictions in future. ++ ++=item Underlying filters or plugin must support C<.cache> (prefetch) ++ ++Very many plugins do not have the concept of prefetching and/or ++do not implement the C<.cache> callback, and so there is no ++way for this filter to issue prefetches. ++ ++You can usually get around this by adding I<--filter=cache> after this ++filter as explained above. ++ ++=item Prefetching the whole disk may load it all into cache ++ ++In particular if you use this filter together with ++L or L, they will cache ++the whole content of the plugin into a temporary file. This may be ++many gigabytes of data, consuming all space in F. Of course ++this is the whole point of using this filter, but you should be aware ++of it. ++ ++If using the cache filter, the total size of the cache can be limited ++(see L). ++ ++=back ++ ++=head1 PARAMETERS ++ ++=over 4 ++ ++=item B ++ ++By default the filter tries to stay ahead of incoming read requests. ++That is to say, it starts prefetching at the beginning of the disk and ++continues incrementally, but if the client issues a read beyond the ++current prefetch point then the filter skips forward and begins ++prefetching after the read. ++ ++However if you set this parameter to false, then this behaviour is ++disabled. The filter simply prefetches sequentially regardless of ++client requests. ++ ++=item B ++ ++By default, if all clients disconnect and then another client ++connects, prefetching resumes at the same place in the disk. (Like ++stopping and starting a clock.) ++ ++If you set this parameter to false, then the filter starts prefetching ++from the beginning of the disk again. ++ ++=item B ++ ++By default the filter scans over the disk once and then stops. ++ ++If you set this parameter to true, then after the disk has been ++prefetched completely, the filter goes back to the beginning and ++starts over, repeating this for as long as nbdkit is running and there ++are clients connected. ++ ++=item BNN ++ ++This parameter controls the prefetch block size. The default is ++C<2M>. This must be a power of 2 and most plugins will have their own ++limits on the amount of data they can prefetch in a single request. ++ ++=back ++ ++=head1 FILES ++ ++=over 4 ++ ++=item F<$filterdir/nbdkit-scan-filter.so> ++ ++The filter. ++ ++Use C to find the location of C<$filterdir>. ++ ++=back ++ ++=head1 VERSION ++ ++C first appeared in nbdkit 1.32. ++ ++=head1 SEE ALSO ++ ++L, ++L, ++L, ++L, ++L, ++L, ++L, ++L, ++L, ++L. ++ ++=head1 AUTHORS ++ ++Richard W.M. Jones ++ ++=head1 COPYRIGHT ++ ++Copyright (C) 2019-2022 Red Hat Inc. +diff --git a/filters/scan/scan.c b/filters/scan/scan.c +new file mode 100644 +index 00000000..ac5b18d2 +--- /dev/null ++++ b/filters/scan/scan.c +@@ -0,0 +1,280 @@ ++/* nbdkit ++ * Copyright (C) 2019-2022 Red Hat Inc. ++ * ++ * Redistribution and use in source and binary forms, with or without ++ * modification, are permitted provided that the following conditions are ++ * met: ++ * ++ * * Redistributions of source code must retain the above copyright ++ * notice, this list of conditions and the following disclaimer. ++ * ++ * * Redistributions in binary form must reproduce the above copyright ++ * notice, this list of conditions and the following disclaimer in the ++ * documentation and/or other materials provided with the distribution. ++ * ++ * * Neither the name of Red Hat nor the names of its contributors may be ++ * used to endorse or promote products derived from this software without ++ * specific prior written permission. ++ * ++ * THIS SOFTWARE IS PROVIDED BY RED HAT AND CONTRIBUTORS ''AS IS'' AND ++ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, ++ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A ++ * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL RED HAT OR ++ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, ++ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT ++ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF ++ * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ++ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, ++ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT ++ * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF ++ * SUCH DAMAGE. ++ */ ++ ++#include ++ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#include ++ ++#include "scan.h" ++ ++#include "cleanup.h" ++#include "ispowerof2.h" ++#include "vector.h" ++ ++static bool scan_ahead = true; ++bool scan_clock = true; ++bool scan_forever = false; ++unsigned scan_size = 2*1024*1024; ++ ++static int thread_model = -1; /* Thread model of the underlying plugin. */ ++ ++/* Per-connection data. */ ++struct scan_handle { ++ bool is_default_export; /* If exportname == "". */ ++ bool running; /* True if background thread is running. */ ++ pthread_t thread; /* The background thread, one per connection. */ ++ struct bgthread_ctrl ctrl; ++}; ++ ++static int ++scan_config (nbdkit_next_config *next, nbdkit_backend *nxdata, ++ const char *key, const char *value) ++{ ++ int r; ++ ++ if (strcmp (key, "scan-ahead") == 0) { ++ r = nbdkit_parse_bool (value); ++ if (r == -1) ++ return -1; ++ scan_ahead = r; ++ return 0; ++ } ++ else if (strcmp (key, "scan-clock") == 0) { ++ r = nbdkit_parse_bool (value); ++ if (r == -1) ++ return -1; ++ scan_clock = r; ++ return 0; ++ } ++ else if (strcmp (key, "scan-forever") == 0) { ++ r = nbdkit_parse_bool (value); ++ if (r == -1) ++ return -1; ++ scan_forever = r; ++ return 0; ++ } ++ else if (strcmp (key, "scan-size") == 0) { ++ scan_size = nbdkit_parse_size (value); ++ if (scan_size == -1) ++ return -1; ++ return 0; ++ } ++ ++ return next (nxdata, key, value); ++} ++ ++static int ++scan_config_complete (nbdkit_next_config_complete *next, nbdkit_backend *nxdata) ++{ ++ if (scan_size < 512 || scan_size > 32*1024*1024 || ++ !is_power_of_2 (scan_size)) { ++ nbdkit_error ("scan-size parameter should be [512..32M] " ++ "and a power of two"); ++ return -1; ++ } ++ ++ return next (nxdata); ++} ++ ++#define scan_config_help \ ++ "scan-ahead=false Skip ahead when client reads faster.\n" \ ++ "scan-clock=false Always start prefetching from beginning.\n" \ ++ "scan-forever=true Scan in a loop while clients connected.\n" \ ++ "scan-size=NN Set scan block size." ++ ++/* We need to hook into .get_ready() so we can read the final thread ++ * model (of the whole server). ++ */ ++static int ++scan_get_ready (int final_thread_model) ++{ ++ thread_model = final_thread_model; ++ return 0; ++} ++ ++static int ++send_command_to_background_thread (struct bgthread_ctrl *ctrl, ++ const struct command cmd) ++{ ++ ACQUIRE_LOCK_FOR_CURRENT_SCOPE (&ctrl->lock); ++ if (command_queue_append (&ctrl->cmds, cmd) == -1) ++ return -1; ++ /* Signal the thread if it could be sleeping on an empty queue. */ ++ if (ctrl->cmds.len == 1) ++ pthread_cond_signal (&ctrl->cond); ++ return 0; ++} ++ ++static void * ++scan_open (nbdkit_next_open *next, nbdkit_context *nxdata, ++ int readonly, const char *exportname, int is_tls) ++{ ++ struct scan_handle *h; ++ ++ if (next (nxdata, readonly, exportname) == -1) ++ return NULL; ++ ++ h = calloc (1, sizeof *h); ++ if (h == NULL) { ++ nbdkit_error ("malloc: %m"); ++ return NULL; ++ } ++ ++ h->is_default_export = strcmp (exportname, "") == 0; ++ return h; ++} ++ ++/* In prepare we check if it's possible to support the scan filter on ++ * this connection (or print a warning), and start the background ++ * thread. ++ */ ++static int ++scan_prepare (nbdkit_next *next, void *handle, int readonly) ++{ ++ struct scan_handle *h = handle; ++ int r, err; ++ ++ if (!h->is_default_export) { ++ nbdkit_error ("scan: warning: not the default export, not scanning"); ++ return 0; ++ } ++ ++ if (thread_model != NBDKIT_THREAD_MODEL_PARALLEL) { ++ nbdkit_error ("scan: warning: underlying plugin does not support " ++ "the PARALLEL thread model, not scanning"); ++ return 0; ++ } ++ ++ /* Call next->can_cache to read the underlying 'can_cache'. */ ++ r = next->can_cache (next); ++ if (r == -1) ++ return -1; ++ if (r != NBDKIT_CACHE_NATIVE) { ++ nbdkit_error ("scan: warning: underlying plugin does not support " ++ "NBD_CMD_CACHE, not scanning; try adding --filter=cache " ++ "after this filter"); ++ return 0; ++ } ++ ++ /* Save the connection in the handle, for the background thread to use. */ ++ h->ctrl.next = next; ++ ++ /* Create the background thread. */ ++ h->ctrl.cmds = (command_queue) empty_vector; ++ pthread_mutex_init (&h->ctrl.lock, NULL); ++ pthread_cond_init (&h->ctrl.cond, NULL); ++ ++ err = pthread_create (&h->thread, NULL, scan_thread, &h->ctrl); ++ if (err != 0) { ++ errno = err; ++ nbdkit_error ("pthread_create: %m"); ++ pthread_cond_destroy (&h->ctrl.cond); ++ pthread_mutex_destroy (&h->ctrl.lock); ++ return -1; ++ } ++ ++ h->running = true; ++ ++ return 0; ++} ++ ++/* Finalize cleans up the thread if it is running. */ ++static int ++scan_finalize (nbdkit_next *next, void *handle) ++{ ++ struct scan_handle *h = handle; ++ const struct command quit_cmd = { .type = CMD_QUIT }; ++ ++ if (!h->running) ++ return 0; ++ ++ send_command_to_background_thread (&h->ctrl, quit_cmd); ++ pthread_join (h->thread, NULL); ++ pthread_cond_destroy (&h->ctrl.cond); ++ pthread_mutex_destroy (&h->ctrl.lock); ++ command_queue_reset (&h->ctrl.cmds); ++ h->running = false; ++ ++ return 0; ++} ++ ++static void ++scan_close (void *handle) ++{ ++ struct scan_handle *h = handle; ++ ++ free (h); ++} ++ ++/* Read data. */ ++static int ++scan_pread (nbdkit_next *next, ++ void *handle, void *buf, uint32_t count, uint64_t offset, ++ uint32_t flags, int *err) ++{ ++ struct scan_handle *h = handle; ++ ++ if (scan_ahead && h->running) { ++ const struct command cmd = ++ { .type = CMD_NOTIFY_PREAD, .offset = offset + count }; ++ ++ if (send_command_to_background_thread (&h->ctrl, cmd) == -1) ++ return -1; ++ } ++ ++ /* Issue the normal read. */ ++ return next->pread (next, buf, count, offset, flags, err); ++} ++ ++static struct nbdkit_filter filter = { ++ .name = "scan", ++ .longname = "nbdkit scan filter", ++ .get_ready = scan_get_ready, ++ .config = scan_config, ++ .config_complete = scan_config_complete, ++ .config_help = scan_config_help, ++ .open = scan_open, ++ .prepare = scan_prepare, ++ .finalize = scan_finalize, ++ .close = scan_close, ++ .pread = scan_pread, ++}; ++ ++NBDKIT_REGISTER_FILTER(filter) +diff --git a/filters/scan/scan.h b/filters/scan/scan.h +new file mode 100644 +index 00000000..7ff39310 +--- /dev/null ++++ b/filters/scan/scan.h +@@ -0,0 +1,64 @@ ++/* nbdkit ++ * Copyright (C) 2019-2022 Red Hat Inc. ++ * ++ * Redistribution and use in source and binary forms, with or without ++ * modification, are permitted provided that the following conditions are ++ * met: ++ * ++ * * Redistributions of source code must retain the above copyright ++ * notice, this list of conditions and the following disclaimer. ++ * ++ * * Redistributions in binary form must reproduce the above copyright ++ * notice, this list of conditions and the following disclaimer in the ++ * documentation and/or other materials provided with the distribution. ++ * ++ * * Neither the name of Red Hat nor the names of its contributors may be ++ * used to endorse or promote products derived from this software without ++ * specific prior written permission. ++ * ++ * THIS SOFTWARE IS PROVIDED BY RED HAT AND CONTRIBUTORS ''AS IS'' AND ++ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, ++ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A ++ * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL RED HAT OR ++ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, ++ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT ++ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF ++ * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ++ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, ++ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT ++ * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF ++ * SUCH DAMAGE. ++ */ ++ ++#ifndef NBDKIT_SCAN_H ++#define NBDKIT_SCAN_H ++ ++#include ++#include ++ ++#include ++ ++#include "vector.h" ++ ++extern bool scan_clock; ++extern bool scan_forever; ++extern unsigned scan_size; ++ ++/* List of commands issued to the background thread. */ ++struct command { ++ enum { CMD_QUIT, CMD_NOTIFY_PREAD } type; ++ uint64_t offset; ++}; ++DEFINE_VECTOR_TYPE(command_queue, struct command); ++ ++struct bgthread_ctrl { ++ command_queue cmds; /* Command queue. */ ++ pthread_mutex_t lock; /* Lock for queue. */ ++ pthread_cond_t cond; /* Condition queue size 0 -> 1. */ ++ nbdkit_next *next; /* For sending cache operations. */ ++}; ++ ++/* Start background thread (one per connection). */ ++extern void *scan_thread (void *vp); ++ ++#endif /* NBDKIT_SCAN_H */ +diff --git a/plugins/ssh/nbdkit-ssh-plugin.pod b/plugins/ssh/nbdkit-ssh-plugin.pod +index 2bc2c4a7..214957d6 100644 +--- a/plugins/ssh/nbdkit-ssh-plugin.pod ++++ b/plugins/ssh/nbdkit-ssh-plugin.pod +@@ -349,6 +349,7 @@ L, + L, + L, + L, ++L, + L, + L, + L, +diff --git a/plugins/torrent/nbdkit-torrent-plugin.pod b/plugins/torrent/nbdkit-torrent-plugin.pod +index 196ce4e9..f09ac3d2 100644 +--- a/plugins/torrent/nbdkit-torrent-plugin.pod ++++ b/plugins/torrent/nbdkit-torrent-plugin.pod +@@ -175,6 +175,7 @@ L, + L, + L, + L, ++L, + L, + L, + L. +diff --git a/plugins/vddk/nbdkit-vddk-plugin.pod b/plugins/vddk/nbdkit-vddk-plugin.pod +index ea5899dc..3991e86b 100644 +--- a/plugins/vddk/nbdkit-vddk-plugin.pod ++++ b/plugins/vddk/nbdkit-vddk-plugin.pod +@@ -733,6 +733,7 @@ L, + L, + L, + L, ++L, + L, + L, + L, +diff --git a/tests/Makefile.am b/tests/Makefile.am +index 5585b3b7..799aa6c2 100644 +--- a/tests/Makefile.am ++++ b/tests/Makefile.am +@@ -1754,6 +1754,16 @@ test_retry_request_mirror_LDADD = \ + $(LIBNBD_LIBS) \ + $(NULL) + ++# scan filter test. ++TESTS += \ ++ test-scan-copy.sh \ ++ test-scan-info.sh \ ++ $(NULL) ++EXTRA_DIST += \ ++ test-scan-copy.sh \ ++ test-scan-info.sh \ ++ $(NULL) ++ + # swab filter test. + TESTS += \ + test-swab-8.sh \ +diff --git a/tests/test-scan-copy.sh b/tests/test-scan-copy.sh +new file mode 100755 +index 00000000..227ad7b2 +--- /dev/null ++++ b/tests/test-scan-copy.sh +@@ -0,0 +1,42 @@ ++#!/usr/bin/env bash ++# nbdkit ++# Copyright (C) 2018-2022 Red Hat Inc. ++# ++# Redistribution and use in source and binary forms, with or without ++# modification, are permitted provided that the following conditions are ++# met: ++# ++# * Redistributions of source code must retain the above copyright ++# notice, this list of conditions and the following disclaimer. ++# ++# * Redistributions in binary form must reproduce the above copyright ++# notice, this list of conditions and the following disclaimer in the ++# documentation and/or other materials provided with the distribution. ++# ++# * Neither the name of Red Hat nor the names of its contributors may be ++# used to endorse or promote products derived from this software without ++# specific prior written permission. ++# ++# THIS SOFTWARE IS PROVIDED BY RED HAT AND CONTRIBUTORS ''AS IS'' AND ++# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, ++# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A ++# PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL RED HAT OR ++# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, ++# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT ++# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF ++# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ++# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, ++# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT ++# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF ++# SUCH DAMAGE. ++ ++source ./functions.sh ++set -e ++set -x ++ ++requires nbdcopy --version ++requires_plugin sparse-random ++requires_filter scan ++ ++nbdkit -fv -U - sparse-random 1M --filter=scan --run 'nbdcopy "$uri" "$uri"' ++nbdkit -fv -U - sparse-random 1G --filter=scan --run 'nbdcopy "$uri" "$uri"' +diff --git a/tests/test-scan-info.sh b/tests/test-scan-info.sh +new file mode 100755 +index 00000000..6b109ca8 +--- /dev/null ++++ b/tests/test-scan-info.sh +@@ -0,0 +1,46 @@ ++#!/usr/bin/env bash ++# nbdkit ++# Copyright (C) 2018-2022 Red Hat Inc. ++# ++# Redistribution and use in source and binary forms, with or without ++# modification, are permitted provided that the following conditions are ++# met: ++# ++# * Redistributions of source code must retain the above copyright ++# notice, this list of conditions and the following disclaimer. ++# ++# * Redistributions in binary form must reproduce the above copyright ++# notice, this list of conditions and the following disclaimer in the ++# documentation and/or other materials provided with the distribution. ++# ++# * Neither the name of Red Hat nor the names of its contributors may be ++# used to endorse or promote products derived from this software without ++# specific prior written permission. ++# ++# THIS SOFTWARE IS PROVIDED BY RED HAT AND CONTRIBUTORS ''AS IS'' AND ++# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, ++# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A ++# PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL RED HAT OR ++# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, ++# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT ++# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF ++# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ++# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, ++# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT ++# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF ++# SUCH DAMAGE. ++ ++source ./functions.sh ++set -e ++set -x ++ ++requires nbdinfo --version ++requires_filter scan ++ ++# We're just testing that there are no problematic races with the ++# background thread. ++ ++nbdkit -fv -U - memory 1 --filter=scan --run 'nbdinfo $uri' ++nbdkit -fv -U - memory 1M --filter=scan --run 'nbdinfo $uri' ++nbdkit -fv -U - memory 1G --filter=scan --run 'nbdinfo $uri' ++nbdkit -fv -U - memory 1G --filter=scan -e test --run 'nbdinfo $uri' +-- +2.31.1 + diff --git a/0014-scan-Remove-condition-variable.patch b/0014-scan-Remove-condition-variable.patch new file mode 100644 index 0000000..7500254 --- /dev/null +++ b/0014-scan-Remove-condition-variable.patch @@ -0,0 +1,67 @@ +From 91677241184ab1aa77adadd612fa069d084863ec Mon Sep 17 00:00:00 2001 +From: "Richard W.M. Jones" +Date: Sat, 14 May 2022 18:54:32 +0100 +Subject: [PATCH] scan: Remove condition variable + +This was copied in from the readahead filter code, but is not actually +needed in this filter because it never has to sleep waiting for a +command. + +Fixes: commit 65c20a09ceacb4431986a2982f2c2e746df63fcb +(cherry picked from commit 43ad586698347997cdfa1bd56bfed0292f89f134) +--- + filters/scan/scan.c | 6 ------ + filters/scan/scan.h | 1 - + 2 files changed, 7 deletions(-) + +diff --git a/filters/scan/scan.c b/filters/scan/scan.c +index ac5b18d2..8a966577 100644 +--- a/filters/scan/scan.c ++++ b/filters/scan/scan.c +@@ -136,9 +136,6 @@ send_command_to_background_thread (struct bgthread_ctrl *ctrl, + ACQUIRE_LOCK_FOR_CURRENT_SCOPE (&ctrl->lock); + if (command_queue_append (&ctrl->cmds, cmd) == -1) + return -1; +- /* Signal the thread if it could be sleeping on an empty queue. */ +- if (ctrl->cmds.len == 1) +- pthread_cond_signal (&ctrl->cond); + return 0; + } + +@@ -199,13 +196,11 @@ scan_prepare (nbdkit_next *next, void *handle, int readonly) + /* Create the background thread. */ + h->ctrl.cmds = (command_queue) empty_vector; + pthread_mutex_init (&h->ctrl.lock, NULL); +- pthread_cond_init (&h->ctrl.cond, NULL); + + err = pthread_create (&h->thread, NULL, scan_thread, &h->ctrl); + if (err != 0) { + errno = err; + nbdkit_error ("pthread_create: %m"); +- pthread_cond_destroy (&h->ctrl.cond); + pthread_mutex_destroy (&h->ctrl.lock); + return -1; + } +@@ -227,7 +222,6 @@ scan_finalize (nbdkit_next *next, void *handle) + + send_command_to_background_thread (&h->ctrl, quit_cmd); + pthread_join (h->thread, NULL); +- pthread_cond_destroy (&h->ctrl.cond); + pthread_mutex_destroy (&h->ctrl.lock); + command_queue_reset (&h->ctrl.cmds); + h->running = false; +diff --git a/filters/scan/scan.h b/filters/scan/scan.h +index 7ff39310..98c0228b 100644 +--- a/filters/scan/scan.h ++++ b/filters/scan/scan.h +@@ -54,7 +54,6 @@ DEFINE_VECTOR_TYPE(command_queue, struct command); + struct bgthread_ctrl { + command_queue cmds; /* Command queue. */ + pthread_mutex_t lock; /* Lock for queue. */ +- pthread_cond_t cond; /* Condition queue size 0 -> 1. */ + nbdkit_next *next; /* For sending cache operations. */ + }; + +-- +2.31.1 + diff --git a/0015-scan-Small-typographical-fix-in-manual.patch b/0015-scan-Small-typographical-fix-in-manual.patch new file mode 100644 index 0000000..c2c6ae0 --- /dev/null +++ b/0015-scan-Small-typographical-fix-in-manual.patch @@ -0,0 +1,57 @@ +From c191f45530d4dd7f978803c0bfa402ca0fc950df Mon Sep 17 00:00:00 2001 +From: "Richard W.M. Jones" +Date: Sat, 14 May 2022 19:02:48 +0100 +Subject: [PATCH] scan: Small typographical fix in manual + +Fixes: commit 65c20a09ceacb4431986a2982f2c2e746df63fcb +(cherry picked from commit 67d4e3437d2e28fa3ce1c4b3818d2b1e7939c5ec) +--- + filters/scan/nbdkit-scan-filter.pod | 12 ++++++------ + 1 file changed, 6 insertions(+), 6 deletions(-) + +diff --git a/filters/scan/nbdkit-scan-filter.pod b/filters/scan/nbdkit-scan-filter.pod +index 4a8d0ef9..2fe9bb80 100644 +--- a/filters/scan/nbdkit-scan-filter.pod ++++ b/filters/scan/nbdkit-scan-filter.pod +@@ -26,8 +26,8 @@ below (after) this filter, giving approximately the same effect. + L can be used instead of nbdkit-cache-filter, if + you add the C option. + +-Various C parameters can be used to tune scanning, although +-the defaults should be suitable in most cases. ++Various parameters can be used to tune scanning, although the defaults ++should be suitable in most cases. + + A similar filter is L. + +@@ -38,23 +38,23 @@ filter will print a warning message if this happens. + + =over 4 + +-=item Thread model must be parallel * ++=item Thread model must be parallel* + + For example L only supports + C, and so this filter cannot perform prefetches in + parallel with the read requests. + +-=item Only scans while clients are connected * ++=item Only scans while clients are connected* + + The current filter only scans while there is at least one client + connected. + +-=item Only scans the default export * ++=item Only scans the default export* + + The current filter only scans the default export and ignores all + clients connecting to the non-default export name. + +-* We may be able to lift these restrictions in future. ++*We may be able to lift these restrictions in future. + + =item Underlying filters or plugin must support C<.cache> (prefetch) + +-- +2.31.1 + diff --git a/0016-ssh-Don-t-reference-readahead-or-scan-filters-from-t.patch b/0016-ssh-Don-t-reference-readahead-or-scan-filters-from-t.patch new file mode 100644 index 0000000..d6e387b --- /dev/null +++ b/0016-ssh-Don-t-reference-readahead-or-scan-filters-from-t.patch @@ -0,0 +1,34 @@ +From 651045d703804d7dafab04a0387ca92573f52467 Mon Sep 17 00:00:00 2001 +From: "Richard W.M. Jones" +Date: Sat, 14 May 2022 20:57:38 +0100 +Subject: [PATCH] ssh: Don't reference readahead or scan filters from this + plugin + +These filters do not support plugins which do not use the parallel +thread model. + +Fixes: commit 2ff548d66ad3eae87868402ec5b3319edd12090f +Fixes: commit 65c20a09ceacb4431986a2982f2c2e746df63fcb +See-also: commit 92fbb76d11b9f17c527debd803aa2505f3642783 +(cherry picked from commit 7eb356719376c4d0b2379cea5d39c81602d2d304) +--- + plugins/ssh/nbdkit-ssh-plugin.pod | 2 -- + 1 file changed, 2 deletions(-) + +diff --git a/plugins/ssh/nbdkit-ssh-plugin.pod b/plugins/ssh/nbdkit-ssh-plugin.pod +index 214957d6..bb922d37 100644 +--- a/plugins/ssh/nbdkit-ssh-plugin.pod ++++ b/plugins/ssh/nbdkit-ssh-plugin.pod +@@ -347,9 +347,7 @@ C first appeared in nbdkit 1.12. + L, + L, + L, +-L, + L, +-L, + L, + L, + L, +-- +2.31.1 + diff --git a/0017-scan-Fix-bound-so-we-don-t-try-to-prefetch-beyond-en.patch b/0017-scan-Fix-bound-so-we-don-t-try-to-prefetch-beyond-en.patch new file mode 100644 index 0000000..f78049c --- /dev/null +++ b/0017-scan-Fix-bound-so-we-don-t-try-to-prefetch-beyond-en.patch @@ -0,0 +1,56 @@ +From f58d2a04338edc647e2334ff58b49508424e3f3b Mon Sep 17 00:00:00 2001 +From: "Richard W.M. Jones" +Date: Tue, 17 May 2022 13:20:17 +0100 +Subject: [PATCH] scan: Fix bound so we don't try to prefetch beyond end of + disk + +An off-by-one error in the bound could cause the filter to try to +prefetch beyond the end of the underlying plugin. This would cause +nbdkit to crash with this assertion failure: + +nbdkit: backend.c:782: backend_cache: Assertion `backend_valid_range (c, offset, count)' failed. + +The sequence of events was: + + - scan filter background thread started + + - client reads to the end of the disk + + - background thread skips ahead to end of disk (offset == size) + + - background thread tries to prefetch from this point + +In the final step the calculations caused to the background thread to +prefetch a scan-size block beyond the end of the plugin. + +Fixes: commit 65c20a09ceacb4431986a2982f2c2e746df63fcb +(cherry picked from commit 953643429b8c57b4dd20a6c0e5b83704ae9a0e88) +--- + filters/scan/bgthread.c | 10 +++++----- + 1 file changed, 5 insertions(+), 5 deletions(-) + +diff --git a/filters/scan/bgthread.c b/filters/scan/bgthread.c +index 384e79b6..5fa5f27f 100644 +--- a/filters/scan/bgthread.c ++++ b/filters/scan/bgthread.c +@@ -113,12 +113,12 @@ scan_thread (void *vp) + } + + adjust_clock (offset); +- if (offset > size) +- continue; + +- /* Issue the next prefetch. */ +- n = MIN (scan_size, size - offset); +- ctrl->next->cache (ctrl->next, n, offset, 0, NULL); ++ if (offset < size) { ++ /* Issue the next prefetch. */ ++ n = MIN (scan_size, size - offset); ++ ctrl->next->cache (ctrl->next, n, offset, 0, NULL); ++ } + } + + if (scan_forever) { +-- +2.31.1 + diff --git a/0018-tests-Add-a-regression-test-for-LUKS-zeroing-crash.patch b/0018-tests-Add-a-regression-test-for-LUKS-zeroing-crash.patch new file mode 100644 index 0000000..123c091 --- /dev/null +++ b/0018-tests-Add-a-regression-test-for-LUKS-zeroing-crash.patch @@ -0,0 +1,110 @@ +From d1d2f43223bcda062d10c8e68776590956892f71 Mon Sep 17 00:00:00 2001 +From: "Richard W.M. Jones" +Date: Fri, 10 Jun 2022 22:11:44 +0100 +Subject: [PATCH] tests: Add a regression test for LUKS zeroing crash + +https://listman.redhat.com/archives/libguestfs/2022-June/029188.html +(cherry picked from commit 7ab2ef96803bfc385f786be82ebfdd4cc977d504) +--- + tests/Makefile.am | 2 ++ + tests/test-luks-copy-zero.sh | 70 ++++++++++++++++++++++++++++++++++++ + 2 files changed, 72 insertions(+) + create mode 100755 tests/test-luks-copy-zero.sh + +diff --git a/tests/Makefile.am b/tests/Makefile.am +index 799aa6c2..0f4b0746 100644 +--- a/tests/Makefile.am ++++ b/tests/Makefile.am +@@ -1601,11 +1601,13 @@ if HAVE_GNUTLS_PBKDF2 + TESTS += \ + test-luks-info.sh \ + test-luks-copy.sh \ ++ test-luks-copy-zero.sh \ + $(NULL) + endif + EXTRA_DIST += \ + test-luks-info.sh \ + test-luks-copy.sh \ ++ test-luks-copy-zero.sh \ + $(NULL) + + # multi-conn filter test. +diff --git a/tests/test-luks-copy-zero.sh b/tests/test-luks-copy-zero.sh +new file mode 100755 +index 00000000..6ff560e3 +--- /dev/null ++++ b/tests/test-luks-copy-zero.sh +@@ -0,0 +1,70 @@ ++#!/usr/bin/env bash ++# nbdkit ++# Copyright (C) 2018-2022 Red Hat Inc. ++# ++# Redistribution and use in source and binary forms, with or without ++# modification, are permitted provided that the following conditions are ++# met: ++# ++# * Redistributions of source code must retain the above copyright ++# notice, this list of conditions and the following disclaimer. ++# ++# * Redistributions in binary form must reproduce the above copyright ++# notice, this list of conditions and the following disclaimer in the ++# documentation and/or other materials provided with the distribution. ++# ++# * Neither the name of Red Hat nor the names of its contributors may be ++# used to endorse or promote products derived from this software without ++# specific prior written permission. ++# ++# THIS SOFTWARE IS PROVIDED BY RED HAT AND CONTRIBUTORS ''AS IS'' AND ++# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, ++# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A ++# PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL RED HAT OR ++# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, ++# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT ++# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF ++# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ++# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, ++# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT ++# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF ++# SUCH DAMAGE. ++ ++# Regression test for: ++# https://listman.redhat.com/archives/libguestfs/2022-June/029188.html ++ ++source ./functions.sh ++set -e ++set -x ++ ++requires qemu-img --version ++requires nbdcopy --version ++requires truncate --version ++requires file --version ++requires_filter luks ++ ++encrypt_disk=luks-copy-zero1.img ++zero_disk=luks-copy-zero2.img ++cleanup_fn rm -f $encrypt_disk $zero_disk ++rm -f $encrypt_disk $zero_disk ++ ++# Create an empty encrypted disk container. ++qemu-img create -f luks \ ++ --object secret,data=123456,id=sec0 \ ++ -o key-secret=sec0 \ ++ $encrypt_disk 100M ++ ++# Create an all zeroes disk of the same size. ++truncate -s 100M $zero_disk ++ ++# Using nbdkit-luks-filter, write the zero disk into the encrypted ++# disk. nbdcopy will do this using NBD_CMD_ZERO operations. ++nbdkit -U - -fv \ ++ file $encrypt_disk --filter=luks passphrase=123456 \ ++ --run "nbdcopy -C 1 $zero_disk \$nbd" ++ ++# Check that the encrypted disk is still a LUKS disk. If zeroing is ++# wrong in the filter it's possible that it writes through to the ++# underlying disk, erasing the container. ++file $encrypt_disk ++file $encrypt_disk | grep "LUKS encrypted file" +-- +2.31.1 + diff --git a/0019-rate-Allow-burstiness-to-be-controlled.patch b/0019-rate-Allow-burstiness-to-be-controlled.patch new file mode 100644 index 0000000..3453169 --- /dev/null +++ b/0019-rate-Allow-burstiness-to-be-controlled.patch @@ -0,0 +1,121 @@ +From c1a7c87fb9710fb29d699d1f39d0da19caf98da0 Mon Sep 17 00:00:00 2001 +From: "Richard W.M. Jones" +Date: Sat, 11 Jun 2022 12:34:02 +0100 +Subject: [PATCH] rate: Allow burstiness to be controlled + +Previously it was fixed at 2.0 seconds. Allowing it to be adjusted +upwards could help with large, lumpy requests. + +(cherry picked from commit f79e951c20510381d5cd83c203c670874a4978f4) +--- + filters/rate/nbdkit-rate-filter.pod | 12 ++++++++++-- + filters/rate/rate.c | 20 +++++++++++++------- + tests/test-rate.sh | 2 +- + 3 files changed, 24 insertions(+), 10 deletions(-) + +diff --git a/filters/rate/nbdkit-rate-filter.pod b/filters/rate/nbdkit-rate-filter.pod +index 8956e641..09ce7dbc 100644 +--- a/filters/rate/nbdkit-rate-filter.pod ++++ b/filters/rate/nbdkit-rate-filter.pod +@@ -9,6 +9,7 @@ nbdkit-rate-filter - limit bandwidth by connection or server + [connection-rate=BITSPERSEC] + [rate-file=FILENAME] + [connection-rate-file=FILENAME] ++ [burstiness=SECS] + + =head1 DESCRIPTION + +@@ -63,6 +64,13 @@ Limit total bandwidth across all connections to C. + Adjust the per-connection or total bandwidth dynamically by writing + C into C. See L below. + ++=item BSECS ++ ++Control the bucket capacity, expressed as a length of time in ++"rate-equivalent seconds" that the client is allowed to burst for ++after a period of inactivity. The default is 2.0 seconds. It's not ++recommended to set this smaller than the default. ++ + =back + + C can be specified as a simple number, or you can use a +@@ -105,8 +113,8 @@ If the size of requests made by your client is much larger than the + rate limit then you can see long, lumpy sleeps in this filter. In the + future we may modify the filter to break up large requests + automatically in order to limit the length of sleeps. Placing the +-L in front of this filter may help in the +-meantime. ++L in front of this filter, or adjusting ++C upwards may help. + + =head1 FILES + +diff --git a/filters/rate/rate.c b/filters/rate/rate.c +index 1a70d212..26082f8c 100644 +--- a/filters/rate/rate.c ++++ b/filters/rate/rate.c +@@ -68,10 +68,9 @@ static char *rate_file = NULL; + + /* Bucket capacity controls the burst rate. It is expressed as the + * length of time in "rate-equivalent seconds" that the client can +- * burst for after a period of inactivity. This could be adjustable +- * in future. ++ * burst for after a period of inactivity. + */ +-#define BUCKET_CAPACITY 2.0 ++static double bucket_capacity = 2.0 /* seconds */; + + /* Global read and write buckets. */ + static struct bucket read_bucket; +@@ -142,6 +141,13 @@ rate_config (nbdkit_next_config *next, nbdkit_backend *nxdata, + return -1; + return 0; + } ++ else if (strcmp (key, "burstiness") == 0) { ++ if (sscanf (value, "%lg", &bucket_capacity) != 1) { ++ nbdkit_error ("burstiness must be a floating point number (seconds)"); ++ return -1; ++ } ++ return 0; ++ } + else + return next (nxdata, key, value); + } +@@ -150,8 +156,8 @@ static int + rate_get_ready (int thread_model) + { + /* Initialize the global buckets. */ +- bucket_init (&read_bucket, rate, BUCKET_CAPACITY); +- bucket_init (&write_bucket, rate, BUCKET_CAPACITY); ++ bucket_init (&read_bucket, rate, bucket_capacity); ++ bucket_init (&write_bucket, rate, bucket_capacity); + + return 0; + } +@@ -178,8 +184,8 @@ rate_open (nbdkit_next_open *next, nbdkit_context *nxdata, + return NULL; + } + +- bucket_init (&h->read_bucket, connection_rate, BUCKET_CAPACITY); +- bucket_init (&h->write_bucket, connection_rate, BUCKET_CAPACITY); ++ bucket_init (&h->read_bucket, connection_rate, bucket_capacity); ++ bucket_init (&h->write_bucket, connection_rate, bucket_capacity); + pthread_mutex_init (&h->read_bucket_lock, NULL); + pthread_mutex_init (&h->write_bucket_lock, NULL); + +diff --git a/tests/test-rate.sh b/tests/test-rate.sh +index 7305c928..ff781c21 100755 +--- a/tests/test-rate.sh ++++ b/tests/test-rate.sh +@@ -56,7 +56,7 @@ nbdkit -U - \ + --filter=blocksize --filter=rate \ + pattern 25M \ + maxdata=65536 \ +- rate=10M \ ++ rate=10M burstiness=2.0 \ + --run 'nbdcopy "$uri" rate.img' + end_t=$SECONDS + +-- +2.31.1 + diff --git a/0020-luks-Check-return-values-from-malloc-more-carefully.patch b/0020-luks-Check-return-values-from-malloc-more-carefully.patch new file mode 100644 index 0000000..f4a8d86 --- /dev/null +++ b/0020-luks-Check-return-values-from-malloc-more-carefully.patch @@ -0,0 +1,104 @@ +From 4e8599886ba4802fef1683811a725e7c4bc4fe72 Mon Sep 17 00:00:00 2001 +From: "Richard W.M. Jones" +Date: Tue, 12 Jul 2022 18:00:38 +0100 +Subject: [PATCH] luks: Check return values from malloc more carefully + +Found by Coverity: + + Error: GCC_ANALYZER_WARNING (CWE-688): [#def53] + nbdkit-1.30.7/filters/luks/luks-encryption.c: scope_hint: In function 'calculate_iv' + nbdkit-1.30.7/filters/luks/luks-encryption.c:175:5: warning[-Wanalyzer-possible-null-argument]: use of possibly-NULL 'iv' where non-null expected + nbdkit-1.30.7/filters/luks/luks-encryption.c:39: included_from: Included from here. + /usr/include/string.h:43:14: note: argument 1 of 'memcpy' must be non-null + # 173| sector32 = (uint32_t) sector; /* truncate to only lower bits */ + # 174| sector32 = htole32 (sector32); + # 175|-> memcpy (iv, §or32, prefixlen); + # 176| memset (iv + prefixlen, 0, ivlen - prefixlen); + # 177| break; + + Error: GCC_ANALYZER_WARNING (CWE-688): [#def54] + nbdkit-1.30.7/filters/luks/luks-encryption.c:184:5: warning[-Wanalyzer-possible-null-argument]: use of possibly-NULL 'iv' where non-null expected + nbdkit-1.30.7/filters/luks/luks-encryption.c:39: included_from: Included from here. + /usr/include/string.h:43:14: note: argument 1 of 'memcpy' must be non-null + # 182| prefixlen = ivlen; + # 183| sector = htole64 (sector); + # 184|-> memcpy (iv, §or, prefixlen); + # 185| memset (iv + prefixlen, 0, ivlen - prefixlen); + # 186| break; + + Error: NULL_RETURNS (CWE-476): [#def55] + nbdkit-1.30.7/filters/luks/luks-encryption.c:498: returned_null: "malloc" returns "NULL" (checked 86 out of 94 times). + nbdkit-1.30.7/filters/luks/luks-encryption.c:498: var_assigned: Assigning: "temp" = "NULL" return value from "malloc". + nbdkit-1.30.7/filters/luks/luks-encryption.c:523: dereference: Dereferencing a pointer that might be "NULL" "temp" when calling "memcpy". [Note: The source code implementation of the function has been overridden by a builtin model.] + # 521| gnutls_hash_deinit (hash, temp); + # 522| + # 523|-> memcpy (&block[i*digest_bytes], temp, blen); + # 524| } + # 525| + +Fixes: commit 468919dce6c5eb57503eacac0f67e5dd87c58e6c +(cherry picked from commit 00c8bbd9e321681843140f697985505de7177f34) +--- + filters/luks/luks-encryption.c | 28 +++++++++++++++++++++++----- + 1 file changed, 23 insertions(+), 5 deletions(-) + +diff --git a/filters/luks/luks-encryption.c b/filters/luks/luks-encryption.c +index 8ee0eb35..19aaf06a 100644 +--- a/filters/luks/luks-encryption.c ++++ b/filters/luks/luks-encryption.c +@@ -495,9 +495,15 @@ af_hash (gnutls_digest_algorithm_t hash_alg, uint8_t *block, size_t len) + size_t digest_bytes = gnutls_hash_get_len (hash_alg); + size_t nr_blocks, last_block_len; + size_t i; +- CLEANUP_FREE uint8_t *temp = malloc (digest_bytes); + int r; + gnutls_hash_hd_t hash; ++ CLEANUP_FREE uint8_t *temp; ++ ++ temp = malloc (digest_bytes); ++ if (!temp) { ++ nbdkit_error ("malloc: %m"); ++ return -1; ++ } + + nr_blocks = len / digest_bytes; + last_block_len = len % digest_bytes; +@@ -874,9 +880,15 @@ int + do_decrypt (struct luks_data *h, gnutls_cipher_hd_t cipher, + uint64_t sector, uint8_t *buf, size_t nr_sectors) + { +- const size_t ivlen = cipher_alg_iv_len (h->cipher_alg, h->cipher_mode); +- CLEANUP_FREE uint8_t *iv = malloc (ivlen); + int r; ++ const size_t ivlen = cipher_alg_iv_len (h->cipher_alg, h->cipher_mode); ++ CLEANUP_FREE uint8_t *iv; ++ ++ iv = malloc (ivlen); ++ if (!iv) { ++ nbdkit_error ("malloc: %m"); ++ return -1; ++ } + + while (nr_sectors) { + calculate_iv (h->ivgen_alg, iv, ivlen, sector); +@@ -902,9 +914,15 @@ int + do_encrypt (struct luks_data *h, gnutls_cipher_hd_t cipher, + uint64_t sector, uint8_t *buf, size_t nr_sectors) + { +- const size_t ivlen = cipher_alg_iv_len (h->cipher_alg, h->cipher_mode); +- CLEANUP_FREE uint8_t *iv = malloc (ivlen); + int r; ++ const size_t ivlen = cipher_alg_iv_len (h->cipher_alg, h->cipher_mode); ++ CLEANUP_FREE uint8_t *iv; ++ ++ iv = malloc (ivlen); ++ if (!iv) { ++ nbdkit_error ("malloc: %m"); ++ return -1; ++ } + + while (nr_sectors) { + calculate_iv (h->ivgen_alg, iv, ivlen, sector); +-- +2.31.1 + diff --git a/0021-luks-Avoid-potential-overflow-when-computing-key-mat.patch b/0021-luks-Avoid-potential-overflow-when-computing-key-mat.patch new file mode 100644 index 0000000..9c95dce --- /dev/null +++ b/0021-luks-Avoid-potential-overflow-when-computing-key-mat.patch @@ -0,0 +1,57 @@ +From 1d593a76796574845d7e32aaadd9f7d1ed4e7987 Mon Sep 17 00:00:00 2001 +From: "Richard W.M. Jones" +Date: Tue, 12 Jul 2022 18:07:25 +0100 +Subject: [PATCH] luks: Avoid potential overflow when computing key material + offset and length + +Found by Coverity: + + Error: OVERFLOW_BEFORE_WIDEN (CWE-190): [#def58] + nbdkit-1.30.7/filters/luks/luks-encryption.c:558: overflow_before_widen: Potentially overflowing expression "h->phdr.master_key_len * h->phdr.keyslot[i].stripes" with type "unsigned int" (32 bits, unsigned) is evaluated using 32-bit arithmetic, and then used in a context that expects an expression of type "uint64_t" (64 bits, unsigned). + nbdkit-1.30.7/filters/luks/luks-encryption.c:558: remediation: To avoid overflow, cast either "h->phdr.master_key_len" or "h->phdr.keyslot[i].stripes" to type "uint64_t". + # 556| uint64_t len, r; + # 557| + # 558|-> len = h->phdr.master_key_len * h->phdr.keyslot[i].stripes; + # 559| r = DIV_ROUND_UP (len, LUKS_SECTOR_SIZE); + # 560| r = ROUND_UP (r, LUKS_ALIGN_KEYSLOTS / LUKS_SECTOR_SIZE); + + Error: OVERFLOW_BEFORE_WIDEN (CWE-190): [#def62] + nbdkit-1.30.7/filters/luks/luks-encryption.c:616: overflow_before_widen: Potentially overflowing expression "ks->key_material_offset * 512U" with type "unsigned int" (32 bits, unsigned) is evaluated using 32-bit arithmetic, and then used in a context that expects an expression of type "uint64_t" (64 bits, unsigned). + nbdkit-1.30.7/filters/luks/luks-encryption.c:616: remediation: To avoid overflow, cast either "ks->key_material_offset" or "512U" to type "uint64_t". + # 614| + # 615| /* Read master key material from plugin. */ + # 616|-> start = ks->key_material_offset * LUKS_SECTOR_SIZE; + # 617| if (next->pread (next, split_key, split_key_len, start, 0, &err) == -1) { + # 618| errno = err; + +Fixes: commit 468919dce6c5eb57503eacac0f67e5dd87c58e6c +(cherry picked from commit 808d88fbc7b58b7c95e05f41fec729cba92ef518) +--- + filters/luks/luks-encryption.c | 4 ++-- + 1 file changed, 2 insertions(+), 2 deletions(-) + +diff --git a/filters/luks/luks-encryption.c b/filters/luks/luks-encryption.c +index 19aaf06a..06435b27 100644 +--- a/filters/luks/luks-encryption.c ++++ b/filters/luks/luks-encryption.c +@@ -561,7 +561,7 @@ key_material_length_in_sectors (struct luks_data *h, size_t i) + { + uint64_t len, r; + +- len = h->phdr.master_key_len * h->phdr.keyslot[i].stripes; ++ len = (uint64_t) h->phdr.master_key_len * h->phdr.keyslot[i].stripes; + r = DIV_ROUND_UP (len, LUKS_SECTOR_SIZE); + r = ROUND_UP (r, LUKS_ALIGN_KEYSLOTS / LUKS_SECTOR_SIZE); + return r; +@@ -619,7 +619,7 @@ try_passphrase_in_keyslot (nbdkit_next *next, struct luks_data *h, + } + + /* Read master key material from plugin. */ +- start = ks->key_material_offset * LUKS_SECTOR_SIZE; ++ start = (uint64_t) ks->key_material_offset * LUKS_SECTOR_SIZE; + if (next->pread (next, split_key, split_key_len, start, 0, &err) == -1) { + errno = err; + return -1; +-- +2.31.1 + diff --git a/0022-luks-Avoid-memory-leak-on-error-path.patch b/0022-luks-Avoid-memory-leak-on-error-path.patch new file mode 100644 index 0000000..22a2708 --- /dev/null +++ b/0022-luks-Avoid-memory-leak-on-error-path.patch @@ -0,0 +1,36 @@ +From ee25c1be953bf385caf23f96384a9834c1f1c250 Mon Sep 17 00:00:00 2001 +From: "Richard W.M. Jones" +Date: Tue, 12 Jul 2022 18:10:30 +0100 +Subject: [PATCH] luks: Avoid memory leak on error path + +Found by Coverity: + + Error: CPPCHECK_WARNING (CWE-401): [#def65] [important] + nbdkit-1.30.7/filters/luks/luks-encryption.c:707: error[memleak]: Memory leak: h + # 705| if (memcmp (h->phdr.magic, expected_magic, LUKS_MAGIC_LEN) != 0) { + # 706| nbdkit_error ("this disk does not contain a LUKS header"); + # 707|-> return NULL; + # 708| } + # 709| h->phdr.version = be16toh (h->phdr.version); + +Fixes: commit 468919dce6c5eb57503eacac0f67e5dd87c58e6c +(cherry picked from commit a345cff137763f105f07bb8942c1bbefd0959cff) +--- + filters/luks/luks-encryption.c | 1 + + 1 file changed, 1 insertion(+) + +diff --git a/filters/luks/luks-encryption.c b/filters/luks/luks-encryption.c +index 06435b27..207a4e46 100644 +--- a/filters/luks/luks-encryption.c ++++ b/filters/luks/luks-encryption.c +@@ -710,6 +710,7 @@ load_header (nbdkit_next *next, const char *passphrase) + + if (memcmp (h->phdr.magic, expected_magic, LUKS_MAGIC_LEN) != 0) { + nbdkit_error ("this disk does not contain a LUKS header"); ++ free (h); + return NULL; + } + h->phdr.version = be16toh (h->phdr.version); +-- +2.31.1 + diff --git a/0023-tests-Hoist-some-EXTRA_DIST-out-of-automake-conditio.patch b/0023-tests-Hoist-some-EXTRA_DIST-out-of-automake-conditio.patch new file mode 100644 index 0000000..0bfe9b4 --- /dev/null +++ b/0023-tests-Hoist-some-EXTRA_DIST-out-of-automake-conditio.patch @@ -0,0 +1,48 @@ +From 5ccf1068703d300c8b5579b3a6ef0e409b5a713e Mon Sep 17 00:00:00 2001 +From: "Richard W.M. Jones" +Date: Tue, 19 Jul 2022 11:56:47 +0100 +Subject: [PATCH] tests: Hoist some EXTRA_DIST out of automake conditionals + +We can fail to add some test files (test.tcl, test.lua) to the tarball +if compiling with those languages disabled, which would cause knock-on +failures when the tarball was used with the languages enabled. We +already fixed this for Ruby etc, this commit fixes it for Tcl and Lua. + +(cherry picked from commit 3b6763c82909c95431ff57c2fe9be1b98316b057) +--- + tests/Makefile.am | 6 ++++-- + 1 file changed, 4 insertions(+), 2 deletions(-) + +diff --git a/tests/Makefile.am b/tests/Makefile.am +index 0f4b0746..2667be32 100644 +--- a/tests/Makefile.am ++++ b/tests/Makefile.am +@@ -1203,10 +1203,11 @@ EXTRA_DIST += \ + $(NULL) + + # Tcl plugin test. ++EXTRA_DIST += test.tcl ++ + if HAVE_TCL + + LIBGUESTFS_TESTS += test-tcl +-EXTRA_DIST += test.tcl + + test_tcl_SOURCES = test-lang-plugins.c test.h + test_tcl_CFLAGS = \ +@@ -1219,10 +1220,11 @@ test_tcl_LDADD = libtest.la $(LIBGUESTFS_LIBS) + endif HAVE_TCL + + # Lua plugin test. ++EXTRA_DIST += test.lua ++ + if HAVE_LUA + + LIBGUESTFS_TESTS += test-lua +-EXTRA_DIST += test.lua + + test_lua_SOURCES = test-lang-plugins.c test.h + test_lua_CFLAGS = \ +-- +2.31.1 + diff --git a/copy-patches.sh b/copy-patches.sh new file mode 100755 index 0000000..299454a --- /dev/null +++ b/copy-patches.sh @@ -0,0 +1,55 @@ +#!/bin/bash - + +set -e + +# Maintainer script to copy patches from the git repo to the current +# directory. Use it like this: +# ./copy-patches.sh + +rhel_version=8.6 + +# Check we're in the right directory. +if [ ! -f nbdkit.spec ]; then + echo "$0: run this from the directory containing 'nbdkit.spec'" + exit 1 +fi + +git_checkout=$HOME/d/nbdkit-rhel-$rhel_version +if [ ! -d $git_checkout ]; then + echo "$0: $git_checkout does not exist" + echo "This script is only for use by the maintainer when preparing a" + echo "nbdkit release on RHEL." + exit 1 +fi + +# Get the base version of nbdkit. +version=`grep '^Version:' nbdkit.spec | awk '{print $2}'` +tag="v$version" + +# Remove any existing patches. +git rm -f [0-9]*.patch ||: +rm -f [0-9]*.patch + +# Get the patches. +(cd $git_checkout; rm -f [0-9]*.patch; git format-patch -N $tag) +mv $git_checkout/[0-9]*.patch . + +# Remove any not to be applied. +rm -f *NOT-FOR-RPM*.patch + +# Add the patches. +git add [0-9]*.patch + +# Print out the patch lines. +echo +echo "--- Copy the following text into nbdkit.spec file" +echo + +echo "# Patches." +for f in [0-9]*.patch; do + n=`echo $f | awk -F- '{print $1}'` + echo "Patch$n: $f" +done + +echo +echo "--- End of text" diff --git a/gating.yaml b/gating.yaml new file mode 100755 index 0000000..648918d --- /dev/null +++ b/gating.yaml @@ -0,0 +1,6 @@ +--- !Policy +product_versions: + - rhel-9 +decision_context: osci_compose_gate +rules: + - !PassingTestCaseRule {test_case_name: osci.brew-build.tier0.functional} diff --git a/libguestfs.keyring b/libguestfs.keyring new file mode 100644 index 0000000..f0508c8 Binary files /dev/null and b/libguestfs.keyring differ diff --git a/nbdkit-1.24.0.tar.gz.sig b/nbdkit-1.24.0.tar.gz.sig new file mode 100644 index 0000000..14ff4e8 --- /dev/null +++ b/nbdkit-1.24.0.tar.gz.sig @@ -0,0 +1,17 @@ +-----BEGIN PGP SIGNATURE----- + +iQJFBAABCAAvFiEE93dPsa0HSn6Mh2fqkXOPc+G3aKAFAl/3RBgRHHJpY2hAYW5u +ZXhpYS5vcmcACgkQkXOPc+G3aKBIIRAAmgoGrmJ8aYO7z+kKgNFjd/p0QxRTZhS/ +ol59ojG6jIzN2x/C2PFbRmPB6HJTEg4anrDX04WrP6R+lID1RrH9pTFQabv0YDQC +z49oeXAqINYHvAqgFUJCwlymd7BHEYUudLlK3yu7gQKxMM+J/2v0glpxrtLM7KlD +vvSZkVfbvHlCWIbMWLWIaRHeoWZIXNOjsAp3uEWN2YgikDoxbXVKoh07JoQx5tJ5 +2U+a/zo4BQuRspjnhmWc252ZF/8d954/L8J+2mKvbRRf2iAmsqPgS+MNi7WKWO4K +w7/urKn0osuOaArs5xYHJnApmJ9U88CzZpoHQkYhcGgnDOipW9ByJRzT41vVQPW5 +IluQODpZUuawWtRIwV/Eoi+LaV2gINAL48Afr02UFYj4gmYQ5TeayLP7NKRQO0VL +jwL4Z3a0cDyUX4i1OArn2ll8THfiog38HfLb70AG1l3P1BVoVVBYWCYbs4xgC9IK +LWkjPKuGXvkGVfZi0nCGdPTOoB1CqCXUvKHXm52FCHg12uJMrBQEivodBoCTbtl0 +fSjULQcfrovUEb4d/rDAX7EgJbFS+1jDnodaFHsmNToo3CqfkMBdhLkxG3XExwjy +OOR34wZssjTLsLlWH/RPucWD25RDy1vdPBska9QvvO7W0p+aOtFbnttkTh5cqs45 +rHg/sDEiaLA= +=OrsS +-----END PGP SIGNATURE----- diff --git a/nbdkit-find-provides b/nbdkit-find-provides new file mode 100755 index 0000000..7013ccd --- /dev/null +++ b/nbdkit-find-provides @@ -0,0 +1,23 @@ +#!/bin/bash - + +# Generate RPM provides automatically for nbdkit packages and filters. +# Copyright (C) 2009-2022 Red Hat Inc. + +# To test: +# find /usr/lib64/nbdkit/plugins | ./nbdkit-find-provides VER REL +# find /usr/lib64/nbdkit/filters | ./nbdkit-find-provides VER REL + +ver="$1" +rel="$2" + +function process_file +{ + if [[ $1 =~ /plugins/nbdkit-.*-plugin ]] || + [[ $1 =~ /filters/nbdkit-.*-filter ]]; then + echo "Provides:" "$(basename $1 .so)" "=" "$ver-$rel" + fi +} + +while read line; do + process_file "$line" +done diff --git a/nbdkit.attr b/nbdkit.attr new file mode 100644 index 0000000..2757679 --- /dev/null +++ b/nbdkit.attr @@ -0,0 +1,3 @@ +%__nbdkit_provides %{_rpmconfigdir}/nbdkit-find-provides %{version} %{release} +%__nbdkit_path %{_libdir}/nbdkit/(plugins|filters)/nbdkit-.*-(plugin|filter)(\.so)?$ +%__nbdkit_flags exeonly diff --git a/nbdkit.spec b/nbdkit.spec new file mode 100644 index 0000000..ec31876 --- /dev/null +++ b/nbdkit.spec @@ -0,0 +1,1569 @@ +%global _hardened_build 1 + +%ifarch aarch64 %{arm} x86_64 ppc %{power64} +%global have_libguestfs 1 +%endif + +# We can only compile the OCaml plugin on platforms which have native +# OCaml support (not bytecode). +%ifarch %{ocaml_native_compiler} +%global have_ocaml 1 +%endif + +# Architectures where the complete test suite must pass. +# +# On all other architectures, a simpler test suite must pass. This +# omits any tests that run full qemu, since running qemu under TCG is +# often broken on non-x86_64 arches. +%global complete_test_arches x86_64 + +# If the test suite is broken on a particular architecture, document +# it as a bug and add it to this list. +%global broken_test_arches aarch64 + +%if 0%{?rhel} == 7 +# On RHEL 7, nothing in the virt stack is shipped on aarch64 and +# libguestfs was not shipped on POWER (fixed in 7.5). We could in +# theory make all of this work by having lots more conditionals, but +# for now limit this package to x86_64 on RHEL. +ExclusiveArch: x86_64 +%endif + +# If we should verify tarball signature with GPGv2. +%global verify_tarball_signature 1 + +# If there are patches which touch autotools files, set this to 1. +%global patches_touch_autotools %{nil} + +# The source directory. +%global source_directory 1.24-stable + +Name: nbdkit +Version: 1.24.0 +Release: 4%{?dist} +Summary: NBD server + +License: BSD +URL: https://github.com/libguestfs/nbdkit + +%if 0%{?rhel} >= 8 +# On RHEL 8+, we cannot build the package on i686 (no virt stack). +ExcludeArch: i686 +%endif + +Source0: http://libguestfs.org/download/nbdkit/%{source_directory}/%{name}-%{version}.tar.gz +%if 0%{verify_tarball_signature} +Source1: http://libguestfs.org/download/nbdkit/%{source_directory}/%{name}-%{version}.tar.gz.sig +# Keyring used to verify tarball signature. +Source2: libguestfs.keyring +%endif + +# Maintainer script which helps with handling patches. +Source3: copy-patches.sh + +# Patches come from this upstream branch: +# https://github.com/libguestfs/nbdkit/tree/rhel-8.6 + +# Patches. +Patch0001: 0001-cache-cow-Fix-data-corruption-in-zero-and-trim-on-un.patch +Patch0002: 0002-server-CVE-2021-3716-reset-structured-replies-on-sta.patch +Patch0003: 0003-server-reset-meta-context-replies-on-starttls.patch +Patch0004: 0004-cow-Fix-for-qemu-6.1-which-requires-backing-format.patch + +%if 0%{patches_touch_autotools} +BuildRequires: autoconf, automake, libtool +%endif + +%ifnarch %{complete_test_arches} +BuildRequires: autoconf, automake, libtool +%endif +BuildRequires: gcc, gcc-c++ +BuildRequires: /usr/bin/pod2man +BuildRequires: gnutls-devel +BuildRequires: libselinux-devel +%if !0%{?rhel} && 0%{?have_libguestfs} +BuildRequires: libguestfs-devel +%endif +BuildRequires: libvirt-devel +BuildRequires: xz-devel +BuildRequires: zlib-devel +BuildRequires: libzstd-devel +BuildRequires: libcurl-devel +BuildRequires: libnbd-devel >= 1.3.11 +BuildRequires: libssh-devel +BuildRequires: e2fsprogs, e2fsprogs-devel +%if !0%{?rhel} +BuildRequires: genisoimage +BuildRequires: rb_libtorrent-devel +%endif +BuildRequires: bash-completion +BuildRequires: perl-devel +BuildRequires: perl(ExtUtils::Embed) +%if !0%{?rhel} +BuildRequires: python3-devel +%else +BuildRequires: platform-python-devel +%endif +%if !0%{?rhel} +%if 0%{?have_ocaml} +# Requires OCaml 4.02.2 which contains fix for +# http://caml.inria.fr/mantis/view.php?id=6693 +BuildRequires: ocaml >= 4.02.2 +BuildRequires: ocaml-ocamldoc +%endif +BuildRequires: ruby-devel +BuildRequires: tcl-devel +BuildRequires: lua-devel +%endif +%if 0%{verify_tarball_signature} +BuildRequires: gnupg2 +%endif + +# Only for running the test suite: +BuildRequires: %{_bindir}/bc +BuildRequires: %{_bindir}/certtool +BuildRequires: %{_bindir}/cut +BuildRequires: expect +BuildRequires: %{_bindir}/hexdump +BuildRequires: %{_sbindir}/ip +BuildRequires: jq +BuildRequires: %{_bindir}/nbdcopy +BuildRequires: %{_bindir}/nbdinfo +BuildRequires: %{_bindir}/nbdsh +BuildRequires: %{_bindir}/qemu-img +BuildRequires: %{_bindir}/qemu-io +BuildRequires: %{_bindir}/qemu-nbd +BuildRequires: %{_sbindir}/sfdisk +BuildRequires: %{_bindir}/socat +BuildRequires: %{_sbindir}/ss +BuildRequires: %{_bindir}/stat +BuildRequires: %{_bindir}/ssh-keygen + +# nbdkit is a metapackage pulling the server and a useful subset +# of the plugins and filters. +Requires: nbdkit-server%{?_isa} = %{version}-%{release} +Requires: nbdkit-basic-plugins%{?_isa} = %{version}-%{release} +Requires: nbdkit-basic-filters%{?_isa} = %{version}-%{release} + + +%description +NBD is a protocol for accessing block devices (hard disks and +disk-like things) over the network. + +nbdkit is a toolkit for creating NBD servers. + +The key features are: + +* Multithreaded NBD server written in C with good performance. + +* Minimal dependencies for the basic server. + +* Liberal license (BSD) allows nbdkit to be linked to proprietary + libraries or included in proprietary code. + +* Well-documented, simple plugin API with a stable ABI guarantee. + Lets you to export "unconventional" block devices easily. + +* You can write plugins in C or many other languages. + +* Filters can be stacked in front of plugins to transform the output. + +'%{name}' is a meta-package which pulls in the core server and a +useful subset of plugins and filters with minimal dependencies. + +If you want just the server, install '%{name}-server'. + +To develop plugins, install the '%{name}-devel' package and start by +reading the nbdkit(1) and nbdkit-plugin(3) manual pages. + + +%package server +Summary: The %{name} server +License: BSD + + +%description server +This package contains the %{name} server with no plugins or filters. + + +%package basic-plugins +Summary: Basic plugins for %{name} +License: BSD + +Requires: %{name}-server%{?_isa} = %{version}-%{release} +Provides: %{name}-data-plugin = %{version}-%{release} +Provides: %{name}-eval-plugin = %{version}-%{release} +Provides: %{name}-file-plugin = %{version}-%{release} +Provides: %{name}-floppy-plugin = %{version}-%{release} +Provides: %{name}-full-plugin = %{version}-%{release} +Provides: %{name}-info-plugin = %{version}-%{release} +Provides: %{name}-memory-plugin = %{version}-%{release} +Provides: %{name}-null-plugin = %{version}-%{release} +Provides: %{name}-ondemand-plugin = %{version}-%{release} +Provides: %{name}-pattern-plugin = %{version}-%{release} +Provides: %{name}-partitioning-plugin = %{version}-%{release} +Provides: %{name}-random-plugin = %{version}-%{release} +Provides: %{name}-sh-plugin = %{version}-%{release} +Provides: %{name}-sparse-random-plugin = %{version}-%{release} +Provides: %{name}-split-plugin = %{version}-%{release} +Provides: %{name}-streaming-plugin = %{version}-%{release} +Provides: %{name}-zero-plugin = %{version}-%{release} + + +%description basic-plugins +This package contains plugins for %{name} which only depend on simple +C libraries: glibc, gnutls, libzstd. Other plugins for nbdkit with +more complex dependencies are packaged separately. + +nbdkit-data-plugin Serve small amounts of data from the command line. + +nbdkit-eval-plugin Write a shell script plugin on the command line. + +nbdkit-file-plugin The normal file plugin for serving files. + +nbdkit-floppy-plugin Create a virtual floppy disk from a directory. + +nbdkit-full-plugin A virtual disk that returns ENOSPC errors. + +nbdkit-info-plugin Serve client and server information. + +nbdkit-memory-plugin A virtual memory plugin. + +nbdkit-null-plugin A null (bitbucket) plugin. + +nbdkit-ondemand-plugin Create filesystems on demand. + +nbdkit-pattern-plugin Fixed test pattern. + +nbdkit-partitioning-plugin Create virtual disks from partitions. + +nbdkit-random-plugin Random content plugin for testing. + +nbdkit-sh-plugin Write plugins as shell scripts or executables. + +nbdkit-sparse-random-plugin Make sparse random disks. + +nbdkit-split-plugin Concatenate one or more files. + +nbdkit-streaming-plugin A streaming file serving plugin. + +nbdkit-zero-plugin Zero-length plugin for testing. + + +%package example-plugins +Summary: Example plugins for %{name} +License: BSD + +Requires: %{name}-server%{?_isa} = %{version}-%{release} +%if !0%{?rhel} +# example4 is written in Perl. +Requires: %{name}-perl-plugin +%endif + + +%description example-plugins +This package contains example plugins for %{name}. + + +# The plugins below have non-trivial dependencies are so are +# packaged separately. + +%if !0%{?rhel} +%package cc-plugin +Summary: Write small inline C plugins and scripts for %{name} +License: BSD + +Requires: %{name}-server%{?_isa} = %{version}-%{release} +Requires: gcc +Requires: %{_bindir}/cat + + +%description cc-plugin +This package contains support for writing inline C plugins and scripts +for %{name}. NOTE this is NOT the right package for writing plugins +in C, install %{name}-devel for that. +%endif + + +%if !0%{?rhel} +%package cdi-plugin +Summary: Containerized Data Import plugin for %{name} +License: BSD + +Requires: %{name}-server%{?_isa} = %{version}-%{release} +Requires: jq +Requires: podman + + +%description cdi-plugin +This package contains Containerized Data Import support for %{name}. +%endif + + +%package curl-plugin +Summary: HTTP/FTP (cURL) plugin for %{name} +License: BSD + +Requires: %{name}-server%{?_isa} = %{version}-%{release} + + +%description curl-plugin +This package contains cURL (HTTP/FTP) support for %{name}. + + +%if !0%{?rhel} && 0%{?have_libguestfs} +%package guestfs-plugin +Summary: libguestfs plugin for %{name} +License: BSD + +Requires: %{name}-server%{?_isa} = %{version}-%{release} + + +%description guestfs-plugin +This package is a libguestfs plugin for %{name}. +%endif + + +%if 0%{?rhel} == 8 +%package gzip-plugin +Summary: GZip plugin for %{name} +License: BSD + +Requires: %{name}-server%{?_isa} = %{version}-%{release} + + +%description gzip-plugin +This package is a gzip plugin for %{name}. +%endif + + +%if !0%{?rhel} +%package iso-plugin +Summary: Virtual ISO 9660 plugin for %{name} +License: BSD + +Requires: %{name}-server%{?_isa} = %{version}-%{release} +Requires: genisoimage + + +%description iso-plugin +This package is a virtual ISO 9660 (CD-ROM) plugin for %{name}. +%endif + + +%if !0%{?rhel} +%package libvirt-plugin +Summary: Libvirt plugin for %{name} +License: BSD + +Requires: %{name}-server%{?_isa} = %{version}-%{release} + + +%description libvirt-plugin +This package is a libvirt plugin for %{name}. It lets you access +libvirt guest disks readonly. It is implemented using the libvirt +virDomainBlockPeek API. +%endif + + +%package linuxdisk-plugin +Summary: Virtual Linux disk plugin for %{name} +License: BSD + +Requires: %{name}-server%{?_isa} = %{version}-%{release} +# for mke2fs +Requires: e2fsprogs + + +%description linuxdisk-plugin +This package is a virtual Linux disk plugin for %{name}. + + +%if !0%{?rhel} +%package lua-plugin +Summary: Lua plugin for %{name} +License: BSD + +Requires: %{name}-server%{?_isa} = %{version}-%{release} + + +%description lua-plugin +This package lets you write Lua plugins for %{name}. +%endif + + +%package nbd-plugin +Summary: NBD proxy / forward plugin for %{name} +License: BSD + +Requires: %{name}-server%{?_isa} = %{version}-%{release} + + +%description nbd-plugin +This package lets you forward NBD connections from %{name} +to another NBD server. + + +%if !0%{?rhel} && 0%{?have_ocaml} +%package ocaml-plugin +Summary: OCaml plugin for %{name} +License: BSD + +Requires: %{name}-server%{?_isa} = %{version}-%{release} + + +%description ocaml-plugin +This package lets you run OCaml plugins for %{name}. + +To compile OCaml plugins you will also need to install +%{name}-ocaml-plugin-devel. + + +%package ocaml-plugin-devel +Summary: OCaml development environment for %{name} +License: BSD + +Requires: %{name}-server%{?_isa} = %{version}-%{release} +Requires: %{name}-ocaml-plugin%{?_isa} = %{version}-%{release} + + +%description ocaml-plugin-devel +This package lets you write OCaml plugins for %{name}. +%endif + + +%if !0%{?rhel} +%package perl-plugin +Summary: Perl plugin for %{name} +License: BSD + +Requires: %{name}-server%{?_isa} = %{version}-%{release} + + +%description perl-plugin +This package lets you write Perl plugins for %{name}. +%endif + + +%package python-plugin +Summary: Python 3 plugin for %{name} +License: BSD + +Requires: %{name}-server%{?_isa} = %{version}-%{release} + + +%description python-plugin +This package lets you write Python 3 plugins for %{name}. + + +%if !0%{?rhel} +%package ruby-plugin +Summary: Ruby plugin for %{name} +License: BSD + +Requires: %{name}-server%{?_isa} = %{version}-%{release} + + +%description ruby-plugin +This package lets you write Ruby plugins for %{name}. +%endif + + +%package ssh-plugin +Summary: SSH plugin for %{name} +License: BSD + +Requires: %{name}-server%{?_isa} = %{version}-%{release} + + +%description ssh-plugin +This package contains SSH support for %{name}. + + +%package tar-plugin +Summary: Tar archive plugin for %{name} +License: BSD + +Requires: %{name}-server%{?_isa} = %{version}-%{release} +Requires: tar + + +%description tar-plugin +This package is a tar archive plugin for %{name}. + + +%if !0%{?rhel} +%package tcl-plugin +Summary: Tcl plugin for %{name} +License: BSD + +Requires: %{name}-server%{?_isa} = %{version}-%{release} + + +%description tcl-plugin +This package lets you write Tcl plugins for %{name}. +%endif + + +%package tmpdisk-plugin +Summary: Remote temporary filesystem disk plugin for %{name} +License: BSD + +Requires: %{name}-server%{?_isa} = %{version}-%{release} +# For mkfs and mke2fs (defaults). +Requires: util-linux, e2fsprogs +# For other filesystems. +Suggests: xfsprogs +%if !0%{?rhel} +Suggests: ntfsprogs, dosfstools +%endif + + +%description tmpdisk-plugin +This package is a remote temporary filesystem disk plugin for %{name}. + + +%if !0%{?rhel} +%package torrent-plugin +Summary: BitTorrent plugin for %{name} +License: BSD + +Requires: %{name}-server%{?_isa} = %{version}-%{release} + + +%description torrent-plugin +This package is a BitTorrent plugin for %{name}. +%endif + + +%ifarch x86_64 +%package vddk-plugin +Summary: VMware VDDK plugin for %{name} +License: BSD + +Requires: %{name}-server%{?_isa} = %{version}-%{release} + + +%description vddk-plugin +This package is a plugin for %{name} which connects to +VMware VDDK for accessing VMware disks and servers. +%endif + + +%package basic-filters +Summary: Basic filters for %{name} +License: BSD + +Requires: %{name}-server%{?_isa} = %{version}-%{release} +Provides: %{name}-blocksize-filter = %{version}-%{release} +Provides: %{name}-cache-filter = %{version}-%{release} +Provides: %{name}-cacheextents-filter = %{version}-%{release} +Provides: %{name}-checkwrite-filter = %{version}-%{release} +Provides: %{name}-cow-filter = %{version}-%{release} +Provides: %{name}-ddrescue-filter = %{version}-%{release} +Provides: %{name}-delay-filter = %{version}-%{release} +Provides: %{name}-error-filter = %{version}-%{release} +Provides: %{name}-exitlast-filter = %{version}-%{release} +Provides: %{name}-exitwhen-filter = %{version}-%{release} +Provides: %{name}-exportname-filter = %{version}-%{release} +Provides: %{name}-extentlist-filter = %{version}-%{release} +Provides: %{name}-fua-filter = %{version}-%{release} +Provides: %{name}-ip-filter = %{version}-%{release} +Provides: %{name}-limit-filter = %{version}-%{release} +Provides: %{name}-log-filter = %{version}-%{release} +Provides: %{name}-nocache-filter = %{version}-%{release} +Provides: %{name}-noextents-filter = %{version}-%{release} +Provides: %{name}-nofilter-filter = %{version}-%{release} +Provides: %{name}-noparallel-filter = %{version}-%{release} +Provides: %{name}-nozero-filter = %{version}-%{release} +Provides: %{name}-offset-filter = %{version}-%{release} +Provides: %{name}-partition-filter = %{version}-%{release} +Provides: %{name}-pause-filter = %{version}-%{release} +Provides: %{name}-rate-filter = %{version}-%{release} +Provides: %{name}-readahead-filter = %{version}-%{release} +Provides: %{name}-retry-filter = %{version}-%{release} +Provides: %{name}-stats-filter = %{version}-%{release} +Provides: %{name}-swab-filter = %{version}-%{release} +Provides: %{name}-tls-fallback-filter = %{version}-%{release} +Provides: %{name}-truncate-filter = %{version}-%{release} + + +%description basic-filters +This package contains filters for %{name} which only depend on simple +C libraries: glibc, gnutls. Other filters for nbdkit with more +complex dependencies are packaged separately. + +nbdkit-blocksize-filter Adjust block size of requests sent to plugins. + +nbdkit-cache-filter Server-side cache. + +nbdkit-cacheextents-filter Cache extents. + +nbdkit-checkwrite-filter Check writes match contents of plugin. + +nbdkit-cow-filter Copy-on-write overlay for read-only plugins. + +nbdkit-ddrescue-filter Filter for serving from ddrescue dump. + +nbdkit-delay-filter Inject read and write delays. + +nbdkit-error-filter Inject errors. + +nbdkit-exitlast-filter Exit on last client connection. + +nbdkit-exitwhen-filter Exit gracefully when an event occurs. + +nbdkit-exportname-filter Adjust export names between client and plugin. + +nbdkit-extentlist-filter Place extent list over a plugin. + +nbdkit-fua-filter Modify flush behaviour in plugins. + +nbdkit-ip-filter Filter clients by IP address. + +nbdkit-limit-filter Limit nr clients that can connect concurrently. + +nbdkit-log-filter Log all transactions to a file. + +nbdkit-nocache-filter Disable cache requests in the underlying plugin. + +nbdkit-noextents-filter Disable extents in the underlying plugin. + +nbdkit-nofilter-filter Passthrough filter. + +nbdkit-noparallel-filter Serialize requests to the underlying plugin. + +nbdkit-nozero-filter Adjust handling of zero requests by plugins. + +nbdkit-offset-filter Serve an offset and range. + +nbdkit-partition-filter Serve a single partition. + +nbdkit-pause-filter Pause NBD requests. + +nbdkit-rate-filter Limit bandwidth by connection or server. + +nbdkit-readahead-filter Prefetch data when reading sequentially. + +nbdkit-retry-filter Reopen connection on error. + +nbdkit-stats-filter Display statistics about operations. + +nbdkit-swab-filter Filter for swapping byte order. + +nbdkit-tls-fallback-filter TLS protection filter. + +nbdkit-truncate-filter Truncate, expand, round up or round down size. + + +%if !0%{?rhel} +%package ext2-filter +Summary: ext2, ext3 and ext4 filesystem support for %{name} +License: BSD + +Requires: %{name}-server%{?_isa} = %{version}-%{release} + +# Remove in Fedora 34: +Provides: %{name}-ext2-plugin = %{version}-%{release} +Obsoletes: %{name}-ext2-plugin <= %{version}-%{release} + + +%description ext2-filter +This package contains ext2, ext3 and ext4 filesystem support for +%{name}. +%endif + + +%package gzip-filter +Summary: GZip filter for %{name} +License: BSD + +Requires: %{name}-server%{?_isa} = %{version}-%{release} + + +%description gzip-filter +This package is a gzip filter for %{name}. + + +%package tar-filter +Summary: Tar archive filter for %{name} +License: BSD + +Requires: %{name}-server%{?_isa} = %{version}-%{release} +Requires: tar + + +%description tar-filter +This package is a tar archive filter for %{name}. + + +%package xz-filter +Summary: XZ filter for %{name} +License: BSD + +Requires: %{name}-server%{?_isa} = %{version}-%{release} + + +%description xz-filter +This package is the xz filter for %{name}. + + +%package devel +Summary: Development files and documentation for %{name} +License: BSD + +Requires: %{name}-server%{?_isa} = %{version}-%{release} +Requires: pkgconfig + + +%description devel +This package contains development files and documentation +for %{name}. Install this package if you want to develop +plugins for %{name}. + + +%package bash-completion +Summary: Bash tab-completion for %{name} +BuildArch: noarch +Requires: bash-completion >= 2.0 +Requires: %{name}-server = %{version}-%{release} + + +%description bash-completion +Install this package if you want intelligent bash tab-completion +for %{name}. + + +%prep +%if 0%{verify_tarball_signature} +tmphome="$(mktemp -d)" +gpgv2 --homedir "$tmphome" --keyring %{SOURCE2} %{SOURCE1} %{SOURCE0} +%endif +%autosetup -p1 +%if 0%{patches_touch_autotools} +autoreconf -i +%endif + +%ifnarch %{complete_test_arches} +# Simplify the test suite so it doesn't require qemu. +sed -i -e 's/^LIBGUESTFS_TESTS/xLIBGUESTFS_TESTS/' tests/Makefile.am +sed -i -e '/^if HAVE_GUESTFISH/,/^endif HAVE_GUESTFISH/d' tests/Makefile.am +autoreconf -i +%endif + + +%build +# Golang bindings are not enabled in the build since they don't +# need to be. Most people would use them by copying the upstream +# package into their vendor/ directory. +# +# %%{__python3} expands to platform-python, so we don't depend on +# the python module (see RHBZ#1659159, RHBZ#1867964). +export PYTHON=%{__python3} +%configure \ + --with-extra='%{name}-%{version}-%{release}' \ + --disable-static \ + --disable-golang \ + --disable-rust \ +%if !0%{?rhel} && 0%{?have_ocaml} + --enable-ocaml \ +%else + --disable-ocaml \ +%endif +%if 0%{?rhel} + --disable-lua \ + --disable-perl \ + --disable-ruby \ + --disable-tcl \ + --without-ext2 \ + --without-iso \ + --without-libvirt \ +%endif +%if !0%{?rhel} && 0%{?have_libguestfs} + --with-libguestfs \ +%else + --without-libguestfs \ +%endif + --with-tls-priority=@NBDKIT,SYSTEM + +# Verify that it picked the correct version of Python +# to avoid RHBZ#1404631 happening again silently. +grep '^PYTHON_VERSION = 3' Makefile + +%make_build + + +%install +%make_install + +# Delete libtool crap. +find $RPM_BUILD_ROOT -name '*.la' -delete + +# If cargo happens to be installed on the machine then the +# rust plugin is built. Delete it if this happens. +rm -f $RPM_BUILD_ROOT%{_mandir}/man3/nbdkit-rust-plugin.3* + +%if 0%{?rhel} != 8 +# Remove the deprecated gzip plugin (use gzip filter instead). +rm $RPM_BUILD_ROOT%{_libdir}/%{name}/plugins/nbdkit-gzip-plugin.so +rm $RPM_BUILD_ROOT%{_mandir}/man1/nbdkit-gzip-plugin.1* +%endif + +%if 0%{?rhel} +# In RHEL, remove some plugins we cannot --disable. +for f in cc cdi torrent; do + rm -f $RPM_BUILD_ROOT%{_libdir}/%{name}/plugins/nbdkit-$f-plugin.so + rm -f $RPM_BUILD_ROOT%{_mandir}/man?/nbdkit-$f-plugin.* +done +rm -f $RPM_BUILD_ROOT%{_libdir}/%{name}/plugins/nbdkit-S3-plugin +rm -f $RPM_BUILD_ROOT%{_mandir}/man1/nbdkit-S3-plugin.1* +%endif + + +%check +%ifnarch %{broken_test_arches} +# Workaround for broken libvirt (RHBZ#1138604). +mkdir -p $HOME/.cache/libvirt + +# tests/test-captive.sh is racy especially on s390x. We need to +# rethink this test upstream. +truncate -s 0 tests/test-captive.sh + +%ifarch s390x +# Temporarily kill tests/test-cache-max-size.sh since it fails +# sometimes on s390x for unclear reasons. +truncate -s 0 tests/test-cache-max-size.sh +%endif + +# Temporarily kill test-nbd-tls.sh and test-nbd-tls-psk.sh +# https://www.redhat.com/archives/libguestfs/2020-March/msg00191.html +truncate -s 0 tests/test-nbd-tls.sh tests/test-nbd-tls-psk.sh + +# Kill tests/test-cc-ocaml.sh. Requires upstream fix (commit bce54e7df25). +truncate -s 0 tests/test-cc-ocaml.sh + +# Make sure we can see the debug messages (RHBZ#1230160). +export LIBGUESTFS_DEBUG=1 +export LIBGUESTFS_TRACE=1 + +%make_build check || { + cat tests/test-suite.log + exit 1 + } +%endif + + +%if 0%{?have_ocaml} +%ldconfig_scriptlets plugin-ocaml +%endif + + +%files +# metapackage so empty + + +%files server +%doc README +%license LICENSE +%{_sbindir}/nbdkit +%dir %{_libdir}/%{name} +%dir %{_libdir}/%{name}/plugins +%dir %{_libdir}/%{name}/filters +%{_mandir}/man1/nbdkit.1* +%{_mandir}/man1/nbdkit-captive.1* +%{_mandir}/man1/nbdkit-client.1* +%{_mandir}/man1/nbdkit-loop.1* +%{_mandir}/man1/nbdkit-probing.1* +%{_mandir}/man1/nbdkit-protocol.1* +%{_mandir}/man1/nbdkit-service.1* +%{_mandir}/man1/nbdkit-security.1* +%{_mandir}/man1/nbdkit-tls.1* + + +%files basic-plugins +%doc README +%license LICENSE +%{_libdir}/%{name}/plugins/nbdkit-data-plugin.so +%{_libdir}/%{name}/plugins/nbdkit-eval-plugin.so +%{_libdir}/%{name}/plugins/nbdkit-file-plugin.so +%{_libdir}/%{name}/plugins/nbdkit-floppy-plugin.so +%{_libdir}/%{name}/plugins/nbdkit-full-plugin.so +%{_libdir}/%{name}/plugins/nbdkit-info-plugin.so +%{_libdir}/%{name}/plugins/nbdkit-memory-plugin.so +%{_libdir}/%{name}/plugins/nbdkit-null-plugin.so +%{_libdir}/%{name}/plugins/nbdkit-ondemand-plugin.so +%{_libdir}/%{name}/plugins/nbdkit-partitioning-plugin.so +%{_libdir}/%{name}/plugins/nbdkit-pattern-plugin.so +%{_libdir}/%{name}/plugins/nbdkit-random-plugin.so +%{_libdir}/%{name}/plugins/nbdkit-sh-plugin.so +%{_libdir}/%{name}/plugins/nbdkit-sparse-random-plugin.so +%{_libdir}/%{name}/plugins/nbdkit-split-plugin.so +%{_libdir}/%{name}/plugins/nbdkit-streaming-plugin.so +%{_libdir}/%{name}/plugins/nbdkit-zero-plugin.so +%{_mandir}/man1/nbdkit-data-plugin.1* +%{_mandir}/man1/nbdkit-eval-plugin.1* +%{_mandir}/man1/nbdkit-file-plugin.1* +%{_mandir}/man1/nbdkit-floppy-plugin.1* +%{_mandir}/man1/nbdkit-full-plugin.1* +%{_mandir}/man1/nbdkit-info-plugin.1* +%{_mandir}/man1/nbdkit-memory-plugin.1* +%{_mandir}/man1/nbdkit-null-plugin.1* +%{_mandir}/man1/nbdkit-ondemand-plugin.1* +%{_mandir}/man1/nbdkit-partitioning-plugin.1* +%{_mandir}/man1/nbdkit-pattern-plugin.1* +%{_mandir}/man1/nbdkit-random-plugin.1* +%{_mandir}/man3/nbdkit-sh-plugin.3* +%{_mandir}/man1/nbdkit-sparse-random-plugin.1* +%{_mandir}/man1/nbdkit-split-plugin.1* +%{_mandir}/man1/nbdkit-streaming-plugin.1* +%{_mandir}/man1/nbdkit-zero-plugin.1* + + +%files example-plugins +%doc README +%license LICENSE +%{_libdir}/%{name}/plugins/nbdkit-example*-plugin.so +%if !0%{?rhel} +%{_libdir}/%{name}/plugins/nbdkit-example4-plugin +%endif +%{_mandir}/man1/nbdkit-example*-plugin.1* + + +%if !0%{?rhel} +%files cc-plugin +%doc README +%license LICENSE +%{_libdir}/%{name}/plugins/nbdkit-cc-plugin.so +%{_mandir}/man3/nbdkit-cc-plugin.3* +%endif + + +%if !0%{?rhel} +%files cdi-plugin +%doc README +%license LICENSE +%{_libdir}/%{name}/plugins/nbdkit-cdi-plugin.so +%{_mandir}/man1/nbdkit-cdi-plugin.1* +%endif + + +%files curl-plugin +%doc README +%license LICENSE +%{_libdir}/%{name}/plugins/nbdkit-curl-plugin.so +%{_mandir}/man1/nbdkit-curl-plugin.1* + + +%if !0%{?rhel} && 0%{?have_libguestfs} +%files guestfs-plugin +%doc README +%license LICENSE +%{_libdir}/%{name}/plugins/nbdkit-guestfs-plugin.so +%{_mandir}/man1/nbdkit-guestfs-plugin.1* +%endif + + +%if 0%{?rhel} == 8 +%files gzip-plugin +%doc README +%license LICENSE +%{_libdir}/%{name}/plugins/nbdkit-gzip-plugin.so +%{_mandir}/man1/nbdkit-gzip-plugin.1* +%endif + + +%if !0%{?rhel} +%files iso-plugin +%doc README +%license LICENSE +%{_libdir}/%{name}/plugins/nbdkit-iso-plugin.so +%{_mandir}/man1/nbdkit-iso-plugin.1* +%endif + + +%if !0%{?rhel} +%files libvirt-plugin +%doc README +%license LICENSE +%{_libdir}/%{name}/plugins/nbdkit-libvirt-plugin.so +%{_mandir}/man1/nbdkit-libvirt-plugin.1* +%endif + + +%files linuxdisk-plugin +%doc README +%license LICENSE +%{_libdir}/%{name}/plugins/nbdkit-linuxdisk-plugin.so +%{_mandir}/man1/nbdkit-linuxdisk-plugin.1* + + +%if !0%{?rhel} +%files lua-plugin +%doc README +%license LICENSE +%{_libdir}/%{name}/plugins/nbdkit-lua-plugin.so +%{_mandir}/man3/nbdkit-lua-plugin.3* +%endif + + +%files nbd-plugin +%doc README +%license LICENSE +%{_libdir}/%{name}/plugins/nbdkit-nbd-plugin.so +%{_mandir}/man1/nbdkit-nbd-plugin.1* + + +%if !0%{?rhel} && 0%{?have_ocaml} +%files ocaml-plugin +%doc README +%license LICENSE +%{_libdir}/libnbdkitocaml.so.* + +%files ocaml-plugin-devel +%{_libdir}/libnbdkitocaml.so +%{_libdir}/ocaml/NBDKit.* +%{_mandir}/man3/nbdkit-ocaml-plugin.3* +%{_mandir}/man3/NBDKit.3* +%endif + + +%if !0%{?rhel} +%files perl-plugin +%doc README +%license LICENSE +%{_libdir}/%{name}/plugins/nbdkit-perl-plugin.so +%{_mandir}/man3/nbdkit-perl-plugin.3* +%endif + + +%files python-plugin +%doc README +%license LICENSE +%{_libdir}/%{name}/plugins/nbdkit-python-plugin.so +%{_mandir}/man3/nbdkit-python-plugin.3* + + +%if !0%{?rhel} +%files ruby-plugin +%doc README +%license LICENSE +%{_libdir}/%{name}/plugins/nbdkit-ruby-plugin.so +%{_mandir}/man3/nbdkit-ruby-plugin.3* +%endif + + +%files ssh-plugin +%doc README +%license LICENSE +%{_libdir}/%{name}/plugins/nbdkit-ssh-plugin.so +%{_mandir}/man1/nbdkit-ssh-plugin.1* + + +%files tar-plugin +%doc README +%license LICENSE +%{_libdir}/%{name}/plugins/nbdkit-tar-plugin.so +%{_mandir}/man1/nbdkit-tar-plugin.1* + + +%if !0%{?rhel} +%files tcl-plugin +%doc README +%license LICENSE +%{_libdir}/%{name}/plugins/nbdkit-tcl-plugin.so +%{_mandir}/man3/nbdkit-tcl-plugin.3* +%endif + + +%files tmpdisk-plugin +%doc README +%license LICENSE +%{_libdir}/%{name}/plugins/nbdkit-tmpdisk-plugin.so +%{_mandir}/man1/nbdkit-tmpdisk-plugin.1* + + +%if !0%{?rhel} +%files torrent-plugin +%doc README +%license LICENSE +%{_libdir}/%{name}/plugins/nbdkit-torrent-plugin.so +%{_mandir}/man1/nbdkit-torrent-plugin.1* +%endif + + +%ifarch x86_64 +%files vddk-plugin +%doc README +%license LICENSE +%{_libdir}/%{name}/plugins/nbdkit-vddk-plugin.so +%{_mandir}/man1/nbdkit-vddk-plugin.1* +%endif + + +%files basic-filters +%doc README +%license LICENSE +%{_libdir}/%{name}/filters/nbdkit-blocksize-filter.so +%{_libdir}/%{name}/filters/nbdkit-cache-filter.so +%{_libdir}/%{name}/filters/nbdkit-cacheextents-filter.so +%{_libdir}/%{name}/filters/nbdkit-checkwrite-filter.so +%{_libdir}/%{name}/filters/nbdkit-cow-filter.so +%{_libdir}/%{name}/filters/nbdkit-ddrescue-filter.so +%{_libdir}/%{name}/filters/nbdkit-delay-filter.so +%{_libdir}/%{name}/filters/nbdkit-error-filter.so +%{_libdir}/%{name}/filters/nbdkit-exitlast-filter.so +%{_libdir}/%{name}/filters/nbdkit-exitwhen-filter.so +%{_libdir}/%{name}/filters/nbdkit-exportname-filter.so +%{_libdir}/%{name}/filters/nbdkit-extentlist-filter.so +%{_libdir}/%{name}/filters/nbdkit-fua-filter.so +%{_libdir}/%{name}/filters/nbdkit-ip-filter.so +%{_libdir}/%{name}/filters/nbdkit-limit-filter.so +%{_libdir}/%{name}/filters/nbdkit-log-filter.so +%{_libdir}/%{name}/filters/nbdkit-nocache-filter.so +%{_libdir}/%{name}/filters/nbdkit-noextents-filter.so +%{_libdir}/%{name}/filters/nbdkit-nofilter-filter.so +%{_libdir}/%{name}/filters/nbdkit-noparallel-filter.so +%{_libdir}/%{name}/filters/nbdkit-nozero-filter.so +%{_libdir}/%{name}/filters/nbdkit-offset-filter.so +%{_libdir}/%{name}/filters/nbdkit-partition-filter.so +%{_libdir}/%{name}/filters/nbdkit-pause-filter.so +%{_libdir}/%{name}/filters/nbdkit-rate-filter.so +%{_libdir}/%{name}/filters/nbdkit-readahead-filter.so +%{_libdir}/%{name}/filters/nbdkit-retry-filter.so +%{_libdir}/%{name}/filters/nbdkit-stats-filter.so +%{_libdir}/%{name}/filters/nbdkit-swab-filter.so +%{_libdir}/%{name}/filters/nbdkit-tls-fallback-filter.so +%{_libdir}/%{name}/filters/nbdkit-truncate-filter.so +%{_mandir}/man1/nbdkit-blocksize-filter.1* +%{_mandir}/man1/nbdkit-cache-filter.1* +%{_mandir}/man1/nbdkit-cacheextents-filter.1* +%{_mandir}/man1/nbdkit-checkwrite-filter.1* +%{_mandir}/man1/nbdkit-cow-filter.1* +%{_mandir}/man1/nbdkit-ddrescue-filter.1* +%{_mandir}/man1/nbdkit-delay-filter.1* +%{_mandir}/man1/nbdkit-error-filter.1* +%{_mandir}/man1/nbdkit-exitlast-filter.1* +%{_mandir}/man1/nbdkit-exitwhen-filter.1* +%{_mandir}/man1/nbdkit-exportname-filter.1* +%{_mandir}/man1/nbdkit-extentlist-filter.1* +%{_mandir}/man1/nbdkit-fua-filter.1* +%{_mandir}/man1/nbdkit-ip-filter.1* +%{_mandir}/man1/nbdkit-limit-filter.1* +%{_mandir}/man1/nbdkit-log-filter.1* +%{_mandir}/man1/nbdkit-nocache-filter.1* +%{_mandir}/man1/nbdkit-noextents-filter.1* +%{_mandir}/man1/nbdkit-nofilter-filter.1* +%{_mandir}/man1/nbdkit-noparallel-filter.1* +%{_mandir}/man1/nbdkit-nozero-filter.1* +%{_mandir}/man1/nbdkit-offset-filter.1* +%{_mandir}/man1/nbdkit-partition-filter.1* +%{_mandir}/man1/nbdkit-pause-filter.1* +%{_mandir}/man1/nbdkit-rate-filter.1* +%{_mandir}/man1/nbdkit-readahead-filter.1* +%{_mandir}/man1/nbdkit-retry-filter.1* +%{_mandir}/man1/nbdkit-stats-filter.1* +%{_mandir}/man1/nbdkit-swab-filter.1* +%{_mandir}/man1/nbdkit-tls-fallback-filter.1* +%{_mandir}/man1/nbdkit-truncate-filter.1* + + +%if !0%{?rhel} +%files ext2-filter +%doc README +%license LICENSE +%{_libdir}/%{name}/filters/nbdkit-ext2-filter.so +%{_mandir}/man1/nbdkit-ext2-filter.1* +%endif + + +%files gzip-filter +%doc README +%license LICENSE +%{_libdir}/%{name}/filters/nbdkit-gzip-filter.so +%{_mandir}/man1/nbdkit-gzip-filter.1* + + +%files tar-filter +%doc README +%license LICENSE +%{_libdir}/%{name}/filters/nbdkit-tar-filter.so +%{_mandir}/man1/nbdkit-tar-filter.1* + + +%files xz-filter +%doc README +%license LICENSE +%{_libdir}/%{name}/filters/nbdkit-xz-filter.so +%{_mandir}/man1/nbdkit-xz-filter.1* + + +%files devel +%doc BENCHMARKING OTHER_PLUGINS README SECURITY TODO +%license LICENSE +# Include the source of the example plugins in the documentation. +%doc plugins/example*/*.c +%if !0%{?rhel} +%doc plugins/example4/nbdkit-example4-plugin +%doc plugins/lua/example.lua +%endif +%if !0%{?rhel} && 0%{?have_ocaml} +%doc plugins/ocaml/example.ml +%endif +%if !0%{?rhel} +%doc plugins/perl/example.pl +%endif +%doc plugins/python/examples/*.py +%if !0%{?rhel} +%doc plugins/ruby/example.rb +%endif +%doc plugins/sh/example.sh +%if !0%{?rhel} +%doc plugins/tcl/example.tcl +%endif +%{_includedir}/nbdkit-common.h +%{_includedir}/nbdkit-filter.h +%{_includedir}/nbdkit-plugin.h +%{_includedir}/nbdkit-version.h +%{_includedir}/nbd-protocol.h +%{_mandir}/man3/nbdkit-filter.3* +%{_mandir}/man3/nbdkit-plugin.3* +%{_mandir}/man1/nbdkit-release-notes-1.*.1* +%{_libdir}/pkgconfig/nbdkit.pc + + +%files bash-completion +%license LICENSE +%dir %{_datadir}/bash-completion/completions +%{_datadir}/bash-completion/completions/nbdkit + + +%changelog +* Wed Jan 26 2022 Richard W.M. Jones - 1.24.0-4 +- Fix build on RHEL 8.6 with qemu >= 6.1 + resolves: rhbz#2045945 + +* Mon Sep 06 2021 Richard W.M. Jones - 1.24.0-3 +- Fix CVE-2021-3716 NBD_OPT_STRUCTURED_REPLY injection on STARTTLS + resolves: rhbz#1994915 + +* Mon Sep 06 2021 Richard W.M. Jones - 1.24.0-2 +- Fix data corruption in zero and trim on unaligned tail + resolves: rhbz#1990135 + +* Thu Sep 2 2021 Danilo C. L. de Paula - 1.24.0-1.el8 +- Resolves: bz#2000225 + (Rebase virt:rhel module:stream based on AV-8.6) + +* Thu Jun 04 2020 Danilo C. L. de Paula - 1.16.2 +- Resolves: bz#1810193 + (Upgrade components in virt:rhel module:stream for RHEL-8.3 release) + +* Mon Apr 27 2020 Danilo C. L. de Paula - 1.16.2 +- Resolves: bz#1810193 + (Upgrade components in virt:rhel module:stream for RHEL-8.3 release) + +* Fri Jun 28 2019 Danilo de Paula - 1.4.2-5 +- Rebuild all virt packages to fix RHEL's upgrade path +- Resolves: rhbz#1695587 + (Ensure modular RPM upgrade path) + +* Mon Dec 17 2018 Richard W.M. Jones - 1.4.2-4 +- Remove misguided LDFLAGS hack which removed server hardening. + https://bugzilla.redhat.com/show_bug.cgi?id=1624149#c6 + resolves: rhbz#1624149 + +* Fri Dec 14 2018 Richard W.M. Jones - 1.4.2-3 +- Use platform-python + resolves: rhbz#1659159 + +* Fri Aug 10 2018 Richard W.M. Jones - 1.4.2-2 +- Add Enhanced Python error reporting + resolves: rhbz#1614750. +- Use copy-patches.sh script. + +* Wed Aug 1 2018 Richard W.M. Jones - 1.4.2-1 +- New stable version 1.4.2. + +* Wed Jul 25 2018 Richard W.M. Jones - 1.4.1-3 +- Enable VDDK plugin on x86-64 only. + +* Fri Jul 20 2018 Richard W.M. Jones - 1.4.1-1 +- New upstream version 1.4.1. +- Small refactorings in the spec file. + +* Fri Jul 6 2018 Richard W.M. Jones - 1.4.0-1 +- New upstream version 1.4.0. +- New plugins: random, zero. +- New bash tab completion subpackage. +- Remove unused build dependencies. + +* Sun Jul 1 2018 Richard W.M. Jones - 1.2.4-3 +- Add all upstream patches since 1.2.4 was released. + +* Tue Jun 12 2018 Richard W.M. Jones - 1.2.4-2 +- Add all upstream patches since 1.2.4 was released. + +* Mon Jun 11 2018 Richard W.M. Jones - 1.2.4-2 +- Disable plugins and filters that we do not want to ship in RHEL 8. + +* Sat Jun 9 2018 Richard W.M. Jones - 1.2.4-1 +- New stable version 1.2.4. +- Remove upstream patches. +- Enable tarball signatures. +- Add upstream patch to fix tests when guestfish not available. + +* Wed Jun 6 2018 Richard W.M. Jones - 1.2.3-1 +- New stable version 1.2.3. +- Add patch to work around libvirt problem with relative socket paths. +- Add patch to fix the xz plugin test with recent guestfish. + +* Sat Apr 21 2018 Richard W.M. Jones - 1.2.2-1 +- New stable version 1.2.2. + +* Mon Apr 9 2018 Richard W.M. Jones - 1.2.1-1 +- New stable version 1.2.1. + +* Fri Apr 6 2018 Richard W.M. Jones - 1.2.0-1 +- Move to stable branch version 1.2.0. + +* Fri Feb 09 2018 Igor Gnatenko - 1.1.28-5 +- Escape macros in %%changelog + +* Thu Feb 08 2018 Fedora Release Engineering - 1.1.28-4 +- Rebuilt for https://fedoraproject.org/wiki/Fedora_28_Mass_Rebuild + +* Wed Jan 31 2018 Igor Gnatenko - 1.1.28-3 +- Switch to %%ldconfig_scriptlets + +* Fri Jan 26 2018 Richard W.M. Jones - 1.1.28-2 +- Run a simplified test suite on all arches. + +* Mon Jan 22 2018 Richard W.M. Jones - 1.1.28-1 +- New upstream version 1.1.28. +- Add two new filters to nbdkit-basic-filters. + +* Sat Jan 20 2018 Björn Esser - 1.1.27-2 +- Rebuilt for switch to libxcrypt + +* Sat Jan 20 2018 Richard W.M. Jones - 1.1.27-1 +- New upstream version 1.1.27. +- Add new subpackage nbdkit-basic-filters containing new filters. + +* Thu Jan 11 2018 Richard W.M. Jones - 1.1.26-2 +- Rebuild against updated Ruby. + +* Sat Dec 23 2017 Richard W.M. Jones - 1.1.26-1 +- New upstream version 1.1.26. +- Add new pkg-config file and dependency. + +* Wed Dec 06 2017 Richard W.M. Jones - 1.1.25-1 +- New upstream version 1.1.25. + +* Tue Dec 05 2017 Richard W.M. Jones - 1.1.24-1 +- New upstream version 1.1.24. +- Add tar plugin (new subpackage nbdkit-plugin-tar). + +* Tue Dec 05 2017 Richard W.M. Jones - 1.1.23-1 +- New upstream version 1.1.23. +- Add example4 plugin. +- Python3 tests require libguestfs so disable on s390x. + +* Sun Dec 03 2017 Richard W.M. Jones - 1.1.22-1 +- New upstream version 1.1.22. +- Enable tests on Fedora. + +* Sat Dec 02 2017 Richard W.M. Jones - 1.1.20-1 +- New upstream version 1.1.20. +- Add nbdkit-split-plugin to basic plugins. + +* Sat Dec 02 2017 Richard W.M. Jones - 1.1.19-2 +- OCaml 4.06.0 rebuild. + +* Thu Nov 30 2017 Richard W.M. Jones - 1.1.19-1 +- New upstream version 1.1.19. +- Combine all the simple plugins in %%{name}-basic-plugins. +- Add memory and null plugins. +- Rename the example plugins subpackage. +- Use %%license instead of %%doc for license file. +- Remove patches now upstream. + +* Wed Nov 29 2017 Richard W.M. Jones - 1.1.18-4 +- Fix Python 3 builds / RHEL macros (RHBZ#1404631). + +* Tue Nov 21 2017 Richard W.M. Jones - 1.1.18-3 +- New upstream version 1.1.18. +- Add NBD forwarding plugin. +- Add libselinux-devel so that SELinux support is enabled in the daemon. +- Apply all patches from upstream since 1.1.18. + +* Fri Oct 20 2017 Richard W.M. Jones - 1.1.16-2 +- New upstream version 1.1.16. +- Disable python3 plugin on RHEL/EPEL <= 7. +- Only ship on x86_64 in RHEL/EPEL <= 7. + +* Wed Sep 27 2017 Richard W.M. Jones - 1.1.15-1 +- New upstream version 1.1.15. +- Enable TLS support. + +* Fri Sep 01 2017 Richard W.M. Jones - 1.1.14-1 +- New upstream version 1.1.14. + +* Fri Aug 25 2017 Richard W.M. Jones - 1.1.13-1 +- New upstream version 1.1.13. +- Remove patches which are all upstream. +- Remove grubby hack, should not be needed with modern supermin. + +* Sat Aug 19 2017 Richard W.M. Jones - 1.1.12-13 +- Rebuild for OCaml 4.05.0. + +* Thu Aug 03 2017 Fedora Release Engineering - 1.1.12-12 +- Rebuilt for https://fedoraproject.org/wiki/Fedora_27_Binutils_Mass_Rebuild + +* Wed Jul 26 2017 Fedora Release Engineering - 1.1.12-11 +- Rebuilt for https://fedoraproject.org/wiki/Fedora_27_Mass_Rebuild + +* Tue Jun 27 2017 Richard W.M. Jones - 1.1.12-10 +- Rebuild for OCaml 4.04.2. + +* Sun Jun 04 2017 Jitka Plesnikova - 1.1.12-9 +- Perl 5.26 rebuild + +* Mon May 15 2017 Richard W.M. Jones - 1.1.12-8 +- Rebuild for OCaml 4.04.1. + +* Fri Feb 10 2017 Fedora Release Engineering - 1.1.12-7 +- Rebuilt for https://fedoraproject.org/wiki/Fedora_26_Mass_Rebuild + +* Thu Jan 12 2017 Vít Ondruch - 1.1.12-6 +- Rebuilt for https://fedoraproject.org/wiki/Changes/Ruby_2.4 + +* Fri Dec 23 2016 Richard W.M. Jones - 1.1.12-5 +- Rebuild for Python 3.6 update. + +* Wed Dec 14 2016 Richard W.M. Jones - 1.1.12-4 +- Fix python3 subpackage so it really uses python3 (RHBZ#1404631). + +* Sat Nov 05 2016 Richard W.M. Jones - 1.1.12-3 +- Rebuild for OCaml 4.04.0. + +* Mon Oct 03 2016 Richard W.M. Jones - 1.1.12-2 +- Compile Python 2 and Python 3 versions of the plugin. + +* Wed Jun 08 2016 Richard W.M. Jones - 1.1.12-1 +- New upstream version 1.1.12 +- Enable Ruby plugin. +- Disable tests on Rawhide because libvirt is broken again (RHBZ#1344016). + +* Wed May 25 2016 Richard W.M. Jones - 1.1.11-10 +- Add another upstream patch since 1.1.11. + +* Mon May 23 2016 Richard W.M. Jones - 1.1.11-9 +- Add all patches upstream since 1.1.11 (fixes RHBZ#1336758). + +* Tue May 17 2016 Jitka Plesnikova - 1.1.11-7 +- Perl 5.24 rebuild + +* Wed Mar 09 2016 Richard W.M. Jones - 1.1.11-6 +- When tests fail, dump out test-suite.log so we can debug it. + +* Fri Feb 05 2016 Richard W.M. Jones - 1.1.11-5 +- Don't run tests on x86, because kernel is broken there + (https://bugzilla.redhat.com/show_bug.cgi?id=1302071) + +* Thu Feb 04 2016 Fedora Release Engineering - 1.1.11-4 +- Rebuilt for https://fedoraproject.org/wiki/Fedora_24_Mass_Rebuild + +* Mon Jan 11 2016 Richard W.M. Jones - 1.1.11-3 +- Add support for newstyle NBD protocol (RHBZ#1297100). + +* Sat Oct 31 2015 Richard W.M. Jones - 1.1.11-1 +- New upstream version 1.1.11. + +* Thu Jul 30 2015 Richard W.M. Jones - 1.1.10-3 +- OCaml 4.02.3 rebuild. + +* Sat Jun 20 2015 Richard W.M. Jones - 1.1.10-2 +- Enable libguestfs plugin on aarch64. + +* Fri Jun 19 2015 Richard W.M. Jones - 1.1.10-1 +- New upstream version. +- Enable now working OCaml plugin (requires OCaml >= 4.02.2). + +* Wed Jun 17 2015 Fedora Release Engineering - 1.1.9-6 +- Rebuilt for https://fedoraproject.org/wiki/Fedora_23_Mass_Rebuild + +* Thu Jun 11 2015 Jitka Plesnikova - 1.1.9-5 +- Perl 5.22 rebuild + +* Wed Jun 10 2015 Richard W.M. Jones - 1.1.9-4 +- Enable debugging messages when running make check. + +* Sat Jun 06 2015 Jitka Plesnikova - 1.1.9-3 +- Perl 5.22 rebuild + +* Tue Oct 14 2014 Richard W.M. Jones - 1.1.9-2 +- New upstream version 1.1.9. +- Add the streaming plugin. +- Include fix for streaming plugin in 1.1.9. + +* Wed Sep 10 2014 Richard W.M. Jones - 1.1.8-4 +- Rebuild for updated Perl in Rawhide. +- Workaround for broken libvirt (RHBZ#1138604). + +* Sun Aug 17 2014 Fedora Release Engineering - 1.1.8-2 +- Rebuilt for https://fedoraproject.org/wiki/Fedora_21_22_Mass_Rebuild + +* Sat Jun 21 2014 Richard W.M. Jones - 1.1.8-1 +- New upstream version 1.1.8. +- Add support for cURL, and new nbdkit-plugin-curl package. + +* Fri Jun 20 2014 Richard W.M. Jones - 1.1.7-1 +- New upstream version 1.1.7. +- Remove patches which are now all upstream. + +* Sat Jun 07 2014 Fedora Release Engineering - 1.1.6-5 +- Rebuilt for https://fedoraproject.org/wiki/Fedora_21_Mass_Rebuild + +* Thu Mar 06 2014 Dan Horák - 1.1.6-4 +- libguestfs is available only on selected arches + +* Fri Feb 21 2014 Richard W.M. Jones - 1.1.6-3 +- Backport some upstream patches, fixing a minor bug and adding more tests. +- Enable the tests since kernel bug is fixed. + +* Sun Feb 16 2014 Richard W.M. Jones - 1.1.6-1 +- New upstream version 1.1.6. + +* Sat Feb 15 2014 Richard W.M. Jones - 1.1.5-2 +- New upstream version 1.1.5. +- Enable the new Python plugin. +- Perl plugin man page moved to section 3. +- Perl now requires ExtUtils::Embed. + +* Mon Feb 10 2014 Richard W.M. Jones - 1.1.4-1 +- New upstream version 1.1.4. +- Enable the new Perl plugin. + +* Sun Aug 4 2013 Richard W.M. Jones - 1.1.3-1 +- New upstream version 1.1.3 which fixes some test problems. +- Disable tests because Rawhide kernel is broken (RHBZ#991808). +- Remove a single quote from description which confused emacs. +- Remove patch, now upstream. + +* Sat Aug 03 2013 Fedora Release Engineering - 1.1.2-4 +- Rebuilt for https://fedoraproject.org/wiki/Fedora_20_Mass_Rebuild + +* Sun Jul 21 2013 Richard W.M. Jones - 1.1.2-3 +- Fix segfault when IPv6 client is used (RHBZ#986601). + +* Tue Jul 16 2013 Richard W.M. Jones - 1.1.2-2 +- New development version 1.1.2. +- Disable the tests on Fedora <= 18. + +* Tue Jun 25 2013 Richard W.M. Jones - 1.1.1-1 +- New development version 1.1.1. +- Add libguestfs plugin. +- Run the test suite. + +* Mon Jun 24 2013 Richard W.M. Jones - 1.0.0-4 +- Initial release. diff --git a/sources b/sources new file mode 100644 index 0000000..b855885 --- /dev/null +++ b/sources @@ -0,0 +1,2 @@ +SHA1 (libguestfs.keyring) = 1bbc40f501a7fef9eef2a39b701a71aee2fea7c4 +SHA1 (nbdkit-1.24.0.tar.gz) = 069720cc0d1502b007652101d293a57d7b4d7c41 diff --git a/tests/basic-test.sh b/tests/basic-test.sh new file mode 100755 index 0000000..07eb459 --- /dev/null +++ b/tests/basic-test.sh @@ -0,0 +1,6 @@ +#!/bin/bash - +set -e +set -x + +# Run nbdkit and check that nbdinfo can connect back to it. +nbdkit -U - memory 1G --run 'nbdinfo "$uri"' diff --git a/tests/tests.yml b/tests/tests.yml new file mode 100755 index 0000000..28c6a43 --- /dev/null +++ b/tests/tests.yml @@ -0,0 +1,12 @@ +- hosts: localhost + roles: + - role: standard-test-basic + tags: + - classic + required_packages: + - libnbd + - nbdkit + tests: + - simple: + dir: . + run: ./basic-test.sh