diff --git a/.gitignore b/.gitignore index 88961c1..4bd7ec0 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,3 @@ -SOURCES/libguestfs.keyring -SOURCES/nbdkit-1.24.0.tar.gz -/nbdkit-1.24.0.tar.gz +/clog +/nbdkit-*.tar.gz +/nbdkit-*.tar.gz.sig diff --git a/0001-ssh-Allow-the-remote-file-to-be-created.patch b/0001-ssh-Allow-the-remote-file-to-be-created.patch deleted file mode 100644 index f3e07e4..0000000 --- a/0001-ssh-Allow-the-remote-file-to-be-created.patch +++ /dev/null @@ -1,293 +0,0 @@ -From 6a2b0aac8be655524ea223e32cac0395fcc9f975 Mon Sep 17 00:00:00 2001 -From: "Richard W.M. Jones" -Date: Fri, 15 Apr 2022 12:08:37 +0100 -Subject: [PATCH] ssh: Allow the remote file to be created - -This adds new parameters, create=(true|false), create-size=SIZE and -create-mode=MODE to create and truncate the remote file. - -Reviewed-by: Laszlo Ersek -(cherry picked from commit 0793f30b1071753532362b2ebf9cb8156a88c3c3) ---- - plugins/ssh/nbdkit-ssh-plugin.pod | 34 ++++++++- - plugins/ssh/ssh.c | 112 +++++++++++++++++++++++++++--- - tests/test-ssh.sh | 13 +++- - 3 files changed, 146 insertions(+), 13 deletions(-) - -diff --git a/plugins/ssh/nbdkit-ssh-plugin.pod b/plugins/ssh/nbdkit-ssh-plugin.pod -index 3f401c15..2bc2c4a7 100644 ---- a/plugins/ssh/nbdkit-ssh-plugin.pod -+++ b/plugins/ssh/nbdkit-ssh-plugin.pod -@@ -5,8 +5,10 @@ nbdkit-ssh-plugin - access disk images over the SSH protocol - =head1 SYNOPSIS - - nbdkit ssh host=HOST [path=]PATH -- [compression=true] [config=CONFIG_FILE] [identity=FILENAME] -- [known-hosts=FILENAME] [password=PASSWORD|-|+FILENAME] -+ [compression=true] [config=CONFIG_FILE] -+ [create=true] [create-mode=MODE] [create-size=SIZE] -+ [identity=FILENAME] [known-hosts=FILENAME] -+ [password=PASSWORD|-|+FILENAME] - [port=PORT] [timeout=SECS] [user=USER] - [verify-remote-host=false] - -@@ -62,6 +64,34 @@ The C parameter is optional. If it is I specified at all - then F<~/.ssh/config> and F are both read. - Missing or unreadable files are ignored. - -+=item B -+ -+(nbdkit E 1.32) -+ -+If set, the remote file will be created. The remote file is created -+on the first NBD connection to nbdkit, not when nbdkit starts up. If -+the file already exists, it will be replaced and any existing content -+lost. -+ -+If using this option, you must use C. C can -+be used to control the permissions of the new file. -+ -+=item BMODE -+ -+(nbdkit E 1.32) -+ -+If using C specify the default permissions of the new -+remote file. You can use octal modes like C or -+C. The default is C<0600>, ie. only readable and -+writable by the remote user. -+ -+=item BSIZE -+ -+(nbdkit E 1.32) -+ -+If using C, specify the virtual size of the new disk. -+C can use modifiers like C<100M> etc. -+ - =item BHOST - - Specify the name or IP address of the remote host. -diff --git a/plugins/ssh/ssh.c b/plugins/ssh/ssh.c -index 39d77e44..5e314cd7 100644 ---- a/plugins/ssh/ssh.c -+++ b/plugins/ssh/ssh.c -@@ -44,6 +44,8 @@ - #include - #include - -+#include -+ - #include - #include - #include -@@ -51,6 +53,7 @@ - #include - - #include "array-size.h" -+#include "cleanup.h" - #include "const-string-vector.h" - #include "minmax.h" - -@@ -64,6 +67,9 @@ static const char *known_hosts = NULL; - static const_string_vector identities = empty_vector; - static uint32_t timeout = 0; - static bool compression = false; -+static bool create = false; -+static int64_t create_size = -1; -+static unsigned create_mode = S_IRUSR | S_IWUSR /* 0600 */; - - /* config can be: - * NULL => parse options from default file -@@ -167,6 +173,27 @@ ssh_config (const char *key, const char *value) - return -1; - compression = r; - } -+ else if (strcmp (key, "create") == 0) { -+ r = nbdkit_parse_bool (value); -+ if (r == -1) -+ return -1; -+ create = r; -+ } -+ else if (strcmp (key, "create-size") == 0) { -+ create_size = nbdkit_parse_size (value); -+ if (create_size == -1) -+ return -1; -+ } -+ else if (strcmp (key, "create-mode") == 0) { -+ r = nbdkit_parse_unsigned (key, value, &create_mode); -+ if (r == -1) -+ return -1; -+ /* OpenSSH checks this too. */ -+ if (create_mode > 0777) { -+ nbdkit_error ("create-mode must be <= 0777"); -+ return -1; -+ } -+ } - - else { - nbdkit_error ("unknown parameter '%s'", key); -@@ -186,6 +213,13 @@ ssh_config_complete (void) - return -1; - } - -+ /* If create=true, create-size must be supplied. */ -+ if (create && create_size == -1) { -+ nbdkit_error ("if using create=true, you must specify the size " -+ "of the new remote file using create-size=SIZE"); -+ return -1; -+ } -+ - return 0; - } - -@@ -200,7 +234,10 @@ ssh_config_complete (void) - "identity= Prepend private key (identity) file.\n" \ - "timeout=SECS Set SSH connection timeout.\n" \ - "verify-remote-host=false Ignore known_hosts.\n" \ -- "compression=true Enable compression." -+ "compression=true Enable compression.\n" \ -+ "create=true Create the remote file.\n" \ -+ "create-mode=MODE Set the permissions of the remote file.\n" \ -+ "create-size=SIZE Set the size of the remote file." - - /* Since we must simulate atomic pread and pwrite using seek + - * read/write, calls on each handle must be serialized. -@@ -329,6 +366,65 @@ authenticate (struct ssh_handle *h) - return -1; - } - -+/* This function opens or creates the remote file (depending on -+ * create=false|true). Parallel connections might call this function -+ * at the same time, and so we must hold a lock to ensure that the -+ * file is created at most once. -+ */ -+static pthread_mutex_t create_lock = PTHREAD_MUTEX_INITIALIZER; -+ -+static sftp_file -+open_or_create_path (ssh_session session, sftp_session sftp, int readonly) -+{ -+ ACQUIRE_LOCK_FOR_CURRENT_SCOPE (&create_lock); -+ int access_type; -+ int r; -+ sftp_file file; -+ -+ access_type = readonly ? O_RDONLY : O_RDWR; -+ if (create) access_type |= O_CREAT | O_TRUNC; -+ -+ file = sftp_open (sftp, path, access_type, S_IRWXU); -+ if (!file) { -+ nbdkit_error ("cannot %s file for %s: %s", -+ create ? "create" : "open", -+ readonly ? "reading" : "writing", -+ ssh_get_error (session)); -+ return NULL; -+ } -+ -+ if (create) { -+ /* There's no sftp_truncate call. However OpenSSH lets you call -+ * SSH_FXP_SETSTAT + SSH_FILEXFER_ATTR_SIZE which invokes -+ * truncate(2) on the server. Libssh doesn't provide a binding -+ * for SSH_FXP_FSETSTAT so we have to pass the session + path. -+ */ -+ struct sftp_attributes_struct attrs = { -+ .flags = SSH_FILEXFER_ATTR_SIZE | -+ SSH_FILEXFER_ATTR_PERMISSIONS, -+ .size = create_size, -+ .permissions = create_mode, -+ }; -+ -+ r = sftp_setstat (sftp, path, &attrs); -+ if (r != SSH_OK) { -+ nbdkit_error ("setstat failed: %s", ssh_get_error (session)); -+ -+ /* Best-effort attempt to delete the remote file on failure. */ -+ r = sftp_unlink (sftp, path); -+ if (r != SSH_OK) -+ nbdkit_debug ("unlink failed: %s", ssh_get_error (session)); -+ -+ return NULL; -+ } -+ } -+ -+ /* On the next connection, don't create or truncate the file. */ -+ create = false; -+ -+ return file; -+} -+ - /* Create the per-connection handle. */ - static void * - ssh_open (int readonly) -@@ -337,7 +433,6 @@ ssh_open (int readonly) - const int set = 1; - size_t i; - int r; -- int access_type; - - h = calloc (1, sizeof *h); - if (h == NULL) { -@@ -471,7 +566,7 @@ ssh_open (int readonly) - if (authenticate (h) == -1) - goto err; - -- /* Open the SFTP connection and file. */ -+ /* Open the SFTP connection. */ - h->sftp = sftp_new (h->session); - if (!h->sftp) { - nbdkit_error ("failed to allocate sftp session: %s", -@@ -484,14 +579,11 @@ ssh_open (int readonly) - ssh_get_error (h->session)); - goto err; - } -- access_type = readonly ? O_RDONLY : O_RDWR; -- h->file = sftp_open (h->sftp, path, access_type, S_IRWXU); -- if (!h->file) { -- nbdkit_error ("cannot open file for %s: %s", -- readonly ? "reading" : "writing", -- ssh_get_error (h->session)); -+ -+ /* Open or create the remote file. */ -+ h->file = open_or_create_path (h->session, h->sftp, readonly); -+ if (!h->file) - goto err; -- } - - nbdkit_debug ("opened libssh handle"); - -diff --git a/tests/test-ssh.sh b/tests/test-ssh.sh -index 6c0ce410..f04b4488 100755 ---- a/tests/test-ssh.sh -+++ b/tests/test-ssh.sh -@@ -36,6 +36,7 @@ set -x - - requires test -f disk - requires nbdcopy --version -+requires stat --version - - # Check that ssh to localhost will work without any passwords or phrases. - # -@@ -48,7 +49,7 @@ then - exit 77 - fi - --files="ssh.img" -+files="ssh.img ssh2.img" - rm -f $files - cleanup_fn rm -f $files - -@@ -59,3 +60,13 @@ nbdkit -v -D ssh.log=2 -U - \ - - # The output should be identical. - cmp disk ssh.img -+ -+# Copy local file 'ssh.img' to newly created "remote" 'ssh2.img' -+size="$(stat -c %s disk)" -+nbdkit -v -D ssh.log=2 -U - \ -+ ssh host=localhost $PWD/ssh2.img \ -+ create=true create-size=$size \ -+ --run 'nbdcopy ssh.img "$uri"' -+ -+# The output should be identical. -+cmp disk ssh2.img --- -2.31.1 - diff --git a/0002-readahead-Rewrite-this-filter-so-it-prefetches-using.patch b/0002-readahead-Rewrite-this-filter-so-it-prefetches-using.patch deleted file mode 100644 index 69d5efa..0000000 --- a/0002-readahead-Rewrite-this-filter-so-it-prefetches-using.patch +++ /dev/null @@ -1,794 +0,0 @@ -From ac40ae11bc9983e11185749b23e793568cb366cc Mon Sep 17 00:00:00 2001 -From: "Richard W.M. Jones" -Date: Sat, 16 Apr 2022 18:39:13 +0100 -Subject: [PATCH] readahead: Rewrite this filter so it prefetches using .cache - -The previous readahead filter did not work well and we have stopped -using it in virt-v2v. However the concept is a good one if we can -make it work. This commit completely rethinks the filter. - -Now, in parallel with the ordinary pread command, we issue a prefetch -(ie. .cache) to the underlying plugin for the data immediately -following the pread. For example a simple sequence of operations: - - t=1 pread (offset=0, count=65536) - t=2 pread (offset=65536, count=65536) - t=3 pread (offset=131072, count=65536) - -would become: - - t=1 pread (offset=0, count=65536) <--\ issued - cache (offset=65536, count=65536) <--/ in parallel - t=2 pread (offset=65536, count=65536) - cache (offset=131072, count=65536) - t=3 pread (offset=131072, count=65536) - cache (offset=196608, count=65536) - -This requires that the underlying filter(s) and plugin chain can -actually do something with the .cache request. If this is not the -case then the filter does nothing (but it will print a warning). For -plugins which don't have native support for prefetching, it is -sufficient to insert nbdkit-cache-filter after this filter. -(nbdkit-cow-filter can also be used with cow-on-cache=true, but that -is more useful for advanced users who are already using the cow -filter). - -The implementation creates a background thread per connection to issue -the parallel .cache requests. This is safer than the alternative (one -background thread in total) since we don't have to deal with the -problem of cache requests being issued with the wrong export name or -stale next pointer. The background thread is controlled by a queue of -commands, with the only possible commands being "cache" or "quit". - -Because the background thread issues parallel requests on the same -connection, the underlying plugin must support the parallel thread -model, otherwise we would be violating the plugin's thread model. It -may be possible in future to open a new connection from the background -thread (but with the same exportname), which would lift this -restriction to at least serialize_requests. Because of the current -limitation, nbdkit-curl-plugin cannot use prefetch. - -(cherry picked from commit 2ff548d66ad3eae87868402ec5b3319edd12090f) ---- - TODO | 22 +- - filters/readahead/Makefile.am | 2 + - filters/readahead/bgthread.c | 76 ++++ - filters/readahead/nbdkit-readahead-filter.pod | 70 +++- - filters/readahead/readahead.c | 338 +++++++++--------- - filters/readahead/readahead.h | 60 ++++ - tests/test-readahead.sh | 2 +- - 7 files changed, 367 insertions(+), 203 deletions(-) - create mode 100644 filters/readahead/bgthread.c - create mode 100644 filters/readahead/readahead.h - -diff --git a/TODO b/TODO -index 5ae21db5..4d2a9796 100644 ---- a/TODO -+++ b/TODO -@@ -62,16 +62,6 @@ General ideas for improvements - continue to keep their non-standard handshake while utilizing nbdkit - to prototype new behaviors in serving the kernel. - --* Background thread for filters. Some filters (readahead, cache and -- proposed scan filter - see below) could be more effective if they -- were able to defer work to a background thread. We finally have -- nbdkit_next_context_open and friends for allowing a background -- thread to have access into the plugin, but still need to worry about -- thread-safety (how much must the filter do vs. nbdkit, to avoid -- calling into the plugin too many times at once) and cleanup -- (spawning the thread during .after_fork is viable, but cleaning it -- up during .unload is too late). -- - * "nbdkit.so": nbdkit as a loadable shared library. The aim of nbdkit - is to make it reusable from other programs (see nbdkit-captive(1)). - If it was a loadable shared library it would be even more reusable. -@@ -228,6 +218,8 @@ Suggestions for filters - * nbdkit-cache-filter should handle ENOSPC errors automatically by - reclaiming blocks from the cache - -+* nbdkit-cache-filter could use a background thread for reclaiming. -+ - * zstd filter was requested as a way to do what we currently do with - xz but saving many hours on compression (at the cost of hundreds of - MBs of extra data) -@@ -240,6 +232,16 @@ Suggestions for filters - could inject a flush after pausing. However this requires that - filter background threads have access to the plugin (see above). - -+nbdkit-readahead-filter: -+ -+* The filter should open a new connection to the plugin per background -+ thread so it is able to work with plugins that use the -+ serialize_requests thread model (like curl). At the moment it makes -+ requests on the same connection, so it requires plugins to use the -+ parallel thread model. -+ -+* It should combine (or avoid) overlapping cache requests. -+ - nbdkit-rate-filter: - - * allow other kinds of traffic shaping such as VBR -diff --git a/filters/readahead/Makefile.am b/filters/readahead/Makefile.am -index ee5bb3fb..187993ae 100644 ---- a/filters/readahead/Makefile.am -+++ b/filters/readahead/Makefile.am -@@ -37,6 +37,8 @@ filter_LTLIBRARIES = nbdkit-readahead-filter.la - - nbdkit_readahead_filter_la_SOURCES = \ - readahead.c \ -+ readahead.h \ -+ bgthread.c \ - $(top_srcdir)/include/nbdkit-filter.h \ - $(NULL) - -diff --git a/filters/readahead/bgthread.c b/filters/readahead/bgthread.c -new file mode 100644 -index 00000000..5894bb5f ---- /dev/null -+++ b/filters/readahead/bgthread.c -@@ -0,0 +1,76 @@ -+/* nbdkit -+ * Copyright (C) 2019-2022 Red Hat Inc. -+ * -+ * Redistribution and use in source and binary forms, with or without -+ * modification, are permitted provided that the following conditions are -+ * met: -+ * -+ * * Redistributions of source code must retain the above copyright -+ * notice, this list of conditions and the following disclaimer. -+ * -+ * * Redistributions in binary form must reproduce the above copyright -+ * notice, this list of conditions and the following disclaimer in the -+ * documentation and/or other materials provided with the distribution. -+ * -+ * * Neither the name of Red Hat nor the names of its contributors may be -+ * used to endorse or promote products derived from this software without -+ * specific prior written permission. -+ * -+ * THIS SOFTWARE IS PROVIDED BY RED HAT AND CONTRIBUTORS ''AS IS'' AND -+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, -+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A -+ * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL RED HAT OR -+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF -+ * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND -+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT -+ * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF -+ * SUCH DAMAGE. -+ */ -+ -+#include -+ -+#include -+#include -+#include -+#include -+ -+#include -+ -+#include "readahead.h" -+ -+#include "cleanup.h" -+ -+void * -+readahead_thread (void *vp) -+{ -+ struct bgthread_ctrl *ctrl = vp; -+ -+ for (;;) { -+ struct command cmd; -+ -+ /* Wait until we are sent at least one command. */ -+ { -+ ACQUIRE_LOCK_FOR_CURRENT_SCOPE (&ctrl->lock); -+ while (ctrl->cmds.len == 0) -+ pthread_cond_wait (&ctrl->cond, &ctrl->lock); -+ cmd = ctrl->cmds.ptr[0]; -+ command_queue_remove (&ctrl->cmds, 0); -+ } -+ -+ switch (cmd.type) { -+ case CMD_QUIT: -+ /* Finish processing and exit the thread. */ -+ return NULL; -+ -+ case CMD_CACHE: -+ /* Issue .cache (readahead) to underlying plugin. We ignore any -+ * errors because there's no way to communicate that back to the -+ * client, and readahead is only advisory. -+ */ -+ cmd.next->cache (cmd.next, cmd.count, cmd.offset, 0, NULL); -+ } -+ } -+} -diff --git a/filters/readahead/nbdkit-readahead-filter.pod b/filters/readahead/nbdkit-readahead-filter.pod -index c220d379..630e5924 100644 ---- a/filters/readahead/nbdkit-readahead-filter.pod -+++ b/filters/readahead/nbdkit-readahead-filter.pod -@@ -1,28 +1,66 @@ - =head1 NAME - --nbdkit-readahead-filter - prefetch data when reading sequentially -+nbdkit-readahead-filter - prefetch data ahead of sequential reads - - =head1 SYNOPSIS - -- nbdkit --filter=readahead plugin -+ nbdkit --filter=readahead PLUGIN -+ -+ nbdkit --filter=readahead --filter=cache PLUGIN -+ -+ nbdkit --filter=readahead --filter=cow PLUGIN cow-on-cache=true - - =head1 DESCRIPTION - - C is a filter that prefetches data when the --client is reading sequentially. -+client is reading. - --A common use for this filter is to accelerate sequential copy --operations (like S>) when plugin requests have a --high overhead (like L). For example: -- -- nbdkit -U - --filter=readahead curl https://example.com/disk.img \ -- --run 'qemu-img convert $nbd disk.img' -+When the client issues a read, this filter issues a parallel prefetch -+(C<.cache>) for subsequent data. Plugins which support this command -+will prefetch the data, making sequential reads faster. For plugins -+which do not support this command, you can inject -+L below (after) this filter, giving -+approximately the same effect. L can be used -+instead of nbdkit-cache-filter, if you add the C -+option. - - The filter uses a simple adaptive algorithm which accelerates --sequential reads, but has a small penalty if the client does random --reads. If the client mixes reads with writes or write-like operations --(trimming, zeroing) then it will work but there can be a large --performance penalty. -+sequential reads and requires no further configuration. -+ -+=head2 Limitations -+ -+In a number of significant cases this filter will do nothing. The -+filter will print a warning message if this happens. -+ -+=over 4 -+ -+=item Thread model must be parallel -+ -+For example L only supports -+C, and so this filter cannot perform prefetches in -+parallel with the read requests. -+ -+We may be able to lift this restriction in future. -+ -+=item Underlying filters or plugin must support C<.cache> (prefetch) -+ -+Very many plugins do not have the concept of prefetching and/or -+do not implement the C<.cache> callback, and so there is no -+way for this filter to issue prefetches. -+ -+You can usually get around this by adding I<--filter=cache> after this -+filter as explained above. It may be necessary to limit the total -+size of the cache (see L). -+ -+=item Clients and kernels may do readahead already -+ -+It may be the case that NBD clients are already issuing -+C (NBD prefetch) commands. It may also be the case -+that your plugin is using local file functions where the kernel is -+doing readahead. In such cases this filter is not necessary and may -+be pessimal. -+ -+=back - - =head1 PARAMETERS - -@@ -50,9 +88,9 @@ C first appeared in nbdkit 1.12. - - L, - L, --L, -+L, -+L, - L, --L, - L, - L, - L, -@@ -64,4 +102,4 @@ Richard W.M. Jones - - =head1 COPYRIGHT - --Copyright (C) 2019 Red Hat Inc. -+Copyright (C) 2019-2022 Red Hat Inc. -diff --git a/filters/readahead/readahead.c b/filters/readahead/readahead.c -index f5552d4c..1d7ae111 100644 ---- a/filters/readahead/readahead.c -+++ b/filters/readahead/readahead.c -@@ -1,5 +1,5 @@ - /* nbdkit -- * Copyright (C) 2019-2021 Red Hat Inc. -+ * Copyright (C) 2019-2022 Red Hat Inc. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are -@@ -34,232 +34,218 @@ - - #include - #include -+#include - #include - #include - #include -- - #include - - #include - -+#include "readahead.h" -+ - #include "cleanup.h" - #include "minmax.h" -- --/* Copied from server/plugins.c. */ --#define MAX_REQUEST_SIZE (64 * 1024 * 1024) -+#include "vector.h" - - /* These could be made configurable in future. */ --#define READAHEAD_MIN 65536 --#define READAHEAD_MAX MAX_REQUEST_SIZE -- --/* This lock protects the global state. */ --static pthread_mutex_t lock = PTHREAD_MUTEX_INITIALIZER; -- --/* The real size of the underlying plugin. */ --static uint64_t size; -+#define READAHEAD_MIN 32768 -+#define READAHEAD_MAX (4*1024*1024) - - /* Size of the readahead window. */ -+static pthread_mutex_t window_lock = PTHREAD_MUTEX_INITIALIZER; - static uint64_t window = READAHEAD_MIN; -+static uint64_t last_offset = 0, last_readahead = 0; - --/* The single prefetch buffer shared by all threads, and its virtual -- * location in the virtual disk. The prefetch buffer grows -- * dynamically as required, but never shrinks. -+static int thread_model = -1; /* Thread model of the underlying plugin. */ -+ -+/* Per-connection data. */ -+struct readahead_handle { -+ int can_cache; /* Can the underlying plugin cache? */ -+ pthread_t thread; /* The background thread, one per connection. */ -+ struct bgthread_ctrl ctrl; -+}; -+ -+/* We have various requirements of the underlying filter(s) + plugin: -+ * - They must support NBDKIT_CACHE_NATIVE (otherwise our requests -+ * would not do anything useful). -+ * - They must use the PARALLEL thread model (otherwise we could -+ * violate their thread model). -+ */ -+static bool -+filter_working (struct readahead_handle *h) -+{ -+ return -+ h->can_cache == NBDKIT_CACHE_NATIVE && -+ thread_model == NBDKIT_THREAD_MODEL_PARALLEL; -+} -+ -+static bool -+suggest_cache_filter (struct readahead_handle *h) -+{ -+ return -+ h->can_cache != NBDKIT_CACHE_NATIVE && -+ thread_model == NBDKIT_THREAD_MODEL_PARALLEL; -+} -+ -+/* We need to hook into .get_ready() so we can read the final thread -+ * model (of the whole server). - */ --static char *buffer = NULL; --static size_t bufsize = 0; --static uint64_t position; --static uint32_t length = 0; -+static int -+readahead_get_ready (int final_thread_model) -+{ -+ thread_model = final_thread_model; -+ return 0; -+} -+ -+static int -+send_command_to_background_thread (struct bgthread_ctrl *ctrl, -+ const struct command cmd) -+{ -+ ACQUIRE_LOCK_FOR_CURRENT_SCOPE (&ctrl->lock); -+ if (command_queue_append (&ctrl->cmds, cmd) == -1) -+ return -1; -+ /* Signal the thread if it could be sleeping on an empty queue. */ -+ if (ctrl->cmds.len == 1) -+ pthread_cond_signal (&ctrl->cond); -+ return 0; -+} -+ -+static void * -+readahead_open (nbdkit_next_open *next, nbdkit_context *nxdata, -+ int readonly, const char *exportname, int is_tls) -+{ -+ struct readahead_handle *h; -+ int err; -+ -+ if (next (nxdata, readonly, exportname) == -1) -+ return NULL; -+ -+ h = malloc (sizeof *h); -+ if (h == NULL) { -+ nbdkit_error ("malloc: %m"); -+ return NULL; -+ } -+ -+ h->ctrl.cmds = (command_queue) empty_vector; -+ pthread_mutex_init (&h->ctrl.lock, NULL); -+ pthread_cond_init (&h->ctrl.cond, NULL); -+ -+ /* Create the background thread. */ -+ err = pthread_create (&h->thread, NULL, readahead_thread, &h->ctrl); -+ if (err != 0) { -+ errno = err; -+ nbdkit_error ("pthread_create: %m"); -+ pthread_cond_destroy (&h->ctrl.cond); -+ pthread_mutex_destroy (&h->ctrl.lock); -+ free (h); -+ return NULL; -+ } -+ -+ return h; -+} - - static void --readahead_unload (void) -+readahead_close (void *handle) - { -- free (buffer); -+ struct readahead_handle *h = handle; -+ const struct command quit_cmd = { .type = CMD_QUIT }; -+ -+ send_command_to_background_thread (&h->ctrl, quit_cmd); -+ pthread_join (h->thread, NULL); -+ pthread_cond_destroy (&h->ctrl.cond); -+ pthread_mutex_destroy (&h->ctrl.lock); -+ command_queue_reset (&h->ctrl.cmds); -+ free (h); - } - --static int64_t readahead_get_size (nbdkit_next *next, void *handle); -- --/* In prepare, force a call to get_size which sets the size global. */ - static int --readahead_prepare (nbdkit_next *next, void *handle, int readonly) -+readahead_can_cache (nbdkit_next *next, void *handle) - { -- int64_t r; -+ struct readahead_handle *h = handle; -+ int r; - -- r = readahead_get_size (next, handle); -- return r >= 0 ? 0 : -1; --} -- --/* Get the size. */ --static int64_t --readahead_get_size (nbdkit_next *next, void *handle) --{ -- int64_t r; -- -- r = next->get_size (next); -+ /* Call next->can_cache to read the underlying 'can_cache'. */ -+ r = next->can_cache (next); - if (r == -1) - return -1; -+ h->can_cache = r; - -- ACQUIRE_LOCK_FOR_CURRENT_SCOPE (&lock); -- size = r; -+ if (!filter_working (h)) { -+ nbdkit_error ("readahead: warning: underlying plugin does not support " -+ "NBD_CMD_CACHE or PARALLEL thread model, so the filter " -+ "won't do anything"); -+ if (suggest_cache_filter (h)) -+ nbdkit_error ("readahead: try adding --filter=cache " -+ "after this filter"); -+ /* This is an error, but that's just to ensure that the warning -+ * above is seen. We don't need to return -1 here. -+ */ -+ } - - return r; - } - --/* Cache */ --static int --readahead_can_cache (nbdkit_next *next, void *handle) --{ -- /* We are already operating as a cache regardless of the plugin's -- * underlying .can_cache, but it's easiest to just rely on nbdkit's -- * behavior of calling .pread for caching. -- */ -- return NBDKIT_CACHE_EMULATE; --} -- - /* Read data. */ -- --static int --fill_readahead (nbdkit_next *next, -- uint32_t count, uint64_t offset, uint32_t flags, int *err) --{ -- position = offset; -- -- /* Read at least window bytes, but if count is larger read that. -- * Note that the count cannot be bigger than the buffer size. -- */ -- length = MAX (count, window); -- -- /* Don't go beyond the end of the underlying file. */ -- length = MIN (length, size - position); -- -- /* Grow the buffer if necessary. */ -- if (bufsize < length) { -- char *new_buffer = realloc (buffer, length); -- if (new_buffer == NULL) { -- *err = errno; -- nbdkit_error ("realloc: %m"); -- return -1; -- } -- buffer = new_buffer; -- bufsize = length; -- } -- -- if (next->pread (next, buffer, length, offset, flags, err) == -1) { -- length = 0; /* failed to fill the prefetch buffer */ -- return -1; -- } -- -- return 0; --} -- - static int - readahead_pread (nbdkit_next *next, - void *handle, void *buf, uint32_t count, uint64_t offset, - uint32_t flags, int *err) - { -- ACQUIRE_LOCK_FOR_CURRENT_SCOPE (&lock); -+ struct readahead_handle *h = handle; - -- while (count > 0) { -- if (length == 0) { -- /* We don't have a prefetch buffer at all. This could be the -- * first request or reset after a miss. -- */ -- window = READAHEAD_MIN; -- if (fill_readahead (next, count, offset, flags, err) == -1) -- return -1; -- } -+ /* If the underlying plugin doesn't support caching then skip that -+ * step completely. The filter will do nothing. -+ */ -+ if (filter_working (h)) { -+ struct command ra_cmd = { .type = CMD_CACHE, .next = NULL }; -+ int64_t size; - -- /* Can we satisfy this request partly or entirely from the prefetch -- * buffer? -- */ -- else if (position <= offset && offset < position + length) { -- uint32_t n = MIN (position - offset + length, count); -- memcpy (buf, &buffer[offset-position], n); -- buf += n; -- offset += n; -- count -= n; -- } -+ size = next->get_size (next); -+ if (size >= 0) { -+ ACQUIRE_LOCK_FOR_CURRENT_SCOPE (&window_lock); - -- /* Does the request start immediately after the prefetch buffer? -- * This is a “hit” allowing us to double the window size. -- */ -- else if (offset == position + length) { -- window = MIN (window * 2, READAHEAD_MAX); -- if (fill_readahead (next, count, offset, flags, err) == -1) -- return -1; -+ /* Generate the asynchronous (background) cache command for -+ * the readahead window. -+ */ -+ ra_cmd.offset = offset + count; -+ if (ra_cmd.offset < size) { -+ ra_cmd.count = MIN (window, size - ra_cmd.offset); -+ ra_cmd.next = next; /* If .next is non-NULL, we'll send it below. */ -+ } -+ -+ /* Should we change the window size? -+ * If the last readahead < current offset, double the window. -+ * If not, but we're still making forward progress, keep the window. -+ * If we're not making forward progress, reduce the window to minimum. -+ */ -+ if (last_readahead < offset) -+ window = MIN (window * 2, READAHEAD_MAX); -+ else if (last_offset < offset) -+ /* leave window unchanged */ ; -+ else -+ window = READAHEAD_MIN; -+ last_offset = offset; -+ last_readahead = ra_cmd.offset + ra_cmd.count; - } - -- /* Else it's a “miss”. Reset everything and start again. */ -- else -- length = 0; -+ if (ra_cmd.next && -+ send_command_to_background_thread (&h->ctrl, ra_cmd) == -1) -+ return -1; - } - -- return 0; --} -- --/* Any writes or write-like operations kill the prefetch buffer. -- * -- * We could do better here, but for the current use case of this -- * filter it doesn't matter. XXX -- */ -- --static void --kill_readahead (void) --{ -- ACQUIRE_LOCK_FOR_CURRENT_SCOPE (&lock); -- window = READAHEAD_MIN; -- length = 0; --} -- --static int --readahead_pwrite (nbdkit_next *next, -- void *handle, -- const void *buf, uint32_t count, uint64_t offset, -- uint32_t flags, int *err) --{ -- kill_readahead (); -- return next->pwrite (next, buf, count, offset, flags, err); --} -- --static int --readahead_trim (nbdkit_next *next, -- void *handle, -- uint32_t count, uint64_t offset, uint32_t flags, -- int *err) --{ -- kill_readahead (); -- return next->trim (next, count, offset, flags, err); --} -- --static int --readahead_zero (nbdkit_next *next, -- void *handle, -- uint32_t count, uint64_t offset, uint32_t flags, -- int *err) --{ -- kill_readahead (); -- return next->zero (next, count, offset, flags, err); --} -- --static int --readahead_flush (nbdkit_next *next, -- void *handle, uint32_t flags, int *err) --{ -- kill_readahead (); -- return next->flush (next, flags, err); -+ /* Issue the synchronous read. */ -+ return next->pread (next, buf, count, offset, flags, err); - } - - static struct nbdkit_filter filter = { - .name = "readahead", - .longname = "nbdkit readahead filter", -- .unload = readahead_unload, -- .prepare = readahead_prepare, -- .get_size = readahead_get_size, -+ .get_ready = readahead_get_ready, -+ .open = readahead_open, -+ .close = readahead_close, - .can_cache = readahead_can_cache, - .pread = readahead_pread, -- .pwrite = readahead_pwrite, -- .trim = readahead_trim, -- .zero = readahead_zero, -- .flush = readahead_flush, - }; - - NBDKIT_REGISTER_FILTER(filter) -diff --git a/filters/readahead/readahead.h b/filters/readahead/readahead.h -new file mode 100644 -index 00000000..a68204d5 ---- /dev/null -+++ b/filters/readahead/readahead.h -@@ -0,0 +1,60 @@ -+/* nbdkit -+ * Copyright (C) 2019-2022 Red Hat Inc. -+ * -+ * Redistribution and use in source and binary forms, with or without -+ * modification, are permitted provided that the following conditions are -+ * met: -+ * -+ * * Redistributions of source code must retain the above copyright -+ * notice, this list of conditions and the following disclaimer. -+ * -+ * * Redistributions in binary form must reproduce the above copyright -+ * notice, this list of conditions and the following disclaimer in the -+ * documentation and/or other materials provided with the distribution. -+ * -+ * * Neither the name of Red Hat nor the names of its contributors may be -+ * used to endorse or promote products derived from this software without -+ * specific prior written permission. -+ * -+ * THIS SOFTWARE IS PROVIDED BY RED HAT AND CONTRIBUTORS ''AS IS'' AND -+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, -+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A -+ * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL RED HAT OR -+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF -+ * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND -+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT -+ * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF -+ * SUCH DAMAGE. -+ */ -+ -+#ifndef NBDKIT_READAHEAD_H -+#define NBDKIT_READAHEAD_H -+ -+#include -+ -+#include -+ -+#include "vector.h" -+ -+/* List of commands issued to the background thread. */ -+struct command { -+ enum { CMD_QUIT, CMD_CACHE } type; -+ nbdkit_next *next; -+ uint64_t offset; -+ uint32_t count; -+}; -+DEFINE_VECTOR_TYPE(command_queue, struct command); -+ -+struct bgthread_ctrl { -+ command_queue cmds; /* Command queue. */ -+ pthread_mutex_t lock; /* Lock for queue. */ -+ pthread_cond_t cond; /* Condition queue size 0 -> 1. */ -+}; -+ -+/* Start background thread (one per connection). */ -+extern void *readahead_thread (void *vp); -+ -+#endif /* NBDKIT_READAHEAD_H */ -diff --git a/tests/test-readahead.sh b/tests/test-readahead.sh -index 7ec7f8e9..17126e5a 100755 ---- a/tests/test-readahead.sh -+++ b/tests/test-readahead.sh -@@ -59,7 +59,7 @@ for i in range(0, 512*10, 512): - echo $((end_t - start_t)) - } - --t1=$(test --filter=readahead) -+t1=$(test --filter=readahead --filter=cache) - t2=$(test) - - # In the t1 case we should make only 1 request into the plugin, --- -2.31.1 - diff --git a/0003-readahead-Fix-test.patch b/0003-readahead-Fix-test.patch deleted file mode 100644 index 52962b7..0000000 --- a/0003-readahead-Fix-test.patch +++ /dev/null @@ -1,120 +0,0 @@ -From b41b7d7ddf6d3fba23ac7978c8b272f2ff84265d Mon Sep 17 00:00:00 2001 -From: "Richard W.M. Jones" -Date: Thu, 21 Apr 2022 16:14:46 +0100 -Subject: [PATCH] readahead: Fix test - -The previous test turned out to be pretty bad at testing the new -filter. A specific problem is that the filter starts a background -thread which issues .cache requests, while on the main connection -.pread requests are being passed through. The test used ---filter=readahead --filter=cache with the cache filter only caching -on .cache requests (since cache-on-read defaults to false), so only -caching requests made by the background thread. - - main thread - client ---- .pread ----- delay-filter -------> plugin - \ - \ background thread - .cache --- cache-filter - -Under very high load, the background thread could be starved. This -means no requests were being cached at all, and all requests were -passing through the delay filter. It would appear that readahead was -failing (which it was, in a way). - -It's not very easy to fix this since readahead is best-effort, but we -can go back to using a simpler plugin that logs reads and caches and -check that they look valid. - -Update: commit 2ff548d66ad3eae87868402ec5b3319edd12090f -(cherry picked from commit db1e3311727c6ecab3264a1811d33db1aa45a4d0) ---- - tests/test-readahead.sh | 61 +++++++++++++++++++++++------------------ - 1 file changed, 35 insertions(+), 26 deletions(-) - -diff --git a/tests/test-readahead.sh b/tests/test-readahead.sh -index 17126e5a..37f4a06f 100755 ---- a/tests/test-readahead.sh -+++ b/tests/test-readahead.sh -@@ -30,43 +30,52 @@ - # OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - # SUCH DAMAGE. - --# Is the readahead filter faster? Copy a blank disk with a custom --# plugin that sleeps on every request. Because the readahead filter --# should result in fewer requests it should run faster. -- - source ./functions.sh - set -e - set -x - --requires_filter delay -+requires_plugin sh - requires nbdsh --version - requires dd iflag=count_bytes > readahead.out -+ ;; -+ pread) -+ echo "$@" >> readahead.out -+ dd if=/dev/zero count=$3 iflag=count_bytes -+ ;; -+ *) -+ exit 2 -+ ;; -+esac -+EOF - -- end_t=$SECONDS -- echo $((end_t - start_t)) --} -+cat readahead.out - --t1=$(test --filter=readahead --filter=cache) --t2=$(test) -- --# In the t1 case we should make only 1 request into the plugin, --# resulting in around 1 sleep period (5 seconds). In the t2 case we --# make 10 requests so sleep for around 50 seconds. t1 should be < t2 --# is every reasonable scenario. --if [ $t1 -ge $t2 ]; then -- echo "$0: readahead filter took longer, should be shorter" -- exit 1 --fi -+# We should see the pread requests, and additional cache requests for -+# the 32K region following each pread request. -+for i in `seq 0 512 $((512*10 - 512))` ; do -+ grep "pread 512 $i" readahead.out -+ grep "cache 32768 $((i+512))" readahead.out -+done --- -2.31.1 - diff --git a/0004-New-filter-luks.patch b/0004-New-filter-luks.patch deleted file mode 100644 index 5d14e25..0000000 --- a/0004-New-filter-luks.patch +++ /dev/null @@ -1,1816 +0,0 @@ -From c19936170cf8b385687cf40f5a9507d87ae08267 Mon Sep 17 00:00:00 2001 -From: "Richard W.M. Jones" -Date: Sat, 30 Apr 2022 12:35:07 +0100 -Subject: [PATCH] New filter: luks - -This filter allows you to open, read and write LUKSv1 disk images, -compatible with the ones used by dm-crypt and qemu. - -(cherry picked from commit 468919dce6c5eb57503eacac0f67e5dd87c58e6c) ---- - TODO | 11 +- - configure.ac | 6 +- - docs/nbdkit-tls.pod | 1 + - filters/luks/Makefile.am | 77 ++ - filters/luks/luks.c | 1263 +++++++++++++++++++++++++++ - filters/luks/nbdkit-luks-filter.pod | 120 +++ - plugins/file/nbdkit-file-plugin.pod | 1 + - tests/Makefile.am | 12 + - tests/test-luks-copy.sh | 125 +++ - tests/test-luks-info.sh | 56 ++ - 10 files changed, 1668 insertions(+), 4 deletions(-) - create mode 100644 filters/luks/Makefile.am - create mode 100644 filters/luks/luks.c - create mode 100644 filters/luks/nbdkit-luks-filter.pod - create mode 100755 tests/test-luks-copy.sh - create mode 100755 tests/test-luks-info.sh - -diff --git a/TODO b/TODO -index 4d2a9796..0f5dc41d 100644 ---- a/TODO -+++ b/TODO -@@ -195,9 +195,6 @@ Suggestions for filters - connections. This may even allow a filter to offer a more parallel - threading model than the underlying plugin. - --* LUKS encrypt/decrypt filter, bonus points if compatible with qemu -- LUKS-encrypted disk images -- - * CBT filter to track dirty blocks. See these links for inspiration: - https://www.cloudandheat.com/block-level-data-tracking-using-davice-mappers-dm-era/ - https://github.com/qemu/qemu/blob/master/docs/interop/bitmaps.rst -@@ -232,6 +229,14 @@ Suggestions for filters - could inject a flush after pausing. However this requires that - filter background threads have access to the plugin (see above). - -+nbdkit-luks-filter: -+ -+* This filter should also support LUKSv2 (and so should qemu). -+ -+* There are some missing features: ESSIV, more ciphers. -+ -+* Implement trim and zero if possible. -+ - nbdkit-readahead-filter: - - * The filter should open a new connection to the plugin per background -diff --git a/configure.ac b/configure.ac -index a402921b..de85b4da 100644 ---- a/configure.ac -+++ b/configure.ac -@@ -127,6 +127,7 @@ filters="\ - ip \ - limit \ - log \ -+ luks \ - multi-conn \ - nocache \ - noextents \ -@@ -614,8 +615,9 @@ PKG_CHECK_MODULES([GNUTLS], [gnutls >= 3.3.0], [ - ], [ - AC_MSG_WARN([gnutls not found or < 3.3.0, TLS support will be disabled.]) - ]) -+AM_CONDITIONAL([HAVE_GNUTLS], [test "x$GNUTLS_LIBS" != "x"]) - --AS_IF([test "$GNUTLS_LIBS" != ""],[ -+AS_IF([test "x$GNUTLS_LIBS" != "x"],[ - AC_MSG_CHECKING([for default TLS session priority string]) - AC_ARG_WITH([tls-priority], - [AS_HELP_STRING([--with-tls-priority=...], -@@ -1383,6 +1385,7 @@ AC_CONFIG_FILES([Makefile - filters/ip/Makefile - filters/limit/Makefile - filters/log/Makefile -+ filters/luks/Makefile - filters/multi-conn/Makefile - filters/nocache/Makefile - filters/noextents/Makefile -@@ -1481,6 +1484,7 @@ echo "Optional filters:" - echo - feature "ext2" test "x$HAVE_EXT2_TRUE" = "x" - feature "gzip" test "x$HAVE_ZLIB_TRUE" = "x" -+feature "LUKS" test "x$HAVE_GNUTLS_TRUE" != "x" - feature "xz" test "x$HAVE_LIBLZMA_TRUE" = "x" - - echo -diff --git a/docs/nbdkit-tls.pod b/docs/nbdkit-tls.pod -index 86f5f984..4d0dc14c 100644 ---- a/docs/nbdkit-tls.pod -+++ b/docs/nbdkit-tls.pod -@@ -364,6 +364,7 @@ More information can be found in L. - =head1 SEE ALSO - - L, -+L, - L, - L, - L, -diff --git a/filters/luks/Makefile.am b/filters/luks/Makefile.am -new file mode 100644 -index 00000000..30089621 ---- /dev/null -+++ b/filters/luks/Makefile.am -@@ -0,0 +1,77 @@ -+# nbdkit -+# Copyright (C) 2019-2022 Red Hat Inc. -+# -+# Redistribution and use in source and binary forms, with or without -+# modification, are permitted provided that the following conditions are -+# met: -+# -+# * Redistributions of source code must retain the above copyright -+# notice, this list of conditions and the following disclaimer. -+# -+# * Redistributions in binary form must reproduce the above copyright -+# notice, this list of conditions and the following disclaimer in the -+# documentation and/or other materials provided with the distribution. -+# -+# * Neither the name of Red Hat nor the names of its contributors may be -+# used to endorse or promote products derived from this software without -+# specific prior written permission. -+# -+# THIS SOFTWARE IS PROVIDED BY RED HAT AND CONTRIBUTORS ''AS IS'' AND -+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, -+# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A -+# PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL RED HAT OR -+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF -+# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND -+# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -+# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT -+# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF -+# SUCH DAMAGE. -+ -+include $(top_srcdir)/common-rules.mk -+ -+EXTRA_DIST = nbdkit-luks-filter.pod -+ -+if HAVE_GNUTLS -+ -+filter_LTLIBRARIES = nbdkit-luks-filter.la -+ -+nbdkit_luks_filter_la_SOURCES = \ -+ luks.c \ -+ $(top_srcdir)/include/nbdkit-filter.h \ -+ $(NULL) -+ -+nbdkit_luks_filter_la_CPPFLAGS = \ -+ -I$(top_srcdir)/include \ -+ -I$(top_srcdir)/common/include \ -+ -I$(top_srcdir)/common/utils \ -+ $(NULL) -+nbdkit_luks_filter_la_CFLAGS = \ -+ $(WARNINGS_CFLAGS) \ -+ $(GNUTLS_CFLAGS) \ -+ $(NULL) -+nbdkit_luks_filter_la_LIBADD = \ -+ $(top_builddir)/common/utils/libutils.la \ -+ $(IMPORT_LIBRARY_ON_WINDOWS) \ -+ $(GNUTLS_LIBS) \ -+ $(NULL) -+nbdkit_luks_filter_la_LDFLAGS = \ -+ -module -avoid-version -shared $(NO_UNDEFINED_ON_WINDOWS) \ -+ -Wl,--version-script=$(top_srcdir)/filters/filters.syms \ -+ $(NULL) -+ -+if HAVE_POD -+ -+man_MANS = nbdkit-luks-filter.1 -+CLEANFILES += $(man_MANS) -+ -+nbdkit-luks-filter.1: nbdkit-luks-filter.pod \ -+ $(top_builddir)/podwrapper.pl -+ $(PODWRAPPER) --section=1 --man $@ \ -+ --html $(top_builddir)/html/$@.html \ -+ $< -+ -+endif HAVE_POD -+ -+endif -diff --git a/filters/luks/luks.c b/filters/luks/luks.c -new file mode 100644 -index 00000000..706a9bd2 ---- /dev/null -+++ b/filters/luks/luks.c -@@ -0,0 +1,1263 @@ -+/* nbdkit -+ * Copyright (C) 2018-2022 Red Hat Inc. -+ * -+ * Redistribution and use in source and binary forms, with or without -+ * modification, are permitted provided that the following conditions are -+ * met: -+ * -+ * * Redistributions of source code must retain the above copyright -+ * notice, this list of conditions and the following disclaimer. -+ * -+ * * Redistributions in binary form must reproduce the above copyright -+ * notice, this list of conditions and the following disclaimer in the -+ * documentation and/or other materials provided with the distribution. -+ * -+ * * Neither the name of Red Hat nor the names of its contributors may be -+ * used to endorse or promote products derived from this software without -+ * specific prior written permission. -+ * -+ * THIS SOFTWARE IS PROVIDED BY RED HAT AND CONTRIBUTORS ''AS IS'' AND -+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, -+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A -+ * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL RED HAT OR -+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF -+ * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND -+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT -+ * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF -+ * SUCH DAMAGE. -+ */ -+ -+#include -+ -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+ -+#include -+ -+#include -+ -+#include "byte-swapping.h" -+#include "cleanup.h" -+#include "isaligned.h" -+#include "minmax.h" -+#include "rounding.h" -+ -+/* LUKSv1 constants. */ -+#define LUKS_MAGIC { 'L', 'U', 'K', 'S', 0xBA, 0xBE } -+#define LUKS_MAGIC_LEN 6 -+#define LUKS_DIGESTSIZE 20 -+#define LUKS_SALTSIZE 32 -+#define LUKS_NUMKEYS 8 -+#define LUKS_KEY_DISABLED 0x0000DEAD -+#define LUKS_KEY_ENABLED 0x00AC71F3 -+#define LUKS_STRIPES 4000 -+#define LUKS_ALIGN_KEYSLOTS 4096 -+#define LUKS_SECTOR_SIZE 512 -+ -+/* Key slot. */ -+struct luks_keyslot { -+ uint32_t active; /* LUKS_KEY_DISABLED|LUKS_KEY_ENABLED */ -+ uint32_t password_iterations; -+ char password_salt[LUKS_SALTSIZE]; -+ uint32_t key_material_offset; -+ uint32_t stripes; -+} __attribute__((__packed__)); -+ -+/* LUKS superblock. */ -+struct luks_phdr { -+ char magic[LUKS_MAGIC_LEN]; /* LUKS_MAGIC */ -+ uint16_t version; /* Only 1 is supported. */ -+ char cipher_name[32]; -+ char cipher_mode[32]; -+ char hash_spec[32]; -+ uint32_t payload_offset; -+ uint32_t master_key_len; -+ uint8_t master_key_digest[LUKS_DIGESTSIZE]; -+ uint8_t master_key_salt[LUKS_SALTSIZE]; -+ uint32_t master_key_digest_iterations; -+ uint8_t uuid[40]; -+ -+ struct luks_keyslot keyslot[LUKS_NUMKEYS]; /* Key slots. */ -+} __attribute__((__packed__)); -+ -+static char *passphrase = NULL; -+ -+static void -+luks_unload (void) -+{ -+ /* XXX We should really store the passphrase (and master key) -+ * in mlock-ed memory. -+ */ -+ if (passphrase) { -+ memset (passphrase, 0, strlen (passphrase)); -+ free (passphrase); -+ } -+} -+ -+static int -+luks_thread_model (void) -+{ -+ return NBDKIT_THREAD_MODEL_PARALLEL; -+} -+ -+static int -+luks_config (nbdkit_next_config *next, nbdkit_backend *nxdata, -+ const char *key, const char *value) -+{ -+ if (strcmp (key, "passphrase") == 0) { -+ if (nbdkit_read_password (value, &passphrase) == -1) -+ return -1; -+ return 0; -+ } -+ -+ return next (nxdata, key, value); -+} -+ -+static int -+luks_config_complete (nbdkit_next_config_complete *next, nbdkit_backend *nxdata) -+{ -+ if (passphrase == NULL) { -+ nbdkit_error ("LUKS \"passphrase\" parameter is missing"); -+ return -1; -+ } -+ return next (nxdata); -+} -+ -+#define luks_config_help \ -+ "passphrase= Secret passphrase." -+ -+enum cipher_mode { -+ CIPHER_MODE_ECB, CIPHER_MODE_CBC, CIPHER_MODE_XTS, CIPHER_MODE_CTR, -+}; -+ -+static enum cipher_mode -+lookup_cipher_mode (const char *str) -+{ -+ if (strcmp (str, "ecb") == 0) -+ return CIPHER_MODE_ECB; -+ if (strcmp (str, "cbc") == 0) -+ return CIPHER_MODE_CBC; -+ if (strcmp (str, "xts") == 0) -+ return CIPHER_MODE_XTS; -+ if (strcmp (str, "ctr") == 0) -+ return CIPHER_MODE_CTR; -+ nbdkit_error ("unknown cipher mode: %s " -+ "(expecting \"ecb\", \"cbc\", \"xts\" or \"ctr\")", str); -+ return -1; -+} -+ -+static const char * -+cipher_mode_to_string (enum cipher_mode v) -+{ -+ switch (v) { -+ case CIPHER_MODE_ECB: return "ecb"; -+ case CIPHER_MODE_CBC: return "cbc"; -+ case CIPHER_MODE_XTS: return "xts"; -+ case CIPHER_MODE_CTR: return "ctr"; -+ default: abort (); -+ } -+} -+ -+enum ivgen { -+ IVGEN_PLAIN, IVGEN_PLAIN64, /* IVGEN_ESSIV, */ -+}; -+ -+static enum ivgen -+lookup_ivgen (const char *str) -+{ -+ if (strcmp (str, "plain") == 0) -+ return IVGEN_PLAIN; -+ if (strcmp (str, "plain64") == 0) -+ return IVGEN_PLAIN64; -+/* -+ if (strcmp (str, "essiv") == 0) -+ return IVGEN_ESSIV; -+*/ -+ nbdkit_error ("unknown IV generation algorithm: %s " -+ "(expecting \"plain\", \"plain64\" etc)", str); -+ return -1; -+} -+ -+static const char * -+ivgen_to_string (enum ivgen v) -+{ -+ switch (v) { -+ case IVGEN_PLAIN: return "plain"; -+ case IVGEN_PLAIN64: return "plain64"; -+ /*case IVGEN_ESSIV: return "essiv";*/ -+ default: abort (); -+ } -+} -+ -+static void -+calculate_iv (enum ivgen v, uint8_t *iv, size_t ivlen, uint64_t sector) -+{ -+ size_t prefixlen; -+ uint32_t sector32; -+ -+ switch (v) { -+ case IVGEN_PLAIN: -+ prefixlen = 4; /* 32 bits */ -+ if (prefixlen > ivlen) -+ prefixlen = ivlen; -+ sector32 = (uint32_t) sector; /* truncate to only lower bits */ -+ sector32 = htole32 (sector32); -+ memcpy (iv, §or32, prefixlen); -+ memset (iv + prefixlen, 0, ivlen - prefixlen); -+ break; -+ -+ case IVGEN_PLAIN64: -+ prefixlen = 8; /* 64 bits */ -+ if (prefixlen > ivlen) -+ prefixlen = ivlen; -+ sector = htole64 (sector); -+ memcpy (iv, §or, prefixlen); -+ memset (iv + prefixlen, 0, ivlen - prefixlen); -+ break; -+ -+ /*case IVGEN_ESSIV:*/ -+ default: abort (); -+ } -+} -+ -+enum cipher_alg { -+ CIPHER_ALG_AES_128, CIPHER_ALG_AES_192, CIPHER_ALG_AES_256, -+}; -+ -+static enum cipher_alg -+lookup_cipher_alg (const char *str, enum cipher_mode mode, int key_bytes) -+{ -+ if (mode == CIPHER_MODE_XTS) -+ key_bytes /= 2; -+ -+ if (strcmp (str, "aes") == 0) { -+ if (key_bytes == 16) -+ return CIPHER_ALG_AES_128; -+ if (key_bytes == 24) -+ return CIPHER_ALG_AES_192; -+ if (key_bytes == 32) -+ return CIPHER_ALG_AES_256; -+ } -+ nbdkit_error ("unknown cipher algorithm: %s (expecting \"aes\", etc)", str); -+ return -1; -+} -+ -+static const char * -+cipher_alg_to_string (enum cipher_alg v) -+{ -+ switch (v) { -+ case CIPHER_ALG_AES_128: return "aes-128"; -+ case CIPHER_ALG_AES_192: return "aes-192"; -+ case CIPHER_ALG_AES_256: return "aes-256"; -+ default: abort (); -+ } -+} -+ -+#if 0 -+static int -+cipher_alg_key_bytes (enum cipher_alg v) -+{ -+ switch (v) { -+ case CIPHER_ALG_AES_128: return 16; -+ case CIPHER_ALG_AES_192: return 24; -+ case CIPHER_ALG_AES_256: return 32; -+ default: abort (); -+ } -+} -+#endif -+ -+static int -+cipher_alg_iv_len (enum cipher_alg v, enum cipher_mode mode) -+{ -+ if (CIPHER_MODE_ECB) -+ return 0; /* Don't need an IV in this mode. */ -+ -+ switch (v) { -+ case CIPHER_ALG_AES_128: -+ case CIPHER_ALG_AES_192: -+ case CIPHER_ALG_AES_256: -+ return 16; -+ default: abort (); -+ } -+} -+ -+static gnutls_digest_algorithm_t -+lookup_hash (const char *str) -+{ -+ if (strcmp (str, "md5") == 0) -+ return GNUTLS_DIG_MD5; -+ if (strcmp (str, "sha1") == 0) -+ return GNUTLS_DIG_SHA1; -+ if (strcmp (str, "sha224") == 0) -+ return GNUTLS_DIG_SHA224; -+ if (strcmp (str, "sha256") == 0) -+ return GNUTLS_DIG_SHA256; -+ if (strcmp (str, "sha384") == 0) -+ return GNUTLS_DIG_SHA384; -+ if (strcmp (str, "sha512") == 0) -+ return GNUTLS_DIG_SHA512; -+ if (strcmp (str, "ripemd160") == 0) -+ return GNUTLS_DIG_RMD160; -+ nbdkit_error ("unknown hash algorithm: %s " -+ "(expecting \"md5\", \"sha1\", \"sha224\", etc)", str); -+ return -1; -+} -+ -+static const char * -+hash_to_string (gnutls_digest_algorithm_t v) -+{ -+ switch (v) { -+ case GNUTLS_DIG_UNKNOWN: return "unknown"; -+ case GNUTLS_DIG_MD5: return "md5"; -+ case GNUTLS_DIG_SHA1: return "sha1"; -+ case GNUTLS_DIG_SHA224: return "sha224"; -+ case GNUTLS_DIG_SHA256: return "sha256"; -+ case GNUTLS_DIG_SHA384: return "sha384"; -+ case GNUTLS_DIG_SHA512: return "sha512"; -+ case GNUTLS_DIG_RMD160: return "ripemd160"; -+ default: abort (); -+ } -+} -+ -+#if 0 -+/* See qemu & dm-crypt implementations for an explanation of what's -+ * going on here. -+ */ -+static enum cipher_alg -+lookup_essiv_cipher (enum cipher_alg cipher_alg, -+ gnutls_digest_algorithm_t ivgen_hash_alg) -+{ -+ int digest_bytes = gnutls_hash_get_len (ivgen_hash_alg); -+ int key_bytes = cipher_alg_key_bytes (cipher_alg); -+ -+ if (digest_bytes == key_bytes) -+ return cipher_alg; -+ -+ switch (cipher_alg) { -+ case CIPHER_ALG_AES_128: -+ case CIPHER_ALG_AES_192: -+ case CIPHER_ALG_AES_256: -+ if (digest_bytes == 16) return CIPHER_ALG_AES_128; -+ if (digest_bytes == 24) return CIPHER_ALG_AES_192; -+ if (digest_bytes == 32) return CIPHER_ALG_AES_256; -+ nbdkit_error ("no %s cipher available with key size %d", -+ "AES", digest_bytes); -+ return -1; -+ default: -+ nbdkit_error ("ESSIV does not support cipher %s", -+ cipher_alg_to_string (cipher_alg)); -+ return -1; -+ } -+} -+#endif -+ -+/* Per-connection handle. */ -+struct handle { -+ /* LUKS header, if necessary byte-swapped into host order. */ -+ struct luks_phdr phdr; -+ -+ /* Decoded algorithm etc. */ -+ enum cipher_alg cipher_alg; -+ enum cipher_mode cipher_mode; -+ gnutls_digest_algorithm_t hash_alg; -+ enum ivgen ivgen_alg; -+ gnutls_digest_algorithm_t ivgen_hash_alg; -+ enum cipher_alg ivgen_cipher_alg; -+ -+ /* GnuTLS algorithm. */ -+ gnutls_cipher_algorithm_t gnutls_cipher; -+ -+ /* If we managed to decrypt one of the keyslots using the passphrase -+ * then this contains the master key, otherwise NULL. -+ */ -+ uint8_t *masterkey; -+}; -+ -+static void * -+luks_open (nbdkit_next_open *next, nbdkit_context *nxdata, -+ int readonly, const char *exportname, int is_tls) -+{ -+ struct handle *h; -+ -+ if (next (nxdata, readonly, exportname) == -1) -+ return NULL; -+ -+ h = calloc (1, sizeof *h); -+ if (h == NULL) { -+ nbdkit_error ("calloc: %m"); -+ return NULL; -+ } -+ -+ return h; -+} -+ -+static void -+luks_close (void *handle) -+{ -+ struct handle *h = handle; -+ -+ if (h->masterkey) { -+ memset (h->masterkey, 0, h->phdr.master_key_len); -+ free (h->masterkey); -+ } -+ free (h); -+} -+ -+/* Perform decryption of a block of data in memory. */ -+static int -+do_decrypt (struct handle *h, gnutls_cipher_hd_t cipher, -+ uint64_t offset, uint8_t *buf, size_t len) -+{ -+ const size_t ivlen = cipher_alg_iv_len (h->cipher_alg, h->cipher_mode); -+ uint64_t sector = offset / LUKS_SECTOR_SIZE; -+ CLEANUP_FREE uint8_t *iv = malloc (ivlen); -+ int r; -+ -+ assert (IS_ALIGNED (offset, LUKS_SECTOR_SIZE)); -+ assert (IS_ALIGNED (len, LUKS_SECTOR_SIZE)); -+ -+ while (len) { -+ calculate_iv (h->ivgen_alg, iv, ivlen, sector); -+ gnutls_cipher_set_iv (cipher, iv, ivlen); -+ r = gnutls_cipher_decrypt2 (cipher, -+ buf, LUKS_SECTOR_SIZE, /* ciphertext */ -+ buf, LUKS_SECTOR_SIZE /* plaintext */); -+ if (r != 0) { -+ nbdkit_error ("gnutls_cipher_decrypt2: %s", gnutls_strerror (r)); -+ return -1; -+ } -+ -+ buf += LUKS_SECTOR_SIZE; -+ offset += LUKS_SECTOR_SIZE; -+ len -= LUKS_SECTOR_SIZE; -+ sector++; -+ } -+ -+ return 0; -+} -+ -+/* Perform encryption of a block of data in memory. */ -+static int -+do_encrypt (struct handle *h, gnutls_cipher_hd_t cipher, -+ uint64_t offset, uint8_t *buf, size_t len) -+{ -+ const size_t ivlen = cipher_alg_iv_len (h->cipher_alg, h->cipher_mode); -+ uint64_t sector = offset / LUKS_SECTOR_SIZE; -+ CLEANUP_FREE uint8_t *iv = malloc (ivlen); -+ int r; -+ -+ assert (IS_ALIGNED (offset, LUKS_SECTOR_SIZE)); -+ assert (IS_ALIGNED (len, LUKS_SECTOR_SIZE)); -+ -+ while (len) { -+ calculate_iv (h->ivgen_alg, iv, ivlen, sector); -+ gnutls_cipher_set_iv (cipher, iv, ivlen); -+ r = gnutls_cipher_encrypt2 (cipher, -+ buf, LUKS_SECTOR_SIZE, /* plaintext */ -+ buf, LUKS_SECTOR_SIZE /* ciphertext */); -+ if (r != 0) { -+ nbdkit_error ("gnutls_cipher_decrypt2: %s", gnutls_strerror (r)); -+ return -1; -+ } -+ -+ buf += LUKS_SECTOR_SIZE; -+ offset += LUKS_SECTOR_SIZE; -+ len -= LUKS_SECTOR_SIZE; -+ sector++; -+ } -+ -+ return 0; -+} -+ -+/* Parse the header fields containing cipher algorithm, mode, etc. */ -+static int -+parse_cipher_strings (struct handle *h) -+{ -+ char cipher_name[33], cipher_mode[33], hash_spec[33]; -+ char *ivgen, *ivhash; -+ -+ /* Copy the header fields locally and ensure they are \0 terminated. */ -+ memcpy (cipher_name, h->phdr.cipher_name, 32); -+ cipher_name[32] = 0; -+ memcpy (cipher_mode, h->phdr.cipher_mode, 32); -+ cipher_mode[32] = 0; -+ memcpy (hash_spec, h->phdr.hash_spec, 32); -+ hash_spec[32] = 0; -+ -+ nbdkit_debug ("LUKS v%" PRIu16 " cipher: %s mode: %s hash: %s " -+ "master key: %" PRIu32 " bits", -+ h->phdr.version, cipher_name, cipher_mode, hash_spec, -+ h->phdr.master_key_len * 8); -+ -+ /* The cipher_mode header has the form: "ciphermode-ivgen[:ivhash]" -+ * QEmu writes: "xts-plain64" -+ */ -+ ivgen = strchr (cipher_mode, '-'); -+ if (!ivgen) { -+ nbdkit_error ("incorrect cipher_mode header, " -+ "expecting mode-ivgenerator but got \"%s\"", cipher_mode); -+ return -1; -+ } -+ *ivgen = '\0'; -+ ivgen++; -+ -+ ivhash = strchr (ivgen, ':'); -+ if (!ivhash) -+ h->ivgen_hash_alg = GNUTLS_DIG_UNKNOWN; -+ else { -+ *ivhash = '\0'; -+ ivhash++; -+ -+ h->ivgen_hash_alg = lookup_hash (ivhash); -+ if (h->ivgen_hash_alg == -1) -+ return -1; -+ } -+ -+ h->cipher_mode = lookup_cipher_mode (cipher_mode); -+ if (h->cipher_mode == -1) -+ return -1; -+ -+ h->cipher_alg = lookup_cipher_alg (cipher_name, h->cipher_mode, -+ h->phdr.master_key_len); -+ if (h->cipher_alg == -1) -+ return -1; -+ -+ h->hash_alg = lookup_hash (hash_spec); -+ if (h->hash_alg == -1) -+ return -1; -+ -+ h->ivgen_alg = lookup_ivgen (ivgen); -+ if (h->ivgen_alg == -1) -+ return -1; -+ -+#if 0 -+ if (h->ivgen_alg == IVGEN_ESSIV) { -+ if (!ivhash) { -+ nbdkit_error ("incorrect IV generator hash specification"); -+ return -1; -+ } -+ h->ivgen_cipher_alg = lookup_essiv_cipher (h->cipher_alg, -+ h->ivgen_hash_alg); -+ if (h->ivgen_cipher_alg == -1) -+ return -1; -+ } -+ else -+#endif -+ h->ivgen_cipher_alg = h->cipher_alg; -+ -+ nbdkit_debug ("LUKS parsed ciphers: %s %s %s %s %s %s", -+ cipher_alg_to_string (h->cipher_alg), -+ cipher_mode_to_string (h->cipher_mode), -+ hash_to_string (h->hash_alg), -+ ivgen_to_string (h->ivgen_alg), -+ hash_to_string (h->ivgen_hash_alg), -+ cipher_alg_to_string (h->ivgen_cipher_alg)); -+ -+ /* GnuTLS combines cipher and block mode into a single value. Not -+ * all possible combinations are available in GnuTLS. See: -+ * https://www.gnutls.org/manual/html_node/Supported-ciphersuites.html -+ */ -+ h->gnutls_cipher = GNUTLS_CIPHER_NULL; -+ switch (h->cipher_mode) { -+ case CIPHER_MODE_XTS: -+ switch (h->cipher_alg) { -+ case CIPHER_ALG_AES_128: -+ h->gnutls_cipher = GNUTLS_CIPHER_AES_128_XTS; -+ break; -+ case CIPHER_ALG_AES_256: -+ h->gnutls_cipher = GNUTLS_CIPHER_AES_256_XTS; -+ break; -+ default: break; -+ } -+ break; -+ case CIPHER_MODE_CBC: -+ switch (h->cipher_alg) { -+ case CIPHER_ALG_AES_128: -+ h->gnutls_cipher = GNUTLS_CIPHER_AES_128_CBC; -+ break; -+ case CIPHER_ALG_AES_192: -+ h->gnutls_cipher = GNUTLS_CIPHER_AES_192_CBC; -+ break; -+ case CIPHER_ALG_AES_256: -+ h->gnutls_cipher = GNUTLS_CIPHER_AES_256_CBC; -+ break; -+ default: break; -+ } -+ default: break; -+ } -+ if (h->gnutls_cipher == GNUTLS_CIPHER_NULL) { -+ nbdkit_error ("cipher algorithm %s in mode %s is not supported by GnuTLS", -+ cipher_alg_to_string (h->cipher_alg), -+ cipher_mode_to_string (h->cipher_mode)); -+ return -1; -+ } -+ -+ return 0; -+} -+ -+/* Anti-Forensic merge operation. */ -+static void -+xor (const uint8_t *in1, const uint8_t *in2, uint8_t *out, size_t len) -+{ -+ size_t i; -+ -+ for (i = 0; i < len; ++i) -+ out[i] = in1[i] ^ in2[i]; -+} -+ -+static int -+af_hash (gnutls_digest_algorithm_t hash_alg, uint8_t *block, size_t len) -+{ -+ size_t digest_bytes = gnutls_hash_get_len (hash_alg); -+ size_t nr_blocks, last_block_len; -+ size_t i; -+ CLEANUP_FREE uint8_t *temp = malloc (digest_bytes); -+ int r; -+ gnutls_hash_hd_t hash; -+ -+ nr_blocks = len / digest_bytes; -+ last_block_len = len % digest_bytes; -+ if (last_block_len != 0) -+ nr_blocks++; -+ else -+ last_block_len = digest_bytes; -+ -+ for (i = 0; i < nr_blocks; ++i) { -+ const uint32_t iv = htobe32 (i); -+ const size_t blen = i < nr_blocks - 1 ? digest_bytes : last_block_len; -+ -+ /* Hash iv + i'th block into temp. */ -+ r = gnutls_hash_init (&hash, hash_alg); -+ if (r != 0) { -+ nbdkit_error ("gnutls_hash_init: %s", gnutls_strerror (r)); -+ return -1; -+ } -+ gnutls_hash (hash, &iv, sizeof iv); -+ gnutls_hash (hash, &block[i*digest_bytes], blen); -+ gnutls_hash_deinit (hash, temp); -+ -+ memcpy (&block[i*digest_bytes], temp, blen); -+ } -+ -+ return 0; -+} -+ -+static int -+afmerge (gnutls_digest_algorithm_t hash_alg, uint32_t stripes, -+ const uint8_t *in, uint8_t *out, size_t outlen) -+{ -+ CLEANUP_FREE uint8_t *block = calloc (1, outlen); -+ size_t i; -+ -+ /* NB: input size is stripes * master_key_len where -+ * master_key_len == outlen -+ */ -+ for (i = 0; i < stripes-1; ++i) { -+ xor (&in[i*outlen], block, block, outlen); -+ if (af_hash (hash_alg, block, outlen) == -1) -+ return -1; -+ } -+ xor (&in[i*outlen], block, out, outlen); -+ return 0; -+} -+ -+/* Length of key material in key slot i (sectors). -+ * -+ * This is basically copied from qemu because the spec description is -+ * unintelligible and apparently doesn't match reality. -+ */ -+static uint64_t -+key_material_length_in_sectors (struct handle *h, size_t i) -+{ -+ uint64_t len, r; -+ -+ len = h->phdr.master_key_len * h->phdr.keyslot[i].stripes; -+ r = DIV_ROUND_UP (len, LUKS_SECTOR_SIZE); -+ r = ROUND_UP (r, LUKS_ALIGN_KEYSLOTS / LUKS_SECTOR_SIZE); -+ return r; -+} -+ -+/* Try the passphrase in key slot i. If this returns true then the -+ * passphrase was able to decrypt the master key, and the master key -+ * has been stored in h->masterkey. -+ */ -+static int -+try_passphrase_in_keyslot (nbdkit_next *next, struct handle *h, size_t i) -+{ -+ struct luks_keyslot *ks = &h->phdr.keyslot[i]; -+ size_t split_key_len; -+ CLEANUP_FREE uint8_t *split_key = NULL; -+ CLEANUP_FREE uint8_t *masterkey = NULL; -+ const gnutls_datum_t key = -+ { (unsigned char *) passphrase, strlen (passphrase) }; -+ const gnutls_datum_t salt = -+ { (unsigned char *) ks->password_salt, LUKS_SALTSIZE }; -+ const gnutls_datum_t msalt = -+ { (unsigned char *) h->phdr.master_key_salt, LUKS_SALTSIZE }; -+ gnutls_datum_t mkey; -+ gnutls_cipher_hd_t cipher; -+ int r, err = 0; -+ uint64_t start; -+ uint8_t key_digest[LUKS_DIGESTSIZE]; -+ -+ if (ks->active != LUKS_KEY_ENABLED) -+ return 0; -+ -+ split_key_len = h->phdr.master_key_len * ks->stripes; -+ split_key = malloc (split_key_len); -+ if (split_key == NULL) { -+ nbdkit_error ("malloc: %m"); -+ return -1; -+ } -+ masterkey = malloc (h->phdr.master_key_len); -+ if (masterkey == NULL) { -+ nbdkit_error ("malloc: %m"); -+ return -1; -+ } -+ -+ /* Hash the passphrase to make a possible masterkey. */ -+ r = gnutls_pbkdf2 (h->hash_alg, &key, &salt, ks->password_iterations, -+ masterkey, h->phdr.master_key_len); -+ if (r != 0) { -+ nbdkit_error ("gnutls_pbkdf2: %s", gnutls_strerror (r)); -+ return -1; -+ } -+ -+ /* Read master key material from plugin. */ -+ start = ks->key_material_offset * LUKS_SECTOR_SIZE; -+ if (next->pread (next, split_key, split_key_len, start, 0, &err) == -1) { -+ errno = err; -+ return -1; -+ } -+ -+ /* Decrypt the (still AFsplit) master key material. */ -+ mkey.data = (unsigned char *) masterkey; -+ mkey.size = h->phdr.master_key_len; -+ r = gnutls_cipher_init (&cipher, h->gnutls_cipher, &mkey, NULL); -+ if (r != 0) { -+ nbdkit_error ("gnutls_cipher_init: %s", gnutls_strerror (r)); -+ return -1; -+ } -+ -+ r = do_decrypt (h, cipher, 0, split_key, split_key_len); -+ gnutls_cipher_deinit (cipher); -+ if (r == -1) -+ return -1; -+ -+ /* Decode AFsplit key to a possible masterkey. */ -+ if (afmerge (h->hash_alg, ks->stripes, split_key, -+ masterkey, h->phdr.master_key_len) == -1) -+ return -1; -+ -+ /* Check if the masterkey is correct by comparing hash of the -+ * masterkey with LUKS header. -+ */ -+ r = gnutls_pbkdf2 (h->hash_alg, &mkey, &msalt, -+ h->phdr.master_key_digest_iterations, -+ key_digest, LUKS_DIGESTSIZE); -+ if (r != 0) { -+ nbdkit_error ("gnutls_pbkdf2: %s", gnutls_strerror (r)); -+ return -1; -+ } -+ -+ if (memcmp (key_digest, h->phdr.master_key_digest, LUKS_DIGESTSIZE) == 0) { -+ /* The passphrase is correct so save the master key in the handle. */ -+ h->masterkey = malloc (h->phdr.master_key_len); -+ if (h->masterkey == NULL) { -+ nbdkit_error ("malloc: %m"); -+ return -1; -+ } -+ memcpy (h->masterkey, masterkey, h->phdr.master_key_len); -+ return 1; -+ } -+ -+ return 0; -+} -+ -+static int -+luks_prepare (nbdkit_next *next, void *handle, int readonly) -+{ -+ static const char expected_magic[] = LUKS_MAGIC; -+ struct handle *h = handle; -+ int64_t size; -+ int err = 0, r; -+ size_t i; -+ struct luks_keyslot *ks; -+ char uuid[41]; -+ -+ /* Check we haven't been called before, this should never happen. */ -+ assert (h->phdr.version == 0); -+ -+ /* Check the struct size matches the documentation. */ -+ assert (sizeof (struct luks_phdr) == 592); -+ -+ /* Check this is a LUKSv1 disk. */ -+ size = next->get_size (next); -+ if (size == -1) -+ return -1; -+ if (size < 16384) { -+ nbdkit_error ("disk is too small to be LUKS-encrypted"); -+ return -1; -+ } -+ -+ /* Read the phdr. */ -+ if (next->pread (next, &h->phdr, sizeof h->phdr, 0, 0, &err) == -1) { -+ errno = err; -+ return -1; -+ } -+ -+ if (memcmp (h->phdr.magic, expected_magic, LUKS_MAGIC_LEN) != 0) { -+ nbdkit_error ("this disk does not contain a LUKS header"); -+ return -1; -+ } -+ h->phdr.version = be16toh (h->phdr.version); -+ if (h->phdr.version != 1) { -+ nbdkit_error ("this disk contains a LUKS version %" PRIu16 " header, " -+ "but this filter only supports LUKSv1", -+ h->phdr.version); -+ return -1; -+ } -+ -+ /* Byte-swap the rest of the header. */ -+ h->phdr.payload_offset = be32toh (h->phdr.payload_offset); -+ h->phdr.master_key_len = be32toh (h->phdr.master_key_len); -+ h->phdr.master_key_digest_iterations = -+ be32toh (h->phdr.master_key_digest_iterations); -+ -+ for (i = 0; i < LUKS_NUMKEYS; ++i) { -+ ks = &h->phdr.keyslot[i]; -+ ks->active = be32toh (ks->active); -+ ks->password_iterations = be32toh (ks->password_iterations); -+ ks->key_material_offset = be32toh (ks->key_material_offset); -+ ks->stripes = be32toh (ks->stripes); -+ } -+ -+ /* Sanity check some fields. */ -+ if (h->phdr.payload_offset >= size / LUKS_SECTOR_SIZE) { -+ nbdkit_error ("bad LUKSv1 header: payload offset points beyond " -+ "the end of the disk"); -+ return -1; -+ } -+ -+ /* We derive several allocations from master_key_len so make sure -+ * it's not insane. -+ */ -+ if (h->phdr.master_key_len > 1024) { -+ nbdkit_error ("bad LUKSv1 header: master key is too long"); -+ return -1; -+ } -+ -+ for (i = 0; i < LUKS_NUMKEYS; ++i) { -+ uint64_t start, len; -+ -+ ks = &h->phdr.keyslot[i]; -+ switch (ks->active) { -+ case LUKS_KEY_ENABLED: -+ if (!ks->stripes) { -+ nbdkit_error ("bad LUKSv1 header: key slot %zu is corrupted", i); -+ return -1; -+ } -+ if (ks->stripes >= 10000) { -+ nbdkit_error ("bad LUKSv1 header: key slot %zu stripes too large", i); -+ return -1; -+ } -+ start = ks->key_material_offset; -+ len = key_material_length_in_sectors (h, i); -+ if (len > 4096) /* bound it at something reasonable */ { -+ nbdkit_error ("bad LUKSv1 header: key slot %zu key material length " -+ "is too large", i); -+ return -1; -+ } -+ if (start * LUKS_SECTOR_SIZE >= size || -+ (start + len) * LUKS_SECTOR_SIZE >= size) { -+ nbdkit_error ("bad LUKSv1 header: key slot %zu key material offset " -+ "points beyond the end of the disk", i); -+ return -1; -+ } -+ if (ks->password_iterations > ULONG_MAX) { -+ nbdkit_error ("bad LUKSv1 header: key slot %zu " -+ "iterations too large", i); -+ return -1; -+ } -+ /*FALLTHROUGH*/ -+ case LUKS_KEY_DISABLED: -+ break; -+ -+ default: -+ nbdkit_error ("bad LUKSv1 header: key slot %zu has " -+ "an invalid active flag", i); -+ return -1; -+ } -+ } -+ -+ /* Decode the ciphers. */ -+ if (parse_cipher_strings (h) == -1) -+ return -1; -+ -+ /* Dump some information about the header. */ -+ memcpy (uuid, h->phdr.uuid, 40); -+ uuid[40] = 0; -+ nbdkit_debug ("LUKS UUID: %s", uuid); -+ -+ for (i = 0; i < LUKS_NUMKEYS; ++i) { -+ uint64_t start, len; -+ -+ ks = &h->phdr.keyslot[i]; -+ if (ks->active == LUKS_KEY_ENABLED) { -+ start = ks->key_material_offset; -+ len = key_material_length_in_sectors (h, i); -+ nbdkit_debug ("LUKS key slot %zu: key material in sectors %" PRIu64 -+ "..%" PRIu64, -+ i, start, start+len-1); -+ } -+ } -+ -+ /* Now try to unlock the master key. */ -+ for (i = 0; i < LUKS_NUMKEYS; ++i) { -+ r = try_passphrase_in_keyslot (next, h, i); -+ if (r == -1) -+ return -1; -+ if (r > 0) -+ goto unlocked; -+ } -+ nbdkit_error ("LUKS passphrase is not correct, " -+ "no key slot could be unlocked"); -+ return -1; -+ -+ unlocked: -+ assert (h->masterkey != NULL); -+ nbdkit_debug ("LUKS unlocked block device with passphrase"); -+ -+ return 0; -+} -+ -+static int64_t -+luks_get_size (nbdkit_next *next, void *handle) -+{ -+ struct handle *h = handle; -+ int64_t size; -+ -+ /* Check that prepare has been called already. */ -+ assert (h->phdr.version > 0); -+ -+ size = next->get_size (next); -+ if (size == -1) -+ return -1; -+ -+ if (size < h->phdr.payload_offset * LUKS_SECTOR_SIZE) { -+ nbdkit_error ("disk too small, or contains an incomplete LUKS partition"); -+ return -1; -+ } -+ -+ size -= h->phdr.payload_offset * LUKS_SECTOR_SIZE; -+ return size; -+} -+ -+/* Whatever the plugin says, several operations are not supported by -+ * this filter: -+ * -+ * - extents -+ * - trim -+ * - zero -+ */ -+static int -+luks_can_extents (nbdkit_next *next, void *handle) -+{ -+ return 0; -+} -+ -+static int -+luks_can_trim (nbdkit_next *next, void *handle) -+{ -+ return 0; -+} -+ -+static int -+luks_can_zero (nbdkit_next *next, void *handle) -+{ -+ return NBDKIT_ZERO_EMULATE; -+} -+ -+static int -+luks_can_fast_zero (nbdkit_next *next, void *handle) -+{ -+ return 0; -+} -+ -+/* Rely on nbdkit to call .pread to emulate .cache calls. We will -+ * respond by decrypting the block which could be stored by the cache -+ * filter or similar on top. -+ */ -+static int -+luks_can_cache (nbdkit_next *next, void *handle) -+{ -+ return NBDKIT_CACHE_EMULATE; -+} -+ -+/* Advertise minimum/preferred sector-sized blocks, although we can in -+ * fact handle any read or write. -+ */ -+static int -+luks_block_size (nbdkit_next *next, void *handle, -+ uint32_t *minimum, uint32_t *preferred, uint32_t *maximum) -+{ -+ if (next->block_size (next, minimum, preferred, maximum) == -1) -+ return -1; -+ -+ if (*minimum == 0) { /* No constraints set by the plugin. */ -+ *minimum = LUKS_SECTOR_SIZE; -+ *preferred = LUKS_SECTOR_SIZE; -+ *maximum = 0xffffffff; -+ } -+ else { -+ *minimum = MAX (*minimum, LUKS_SECTOR_SIZE); -+ *preferred = MAX (*minimum, MAX (*preferred, LUKS_SECTOR_SIZE)); -+ } -+ return 0; -+} -+ -+/* Decrypt data. */ -+static int -+luks_pread (nbdkit_next *next, void *handle, -+ void *buf, uint32_t count, uint64_t offset, -+ uint32_t flags, int *err) -+{ -+ struct handle *h = handle; -+ const uint64_t payload_offset = h->phdr.payload_offset * LUKS_SECTOR_SIZE; -+ CLEANUP_FREE uint8_t *sector = NULL; -+ uint64_t sectnum, sectoffs; -+ const gnutls_datum_t mkey = -+ { (unsigned char *) h->masterkey, h->phdr.master_key_len }; -+ gnutls_cipher_hd_t cipher; -+ int r; -+ -+ if (!h->masterkey) { -+ *err = EIO; -+ return -1; -+ } -+ -+ if (!IS_ALIGNED (count | offset, LUKS_SECTOR_SIZE)) { -+ sector = malloc (LUKS_SECTOR_SIZE); -+ if (sector == NULL) { -+ *err = errno; -+ nbdkit_error ("malloc: %m"); -+ return -1; -+ } -+ } -+ -+ r = gnutls_cipher_init (&cipher, h->gnutls_cipher, &mkey, NULL); -+ if (r != 0) { -+ nbdkit_error ("gnutls_cipher_init: %s", gnutls_strerror (r)); -+ *err = EIO; -+ return -1; -+ } -+ -+ sectnum = offset / LUKS_SECTOR_SIZE; /* sector number */ -+ sectoffs = offset % LUKS_SECTOR_SIZE; /* offset within the sector */ -+ -+ /* Unaligned head */ -+ if (sectoffs) { -+ uint64_t n = MIN (LUKS_SECTOR_SIZE - sectoffs, count); -+ -+ assert (sector); -+ if (next->pread (next, sector, LUKS_SECTOR_SIZE, -+ sectnum * LUKS_SECTOR_SIZE + payload_offset, -+ flags, err) == -1) -+ goto err; -+ -+ if (do_decrypt (h, cipher, offset & ~LUKS_SECTOR_SIZE, -+ sector, LUKS_SECTOR_SIZE) == -1) -+ goto err; -+ -+ memcpy (buf, §or[sectoffs], n); -+ -+ buf += n; -+ count -= n; -+ offset += n; -+ sectnum++; -+ } -+ -+ /* Aligned body */ -+ while (count >= LUKS_SECTOR_SIZE) { -+ if (next->pread (next, buf, LUKS_SECTOR_SIZE, -+ sectnum * LUKS_SECTOR_SIZE + payload_offset, -+ flags, err) == -1) -+ goto err; -+ -+ if (do_decrypt (h, cipher, offset, buf, LUKS_SECTOR_SIZE) == -1) -+ goto err; -+ -+ buf += LUKS_SECTOR_SIZE; -+ count -= LUKS_SECTOR_SIZE; -+ offset += LUKS_SECTOR_SIZE; -+ sectnum++; -+ } -+ -+ /* Unaligned tail */ -+ if (count) { -+ assert (sector); -+ if (next->pread (next, sector, LUKS_SECTOR_SIZE, -+ sectnum * LUKS_SECTOR_SIZE + payload_offset, -+ flags, err) == -1) -+ goto err; -+ -+ if (do_decrypt (h, cipher, offset, sector, LUKS_SECTOR_SIZE) == -1) -+ goto err; -+ -+ memcpy (buf, sector, count); -+ } -+ -+ gnutls_cipher_deinit (cipher); -+ return 0; -+ -+ err: -+ gnutls_cipher_deinit (cipher); -+ return -1; -+} -+ -+/* Lock preventing read-modify-write cycles from overlapping. */ -+static pthread_mutex_t read_modify_write_lock = PTHREAD_MUTEX_INITIALIZER; -+ -+/* Encrypt data. */ -+static int -+luks_pwrite (nbdkit_next *next, void *handle, -+ const void *buf, uint32_t count, uint64_t offset, -+ uint32_t flags, int *err) -+{ -+ struct handle *h = handle; -+ const uint64_t payload_offset = h->phdr.payload_offset * LUKS_SECTOR_SIZE; -+ CLEANUP_FREE uint8_t *sector = NULL; -+ uint64_t sectnum, sectoffs; -+ const gnutls_datum_t mkey = -+ { (unsigned char *) h->masterkey, h->phdr.master_key_len }; -+ gnutls_cipher_hd_t cipher; -+ int r; -+ -+ if (!h->masterkey) { -+ *err = EIO; -+ return -1; -+ } -+ -+ sector = malloc (LUKS_SECTOR_SIZE); -+ if (sector == NULL) { -+ *err = errno; -+ nbdkit_error ("malloc: %m"); -+ return -1; -+ } -+ -+ r = gnutls_cipher_init (&cipher, h->gnutls_cipher, &mkey, NULL); -+ if (r != 0) { -+ nbdkit_error ("gnutls_cipher_init: %s", gnutls_strerror (r)); -+ *err = EIO; -+ return -1; -+ } -+ -+ sectnum = offset / LUKS_SECTOR_SIZE; /* sector number */ -+ sectoffs = offset % LUKS_SECTOR_SIZE; /* offset within the sector */ -+ -+ /* Unaligned head */ -+ if (sectoffs) { -+ ACQUIRE_LOCK_FOR_CURRENT_SCOPE (&read_modify_write_lock); -+ -+ uint64_t n = MIN (LUKS_SECTOR_SIZE - sectoffs, count); -+ -+ if (next->pread (next, sector, LUKS_SECTOR_SIZE, -+ sectnum * LUKS_SECTOR_SIZE + payload_offset, -+ flags, err) == -1) -+ goto err; -+ -+ memcpy (§or[sectoffs], buf, n); -+ -+ if (do_encrypt (h, cipher, offset & ~LUKS_SECTOR_SIZE, -+ sector, LUKS_SECTOR_SIZE) == -1) -+ goto err; -+ -+ if (next->pwrite (next, sector, LUKS_SECTOR_SIZE, -+ sectnum * LUKS_SECTOR_SIZE + payload_offset, -+ flags, err) == -1) -+ goto err; -+ -+ buf += n; -+ count -= n; -+ offset += n; -+ sectnum++; -+ } -+ -+ /* Aligned body */ -+ while (count >= LUKS_SECTOR_SIZE) { -+ memcpy (sector, buf, LUKS_SECTOR_SIZE); -+ -+ if (do_encrypt (h, cipher, offset, sector, LUKS_SECTOR_SIZE) == -1) -+ goto err; -+ -+ if (next->pwrite (next, sector, LUKS_SECTOR_SIZE, -+ sectnum * LUKS_SECTOR_SIZE + payload_offset, -+ flags, err) == -1) -+ goto err; -+ -+ buf += LUKS_SECTOR_SIZE; -+ count -= LUKS_SECTOR_SIZE; -+ offset += LUKS_SECTOR_SIZE; -+ sectnum++; -+ } -+ -+ /* Unaligned tail */ -+ if (count) { -+ ACQUIRE_LOCK_FOR_CURRENT_SCOPE (&read_modify_write_lock); -+ -+ if (next->pread (next, sector, LUKS_SECTOR_SIZE, -+ sectnum * LUKS_SECTOR_SIZE + payload_offset, -+ flags, err) == -1) -+ goto err; -+ -+ memcpy (sector, buf, count); -+ -+ if (do_encrypt (h, cipher, offset, sector, LUKS_SECTOR_SIZE) == -1) -+ goto err; -+ -+ if (next->pwrite (next, sector, LUKS_SECTOR_SIZE, -+ sectnum * LUKS_SECTOR_SIZE + payload_offset, -+ flags, err) == -1) -+ goto err; -+ } -+ -+ gnutls_cipher_deinit (cipher); -+ return 0; -+ -+ err: -+ gnutls_cipher_deinit (cipher); -+ return -1; -+} -+ -+static struct nbdkit_filter filter = { -+ .name = "luks", -+ .longname = "nbdkit luks filter", -+ .unload = luks_unload, -+ .thread_model = luks_thread_model, -+ .config = luks_config, -+ .config_complete = luks_config_complete, -+ .config_help = luks_config_help, -+ .open = luks_open, -+ .close = luks_close, -+ .prepare = luks_prepare, -+ .get_size = luks_get_size, -+ .can_extents = luks_can_extents, -+ .can_trim = luks_can_trim, -+ .can_zero = luks_can_zero, -+ .can_fast_zero = luks_can_fast_zero, -+ .can_cache = luks_can_cache, -+ .block_size = luks_block_size, -+ .pread = luks_pread, -+ .pwrite = luks_pwrite, -+}; -+ -+NBDKIT_REGISTER_FILTER(filter) -diff --git a/filters/luks/nbdkit-luks-filter.pod b/filters/luks/nbdkit-luks-filter.pod -new file mode 100644 -index 00000000..56e51561 ---- /dev/null -+++ b/filters/luks/nbdkit-luks-filter.pod -@@ -0,0 +1,120 @@ -+=head1 NAME -+ -+nbdkit-luks-filter - read and write LUKS-encrypted disks and partitions -+ -+=head1 SYNOPSIS -+ -+ nbdkit file encrypted-disk.img --filter=luks passphrase=+/tmp/secret -+ -+=head1 DESCRIPTION -+ -+C is a filter for L which transparently -+opens a LUKS-encrypted disk image. LUKS ("Linux Unified Key Setup") -+is the Full Disk Encryption (FDE) system commonly used by Linux -+systems. This filter is compatible with LUKSv1 as implemented by the -+Linux kernel (dm_crypt), and by qemu. -+ -+You can place this filter on top of L to -+decrypt a local file: -+ -+ nbdkit file encrypted-disk.img --filter=luks passphrase=+/tmp/secret -+ -+If LUKS is present inside a partition in the disk image then you will -+have to combine this filter with L. The -+order of the filters is important: -+ -+ nbdkit file encrypted-disk.img \ -+ --filter=luks passphrase=+/tmp/secret \ -+ --filter=partition partition=1 -+ -+This filter also works on top of other plugins such as -+L: -+ -+ nbdkit curl https://example.com/encrypted-disk.img \ -+ --filter=luks passphrase=+/tmp/secret -+ -+The web server sees only the encrypted data. Without knowing the -+passphrase, the web server cannot access the decrypted disk. Only -+encrypted data is sent over the HTTP connection. nbdkit itself will -+serve I disk data over the NBD connection (if this is a -+problem see L, or use a Unix domain socket I<-U>). -+ -+The passphrase can be stored in a file (as shown), passed directly on -+the command line (insecure), entered interactively, or passed to -+nbdkit over a file descriptor. -+ -+This filter can read and write LUKSv1. It cannot create disks, change -+passphrases, add keyslots, etc. To do that, you can use ordinary -+Linux tools like L. Note you must force LUKSv1 -+(eg. using cryptsetup I<--type luks1>). L can also -+create compatible disk images: -+ -+ qemu-img create -f luks \ -+ --object secret,data=SECRET,id=sec0 \ -+ -o key-secret=sec0 \ -+ encrypted-disk.img 1G -+ -+=head1 PARAMETERS -+ -+=over 4 -+ -+=item BSECRET -+ -+Use the secret passphrase when decrypting the disk. -+ -+Note that passing this on the command line is not secure on shared -+machines. -+ -+=item B -+ -+Ask for the passphrase (interactively) when nbdkit starts up. -+ -+=item BFILENAME -+ -+Read the passphrase from the named file. This is a secure method to -+supply a passphrase, as long as you set the permissions on the file -+appropriately. -+ -+=item BFD -+ -+Read the passphrase from file descriptor number C, inherited from -+the parent process when nbdkit starts up. This is also a secure -+method to supply a passphrase. -+ -+=back -+ -+=head1 FILES -+ -+=over 4 -+ -+=item F<$filterdir/nbdkit-luks-filter.so> -+ -+The plugin. -+ -+Use C to find the location of C<$filterdir>. -+ -+=back -+ -+=head1 VERSION -+ -+C first appeared in nbdkit 1.32. -+ -+=head1 SEE ALSO -+ -+L, -+L, -+L, -+L, -+L, -+L, -+L, -+L, -+L. -+ -+=head1 AUTHORS -+ -+Richard W.M. Jones -+ -+=head1 COPYRIGHT -+ -+Copyright (C) 2013-2022 Red Hat Inc. -diff --git a/plugins/file/nbdkit-file-plugin.pod b/plugins/file/nbdkit-file-plugin.pod -index f8f0e198..b95e7349 100644 ---- a/plugins/file/nbdkit-file-plugin.pod -+++ b/plugins/file/nbdkit-file-plugin.pod -@@ -223,6 +223,7 @@ L, - L, - L, - L, -+L, - L. - - =head1 AUTHORS -diff --git a/tests/Makefile.am b/tests/Makefile.am -index b310e8a2..c29453ba 100644 ---- a/tests/Makefile.am -+++ b/tests/Makefile.am -@@ -1596,6 +1596,18 @@ EXTRA_DIST += \ - test-log-script-info.sh \ - $(NULL) - -+# luks filter test. -+if HAVE_GNUTLS -+TESTS += \ -+ test-luks-info.sh \ -+ test-luks-copy.sh \ -+ $(NULL) -+endif -+EXTRA_DIST += \ -+ test-luks-info.sh \ -+ test-luks-copy.sh \ -+ $(NULL) -+ - # multi-conn filter test. - TESTS += \ - test-multi-conn.sh \ -diff --git a/tests/test-luks-copy.sh b/tests/test-luks-copy.sh -new file mode 100755 -index 00000000..99f300d0 ---- /dev/null -+++ b/tests/test-luks-copy.sh -@@ -0,0 +1,125 @@ -+#!/usr/bin/env bash -+# nbdkit -+# Copyright (C) 2018-2022 Red Hat Inc. -+# -+# Redistribution and use in source and binary forms, with or without -+# modification, are permitted provided that the following conditions are -+# met: -+# -+# * Redistributions of source code must retain the above copyright -+# notice, this list of conditions and the following disclaimer. -+# -+# * Redistributions in binary form must reproduce the above copyright -+# notice, this list of conditions and the following disclaimer in the -+# documentation and/or other materials provided with the distribution. -+# -+# * Neither the name of Red Hat nor the names of its contributors may be -+# used to endorse or promote products derived from this software without -+# specific prior written permission. -+# -+# THIS SOFTWARE IS PROVIDED BY RED HAT AND CONTRIBUTORS ''AS IS'' AND -+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, -+# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A -+# PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL RED HAT OR -+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF -+# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND -+# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -+# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT -+# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF -+# SUCH DAMAGE. -+ -+source ./functions.sh -+set -e -+set -x -+ -+requires nbdcopy --version -+requires nbdsh --version -+requires_nbdsh_uri -+requires qemu-img --version -+requires bash -c 'qemu-img --help | grep -- --target-image-opts' -+requires hexdump --version -+requires truncate --version -+requires_filter luks -+ -+encrypt_disk=luks-copy1.img -+plain_disk=luks-copy2.img -+pid=luks-copy.pid -+sock=$(mktemp -u /tmp/nbdkit-test-sock.XXXXXX) -+cleanup_fn rm -f $encrypt_disk $plain_disk $pid $sock -+rm -f $encrypt_disk $plain_disk $pid $sock -+ -+# Create an empty encrypted disk container. -+# -+# NB: This is complicated because qemu doesn't create an all-zeroes -+# plaintext disk for some reason when you use create -f luks. It -+# starts with random plaintext. -+# -+# https://stackoverflow.com/a/44669936 -+qemu-img create -f luks \ -+ --object secret,data=123456,id=sec0 \ -+ -o key-secret=sec0 \ -+ $encrypt_disk 10M -+truncate -s 10M $plain_disk -+qemu-img convert --target-image-opts -n \ -+ --object secret,data=123456,id=sec0 \ -+ $plain_disk \ -+ driver=luks,file.filename=$encrypt_disk,key-secret=sec0 -+rm $plain_disk -+ -+# Start nbdkit on the encrypted disk. -+start_nbdkit -P $pid -U $sock \ -+ file $encrypt_disk --filter=luks passphrase=123456 -+uri="nbd+unix:///?socket=$sock" -+ -+# Copy the whole disk out. It should be empty. -+nbdcopy "$uri" $plain_disk -+ -+if [ "$(hexdump -C $plain_disk)" != '00000000 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| -+* -+00a00000' ]; then -+ echo "$0: expected plaintext disk to be empty" -+ exit 1 -+fi -+ -+# Use nbdsh to overwrite with some known data and check we can read -+# back what we wrote. -+nbdsh -u "$uri" \ -+ -c 'h.pwrite(b"1"*65536, 0)' \ -+ -c 'h.pwrite(b"2"*65536, 128*1024)' \ -+ -c 'h.pwrite(b"3"*65536, 9*1024*1024)' \ -+ -c 'buf = h.pread(65536, 0)' \ -+ -c 'assert buf == b"1"*65536' \ -+ -c 'buf = h.pread(65536, 65536)' \ -+ -c 'assert buf == bytearray(65536)' \ -+ -c 'buf = h.pread(65536, 128*1024)' \ -+ -c 'assert buf == b"2"*65536' \ -+ -c 'buf = h.pread(65536, 9*1024*1024)' \ -+ -c 'assert buf == b"3"*65536' \ -+ -c 'h.flush()' -+ -+# Use qemu to copy out the whole disk. Note we called flush() above -+# so the disk should be synchronised. -+qemu-img convert --image-opts \ -+ --object secret,data=123456,id=sec0 \ -+ driver=luks,file.filename=$encrypt_disk,key-secret=sec0 \ -+ $plain_disk -+ -+# Check the contents are expected. -+if [ "$(hexdump -C $plain_disk)" != '00000000 31 31 31 31 31 31 31 31 31 31 31 31 31 31 31 31 |1111111111111111| -+* -+00010000 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| -+* -+00020000 32 32 32 32 32 32 32 32 32 32 32 32 32 32 32 32 |2222222222222222| -+* -+00030000 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| -+* -+00900000 33 33 33 33 33 33 33 33 33 33 33 33 33 33 33 33 |3333333333333333| -+* -+00910000 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| -+* -+00a00000' ]; then -+ echo "$0: unexpected content" -+ exit 1 -+fi -diff --git a/tests/test-luks-info.sh b/tests/test-luks-info.sh -new file mode 100755 -index 00000000..3eff657b ---- /dev/null -+++ b/tests/test-luks-info.sh -@@ -0,0 +1,56 @@ -+#!/usr/bin/env bash -+# nbdkit -+# Copyright (C) 2018-2022 Red Hat Inc. -+# -+# Redistribution and use in source and binary forms, with or without -+# modification, are permitted provided that the following conditions are -+# met: -+# -+# * Redistributions of source code must retain the above copyright -+# notice, this list of conditions and the following disclaimer. -+# -+# * Redistributions in binary form must reproduce the above copyright -+# notice, this list of conditions and the following disclaimer in the -+# documentation and/or other materials provided with the distribution. -+# -+# * Neither the name of Red Hat nor the names of its contributors may be -+# used to endorse or promote products derived from this software without -+# specific prior written permission. -+# -+# THIS SOFTWARE IS PROVIDED BY RED HAT AND CONTRIBUTORS ''AS IS'' AND -+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, -+# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A -+# PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL RED HAT OR -+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF -+# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND -+# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -+# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT -+# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF -+# SUCH DAMAGE. -+ -+source ./functions.sh -+set -e -+set -x -+ -+requires nbdinfo --version -+requires qemu-img --version -+requires_filter luks -+ -+disk=luks-info.img -+info=luks-info.log -+cleanup_fn rm -f $disk $info -+rm -f $disk $info -+ -+qemu-img create -f luks \ -+ --object secret,data=123456,id=sec0 \ -+ -o key-secret=sec0 \ -+ $disk 10M -+ -+nbdkit -U - file $disk --filter=luks passphrase=123456 \ -+ --run 'nbdinfo $uri' > $info -+cat $info -+ -+# Check the size is 10M (so it doesn't include the LUKS header). -+grep "10485760" $info --- -2.31.1 - diff --git a/0005-luks-Disable-filter-with-old-GnuTLS-in-Debian-10.patch b/0005-luks-Disable-filter-with-old-GnuTLS-in-Debian-10.patch deleted file mode 100644 index 2ff4e07..0000000 --- a/0005-luks-Disable-filter-with-old-GnuTLS-in-Debian-10.patch +++ /dev/null @@ -1,95 +0,0 @@ -From 66daae1a7daf680e06f884e9af6a14830263c932 Mon Sep 17 00:00:00 2001 -From: "Richard W.M. Jones" -Date: Sun, 8 May 2022 12:13:39 +0100 -Subject: [PATCH] luks: Disable filter with old GnuTLS in Debian 10 -MIME-Version: 1.0 -Content-Type: text/plain; charset=UTF-8 -Content-Transfer-Encoding: 8bit - -On Debian 10: - -luks.c: In function ‘parse_cipher_strings’: -luks.c:574:26: error: ‘GNUTLS_CIPHER_AES_128_XTS’ undeclared (first use in this function); did you mean ‘GNUTLS_CIPHER_AES_128_CCM’? - h->gnutls_cipher = GNUTLS_CIPHER_AES_128_XTS; - ^~~~~~~~~~~~~~~~~~~~~~~~~ - GNUTLS_CIPHER_AES_128_CCM -luks.c:574:26: note: each undeclared identifier is reported only once for each function it appears in -luks.c:577:26: error: ‘GNUTLS_CIPHER_AES_256_XTS’ undeclared (first use in this function); did you mean ‘GNUTLS_CIPHER_AES_256_CCM’? - h->gnutls_cipher = GNUTLS_CIPHER_AES_256_XTS; - ^~~~~~~~~~~~~~~~~~~~~~~~~ - GNUTLS_CIPHER_AES_256_CCM -luks.c: In function ‘try_passphrase_in_keyslot’: -luks.c:728:7: error: implicit declaration of function ‘gnutls_pbkdf2’; did you mean ‘gnutls_prf’? [-Werror=implicit-function-declaration] - r = gnutls_pbkdf2 (h->hash_alg, &key, &salt, ks->password_iterations, - ^~~~~~~~~~~~~ - gnutls_prf - -Because gnutls_pbkdf2 is missing there's no chance of making this -filter work on this platform so it's best to compile it out. - -Fixes: commit 468919dce6c5eb57503eacac0f67e5dd87c58e6c -(cherry picked from commit f9f67e483f4aad19ad6101163d32562f13504ca7) ---- - configure.ac | 5 ++++- - filters/luks/Makefile.am | 2 +- - tests/Makefile.am | 2 +- - 3 files changed, 6 insertions(+), 3 deletions(-) - -diff --git a/configure.ac b/configure.ac -index de85b4da..1d209f67 100644 ---- a/configure.ac -+++ b/configure.ac -@@ -636,12 +636,15 @@ AS_IF([test "x$GNUTLS_LIBS" != "x"],[ - gnutls_certificate_set_known_dh_params \ - gnutls_group_get \ - gnutls_group_get_name \ -+ gnutls_pbkdf2 \ - gnutls_session_set_verify_cert \ - gnutls_srp_server_get_username \ - gnutls_transport_is_ktls_enabled \ - ]) - LIBS="$old_LIBS" - ]) -+AM_CONDITIONAL([HAVE_GNUTLS_PBKDF2], -+ [test "x$GNUTLS_LIBS" != "x" && test "x$ac_cv_func_gnutls_pbkdf2" = xyes]) - - AC_ARG_ENABLE([linuxdisk], - [AS_HELP_STRING([--disable-linuxdisk], -@@ -1484,7 +1487,7 @@ echo "Optional filters:" - echo - feature "ext2" test "x$HAVE_EXT2_TRUE" = "x" - feature "gzip" test "x$HAVE_ZLIB_TRUE" = "x" --feature "LUKS" test "x$HAVE_GNUTLS_TRUE" != "x" -+feature "luks" test "x$HAVE_GNUTLS_PBKDF2_TRUE" = "x" - feature "xz" test "x$HAVE_LIBLZMA_TRUE" = "x" - - echo -diff --git a/filters/luks/Makefile.am b/filters/luks/Makefile.am -index 30089621..622e5c3d 100644 ---- a/filters/luks/Makefile.am -+++ b/filters/luks/Makefile.am -@@ -33,7 +33,7 @@ include $(top_srcdir)/common-rules.mk - - EXTRA_DIST = nbdkit-luks-filter.pod - --if HAVE_GNUTLS -+if HAVE_GNUTLS_PBKDF2 - - filter_LTLIBRARIES = nbdkit-luks-filter.la - -diff --git a/tests/Makefile.am b/tests/Makefile.am -index c29453ba..5585b3b7 100644 ---- a/tests/Makefile.am -+++ b/tests/Makefile.am -@@ -1597,7 +1597,7 @@ EXTRA_DIST += \ - $(NULL) - - # luks filter test. --if HAVE_GNUTLS -+if HAVE_GNUTLS_PBKDF2 - TESTS += \ - test-luks-info.sh \ - test-luks-copy.sh \ --- -2.31.1 - diff --git a/0006-luks-Various-fixes-for-Clang.patch b/0006-luks-Various-fixes-for-Clang.patch deleted file mode 100644 index c0f7f2f..0000000 --- a/0006-luks-Various-fixes-for-Clang.patch +++ /dev/null @@ -1,71 +0,0 @@ -From b3c05065801c723966a3e8d93c9b84e808ff38b9 Mon Sep 17 00:00:00 2001 -From: "Richard W.M. Jones" -Date: Sun, 8 May 2022 12:30:09 +0100 -Subject: [PATCH] luks: Various fixes for Clang - -With Clang: - -luks.c:728:25: error: implicit conversion from enumeration type 'gnutls_digest_algorithm_t' to different enumeration type 'gnutls_mac_algorithm_t' [-Werror,-Wenum-conversion] - r = gnutls_pbkdf2 (h->hash_alg, &key, &salt, ks->password_iterations, - ~~~~~~~~~~~~~ ~~~^~~~~~~~ -luks.c:764:25: error: implicit conversion from enumeration type 'gnutls_digest_algorithm_t' to different enumeration type 'gnutls_mac_algorithm_t' [-Werror,-Wenum-conversion] - r = gnutls_pbkdf2 (h->hash_alg, &mkey, &msalt, - ~~~~~~~~~~~~~ ~~~^~~~~~~~ -luks.c:886:35: error: result of comparison of constant 18446744073709551615 with expression of type 'uint32_t' (aka 'unsigned int') is always false [-Werror,-Wtautological-constant-out-of-range-compare] - if (ks->password_iterations > ULONG_MAX) { - ~~~~~~~~~~~~~~~~~~~~~~~ ^ ~~~~~~~~~ - -Fixes: commit 468919dce6c5eb57503eacac0f67e5dd87c58e6c -(cherry picked from commit 87d488ede9101a2effc71cd1851bf4a4caa521d2) ---- - filters/luks/luks.c | 13 ++++++------- - 1 file changed, 6 insertions(+), 7 deletions(-) - -diff --git a/filters/luks/luks.c b/filters/luks/luks.c -index 706a9bd2..cc619698 100644 ---- a/filters/luks/luks.c -+++ b/filters/luks/luks.c -@@ -693,6 +693,10 @@ key_material_length_in_sectors (struct handle *h, size_t i) - static int - try_passphrase_in_keyslot (nbdkit_next *next, struct handle *h, size_t i) - { -+ /* I believe this is supposed to be safe, looking at the GnuTLS -+ * header file. -+ */ -+ const gnutls_mac_algorithm_t mac = (gnutls_mac_algorithm_t) h->hash_alg; - struct luks_keyslot *ks = &h->phdr.keyslot[i]; - size_t split_key_len; - CLEANUP_FREE uint8_t *split_key = NULL; -@@ -725,7 +729,7 @@ try_passphrase_in_keyslot (nbdkit_next *next, struct handle *h, size_t i) - } - - /* Hash the passphrase to make a possible masterkey. */ -- r = gnutls_pbkdf2 (h->hash_alg, &key, &salt, ks->password_iterations, -+ r = gnutls_pbkdf2 (mac, &key, &salt, ks->password_iterations, - masterkey, h->phdr.master_key_len); - if (r != 0) { - nbdkit_error ("gnutls_pbkdf2: %s", gnutls_strerror (r)); -@@ -761,7 +765,7 @@ try_passphrase_in_keyslot (nbdkit_next *next, struct handle *h, size_t i) - /* Check if the masterkey is correct by comparing hash of the - * masterkey with LUKS header. - */ -- r = gnutls_pbkdf2 (h->hash_alg, &mkey, &msalt, -+ r = gnutls_pbkdf2 (mac, &mkey, &msalt, - h->phdr.master_key_digest_iterations, - key_digest, LUKS_DIGESTSIZE); - if (r != 0) { -@@ -883,11 +887,6 @@ luks_prepare (nbdkit_next *next, void *handle, int readonly) - "points beyond the end of the disk", i); - return -1; - } -- if (ks->password_iterations > ULONG_MAX) { -- nbdkit_error ("bad LUKSv1 header: key slot %zu " -- "iterations too large", i); -- return -1; -- } - /*FALLTHROUGH*/ - case LUKS_KEY_DISABLED: - break; --- -2.31.1 - diff --git a/0007-luks-Link-with-libcompat-on-Windows.patch b/0007-luks-Link-with-libcompat-on-Windows.patch deleted file mode 100644 index f93465b..0000000 --- a/0007-luks-Link-with-libcompat-on-Windows.patch +++ /dev/null @@ -1,43 +0,0 @@ -From 9416effd73a5cb2e1c929449fca88fd7152aa1be Mon Sep 17 00:00:00 2001 -From: "Richard W.M. Jones" -Date: Sun, 8 May 2022 12:38:00 +0100 -Subject: [PATCH] luks: Link with libcompat on Windows - -/usr/lib/gcc/x86_64-w64-mingw32/11.2.1/../../../../x86_64-w64-mingw32/bin/ld: ../../common/utils/.libs/libutils.a(libutils_la-full-rw.o): in function `full_pread': -/builds/nbdkit/nbdkit/common/utils/full-rw.c:53: undefined reference to `pread' -/usr/lib/gcc/x86_64-w64-mingw32/11.2.1/../../../../x86_64-w64-mingw32/bin/ld: ../../common/utils/.libs/libutils.a(libutils_la-full-rw.o): in function `full_pwrite': -/builds/nbdkit/nbdkit/common/utils/full-rw.c:76: undefined reference to `pwrite' -/usr/lib/gcc/x86_64-w64-mingw32/11.2.1/../../../../x86_64-w64-mingw32/bin/ld: ../../common/utils/.libs/libutils.a(libutils_la-vector.o): in function `generic_vector_reserve_page_aligned': -/builds/nbdkit/nbdkit/common/utils/vector.c:112: undefined reference to `sysconf' -/usr/lib/gcc/x86_64-w64-mingw32/11.2.1/../../../../x86_64-w64-mingw32/bin/ld: /builds/nbdkit/nbdkit/common/utils/vector.c:134: undefined reference to `posix_memalign' -collect2: error: ld returned 1 exit status - -Fixes: commit 468919dce6c5eb57503eacac0f67e5dd87c58e6c -(cherry picked from commit 4a28c4c46aedf270929a62a1c5ecf2c1129cd456) ---- - filters/luks/Makefile.am | 2 ++ - 1 file changed, 2 insertions(+) - -diff --git a/filters/luks/Makefile.am b/filters/luks/Makefile.am -index 622e5c3d..2688f696 100644 ---- a/filters/luks/Makefile.am -+++ b/filters/luks/Makefile.am -@@ -45,6 +45,7 @@ nbdkit_luks_filter_la_SOURCES = \ - nbdkit_luks_filter_la_CPPFLAGS = \ - -I$(top_srcdir)/include \ - -I$(top_srcdir)/common/include \ -+ -I$(top_srcdir)/common/replacements \ - -I$(top_srcdir)/common/utils \ - $(NULL) - nbdkit_luks_filter_la_CFLAGS = \ -@@ -53,6 +54,7 @@ nbdkit_luks_filter_la_CFLAGS = \ - $(NULL) - nbdkit_luks_filter_la_LIBADD = \ - $(top_builddir)/common/utils/libutils.la \ -+ $(top_builddir)/common/replacements/libcompat.la \ - $(IMPORT_LIBRARY_ON_WINDOWS) \ - $(GNUTLS_LIBS) \ - $(NULL) --- -2.31.1 - diff --git a/0008-luks-Refactor-the-filter.patch b/0008-luks-Refactor-the-filter.patch deleted file mode 100644 index feaf986..0000000 --- a/0008-luks-Refactor-the-filter.patch +++ /dev/null @@ -1,2096 +0,0 @@ -From e8279107801bb93303b22e1b927929ce18279dc5 Mon Sep 17 00:00:00 2001 -From: "Richard W.M. Jones" -Date: Sun, 8 May 2022 16:13:13 +0100 -Subject: [PATCH] luks: Refactor the filter - -Move the LUKS-specific code out into a separate file. This is mostly -pure refactoring to tidy things up. - -(cherry picked from commit 2159d85e0bed1943542da58b43c91f2caa096d6c) ---- - filters/luks/Makefile.am | 2 + - filters/luks/luks-encryption.c | 926 +++++++++++++++++++++++++++++++++ - filters/luks/luks-encryption.h | 78 +++ - filters/luks/luks.c | 874 ++----------------------------- - 4 files changed, 1037 insertions(+), 843 deletions(-) - create mode 100644 filters/luks/luks-encryption.c - create mode 100644 filters/luks/luks-encryption.h - -diff --git a/filters/luks/Makefile.am b/filters/luks/Makefile.am -index 2688f696..0894ac8b 100644 ---- a/filters/luks/Makefile.am -+++ b/filters/luks/Makefile.am -@@ -38,6 +38,8 @@ if HAVE_GNUTLS_PBKDF2 - filter_LTLIBRARIES = nbdkit-luks-filter.la - - nbdkit_luks_filter_la_SOURCES = \ -+ luks-encryption.c \ -+ luks-encryption.h \ - luks.c \ - $(top_srcdir)/include/nbdkit-filter.h \ - $(NULL) -diff --git a/filters/luks/luks-encryption.c b/filters/luks/luks-encryption.c -new file mode 100644 -index 00000000..8ee0eb35 ---- /dev/null -+++ b/filters/luks/luks-encryption.c -@@ -0,0 +1,926 @@ -+/* nbdkit -+ * Copyright (C) 2018-2022 Red Hat Inc. -+ * -+ * Redistribution and use in source and binary forms, with or without -+ * modification, are permitted provided that the following conditions are -+ * met: -+ * -+ * * Redistributions of source code must retain the above copyright -+ * notice, this list of conditions and the following disclaimer. -+ * -+ * * Redistributions in binary form must reproduce the above copyright -+ * notice, this list of conditions and the following disclaimer in the -+ * documentation and/or other materials provided with the distribution. -+ * -+ * * Neither the name of Red Hat nor the names of its contributors may be -+ * used to endorse or promote products derived from this software without -+ * specific prior written permission. -+ * -+ * THIS SOFTWARE IS PROVIDED BY RED HAT AND CONTRIBUTORS ''AS IS'' AND -+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, -+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A -+ * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL RED HAT OR -+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF -+ * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND -+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT -+ * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF -+ * SUCH DAMAGE. -+ */ -+ -+#include -+ -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+ -+#include -+ -+#include -+ -+#include "luks-encryption.h" -+ -+#include "byte-swapping.h" -+#include "cleanup.h" -+#include "isaligned.h" -+#include "rounding.h" -+ -+/* LUKSv1 constants. */ -+#define LUKS_MAGIC { 'L', 'U', 'K', 'S', 0xBA, 0xBE } -+#define LUKS_MAGIC_LEN 6 -+#define LUKS_DIGESTSIZE 20 -+#define LUKS_SALTSIZE 32 -+#define LUKS_NUMKEYS 8 -+#define LUKS_KEY_DISABLED 0x0000DEAD -+#define LUKS_KEY_ENABLED 0x00AC71F3 -+#define LUKS_STRIPES 4000 -+#define LUKS_ALIGN_KEYSLOTS 4096 -+ -+/* Key slot. */ -+struct luks_keyslot { -+ uint32_t active; /* LUKS_KEY_DISABLED|LUKS_KEY_ENABLED */ -+ uint32_t password_iterations; -+ char password_salt[LUKS_SALTSIZE]; -+ uint32_t key_material_offset; -+ uint32_t stripes; -+} __attribute__((__packed__)); -+ -+/* LUKS superblock. */ -+struct luks_phdr { -+ char magic[LUKS_MAGIC_LEN]; /* LUKS_MAGIC */ -+ uint16_t version; /* Only 1 is supported. */ -+ char cipher_name[32]; -+ char cipher_mode[32]; -+ char hash_spec[32]; -+ uint32_t payload_offset; -+ uint32_t master_key_len; -+ uint8_t master_key_digest[LUKS_DIGESTSIZE]; -+ uint8_t master_key_salt[LUKS_SALTSIZE]; -+ uint32_t master_key_digest_iterations; -+ uint8_t uuid[40]; -+ -+ struct luks_keyslot keyslot[LUKS_NUMKEYS]; /* Key slots. */ -+} __attribute__((__packed__)); -+ -+/* Block cipher mode of operation. -+ * https://en.wikipedia.org/wiki/Block_cipher_mode_of_operation -+ */ -+enum cipher_mode { -+ CIPHER_MODE_ECB, CIPHER_MODE_CBC, CIPHER_MODE_XTS, CIPHER_MODE_CTR, -+}; -+ -+static enum cipher_mode -+lookup_cipher_mode (const char *str) -+{ -+ if (strcmp (str, "ecb") == 0) -+ return CIPHER_MODE_ECB; -+ if (strcmp (str, "cbc") == 0) -+ return CIPHER_MODE_CBC; -+ if (strcmp (str, "xts") == 0) -+ return CIPHER_MODE_XTS; -+ if (strcmp (str, "ctr") == 0) -+ return CIPHER_MODE_CTR; -+ nbdkit_error ("unknown cipher mode: %s " -+ "(expecting \"ecb\", \"cbc\", \"xts\" or \"ctr\")", str); -+ return -1; -+} -+ -+static const char * -+cipher_mode_to_string (enum cipher_mode v) -+{ -+ switch (v) { -+ case CIPHER_MODE_ECB: return "ecb"; -+ case CIPHER_MODE_CBC: return "cbc"; -+ case CIPHER_MODE_XTS: return "xts"; -+ case CIPHER_MODE_CTR: return "ctr"; -+ default: abort (); -+ } -+} -+ -+/* Methods used by LUKS to generate initial vectors. -+ * -+ * ESSIV is a bit more complicated to implement. It is supported by -+ * qemu but not by us. -+ */ -+enum ivgen { -+ IVGEN_PLAIN, IVGEN_PLAIN64, /* IVGEN_ESSIV, */ -+}; -+ -+static enum ivgen -+lookup_ivgen (const char *str) -+{ -+ if (strcmp (str, "plain") == 0) -+ return IVGEN_PLAIN; -+ if (strcmp (str, "plain64") == 0) -+ return IVGEN_PLAIN64; -+/* -+ if (strcmp (str, "essiv") == 0) -+ return IVGEN_ESSIV; -+*/ -+ nbdkit_error ("unknown IV generation algorithm: %s " -+ "(expecting \"plain\", \"plain64\" etc)", str); -+ return -1; -+} -+ -+static const char * -+ivgen_to_string (enum ivgen v) -+{ -+ switch (v) { -+ case IVGEN_PLAIN: return "plain"; -+ case IVGEN_PLAIN64: return "plain64"; -+ /*case IVGEN_ESSIV: return "essiv";*/ -+ default: abort (); -+ } -+} -+ -+static void -+calculate_iv (enum ivgen v, uint8_t *iv, size_t ivlen, uint64_t sector) -+{ -+ size_t prefixlen; -+ uint32_t sector32; -+ -+ switch (v) { -+ case IVGEN_PLAIN: -+ prefixlen = 4; /* 32 bits */ -+ if (prefixlen > ivlen) -+ prefixlen = ivlen; -+ sector32 = (uint32_t) sector; /* truncate to only lower bits */ -+ sector32 = htole32 (sector32); -+ memcpy (iv, §or32, prefixlen); -+ memset (iv + prefixlen, 0, ivlen - prefixlen); -+ break; -+ -+ case IVGEN_PLAIN64: -+ prefixlen = 8; /* 64 bits */ -+ if (prefixlen > ivlen) -+ prefixlen = ivlen; -+ sector = htole64 (sector); -+ memcpy (iv, §or, prefixlen); -+ memset (iv + prefixlen, 0, ivlen - prefixlen); -+ break; -+ -+ /*case IVGEN_ESSIV:*/ -+ default: abort (); -+ } -+} -+ -+/* Cipher algorithm. -+ * -+ * qemu in theory supports many more, but with the GnuTLS backend only -+ * AES is supported. The kernel seems to only support AES for LUKSv1. -+ */ -+enum cipher_alg { -+ CIPHER_ALG_AES_128, CIPHER_ALG_AES_192, CIPHER_ALG_AES_256, -+}; -+ -+static enum cipher_alg -+lookup_cipher_alg (const char *str, enum cipher_mode mode, int key_bytes) -+{ -+ if (mode == CIPHER_MODE_XTS) -+ key_bytes /= 2; -+ -+ if (strcmp (str, "aes") == 0) { -+ if (key_bytes == 16) -+ return CIPHER_ALG_AES_128; -+ if (key_bytes == 24) -+ return CIPHER_ALG_AES_192; -+ if (key_bytes == 32) -+ return CIPHER_ALG_AES_256; -+ } -+ nbdkit_error ("unknown cipher algorithm: %s (expecting \"aes\", etc)", str); -+ return -1; -+} -+ -+static const char * -+cipher_alg_to_string (enum cipher_alg v) -+{ -+ switch (v) { -+ case CIPHER_ALG_AES_128: return "aes-128"; -+ case CIPHER_ALG_AES_192: return "aes-192"; -+ case CIPHER_ALG_AES_256: return "aes-256"; -+ default: abort (); -+ } -+} -+ -+#if 0 -+static int -+cipher_alg_key_bytes (enum cipher_alg v) -+{ -+ switch (v) { -+ case CIPHER_ALG_AES_128: return 16; -+ case CIPHER_ALG_AES_192: return 24; -+ case CIPHER_ALG_AES_256: return 32; -+ default: abort (); -+ } -+} -+#endif -+ -+static int -+cipher_alg_iv_len (enum cipher_alg v, enum cipher_mode mode) -+{ -+ if (CIPHER_MODE_ECB) -+ return 0; /* Don't need an IV in this mode. */ -+ -+ switch (v) { -+ case CIPHER_ALG_AES_128: -+ case CIPHER_ALG_AES_192: -+ case CIPHER_ALG_AES_256: -+ return 16; -+ default: abort (); -+ } -+} -+ -+/* Hash, eg.MD5, SHA1 etc. -+ * -+ * We reuse the GnuTLS digest algorithm enum here since it supports at -+ * least all the ones that LUKSv1 does. -+ */ -+static gnutls_digest_algorithm_t -+lookup_hash (const char *str) -+{ -+ if (strcmp (str, "md5") == 0) -+ return GNUTLS_DIG_MD5; -+ if (strcmp (str, "sha1") == 0) -+ return GNUTLS_DIG_SHA1; -+ if (strcmp (str, "sha224") == 0) -+ return GNUTLS_DIG_SHA224; -+ if (strcmp (str, "sha256") == 0) -+ return GNUTLS_DIG_SHA256; -+ if (strcmp (str, "sha384") == 0) -+ return GNUTLS_DIG_SHA384; -+ if (strcmp (str, "sha512") == 0) -+ return GNUTLS_DIG_SHA512; -+ if (strcmp (str, "ripemd160") == 0) -+ return GNUTLS_DIG_RMD160; -+ nbdkit_error ("unknown hash algorithm: %s " -+ "(expecting \"md5\", \"sha1\", \"sha224\", etc)", str); -+ return -1; -+} -+ -+static const char * -+hash_to_string (gnutls_digest_algorithm_t v) -+{ -+ switch (v) { -+ case GNUTLS_DIG_UNKNOWN: return "unknown"; -+ case GNUTLS_DIG_MD5: return "md5"; -+ case GNUTLS_DIG_SHA1: return "sha1"; -+ case GNUTLS_DIG_SHA224: return "sha224"; -+ case GNUTLS_DIG_SHA256: return "sha256"; -+ case GNUTLS_DIG_SHA384: return "sha384"; -+ case GNUTLS_DIG_SHA512: return "sha512"; -+ case GNUTLS_DIG_RMD160: return "ripemd160"; -+ default: abort (); -+ } -+} -+ -+#if 0 -+/* See qemu & dm-crypt implementations for an explanation of what's -+ * going on here. -+ */ -+enum cipher_alg -+lookup_essiv_cipher (enum cipher_alg cipher_alg, -+ gnutls_digest_algorithm_t ivgen_hash_alg) -+{ -+ int digest_bytes = gnutls_hash_get_len (ivgen_hash_alg); -+ int key_bytes = cipher_alg_key_bytes (cipher_alg); -+ -+ if (digest_bytes == key_bytes) -+ return cipher_alg; -+ -+ switch (cipher_alg) { -+ case CIPHER_ALG_AES_128: -+ case CIPHER_ALG_AES_192: -+ case CIPHER_ALG_AES_256: -+ if (digest_bytes == 16) return CIPHER_ALG_AES_128; -+ if (digest_bytes == 24) return CIPHER_ALG_AES_192; -+ if (digest_bytes == 32) return CIPHER_ALG_AES_256; -+ nbdkit_error ("no %s cipher available with key size %d", -+ "AES", digest_bytes); -+ return -1; -+ default: -+ nbdkit_error ("ESSIV does not support cipher %s", -+ cipher_alg_to_string (cipher_alg)); -+ return -1; -+ } -+} -+#endif -+ -+/* Per-connection data. */ -+struct luks_data { -+ /* LUKS header, if necessary byte-swapped into host order. */ -+ struct luks_phdr phdr; -+ -+ /* Decoded algorithm etc. */ -+ enum cipher_alg cipher_alg; -+ enum cipher_mode cipher_mode; -+ gnutls_digest_algorithm_t hash_alg; -+ enum ivgen ivgen_alg; -+ gnutls_digest_algorithm_t ivgen_hash_alg; -+ enum cipher_alg ivgen_cipher_alg; -+ -+ /* GnuTLS algorithm. */ -+ gnutls_cipher_algorithm_t gnutls_cipher; -+ -+ /* If we managed to decrypt one of the keyslots using the passphrase -+ * then this contains the master key, otherwise NULL. -+ */ -+ uint8_t *masterkey; -+}; -+ -+/* Parse the header fields containing cipher algorithm, mode, etc. */ -+static int -+parse_cipher_strings (struct luks_data *h) -+{ -+ char cipher_name[33], cipher_mode[33], hash_spec[33]; -+ char *ivgen, *ivhash; -+ -+ /* Copy the header fields locally and ensure they are \0 terminated. */ -+ memcpy (cipher_name, h->phdr.cipher_name, 32); -+ cipher_name[32] = 0; -+ memcpy (cipher_mode, h->phdr.cipher_mode, 32); -+ cipher_mode[32] = 0; -+ memcpy (hash_spec, h->phdr.hash_spec, 32); -+ hash_spec[32] = 0; -+ -+ nbdkit_debug ("LUKS v%" PRIu16 " cipher: %s mode: %s hash: %s " -+ "master key: %" PRIu32 " bits", -+ h->phdr.version, cipher_name, cipher_mode, hash_spec, -+ h->phdr.master_key_len * 8); -+ -+ /* The cipher_mode header has the form: "ciphermode-ivgen[:ivhash]" -+ * QEmu writes: "xts-plain64" -+ */ -+ ivgen = strchr (cipher_mode, '-'); -+ if (!ivgen) { -+ nbdkit_error ("incorrect cipher_mode header, " -+ "expecting mode-ivgenerator but got \"%s\"", cipher_mode); -+ return -1; -+ } -+ *ivgen = '\0'; -+ ivgen++; -+ -+ ivhash = strchr (ivgen, ':'); -+ if (!ivhash) -+ h->ivgen_hash_alg = GNUTLS_DIG_UNKNOWN; -+ else { -+ *ivhash = '\0'; -+ ivhash++; -+ -+ h->ivgen_hash_alg = lookup_hash (ivhash); -+ if (h->ivgen_hash_alg == -1) -+ return -1; -+ } -+ -+ h->cipher_mode = lookup_cipher_mode (cipher_mode); -+ if (h->cipher_mode == -1) -+ return -1; -+ -+ h->cipher_alg = lookup_cipher_alg (cipher_name, h->cipher_mode, -+ h->phdr.master_key_len); -+ if (h->cipher_alg == -1) -+ return -1; -+ -+ h->hash_alg = lookup_hash (hash_spec); -+ if (h->hash_alg == -1) -+ return -1; -+ -+ h->ivgen_alg = lookup_ivgen (ivgen); -+ if (h->ivgen_alg == -1) -+ return -1; -+ -+#if 0 -+ if (h->ivgen_alg == IVGEN_ESSIV) { -+ if (!ivhash) { -+ nbdkit_error ("incorrect IV generator hash specification"); -+ return -1; -+ } -+ h->ivgen_cipher_alg = lookup_essiv_cipher (h->cipher_alg, -+ h->ivgen_hash_alg); -+ if (h->ivgen_cipher_alg == -1) -+ return -1; -+ } -+ else -+#endif -+ h->ivgen_cipher_alg = h->cipher_alg; -+ -+ nbdkit_debug ("LUKS parsed ciphers: %s %s %s %s %s %s", -+ cipher_alg_to_string (h->cipher_alg), -+ cipher_mode_to_string (h->cipher_mode), -+ hash_to_string (h->hash_alg), -+ ivgen_to_string (h->ivgen_alg), -+ hash_to_string (h->ivgen_hash_alg), -+ cipher_alg_to_string (h->ivgen_cipher_alg)); -+ -+ /* GnuTLS combines cipher and block mode into a single value. Not -+ * all possible combinations are available in GnuTLS. See: -+ * https://www.gnutls.org/manual/html_node/Supported-ciphersuites.html -+ */ -+ h->gnutls_cipher = GNUTLS_CIPHER_NULL; -+ switch (h->cipher_mode) { -+ case CIPHER_MODE_XTS: -+ switch (h->cipher_alg) { -+ case CIPHER_ALG_AES_128: -+ h->gnutls_cipher = GNUTLS_CIPHER_AES_128_XTS; -+ break; -+ case CIPHER_ALG_AES_256: -+ h->gnutls_cipher = GNUTLS_CIPHER_AES_256_XTS; -+ break; -+ default: break; -+ } -+ break; -+ case CIPHER_MODE_CBC: -+ switch (h->cipher_alg) { -+ case CIPHER_ALG_AES_128: -+ h->gnutls_cipher = GNUTLS_CIPHER_AES_128_CBC; -+ break; -+ case CIPHER_ALG_AES_192: -+ h->gnutls_cipher = GNUTLS_CIPHER_AES_192_CBC; -+ break; -+ case CIPHER_ALG_AES_256: -+ h->gnutls_cipher = GNUTLS_CIPHER_AES_256_CBC; -+ break; -+ default: break; -+ } -+ default: break; -+ } -+ if (h->gnutls_cipher == GNUTLS_CIPHER_NULL) { -+ nbdkit_error ("cipher algorithm %s in mode %s is not supported by GnuTLS", -+ cipher_alg_to_string (h->cipher_alg), -+ cipher_mode_to_string (h->cipher_mode)); -+ return -1; -+ } -+ -+ return 0; -+} -+ -+/* Anti-Forensic merge operation. */ -+static void -+xor (const uint8_t *in1, const uint8_t *in2, uint8_t *out, size_t len) -+{ -+ size_t i; -+ -+ for (i = 0; i < len; ++i) -+ out[i] = in1[i] ^ in2[i]; -+} -+ -+static int -+af_hash (gnutls_digest_algorithm_t hash_alg, uint8_t *block, size_t len) -+{ -+ size_t digest_bytes = gnutls_hash_get_len (hash_alg); -+ size_t nr_blocks, last_block_len; -+ size_t i; -+ CLEANUP_FREE uint8_t *temp = malloc (digest_bytes); -+ int r; -+ gnutls_hash_hd_t hash; -+ -+ nr_blocks = len / digest_bytes; -+ last_block_len = len % digest_bytes; -+ if (last_block_len != 0) -+ nr_blocks++; -+ else -+ last_block_len = digest_bytes; -+ -+ for (i = 0; i < nr_blocks; ++i) { -+ const uint32_t iv = htobe32 (i); -+ const size_t blen = i < nr_blocks - 1 ? digest_bytes : last_block_len; -+ -+ /* Hash iv + i'th block into temp. */ -+ r = gnutls_hash_init (&hash, hash_alg); -+ if (r != 0) { -+ nbdkit_error ("gnutls_hash_init: %s", gnutls_strerror (r)); -+ return -1; -+ } -+ gnutls_hash (hash, &iv, sizeof iv); -+ gnutls_hash (hash, &block[i*digest_bytes], blen); -+ gnutls_hash_deinit (hash, temp); -+ -+ memcpy (&block[i*digest_bytes], temp, blen); -+ } -+ -+ return 0; -+} -+ -+static int -+afmerge (gnutls_digest_algorithm_t hash_alg, uint32_t stripes, -+ const uint8_t *in, uint8_t *out, size_t outlen) -+{ -+ CLEANUP_FREE uint8_t *block = calloc (1, outlen); -+ size_t i; -+ -+ /* NB: input size is stripes * master_key_len where -+ * master_key_len == outlen -+ */ -+ for (i = 0; i < stripes-1; ++i) { -+ xor (&in[i*outlen], block, block, outlen); -+ if (af_hash (hash_alg, block, outlen) == -1) -+ return -1; -+ } -+ xor (&in[i*outlen], block, out, outlen); -+ return 0; -+} -+ -+/* Length of key material in key slot i (sectors). -+ * -+ * This is basically copied from qemu because the spec description is -+ * unintelligible and apparently doesn't match reality. -+ */ -+static uint64_t -+key_material_length_in_sectors (struct luks_data *h, size_t i) -+{ -+ uint64_t len, r; -+ -+ len = h->phdr.master_key_len * h->phdr.keyslot[i].stripes; -+ r = DIV_ROUND_UP (len, LUKS_SECTOR_SIZE); -+ r = ROUND_UP (r, LUKS_ALIGN_KEYSLOTS / LUKS_SECTOR_SIZE); -+ return r; -+} -+ -+/* Try the passphrase in key slot i. If this returns true then the -+ * passphrase was able to decrypt the master key, and the master key -+ * has been stored in h->masterkey. -+ */ -+static int -+try_passphrase_in_keyslot (nbdkit_next *next, struct luks_data *h, -+ size_t i, const char *passphrase) -+{ -+ /* I believe this is supposed to be safe, looking at the GnuTLS -+ * header file. -+ */ -+ const gnutls_mac_algorithm_t mac = (gnutls_mac_algorithm_t) h->hash_alg; -+ struct luks_keyslot *ks = &h->phdr.keyslot[i]; -+ size_t split_key_len; -+ CLEANUP_FREE uint8_t *split_key = NULL; -+ CLEANUP_FREE uint8_t *masterkey = NULL; -+ const gnutls_datum_t key = -+ { (unsigned char *) passphrase, strlen (passphrase) }; -+ const gnutls_datum_t salt = -+ { (unsigned char *) ks->password_salt, LUKS_SALTSIZE }; -+ const gnutls_datum_t msalt = -+ { (unsigned char *) h->phdr.master_key_salt, LUKS_SALTSIZE }; -+ gnutls_datum_t mkey; -+ gnutls_cipher_hd_t cipher; -+ int r, err = 0; -+ uint64_t start; -+ uint8_t key_digest[LUKS_DIGESTSIZE]; -+ -+ if (ks->active != LUKS_KEY_ENABLED) -+ return 0; -+ -+ split_key_len = h->phdr.master_key_len * ks->stripes; -+ split_key = malloc (split_key_len); -+ if (split_key == NULL) { -+ nbdkit_error ("malloc: %m"); -+ return -1; -+ } -+ masterkey = malloc (h->phdr.master_key_len); -+ if (masterkey == NULL) { -+ nbdkit_error ("malloc: %m"); -+ return -1; -+ } -+ -+ /* Hash the passphrase to make a possible masterkey. */ -+ r = gnutls_pbkdf2 (mac, &key, &salt, ks->password_iterations, -+ masterkey, h->phdr.master_key_len); -+ if (r != 0) { -+ nbdkit_error ("gnutls_pbkdf2: %s", gnutls_strerror (r)); -+ return -1; -+ } -+ -+ /* Read master key material from plugin. */ -+ start = ks->key_material_offset * LUKS_SECTOR_SIZE; -+ if (next->pread (next, split_key, split_key_len, start, 0, &err) == -1) { -+ errno = err; -+ return -1; -+ } -+ -+ /* Decrypt the (still AFsplit) master key material. */ -+ mkey.data = (unsigned char *) masterkey; -+ mkey.size = h->phdr.master_key_len; -+ r = gnutls_cipher_init (&cipher, h->gnutls_cipher, &mkey, NULL); -+ if (r != 0) { -+ nbdkit_error ("gnutls_cipher_init: %s", gnutls_strerror (r)); -+ return -1; -+ } -+ -+ r = do_decrypt (h, cipher, 0, split_key, split_key_len / LUKS_SECTOR_SIZE); -+ gnutls_cipher_deinit (cipher); -+ if (r == -1) -+ return -1; -+ -+ /* Decode AFsplit key to a possible masterkey. */ -+ if (afmerge (h->hash_alg, ks->stripes, split_key, -+ masterkey, h->phdr.master_key_len) == -1) -+ return -1; -+ -+ /* Check if the masterkey is correct by comparing hash of the -+ * masterkey with LUKS header. -+ */ -+ r = gnutls_pbkdf2 (mac, &mkey, &msalt, -+ h->phdr.master_key_digest_iterations, -+ key_digest, LUKS_DIGESTSIZE); -+ if (r != 0) { -+ nbdkit_error ("gnutls_pbkdf2: %s", gnutls_strerror (r)); -+ return -1; -+ } -+ -+ if (memcmp (key_digest, h->phdr.master_key_digest, LUKS_DIGESTSIZE) == 0) { -+ /* The passphrase is correct so save the master key in the handle. */ -+ h->masterkey = malloc (h->phdr.master_key_len); -+ if (h->masterkey == NULL) { -+ nbdkit_error ("malloc: %m"); -+ return -1; -+ } -+ memcpy (h->masterkey, masterkey, h->phdr.master_key_len); -+ return 1; -+ } -+ -+ return 0; -+} -+ -+struct luks_data * -+load_header (nbdkit_next *next, const char *passphrase) -+{ -+ static const char expected_magic[] = LUKS_MAGIC; -+ struct luks_data *h; -+ int64_t size; -+ int err = 0, r; -+ size_t i; -+ struct luks_keyslot *ks; -+ char uuid[41]; -+ -+ h = calloc (1, sizeof *h); -+ if (h == NULL) { -+ nbdkit_error ("calloc: %m"); -+ return NULL; -+ } -+ -+ /* Check the struct size matches the documentation. */ -+ assert (sizeof (struct luks_phdr) == 592); -+ -+ /* Check this is a LUKSv1 disk. */ -+ size = next->get_size (next); -+ if (size == -1) { -+ free (h); -+ return NULL; -+ } -+ if (size < 16384) { -+ nbdkit_error ("disk is too small to be LUKS-encrypted"); -+ free (h); -+ return NULL; -+ } -+ -+ /* Read the phdr. */ -+ if (next->pread (next, &h->phdr, sizeof h->phdr, 0, 0, &err) == -1) { -+ free (h); -+ errno = err; -+ return NULL; -+ } -+ -+ if (memcmp (h->phdr.magic, expected_magic, LUKS_MAGIC_LEN) != 0) { -+ nbdkit_error ("this disk does not contain a LUKS header"); -+ return NULL; -+ } -+ h->phdr.version = be16toh (h->phdr.version); -+ if (h->phdr.version != 1) { -+ nbdkit_error ("this disk contains a LUKS version %" PRIu16 " header, " -+ "but this filter only supports LUKSv1", -+ h->phdr.version); -+ free (h); -+ return NULL; -+ } -+ -+ /* Byte-swap the rest of the header. */ -+ h->phdr.payload_offset = be32toh (h->phdr.payload_offset); -+ h->phdr.master_key_len = be32toh (h->phdr.master_key_len); -+ h->phdr.master_key_digest_iterations = -+ be32toh (h->phdr.master_key_digest_iterations); -+ -+ for (i = 0; i < LUKS_NUMKEYS; ++i) { -+ ks = &h->phdr.keyslot[i]; -+ ks->active = be32toh (ks->active); -+ ks->password_iterations = be32toh (ks->password_iterations); -+ ks->key_material_offset = be32toh (ks->key_material_offset); -+ ks->stripes = be32toh (ks->stripes); -+ } -+ -+ /* Sanity check some fields. */ -+ if (h->phdr.payload_offset >= size / LUKS_SECTOR_SIZE) { -+ nbdkit_error ("bad LUKSv1 header: payload offset points beyond " -+ "the end of the disk"); -+ free (h); -+ return NULL; -+ } -+ -+ /* We derive several allocations from master_key_len so make sure -+ * it's not insane. -+ */ -+ if (h->phdr.master_key_len > 1024) { -+ nbdkit_error ("bad LUKSv1 header: master key is too long"); -+ free (h); -+ return NULL; -+ } -+ -+ for (i = 0; i < LUKS_NUMKEYS; ++i) { -+ uint64_t start, len; -+ -+ ks = &h->phdr.keyslot[i]; -+ switch (ks->active) { -+ case LUKS_KEY_ENABLED: -+ if (!ks->stripes) { -+ nbdkit_error ("bad LUKSv1 header: key slot %zu is corrupted", i); -+ free (h); -+ return NULL; -+ } -+ if (ks->stripes >= 10000) { -+ nbdkit_error ("bad LUKSv1 header: key slot %zu stripes too large", i); -+ return NULL; -+ } -+ start = ks->key_material_offset; -+ len = key_material_length_in_sectors (h, i); -+ if (len > 4096) /* bound it at something reasonable */ { -+ nbdkit_error ("bad LUKSv1 header: key slot %zu key material length " -+ "is too large", i); -+ free (h); -+ return NULL; -+ } -+ if (start * LUKS_SECTOR_SIZE >= size || -+ (start + len) * LUKS_SECTOR_SIZE >= size) { -+ nbdkit_error ("bad LUKSv1 header: key slot %zu key material offset " -+ "points beyond the end of the disk", i); -+ free (h); -+ return NULL; -+ } -+ /*FALLTHROUGH*/ -+ case LUKS_KEY_DISABLED: -+ break; -+ -+ default: -+ nbdkit_error ("bad LUKSv1 header: key slot %zu has " -+ "an invalid active flag", i); -+ return NULL; -+ } -+ } -+ -+ /* Decode the ciphers. */ -+ if (parse_cipher_strings (h) == -1) { -+ free (h); -+ return NULL; -+ } -+ -+ /* Dump some information about the header. */ -+ memcpy (uuid, h->phdr.uuid, 40); -+ uuid[40] = 0; -+ nbdkit_debug ("LUKS UUID: %s", uuid); -+ -+ for (i = 0; i < LUKS_NUMKEYS; ++i) { -+ uint64_t start, len; -+ -+ ks = &h->phdr.keyslot[i]; -+ if (ks->active == LUKS_KEY_ENABLED) { -+ start = ks->key_material_offset; -+ len = key_material_length_in_sectors (h, i); -+ nbdkit_debug ("LUKS key slot %zu: key material in sectors %" PRIu64 -+ "..%" PRIu64, -+ i, start, start+len-1); -+ } -+ } -+ -+ /* Now try to unlock the master key. */ -+ for (i = 0; i < LUKS_NUMKEYS; ++i) { -+ r = try_passphrase_in_keyslot (next, h, i, passphrase); -+ if (r == -1) { -+ free (h); -+ return NULL; -+ } -+ if (r > 0) -+ goto unlocked; -+ } -+ nbdkit_error ("LUKS passphrase is not correct, " -+ "no key slot could be unlocked"); -+ free (h); -+ return NULL; -+ -+ unlocked: -+ assert (h->masterkey != NULL); -+ nbdkit_debug ("LUKS unlocked block device with passphrase"); -+ -+ return h; -+} -+ -+/* Free fields in the handle that might have been set by load_header. */ -+void -+free_luks_data (struct luks_data *h) -+{ -+ if (h->masterkey) { -+ memset (h->masterkey, 0, h->phdr.master_key_len); -+ free (h->masterkey); -+ } -+ free (h); -+} -+ -+uint64_t -+get_payload_offset (struct luks_data *h) -+{ -+ return h->phdr.payload_offset; -+} -+ -+gnutls_cipher_hd_t -+create_cipher (struct luks_data *h) -+{ -+ gnutls_datum_t mkey; -+ gnutls_cipher_hd_t cipher; -+ int r; -+ -+ assert (h->masterkey != NULL); -+ -+ mkey.data = (unsigned char *) h->masterkey; -+ mkey.size = h->phdr.master_key_len; -+ r = gnutls_cipher_init (&cipher, h->gnutls_cipher, &mkey, NULL); -+ if (r != 0) { -+ nbdkit_error ("gnutls_cipher_init: %s", gnutls_strerror (r)); -+ return NULL; -+ } -+ return cipher; -+} -+ -+/* Perform decryption of a block of data in memory. */ -+int -+do_decrypt (struct luks_data *h, gnutls_cipher_hd_t cipher, -+ uint64_t sector, uint8_t *buf, size_t nr_sectors) -+{ -+ const size_t ivlen = cipher_alg_iv_len (h->cipher_alg, h->cipher_mode); -+ CLEANUP_FREE uint8_t *iv = malloc (ivlen); -+ int r; -+ -+ while (nr_sectors) { -+ calculate_iv (h->ivgen_alg, iv, ivlen, sector); -+ gnutls_cipher_set_iv (cipher, iv, ivlen); -+ r = gnutls_cipher_decrypt2 (cipher, -+ buf, LUKS_SECTOR_SIZE, /* ciphertext */ -+ buf, LUKS_SECTOR_SIZE /* plaintext */); -+ if (r != 0) { -+ nbdkit_error ("gnutls_cipher_decrypt2: %s", gnutls_strerror (r)); -+ return -1; -+ } -+ -+ buf += LUKS_SECTOR_SIZE; -+ nr_sectors--; -+ sector++; -+ } -+ -+ return 0; -+} -+ -+/* Perform encryption of a block of data in memory. */ -+int -+do_encrypt (struct luks_data *h, gnutls_cipher_hd_t cipher, -+ uint64_t sector, uint8_t *buf, size_t nr_sectors) -+{ -+ const size_t ivlen = cipher_alg_iv_len (h->cipher_alg, h->cipher_mode); -+ CLEANUP_FREE uint8_t *iv = malloc (ivlen); -+ int r; -+ -+ while (nr_sectors) { -+ calculate_iv (h->ivgen_alg, iv, ivlen, sector); -+ gnutls_cipher_set_iv (cipher, iv, ivlen); -+ r = gnutls_cipher_encrypt2 (cipher, -+ buf, LUKS_SECTOR_SIZE, /* plaintext */ -+ buf, LUKS_SECTOR_SIZE /* ciphertext */); -+ if (r != 0) { -+ nbdkit_error ("gnutls_cipher_decrypt2: %s", gnutls_strerror (r)); -+ return -1; -+ } -+ -+ buf += LUKS_SECTOR_SIZE; -+ nr_sectors--; -+ sector++; -+ } -+ -+ return 0; -+} -diff --git a/filters/luks/luks-encryption.h b/filters/luks/luks-encryption.h -new file mode 100644 -index 00000000..3f8b9c9e ---- /dev/null -+++ b/filters/luks/luks-encryption.h -@@ -0,0 +1,78 @@ -+/* nbdkit -+ * Copyright (C) 2013-2022 Red Hat Inc. -+ * -+ * Redistribution and use in source and binary forms, with or without -+ * modification, are permitted provided that the following conditions are -+ * met: -+ * -+ * * Redistributions of source code must retain the above copyright -+ * notice, this list of conditions and the following disclaimer. -+ * -+ * * Redistributions in binary form must reproduce the above copyright -+ * notice, this list of conditions and the following disclaimer in the -+ * documentation and/or other materials provided with the distribution. -+ * -+ * * Neither the name of Red Hat nor the names of its contributors may be -+ * used to endorse or promote products derived from this software without -+ * specific prior written permission. -+ * -+ * THIS SOFTWARE IS PROVIDED BY RED HAT AND CONTRIBUTORS ''AS IS'' AND -+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, -+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A -+ * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL RED HAT OR -+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF -+ * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND -+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT -+ * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF -+ * SUCH DAMAGE. -+ */ -+ -+/* This header file defines the file format used by LUKSv1. See also: -+ * https://gitlab.com/cryptsetup/cryptsetup/-/wikis/LUKS-standard/on-disk-format.pdf -+ * Note we do not yet support LUKSv2. -+ */ -+ -+#ifndef NBDKIT_LUKS_ENCRYPTION_H -+#define NBDKIT_LUKS_ENCRYPTION_H -+ -+#include -+#include -+ -+#define LUKS_SECTOR_SIZE 512 -+ -+/* Per-connection data. */ -+struct luks_data; -+ -+/* Load the LUKS header, parse the algorithms, unlock the masterkey -+ * using the passphrase, initialize all the fields in the handle. -+ * -+ * This function may call next->pread (many times). -+ */ -+extern struct luks_data *load_header (nbdkit_next *next, -+ const char *passphrase); -+ -+/* Free the handle and all fields inside it. */ -+extern void free_luks_data (struct luks_data *h); -+ -+/* Get the offset where the encrypted data starts (in sectors). */ -+extern uint64_t get_payload_offset (struct luks_data *h); -+ -+/* Create an GnuTLS cipher, initialized with the master key. Must be -+ * freed by the caller using gnutls_cipher_deinit. -+ */ -+extern gnutls_cipher_hd_t create_cipher (struct luks_data *h); -+ -+/* Perform decryption/encryption of a block of memory in-place. -+ * -+ * 'sector' is the sector number on disk, used to calculate IVs. (The -+ * keyslots also use these functions, but sector must be 0). -+ */ -+extern int do_decrypt (struct luks_data *h, gnutls_cipher_hd_t cipher, -+ uint64_t sector, uint8_t *buf, size_t nr_sectors); -+extern int do_encrypt (struct luks_data *h, gnutls_cipher_hd_t cipher, -+ uint64_t sector, uint8_t *buf, size_t nr_sectors); -+ -+#endif /* NBDKIT_LUKS_ENCRYPTION_H */ -diff --git a/filters/luks/luks.c b/filters/luks/luks.c -index cc619698..8ad3f4ec 100644 ---- a/filters/luks/luks.c -+++ b/filters/luks/luks.c -@@ -45,49 +45,11 @@ - - #include - --#include "byte-swapping.h" -+#include "luks-encryption.h" -+ - #include "cleanup.h" - #include "isaligned.h" - #include "minmax.h" --#include "rounding.h" -- --/* LUKSv1 constants. */ --#define LUKS_MAGIC { 'L', 'U', 'K', 'S', 0xBA, 0xBE } --#define LUKS_MAGIC_LEN 6 --#define LUKS_DIGESTSIZE 20 --#define LUKS_SALTSIZE 32 --#define LUKS_NUMKEYS 8 --#define LUKS_KEY_DISABLED 0x0000DEAD --#define LUKS_KEY_ENABLED 0x00AC71F3 --#define LUKS_STRIPES 4000 --#define LUKS_ALIGN_KEYSLOTS 4096 --#define LUKS_SECTOR_SIZE 512 -- --/* Key slot. */ --struct luks_keyslot { -- uint32_t active; /* LUKS_KEY_DISABLED|LUKS_KEY_ENABLED */ -- uint32_t password_iterations; -- char password_salt[LUKS_SALTSIZE]; -- uint32_t key_material_offset; -- uint32_t stripes; --} __attribute__((__packed__)); -- --/* LUKS superblock. */ --struct luks_phdr { -- char magic[LUKS_MAGIC_LEN]; /* LUKS_MAGIC */ -- uint16_t version; /* Only 1 is supported. */ -- char cipher_name[32]; -- char cipher_mode[32]; -- char hash_spec[32]; -- uint32_t payload_offset; -- uint32_t master_key_len; -- uint8_t master_key_digest[LUKS_DIGESTSIZE]; -- uint8_t master_key_salt[LUKS_SALTSIZE]; -- uint32_t master_key_digest_iterations; -- uint8_t uuid[40]; -- -- struct luks_keyslot keyslot[LUKS_NUMKEYS]; /* Key slots. */ --} __attribute__((__packed__)); - - static char *passphrase = NULL; - -@@ -135,251 +97,9 @@ luks_config_complete (nbdkit_next_config_complete *next, nbdkit_backend *nxdata) - #define luks_config_help \ - "passphrase= Secret passphrase." - --enum cipher_mode { -- CIPHER_MODE_ECB, CIPHER_MODE_CBC, CIPHER_MODE_XTS, CIPHER_MODE_CTR, --}; -- --static enum cipher_mode --lookup_cipher_mode (const char *str) --{ -- if (strcmp (str, "ecb") == 0) -- return CIPHER_MODE_ECB; -- if (strcmp (str, "cbc") == 0) -- return CIPHER_MODE_CBC; -- if (strcmp (str, "xts") == 0) -- return CIPHER_MODE_XTS; -- if (strcmp (str, "ctr") == 0) -- return CIPHER_MODE_CTR; -- nbdkit_error ("unknown cipher mode: %s " -- "(expecting \"ecb\", \"cbc\", \"xts\" or \"ctr\")", str); -- return -1; --} -- --static const char * --cipher_mode_to_string (enum cipher_mode v) --{ -- switch (v) { -- case CIPHER_MODE_ECB: return "ecb"; -- case CIPHER_MODE_CBC: return "cbc"; -- case CIPHER_MODE_XTS: return "xts"; -- case CIPHER_MODE_CTR: return "ctr"; -- default: abort (); -- } --} -- --enum ivgen { -- IVGEN_PLAIN, IVGEN_PLAIN64, /* IVGEN_ESSIV, */ --}; -- --static enum ivgen --lookup_ivgen (const char *str) --{ -- if (strcmp (str, "plain") == 0) -- return IVGEN_PLAIN; -- if (strcmp (str, "plain64") == 0) -- return IVGEN_PLAIN64; --/* -- if (strcmp (str, "essiv") == 0) -- return IVGEN_ESSIV; --*/ -- nbdkit_error ("unknown IV generation algorithm: %s " -- "(expecting \"plain\", \"plain64\" etc)", str); -- return -1; --} -- --static const char * --ivgen_to_string (enum ivgen v) --{ -- switch (v) { -- case IVGEN_PLAIN: return "plain"; -- case IVGEN_PLAIN64: return "plain64"; -- /*case IVGEN_ESSIV: return "essiv";*/ -- default: abort (); -- } --} -- --static void --calculate_iv (enum ivgen v, uint8_t *iv, size_t ivlen, uint64_t sector) --{ -- size_t prefixlen; -- uint32_t sector32; -- -- switch (v) { -- case IVGEN_PLAIN: -- prefixlen = 4; /* 32 bits */ -- if (prefixlen > ivlen) -- prefixlen = ivlen; -- sector32 = (uint32_t) sector; /* truncate to only lower bits */ -- sector32 = htole32 (sector32); -- memcpy (iv, §or32, prefixlen); -- memset (iv + prefixlen, 0, ivlen - prefixlen); -- break; -- -- case IVGEN_PLAIN64: -- prefixlen = 8; /* 64 bits */ -- if (prefixlen > ivlen) -- prefixlen = ivlen; -- sector = htole64 (sector); -- memcpy (iv, §or, prefixlen); -- memset (iv + prefixlen, 0, ivlen - prefixlen); -- break; -- -- /*case IVGEN_ESSIV:*/ -- default: abort (); -- } --} -- --enum cipher_alg { -- CIPHER_ALG_AES_128, CIPHER_ALG_AES_192, CIPHER_ALG_AES_256, --}; -- --static enum cipher_alg --lookup_cipher_alg (const char *str, enum cipher_mode mode, int key_bytes) --{ -- if (mode == CIPHER_MODE_XTS) -- key_bytes /= 2; -- -- if (strcmp (str, "aes") == 0) { -- if (key_bytes == 16) -- return CIPHER_ALG_AES_128; -- if (key_bytes == 24) -- return CIPHER_ALG_AES_192; -- if (key_bytes == 32) -- return CIPHER_ALG_AES_256; -- } -- nbdkit_error ("unknown cipher algorithm: %s (expecting \"aes\", etc)", str); -- return -1; --} -- --static const char * --cipher_alg_to_string (enum cipher_alg v) --{ -- switch (v) { -- case CIPHER_ALG_AES_128: return "aes-128"; -- case CIPHER_ALG_AES_192: return "aes-192"; -- case CIPHER_ALG_AES_256: return "aes-256"; -- default: abort (); -- } --} -- --#if 0 --static int --cipher_alg_key_bytes (enum cipher_alg v) --{ -- switch (v) { -- case CIPHER_ALG_AES_128: return 16; -- case CIPHER_ALG_AES_192: return 24; -- case CIPHER_ALG_AES_256: return 32; -- default: abort (); -- } --} --#endif -- --static int --cipher_alg_iv_len (enum cipher_alg v, enum cipher_mode mode) --{ -- if (CIPHER_MODE_ECB) -- return 0; /* Don't need an IV in this mode. */ -- -- switch (v) { -- case CIPHER_ALG_AES_128: -- case CIPHER_ALG_AES_192: -- case CIPHER_ALG_AES_256: -- return 16; -- default: abort (); -- } --} -- --static gnutls_digest_algorithm_t --lookup_hash (const char *str) --{ -- if (strcmp (str, "md5") == 0) -- return GNUTLS_DIG_MD5; -- if (strcmp (str, "sha1") == 0) -- return GNUTLS_DIG_SHA1; -- if (strcmp (str, "sha224") == 0) -- return GNUTLS_DIG_SHA224; -- if (strcmp (str, "sha256") == 0) -- return GNUTLS_DIG_SHA256; -- if (strcmp (str, "sha384") == 0) -- return GNUTLS_DIG_SHA384; -- if (strcmp (str, "sha512") == 0) -- return GNUTLS_DIG_SHA512; -- if (strcmp (str, "ripemd160") == 0) -- return GNUTLS_DIG_RMD160; -- nbdkit_error ("unknown hash algorithm: %s " -- "(expecting \"md5\", \"sha1\", \"sha224\", etc)", str); -- return -1; --} -- --static const char * --hash_to_string (gnutls_digest_algorithm_t v) --{ -- switch (v) { -- case GNUTLS_DIG_UNKNOWN: return "unknown"; -- case GNUTLS_DIG_MD5: return "md5"; -- case GNUTLS_DIG_SHA1: return "sha1"; -- case GNUTLS_DIG_SHA224: return "sha224"; -- case GNUTLS_DIG_SHA256: return "sha256"; -- case GNUTLS_DIG_SHA384: return "sha384"; -- case GNUTLS_DIG_SHA512: return "sha512"; -- case GNUTLS_DIG_RMD160: return "ripemd160"; -- default: abort (); -- } --} -- --#if 0 --/* See qemu & dm-crypt implementations for an explanation of what's -- * going on here. -- */ --static enum cipher_alg --lookup_essiv_cipher (enum cipher_alg cipher_alg, -- gnutls_digest_algorithm_t ivgen_hash_alg) --{ -- int digest_bytes = gnutls_hash_get_len (ivgen_hash_alg); -- int key_bytes = cipher_alg_key_bytes (cipher_alg); -- -- if (digest_bytes == key_bytes) -- return cipher_alg; -- -- switch (cipher_alg) { -- case CIPHER_ALG_AES_128: -- case CIPHER_ALG_AES_192: -- case CIPHER_ALG_AES_256: -- if (digest_bytes == 16) return CIPHER_ALG_AES_128; -- if (digest_bytes == 24) return CIPHER_ALG_AES_192; -- if (digest_bytes == 32) return CIPHER_ALG_AES_256; -- nbdkit_error ("no %s cipher available with key size %d", -- "AES", digest_bytes); -- return -1; -- default: -- nbdkit_error ("ESSIV does not support cipher %s", -- cipher_alg_to_string (cipher_alg)); -- return -1; -- } --} --#endif -- - /* Per-connection handle. */ - struct handle { -- /* LUKS header, if necessary byte-swapped into host order. */ -- struct luks_phdr phdr; -- -- /* Decoded algorithm etc. */ -- enum cipher_alg cipher_alg; -- enum cipher_mode cipher_mode; -- gnutls_digest_algorithm_t hash_alg; -- enum ivgen ivgen_alg; -- gnutls_digest_algorithm_t ivgen_hash_alg; -- enum cipher_alg ivgen_cipher_alg; -- -- /* GnuTLS algorithm. */ -- gnutls_cipher_algorithm_t gnutls_cipher; -- -- /* If we managed to decrypt one of the keyslots using the passphrase -- * then this contains the master key, otherwise NULL. -- */ -- uint8_t *masterkey; -+ struct luks_data *h; - }; - - static void * -@@ -405,536 +125,21 @@ luks_close (void *handle) - { - struct handle *h = handle; - -- if (h->masterkey) { -- memset (h->masterkey, 0, h->phdr.master_key_len); -- free (h->masterkey); -- } -+ free_luks_data (h->h); - free (h); - } - --/* Perform decryption of a block of data in memory. */ --static int --do_decrypt (struct handle *h, gnutls_cipher_hd_t cipher, -- uint64_t offset, uint8_t *buf, size_t len) --{ -- const size_t ivlen = cipher_alg_iv_len (h->cipher_alg, h->cipher_mode); -- uint64_t sector = offset / LUKS_SECTOR_SIZE; -- CLEANUP_FREE uint8_t *iv = malloc (ivlen); -- int r; -- -- assert (IS_ALIGNED (offset, LUKS_SECTOR_SIZE)); -- assert (IS_ALIGNED (len, LUKS_SECTOR_SIZE)); -- -- while (len) { -- calculate_iv (h->ivgen_alg, iv, ivlen, sector); -- gnutls_cipher_set_iv (cipher, iv, ivlen); -- r = gnutls_cipher_decrypt2 (cipher, -- buf, LUKS_SECTOR_SIZE, /* ciphertext */ -- buf, LUKS_SECTOR_SIZE /* plaintext */); -- if (r != 0) { -- nbdkit_error ("gnutls_cipher_decrypt2: %s", gnutls_strerror (r)); -- return -1; -- } -- -- buf += LUKS_SECTOR_SIZE; -- offset += LUKS_SECTOR_SIZE; -- len -= LUKS_SECTOR_SIZE; -- sector++; -- } -- -- return 0; --} -- --/* Perform encryption of a block of data in memory. */ --static int --do_encrypt (struct handle *h, gnutls_cipher_hd_t cipher, -- uint64_t offset, uint8_t *buf, size_t len) --{ -- const size_t ivlen = cipher_alg_iv_len (h->cipher_alg, h->cipher_mode); -- uint64_t sector = offset / LUKS_SECTOR_SIZE; -- CLEANUP_FREE uint8_t *iv = malloc (ivlen); -- int r; -- -- assert (IS_ALIGNED (offset, LUKS_SECTOR_SIZE)); -- assert (IS_ALIGNED (len, LUKS_SECTOR_SIZE)); -- -- while (len) { -- calculate_iv (h->ivgen_alg, iv, ivlen, sector); -- gnutls_cipher_set_iv (cipher, iv, ivlen); -- r = gnutls_cipher_encrypt2 (cipher, -- buf, LUKS_SECTOR_SIZE, /* plaintext */ -- buf, LUKS_SECTOR_SIZE /* ciphertext */); -- if (r != 0) { -- nbdkit_error ("gnutls_cipher_decrypt2: %s", gnutls_strerror (r)); -- return -1; -- } -- -- buf += LUKS_SECTOR_SIZE; -- offset += LUKS_SECTOR_SIZE; -- len -= LUKS_SECTOR_SIZE; -- sector++; -- } -- -- return 0; --} -- --/* Parse the header fields containing cipher algorithm, mode, etc. */ --static int --parse_cipher_strings (struct handle *h) --{ -- char cipher_name[33], cipher_mode[33], hash_spec[33]; -- char *ivgen, *ivhash; -- -- /* Copy the header fields locally and ensure they are \0 terminated. */ -- memcpy (cipher_name, h->phdr.cipher_name, 32); -- cipher_name[32] = 0; -- memcpy (cipher_mode, h->phdr.cipher_mode, 32); -- cipher_mode[32] = 0; -- memcpy (hash_spec, h->phdr.hash_spec, 32); -- hash_spec[32] = 0; -- -- nbdkit_debug ("LUKS v%" PRIu16 " cipher: %s mode: %s hash: %s " -- "master key: %" PRIu32 " bits", -- h->phdr.version, cipher_name, cipher_mode, hash_spec, -- h->phdr.master_key_len * 8); -- -- /* The cipher_mode header has the form: "ciphermode-ivgen[:ivhash]" -- * QEmu writes: "xts-plain64" -- */ -- ivgen = strchr (cipher_mode, '-'); -- if (!ivgen) { -- nbdkit_error ("incorrect cipher_mode header, " -- "expecting mode-ivgenerator but got \"%s\"", cipher_mode); -- return -1; -- } -- *ivgen = '\0'; -- ivgen++; -- -- ivhash = strchr (ivgen, ':'); -- if (!ivhash) -- h->ivgen_hash_alg = GNUTLS_DIG_UNKNOWN; -- else { -- *ivhash = '\0'; -- ivhash++; -- -- h->ivgen_hash_alg = lookup_hash (ivhash); -- if (h->ivgen_hash_alg == -1) -- return -1; -- } -- -- h->cipher_mode = lookup_cipher_mode (cipher_mode); -- if (h->cipher_mode == -1) -- return -1; -- -- h->cipher_alg = lookup_cipher_alg (cipher_name, h->cipher_mode, -- h->phdr.master_key_len); -- if (h->cipher_alg == -1) -- return -1; -- -- h->hash_alg = lookup_hash (hash_spec); -- if (h->hash_alg == -1) -- return -1; -- -- h->ivgen_alg = lookup_ivgen (ivgen); -- if (h->ivgen_alg == -1) -- return -1; -- --#if 0 -- if (h->ivgen_alg == IVGEN_ESSIV) { -- if (!ivhash) { -- nbdkit_error ("incorrect IV generator hash specification"); -- return -1; -- } -- h->ivgen_cipher_alg = lookup_essiv_cipher (h->cipher_alg, -- h->ivgen_hash_alg); -- if (h->ivgen_cipher_alg == -1) -- return -1; -- } -- else --#endif -- h->ivgen_cipher_alg = h->cipher_alg; -- -- nbdkit_debug ("LUKS parsed ciphers: %s %s %s %s %s %s", -- cipher_alg_to_string (h->cipher_alg), -- cipher_mode_to_string (h->cipher_mode), -- hash_to_string (h->hash_alg), -- ivgen_to_string (h->ivgen_alg), -- hash_to_string (h->ivgen_hash_alg), -- cipher_alg_to_string (h->ivgen_cipher_alg)); -- -- /* GnuTLS combines cipher and block mode into a single value. Not -- * all possible combinations are available in GnuTLS. See: -- * https://www.gnutls.org/manual/html_node/Supported-ciphersuites.html -- */ -- h->gnutls_cipher = GNUTLS_CIPHER_NULL; -- switch (h->cipher_mode) { -- case CIPHER_MODE_XTS: -- switch (h->cipher_alg) { -- case CIPHER_ALG_AES_128: -- h->gnutls_cipher = GNUTLS_CIPHER_AES_128_XTS; -- break; -- case CIPHER_ALG_AES_256: -- h->gnutls_cipher = GNUTLS_CIPHER_AES_256_XTS; -- break; -- default: break; -- } -- break; -- case CIPHER_MODE_CBC: -- switch (h->cipher_alg) { -- case CIPHER_ALG_AES_128: -- h->gnutls_cipher = GNUTLS_CIPHER_AES_128_CBC; -- break; -- case CIPHER_ALG_AES_192: -- h->gnutls_cipher = GNUTLS_CIPHER_AES_192_CBC; -- break; -- case CIPHER_ALG_AES_256: -- h->gnutls_cipher = GNUTLS_CIPHER_AES_256_CBC; -- break; -- default: break; -- } -- default: break; -- } -- if (h->gnutls_cipher == GNUTLS_CIPHER_NULL) { -- nbdkit_error ("cipher algorithm %s in mode %s is not supported by GnuTLS", -- cipher_alg_to_string (h->cipher_alg), -- cipher_mode_to_string (h->cipher_mode)); -- return -1; -- } -- -- return 0; --} -- --/* Anti-Forensic merge operation. */ --static void --xor (const uint8_t *in1, const uint8_t *in2, uint8_t *out, size_t len) --{ -- size_t i; -- -- for (i = 0; i < len; ++i) -- out[i] = in1[i] ^ in2[i]; --} -- --static int --af_hash (gnutls_digest_algorithm_t hash_alg, uint8_t *block, size_t len) --{ -- size_t digest_bytes = gnutls_hash_get_len (hash_alg); -- size_t nr_blocks, last_block_len; -- size_t i; -- CLEANUP_FREE uint8_t *temp = malloc (digest_bytes); -- int r; -- gnutls_hash_hd_t hash; -- -- nr_blocks = len / digest_bytes; -- last_block_len = len % digest_bytes; -- if (last_block_len != 0) -- nr_blocks++; -- else -- last_block_len = digest_bytes; -- -- for (i = 0; i < nr_blocks; ++i) { -- const uint32_t iv = htobe32 (i); -- const size_t blen = i < nr_blocks - 1 ? digest_bytes : last_block_len; -- -- /* Hash iv + i'th block into temp. */ -- r = gnutls_hash_init (&hash, hash_alg); -- if (r != 0) { -- nbdkit_error ("gnutls_hash_init: %s", gnutls_strerror (r)); -- return -1; -- } -- gnutls_hash (hash, &iv, sizeof iv); -- gnutls_hash (hash, &block[i*digest_bytes], blen); -- gnutls_hash_deinit (hash, temp); -- -- memcpy (&block[i*digest_bytes], temp, blen); -- } -- -- return 0; --} -- --static int --afmerge (gnutls_digest_algorithm_t hash_alg, uint32_t stripes, -- const uint8_t *in, uint8_t *out, size_t outlen) --{ -- CLEANUP_FREE uint8_t *block = calloc (1, outlen); -- size_t i; -- -- /* NB: input size is stripes * master_key_len where -- * master_key_len == outlen -- */ -- for (i = 0; i < stripes-1; ++i) { -- xor (&in[i*outlen], block, block, outlen); -- if (af_hash (hash_alg, block, outlen) == -1) -- return -1; -- } -- xor (&in[i*outlen], block, out, outlen); -- return 0; --} -- --/* Length of key material in key slot i (sectors). -- * -- * This is basically copied from qemu because the spec description is -- * unintelligible and apparently doesn't match reality. -- */ --static uint64_t --key_material_length_in_sectors (struct handle *h, size_t i) --{ -- uint64_t len, r; -- -- len = h->phdr.master_key_len * h->phdr.keyslot[i].stripes; -- r = DIV_ROUND_UP (len, LUKS_SECTOR_SIZE); -- r = ROUND_UP (r, LUKS_ALIGN_KEYSLOTS / LUKS_SECTOR_SIZE); -- return r; --} -- --/* Try the passphrase in key slot i. If this returns true then the -- * passphrase was able to decrypt the master key, and the master key -- * has been stored in h->masterkey. -- */ --static int --try_passphrase_in_keyslot (nbdkit_next *next, struct handle *h, size_t i) --{ -- /* I believe this is supposed to be safe, looking at the GnuTLS -- * header file. -- */ -- const gnutls_mac_algorithm_t mac = (gnutls_mac_algorithm_t) h->hash_alg; -- struct luks_keyslot *ks = &h->phdr.keyslot[i]; -- size_t split_key_len; -- CLEANUP_FREE uint8_t *split_key = NULL; -- CLEANUP_FREE uint8_t *masterkey = NULL; -- const gnutls_datum_t key = -- { (unsigned char *) passphrase, strlen (passphrase) }; -- const gnutls_datum_t salt = -- { (unsigned char *) ks->password_salt, LUKS_SALTSIZE }; -- const gnutls_datum_t msalt = -- { (unsigned char *) h->phdr.master_key_salt, LUKS_SALTSIZE }; -- gnutls_datum_t mkey; -- gnutls_cipher_hd_t cipher; -- int r, err = 0; -- uint64_t start; -- uint8_t key_digest[LUKS_DIGESTSIZE]; -- -- if (ks->active != LUKS_KEY_ENABLED) -- return 0; -- -- split_key_len = h->phdr.master_key_len * ks->stripes; -- split_key = malloc (split_key_len); -- if (split_key == NULL) { -- nbdkit_error ("malloc: %m"); -- return -1; -- } -- masterkey = malloc (h->phdr.master_key_len); -- if (masterkey == NULL) { -- nbdkit_error ("malloc: %m"); -- return -1; -- } -- -- /* Hash the passphrase to make a possible masterkey. */ -- r = gnutls_pbkdf2 (mac, &key, &salt, ks->password_iterations, -- masterkey, h->phdr.master_key_len); -- if (r != 0) { -- nbdkit_error ("gnutls_pbkdf2: %s", gnutls_strerror (r)); -- return -1; -- } -- -- /* Read master key material from plugin. */ -- start = ks->key_material_offset * LUKS_SECTOR_SIZE; -- if (next->pread (next, split_key, split_key_len, start, 0, &err) == -1) { -- errno = err; -- return -1; -- } -- -- /* Decrypt the (still AFsplit) master key material. */ -- mkey.data = (unsigned char *) masterkey; -- mkey.size = h->phdr.master_key_len; -- r = gnutls_cipher_init (&cipher, h->gnutls_cipher, &mkey, NULL); -- if (r != 0) { -- nbdkit_error ("gnutls_cipher_init: %s", gnutls_strerror (r)); -- return -1; -- } -- -- r = do_decrypt (h, cipher, 0, split_key, split_key_len); -- gnutls_cipher_deinit (cipher); -- if (r == -1) -- return -1; -- -- /* Decode AFsplit key to a possible masterkey. */ -- if (afmerge (h->hash_alg, ks->stripes, split_key, -- masterkey, h->phdr.master_key_len) == -1) -- return -1; -- -- /* Check if the masterkey is correct by comparing hash of the -- * masterkey with LUKS header. -- */ -- r = gnutls_pbkdf2 (mac, &mkey, &msalt, -- h->phdr.master_key_digest_iterations, -- key_digest, LUKS_DIGESTSIZE); -- if (r != 0) { -- nbdkit_error ("gnutls_pbkdf2: %s", gnutls_strerror (r)); -- return -1; -- } -- -- if (memcmp (key_digest, h->phdr.master_key_digest, LUKS_DIGESTSIZE) == 0) { -- /* The passphrase is correct so save the master key in the handle. */ -- h->masterkey = malloc (h->phdr.master_key_len); -- if (h->masterkey == NULL) { -- nbdkit_error ("malloc: %m"); -- return -1; -- } -- memcpy (h->masterkey, masterkey, h->phdr.master_key_len); -- return 1; -- } -- -- return 0; --} -- - static int - luks_prepare (nbdkit_next *next, void *handle, int readonly) - { -- static const char expected_magic[] = LUKS_MAGIC; - struct handle *h = handle; -- int64_t size; -- int err = 0, r; -- size_t i; -- struct luks_keyslot *ks; -- char uuid[41]; - - /* Check we haven't been called before, this should never happen. */ -- assert (h->phdr.version == 0); -+ assert (h->h == NULL); - -- /* Check the struct size matches the documentation. */ -- assert (sizeof (struct luks_phdr) == 592); -- -- /* Check this is a LUKSv1 disk. */ -- size = next->get_size (next); -- if (size == -1) -- return -1; -- if (size < 16384) { -- nbdkit_error ("disk is too small to be LUKS-encrypted"); -- return -1; -- } -- -- /* Read the phdr. */ -- if (next->pread (next, &h->phdr, sizeof h->phdr, 0, 0, &err) == -1) { -- errno = err; -- return -1; -- } -- -- if (memcmp (h->phdr.magic, expected_magic, LUKS_MAGIC_LEN) != 0) { -- nbdkit_error ("this disk does not contain a LUKS header"); -+ h->h = load_header (next, passphrase); -+ if (h->h == NULL) - return -1; -- } -- h->phdr.version = be16toh (h->phdr.version); -- if (h->phdr.version != 1) { -- nbdkit_error ("this disk contains a LUKS version %" PRIu16 " header, " -- "but this filter only supports LUKSv1", -- h->phdr.version); -- return -1; -- } -- -- /* Byte-swap the rest of the header. */ -- h->phdr.payload_offset = be32toh (h->phdr.payload_offset); -- h->phdr.master_key_len = be32toh (h->phdr.master_key_len); -- h->phdr.master_key_digest_iterations = -- be32toh (h->phdr.master_key_digest_iterations); -- -- for (i = 0; i < LUKS_NUMKEYS; ++i) { -- ks = &h->phdr.keyslot[i]; -- ks->active = be32toh (ks->active); -- ks->password_iterations = be32toh (ks->password_iterations); -- ks->key_material_offset = be32toh (ks->key_material_offset); -- ks->stripes = be32toh (ks->stripes); -- } -- -- /* Sanity check some fields. */ -- if (h->phdr.payload_offset >= size / LUKS_SECTOR_SIZE) { -- nbdkit_error ("bad LUKSv1 header: payload offset points beyond " -- "the end of the disk"); -- return -1; -- } -- -- /* We derive several allocations from master_key_len so make sure -- * it's not insane. -- */ -- if (h->phdr.master_key_len > 1024) { -- nbdkit_error ("bad LUKSv1 header: master key is too long"); -- return -1; -- } -- -- for (i = 0; i < LUKS_NUMKEYS; ++i) { -- uint64_t start, len; -- -- ks = &h->phdr.keyslot[i]; -- switch (ks->active) { -- case LUKS_KEY_ENABLED: -- if (!ks->stripes) { -- nbdkit_error ("bad LUKSv1 header: key slot %zu is corrupted", i); -- return -1; -- } -- if (ks->stripes >= 10000) { -- nbdkit_error ("bad LUKSv1 header: key slot %zu stripes too large", i); -- return -1; -- } -- start = ks->key_material_offset; -- len = key_material_length_in_sectors (h, i); -- if (len > 4096) /* bound it at something reasonable */ { -- nbdkit_error ("bad LUKSv1 header: key slot %zu key material length " -- "is too large", i); -- return -1; -- } -- if (start * LUKS_SECTOR_SIZE >= size || -- (start + len) * LUKS_SECTOR_SIZE >= size) { -- nbdkit_error ("bad LUKSv1 header: key slot %zu key material offset " -- "points beyond the end of the disk", i); -- return -1; -- } -- /*FALLTHROUGH*/ -- case LUKS_KEY_DISABLED: -- break; -- -- default: -- nbdkit_error ("bad LUKSv1 header: key slot %zu has " -- "an invalid active flag", i); -- return -1; -- } -- } -- -- /* Decode the ciphers. */ -- if (parse_cipher_strings (h) == -1) -- return -1; -- -- /* Dump some information about the header. */ -- memcpy (uuid, h->phdr.uuid, 40); -- uuid[40] = 0; -- nbdkit_debug ("LUKS UUID: %s", uuid); -- -- for (i = 0; i < LUKS_NUMKEYS; ++i) { -- uint64_t start, len; -- -- ks = &h->phdr.keyslot[i]; -- if (ks->active == LUKS_KEY_ENABLED) { -- start = ks->key_material_offset; -- len = key_material_length_in_sectors (h, i); -- nbdkit_debug ("LUKS key slot %zu: key material in sectors %" PRIu64 -- "..%" PRIu64, -- i, start, start+len-1); -- } -- } -- -- /* Now try to unlock the master key. */ -- for (i = 0; i < LUKS_NUMKEYS; ++i) { -- r = try_passphrase_in_keyslot (next, h, i); -- if (r == -1) -- return -1; -- if (r > 0) -- goto unlocked; -- } -- nbdkit_error ("LUKS passphrase is not correct, " -- "no key slot could be unlocked"); -- return -1; -- -- unlocked: -- assert (h->masterkey != NULL); -- nbdkit_debug ("LUKS unlocked block device with passphrase"); - - return 0; - } -@@ -946,19 +151,20 @@ luks_get_size (nbdkit_next *next, void *handle) - int64_t size; - - /* Check that prepare has been called already. */ -- assert (h->phdr.version > 0); -+ assert (h->h != NULL); -+ -+ const uint64_t payload_offset = get_payload_offset (h->h) * LUKS_SECTOR_SIZE; - - size = next->get_size (next); - if (size == -1) - return -1; - -- if (size < h->phdr.payload_offset * LUKS_SECTOR_SIZE) { -+ if (size < payload_offset) { - nbdkit_error ("disk too small, or contains an incomplete LUKS partition"); - return -1; - } - -- size -= h->phdr.payload_offset * LUKS_SECTOR_SIZE; -- return size; -+ return size - payload_offset; - } - - /* Whatever the plugin says, several operations are not supported by -@@ -1031,15 +237,12 @@ luks_pread (nbdkit_next *next, void *handle, - uint32_t flags, int *err) - { - struct handle *h = handle; -- const uint64_t payload_offset = h->phdr.payload_offset * LUKS_SECTOR_SIZE; -+ const uint64_t payload_offset = get_payload_offset (h->h) * LUKS_SECTOR_SIZE; - CLEANUP_FREE uint8_t *sector = NULL; - uint64_t sectnum, sectoffs; -- const gnutls_datum_t mkey = -- { (unsigned char *) h->masterkey, h->phdr.master_key_len }; - gnutls_cipher_hd_t cipher; -- int r; - -- if (!h->masterkey) { -+ if (!h->h) { - *err = EIO; - return -1; - } -@@ -1053,16 +256,13 @@ luks_pread (nbdkit_next *next, void *handle, - } - } - -- r = gnutls_cipher_init (&cipher, h->gnutls_cipher, &mkey, NULL); -- if (r != 0) { -- nbdkit_error ("gnutls_cipher_init: %s", gnutls_strerror (r)); -- *err = EIO; -- return -1; -- } -- - sectnum = offset / LUKS_SECTOR_SIZE; /* sector number */ - sectoffs = offset % LUKS_SECTOR_SIZE; /* offset within the sector */ - -+ cipher = create_cipher (h->h); -+ if (!cipher) -+ return -1; -+ - /* Unaligned head */ - if (sectoffs) { - uint64_t n = MIN (LUKS_SECTOR_SIZE - sectoffs, count); -@@ -1073,15 +273,13 @@ luks_pread (nbdkit_next *next, void *handle, - flags, err) == -1) - goto err; - -- if (do_decrypt (h, cipher, offset & ~LUKS_SECTOR_SIZE, -- sector, LUKS_SECTOR_SIZE) == -1) -+ if (do_decrypt (h->h, cipher, sectnum, sector, 1) == -1) - goto err; - - memcpy (buf, §or[sectoffs], n); - - buf += n; - count -= n; -- offset += n; - sectnum++; - } - -@@ -1092,12 +290,11 @@ luks_pread (nbdkit_next *next, void *handle, - flags, err) == -1) - goto err; - -- if (do_decrypt (h, cipher, offset, buf, LUKS_SECTOR_SIZE) == -1) -+ if (do_decrypt (h->h, cipher, sectnum, buf, 1) == -1) - goto err; - - buf += LUKS_SECTOR_SIZE; - count -= LUKS_SECTOR_SIZE; -- offset += LUKS_SECTOR_SIZE; - sectnum++; - } - -@@ -1109,7 +306,7 @@ luks_pread (nbdkit_next *next, void *handle, - flags, err) == -1) - goto err; - -- if (do_decrypt (h, cipher, offset, sector, LUKS_SECTOR_SIZE) == -1) -+ if (do_decrypt (h->h, cipher, sectnum, sector, 1) == -1) - goto err; - - memcpy (buf, sector, count); -@@ -1120,7 +317,7 @@ luks_pread (nbdkit_next *next, void *handle, - - err: - gnutls_cipher_deinit (cipher); -- return -1; -+ goto err; - } - - /* Lock preventing read-modify-write cycles from overlapping. */ -@@ -1133,15 +330,12 @@ luks_pwrite (nbdkit_next *next, void *handle, - uint32_t flags, int *err) - { - struct handle *h = handle; -- const uint64_t payload_offset = h->phdr.payload_offset * LUKS_SECTOR_SIZE; -+ const uint64_t payload_offset = get_payload_offset (h->h) * LUKS_SECTOR_SIZE; - CLEANUP_FREE uint8_t *sector = NULL; - uint64_t sectnum, sectoffs; -- const gnutls_datum_t mkey = -- { (unsigned char *) h->masterkey, h->phdr.master_key_len }; - gnutls_cipher_hd_t cipher; -- int r; - -- if (!h->masterkey) { -+ if (!h->h) { - *err = EIO; - return -1; - } -@@ -1153,16 +347,13 @@ luks_pwrite (nbdkit_next *next, void *handle, - return -1; - } - -- r = gnutls_cipher_init (&cipher, h->gnutls_cipher, &mkey, NULL); -- if (r != 0) { -- nbdkit_error ("gnutls_cipher_init: %s", gnutls_strerror (r)); -- *err = EIO; -- return -1; -- } -- - sectnum = offset / LUKS_SECTOR_SIZE; /* sector number */ - sectoffs = offset % LUKS_SECTOR_SIZE; /* offset within the sector */ - -+ cipher = create_cipher (h->h); -+ if (!cipher) -+ return -1; -+ - /* Unaligned head */ - if (sectoffs) { - ACQUIRE_LOCK_FOR_CURRENT_SCOPE (&read_modify_write_lock); -@@ -1176,8 +367,7 @@ luks_pwrite (nbdkit_next *next, void *handle, - - memcpy (§or[sectoffs], buf, n); - -- if (do_encrypt (h, cipher, offset & ~LUKS_SECTOR_SIZE, -- sector, LUKS_SECTOR_SIZE) == -1) -+ if (do_encrypt (h->h, cipher, sectnum, sector, 1) == -1) - goto err; - - if (next->pwrite (next, sector, LUKS_SECTOR_SIZE, -@@ -1187,7 +377,6 @@ luks_pwrite (nbdkit_next *next, void *handle, - - buf += n; - count -= n; -- offset += n; - sectnum++; - } - -@@ -1195,7 +384,7 @@ luks_pwrite (nbdkit_next *next, void *handle, - while (count >= LUKS_SECTOR_SIZE) { - memcpy (sector, buf, LUKS_SECTOR_SIZE); - -- if (do_encrypt (h, cipher, offset, sector, LUKS_SECTOR_SIZE) == -1) -+ if (do_encrypt (h->h, cipher, sectnum, sector, 1) == -1) - goto err; - - if (next->pwrite (next, sector, LUKS_SECTOR_SIZE, -@@ -1205,7 +394,6 @@ luks_pwrite (nbdkit_next *next, void *handle, - - buf += LUKS_SECTOR_SIZE; - count -= LUKS_SECTOR_SIZE; -- offset += LUKS_SECTOR_SIZE; - sectnum++; - } - -@@ -1220,7 +408,7 @@ luks_pwrite (nbdkit_next *next, void *handle, - - memcpy (sector, buf, count); - -- if (do_encrypt (h, cipher, offset, sector, LUKS_SECTOR_SIZE) == -1) -+ if (do_encrypt (h->h, cipher, sectnum, sector, 1) == -1) - goto err; - - if (next->pwrite (next, sector, LUKS_SECTOR_SIZE, --- -2.31.1 - diff --git a/0009-tests-luks-Reduce-time-taken-to-run-these-tests.patch b/0009-tests-luks-Reduce-time-taken-to-run-these-tests.patch deleted file mode 100644 index 34419d8..0000000 --- a/0009-tests-luks-Reduce-time-taken-to-run-these-tests.patch +++ /dev/null @@ -1,101 +0,0 @@ -From 387bd4c6fee8ab339fd04e0b841b0c67e6020c8a Mon Sep 17 00:00:00 2001 -From: "Richard W.M. Jones" -Date: Sun, 8 May 2022 18:05:45 +0100 -Subject: [PATCH] tests: luks: Reduce time taken to run these tests - -Under valgrind they ran very slowly. Turns out valgrinding over -GnuTLS hashing code is not pretty. About half the time seems to be -taken opening the keyslot, and the rest copying the data. - -This change reduces the time (under valgrind) from 15 minutes 45 seconds -to about 6 mins 30 seconds. - -(cherry picked from commit 7320ae5dba476171a024ca44b889b3474302dc40) ---- - tests/test-luks-copy.sh | 18 +++++++++--------- - tests/test-luks-info.sh | 6 +++--- - 2 files changed, 12 insertions(+), 12 deletions(-) - -diff --git a/tests/test-luks-copy.sh b/tests/test-luks-copy.sh -index 99f300d0..01801811 100755 ---- a/tests/test-luks-copy.sh -+++ b/tests/test-luks-copy.sh -@@ -60,8 +60,8 @@ rm -f $encrypt_disk $plain_disk $pid $sock - qemu-img create -f luks \ - --object secret,data=123456,id=sec0 \ - -o key-secret=sec0 \ -- $encrypt_disk 10M --truncate -s 10M $plain_disk -+ $encrypt_disk 1M -+truncate -s 1M $plain_disk - qemu-img convert --target-image-opts -n \ - --object secret,data=123456,id=sec0 \ - $plain_disk \ -@@ -74,11 +74,11 @@ start_nbdkit -P $pid -U $sock \ - uri="nbd+unix:///?socket=$sock" - - # Copy the whole disk out. It should be empty. --nbdcopy "$uri" $plain_disk -+nbdcopy -C 1 "$uri" $plain_disk - - if [ "$(hexdump -C $plain_disk)" != '00000000 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| - * --00a00000' ]; then -+00100000' ]; then - echo "$0: expected plaintext disk to be empty" - exit 1 - fi -@@ -88,14 +88,14 @@ fi - nbdsh -u "$uri" \ - -c 'h.pwrite(b"1"*65536, 0)' \ - -c 'h.pwrite(b"2"*65536, 128*1024)' \ -- -c 'h.pwrite(b"3"*65536, 9*1024*1024)' \ -+ -c 'h.pwrite(b"3"*65536, 900*1024)' \ - -c 'buf = h.pread(65536, 0)' \ - -c 'assert buf == b"1"*65536' \ - -c 'buf = h.pread(65536, 65536)' \ - -c 'assert buf == bytearray(65536)' \ - -c 'buf = h.pread(65536, 128*1024)' \ - -c 'assert buf == b"2"*65536' \ -- -c 'buf = h.pread(65536, 9*1024*1024)' \ -+ -c 'buf = h.pread(65536, 900*1024)' \ - -c 'assert buf == b"3"*65536' \ - -c 'h.flush()' - -@@ -115,11 +115,11 @@ if [ "$(hexdump -C $plain_disk)" != '00000000 31 31 31 31 31 31 31 31 31 31 31 - * - 00030000 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| - * --00900000 33 33 33 33 33 33 33 33 33 33 33 33 33 33 33 33 |3333333333333333| -+000e1000 33 33 33 33 33 33 33 33 33 33 33 33 33 33 33 33 |3333333333333333| - * --00910000 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| -+000f1000 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| - * --00a00000' ]; then -+00100000' ]; then - echo "$0: unexpected content" - exit 1 - fi -diff --git a/tests/test-luks-info.sh b/tests/test-luks-info.sh -index 3eff657b..ef141ecd 100755 ---- a/tests/test-luks-info.sh -+++ b/tests/test-luks-info.sh -@@ -46,11 +46,11 @@ rm -f $disk $info - qemu-img create -f luks \ - --object secret,data=123456,id=sec0 \ - -o key-secret=sec0 \ -- $disk 10M -+ $disk 1M - - nbdkit -U - file $disk --filter=luks passphrase=123456 \ - --run 'nbdinfo $uri' > $info - cat $info - --# Check the size is 10M (so it doesn't include the LUKS header). --grep "10485760" $info -+# Check the size is 1M (so it doesn't include the LUKS header). -+grep "1048576" $info --- -2.31.1 - diff --git a/0010-Add-nbdkit.parse_size-Python-function.patch b/0010-Add-nbdkit.parse_size-Python-function.patch deleted file mode 100644 index 5cb340b..0000000 --- a/0010-Add-nbdkit.parse_size-Python-function.patch +++ /dev/null @@ -1,112 +0,0 @@ -From 52ee1dab95436128b44c37cc495022ff90108b2e Mon Sep 17 00:00:00 2001 -From: Nikolaus Rath -Date: Mon, 9 May 2022 10:04:30 +0100 -Subject: [PATCH] Add nbdkit.parse_size() Python function. - -This enables Python plugins to parse sizes the same way as C plugins. - -I'm not sure about the best way to test this - input is appreciated. - -I'm not too happy with the way this code is tested. It workes, but putting the tests into -test-python-plugin.py feels misplaced: this file is intended to support the unit tests in -test_python.py, not run its own unit tests. - -(cherry picked from commit 1b7d72542be68e254c1ef86ecb1a82b05c78ff63) ---- - plugins/python/modfunctions.c | 21 +++++++++++++++++++++ - plugins/python/nbdkit-python-plugin.pod | 5 +++++ - tests/test-python-plugin.py | 19 +++++++++++++++++++ - 3 files changed, 45 insertions(+) - -diff --git a/plugins/python/modfunctions.c b/plugins/python/modfunctions.c -index fffbaab2..46b0c904 100644 ---- a/plugins/python/modfunctions.c -+++ b/plugins/python/modfunctions.c -@@ -93,11 +93,32 @@ do_shutdown (PyObject *self, PyObject *args) - Py_RETURN_NONE; - } - -+/* nbdkit.parse_size */ -+static PyObject * -+parse_size (PyObject *self, PyObject *args) -+{ -+ const char *s; -+ if (!PyArg_ParseTuple (args, "s", &s)) { -+ PyErr_SetString (PyExc_TypeError, "Expected string, got something else"); -+ return NULL; -+ } -+ -+ int64_t size = nbdkit_parse_size(s); -+ if (size == -1) { -+ PyErr_SetString (PyExc_ValueError, "Unable to parse string as size"); -+ return NULL; -+ } -+ -+ return PyLong_FromSize_t((size_t)size); -+} -+ - static PyMethodDef NbdkitMethods[] = { - { "debug", debug, METH_VARARGS, - "Print a debug message" }, - { "export_name", export_name, METH_VARARGS, - "Return the optional export name negotiated with the client" }, -+ { "parse_size", parse_size, METH_VARARGS, -+ "Parse human-readable size strings into bytes" }, - { "set_error", set_error, METH_VARARGS, - "Store an errno value prior to throwing an exception" }, - { "shutdown", do_shutdown, METH_VARARGS, -diff --git a/plugins/python/nbdkit-python-plugin.pod b/plugins/python/nbdkit-python-plugin.pod -index 051b0237..ccc9406f 100644 ---- a/plugins/python/nbdkit-python-plugin.pod -+++ b/plugins/python/nbdkit-python-plugin.pod -@@ -131,6 +131,11 @@ Record C as the reason you are about to throw an exception. C - should correspond to usual errno values, where it may help to - C. - -+=head3 C -+ -+Parse a string (such as "100M") into a size in bytes. Wraps the -+C C function. -+ - =head3 C - - Request asynchronous server shutdown. -diff --git a/tests/test-python-plugin.py b/tests/test-python-plugin.py -index 0b34d532..d4f379fc 100644 ---- a/tests/test-python-plugin.py -+++ b/tests/test-python-plugin.py -@@ -34,12 +34,31 @@ - import nbdkit - import pickle - import base64 -+import unittest - - API_VERSION = 2 - - cfg = {} - - -+# Not nice, but there doesn't seem to be a better way of putting this -+class TestAPI(unittest.TestCase): -+ -+ def test_parse_size(self): -+ self.assertEqual(nbdkit.parse_size('511'), 511) -+ self.assertEqual(nbdkit.parse_size('7k'), 7*1024) -+ self.assertEqual(nbdkit.parse_size('17M'), 17*1024*1024) -+ -+ with self.assertRaises(TypeError): -+ nbdkit.parse_size(17) -+ -+ with self.assertRaises(ValueError): -+ nbdkit.parse_size('foo') -+ -+ -+TestAPI().test_parse_size() -+ -+ - def config(k, v): - global cfg - if k == "cfg": --- -2.31.1 - diff --git a/0011-cache-Fix-cross-reference-nbdkit-readahead-filter.patch b/0011-cache-Fix-cross-reference-nbdkit-readahead-filter.patch deleted file mode 100644 index ed4a4e3..0000000 --- a/0011-cache-Fix-cross-reference-nbdkit-readahead-filter.patch +++ /dev/null @@ -1,34 +0,0 @@ -From 644e0ed6333cf5fe2c1e39da157e8f1ce97267b9 Mon Sep 17 00:00:00 2001 -From: "Richard W.M. Jones" -Date: Sat, 14 May 2022 13:47:19 +0100 -Subject: [PATCH] cache: Fix cross-reference nbdkit-readahead-filter - -After the readahead filter was reimplemented so that it only issues -cache requests, the two filters should be used together, not as -alternatives. Update the documentation of the cache filter to make -this clear. - -Fixes: commit 2ff548d66ad3eae87868402ec5b3319edd12090f -(cherry picked from commit 894771f39a8fd2632caad00e497146d69cac4bac) ---- - filters/cache/nbdkit-cache-filter.pod | 4 ++-- - 1 file changed, 2 insertions(+), 2 deletions(-) - -diff --git a/filters/cache/nbdkit-cache-filter.pod b/filters/cache/nbdkit-cache-filter.pod -index d85fef09..f4234e1a 100644 ---- a/filters/cache/nbdkit-cache-filter.pod -+++ b/filters/cache/nbdkit-cache-filter.pod -@@ -28,8 +28,8 @@ loss, as the name suggests). - - This filter only caches image contents. To cache image metadata, use - L between this filter and the plugin. --To accelerate sequential reads, use L --instead. -+To accelerate sequential reads, use L on -+top of this filter. - - =head1 PARAMETERS - --- -2.31.1 - diff --git a/0012-curl-Don-t-document-curl-plugin-readahead-filter.patch b/0012-curl-Don-t-document-curl-plugin-readahead-filter.patch deleted file mode 100644 index 4c8288e..0000000 --- a/0012-curl-Don-t-document-curl-plugin-readahead-filter.patch +++ /dev/null @@ -1,48 +0,0 @@ -From 4a7e5169935c8850fddcea8da79639ded907c549 Mon Sep 17 00:00:00 2001 -From: "Richard W.M. Jones" -Date: Sat, 14 May 2022 14:00:16 +0100 -Subject: [PATCH] curl: Don't document curl plugin + readahead filter - -nbdkit readahead filter does not support plugins which do not use the -parallel thread model. - -Fixes: commit 2ff548d66ad3eae87868402ec5b3319edd12090f -(cherry picked from commit 92fbb76d11b9f17c527debd803aa2505f3642783) ---- - docs/nbdkit-captive.pod | 7 ------- - plugins/curl/nbdkit-curl-plugin.pod | 1 - - 2 files changed, 8 deletions(-) - -diff --git a/docs/nbdkit-captive.pod b/docs/nbdkit-captive.pod -index eafe36d8..d41a824d 100644 ---- a/docs/nbdkit-captive.pod -+++ b/docs/nbdkit-captive.pod -@@ -110,13 +110,6 @@ an embedded disk image. To copy it out: - - nbdkit -U - example1 --run 'qemu-img convert $nbd disk.img' - --If plugin requests have a high overhead (for example making HTTP --requests to a remote server), adding L may --help performance: -- -- nbdkit -U - --filter=readahead curl https://example.com/disk.img \ -- --run 'qemu-img convert $nbd disk.img' -- - If the source suffers from temporary network failures - L or L may - help. -diff --git a/plugins/curl/nbdkit-curl-plugin.pod b/plugins/curl/nbdkit-curl-plugin.pod -index 54fce66c..fc422ca2 100644 ---- a/plugins/curl/nbdkit-curl-plugin.pod -+++ b/plugins/curl/nbdkit-curl-plugin.pod -@@ -509,7 +509,6 @@ L, - L, - L, - L, --L, - L, - L, - L, --- -2.31.1 - diff --git a/0013-New-filter-scan.patch b/0013-New-filter-scan.patch deleted file mode 100644 index 527a597..0000000 --- a/0013-New-filter-scan.patch +++ /dev/null @@ -1,1021 +0,0 @@ -From 8bfe6512d07caf778fd001425435b048c45513eb Mon Sep 17 00:00:00 2001 -From: "Richard W.M. Jones" -Date: Sat, 14 May 2022 13:46:56 +0100 -Subject: [PATCH] New filter: scan - -This filter will simply scan across the disk issuing a series of cache -requests to the underlying plugin. It is similar in scope and usage -to the new nbdkit-readahead-filter. - -(cherry picked from commit 65c20a09ceacb4431986a2982f2c2e746df63fcb) ---- - TODO | 8 - - configure.ac | 2 + - filters/cache/nbdkit-cache-filter.pod | 4 +- - .../nbdkit-cacheextents-filter.pod | 1 + - filters/readahead/nbdkit-readahead-filter.pod | 5 + - filters/scan/Makefile.am | 72 +++++ - filters/scan/bgthread.c | 131 ++++++++ - filters/scan/nbdkit-scan-filter.pod | 159 ++++++++++ - filters/scan/scan.c | 280 ++++++++++++++++++ - filters/scan/scan.h | 64 ++++ - plugins/ssh/nbdkit-ssh-plugin.pod | 1 + - plugins/torrent/nbdkit-torrent-plugin.pod | 1 + - plugins/vddk/nbdkit-vddk-plugin.pod | 1 + - tests/Makefile.am | 10 + - tests/test-scan-copy.sh | 42 +++ - tests/test-scan-info.sh | 46 +++ - 16 files changed, 817 insertions(+), 10 deletions(-) - create mode 100644 filters/scan/Makefile.am - create mode 100644 filters/scan/bgthread.c - create mode 100644 filters/scan/nbdkit-scan-filter.pod - create mode 100644 filters/scan/scan.c - create mode 100644 filters/scan/scan.h - create mode 100755 tests/test-scan-copy.sh - create mode 100755 tests/test-scan-info.sh - -diff --git a/TODO b/TODO -index 0f5dc41d..8600d9e4 100644 ---- a/TODO -+++ b/TODO -@@ -182,14 +182,6 @@ Python: - Suggestions for filters - ----------------------- - --* Add scan filter. This would be placed on top of cache filters and -- would scan (read) the whole disk in the background, ensuring it is -- copied into the cache. Useful if you have a slow plugin, limited -- size device, and lots of local disk space, especially if you know -- that the NBD clients will eventually read all of the device. RWMJ -- wrote an implementation of this but it doesn't work well without a -- background thread. -- - * Add shared filter. Take advantage of filter context APIs to open a - single context into the backend shared among multiple client - connections. This may even allow a filter to offer a more parallel -diff --git a/configure.ac b/configure.ac -index 1d209f67..466dbd9b 100644 ---- a/configure.ac -+++ b/configure.ac -@@ -142,6 +142,7 @@ filters="\ - readahead \ - retry \ - retry-request \ -+ scan \ - stats \ - swab \ - tar \ -@@ -1403,6 +1404,7 @@ AC_CONFIG_FILES([Makefile - filters/readahead/Makefile - filters/retry/Makefile - filters/retry-request/Makefile -+ filters/scan/Makefile - filters/stats/Makefile - filters/swab/Makefile - filters/tar/Makefile -diff --git a/filters/cache/nbdkit-cache-filter.pod b/filters/cache/nbdkit-cache-filter.pod -index f4234e1a..935804b5 100644 ---- a/filters/cache/nbdkit-cache-filter.pod -+++ b/filters/cache/nbdkit-cache-filter.pod -@@ -28,8 +28,8 @@ loss, as the name suggests). - - This filter only caches image contents. To cache image metadata, use - L between this filter and the plugin. --To accelerate sequential reads, use L on --top of this filter. -+To accelerate sequential reads, use L or -+L on top of this filter. - - =head1 PARAMETERS - -diff --git a/filters/cacheextents/nbdkit-cacheextents-filter.pod b/filters/cacheextents/nbdkit-cacheextents-filter.pod -index bb2514a4..6464eac2 100644 ---- a/filters/cacheextents/nbdkit-cacheextents-filter.pod -+++ b/filters/cacheextents/nbdkit-cacheextents-filter.pod -@@ -54,6 +54,7 @@ L, - L, - L, - L, -+L, - L, - L, - L. -diff --git a/filters/readahead/nbdkit-readahead-filter.pod b/filters/readahead/nbdkit-readahead-filter.pod -index 630e5924..99d64dfb 100644 ---- a/filters/readahead/nbdkit-readahead-filter.pod -+++ b/filters/readahead/nbdkit-readahead-filter.pod -@@ -27,6 +27,10 @@ option. - The filter uses a simple adaptive algorithm which accelerates - sequential reads and requires no further configuration. - -+A similar filter is L which reads ahead over -+the whole disk, useful if you know that the client will be reading -+sequentially across most or all of the disk. -+ - =head2 Limitations - - In a number of significant cases this filter will do nothing. The -@@ -91,6 +95,7 @@ L, - L, - L, - L, -+L, - L, - L, - L, -diff --git a/filters/scan/Makefile.am b/filters/scan/Makefile.am -new file mode 100644 -index 00000000..d4aabfc6 ---- /dev/null -+++ b/filters/scan/Makefile.am -@@ -0,0 +1,72 @@ -+# nbdkit -+# Copyright (C) 2019-2021 Red Hat Inc. -+# -+# Redistribution and use in source and binary forms, with or without -+# modification, are permitted provided that the following conditions are -+# met: -+# -+# * Redistributions of source code must retain the above copyright -+# notice, this list of conditions and the following disclaimer. -+# -+# * Redistributions in binary form must reproduce the above copyright -+# notice, this list of conditions and the following disclaimer in the -+# documentation and/or other materials provided with the distribution. -+# -+# * Neither the name of Red Hat nor the names of its contributors may be -+# used to endorse or promote products derived from this software without -+# specific prior written permission. -+# -+# THIS SOFTWARE IS PROVIDED BY RED HAT AND CONTRIBUTORS ''AS IS'' AND -+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, -+# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A -+# PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL RED HAT OR -+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF -+# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND -+# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -+# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT -+# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF -+# SUCH DAMAGE. -+ -+include $(top_srcdir)/common-rules.mk -+ -+EXTRA_DIST = nbdkit-scan-filter.pod -+ -+filter_LTLIBRARIES = nbdkit-scan-filter.la -+ -+nbdkit_scan_filter_la_SOURCES = \ -+ scan.c \ -+ scan.h \ -+ bgthread.c \ -+ $(top_srcdir)/include/nbdkit-filter.h \ -+ $(NULL) -+ -+nbdkit_scan_filter_la_CPPFLAGS = \ -+ -I$(top_srcdir)/include \ -+ -I$(top_srcdir)/common/include \ -+ -I$(top_srcdir)/common/utils \ -+ $(NULL) -+nbdkit_scan_filter_la_CFLAGS = $(WARNINGS_CFLAGS) -+nbdkit_scan_filter_la_LDFLAGS = \ -+ -module -avoid-version -shared $(NO_UNDEFINED_ON_WINDOWS) \ -+ -Wl,--version-script=$(top_srcdir)/filters/filters.syms \ -+ $(NULL) -+nbdkit_scan_filter_la_LIBADD = \ -+ $(top_builddir)/common/utils/libutils.la \ -+ $(top_builddir)/common/replacements/libcompat.la \ -+ $(IMPORT_LIBRARY_ON_WINDOWS) \ -+ $(NULL) -+ -+if HAVE_POD -+ -+man_MANS = nbdkit-scan-filter.1 -+CLEANFILES += $(man_MANS) -+ -+nbdkit-scan-filter.1: nbdkit-scan-filter.pod \ -+ $(top_builddir)/podwrapper.pl -+ $(PODWRAPPER) --section=1 --man $@ \ -+ --html $(top_builddir)/html/$@.html \ -+ $< -+ -+endif HAVE_POD -diff --git a/filters/scan/bgthread.c b/filters/scan/bgthread.c -new file mode 100644 -index 00000000..384e79b6 ---- /dev/null -+++ b/filters/scan/bgthread.c -@@ -0,0 +1,131 @@ -+/* nbdkit -+ * Copyright (C) 2019-2022 Red Hat Inc. -+ * -+ * Redistribution and use in source and binary forms, with or without -+ * modification, are permitted provided that the following conditions are -+ * met: -+ * -+ * * Redistributions of source code must retain the above copyright -+ * notice, this list of conditions and the following disclaimer. -+ * -+ * * Redistributions in binary form must reproduce the above copyright -+ * notice, this list of conditions and the following disclaimer in the -+ * documentation and/or other materials provided with the distribution. -+ * -+ * * Neither the name of Red Hat nor the names of its contributors may be -+ * used to endorse or promote products derived from this software without -+ * specific prior written permission. -+ * -+ * THIS SOFTWARE IS PROVIDED BY RED HAT AND CONTRIBUTORS ''AS IS'' AND -+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, -+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A -+ * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL RED HAT OR -+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF -+ * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND -+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT -+ * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF -+ * SUCH DAMAGE. -+ */ -+ -+#include -+ -+#include -+#include -+#include -+#include -+ -+#include -+ -+#include "scan.h" -+ -+#include "cleanup.h" -+#include "minmax.h" -+ -+static pthread_mutex_t clock_lock; -+static uint64_t clock_ = 0; -+ -+static void -+adjust_clock (uint64_t offset) -+{ -+ ACQUIRE_LOCK_FOR_CURRENT_SCOPE (&clock_lock); -+ if (clock_ < offset) -+ clock_ = offset; -+} -+ -+static void -+reset_clock (uint64_t offset) -+{ -+ ACQUIRE_LOCK_FOR_CURRENT_SCOPE (&clock_lock); -+ clock_ = 0; -+} -+ -+static uint64_t -+get_starting_offset (void) -+{ -+ ACQUIRE_LOCK_FOR_CURRENT_SCOPE (&clock_lock); -+ return scan_clock ? clock_ : 0; -+} -+ -+void * -+scan_thread (void *vp) -+{ -+ struct bgthread_ctrl *ctrl = vp; -+ uint64_t offset, size; -+ int64_t r; -+ -+ assert (ctrl->next != NULL); -+ -+ /* Get the size of the underlying plugin. Exit the thread on error -+ * because there's not much we can do without knowing the size. -+ */ -+ r = ctrl->next->get_size (ctrl->next); -+ if (r == -1) -+ return NULL; -+ size = r; -+ -+ /* Start scanning. */ -+ start: -+ for (offset = get_starting_offset (); offset < size; offset += scan_size) { -+ uint64_t n; -+ -+ /* Execute any commands in the queue. */ -+ { -+ ACQUIRE_LOCK_FOR_CURRENT_SCOPE (&ctrl->lock); -+ struct command cmd; -+ -+ while (ctrl->cmds.len) { -+ cmd = ctrl->cmds.ptr[0]; -+ command_queue_remove (&ctrl->cmds, 0); -+ -+ switch (cmd.type) { -+ case CMD_QUIT: -+ nbdkit_debug ("scan: exiting background thread on connection close"); -+ return NULL; -+ -+ case CMD_NOTIFY_PREAD: -+ if (offset < cmd.offset) -+ offset = cmd.offset; -+ } -+ } -+ } -+ -+ adjust_clock (offset); -+ if (offset > size) -+ continue; -+ -+ /* Issue the next prefetch. */ -+ n = MIN (scan_size, size - offset); -+ ctrl->next->cache (ctrl->next, n, offset, 0, NULL); -+ } -+ -+ if (scan_forever) { -+ reset_clock (offset); -+ goto start; -+ } -+ -+ nbdkit_debug ("scan: finished scanning the plugin"); -+ return NULL; -+} -diff --git a/filters/scan/nbdkit-scan-filter.pod b/filters/scan/nbdkit-scan-filter.pod -new file mode 100644 -index 00000000..4a8d0ef9 ---- /dev/null -+++ b/filters/scan/nbdkit-scan-filter.pod -@@ -0,0 +1,159 @@ -+=head1 NAME -+ -+nbdkit-scan-filter - scan disk prefetching data ahead of sequential reads -+ -+=head1 SYNOPSIS -+ -+ nbdkit --filter=scan PLUGIN [scan-ahead=false] [scan-clock=false] -+ [scan-forever=true] [scan-size=]NN -+ -+ nbdkit --filter=scan --filter=cache PLUGIN -+ -+ nbdkit --filter=scan --filter=cow PLUGIN cow-on-cache=true -+ -+=head1 DESCRIPTION -+ -+C is a filter that scans the disk prefetching -+data. It is sometimes useful if you expect that the client will read -+the disk sequentially. -+ -+The basic operation of the filter is that when a client connects, the -+filter will start issuing C<.cache> (prefetch) requests to the plugin -+across the whole disk. Plugins which support this command will -+prefetch the data, making subsequent reads faster. For plugins which -+do not support this command, you can inject L -+below (after) this filter, giving approximately the same effect. -+L can be used instead of nbdkit-cache-filter, if -+you add the C option. -+ -+Various C parameters can be used to tune scanning, although -+the defaults should be suitable in most cases. -+ -+A similar filter is L. -+ -+=head2 Limitations -+ -+In a number of significant cases this filter will do nothing. The -+filter will print a warning message if this happens. -+ -+=over 4 -+ -+=item Thread model must be parallel * -+ -+For example L only supports -+C, and so this filter cannot perform prefetches in -+parallel with the read requests. -+ -+=item Only scans while clients are connected * -+ -+The current filter only scans while there is at least one client -+connected. -+ -+=item Only scans the default export * -+ -+The current filter only scans the default export and ignores all -+clients connecting to the non-default export name. -+ -+* We may be able to lift these restrictions in future. -+ -+=item Underlying filters or plugin must support C<.cache> (prefetch) -+ -+Very many plugins do not have the concept of prefetching and/or -+do not implement the C<.cache> callback, and so there is no -+way for this filter to issue prefetches. -+ -+You can usually get around this by adding I<--filter=cache> after this -+filter as explained above. -+ -+=item Prefetching the whole disk may load it all into cache -+ -+In particular if you use this filter together with -+L or L, they will cache -+the whole content of the plugin into a temporary file. This may be -+many gigabytes of data, consuming all space in F. Of course -+this is the whole point of using this filter, but you should be aware -+of it. -+ -+If using the cache filter, the total size of the cache can be limited -+(see L). -+ -+=back -+ -+=head1 PARAMETERS -+ -+=over 4 -+ -+=item B -+ -+By default the filter tries to stay ahead of incoming read requests. -+That is to say, it starts prefetching at the beginning of the disk and -+continues incrementally, but if the client issues a read beyond the -+current prefetch point then the filter skips forward and begins -+prefetching after the read. -+ -+However if you set this parameter to false, then this behaviour is -+disabled. The filter simply prefetches sequentially regardless of -+client requests. -+ -+=item B -+ -+By default, if all clients disconnect and then another client -+connects, prefetching resumes at the same place in the disk. (Like -+stopping and starting a clock.) -+ -+If you set this parameter to false, then the filter starts prefetching -+from the beginning of the disk again. -+ -+=item B -+ -+By default the filter scans over the disk once and then stops. -+ -+If you set this parameter to true, then after the disk has been -+prefetched completely, the filter goes back to the beginning and -+starts over, repeating this for as long as nbdkit is running and there -+are clients connected. -+ -+=item BNN -+ -+This parameter controls the prefetch block size. The default is -+C<2M>. This must be a power of 2 and most plugins will have their own -+limits on the amount of data they can prefetch in a single request. -+ -+=back -+ -+=head1 FILES -+ -+=over 4 -+ -+=item F<$filterdir/nbdkit-scan-filter.so> -+ -+The filter. -+ -+Use C to find the location of C<$filterdir>. -+ -+=back -+ -+=head1 VERSION -+ -+C first appeared in nbdkit 1.32. -+ -+=head1 SEE ALSO -+ -+L, -+L, -+L, -+L, -+L, -+L, -+L, -+L, -+L, -+L. -+ -+=head1 AUTHORS -+ -+Richard W.M. Jones -+ -+=head1 COPYRIGHT -+ -+Copyright (C) 2019-2022 Red Hat Inc. -diff --git a/filters/scan/scan.c b/filters/scan/scan.c -new file mode 100644 -index 00000000..ac5b18d2 ---- /dev/null -+++ b/filters/scan/scan.c -@@ -0,0 +1,280 @@ -+/* nbdkit -+ * Copyright (C) 2019-2022 Red Hat Inc. -+ * -+ * Redistribution and use in source and binary forms, with or without -+ * modification, are permitted provided that the following conditions are -+ * met: -+ * -+ * * Redistributions of source code must retain the above copyright -+ * notice, this list of conditions and the following disclaimer. -+ * -+ * * Redistributions in binary form must reproduce the above copyright -+ * notice, this list of conditions and the following disclaimer in the -+ * documentation and/or other materials provided with the distribution. -+ * -+ * * Neither the name of Red Hat nor the names of its contributors may be -+ * used to endorse or promote products derived from this software without -+ * specific prior written permission. -+ * -+ * THIS SOFTWARE IS PROVIDED BY RED HAT AND CONTRIBUTORS ''AS IS'' AND -+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, -+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A -+ * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL RED HAT OR -+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF -+ * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND -+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT -+ * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF -+ * SUCH DAMAGE. -+ */ -+ -+#include -+ -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+ -+#include -+ -+#include "scan.h" -+ -+#include "cleanup.h" -+#include "ispowerof2.h" -+#include "vector.h" -+ -+static bool scan_ahead = true; -+bool scan_clock = true; -+bool scan_forever = false; -+unsigned scan_size = 2*1024*1024; -+ -+static int thread_model = -1; /* Thread model of the underlying plugin. */ -+ -+/* Per-connection data. */ -+struct scan_handle { -+ bool is_default_export; /* If exportname == "". */ -+ bool running; /* True if background thread is running. */ -+ pthread_t thread; /* The background thread, one per connection. */ -+ struct bgthread_ctrl ctrl; -+}; -+ -+static int -+scan_config (nbdkit_next_config *next, nbdkit_backend *nxdata, -+ const char *key, const char *value) -+{ -+ int r; -+ -+ if (strcmp (key, "scan-ahead") == 0) { -+ r = nbdkit_parse_bool (value); -+ if (r == -1) -+ return -1; -+ scan_ahead = r; -+ return 0; -+ } -+ else if (strcmp (key, "scan-clock") == 0) { -+ r = nbdkit_parse_bool (value); -+ if (r == -1) -+ return -1; -+ scan_clock = r; -+ return 0; -+ } -+ else if (strcmp (key, "scan-forever") == 0) { -+ r = nbdkit_parse_bool (value); -+ if (r == -1) -+ return -1; -+ scan_forever = r; -+ return 0; -+ } -+ else if (strcmp (key, "scan-size") == 0) { -+ scan_size = nbdkit_parse_size (value); -+ if (scan_size == -1) -+ return -1; -+ return 0; -+ } -+ -+ return next (nxdata, key, value); -+} -+ -+static int -+scan_config_complete (nbdkit_next_config_complete *next, nbdkit_backend *nxdata) -+{ -+ if (scan_size < 512 || scan_size > 32*1024*1024 || -+ !is_power_of_2 (scan_size)) { -+ nbdkit_error ("scan-size parameter should be [512..32M] " -+ "and a power of two"); -+ return -1; -+ } -+ -+ return next (nxdata); -+} -+ -+#define scan_config_help \ -+ "scan-ahead=false Skip ahead when client reads faster.\n" \ -+ "scan-clock=false Always start prefetching from beginning.\n" \ -+ "scan-forever=true Scan in a loop while clients connected.\n" \ -+ "scan-size=NN Set scan block size." -+ -+/* We need to hook into .get_ready() so we can read the final thread -+ * model (of the whole server). -+ */ -+static int -+scan_get_ready (int final_thread_model) -+{ -+ thread_model = final_thread_model; -+ return 0; -+} -+ -+static int -+send_command_to_background_thread (struct bgthread_ctrl *ctrl, -+ const struct command cmd) -+{ -+ ACQUIRE_LOCK_FOR_CURRENT_SCOPE (&ctrl->lock); -+ if (command_queue_append (&ctrl->cmds, cmd) == -1) -+ return -1; -+ /* Signal the thread if it could be sleeping on an empty queue. */ -+ if (ctrl->cmds.len == 1) -+ pthread_cond_signal (&ctrl->cond); -+ return 0; -+} -+ -+static void * -+scan_open (nbdkit_next_open *next, nbdkit_context *nxdata, -+ int readonly, const char *exportname, int is_tls) -+{ -+ struct scan_handle *h; -+ -+ if (next (nxdata, readonly, exportname) == -1) -+ return NULL; -+ -+ h = calloc (1, sizeof *h); -+ if (h == NULL) { -+ nbdkit_error ("malloc: %m"); -+ return NULL; -+ } -+ -+ h->is_default_export = strcmp (exportname, "") == 0; -+ return h; -+} -+ -+/* In prepare we check if it's possible to support the scan filter on -+ * this connection (or print a warning), and start the background -+ * thread. -+ */ -+static int -+scan_prepare (nbdkit_next *next, void *handle, int readonly) -+{ -+ struct scan_handle *h = handle; -+ int r, err; -+ -+ if (!h->is_default_export) { -+ nbdkit_error ("scan: warning: not the default export, not scanning"); -+ return 0; -+ } -+ -+ if (thread_model != NBDKIT_THREAD_MODEL_PARALLEL) { -+ nbdkit_error ("scan: warning: underlying plugin does not support " -+ "the PARALLEL thread model, not scanning"); -+ return 0; -+ } -+ -+ /* Call next->can_cache to read the underlying 'can_cache'. */ -+ r = next->can_cache (next); -+ if (r == -1) -+ return -1; -+ if (r != NBDKIT_CACHE_NATIVE) { -+ nbdkit_error ("scan: warning: underlying plugin does not support " -+ "NBD_CMD_CACHE, not scanning; try adding --filter=cache " -+ "after this filter"); -+ return 0; -+ } -+ -+ /* Save the connection in the handle, for the background thread to use. */ -+ h->ctrl.next = next; -+ -+ /* Create the background thread. */ -+ h->ctrl.cmds = (command_queue) empty_vector; -+ pthread_mutex_init (&h->ctrl.lock, NULL); -+ pthread_cond_init (&h->ctrl.cond, NULL); -+ -+ err = pthread_create (&h->thread, NULL, scan_thread, &h->ctrl); -+ if (err != 0) { -+ errno = err; -+ nbdkit_error ("pthread_create: %m"); -+ pthread_cond_destroy (&h->ctrl.cond); -+ pthread_mutex_destroy (&h->ctrl.lock); -+ return -1; -+ } -+ -+ h->running = true; -+ -+ return 0; -+} -+ -+/* Finalize cleans up the thread if it is running. */ -+static int -+scan_finalize (nbdkit_next *next, void *handle) -+{ -+ struct scan_handle *h = handle; -+ const struct command quit_cmd = { .type = CMD_QUIT }; -+ -+ if (!h->running) -+ return 0; -+ -+ send_command_to_background_thread (&h->ctrl, quit_cmd); -+ pthread_join (h->thread, NULL); -+ pthread_cond_destroy (&h->ctrl.cond); -+ pthread_mutex_destroy (&h->ctrl.lock); -+ command_queue_reset (&h->ctrl.cmds); -+ h->running = false; -+ -+ return 0; -+} -+ -+static void -+scan_close (void *handle) -+{ -+ struct scan_handle *h = handle; -+ -+ free (h); -+} -+ -+/* Read data. */ -+static int -+scan_pread (nbdkit_next *next, -+ void *handle, void *buf, uint32_t count, uint64_t offset, -+ uint32_t flags, int *err) -+{ -+ struct scan_handle *h = handle; -+ -+ if (scan_ahead && h->running) { -+ const struct command cmd = -+ { .type = CMD_NOTIFY_PREAD, .offset = offset + count }; -+ -+ if (send_command_to_background_thread (&h->ctrl, cmd) == -1) -+ return -1; -+ } -+ -+ /* Issue the normal read. */ -+ return next->pread (next, buf, count, offset, flags, err); -+} -+ -+static struct nbdkit_filter filter = { -+ .name = "scan", -+ .longname = "nbdkit scan filter", -+ .get_ready = scan_get_ready, -+ .config = scan_config, -+ .config_complete = scan_config_complete, -+ .config_help = scan_config_help, -+ .open = scan_open, -+ .prepare = scan_prepare, -+ .finalize = scan_finalize, -+ .close = scan_close, -+ .pread = scan_pread, -+}; -+ -+NBDKIT_REGISTER_FILTER(filter) -diff --git a/filters/scan/scan.h b/filters/scan/scan.h -new file mode 100644 -index 00000000..7ff39310 ---- /dev/null -+++ b/filters/scan/scan.h -@@ -0,0 +1,64 @@ -+/* nbdkit -+ * Copyright (C) 2019-2022 Red Hat Inc. -+ * -+ * Redistribution and use in source and binary forms, with or without -+ * modification, are permitted provided that the following conditions are -+ * met: -+ * -+ * * Redistributions of source code must retain the above copyright -+ * notice, this list of conditions and the following disclaimer. -+ * -+ * * Redistributions in binary form must reproduce the above copyright -+ * notice, this list of conditions and the following disclaimer in the -+ * documentation and/or other materials provided with the distribution. -+ * -+ * * Neither the name of Red Hat nor the names of its contributors may be -+ * used to endorse or promote products derived from this software without -+ * specific prior written permission. -+ * -+ * THIS SOFTWARE IS PROVIDED BY RED HAT AND CONTRIBUTORS ''AS IS'' AND -+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, -+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A -+ * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL RED HAT OR -+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF -+ * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND -+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT -+ * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF -+ * SUCH DAMAGE. -+ */ -+ -+#ifndef NBDKIT_SCAN_H -+#define NBDKIT_SCAN_H -+ -+#include -+#include -+ -+#include -+ -+#include "vector.h" -+ -+extern bool scan_clock; -+extern bool scan_forever; -+extern unsigned scan_size; -+ -+/* List of commands issued to the background thread. */ -+struct command { -+ enum { CMD_QUIT, CMD_NOTIFY_PREAD } type; -+ uint64_t offset; -+}; -+DEFINE_VECTOR_TYPE(command_queue, struct command); -+ -+struct bgthread_ctrl { -+ command_queue cmds; /* Command queue. */ -+ pthread_mutex_t lock; /* Lock for queue. */ -+ pthread_cond_t cond; /* Condition queue size 0 -> 1. */ -+ nbdkit_next *next; /* For sending cache operations. */ -+}; -+ -+/* Start background thread (one per connection). */ -+extern void *scan_thread (void *vp); -+ -+#endif /* NBDKIT_SCAN_H */ -diff --git a/plugins/ssh/nbdkit-ssh-plugin.pod b/plugins/ssh/nbdkit-ssh-plugin.pod -index 2bc2c4a7..214957d6 100644 ---- a/plugins/ssh/nbdkit-ssh-plugin.pod -+++ b/plugins/ssh/nbdkit-ssh-plugin.pod -@@ -349,6 +349,7 @@ L, - L, - L, - L, -+L, - L, - L, - L, -diff --git a/plugins/torrent/nbdkit-torrent-plugin.pod b/plugins/torrent/nbdkit-torrent-plugin.pod -index 196ce4e9..f09ac3d2 100644 ---- a/plugins/torrent/nbdkit-torrent-plugin.pod -+++ b/plugins/torrent/nbdkit-torrent-plugin.pod -@@ -175,6 +175,7 @@ L, - L, - L, - L, -+L, - L, - L, - L. -diff --git a/plugins/vddk/nbdkit-vddk-plugin.pod b/plugins/vddk/nbdkit-vddk-plugin.pod -index ea5899dc..3991e86b 100644 ---- a/plugins/vddk/nbdkit-vddk-plugin.pod -+++ b/plugins/vddk/nbdkit-vddk-plugin.pod -@@ -733,6 +733,7 @@ L, - L, - L, - L, -+L, - L, - L, - L, -diff --git a/tests/Makefile.am b/tests/Makefile.am -index 5585b3b7..799aa6c2 100644 ---- a/tests/Makefile.am -+++ b/tests/Makefile.am -@@ -1754,6 +1754,16 @@ test_retry_request_mirror_LDADD = \ - $(LIBNBD_LIBS) \ - $(NULL) - -+# scan filter test. -+TESTS += \ -+ test-scan-copy.sh \ -+ test-scan-info.sh \ -+ $(NULL) -+EXTRA_DIST += \ -+ test-scan-copy.sh \ -+ test-scan-info.sh \ -+ $(NULL) -+ - # swab filter test. - TESTS += \ - test-swab-8.sh \ -diff --git a/tests/test-scan-copy.sh b/tests/test-scan-copy.sh -new file mode 100755 -index 00000000..227ad7b2 ---- /dev/null -+++ b/tests/test-scan-copy.sh -@@ -0,0 +1,42 @@ -+#!/usr/bin/env bash -+# nbdkit -+# Copyright (C) 2018-2022 Red Hat Inc. -+# -+# Redistribution and use in source and binary forms, with or without -+# modification, are permitted provided that the following conditions are -+# met: -+# -+# * Redistributions of source code must retain the above copyright -+# notice, this list of conditions and the following disclaimer. -+# -+# * Redistributions in binary form must reproduce the above copyright -+# notice, this list of conditions and the following disclaimer in the -+# documentation and/or other materials provided with the distribution. -+# -+# * Neither the name of Red Hat nor the names of its contributors may be -+# used to endorse or promote products derived from this software without -+# specific prior written permission. -+# -+# THIS SOFTWARE IS PROVIDED BY RED HAT AND CONTRIBUTORS ''AS IS'' AND -+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, -+# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A -+# PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL RED HAT OR -+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF -+# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND -+# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -+# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT -+# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF -+# SUCH DAMAGE. -+ -+source ./functions.sh -+set -e -+set -x -+ -+requires nbdcopy --version -+requires_plugin sparse-random -+requires_filter scan -+ -+nbdkit -fv -U - sparse-random 1M --filter=scan --run 'nbdcopy "$uri" "$uri"' -+nbdkit -fv -U - sparse-random 1G --filter=scan --run 'nbdcopy "$uri" "$uri"' -diff --git a/tests/test-scan-info.sh b/tests/test-scan-info.sh -new file mode 100755 -index 00000000..6b109ca8 ---- /dev/null -+++ b/tests/test-scan-info.sh -@@ -0,0 +1,46 @@ -+#!/usr/bin/env bash -+# nbdkit -+# Copyright (C) 2018-2022 Red Hat Inc. -+# -+# Redistribution and use in source and binary forms, with or without -+# modification, are permitted provided that the following conditions are -+# met: -+# -+# * Redistributions of source code must retain the above copyright -+# notice, this list of conditions and the following disclaimer. -+# -+# * Redistributions in binary form must reproduce the above copyright -+# notice, this list of conditions and the following disclaimer in the -+# documentation and/or other materials provided with the distribution. -+# -+# * Neither the name of Red Hat nor the names of its contributors may be -+# used to endorse or promote products derived from this software without -+# specific prior written permission. -+# -+# THIS SOFTWARE IS PROVIDED BY RED HAT AND CONTRIBUTORS ''AS IS'' AND -+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, -+# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A -+# PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL RED HAT OR -+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF -+# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND -+# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -+# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT -+# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF -+# SUCH DAMAGE. -+ -+source ./functions.sh -+set -e -+set -x -+ -+requires nbdinfo --version -+requires_filter scan -+ -+# We're just testing that there are no problematic races with the -+# background thread. -+ -+nbdkit -fv -U - memory 1 --filter=scan --run 'nbdinfo $uri' -+nbdkit -fv -U - memory 1M --filter=scan --run 'nbdinfo $uri' -+nbdkit -fv -U - memory 1G --filter=scan --run 'nbdinfo $uri' -+nbdkit -fv -U - memory 1G --filter=scan -e test --run 'nbdinfo $uri' --- -2.31.1 - diff --git a/0014-scan-Remove-condition-variable.patch b/0014-scan-Remove-condition-variable.patch deleted file mode 100644 index 7500254..0000000 --- a/0014-scan-Remove-condition-variable.patch +++ /dev/null @@ -1,67 +0,0 @@ -From 91677241184ab1aa77adadd612fa069d084863ec Mon Sep 17 00:00:00 2001 -From: "Richard W.M. Jones" -Date: Sat, 14 May 2022 18:54:32 +0100 -Subject: [PATCH] scan: Remove condition variable - -This was copied in from the readahead filter code, but is not actually -needed in this filter because it never has to sleep waiting for a -command. - -Fixes: commit 65c20a09ceacb4431986a2982f2c2e746df63fcb -(cherry picked from commit 43ad586698347997cdfa1bd56bfed0292f89f134) ---- - filters/scan/scan.c | 6 ------ - filters/scan/scan.h | 1 - - 2 files changed, 7 deletions(-) - -diff --git a/filters/scan/scan.c b/filters/scan/scan.c -index ac5b18d2..8a966577 100644 ---- a/filters/scan/scan.c -+++ b/filters/scan/scan.c -@@ -136,9 +136,6 @@ send_command_to_background_thread (struct bgthread_ctrl *ctrl, - ACQUIRE_LOCK_FOR_CURRENT_SCOPE (&ctrl->lock); - if (command_queue_append (&ctrl->cmds, cmd) == -1) - return -1; -- /* Signal the thread if it could be sleeping on an empty queue. */ -- if (ctrl->cmds.len == 1) -- pthread_cond_signal (&ctrl->cond); - return 0; - } - -@@ -199,13 +196,11 @@ scan_prepare (nbdkit_next *next, void *handle, int readonly) - /* Create the background thread. */ - h->ctrl.cmds = (command_queue) empty_vector; - pthread_mutex_init (&h->ctrl.lock, NULL); -- pthread_cond_init (&h->ctrl.cond, NULL); - - err = pthread_create (&h->thread, NULL, scan_thread, &h->ctrl); - if (err != 0) { - errno = err; - nbdkit_error ("pthread_create: %m"); -- pthread_cond_destroy (&h->ctrl.cond); - pthread_mutex_destroy (&h->ctrl.lock); - return -1; - } -@@ -227,7 +222,6 @@ scan_finalize (nbdkit_next *next, void *handle) - - send_command_to_background_thread (&h->ctrl, quit_cmd); - pthread_join (h->thread, NULL); -- pthread_cond_destroy (&h->ctrl.cond); - pthread_mutex_destroy (&h->ctrl.lock); - command_queue_reset (&h->ctrl.cmds); - h->running = false; -diff --git a/filters/scan/scan.h b/filters/scan/scan.h -index 7ff39310..98c0228b 100644 ---- a/filters/scan/scan.h -+++ b/filters/scan/scan.h -@@ -54,7 +54,6 @@ DEFINE_VECTOR_TYPE(command_queue, struct command); - struct bgthread_ctrl { - command_queue cmds; /* Command queue. */ - pthread_mutex_t lock; /* Lock for queue. */ -- pthread_cond_t cond; /* Condition queue size 0 -> 1. */ - nbdkit_next *next; /* For sending cache operations. */ - }; - --- -2.31.1 - diff --git a/0015-scan-Small-typographical-fix-in-manual.patch b/0015-scan-Small-typographical-fix-in-manual.patch deleted file mode 100644 index c2c6ae0..0000000 --- a/0015-scan-Small-typographical-fix-in-manual.patch +++ /dev/null @@ -1,57 +0,0 @@ -From c191f45530d4dd7f978803c0bfa402ca0fc950df Mon Sep 17 00:00:00 2001 -From: "Richard W.M. Jones" -Date: Sat, 14 May 2022 19:02:48 +0100 -Subject: [PATCH] scan: Small typographical fix in manual - -Fixes: commit 65c20a09ceacb4431986a2982f2c2e746df63fcb -(cherry picked from commit 67d4e3437d2e28fa3ce1c4b3818d2b1e7939c5ec) ---- - filters/scan/nbdkit-scan-filter.pod | 12 ++++++------ - 1 file changed, 6 insertions(+), 6 deletions(-) - -diff --git a/filters/scan/nbdkit-scan-filter.pod b/filters/scan/nbdkit-scan-filter.pod -index 4a8d0ef9..2fe9bb80 100644 ---- a/filters/scan/nbdkit-scan-filter.pod -+++ b/filters/scan/nbdkit-scan-filter.pod -@@ -26,8 +26,8 @@ below (after) this filter, giving approximately the same effect. - L can be used instead of nbdkit-cache-filter, if - you add the C option. - --Various C parameters can be used to tune scanning, although --the defaults should be suitable in most cases. -+Various parameters can be used to tune scanning, although the defaults -+should be suitable in most cases. - - A similar filter is L. - -@@ -38,23 +38,23 @@ filter will print a warning message if this happens. - - =over 4 - --=item Thread model must be parallel * -+=item Thread model must be parallel* - - For example L only supports - C, and so this filter cannot perform prefetches in - parallel with the read requests. - --=item Only scans while clients are connected * -+=item Only scans while clients are connected* - - The current filter only scans while there is at least one client - connected. - --=item Only scans the default export * -+=item Only scans the default export* - - The current filter only scans the default export and ignores all - clients connecting to the non-default export name. - --* We may be able to lift these restrictions in future. -+*We may be able to lift these restrictions in future. - - =item Underlying filters or plugin must support C<.cache> (prefetch) - --- -2.31.1 - diff --git a/0016-ssh-Don-t-reference-readahead-or-scan-filters-from-t.patch b/0016-ssh-Don-t-reference-readahead-or-scan-filters-from-t.patch deleted file mode 100644 index d6e387b..0000000 --- a/0016-ssh-Don-t-reference-readahead-or-scan-filters-from-t.patch +++ /dev/null @@ -1,34 +0,0 @@ -From 651045d703804d7dafab04a0387ca92573f52467 Mon Sep 17 00:00:00 2001 -From: "Richard W.M. Jones" -Date: Sat, 14 May 2022 20:57:38 +0100 -Subject: [PATCH] ssh: Don't reference readahead or scan filters from this - plugin - -These filters do not support plugins which do not use the parallel -thread model. - -Fixes: commit 2ff548d66ad3eae87868402ec5b3319edd12090f -Fixes: commit 65c20a09ceacb4431986a2982f2c2e746df63fcb -See-also: commit 92fbb76d11b9f17c527debd803aa2505f3642783 -(cherry picked from commit 7eb356719376c4d0b2379cea5d39c81602d2d304) ---- - plugins/ssh/nbdkit-ssh-plugin.pod | 2 -- - 1 file changed, 2 deletions(-) - -diff --git a/plugins/ssh/nbdkit-ssh-plugin.pod b/plugins/ssh/nbdkit-ssh-plugin.pod -index 214957d6..bb922d37 100644 ---- a/plugins/ssh/nbdkit-ssh-plugin.pod -+++ b/plugins/ssh/nbdkit-ssh-plugin.pod -@@ -347,9 +347,7 @@ C first appeared in nbdkit 1.12. - L, - L, - L, --L, - L, --L, - L, - L, - L, --- -2.31.1 - diff --git a/0017-scan-Fix-bound-so-we-don-t-try-to-prefetch-beyond-en.patch b/0017-scan-Fix-bound-so-we-don-t-try-to-prefetch-beyond-en.patch deleted file mode 100644 index f78049c..0000000 --- a/0017-scan-Fix-bound-so-we-don-t-try-to-prefetch-beyond-en.patch +++ /dev/null @@ -1,56 +0,0 @@ -From f58d2a04338edc647e2334ff58b49508424e3f3b Mon Sep 17 00:00:00 2001 -From: "Richard W.M. Jones" -Date: Tue, 17 May 2022 13:20:17 +0100 -Subject: [PATCH] scan: Fix bound so we don't try to prefetch beyond end of - disk - -An off-by-one error in the bound could cause the filter to try to -prefetch beyond the end of the underlying plugin. This would cause -nbdkit to crash with this assertion failure: - -nbdkit: backend.c:782: backend_cache: Assertion `backend_valid_range (c, offset, count)' failed. - -The sequence of events was: - - - scan filter background thread started - - - client reads to the end of the disk - - - background thread skips ahead to end of disk (offset == size) - - - background thread tries to prefetch from this point - -In the final step the calculations caused to the background thread to -prefetch a scan-size block beyond the end of the plugin. - -Fixes: commit 65c20a09ceacb4431986a2982f2c2e746df63fcb -(cherry picked from commit 953643429b8c57b4dd20a6c0e5b83704ae9a0e88) ---- - filters/scan/bgthread.c | 10 +++++----- - 1 file changed, 5 insertions(+), 5 deletions(-) - -diff --git a/filters/scan/bgthread.c b/filters/scan/bgthread.c -index 384e79b6..5fa5f27f 100644 ---- a/filters/scan/bgthread.c -+++ b/filters/scan/bgthread.c -@@ -113,12 +113,12 @@ scan_thread (void *vp) - } - - adjust_clock (offset); -- if (offset > size) -- continue; - -- /* Issue the next prefetch. */ -- n = MIN (scan_size, size - offset); -- ctrl->next->cache (ctrl->next, n, offset, 0, NULL); -+ if (offset < size) { -+ /* Issue the next prefetch. */ -+ n = MIN (scan_size, size - offset); -+ ctrl->next->cache (ctrl->next, n, offset, 0, NULL); -+ } - } - - if (scan_forever) { --- -2.31.1 - diff --git a/0018-tests-Add-a-regression-test-for-LUKS-zeroing-crash.patch b/0018-tests-Add-a-regression-test-for-LUKS-zeroing-crash.patch deleted file mode 100644 index 123c091..0000000 --- a/0018-tests-Add-a-regression-test-for-LUKS-zeroing-crash.patch +++ /dev/null @@ -1,110 +0,0 @@ -From d1d2f43223bcda062d10c8e68776590956892f71 Mon Sep 17 00:00:00 2001 -From: "Richard W.M. Jones" -Date: Fri, 10 Jun 2022 22:11:44 +0100 -Subject: [PATCH] tests: Add a regression test for LUKS zeroing crash - -https://listman.redhat.com/archives/libguestfs/2022-June/029188.html -(cherry picked from commit 7ab2ef96803bfc385f786be82ebfdd4cc977d504) ---- - tests/Makefile.am | 2 ++ - tests/test-luks-copy-zero.sh | 70 ++++++++++++++++++++++++++++++++++++ - 2 files changed, 72 insertions(+) - create mode 100755 tests/test-luks-copy-zero.sh - -diff --git a/tests/Makefile.am b/tests/Makefile.am -index 799aa6c2..0f4b0746 100644 ---- a/tests/Makefile.am -+++ b/tests/Makefile.am -@@ -1601,11 +1601,13 @@ if HAVE_GNUTLS_PBKDF2 - TESTS += \ - test-luks-info.sh \ - test-luks-copy.sh \ -+ test-luks-copy-zero.sh \ - $(NULL) - endif - EXTRA_DIST += \ - test-luks-info.sh \ - test-luks-copy.sh \ -+ test-luks-copy-zero.sh \ - $(NULL) - - # multi-conn filter test. -diff --git a/tests/test-luks-copy-zero.sh b/tests/test-luks-copy-zero.sh -new file mode 100755 -index 00000000..6ff560e3 ---- /dev/null -+++ b/tests/test-luks-copy-zero.sh -@@ -0,0 +1,70 @@ -+#!/usr/bin/env bash -+# nbdkit -+# Copyright (C) 2018-2022 Red Hat Inc. -+# -+# Redistribution and use in source and binary forms, with or without -+# modification, are permitted provided that the following conditions are -+# met: -+# -+# * Redistributions of source code must retain the above copyright -+# notice, this list of conditions and the following disclaimer. -+# -+# * Redistributions in binary form must reproduce the above copyright -+# notice, this list of conditions and the following disclaimer in the -+# documentation and/or other materials provided with the distribution. -+# -+# * Neither the name of Red Hat nor the names of its contributors may be -+# used to endorse or promote products derived from this software without -+# specific prior written permission. -+# -+# THIS SOFTWARE IS PROVIDED BY RED HAT AND CONTRIBUTORS ''AS IS'' AND -+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, -+# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A -+# PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL RED HAT OR -+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF -+# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND -+# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -+# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT -+# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF -+# SUCH DAMAGE. -+ -+# Regression test for: -+# https://listman.redhat.com/archives/libguestfs/2022-June/029188.html -+ -+source ./functions.sh -+set -e -+set -x -+ -+requires qemu-img --version -+requires nbdcopy --version -+requires truncate --version -+requires file --version -+requires_filter luks -+ -+encrypt_disk=luks-copy-zero1.img -+zero_disk=luks-copy-zero2.img -+cleanup_fn rm -f $encrypt_disk $zero_disk -+rm -f $encrypt_disk $zero_disk -+ -+# Create an empty encrypted disk container. -+qemu-img create -f luks \ -+ --object secret,data=123456,id=sec0 \ -+ -o key-secret=sec0 \ -+ $encrypt_disk 100M -+ -+# Create an all zeroes disk of the same size. -+truncate -s 100M $zero_disk -+ -+# Using nbdkit-luks-filter, write the zero disk into the encrypted -+# disk. nbdcopy will do this using NBD_CMD_ZERO operations. -+nbdkit -U - -fv \ -+ file $encrypt_disk --filter=luks passphrase=123456 \ -+ --run "nbdcopy -C 1 $zero_disk \$nbd" -+ -+# Check that the encrypted disk is still a LUKS disk. If zeroing is -+# wrong in the filter it's possible that it writes through to the -+# underlying disk, erasing the container. -+file $encrypt_disk -+file $encrypt_disk | grep "LUKS encrypted file" --- -2.31.1 - diff --git a/0019-rate-Allow-burstiness-to-be-controlled.patch b/0019-rate-Allow-burstiness-to-be-controlled.patch deleted file mode 100644 index 3453169..0000000 --- a/0019-rate-Allow-burstiness-to-be-controlled.patch +++ /dev/null @@ -1,121 +0,0 @@ -From c1a7c87fb9710fb29d699d1f39d0da19caf98da0 Mon Sep 17 00:00:00 2001 -From: "Richard W.M. Jones" -Date: Sat, 11 Jun 2022 12:34:02 +0100 -Subject: [PATCH] rate: Allow burstiness to be controlled - -Previously it was fixed at 2.0 seconds. Allowing it to be adjusted -upwards could help with large, lumpy requests. - -(cherry picked from commit f79e951c20510381d5cd83c203c670874a4978f4) ---- - filters/rate/nbdkit-rate-filter.pod | 12 ++++++++++-- - filters/rate/rate.c | 20 +++++++++++++------- - tests/test-rate.sh | 2 +- - 3 files changed, 24 insertions(+), 10 deletions(-) - -diff --git a/filters/rate/nbdkit-rate-filter.pod b/filters/rate/nbdkit-rate-filter.pod -index 8956e641..09ce7dbc 100644 ---- a/filters/rate/nbdkit-rate-filter.pod -+++ b/filters/rate/nbdkit-rate-filter.pod -@@ -9,6 +9,7 @@ nbdkit-rate-filter - limit bandwidth by connection or server - [connection-rate=BITSPERSEC] - [rate-file=FILENAME] - [connection-rate-file=FILENAME] -+ [burstiness=SECS] - - =head1 DESCRIPTION - -@@ -63,6 +64,13 @@ Limit total bandwidth across all connections to C. - Adjust the per-connection or total bandwidth dynamically by writing - C into C. See L below. - -+=item BSECS -+ -+Control the bucket capacity, expressed as a length of time in -+"rate-equivalent seconds" that the client is allowed to burst for -+after a period of inactivity. The default is 2.0 seconds. It's not -+recommended to set this smaller than the default. -+ - =back - - C can be specified as a simple number, or you can use a -@@ -105,8 +113,8 @@ If the size of requests made by your client is much larger than the - rate limit then you can see long, lumpy sleeps in this filter. In the - future we may modify the filter to break up large requests - automatically in order to limit the length of sleeps. Placing the --L in front of this filter may help in the --meantime. -+L in front of this filter, or adjusting -+C upwards may help. - - =head1 FILES - -diff --git a/filters/rate/rate.c b/filters/rate/rate.c -index 1a70d212..26082f8c 100644 ---- a/filters/rate/rate.c -+++ b/filters/rate/rate.c -@@ -68,10 +68,9 @@ static char *rate_file = NULL; - - /* Bucket capacity controls the burst rate. It is expressed as the - * length of time in "rate-equivalent seconds" that the client can -- * burst for after a period of inactivity. This could be adjustable -- * in future. -+ * burst for after a period of inactivity. - */ --#define BUCKET_CAPACITY 2.0 -+static double bucket_capacity = 2.0 /* seconds */; - - /* Global read and write buckets. */ - static struct bucket read_bucket; -@@ -142,6 +141,13 @@ rate_config (nbdkit_next_config *next, nbdkit_backend *nxdata, - return -1; - return 0; - } -+ else if (strcmp (key, "burstiness") == 0) { -+ if (sscanf (value, "%lg", &bucket_capacity) != 1) { -+ nbdkit_error ("burstiness must be a floating point number (seconds)"); -+ return -1; -+ } -+ return 0; -+ } - else - return next (nxdata, key, value); - } -@@ -150,8 +156,8 @@ static int - rate_get_ready (int thread_model) - { - /* Initialize the global buckets. */ -- bucket_init (&read_bucket, rate, BUCKET_CAPACITY); -- bucket_init (&write_bucket, rate, BUCKET_CAPACITY); -+ bucket_init (&read_bucket, rate, bucket_capacity); -+ bucket_init (&write_bucket, rate, bucket_capacity); - - return 0; - } -@@ -178,8 +184,8 @@ rate_open (nbdkit_next_open *next, nbdkit_context *nxdata, - return NULL; - } - -- bucket_init (&h->read_bucket, connection_rate, BUCKET_CAPACITY); -- bucket_init (&h->write_bucket, connection_rate, BUCKET_CAPACITY); -+ bucket_init (&h->read_bucket, connection_rate, bucket_capacity); -+ bucket_init (&h->write_bucket, connection_rate, bucket_capacity); - pthread_mutex_init (&h->read_bucket_lock, NULL); - pthread_mutex_init (&h->write_bucket_lock, NULL); - -diff --git a/tests/test-rate.sh b/tests/test-rate.sh -index 7305c928..ff781c21 100755 ---- a/tests/test-rate.sh -+++ b/tests/test-rate.sh -@@ -56,7 +56,7 @@ nbdkit -U - \ - --filter=blocksize --filter=rate \ - pattern 25M \ - maxdata=65536 \ -- rate=10M \ -+ rate=10M burstiness=2.0 \ - --run 'nbdcopy "$uri" rate.img' - end_t=$SECONDS - --- -2.31.1 - diff --git a/0020-luks-Check-return-values-from-malloc-more-carefully.patch b/0020-luks-Check-return-values-from-malloc-more-carefully.patch deleted file mode 100644 index f4a8d86..0000000 --- a/0020-luks-Check-return-values-from-malloc-more-carefully.patch +++ /dev/null @@ -1,104 +0,0 @@ -From 4e8599886ba4802fef1683811a725e7c4bc4fe72 Mon Sep 17 00:00:00 2001 -From: "Richard W.M. Jones" -Date: Tue, 12 Jul 2022 18:00:38 +0100 -Subject: [PATCH] luks: Check return values from malloc more carefully - -Found by Coverity: - - Error: GCC_ANALYZER_WARNING (CWE-688): [#def53] - nbdkit-1.30.7/filters/luks/luks-encryption.c: scope_hint: In function 'calculate_iv' - nbdkit-1.30.7/filters/luks/luks-encryption.c:175:5: warning[-Wanalyzer-possible-null-argument]: use of possibly-NULL 'iv' where non-null expected - nbdkit-1.30.7/filters/luks/luks-encryption.c:39: included_from: Included from here. - /usr/include/string.h:43:14: note: argument 1 of 'memcpy' must be non-null - # 173| sector32 = (uint32_t) sector; /* truncate to only lower bits */ - # 174| sector32 = htole32 (sector32); - # 175|-> memcpy (iv, §or32, prefixlen); - # 176| memset (iv + prefixlen, 0, ivlen - prefixlen); - # 177| break; - - Error: GCC_ANALYZER_WARNING (CWE-688): [#def54] - nbdkit-1.30.7/filters/luks/luks-encryption.c:184:5: warning[-Wanalyzer-possible-null-argument]: use of possibly-NULL 'iv' where non-null expected - nbdkit-1.30.7/filters/luks/luks-encryption.c:39: included_from: Included from here. - /usr/include/string.h:43:14: note: argument 1 of 'memcpy' must be non-null - # 182| prefixlen = ivlen; - # 183| sector = htole64 (sector); - # 184|-> memcpy (iv, §or, prefixlen); - # 185| memset (iv + prefixlen, 0, ivlen - prefixlen); - # 186| break; - - Error: NULL_RETURNS (CWE-476): [#def55] - nbdkit-1.30.7/filters/luks/luks-encryption.c:498: returned_null: "malloc" returns "NULL" (checked 86 out of 94 times). - nbdkit-1.30.7/filters/luks/luks-encryption.c:498: var_assigned: Assigning: "temp" = "NULL" return value from "malloc". - nbdkit-1.30.7/filters/luks/luks-encryption.c:523: dereference: Dereferencing a pointer that might be "NULL" "temp" when calling "memcpy". [Note: The source code implementation of the function has been overridden by a builtin model.] - # 521| gnutls_hash_deinit (hash, temp); - # 522| - # 523|-> memcpy (&block[i*digest_bytes], temp, blen); - # 524| } - # 525| - -Fixes: commit 468919dce6c5eb57503eacac0f67e5dd87c58e6c -(cherry picked from commit 00c8bbd9e321681843140f697985505de7177f34) ---- - filters/luks/luks-encryption.c | 28 +++++++++++++++++++++++----- - 1 file changed, 23 insertions(+), 5 deletions(-) - -diff --git a/filters/luks/luks-encryption.c b/filters/luks/luks-encryption.c -index 8ee0eb35..19aaf06a 100644 ---- a/filters/luks/luks-encryption.c -+++ b/filters/luks/luks-encryption.c -@@ -495,9 +495,15 @@ af_hash (gnutls_digest_algorithm_t hash_alg, uint8_t *block, size_t len) - size_t digest_bytes = gnutls_hash_get_len (hash_alg); - size_t nr_blocks, last_block_len; - size_t i; -- CLEANUP_FREE uint8_t *temp = malloc (digest_bytes); - int r; - gnutls_hash_hd_t hash; -+ CLEANUP_FREE uint8_t *temp; -+ -+ temp = malloc (digest_bytes); -+ if (!temp) { -+ nbdkit_error ("malloc: %m"); -+ return -1; -+ } - - nr_blocks = len / digest_bytes; - last_block_len = len % digest_bytes; -@@ -874,9 +880,15 @@ int - do_decrypt (struct luks_data *h, gnutls_cipher_hd_t cipher, - uint64_t sector, uint8_t *buf, size_t nr_sectors) - { -- const size_t ivlen = cipher_alg_iv_len (h->cipher_alg, h->cipher_mode); -- CLEANUP_FREE uint8_t *iv = malloc (ivlen); - int r; -+ const size_t ivlen = cipher_alg_iv_len (h->cipher_alg, h->cipher_mode); -+ CLEANUP_FREE uint8_t *iv; -+ -+ iv = malloc (ivlen); -+ if (!iv) { -+ nbdkit_error ("malloc: %m"); -+ return -1; -+ } - - while (nr_sectors) { - calculate_iv (h->ivgen_alg, iv, ivlen, sector); -@@ -902,9 +914,15 @@ int - do_encrypt (struct luks_data *h, gnutls_cipher_hd_t cipher, - uint64_t sector, uint8_t *buf, size_t nr_sectors) - { -- const size_t ivlen = cipher_alg_iv_len (h->cipher_alg, h->cipher_mode); -- CLEANUP_FREE uint8_t *iv = malloc (ivlen); - int r; -+ const size_t ivlen = cipher_alg_iv_len (h->cipher_alg, h->cipher_mode); -+ CLEANUP_FREE uint8_t *iv; -+ -+ iv = malloc (ivlen); -+ if (!iv) { -+ nbdkit_error ("malloc: %m"); -+ return -1; -+ } - - while (nr_sectors) { - calculate_iv (h->ivgen_alg, iv, ivlen, sector); --- -2.31.1 - diff --git a/0021-luks-Avoid-potential-overflow-when-computing-key-mat.patch b/0021-luks-Avoid-potential-overflow-when-computing-key-mat.patch deleted file mode 100644 index 9c95dce..0000000 --- a/0021-luks-Avoid-potential-overflow-when-computing-key-mat.patch +++ /dev/null @@ -1,57 +0,0 @@ -From 1d593a76796574845d7e32aaadd9f7d1ed4e7987 Mon Sep 17 00:00:00 2001 -From: "Richard W.M. Jones" -Date: Tue, 12 Jul 2022 18:07:25 +0100 -Subject: [PATCH] luks: Avoid potential overflow when computing key material - offset and length - -Found by Coverity: - - Error: OVERFLOW_BEFORE_WIDEN (CWE-190): [#def58] - nbdkit-1.30.7/filters/luks/luks-encryption.c:558: overflow_before_widen: Potentially overflowing expression "h->phdr.master_key_len * h->phdr.keyslot[i].stripes" with type "unsigned int" (32 bits, unsigned) is evaluated using 32-bit arithmetic, and then used in a context that expects an expression of type "uint64_t" (64 bits, unsigned). - nbdkit-1.30.7/filters/luks/luks-encryption.c:558: remediation: To avoid overflow, cast either "h->phdr.master_key_len" or "h->phdr.keyslot[i].stripes" to type "uint64_t". - # 556| uint64_t len, r; - # 557| - # 558|-> len = h->phdr.master_key_len * h->phdr.keyslot[i].stripes; - # 559| r = DIV_ROUND_UP (len, LUKS_SECTOR_SIZE); - # 560| r = ROUND_UP (r, LUKS_ALIGN_KEYSLOTS / LUKS_SECTOR_SIZE); - - Error: OVERFLOW_BEFORE_WIDEN (CWE-190): [#def62] - nbdkit-1.30.7/filters/luks/luks-encryption.c:616: overflow_before_widen: Potentially overflowing expression "ks->key_material_offset * 512U" with type "unsigned int" (32 bits, unsigned) is evaluated using 32-bit arithmetic, and then used in a context that expects an expression of type "uint64_t" (64 bits, unsigned). - nbdkit-1.30.7/filters/luks/luks-encryption.c:616: remediation: To avoid overflow, cast either "ks->key_material_offset" or "512U" to type "uint64_t". - # 614| - # 615| /* Read master key material from plugin. */ - # 616|-> start = ks->key_material_offset * LUKS_SECTOR_SIZE; - # 617| if (next->pread (next, split_key, split_key_len, start, 0, &err) == -1) { - # 618| errno = err; - -Fixes: commit 468919dce6c5eb57503eacac0f67e5dd87c58e6c -(cherry picked from commit 808d88fbc7b58b7c95e05f41fec729cba92ef518) ---- - filters/luks/luks-encryption.c | 4 ++-- - 1 file changed, 2 insertions(+), 2 deletions(-) - -diff --git a/filters/luks/luks-encryption.c b/filters/luks/luks-encryption.c -index 19aaf06a..06435b27 100644 ---- a/filters/luks/luks-encryption.c -+++ b/filters/luks/luks-encryption.c -@@ -561,7 +561,7 @@ key_material_length_in_sectors (struct luks_data *h, size_t i) - { - uint64_t len, r; - -- len = h->phdr.master_key_len * h->phdr.keyslot[i].stripes; -+ len = (uint64_t) h->phdr.master_key_len * h->phdr.keyslot[i].stripes; - r = DIV_ROUND_UP (len, LUKS_SECTOR_SIZE); - r = ROUND_UP (r, LUKS_ALIGN_KEYSLOTS / LUKS_SECTOR_SIZE); - return r; -@@ -619,7 +619,7 @@ try_passphrase_in_keyslot (nbdkit_next *next, struct luks_data *h, - } - - /* Read master key material from plugin. */ -- start = ks->key_material_offset * LUKS_SECTOR_SIZE; -+ start = (uint64_t) ks->key_material_offset * LUKS_SECTOR_SIZE; - if (next->pread (next, split_key, split_key_len, start, 0, &err) == -1) { - errno = err; - return -1; --- -2.31.1 - diff --git a/0022-luks-Avoid-memory-leak-on-error-path.patch b/0022-luks-Avoid-memory-leak-on-error-path.patch deleted file mode 100644 index 22a2708..0000000 --- a/0022-luks-Avoid-memory-leak-on-error-path.patch +++ /dev/null @@ -1,36 +0,0 @@ -From ee25c1be953bf385caf23f96384a9834c1f1c250 Mon Sep 17 00:00:00 2001 -From: "Richard W.M. Jones" -Date: Tue, 12 Jul 2022 18:10:30 +0100 -Subject: [PATCH] luks: Avoid memory leak on error path - -Found by Coverity: - - Error: CPPCHECK_WARNING (CWE-401): [#def65] [important] - nbdkit-1.30.7/filters/luks/luks-encryption.c:707: error[memleak]: Memory leak: h - # 705| if (memcmp (h->phdr.magic, expected_magic, LUKS_MAGIC_LEN) != 0) { - # 706| nbdkit_error ("this disk does not contain a LUKS header"); - # 707|-> return NULL; - # 708| } - # 709| h->phdr.version = be16toh (h->phdr.version); - -Fixes: commit 468919dce6c5eb57503eacac0f67e5dd87c58e6c -(cherry picked from commit a345cff137763f105f07bb8942c1bbefd0959cff) ---- - filters/luks/luks-encryption.c | 1 + - 1 file changed, 1 insertion(+) - -diff --git a/filters/luks/luks-encryption.c b/filters/luks/luks-encryption.c -index 06435b27..207a4e46 100644 ---- a/filters/luks/luks-encryption.c -+++ b/filters/luks/luks-encryption.c -@@ -710,6 +710,7 @@ load_header (nbdkit_next *next, const char *passphrase) - - if (memcmp (h->phdr.magic, expected_magic, LUKS_MAGIC_LEN) != 0) { - nbdkit_error ("this disk does not contain a LUKS header"); -+ free (h); - return NULL; - } - h->phdr.version = be16toh (h->phdr.version); --- -2.31.1 - diff --git a/0023-tests-Hoist-some-EXTRA_DIST-out-of-automake-conditio.patch b/0023-tests-Hoist-some-EXTRA_DIST-out-of-automake-conditio.patch deleted file mode 100644 index 0bfe9b4..0000000 --- a/0023-tests-Hoist-some-EXTRA_DIST-out-of-automake-conditio.patch +++ /dev/null @@ -1,48 +0,0 @@ -From 5ccf1068703d300c8b5579b3a6ef0e409b5a713e Mon Sep 17 00:00:00 2001 -From: "Richard W.M. Jones" -Date: Tue, 19 Jul 2022 11:56:47 +0100 -Subject: [PATCH] tests: Hoist some EXTRA_DIST out of automake conditionals - -We can fail to add some test files (test.tcl, test.lua) to the tarball -if compiling with those languages disabled, which would cause knock-on -failures when the tarball was used with the languages enabled. We -already fixed this for Ruby etc, this commit fixes it for Tcl and Lua. - -(cherry picked from commit 3b6763c82909c95431ff57c2fe9be1b98316b057) ---- - tests/Makefile.am | 6 ++++-- - 1 file changed, 4 insertions(+), 2 deletions(-) - -diff --git a/tests/Makefile.am b/tests/Makefile.am -index 0f4b0746..2667be32 100644 ---- a/tests/Makefile.am -+++ b/tests/Makefile.am -@@ -1203,10 +1203,11 @@ EXTRA_DIST += \ - $(NULL) - - # Tcl plugin test. -+EXTRA_DIST += test.tcl -+ - if HAVE_TCL - - LIBGUESTFS_TESTS += test-tcl --EXTRA_DIST += test.tcl - - test_tcl_SOURCES = test-lang-plugins.c test.h - test_tcl_CFLAGS = \ -@@ -1219,10 +1220,11 @@ test_tcl_LDADD = libtest.la $(LIBGUESTFS_LIBS) - endif HAVE_TCL - - # Lua plugin test. -+EXTRA_DIST += test.lua -+ - if HAVE_LUA - - LIBGUESTFS_TESTS += test-lua --EXTRA_DIST += test.lua - - test_lua_SOURCES = test-lang-plugins.c test.h - test_lua_CFLAGS = \ --- -2.31.1 - diff --git a/gating.yaml b/gating.yaml deleted file mode 100755 index 648918d..0000000 --- a/gating.yaml +++ /dev/null @@ -1,6 +0,0 @@ ---- !Policy -product_versions: - - rhel-9 -decision_context: osci_compose_gate -rules: - - !PassingTestCaseRule {test_case_name: osci.brew-build.tier0.functional} diff --git a/libguestfs.keyring b/libguestfs.keyring index f0508c8..bb3eb55 100644 Binary files a/libguestfs.keyring and b/libguestfs.keyring differ diff --git a/nbdkit-1.24.0.tar.gz.sig b/nbdkit-1.24.0.tar.gz.sig deleted file mode 100644 index 14ff4e8..0000000 --- a/nbdkit-1.24.0.tar.gz.sig +++ /dev/null @@ -1,17 +0,0 @@ ------BEGIN PGP SIGNATURE----- - -iQJFBAABCAAvFiEE93dPsa0HSn6Mh2fqkXOPc+G3aKAFAl/3RBgRHHJpY2hAYW5u -ZXhpYS5vcmcACgkQkXOPc+G3aKBIIRAAmgoGrmJ8aYO7z+kKgNFjd/p0QxRTZhS/ -ol59ojG6jIzN2x/C2PFbRmPB6HJTEg4anrDX04WrP6R+lID1RrH9pTFQabv0YDQC -z49oeXAqINYHvAqgFUJCwlymd7BHEYUudLlK3yu7gQKxMM+J/2v0glpxrtLM7KlD -vvSZkVfbvHlCWIbMWLWIaRHeoWZIXNOjsAp3uEWN2YgikDoxbXVKoh07JoQx5tJ5 -2U+a/zo4BQuRspjnhmWc252ZF/8d954/L8J+2mKvbRRf2iAmsqPgS+MNi7WKWO4K -w7/urKn0osuOaArs5xYHJnApmJ9U88CzZpoHQkYhcGgnDOipW9ByJRzT41vVQPW5 -IluQODpZUuawWtRIwV/Eoi+LaV2gINAL48Afr02UFYj4gmYQ5TeayLP7NKRQO0VL -jwL4Z3a0cDyUX4i1OArn2ll8THfiog38HfLb70AG1l3P1BVoVVBYWCYbs4xgC9IK -LWkjPKuGXvkGVfZi0nCGdPTOoB1CqCXUvKHXm52FCHg12uJMrBQEivodBoCTbtl0 -fSjULQcfrovUEb4d/rDAX7EgJbFS+1jDnodaFHsmNToo3CqfkMBdhLkxG3XExwjy -OOR34wZssjTLsLlWH/RPucWD25RDy1vdPBska9QvvO7W0p+aOtFbnttkTh5cqs45 -rHg/sDEiaLA= -=OrsS ------END PGP SIGNATURE----- diff --git a/nbdkit-find-provides b/nbdkit-find-provides deleted file mode 100755 index 7013ccd..0000000 --- a/nbdkit-find-provides +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/bash - - -# Generate RPM provides automatically for nbdkit packages and filters. -# Copyright (C) 2009-2022 Red Hat Inc. - -# To test: -# find /usr/lib64/nbdkit/plugins | ./nbdkit-find-provides VER REL -# find /usr/lib64/nbdkit/filters | ./nbdkit-find-provides VER REL - -ver="$1" -rel="$2" - -function process_file -{ - if [[ $1 =~ /plugins/nbdkit-.*-plugin ]] || - [[ $1 =~ /filters/nbdkit-.*-filter ]]; then - echo "Provides:" "$(basename $1 .so)" "=" "$ver-$rel" - fi -} - -while read line; do - process_file "$line" -done diff --git a/nbdkit.attr b/nbdkit.attr deleted file mode 100644 index 2757679..0000000 --- a/nbdkit.attr +++ /dev/null @@ -1,3 +0,0 @@ -%__nbdkit_provides %{_rpmconfigdir}/nbdkit-find-provides %{version} %{release} -%__nbdkit_path %{_libdir}/nbdkit/(plugins|filters)/nbdkit-.*-(plugin|filter)(\.so)?$ -%__nbdkit_flags exeonly diff --git a/sources b/sources index f9e34cc..bffaaa7 100644 --- a/sources +++ b/sources @@ -1,2 +1,2 @@ -SHA512 (libguestfs.keyring) = 297a15edc7c220222b9f650e0a9361ae132d3f0fed04aeb2237a1d9c3f6dac6f336846434f66480faed72635a33f659e849b052e74b88d1508aeff03f8c9a2ac SHA512 (nbdkit-1.24.0.tar.gz) = cef2976163856b0c46148f9953764ed9ddf59fd2644380375ab34b7a10e981c3bebf892b63771b20a4a34f6ffb59d80f46204a38aa9fd0cd9fef82ff098b35d8 +SHA512 (nbdkit-1.24.0.tar.gz.sig) = 71888a984f3dcb73b87903a9e56bf5a6c6f8114907cb092185e0e3d44ce0406ae62e9671de87fd000f72f73fe732f23c69cdbb2eafa496884846a39fc70f90fa diff --git a/tests/basic-test.sh b/tests/basic-test.sh deleted file mode 100755 index 07eb459..0000000 --- a/tests/basic-test.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash - -set -e -set -x - -# Run nbdkit and check that nbdinfo can connect back to it. -nbdkit -U - memory 1G --run 'nbdinfo "$uri"' diff --git a/tests/tests.yml b/tests/tests.yml deleted file mode 100755 index 28c6a43..0000000 --- a/tests/tests.yml +++ /dev/null @@ -1,12 +0,0 @@ -- hosts: localhost - roles: - - role: standard-test-basic - tags: - - classic - required_packages: - - libnbd - - nbdkit - tests: - - simple: - dir: . - run: ./basic-test.sh