From cdb7ffa3a378a1832c92f40aef9e7dd6d8d093ff Mon Sep 17 00:00:00 2001 From: "Richard W.M. Jones" Date: Wed, 9 Jul 2025 09:48:35 +0100 Subject: [PATCH] curl, ssh, vddk, file: Add nbdkit-count-filter nbdkit-count-filter, added in nbdkit 1.45.1, prints a debug message which just records the number of bytes read, written, trimmed or zeroed. This is useful for determining how many bytes were actually copied over the wire or written to disk, and that's a common question that we currently have no easy way to answer. For more details about this filter see: https://gitlab.com/nbdkit/nbdkit/-/commit/3512c3ce9308b4d940119ac6cc87f1baa9afb655 https://libguestfs.org/nbdkit-count-filter.1.html Use this filter (if available) unconditionally for all inputs that read over the network, and for the common file-based outputs (specifically -o kubevirt). It is only enabled if we're producing debug output (ie. conversion logs). --- README | 1 + input/input_vddk.ml | 7 +++++++ input/nbdkit_curl.ml | 7 +++++++ input/nbdkit_ssh.ml | 7 +++++++ output/output.ml | 2 ++ 5 files changed, 24 insertions(+) diff --git a/README b/README index 1cdbef15..407869b4 100644 --- a/README +++ b/README @@ -69,6 +69,7 @@ REQUIREMENTS + nbdkit-vddk-plugin + nbdkit-blocksize-filter + + nbdkit-count-filter + nbdkit-cow-filter + nbdkit-multi-conn-filter + nbdkit-rate-filter diff --git a/input/input_vddk.ml b/input/input_vddk.ml index 12f43729..e88befa2 100644 --- a/input/input_vddk.ml +++ b/input/input_vddk.ml @@ -313,6 +313,13 @@ See also the virt-v2v-input-vmware(1) manual.") libNN *) Nbdkit.add_filter_if_available cmd "retry"; + (* Add the count filter if available, to report bytes read. + * Since it writes a debug message, only do this if verbose. + * This should be close to the plugin so we're reporting what + * is read over the wire. + *) + if verbose () then Nbdkit.add_filter_if_available cmd "count"; + (* Split very large requests to avoid out of memory errors on the * server. Since we're using this filter, also add minblock=512 * although it will make no difference. diff --git a/input/nbdkit_curl.ml b/input/nbdkit_curl.ml index d8c832b8..f8a0213e 100644 --- a/input/nbdkit_curl.ml +++ b/input/nbdkit_curl.ml @@ -58,6 +58,13 @@ let create_curl ?bandwidth ?cookie_script ?cookie_script_renew ?cor *) Nbdkit.add_filter_if_available cmd "retry"; + (* Add the count filter if available, to report bytes read. + * Since it writes a debug message, only do this if verbose. + * This should be close to the plugin so we're reporting what + * is read over the wire. + *) + if verbose () then Nbdkit.add_filter_if_available cmd "count"; + (* IMPORTANT! Add the COW filter. It must be furthest away * except for the rate filter. *) diff --git a/input/nbdkit_ssh.ml b/input/nbdkit_ssh.ml index e6d99ede..4e4c918f 100644 --- a/input/nbdkit_ssh.ml +++ b/input/nbdkit_ssh.ml @@ -54,6 +54,13 @@ let create_ssh ?bandwidth ?cor ?(retry=true) if retry then Nbdkit.add_filter_if_available cmd "retry"; + (* Add the count filter if available, to report bytes read. + * Since it writes a debug message, only do this if verbose. + * This should be close to the plugin so we're reporting what + * is read over the wire. + *) + if verbose () then Nbdkit.add_filter_if_available cmd "count"; + (* IMPORTANT! Add the COW filter. It must be furthest away * except for the rate filter. *) diff --git a/output/output.ml b/output/output.ml index b3629372..65325538 100644 --- a/output/output.ml +++ b/output/output.ml @@ -98,6 +98,7 @@ let output_to_local_file ?(changeuid = fun f -> f ()) ?(compressed = false) let cmd = Nbdkit.create "file" in Nbdkit.add_arg cmd "file" filename; Nbdkit.add_arg cmd "cache" "none"; + if verbose () then Nbdkit.add_filter_if_available cmd "count"; let _, pid = Nbdkit.run_unix socket cmd in pid @@ -197,6 +198,7 @@ let create_local_output_disks dir let cmd = Nbdkit.create "file" in Nbdkit.add_arg cmd "dir" output_storage; Nbdkit.add_arg cmd "cache" "none"; + if verbose () then Nbdkit.add_filter_if_available cmd "count"; let _, pid = Nbdkit.run_unix socket cmd in On_exit.kill pid;