278 lines
9.4 KiB
Diff
278 lines
9.4 KiB
Diff
From de71ce60bff586dad2d554d3de344dd7d2d13c86 Mon Sep 17 00:00:00 2001
|
||
From: "Richard W.M. Jones" <rjones@redhat.com>
|
||
Date: Mon, 2 Dec 2024 15:22:43 +0000
|
||
Subject: [PATCH] v2v: Implement --parallel=N for parallel disk copies
|
||
|
||
When set, run up to N copies of nbdcopy in parallel. This only
|
||
applies for guests with multiple disks.
|
||
|
||
The default, as for all older versions of virt-v2v, is to copy disks
|
||
one at a time.
|
||
|
||
(cherry picked from commit fd1148f79581b148525eb12154aef7603ccf0baa)
|
||
(cherry picked from commit 037a603c2d5cf9d2d5f8157116dbf14945277dc2)
|
||
---
|
||
docs/virt-v2v.pod | 13 +++++++
|
||
lib/utils.ml | 6 ++++
|
||
lib/utils.mli | 4 +++
|
||
tests/Makefile.am | 2 ++
|
||
tests/test-v2v-i-disk-parallel.sh | 54 +++++++++++++++++++++++++++++
|
||
v2v/v2v.ml | 56 +++++++++++++++++++++++++------
|
||
6 files changed, 125 insertions(+), 10 deletions(-)
|
||
create mode 100755 tests/test-v2v-i-disk-parallel.sh
|
||
|
||
diff --git a/docs/virt-v2v.pod b/docs/virt-v2v.pod
|
||
index de92db84..f0857ebf 100644
|
||
--- a/docs/virt-v2v.pod
|
||
+++ b/docs/virt-v2v.pod
|
||
@@ -556,6 +556,19 @@ The directory must exist.
|
||
|
||
For I<-o openstack>, this is the optional Cinder volume type.
|
||
|
||
+=item B<--parallel> N
|
||
+
|
||
+Enable parallel copying if the guest has multiple disks. I<N> is the
|
||
+maximum number of parallel L<nbdcopy(1)> instances to run.
|
||
+
|
||
+The default is to run at most one instance of nbdcopy
|
||
+(ie. I<--parallel=1>). All versions of virt-v2v E<le> 2.7.2 also did
|
||
+disk copies one at a time.
|
||
+
|
||
+Within each guest disk, nbdcopy tries to copy in parallel if the
|
||
+underlying endpoints support that. This is not affected by this
|
||
+command line option. See the L<nbdcopy(1)> manual page for details.
|
||
+
|
||
=item B<--print-source>
|
||
|
||
Print information about the source guest and stop. This option is
|
||
diff --git a/lib/utils.ml b/lib/utils.ml
|
||
index c4cfd89b..f2da9e80 100644
|
||
--- a/lib/utils.ml
|
||
+++ b/lib/utils.ml
|
||
@@ -29,6 +29,12 @@ let large_tmpdir =
|
||
try Sys.getenv "VIRT_V2V_TMPDIR"
|
||
with Not_found -> (open_guestfs ())#get_cachedir ()
|
||
|
||
+let string_of_process_status = function
|
||
+ | Unix.WEXITED 0 -> s_"success"
|
||
+ | WEXITED i -> sprintf (f_"exited with non-zero error code %d") i
|
||
+ | WSIGNALED i -> sprintf (f_"signalled by signal %d") i
|
||
+ | WSTOPPED i -> sprintf (f_"stopped by signal %d") i
|
||
+
|
||
(* Is SELinux enabled and enforcing on the host? *)
|
||
let have_selinux =
|
||
0 = Sys.command "getenforce 2>/dev/null | grep -isq Enforcing"
|
||
diff --git a/lib/utils.mli b/lib/utils.mli
|
||
index afe61a4e..e7ee13d1 100644
|
||
--- a/lib/utils.mli
|
||
+++ b/lib/utils.mli
|
||
@@ -23,6 +23,10 @@ val large_tmpdir : string
|
||
such as overlays in this directory. Small temporary files can
|
||
use the default behaviour eg. of {!Filename.temp_file} *)
|
||
|
||
+val string_of_process_status : Unix.process_status -> string
|
||
+(** Convert a process status (such as returned by {!Unix.wait}) into
|
||
+ a printable string. *)
|
||
+
|
||
val have_selinux : bool
|
||
(** True if SELinux is enabled and enforcing on the host. *)
|
||
|
||
diff --git a/tests/Makefile.am b/tests/Makefile.am
|
||
index 2cf57248..0035262e 100644
|
||
--- a/tests/Makefile.am
|
||
+++ b/tests/Makefile.am
|
||
@@ -74,6 +74,7 @@ TESTS = \
|
||
test-v2v-cdrom.sh \
|
||
test-v2v-floppy.sh \
|
||
test-v2v-i-disk.sh \
|
||
+ test-v2v-i-disk-parallel.sh \
|
||
test-v2v-i-ova.sh \
|
||
test-v2v-inspector.sh \
|
||
test-v2v-mac.sh \
|
||
@@ -185,6 +186,7 @@ EXTRA_DIST += \
|
||
test-v2v-floppy.expected \
|
||
test-v2v-floppy.sh \
|
||
test-v2v-i-disk.sh \
|
||
+ test-v2v-i-disk-parallel.sh \
|
||
test-v2v-i-ova-as-root.ovf \
|
||
test-v2v-i-ova-as-root.sh \
|
||
test-v2v-i-ova-bad-sha1.sh \
|
||
diff --git a/tests/test-v2v-i-disk-parallel.sh b/tests/test-v2v-i-disk-parallel.sh
|
||
new file mode 100755
|
||
index 00000000..a6470fdd
|
||
--- /dev/null
|
||
+++ b/tests/test-v2v-i-disk-parallel.sh
|
||
@@ -0,0 +1,54 @@
|
||
+#!/bin/bash -
|
||
+# libguestfs virt-v2v test script
|
||
+# Copyright (C) 2014-2024 Red Hat Inc.
|
||
+#
|
||
+# This program is free software; you can redistribute it and/or modify
|
||
+# it under the terms of the GNU General Public License as published by
|
||
+# the Free Software Foundation; either version 2 of the License, or
|
||
+# (at your option) any later version.
|
||
+#
|
||
+# This program is distributed in the hope that it will be useful,
|
||
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||
+# GNU General Public License for more details.
|
||
+#
|
||
+# You should have received a copy of the GNU General Public License
|
||
+# along with this program; if not, write to the Free Software
|
||
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||
+
|
||
+# Test --parallel option.
|
||
+
|
||
+set -e
|
||
+
|
||
+source ./functions.sh
|
||
+set -e
|
||
+set -x
|
||
+
|
||
+skip_if_skipped
|
||
+windows=../test-data/phony-guests/windows.img
|
||
+requires test -f $windows
|
||
+
|
||
+export VIRT_TOOLS_DATA_DIR="$srcdir/../test-data/fake-virt-tools"
|
||
+
|
||
+d=test-v2v-i-disk-parallel.d
|
||
+rm -rf $d
|
||
+cleanup_fn rm -rf $d
|
||
+mkdir $d
|
||
+
|
||
+truncate -s $((100*1024*1024)) $d/disk-2.img $d/disk-3.img $d/disk-4.img
|
||
+
|
||
+$VG virt-v2v --debug-gc \
|
||
+ --parallel=2 \
|
||
+ -i disk \
|
||
+ $windows \
|
||
+ $d/disk-2.img \
|
||
+ $d/disk-3.img \
|
||
+ $d/disk-4.img \
|
||
+ -o local -os $d
|
||
+
|
||
+# Test the libvirt XML metadata and output disks were created.
|
||
+test -f $d/windows.xml
|
||
+test -f $d/windows-sda
|
||
+test -f $d/windows-sdb
|
||
+test -f $d/windows-sdc
|
||
+test -f $d/windows-sdd
|
||
diff --git a/v2v/v2v.ml b/v2v/v2v.ml
|
||
index 8be1e5cb..b7dbce3a 100644
|
||
--- a/v2v/v2v.ml
|
||
+++ b/v2v/v2v.ml
|
||
@@ -79,6 +79,7 @@ let rec main () =
|
||
)
|
||
in
|
||
|
||
+ let parallel = ref 1 in
|
||
let network_map = Networks.create () in
|
||
let static_ips = ref [] in
|
||
let rec add_network str =
|
||
@@ -259,6 +260,8 @@ let rec main () =
|
||
s_"Set output storage location";
|
||
[ L"password-file" ], Getopt.String ("filename", set_string_option_once "-ip" input_password),
|
||
s_"Same as ‘-ip filename’";
|
||
+ [ L"parallel" ], Getopt.Set_int ("N", parallel),
|
||
+ s_"Run up to N instances of nbdcopy in parallel";
|
||
[ L"print-source" ], Getopt.Set print_source,
|
||
s_"Print source and stop";
|
||
[ L"root" ], Getopt.String ("ask|... ", set_root_choice),
|
||
@@ -346,6 +349,7 @@ read the man page virt-v2v(1).
|
||
| `Preallocated -> Types.Preallocated in
|
||
let output_mode = !output_mode in
|
||
let output_name = !output_name in
|
||
+ let parallel = !parallel in
|
||
let print_source = !print_source in
|
||
let root_choice = !root_choice in
|
||
let static_ips = !static_ips in
|
||
@@ -365,6 +369,7 @@ read the man page virt-v2v(1).
|
||
pr "mac-option\n";
|
||
pr "bandwidth-option\n";
|
||
pr "mac-ip-option\n";
|
||
+ pr "parallel-option\n";
|
||
pr "customize-ops\n";
|
||
pr "input:disk\n";
|
||
pr "input:libvirt\n";
|
||
@@ -551,12 +556,15 @@ read the man page virt-v2v(1).
|
||
else
|
||
List.rev acc
|
||
in
|
||
- let disks = loop [] 0 in
|
||
- let nr_disks = List.length disks in
|
||
+ let disks = ref (loop [] 0) in
|
||
+ let nr_disks = List.length !disks in
|
||
|
||
(* Copy the disks. *)
|
||
- List.iter (
|
||
- fun (i, input_socket, output_socket) ->
|
||
+ let nbdcopy_pids = ref [] in
|
||
+ let rec copy_loop () =
|
||
+ if List.length !nbdcopy_pids < parallel && !disks <> [] then (
|
||
+ (* Schedule another nbdcopy process. *)
|
||
+ let i, input_socket, output_socket = List.pop_front disks in
|
||
message (f_"Copying disk %d/%d") (i+1) nr_disks;
|
||
|
||
let request_size = Output_module.request_size
|
||
@@ -576,8 +584,33 @@ read the man page virt-v2v(1).
|
||
flush Stdlib.stderr
|
||
);
|
||
|
||
- nbdcopy ?request_size output_alloc input_uri output_uri
|
||
- ) disks;
|
||
+ let pid = nbdcopy ?request_size output_alloc input_uri output_uri in
|
||
+ List.push_front pid nbdcopy_pids;
|
||
+
|
||
+ copy_loop ();
|
||
+ )
|
||
+ else if !nbdcopy_pids <> [] then (
|
||
+ (* Wait for one nbdcopy instance to exit. *)
|
||
+ let pid, status = wait () in
|
||
+ (* If this internal error turns up in real world scenarios then
|
||
+ * we may need to change the [wait] above so it only waits on
|
||
+ * the nbdcopy PIDs.
|
||
+ *)
|
||
+ if not (List.mem pid !nbdcopy_pids) then
|
||
+ error (f_"internal error: wait returned unexpected \
|
||
+ process ID %d status \"%s\"")
|
||
+ pid (string_of_process_status status);
|
||
+ nbdcopy_pids := List.filter ((<>) pid) !nbdcopy_pids;
|
||
+ (match status with
|
||
+ | WEXITED 0 -> copy_loop ()
|
||
+ | WEXITED _ | WSIGNALED _ | WSTOPPED _ ->
|
||
+ error "nbdcopy %s" (string_of_process_status status)
|
||
+ );
|
||
+ )
|
||
+ in
|
||
+ copy_loop ();
|
||
+ assert (!disks == []);
|
||
+ assert (!nbdcopy_pids == []);
|
||
|
||
(* End of copying phase. *)
|
||
unlink (v2vdir // "copy");
|
||
@@ -615,6 +648,7 @@ and check_host_free_space () =
|
||
\"Minimum free space check in the host\".")
|
||
large_tmpdir (human_size free_space)
|
||
|
||
+(* Start nbdcopy as a background process, returning the PID. *)
|
||
and nbdcopy ?request_size output_alloc input_uri output_uri =
|
||
(* XXX It's possible that some output modes know whether
|
||
* --target-is-zero which would be a useful optimization.
|
||
@@ -642,10 +676,12 @@ and nbdcopy ?request_size output_alloc input_uri output_uri =
|
||
if not (quiet ()) then List.push_back cmd "--progress";
|
||
if output_alloc = Types.Preallocated then List.push_back cmd "--allocated";
|
||
|
||
- let cmd = !cmd in
|
||
-
|
||
- if run_command cmd <> 0 then
|
||
- error (f_"nbdcopy command failed, see earlier error messages")
|
||
+ let args = Array.of_list !cmd in
|
||
+ match fork () with
|
||
+ | 0 ->
|
||
+ (* Child process (nbdcopy). *)
|
||
+ execvp "nbdcopy" args
|
||
+ | pid -> pid
|
||
|
||
(* Run nbdinfo on a URI and collect the information. However don't
|
||
* fail if nbdinfo is not installed since this is just used for debugging.
|