WHATS_NEW | 3 + configure | 25 +++ configure.ac | 15 ++ include/configure.h.in | 3 + man/Makefile.in | 7 +- scripts/Makefile.in | 4 + scripts/vdoimport.sh | 376 ++++++++++++++++++++++++++++++++++++++++++++++ test/Makefile.in | 1 + test/shell/vdo-convert.sh | 110 ++++++++++++++ 9 files changed, 543 insertions(+), 1 deletion(-) create mode 100755 scripts/vdoimport.sh create mode 100644 test/shell/vdo-convert.sh diff --git a/WHATS_NEW b/WHATS_NEW index 04c6dcd..5806ecb 100644 --- a/WHATS_NEW +++ b/WHATS_NEW @@ -1,5 +1,8 @@ Version 2.03.13 - =============================== + Add vdoimport tool to support conversion of VDO volumes. + Support configurable allocation/vdo_pool_header_size. + Fix handling of lvconvert --type vdo-pool --virtualsize. Fix load of kvdo target when it is not present in memory (2.03.12). Version 2.03.12 - 07th May 2021 diff --git a/configure b/configure index 7c6bd48..661702d 100755 --- a/configure +++ b/configure @@ -643,6 +643,8 @@ WRITE_INSTALL WRITECACHE VDO_LIB VDO_INCLUDE +VDOIMPORT_PATH +VDOIMPORT VDO VALGRIND_POOL USRSBINDIR @@ -966,6 +968,7 @@ enable_dbus_service enable_pkgconfig enable_write_install enable_fsadm +enable_vdoimport enable_blkdeactivate enable_dmeventd enable_selinux @@ -1701,6 +1704,7 @@ Optional Features: --enable-pkgconfig install pkgconfig support --enable-write_install install user writable files --disable-fsadm disable fsadm + --disable-vdoimport disable vdoimport --disable-blkdeactivate disable blkdeactivate --enable-dmeventd enable the device-mapper event daemon --disable-selinux disable selinux support @@ -3128,6 +3132,7 @@ case "$host_os" in DM_IOCTLS=yes SELINUX=yes FSADM=yes + VDOIMPORT=yes BLKDEACTIVATE=yes ;; darwin*) @@ -3141,6 +3146,7 @@ case "$host_os" in DM_IOCTLS=no SELINUX=no FSADM=no + VDOIMPORT=no BLKDEACTIVATE=no ;; *) @@ -12371,6 +12377,18 @@ fi { $as_echo "$as_me:${as_lineno-$LINENO}: result: $FSADM" >&5 $as_echo "$FSADM" >&6; } + +################################################################################ +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to install vdoimport" >&5 +$as_echo_n "checking whether to install vdoimport... " >&6; } +# Check whether --enable-vdoimport was given. +if test "${enable_vdoimport+set}" = set; then : + enableval=$enable_vdoimport; VDOIMPORT=$enableval +fi + +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $VDOIMPORT" >&5 +$as_echo "$VDOIMPORT" >&6; } + ################################################################################ { $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to install blkdeactivate" >&5 $as_echo_n "checking whether to install blkdeactivate... " >&6; } @@ -13857,6 +13875,13 @@ cat >>confdefs.h <<_ACEOF _ACEOF +VDOIMPORT_PATH="$SBINDIR/vdoimport" + +cat >>confdefs.h <<_ACEOF +#define VDOIMPORT_PATH "$VDOIMPORT_PATH" +_ACEOF + + ################################################################################ if test "$BUILD_DMEVENTD" = yes; then diff --git a/configure.ac b/configure.ac index 1a49e7f..5a8b486 100644 --- a/configure.ac +++ b/configure.ac @@ -45,6 +45,7 @@ case "$host_os" in DM_IOCTLS=yes SELINUX=yes FSADM=yes + VDOIMPORT=yes BLKDEACTIVATE=yes ;; darwin*) @@ -58,6 +59,7 @@ case "$host_os" in DM_IOCTLS=no SELINUX=no FSADM=no + VDOIMPORT=no BLKDEACTIVATE=no ;; *) @@ -1291,6 +1293,14 @@ AC_ARG_ENABLE(fsadm, AC_HELP_STRING([--disable-fsadm], [disable fsadm]), FSADM=$enableval) AC_MSG_RESULT($FSADM) + +################################################################################ +dnl -- Enable vdoimport +AC_MSG_CHECKING(whether to install vdoimport) +AC_ARG_ENABLE(vdoimport, AC_HELP_STRING([--disable-vdoimport], [disable vdoimport]), + VDOIMPORT=$enableval) +AC_MSG_RESULT($VDOIMPORT) + ################################################################################ dnl -- Enable blkdeactivate AC_MSG_CHECKING(whether to install blkdeactivate) @@ -1646,6 +1656,9 @@ USRSBINDIR="$(eval echo $(eval echo $usrsbindir))" FSADM_PATH="$SBINDIR/fsadm" AC_DEFINE_UNQUOTED(FSADM_PATH, ["$FSADM_PATH"], [Path to fsadm binary.]) +VDOIMPORT_PATH="$SBINDIR/vdoimport" +AC_DEFINE_UNQUOTED(VDOIMPORT_PATH, ["$VDOIMPORT_PATH"], [Path to vdoimport binary.]) + ################################################################################ dnl -- dmeventd pidfile and executable path if test "$BUILD_DMEVENTD" = yes; then @@ -1882,6 +1895,8 @@ AC_SUBST(SILENT_RULES) AC_SUBST(USRSBINDIR) AC_SUBST(VALGRIND_POOL) AC_SUBST(VDO) +AC_SUBST(VDOIMPORT) +AC_SUBST(VDOIMPORT_PATH) AC_SUBST(VDO_FORMAT_CMD) AC_SUBST(VDO_INCLUDE) AC_SUBST(VDO_LIB) diff --git a/include/configure.h.in b/include/configure.h.in index 671d201..6df8d89 100644 --- a/include/configure.h.in +++ b/include/configure.h.in @@ -684,6 +684,9 @@ /* Enable a valgrind aware build of pool */ #undef VALGRIND_POOL +/* Path to vdoimport binary. */ +#undef VDOIMPORT_PATH + /* The path to 'vdoformat', if available. */ #undef VDO_FORMAT_CMD diff --git a/man/Makefile.in b/man/Makefile.in index 29afc77..d60a92c 100644 --- a/man/Makefile.in +++ b/man/Makefile.in @@ -23,6 +23,7 @@ else endif FSADMMAN = fsadm.8 +VDOIMPORTMAN = vdoimport.8 BLKDEACTIVATEMAN = blkdeactivate.8 DMEVENTDMAN = dmeventd.8 DMFILEMAPDMAN = dmfilemapd.8 @@ -50,7 +51,7 @@ MAN8SYSTEMD_GENERATORS=lvm2-activation-generator.8 ifeq (,$(findstring $(MAKECMDGOALS), distclean all_man install_all_man)) MAN7 += lvmcache.7 lvmthin.7 lvmvdo.7 - MAN8+=$(FSADMMAN) $(LVMPOLLDMAN) $(LVMLOCKDMAN) $(LVMDBUSDMAN) + MAN8+=$(FSADMMAN) $(LVMPOLLDMAN) $(LVMLOCKDMAN) $(LVMDBUSDMAN) $(VDOIMPORTMAN) MAN8DM+=$(BLKDEACTIVATEMAN) $(DMEVENTDMAN) $(DMFILEMAPDMAN) MAN8CLUSTER+=$(CMIRRORDMAN) else @@ -58,6 +59,10 @@ else MAN8+=$(FSADMMAN) endif + ifeq ("@VDOIMPORT@", "yes") + MAN8+=$(VDOIMPORTMAN) + endif + ifeq ("@BUILD_LVMDBUSD@", "yes") MAN8+=$(LVMDBUSDMAN) endif diff --git a/scripts/Makefile.in b/scripts/Makefile.in index e8f6742..1fe88ca 100644 --- a/scripts/Makefile.in +++ b/scripts/Makefile.in @@ -31,6 +31,10 @@ ifeq ("@FSADM@", "yes") LVM_SCRIPTS += fsadm.sh endif +ifeq ("@VDOIMPORT@", "yes") + LVM_SCRIPTS += vdoimport.sh +endif + ifeq ("@BLKDEACTIVATE@", "yes") DM_SCRIPTS += blkdeactivate.sh endif diff --git a/scripts/vdoimport.sh b/scripts/vdoimport.sh new file mode 100755 index 0000000..ef96591 --- /dev/null +++ b/scripts/vdoimport.sh @@ -0,0 +1,376 @@ +#!/bin/bash +# +# Copyright (C) 2021 Red Hat, Inc. All rights reserved. +# +# This file is part of LVM2. +# +# This copyrighted material is made available to anyone wishing to use, +# modify, copy, or redistribute it subject to the terms and conditions +# of the GNU General Public License v.2. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +# +# Author: Zdenek Kabelac +# +# Script for converting VDO volumes to lvm2 VDO LVs +# +# Needed utilities: +# lvm, dmsetup, +# vdo, vdo2lvm, +# grep, awk, sed, blockdev, readlink, mkdir +# +# Conversion is using 'vdo convert' support from VDO manager to move +# existing VDO header by 2M which makes space to place in PV header +# and VG metadata area, and then create VDOPOOL LV and VDO LV in such VG. +# + +set -euE -o pipefail + +TOOL=vdoimport + +_SAVEPATH=$PATH +PATH="/sbin:/usr/sbin:/bin:/usr/sbin:$PATH" + +# user may override lvm location by setting LVM_BINARY +LVM=${LVM_BINARY:-lvm} +VDO=${VDO_BINARY:-vdo} +VDOCONF=${VDOCONF:-} +BLOCKDEV="blockdev" +READLINK="readlink" +READLINK_E="-e" +MKDIR="mkdir" + +TEMPDIR="${TMPDIR:-/tmp}/${TOOL}_${RANDOM}$$" +DM_DEV_DIR="${DM_DEV_DIR:-/dev}" + +DRY=0 +VERB="" +FORCE="" +YES="" + +# default name for converted VG and its VDO LV +NAME="vdovg/vdolvol" + +# help message +tool_usage() { + echo "${TOOL}: Utility to convert VDO volume to VDO LV." + echo + echo " ${TOOL} [options] " + echo + echo " Options:" + echo " -f | --force Bypass sanity checks" + echo " -h | --help Show this help message" + echo " -n | --name Specifies VG/LV name for converted VDO volume" + echo " -v | --verbose Be verbose" + echo " -y | --yes Answer \"yes\" at any prompts" + echo " --dry-run Print commands without running them" + + exit +} + +verbose() { + test -z "$VERB" || echo "$TOOL:" "$@" +} + +# Support multi-line error messages +error() { + for i in "$@" ; do + echo "$TOOL: $i" >&2 + done + cleanup 1 +} + +dry() { + if [ "$DRY" -ne 0 ]; then + verbose "Dry execution" "$@" + return 0 + fi + verbose "Executing" "$@" + "$@" +} + +cleanup() { + trap '' 2 + + rm -rf "$TEMPDIR" + # error exit status for break + exit "${1:-1}" +} + +get_enabled_value_() { + case "$1" in + enabled) echo "1" ;; + *) echo "0" ;; + esac +} + +get_kb_size_with_unit_() { + case "$1" in + *[kK]) echo $(( ${1%[kK]} )) ;; + *[mM]) echo $(( ${1%[mM]} * 1024 )) ;; + *[gG]) echo $(( ${1%[gG]} * 1024 * 1024 )) ;; + *[tT]) echo $(( ${1%[tT]} * 1024 * 1024 * 1024 )) ;; + *[pP]) echo $(( ${1%[pP]} * 1024 * 1024 * 1024 * 1024 )) ;; + esac +} + +get_mb_size_with_unit_() { + case "$1" in + *[mM]) echo $(( ${1%[mM]} )) ;; + *[gG]) echo $(( ${1%[gG]} * 1024 )) ;; + *[tT]) echo $(( ${1%[tT]} * 1024 * 1024 )) ;; + *[pP]) echo $(( ${1%[pP]} * 1024 * 1024 * 1024 )) ;; + esac +} + +# Figure out largest possible extent size usable for VG +# $1 physical size +# $2 logical size +get_largest_extent_size_() { + local max=4 + local i + local d + + for i in 8 16 32 64 128 256 512 1024 2048 4096 ; do + d=$(( $1 / i )) + test $(( d * i )) -eq "$1" || break + d=$(( $2 / i )) + test $(( d * i )) -eq "$2" || break + max=$i + done + echo "$max" +} + +# detect LV on the given device +# dereference device name if it is symbolic link +detect_lv_() { + local DEVICE=$1 + local MAJOR + local MINOR + local SYSVOLUME + local MAJORMINOR + + DEVICE=${1/#"${DM_DEV_DIR}/"/} + DEVICE=$("$READLINK" $READLINK_E "$DM_DEV_DIR/$DEVICE") + test -n "$DEVICE" || error "Cannot get readlink \"$1\"." + RDEVICE=$DEVICE + case "$RDEVICE" in + # hardcoded /dev since udev does not create these entries elsewhere + /dev/dm-[0-9]*) + read -r <"/sys/block/${RDEVICE#/dev/}/dm/name" SYSVOLUME 2>&1 && DEVICE="$DM_DEV_DIR/mapper/$SYSVOLUME" + read -r <"/sys/block/${RDEVICE#/dev/}/dev" MAJORMINOR 2>&1 || error "Cannot get major:minor for \"$DEVICE\"." + MAJOR=${MAJORMINOR%%:*} + MINOR=${MAJORMINOR##*:} + ;; + *) + STAT=$(stat --format "MAJOR=\$((0x%t)) MINOR=\$((0x%T))" "$RDEVICE") + test -n "$STAT" || error "Cannot get major:minor for \"$DEVICE\"." + eval "$STAT" + ;; + esac + + eval "$(dmsetup info -c -j "$MAJOR" -m "$MINOR" -o uuid,name --noheadings --nameprefixes --separator ' ')" +} + +# parse yaml config files into 'prefix_yaml_part_names=("value")' strings +parse_yaml_() { + local yaml_file=$1 + local prefix=$2 + local s + local w + local fs + + s='[[:space:]]*' + w='[a-zA-Z0-9_.-]*' + fs="$(echo @|tr @ '\034')" + + ( + sed -ne '/^--/s|--||g; s|\"|\\\"|g; s/[[:space:]]*$//g;' \ + -e 's/\$/\\\$/g' \ + -e "/#.*[\"\']/!s| #.*||g; /^#/s|#.*||g;" \ + -e "s|^\($s\)\($w\)$s:$s\"\(.*\)\"$s\$|\1$fs\2$fs\3|p" \ + -e "s|^\($s\)\($w\)${s}[:-]$s\(.*\)$s\$|\1$fs\2$fs\3|p" | + + awk -F"$fs" '{ + indent = length($1)/2; + if (length($2) == 0) { conj[indent]="+";} else {conj[indent]="";} + vname[indent] = $2; + for (i in vname) {if (i > indent) {delete vname[i]}} + if (length($3) > 0) { + vn=""; for (i=0; i"$TEMPDIR/vdoconf.yml" + + VDONAME=$(awk -v DNAME="$DEVICE" '/.*VDOService$/ {VNAME=substr($1, 0, length($1) - 1)} /[[:space:]]*device:/ { if ($2 ~ DNAME) {print VNAME}}' "$TEMPDIR/vdoconf.yml") + TRVDONAME=$(echo "$VDONAME" | tr '-' '_') + + # When VDO volume is 'active', check it's not mounted/being used + eval "$(dmsetup info -c -o open "$VDONAME" --noheadings --nameprefixes || true)" + test "${DM_OPEN:-0}" -eq 0 || error "Cannot converted VDO volume \"$VDONAME\" which is in use!" + + #parse_yaml_ "$TEMPDIR/vdoconf.yml" _ + eval "$(parse_yaml_ "$TEMPDIR/vdoconf.yml" _ | grep "$TRVDONAME" | sed -e "s/_config_vdos_$TRVDONAME/vdo/g")" + + vdo_logicalSize=$(get_kb_size_with_unit_ "$vdo_logicalSize") + vdo_physicalSize=$(get_kb_size_with_unit_ "$vdo_physicalSize") + + verbose "Going to convert physical sized VDO device $vdo_physicalSize KiB." + verbose "With logical volume of size $vdo_logicalSize KiB." + + PARAMS=$(cat <