diff --git a/SOURCES/0069-vdo-ensure-VDO-config-is-removed.patch b/SOURCES/0069-vdo-ensure-VDO-config-is-removed.patch new file mode 100644 index 0000000..7a7e9b3 --- /dev/null +++ b/SOURCES/0069-vdo-ensure-VDO-config-is-removed.patch @@ -0,0 +1,35 @@ +From c41468a8acff9fd71bf774c821ad7c92623889da Mon Sep 17 00:00:00 2001 +From: Andrew Walsh +Date: Mon, 15 Nov 2021 10:49:06 -0500 +Subject: [PATCH 069/115] vdo: ensure VDO config is removed + +Make sure to remove the VDO config after conversion +of LVM-backed VDO. + +Addresses point 3 in rhbz#1987024#c5 + +(cherry picked from commit 522561e64b5fe73cb9d01c2ee2d4b7624b0ddff4) +--- + scripts/lvm_import_vdo.sh | 6 ++++++ + 1 file changed, 6 insertions(+) + +diff --git a/scripts/lvm_import_vdo.sh b/scripts/lvm_import_vdo.sh +index beb55dbdb..cc09187e9 100755 +--- a/scripts/lvm_import_vdo.sh ++++ b/scripts/lvm_import_vdo.sh +@@ -405,6 +405,12 @@ EOF + verbose "Converting to VDO pool." + dry "$LVM" lvconvert $YES $VERB $FORCE --config "$PARAMS" -Zn -V "${vdo_logicalSize}k" -n "$LVNAME" --type vdo-pool "$VGNAME/${LVNAME}_vpool" + ++ # Note: that this is spelled OPPOSITE the other $IS_LV checks. ++ if [ "$IS_LV" = "1" ]; then ++ verbose "Removing now-unused VDO entry from VDO config." ++ dry "$VDO" remove $VDOCONF --force --verbose --name "$VDONAME" ++ fi ++ + rm -fr "$TEMPDIR" + } + +-- +2.41.0 + diff --git a/SOURCES/0070-vdo-fix-conversion-of-vdo_slab_size_mb.patch b/SOURCES/0070-vdo-fix-conversion-of-vdo_slab_size_mb.patch new file mode 100644 index 0000000..ade23f5 --- /dev/null +++ b/SOURCES/0070-vdo-fix-conversion-of-vdo_slab_size_mb.patch @@ -0,0 +1,50 @@ +From a147a14813d576a11ed2f9ff08090fad874e418a Mon Sep 17 00:00:00 2001 +From: Zdenek Kabelac +Date: Tue, 31 May 2022 22:48:38 +0200 +Subject: [PATCH 070/115] vdo: fix conversion of vdo_slab_size_mb + +When converting VDO volume, the parameter vdo_slabSize was +incorrectly copied as vdo_blockMapCacheSize, however this parameter +is then no longer used for any table line creation so the wrong +value was only stored in metadata. + +Also use just single get_kb_size_with_unit_ and remove it's duplicate +functionality with get_mb_size_with_unit_. + +Use $VERB for vdo remove call. + +(cherry picked from commit 1b070f366ba57a6eb24df03241284732db5047e9) +--- + WHATS_NEW | 3 ++- + scripts/lvm_import_vdo.sh | 2 +- + 2 files changed, 3 insertions(+), 2 deletions(-) + +diff --git a/WHATS_NEW b/WHATS_NEW +index bffd24648..705fbde74 100644 +--- a/WHATS_NEW ++++ b/WHATS_NEW +@@ -1,6 +1,7 @@ + Version 2.03.17 - + =============================== +- Fix lvconvert --test --type vdo-pool execution. ++ Fix lvconvert --test --type vdo-pool execution. ++ Fix vdo_slab_size_mb value for converted VDO volume. + + Version 2.03.16 - + ==================================== +diff --git a/scripts/lvm_import_vdo.sh b/scripts/lvm_import_vdo.sh +index cc09187e9..c10b3b050 100755 +--- a/scripts/lvm_import_vdo.sh ++++ b/scripts/lvm_import_vdo.sh +@@ -408,7 +408,7 @@ EOF + # Note: that this is spelled OPPOSITE the other $IS_LV checks. + if [ "$IS_LV" = "1" ]; then + verbose "Removing now-unused VDO entry from VDO config." +- dry "$VDO" remove $VDOCONF --force --verbose --name "$VDONAME" ++ dry "$VDO" remove $VDOCONF $VERB --force --name "$VDONAME" + fi + + rm -fr "$TEMPDIR" +-- +2.41.0 + diff --git a/SOURCES/0071-tests-add-lvm_vdo_wrapper.patch b/SOURCES/0071-tests-add-lvm_vdo_wrapper.patch new file mode 100644 index 0000000..9ec24fb --- /dev/null +++ b/SOURCES/0071-tests-add-lvm_vdo_wrapper.patch @@ -0,0 +1,396 @@ +From 2456b39b345a589949528bd439052a776d274d63 Mon Sep 17 00:00:00 2001 +From: Zdenek Kabelac +Date: Tue, 31 May 2022 22:45:29 +0200 +Subject: [PATCH 071/115] tests: add lvm_vdo_wrapper + +Introduce a replacement vdo manager wrapper for testing. +When using test suite on a system without vdo manager (which has got +deprecated) - we still need its functionality to prepare 'vdo volume' +for testing lvm_import_vdo. + +Wrapper currently need 2 binaries from older 'vdo 6.2' package - +to be named: +oldvdoformat - format VDO metadata with older format +oldvdoprepareforlvm - shift vdo metadata by 1MiB + +(cherry picked from commit 2ecfd503edadaf5d46115826c629754bf9fd573f) +--- + test/Makefile.in | 1 + + test/lib/lvm_vdo_wrapper.sh | 353 ++++++++++++++++++++++++++++++++++++ + 2 files changed, 354 insertions(+) + create mode 100755 test/lib/lvm_vdo_wrapper.sh + +diff --git a/test/Makefile.in b/test/Makefile.in +index ecb9e4264..f69dc97b6 100644 +--- a/test/Makefile.in ++++ b/test/Makefile.in +@@ -369,6 +369,7 @@ LIB = $(addprefix lib/, $(LIB_SECURETEST) $(LIB_DMSECURETEST) $(LIB_SHARED) $(LI + $(Q) $(LN_S) -f $(abs_top_srcdir)/conf/thin-performance.profile lib/ + $(Q) $(LN_S) -f $(abs_top_srcdir)/scripts/fsadm.sh lib/fsadm + $(Q) $(LN_S) -f $(abs_top_srcdir)/scripts/lvm_import_vdo.sh lib/lvm_import_vdo ++ $(Q) which vdo || $(LN_S) -f $(abs_top_srcdir)/test/lib/lvm_vdo_wrapper.sh lib/vdo + @test "$(srcdir)" = . || \ + for i in $(LIB_LVMLOCKD_CONF) $(LIB_MKE2FS_CONF); do \ + test -n "$(Q)" || echo "$(LN_S) -f $(abs_top_srcdir)/test/lib/$$i lib/"; \ +diff --git a/test/lib/lvm_vdo_wrapper.sh b/test/lib/lvm_vdo_wrapper.sh +new file mode 100755 +index 000000000..d622d6456 +--- /dev/null ++++ b/test/lib/lvm_vdo_wrapper.sh +@@ -0,0 +1,353 @@ ++#!/bin/bash ++# ++# Wrapper script for 'naive' emulation of vdo manager tool for systems ++# that no longer have this tool present ++# ++ ++set -euE -o pipefail ++ ++# tool for formating 'old' VDO metadata format ++LVM_VDO_FORMAT=${LVM_VDO_FORMAT-"oldvdoformat"} ++# tool for shifting VDO metadata header by 2MiB ++LVM_VDO_PREPARE=${LVM_VDO_PREPARE-"oldvdoprepareforlvm"} ++# default vdo conf file ++LVM_VDO_DEFAULT_CONF=${LVM_VDO_DEFAULT_CONF-"/tmp/vdoconf.yml"} ++ ++vdo_die_() { ++ echo -e "$@" >&2 ++ return 1 ++} ++ ++vdo_verbose_() { ++ test -z "$vdo_verbose" || echo "$0:" "$@" ++} ++ ++vdo_dry_() { ++ if test -n "$vdo_dry"; then ++ vdo_verbose_ "Dry execution" "$@" ++ return 0 ++ fi ++ vdo_verbose_ "Executing" "$@" ++ "$@" ++} ++ ++vdo_get_kb_size_with_unit_() { ++ local sz=${2-1} # 2nd. arg as unit - default 'k' ++ ++ case "$sz" in ++ [mM]) sz=1024 ;; ++ esac ++ ++ case "$1" in ++ *[mM]) sz=1024 ;; ++ *[gG]) sz=$(( 1024 * 1024 )) ;; ++ *[tT]) sz=$(( 1024 * 1024 * 1024 )) ;; ++ *[pP]) sz=$(( 1024 * 1024 * 1024 * 1024 )) ;; ++ esac ++ ++ echo $(( sz * ${1%[kKmMgGtTpP]} )) ++} ++ ++# ++# Emulate functionality of deprecated 'vdo create' ++# ++vdo_create_() { ++local cachesize= ++local devsize= ++local emulate512=disabled ++local logicalsize= ++local maxdiscardsize= ++local slabbits=0 # 4k ++local slabsize= ++local sparse= ++local table= ++local vdo_compression_msg= ++local vdo_dry= ++local vdo_index_msg= ++local vdo_logicalBlockSize= ++local vdo_verbose= ++ ++local vdo_ackThreads=${vdo_ackThreads-1} ++local vdo_bioRotationInterval=${vdo_bioRotationInterval-64} ++local vdo_bioThreads=${vdo_bioThreads-4} ++local vdo_blockMapCacheSize=${vdo_blockMapCacheSize-128M} ++local vdo_blockMapPeriod=${vdo_blockMapPeriod-16380} ++local vdo_compression=${vdo_compression-enabled} ++local vdo_confFile=$LVM_VDO_DEFAULT_CONF # place some file in /tmp ++local vdo_cpuThreads=${vdo_cpuThreads-2} ++local vdo_deduplication=${vdo_deduplication-enabled} ++local vdo_hashZoneThreads=${vdo_hashZoneThreads-1} ++local vdo_indexCfreq=${vdo_indexCfreq-0} ++local vdo_indexMemory=${vdo_indexMemory-0.25} ++local vdo_indexSparse=${vdo_indexSparse-disabled} ++local vdo_indexThreads=${vdo_indexThreads-0} ++local vdo_logicalSize=${vdo_logicalSize-0} ++local vdo_logicalThreads=${vdo_logicalThreads-1} ++local vdo_maxDiscardSize=${vdo_maxDiscardSize-4K} ++local vdo_name=${vdo_name-VDONAME} ++local vdo_physicalThreads=${vdo_physicalThreads-1} ++local vdo_slabSize=${vdo_slabSize-2G} ++local vdo_uuid="VDO-$(uuidgen || echo \"f7a3ecdc-40a0-4e43-814c-4a7039a75de4\")" ++local vdo_writePolicy=${vdo_writePolicy-auto} ++ ++while [ "$#" -ne 0 ] ++do ++ case "$1" in ++ "--blockMapCacheSize") shift; vdo_blockMapCacheSize=$1 ;; ++ "--blockMapPeriod") shift; vdo_blockMapPeriod=$1 ;; ++ "--compression") shift; vdo_compression=$1 ;; ++ "--confFile"|"-f") shift; vdo_confFile=$1 ;; ++ "--deduplication") shift; vdo_deduplication=$1 ;; ++ "--device") shift; vdo_device=$1 ;; ++ "--emulate512") shift; emulate512=$1 ;; ++ "--indexMem") shift; vdo_indexMemory=$1 ;; ++ "--maxDiscardSize") shift; vdo_maxDiscardSize=$1 ;; ++ "--name"|"-n") shift; vdo_name=$1 ;; ++ "--sparseIndex") shift; vdo_indexSparse=$1 ;; ++ "--uuid") shift ;; # ignored ++ "--vdoAckThreads") shift; vdo_ackThreads=$1 ;; ++ "--vdoBioRotationInterval") shift; vdo_bioRotationInterval=$1 ;; ++ "--vdoBioThreads") shift; vdo_bioThreads=$1 ;; ++ "--vdoCpuThreads") shift; vdo_cpuThreads=$1 ;; ++ "--vdoHashZoneThreads") shift; vdo_hashZoneThreads=$1 ;; ++ "--vdoLogicalSize") shift; vdo_logicalSize=$1 ;; ++ "--vdoLogicalThreads") shift; vdo_logicalThreads=$1 ;; ++ "--vdoLogLevel") shift ;; # ignored ++ "--vdoPhysicalThreads") shift; vdo_physicalSize=$1 ;; ++ "--vdoSlabSize") shift; vdo_slabSize=$1 ;; ++ "--verbose"|"-d"|"--debug") vdo_verbose="-v" ;; ++ "--writePolicy") shift; vdo_writePolicy=$1 ;; ++ esac ++ shift ++done ++ ++# Convert when set ++case "$emulate512" in ++ "enabled") vdo_logicalBlockSize=512 ;; ++ "disabled") vdo_logicalBlockSize=4096 ;; ++ *) vdo_die_ "Invalid emulate512 setting." ++esac ++ ++case "$vdo_deduplication" in ++ "enabled") vdo_index_msg="index-enable" ;; ++ "disabled") vdo_index_msg="index-disable";; ++ *) vdo_die_ "Invalid deduplication setting." ++esac ++ ++case "$vdo_compression" in ++ "enabled") vdo_compression_msg="compression on" ;; ++ "disabled") vdo_compression_msg="compression off";; ++ *) vdo_die_ "Invalid compression setting." ++esac ++ ++test -n "${vdo_device-}" || vdo_die_ "VDO device is missing" ++ ++blkid -s UUID -o value "${vdo_device}" || true ++ ++devsize=$(blockdev --getsize64 "$vdo_device") ++devsize=$(( devsize / 4096 )) # convert to 4KiB units ++ ++logicalsize=$(vdo_get_kb_size_with_unit_ "$vdo_logicalSize" M) ++logicalsize=$(( logicalsize * 2 )) # 512B units ++ ++cachesize=$(vdo_get_kb_size_with_unit_ "$vdo_blockMapCacheSize" M) ++cachesize=$(( cachesize / 4 )) # 4KiB units ++ ++maxdiscardsize=$(vdo_get_kb_size_with_unit_ "$vdo_maxDiscardSize" M) ++maxdiscardsize=$(( maxdiscardsize / 4 )) # 4KiB units ++ ++test -e "$vdo_confFile" || { ++ cat > "$vdo_confFile" <> "$vdo_confFile" </dev/null || return 0 ++vdo_dry_ dmsetup remove $vdo_force "$vdo_name" || true ++} ++ ++# ++# vdo remove ++# ++vdo_remove_() { ++local vdo_confFile=$LVM_VDO_DEFAULT_CONF ++local vdo_name= ++ ++vdo_stop_ "$@" ++while [ "$#" -ne 0 ] ++do ++ case "$1" in ++ "--confFile"|"-f") shift; vdo_confFile=$1 ;; ++ "--name"|"-n") shift; vdo_name=$1 ;; ++ esac ++ shift ++done ++ ++# remove entry from conf file ++awk -v vdovolname="$vdo_name" 'BEGIN { have=0 } ++ $0 ~ "!VDOService" { have=0 } ++ $0 ~ vdovolname":" { have=1 } ++ { if (have==0) { print } ;} ++ ' "$vdo_confFile" >"${vdo_confFile}.new" ++ ++mv "${vdo_confFile}.new" "$vdo_confFile" ++grep "!VDOService" "$vdo_confFile" || rm -f "$vdo_confFile" ++} ++ ++ ++# ++# print_config_file ++# ++vdo_print_config_file_() { ++local vdo_confFile=$LVM_VDO_DEFAULT_CONF ++ ++while [ "$#" -ne 0 ] ++do ++ case "$1" in ++ "--confFile"|"-f") shift; vdo_confFile=$1 ;; ++ "--verbose"|"-d"|"--debug") ;; ++ "--logfile") shift ;; # ignore ++ esac ++ shift ++done ++ ++cat "$vdo_confFile" ++} ++ ++# ++# vdo convert ++# ++vdo_convert_() { ++local vdo_confFile=$LVM_VDO_DEFAULT_CONF ++local vdo_dry= ++local vdo_force= ++local vdo_name= ++local vdo_verbose= ++local vdo_device= ++ ++while [ "$#" -ne 0 ] ++do ++ case "$1" in ++ "--confFile"|"-f") shift; vdo_confFile=$1 ;; ++ "--name"|"-n") shift; vdo_name=$1 ;; ++ "--verbose"|"-d"|"--debug") vdo_verbose="-v" ;; ++ "--force") vdo_force="--force" ;; ++ esac ++ shift ++done ++ ++vdo_device=$(awk -v vdovolname="$vdo_name" 'BEGIN { have=0 } ++ $0 ~ "!VDOService" { have=0 } ++ $0 ~ vdovolname":" { have=1 } ++ { if (have==1 && $0 ~ "device:" ) { print $2 } ;}'\ ++ "$vdo_confFile") ++ ++#dmsetup status --target vdo "$vdo_name" || true ++vdo_dry_ "$LVM_VDO_PREPARE" "$vdo_device" ++vdo_dry_ vdo_remove_ -f "$vdo_confFile" -n "$vdo_name" || true ++} ++ ++# ++# MAIN ++# ++case "$1" in ++ "create") shift; vdo_create_ "$@" ;; ++ "remove") shift; vdo_remove_ "$@" ;; ++ "stop") shift; vdo_stop_ "$@" ;; ++ "convert") shift; vdo_convert_ "$@" ;; ++ "printConfigFile") shift; vdo_print_config_file_ "$@" ;; ++esac +-- +2.41.0 + diff --git a/SOURCES/0072-tests-update-for-wrapper.patch b/SOURCES/0072-tests-update-for-wrapper.patch new file mode 100644 index 0000000..21631ee --- /dev/null +++ b/SOURCES/0072-tests-update-for-wrapper.patch @@ -0,0 +1,84 @@ +From 64a78a2904c0481de8c1f9894cedcbd5f6525287 Mon Sep 17 00:00:00 2001 +From: Zdenek Kabelac +Date: Tue, 7 Jun 2022 16:52:30 +0200 +Subject: [PATCH 072/115] tests: update for wrapper + +Update calling vdo manager since our vdo wrapper has a simple shell +arg parser so it needs args without '=' + +Also correct using DM_DEV_DIR for 'pvcreate' + +(cherry picked from commit 4a498512077b4fe5cf6b806a91228bd23e513123) +--- + test/shell/vdo-convert.sh | 13 +++++++------ + 1 file changed, 7 insertions(+), 6 deletions(-) + +diff --git a/test/shell/vdo-convert.sh b/test/shell/vdo-convert.sh +index 2a2026c2e..8b03344a7 100644 +--- a/test/shell/vdo-convert.sh ++++ b/test/shell/vdo-convert.sh +@@ -54,7 +54,7 @@ vgcreate $vg "$dev1" + + lvcreate -L5G -n $lv1 $vg + +-vdo create $VDOCONF --name "$VDONAME" --device="$DM_DEV_DIR/$vg/$lv1" --vdoLogicalSize=10G ++vdo create $VDOCONF --name "$VDONAME" --device "$DM_DEV_DIR/$vg/$lv1" --vdoSlabSize 128M --vdoLogicalSize 10G + + mkfs -E nodiscard "$DM_DEV_DIR/mapper/$VDONAME" + +@@ -75,7 +75,7 @@ lvremove -f $vg + # Test user can specify different VDO LV name (so the original LV is renamed) + lvcreate -y -L5G -n $lv1 $vg + +-vdo create $VDOCONF --name "$VDONAME" --device="$DM_DEV_DIR/$vg/$lv1" --vdoLogicalSize=10G ++vdo create $VDOCONF --name "$VDONAME" --device "$DM_DEV_DIR/$vg/$lv1" --vdoSlabSize 128M --vdoLogicalSize 10G + + lvm_import_vdo -y --name $vg/$lv2 "$DM_DEV_DIR/$vg/$lv1" + +@@ -95,7 +95,7 @@ vgcreate $vg2 "$dev2" + # + # Check conversion of VDO volume on non-LV device and with >2T size + # +-vdo create $VDOCONF --name "$VDONAME" --device="$dev1" --vdoLogicalSize=3T ++vdo create $VDOCONF --name "$VDONAME" --device "$dev1" --vdoSlabSize 128M --vdoLogicalSize 3T + + # Fail with an already existing volume group $vg2 + not lvm_import_vdo --dry-run -y -v --name $vg2/$lv1 "$dev1" |& tee err +@@ -117,7 +117,7 @@ vgremove -f $vg + aux teardown_devs + aux prepare_devs 1 23456 + +-vdo create $VDOCONF --name "$VDONAME" --device="$dev1" --vdoLogicalSize=23G ++vdo create $VDOCONF --name "$VDONAME" --device "$dev1" --vdoSlabSize 128M --vdoLogicalSize 23G + + mkfs -E nodiscard "$DM_DEV_DIR/mapper/$VDONAME" + +@@ -137,6 +137,7 @@ aux prepare_loop 60000 || skip + + test -f LOOP + LOOP=$(< LOOP) ++LOOP="${DM_DEV_DIR}/${LOOP##/dev/}" + + aux extend_filter "a|$LOOP|" + aux extend_devices "$LOOP" +@@ -155,7 +156,7 @@ aux extend_devices "$LOOP" + # + # automate... + # +-vdo create $VDOCONF --name "$VDONAME" --device="$LOOP" --vdoLogicalSize=23G \ ++vdo create $VDOCONF --name "$VDONAME" --device "$LOOP" --vdoSlabSize 128M --vdoLogicalSize 23G\ + --blockMapCacheSize 192 \ + --blockMapPeriod 2048 \ + --emulate512 disabled \ +@@ -173,7 +174,7 @@ vdo create $VDOCONF --name "$VDONAME" --device="$LOOP" --vdoLogicalSize=23G \ + # Get VDO table line + dmsetup table "$VDONAME" | tr " " "\n" | sed -e '5,6d' -e '12d' | tee vdo-orig + +-DM_DEV_DIR="" lvm_import_vdo -y --name $vg/$lv "$LOOP" ++lvm_import_vdo -y --name $vg/$lv "$LOOP" + lvs -a $vg + + dmsetup table "$vg-${lv}_vpool-vpool" | tr " " "\n" | sed -e '5,6d' -e '12d' | tee new-vdo-lv +-- +2.41.0 + diff --git a/SOURCES/0073-lvmdbusd-Change-unit-test-vdo-minimum-size.patch b/SOURCES/0073-lvmdbusd-Change-unit-test-vdo-minimum-size.patch new file mode 100644 index 0000000..a75ce58 --- /dev/null +++ b/SOURCES/0073-lvmdbusd-Change-unit-test-vdo-minimum-size.patch @@ -0,0 +1,73 @@ +From 8702e97022753c9f21c85af6deecea76b179911d Mon Sep 17 00:00:00 2001 +From: Tony Asleson +Date: Wed, 25 May 2022 16:03:27 -0500 +Subject: [PATCH 073/115] lvmdbusd: Change unit test vdo minimum size + +(cherry picked from commit 47c61907b4adbdead50f5bb5ac95c0f5d0fe263e) +--- + test/dbus/lvmdbustest.py | 14 +++++++++----- + 1 file changed, 9 insertions(+), 5 deletions(-) + +diff --git a/test/dbus/lvmdbustest.py b/test/dbus/lvmdbustest.py +index 6d692223f..3eef77fd7 100755 +--- a/test/dbus/lvmdbustest.py ++++ b/test/dbus/lvmdbustest.py +@@ -23,6 +23,9 @@ import os + + g_tmo = 0 + ++# Approx. min size ++VDO_MIN_SIZE = mib(8192) ++ + # Prefix on created objects to enable easier clean-up + g_prefix = os.getenv('PREFIX', '') + +@@ -1155,7 +1158,7 @@ class TestDbusService(unittest.TestCase): + return + + # This may not pass +- for i in [48, 64, 128]: ++ for i in [64, 128]: + yes = self._test_expired_timer(i) + if yes: + break +@@ -1907,8 +1910,8 @@ class TestDbusService(unittest.TestCase): + vdo_pool_object_path = self.handle_return( + vg_proxy.VgVdo.CreateVdoPoolandLv( + pool_name, lv_name, +- dbus.UInt64(mib(4096)), # Appears to be minimum size +- dbus.UInt64(mib(8192)), ++ dbus.UInt64(VDO_MIN_SIZE), ++ dbus.UInt64(VDO_MIN_SIZE * 2), + dbus.Int32(g_tmo), + EOD)) + +@@ -1950,7 +1953,7 @@ class TestDbusService(unittest.TestCase): + vg_proxy = self._vg_create(vg_prefix="vdo_conv_") + lv = self._test_lv_create( + vg_proxy.Vg.LvCreate, +- (dbus.String(pool_name), dbus.UInt64(mib(4096)), ++ (dbus.String(pool_name), dbus.UInt64(VDO_MIN_SIZE), + dbus.Array([], signature='(ott)'), dbus.Int32(g_tmo), + EOD), vg_proxy.Vg, LV_BASE_INT) + lv_obj_path = self._lookup("%s/%s" % (vg_proxy.Vg.Name, pool_name)) +@@ -1959,7 +1962,7 @@ class TestDbusService(unittest.TestCase): + vdo_pool_path = self.handle_return( + vg_proxy.VgVdo.CreateVdoPool( + dbus.ObjectPath(lv.object_path), lv_name, +- dbus.UInt64(mib(8192)), ++ dbus.UInt64(VDO_MIN_SIZE), + dbus.Int32(g_tmo), + EOD)) + +@@ -2083,6 +2086,7 @@ if __name__ == '__main__': + std_err_print('\n*** Testing only lvm shell mode ***\n') + + for g_tmo in [0, 15]: ++ std_err_print('Testing TMO=%d\n' % g_tmo) + if mode == 0: + if set_execution(False, r): + r.register_result(unittest.main(exit=False)) +-- +2.41.0 + diff --git a/SOURCES/0074-vdo-fix-conversion-of-vdo_slab_size_mb-2nd.patch b/SOURCES/0074-vdo-fix-conversion-of-vdo_slab_size_mb-2nd.patch new file mode 100644 index 0000000..812d17a --- /dev/null +++ b/SOURCES/0074-vdo-fix-conversion-of-vdo_slab_size_mb-2nd.patch @@ -0,0 +1,31 @@ +From e82e23447f830bd6c38bbefa600212e84cdb0f27 Mon Sep 17 00:00:00 2001 +From: Zdenek Kabelac +Date: Mon, 11 Jul 2022 01:07:24 +0200 +Subject: [PATCH 074/115] vdo: fix conversion of vdo_slab_size_mb 2nd + +Patch 1b070f366ba57a6eb24df03241284732db5047e9 should have +been already fixing this issue but since it the incorrect +patch rebasing the change to vdo_slabSize got lost. +So again now with explicit one-line patch. + +(cherry picked from commit d2667bc25bccaf0f70cc2ded0fd3f25a79cb4f6c) +--- + scripts/lvm_import_vdo.sh | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/scripts/lvm_import_vdo.sh b/scripts/lvm_import_vdo.sh +index c10b3b050..c4c1d152e 100755 +--- a/scripts/lvm_import_vdo.sh ++++ b/scripts/lvm_import_vdo.sh +@@ -324,7 +324,7 @@ allocation { + vdo_check_point_frequency = $vdo_indexCfreq + vdo_use_sparse_index = $(get_enabled_value_ "$vdo_indexSparse") + vdo_index_memory_size_mb = $(awk "BEGIN {print $vdo_indexMemory * 1024}") +- vdo_slab_size_mb = $(( $(get_kb_size_with_unit_ "$vdo_blockMapCacheSize") / 1024 )) ++ vdo_slab_size_mb = $(( $(get_kb_size_with_unit_ "$vdo_slabSize") / 1024 )) + vdo_ack_threads = $vdo_ackThreads + vdo_bio_threads = $vdo_bioThreads + vdo_bio_rotation = $vdo_bioRotationInterval +-- +2.41.0 + diff --git a/SOURCES/0075-vdo-update-info-about-memory.patch b/SOURCES/0075-vdo-update-info-about-memory.patch new file mode 100644 index 0000000..84d0325 --- /dev/null +++ b/SOURCES/0075-vdo-update-info-about-memory.patch @@ -0,0 +1,49 @@ +From 1a39e0c2c24c7c505753658f9a74bf962fdf013b Mon Sep 17 00:00:00 2001 +From: Zdenek Kabelac +Date: Mon, 4 Jul 2022 15:00:26 +0200 +Subject: [PATCH 075/115] vdo: update info about memory + +Add more info about kernel target memory allocation associated with +VDO pool usage. + +(cherry picked from commit f445624c339a5c1436a47d2b51046869f183eb03) +--- + man/lvmvdo.7_main | 15 ++++++++++++--- + 1 file changed, 12 insertions(+), 3 deletions(-) + +diff --git a/man/lvmvdo.7_main b/man/lvmvdo.7_main +index 14bd640b5..8c3e3eeaa 100644 +--- a/man/lvmvdo.7_main ++++ b/man/lvmvdo.7_main +@@ -12,7 +12,7 @@ for primary storage. + .P + Deduplication is a technique for reducing the consumption of storage + resources by eliminating multiple copies of duplicate blocks. Compression +-takes the individual unique blocks and shrinks them. ++takes the individual unique blocks and shrinks them. + These reduced blocks are then efficiently packed together into + physical blocks. Thin provisioning manages the mapping from logical blocks + presented by VDO to where the data has actually been physically stored, +@@ -358,8 +358,17 @@ take otherwise as device is already expected to be empty. + . + .SS \n+[step]. Memory usage + . +-The VDO target requires 370 MiB of RAM plus an additional 268 MiB +-per each 1 TiB of physical storage managed by the volume. ++The VDO target requires 38 MiB of RAM and several variable amounts: ++.IP \(bu 2 ++1.15 MiB of RAM for each 1 MiB of configured block map cache size. ++The block map cache requires a minimum of 150 MiB RAM. ++.br ++.IP \(bu ++1.6 MiB of RAM for each 1 TiB of logical space. ++.br ++.IP \(bu ++268 MiB of RAM for each 1 TiB of physical storage managed by the volume. ++.br + .P + UDS requires a minimum of 250 MiB of RAM, + which is also the default amount that deduplication uses. +-- +2.41.0 + diff --git a/SOURCES/0076-vdo-check-vdo-memory-constrains.patch b/SOURCES/0076-vdo-check-vdo-memory-constrains.patch new file mode 100644 index 0000000..430ef61 --- /dev/null +++ b/SOURCES/0076-vdo-check-vdo-memory-constrains.patch @@ -0,0 +1,246 @@ +From 783213e4da16a70f5e3d5efae2d409b92d899fc9 Mon Sep 17 00:00:00 2001 +From: Zdenek Kabelac +Date: Fri, 8 Jul 2022 23:33:29 +0200 +Subject: [PATCH 076/115] vdo: check vdo memory constrains + +Add function to check for avaialble memory for particular VDO +configuration - to avoid unnecessary machine swapping for configs +that will not fit into memory (possibly in locked section). + +Formula tries to estimate RAM size machine can use also with +swapping for kernel target - but still leaving some amount of +usable RAM. + +Estimation is based on documented RAM usage of VDO target. + +If the /proc/meminfo would be theoretically unavailable, try to use +'sysinfo()' function, however this is giving only free RAM without +the knowledge about how much RAM could be eventually swapped. + +TODO: move _get_memory_info() into generic lvm2 API function used +by other targets with non-trivial memory requirements. + +(cherry picked from commit ebad057579aeff0980a1b8af7eaacd56e62ed0c9) +--- + lib/metadata/metadata-exported.h | 2 + + lib/metadata/vdo_manip.c | 144 +++++++++++++++++++++++++++++++ + lib/vdo/vdo.c | 8 +- + tools/lvcreate.c | 4 + + 4 files changed, 156 insertions(+), 2 deletions(-) + +diff --git a/lib/metadata/metadata-exported.h b/lib/metadata/metadata-exported.h +index 7bac5b900..449c8d014 100644 +--- a/lib/metadata/metadata-exported.h ++++ b/lib/metadata/metadata-exported.h +@@ -1377,6 +1377,8 @@ int fill_vdo_target_params(struct cmd_context *cmd, + struct dm_vdo_target_params *vtp, + uint64_t *vdo_pool_header_size, + struct profile *profile); ++int check_vdo_constrains(struct cmd_context *cmd, uint64_t physical_size, ++ uint64_t virtual_size, struct dm_vdo_target_params *vtp); + /* -- metadata/vdo_manip.c */ + + struct logical_volume *find_pvmove_lv(struct volume_group *vg, +diff --git a/lib/metadata/vdo_manip.c b/lib/metadata/vdo_manip.c +index 11a119a68..9f449ef8b 100644 +--- a/lib/metadata/vdo_manip.c ++++ b/lib/metadata/vdo_manip.c +@@ -23,6 +23,8 @@ + #include "lib/config/defaults.h" + #include "lib/misc/lvm-exec.h" + ++#include // sysinfo ++ + const char *get_vdo_compression_state_name(enum dm_vdo_compression_state state) + { + switch (state) { +@@ -516,3 +518,145 @@ int fill_vdo_target_params(struct cmd_context *cmd, + + return 1; + } ++ ++static int _get_sysinfo_memory(uint64_t *total_mb, uint64_t *available_mb) ++{ ++ struct sysinfo si = { 0 }; ++ ++ *total_mb = *available_mb = UINT64_MAX; ++ ++ if (sysinfo(&si) != 0) ++ return 0; ++ ++ log_debug("Sysinfo free:%lu bufferram:%lu sharedram:%lu freehigh:%lu unit:%u.", ++ si.freeram >> 20, si.bufferram >> 20, si.sharedram >> 20, ++ si.freehigh >> 20, si.mem_unit); ++ ++ *available_mb = ((uint64_t)(si.freeram + si.bufferram) * si.mem_unit) >> 30; ++ *total_mb = si.totalram >> 30; ++ ++ return 1; ++} ++ ++typedef struct mem_table_s { ++ const char *name; ++ uint64_t *value; ++} mem_table_t; ++ ++static int _compare_mem_table_s(const void *a, const void *b){ ++ return strcmp(((const mem_table_t*)a)->name, ((const mem_table_t*)b)->name); ++} ++ ++static int _get_memory_info(uint64_t *total_mb, uint64_t *available_mb) ++{ ++ uint64_t anon_pages, mem_available, mem_free, mem_total, shmem, swap_free; ++ uint64_t can_swap; ++ mem_table_t mt[] = { ++ { "AnonPages", &anon_pages }, ++ { "MemAvailable", &mem_available }, ++ { "MemFree", &mem_free }, ++ { "MemTotal", &mem_total }, ++ { "Shmem", &shmem }, ++ { "SwapFree", &swap_free }, ++ }; ++ ++ char line[128], namebuf[32], *e, *tail; ++ FILE *fp; ++ mem_table_t findme = { namebuf, NULL }; ++ mem_table_t *found; ++ ++ if (!(fp = fopen("/proc/meminfo", "r"))) ++ return _get_sysinfo_memory(total_mb, available_mb); ++ ++ while (fgets(line, sizeof(line), fp)) { ++ if (!(e = strchr(line, ':'))) ++ break; ++ ++ if ((++e - line) > sizeof(namebuf)) ++ continue; // something too long ++ ++ (void)dm_strncpy((char*)findme.name, line, e - line); ++ ++ found = bsearch(&findme, mt, DM_ARRAY_SIZE(mt), sizeof(mem_table_t), ++ _compare_mem_table_s); ++ if (!found) ++ continue; // not interesting ++ ++ *(found->value) = (uint64_t) strtoull(e, &tail, 10); ++ ++ if ((e == tail) || errno) ++ log_debug("Failing to parse value from %s.", line); ++ else ++ log_debug("Parsed %s = " FMTu64 " KiB.", found->name, *(found->value)); ++ } ++ (void)fclose(fp); ++ ++ // use at most 2/3 of swap space to keep machine usable ++ can_swap = (anon_pages + shmem) * 2 / 3; ++ swap_free = swap_free * 2 / 3; ++ ++ if (can_swap > swap_free) ++ can_swap = swap_free; ++ ++ // TODO: add more constrains, i.e. 3/4 of physical RAM... ++ ++ *total_mb = mem_total >> 10; ++ *available_mb = (mem_available + can_swap) >> 10; ++ ++ return 1; ++} ++ ++static uint64_t _round_1024(uint64_t s) ++{ ++ return (s + ((1 << 10) - 1)) >> 10; ++} ++ ++static uint64_t _round_sectors_to_tib(uint64_t s) ++{ ++ return (s + ((UINT64_C(1) << (40 - SECTOR_SHIFT)) - 1)) >> (40 - SECTOR_SHIFT); ++} ++ ++int check_vdo_constrains(struct cmd_context *cmd, uint64_t physical_size, ++ uint64_t virtual_size, struct dm_vdo_target_params *vtp) ++{ ++ uint64_t req_mb, total_mb, available_mb; ++ uint64_t phy_mb = _round_sectors_to_tib(UINT64_C(268) * physical_size); // 268 MiB per 1 TiB of physical size ++ uint64_t virt_mb = _round_1024(UINT64_C(1638) * _round_sectors_to_tib(virtual_size)); // 1.6 MiB per 1 TiB ++ uint64_t cache_mb = _round_1024(UINT64_C(1177) * vtp->block_map_cache_size_mb); // 1.15 MiB per 1 MiB cache size ++ char msg[512]; ++ ++ if (cache_mb < 150) ++ cache_mb = 150; // always at least 150 MiB for block map ++ ++ // total required memory for VDO target ++ req_mb = 38 + vtp->index_memory_size_mb + virt_mb + phy_mb + cache_mb; ++ ++ _get_memory_info(&total_mb, &available_mb); ++ ++ (void)snprintf(msg, sizeof(msg), "VDO configuration needs %s RAM for physical volume size %s, " ++ "%s RAM for virtual volume size %s, %s RAM for block map cache size %s and " ++ "%s RAM for index memory.", ++ display_size(cmd, phy_mb << (20 - SECTOR_SHIFT)), ++ display_size(cmd, physical_size), ++ display_size(cmd, virt_mb << (20 - SECTOR_SHIFT)), ++ display_size(cmd, virtual_size), ++ display_size(cmd, cache_mb << (20 - SECTOR_SHIFT)), ++ display_size(cmd, ((uint64_t)vtp->block_map_cache_size_mb) << (20 - SECTOR_SHIFT)), ++ display_size(cmd, ((uint64_t)vtp->index_memory_size_mb) << (20 - SECTOR_SHIFT))); ++ ++ if (req_mb > available_mb) { ++ log_error("Not enough free memory for VDO target. %s RAM is required, but only %s RAM is available.", ++ display_size(cmd, req_mb << (20 - SECTOR_SHIFT)), ++ display_size(cmd, available_mb << (20 - SECTOR_SHIFT))); ++ log_print_unless_silent("%s", msg); ++ return 0; ++ } ++ ++ log_debug("VDO requires %s RAM, currently available %s RAM.", ++ display_size(cmd, req_mb << (20 - SECTOR_SHIFT)), ++ display_size(cmd, available_mb << (20 - SECTOR_SHIFT))); ++ ++ log_verbose("%s", msg); ++ ++ return 1; ++} +diff --git a/lib/vdo/vdo.c b/lib/vdo/vdo.c +index 52e9443ea..b9bcc044f 100644 +--- a/lib/vdo/vdo.c ++++ b/lib/vdo/vdo.c +@@ -355,8 +355,8 @@ static int _vdo_pool_target_status_compatible(const char *type) + } + + static int _vdo_pool_add_target_line(struct dev_manager *dm, +- struct dm_pool *mem __attribute__((unused)), +- struct cmd_context *cmd __attribute__((unused)), ++ struct dm_pool *mem, ++ struct cmd_context *cmd, + void **target_state __attribute__((unused)), + struct lv_segment *seg, + const struct lv_activate_opts *laopts __attribute__((unused)), +@@ -369,6 +369,10 @@ static int _vdo_pool_add_target_line(struct dev_manager *dm, + log_error(INTERNAL_ERROR "Passed segment is not VDO pool."); + return 0; + } ++ ++ if (!check_vdo_constrains(cmd, seg->lv->size, seg_lv(seg, 0)->size, &seg->vdo_params)) ++ return_0; ++ + if (!(vdo_pool_name = dm_build_dm_name(mem, seg->lv->vg->name, seg->lv->name, lv_layer(seg->lv)))) + return_0; + +diff --git a/tools/lvcreate.c b/tools/lvcreate.c +index 8de6f3408..fb57d84bd 100644 +--- a/tools/lvcreate.c ++++ b/tools/lvcreate.c +@@ -1729,6 +1729,10 @@ static int _lvcreate_single(struct cmd_context *cmd, const char *vg_name, + if (!_update_extents_params(vg, lp, lcp)) + goto_out; + ++ if (seg_is_vdo(lp) && !check_vdo_constrains(cmd, (uint64_t)lp->extents * vg->extent_size, ++ lcp->virtual_size, &lp->vdo_params)) ++ return_0; ++ + if (seg_is_thin(lp) && !_validate_internal_thin_processing(lp)) + goto_out; + +-- +2.41.0 + diff --git a/SOURCES/0077-vdo-add-reformating-to-extent-size-aligned-virtual-s.patch b/SOURCES/0077-vdo-add-reformating-to-extent-size-aligned-virtual-s.patch new file mode 100644 index 0000000..95f738d --- /dev/null +++ b/SOURCES/0077-vdo-add-reformating-to-extent-size-aligned-virtual-s.patch @@ -0,0 +1,79 @@ +From d2a2720ccb88d99c76423ebd5c3bb1f13dc60ab1 Mon Sep 17 00:00:00 2001 +From: Zdenek Kabelac +Date: Sat, 9 Jul 2022 00:42:01 +0200 +Subject: [PATCH 077/115] vdo: add reformating to extent size aligned virtual + size + +Newer VDO kernel target require to have matching virtual size - this +however cause incompatiblity when lvcreate is let to format VDO data +device and read the usable size from vdoformat. +Altough this is a kernel regression and will likely get fixed, +lvm2 can actually reformat VDO device to use properly aligned VDO LV +size to make this problem disappear. + +(cherry picked from commit a477490e812639fed3be495f215fcf1a7b65b7ee) +--- + lib/metadata/vdo_manip.c | 22 +++++++++++++++++++--- + 1 file changed, 19 insertions(+), 3 deletions(-) + +diff --git a/lib/metadata/vdo_manip.c b/lib/metadata/vdo_manip.c +index 9f449ef8b..f3a4a9534 100644 +--- a/lib/metadata/vdo_manip.c ++++ b/lib/metadata/vdo_manip.c +@@ -227,10 +227,11 @@ static int _format_vdo_pool_data_lv(struct logical_volume *data_lv, + const struct dm_vdo_target_params *vtp, + uint64_t *logical_size) + { +- char *dpath; ++ char *dpath, *c; + const struct dm_config_node *cn; + const struct dm_config_value *cv; + struct pipe_data pdata; ++ uint64_t logical_size_aligned = 1; + FILE *f; + uint64_t lb; + unsigned slabbits; +@@ -247,7 +248,9 @@ static int _format_vdo_pool_data_lv(struct logical_volume *data_lv, + return 0; + } + ++reformat: + if (*logical_size) { ++ logical_size_aligned = 0; + if (dm_snprintf(buf_args[args], sizeof(buf_args[0]), "--logical-size=" FMTu64 "K", + (*logical_size / 2)) < 0) + return_0; +@@ -332,8 +335,8 @@ static int _format_vdo_pool_data_lv(struct logical_volume *data_lv, + log_verbose("Available VDO logical blocks " FMTu64 " (%s).", + lb, display_size(data_lv->vg->cmd, *logical_size)); + } +- if ((dpath = strchr(buf, '\n'))) +- *dpath = 0; /* cut last '\n' away */ ++ if ((c = strchr(buf, '\n'))) ++ *c = 0; /* cut last '\n' away */ + if (buf[0]) + log_print(" %s", buf); /* Print vdo_format messages */ + } +@@ -348,6 +351,19 @@ static int _format_vdo_pool_data_lv(struct logical_volume *data_lv, + return 0; + } + ++ if (logical_size_aligned) { ++ // align obtained size to extent size ++ logical_size_aligned = *logical_size / data_lv->vg->extent_size * data_lv->vg->extent_size; ++ if (*logical_size != logical_size_aligned) { ++ *logical_size = logical_size_aligned; ++ argv[1] = (char*) "--force"; ++ args = 2; ++ log_verbose("Reformating VDO to align virtual size %s by extent size.", ++ display_size(data_lv->vg->cmd, *logical_size)); ++ goto reformat; ++ } ++ } ++ + return 1; + } + +-- +2.41.0 + diff --git a/SOURCES/0078-vdo-support-v4-kernel-target-line.patch b/SOURCES/0078-vdo-support-v4-kernel-target-line.patch new file mode 100644 index 0000000..71543a5 --- /dev/null +++ b/SOURCES/0078-vdo-support-v4-kernel-target-line.patch @@ -0,0 +1,172 @@ +From c160f54ec5cc26b78db38058f1a8bd63da9e225d Mon Sep 17 00:00:00 2001 +From: Zdenek Kabelac +Date: Fri, 8 Jul 2022 23:35:06 +0200 +Subject: [PATCH 078/115] vdo: support v4 kernel target line + +Check and use new available table line v4, if kernel supports it. + +(cherry picked from commit 1c18ed3b4ab2f2d5a15995b8f0a18d7d1c4d60ca) +--- + device_mapper/all.h | 1 + + device_mapper/libdm-deptree.c | 37 +++++++++++++++++++++++------------ + lib/metadata/segtype.h | 1 + + lib/vdo/vdo.c | 10 ++++++++-- + test/shell/vdo-convert.sh | 2 ++ + 5 files changed, 37 insertions(+), 14 deletions(-) + +diff --git a/device_mapper/all.h b/device_mapper/all.h +index 17f78d989..e45f5923a 100644 +--- a/device_mapper/all.h ++++ b/device_mapper/all.h +@@ -1020,6 +1020,7 @@ int dm_tree_node_add_integrity_target(struct dm_tree_node *node, + */ + int dm_tree_node_add_vdo_target(struct dm_tree_node *node, + uint64_t size, ++ uint32_t vdo_version, + const char *vdo_pool_name, + const char *data_uuid, + uint64_t data_size, +diff --git a/device_mapper/libdm-deptree.c b/device_mapper/libdm-deptree.c +index e4bf4c814..2d382037c 100644 +--- a/device_mapper/libdm-deptree.c ++++ b/device_mapper/libdm-deptree.c +@@ -214,6 +214,7 @@ struct load_segment { + uint32_t device_id; /* Thin */ + + // VDO params ++ uint32_t vdo_version; /* VDO - version of target table line */ + struct dm_tree_node *vdo_data; /* VDO */ + struct dm_vdo_target_params vdo_params; /* VDO */ + const char *vdo_name; /* VDO - device name is ALSO passed as table arg */ +@@ -2865,18 +2866,28 @@ static int _vdo_emit_segment_line(struct dm_task *dmt, + return 0; + } + +- EMIT_PARAMS(pos, "V2 %s " FMTu64 " %u " FMTu64 " %u %s %s %s " +- "maxDiscard %u ack %u bio %u bioRotationInterval %u cpu %u hash %u logical %u physical %u", +- data_dev, +- seg->vdo_data_size / 8, // this parameter is in 4K units +- seg->vdo_params.minimum_io_size * UINT32_C(512), // sector to byte units +- seg->vdo_params.block_map_cache_size_mb * UINT64_C(256), // 1MiB -> 4KiB units +- seg->vdo_params.block_map_era_length, +- seg->vdo_params.use_metadata_hints ? "on" : "off" , +- (seg->vdo_params.write_policy == DM_VDO_WRITE_POLICY_SYNC) ? "sync" : +- (seg->vdo_params.write_policy == DM_VDO_WRITE_POLICY_ASYNC) ? "async" : +- (seg->vdo_params.write_policy == DM_VDO_WRITE_POLICY_ASYNC_UNSAFE) ? "async-unsafe" : "auto", // policy +- seg->vdo_name, ++ if (seg->vdo_version < 4) { ++ EMIT_PARAMS(pos, "V2 %s " FMTu64 " %u " FMTu64 " %u %s %s %s ", ++ data_dev, ++ seg->vdo_data_size / 8, // this parameter is in 4K units ++ seg->vdo_params.minimum_io_size * UINT32_C(512), // sector to byte units ++ seg->vdo_params.block_map_cache_size_mb * UINT64_C(256), // 1MiB -> 4KiB units ++ seg->vdo_params.block_map_era_length, ++ seg->vdo_params.use_metadata_hints ? "on" : "off" , ++ (seg->vdo_params.write_policy == DM_VDO_WRITE_POLICY_SYNC) ? "sync" : ++ (seg->vdo_params.write_policy == DM_VDO_WRITE_POLICY_ASYNC) ? "async" : ++ (seg->vdo_params.write_policy == DM_VDO_WRITE_POLICY_ASYNC_UNSAFE) ? "async-unsafe" : "auto", // policy ++ seg->vdo_name); ++ } else { ++ EMIT_PARAMS(pos, "V4 %s " FMTu64 " %u " FMTu64 " %u ", ++ data_dev, ++ seg->vdo_data_size / 8, // this parameter is in 4K units ++ seg->vdo_params.minimum_io_size * UINT32_C(512), // sector to byte units ++ seg->vdo_params.block_map_cache_size_mb * UINT64_C(256), // 1MiB -> 4KiB units ++ seg->vdo_params.block_map_era_length); ++ } ++ ++ EMIT_PARAMS(pos, "maxDiscard %u ack %u bio %u bioRotationInterval %u cpu %u hash %u logical %u physical %u", + seg->vdo_params.max_discard, + seg->vdo_params.ack_threads, + seg->vdo_params.bio_threads, +@@ -4323,6 +4334,7 @@ void dm_tree_node_set_callback(struct dm_tree_node *dnode, + + int dm_tree_node_add_vdo_target(struct dm_tree_node *node, + uint64_t size, ++ uint32_t vdo_version, + const char *vdo_pool_name, + const char *data_uuid, + uint64_t data_size, +@@ -4344,6 +4356,7 @@ int dm_tree_node_add_vdo_target(struct dm_tree_node *node, + if (!_link_tree_nodes(node, seg->vdo_data)) + return_0; + ++ seg->vdo_version = vdo_version; + seg->vdo_params = *vtp; + seg->vdo_name = vdo_pool_name; + seg->vdo_data_size = data_size; +diff --git a/lib/metadata/segtype.h b/lib/metadata/segtype.h +index 2f4949267..3e52f04a1 100644 +--- a/lib/metadata/segtype.h ++++ b/lib/metadata/segtype.h +@@ -353,6 +353,7 @@ int init_vdo_segtypes(struct cmd_context *cmd, struct segtype_library *seglib); + #endif + + #define VDO_FEATURE_ONLINE_RENAME (1U << 0) /* version 6.2.3 */ ++#define VDO_FEATURE_VERSION4 (1U << 1) /* version 8.2.0 */ + + int init_writecache_segtypes(struct cmd_context *cmd, struct segtype_library *seglib); + +diff --git a/lib/vdo/vdo.c b/lib/vdo/vdo.c +index b9bcc044f..9efb424f0 100644 +--- a/lib/vdo/vdo.c ++++ b/lib/vdo/vdo.c +@@ -1,5 +1,5 @@ + /* +- * Copyright (C) 2018-2019 Red Hat, Inc. All rights reserved. ++ * Copyright (C) 2018-2022 Red Hat, Inc. All rights reserved. + * + * This file is part of LVM2. + * +@@ -364,6 +364,10 @@ static int _vdo_pool_add_target_line(struct dev_manager *dm, + uint32_t *pvmove_mirror_count __attribute__((unused))) + { + char *vdo_pool_name, *data_uuid; ++ unsigned attrs = 0; ++ ++ if (seg->segtype->ops->target_present) ++ seg->segtype->ops->target_present(cmd, NULL, &attrs); + + if (!seg_is_vdo_pool(seg)) { + log_error(INTERNAL_ERROR "Passed segment is not VDO pool."); +@@ -381,6 +385,7 @@ static int _vdo_pool_add_target_line(struct dev_manager *dm, + + /* VDO uses virtual size instead of its physical size */ + if (!dm_tree_node_add_vdo_target(node, get_vdo_pool_virtual_size(seg), ++ !(attrs & VDO_FEATURE_VERSION4) ? 2 : 4, + vdo_pool_name, data_uuid, seg_lv(seg, 0)->size, + &seg->vdo_params)) + return_0; +@@ -390,7 +395,7 @@ static int _vdo_pool_add_target_line(struct dev_manager *dm, + + static int _vdo_target_present(struct cmd_context *cmd, + const struct lv_segment *seg __attribute__((unused)), +- unsigned *attributes __attribute__((unused))) ++ unsigned *attributes) + { + /* List of features with their kernel target version */ + static const struct feature { +@@ -401,6 +406,7 @@ static int _vdo_target_present(struct cmd_context *cmd, + const char *feature; + } _features[] = { + { 6, 2, 3, VDO_FEATURE_ONLINE_RENAME, "online_rename" }, ++ { 8, 2, 0, VDO_FEATURE_VERSION4, "version4" }, + }; + static const char _lvmconf[] = "global/vdo_disabled_features"; + static int _vdo_checked = 0; +diff --git a/test/shell/vdo-convert.sh b/test/shell/vdo-convert.sh +index 8b03344a7..f1d04d596 100644 +--- a/test/shell/vdo-convert.sh ++++ b/test/shell/vdo-convert.sh +@@ -174,6 +174,8 @@ vdo create $VDOCONF --name "$VDONAME" --device "$LOOP" --vdoSlabSize 128M --vdoL + # Get VDO table line + dmsetup table "$VDONAME" | tr " " "\n" | sed -e '5,6d' -e '12d' | tee vdo-orig + ++aux lvmconf 'global/vdo_disabled_features = [ "version4" ]' ++ + lvm_import_vdo -y --name $vg/$lv "$LOOP" + lvs -a $vg + +-- +2.41.0 + diff --git a/SOURCES/0079-vdo-enhance-lvcreate-validation.patch b/SOURCES/0079-vdo-enhance-lvcreate-validation.patch new file mode 100644 index 0000000..9d8ae9f --- /dev/null +++ b/SOURCES/0079-vdo-enhance-lvcreate-validation.patch @@ -0,0 +1,171 @@ +From aa75359c7c1baae43349416f3761754507cd5c1a Mon Sep 17 00:00:00 2001 +From: Zdenek Kabelac +Date: Fri, 8 Jul 2022 23:38:34 +0200 +Subject: [PATCH 079/115] vdo: enhance lvcreate validation + +When creating VDO pool based of % values, lvm2 is now more clever +and avoids to create 'unsupportable' sizes of physical backend +volumes as 16TiB is maximum size supported by VDO target +(and also limited by maximum supportable slabs (8192) based on slab +size. + +If the requested virtual size is approaching max supported size 4PiB, +switch header size to 0. + +(cherry picked from commit e2e31d9acf1b96ab741c22dc0a2fefd672996d3a) +--- + lib/metadata/metadata-exported.h | 2 ++ + lib/metadata/vdo_manip.c | 14 ++++++++++ + tools/lvcreate.c | 48 ++++++++++++++++++++++++++------ + 3 files changed, 56 insertions(+), 8 deletions(-) + +diff --git a/lib/metadata/metadata-exported.h b/lib/metadata/metadata-exported.h +index 449c8d014..f735baa55 100644 +--- a/lib/metadata/metadata-exported.h ++++ b/lib/metadata/metadata-exported.h +@@ -1364,6 +1364,8 @@ const char *get_vdo_operating_mode_name(enum dm_vdo_operating_mode mode); + const char *get_vdo_write_policy_name(enum dm_vdo_write_policy policy); + uint64_t get_vdo_pool_virtual_size(const struct lv_segment *vdo_pool_seg); + int update_vdo_pool_virtual_size(struct lv_segment *vdo_pool_seg); ++uint32_t get_vdo_pool_max_extents(const struct dm_vdo_target_params *vtp, ++ uint32_t extent_size); + int parse_vdo_pool_status(struct dm_pool *mem, const struct logical_volume *vdo_pool_lv, + const char *params, const struct dm_info *dminfo, + struct lv_status_vdo *status); +diff --git a/lib/metadata/vdo_manip.c b/lib/metadata/vdo_manip.c +index f3a4a9534..4ccde40b1 100644 +--- a/lib/metadata/vdo_manip.c ++++ b/lib/metadata/vdo_manip.c +@@ -127,6 +127,20 @@ int update_vdo_pool_virtual_size(struct lv_segment *vdo_pool_seg) + return 1; + } + ++uint32_t get_vdo_pool_max_extents(const struct dm_vdo_target_params *vtp, ++ uint32_t extent_size) ++{ ++ uint64_t max_extents = (DM_VDO_PHYSICAL_SIZE_MAXIMUM + extent_size - 1) / extent_size; ++ uint64_t max_slab_extents = ((extent_size - 1 + DM_VDO_SLABS_MAXIMUM * ++ ((uint64_t)vtp->slab_size_mb << (20 - SECTOR_SHIFT))) / ++ extent_size); ++ ++ max_extents = (max_slab_extents < max_extents) ? max_slab_extents : max_extents; ++ ++ return (max_extents > UINT32_MAX) ? UINT32_MAX : (uint32_t)max_extents; ++} ++ ++ + static int _sysfs_get_kvdo_value(const char *dm_name, const struct dm_info *dminfo, + const char *vdo_param, uint64_t *value) + { +diff --git a/tools/lvcreate.c b/tools/lvcreate.c +index fb57d84bd..3eee5de90 100644 +--- a/tools/lvcreate.c ++++ b/tools/lvcreate.c +@@ -253,6 +253,7 @@ static int _update_extents_params(struct volume_group *vg, + uint32_t stripesize_extents; + uint32_t extents; + uint32_t base_calc_extents; ++ uint32_t vdo_pool_max_extents; + + if (lcp->size && + !(lp->extents = extents_from_size(vg->cmd, lcp->size, +@@ -322,6 +323,23 @@ static int _update_extents_params(struct volume_group *vg, + return 0; + } + ++ if (seg_is_vdo(lp)) { ++ vdo_pool_max_extents = get_vdo_pool_max_extents(&lp->vdo_params, vg->extent_size); ++ if (extents > vdo_pool_max_extents) { ++ if (lcp->percent == PERCENT_NONE) { ++ log_error("Can't use %s size. Maximal supported VDO POOL volume size with slab size %s is %s.", ++ display_size(vg->cmd, (uint64_t)vg->extent_size * extents), ++ display_size(vg->cmd, (uint64_t)lp->vdo_params.slab_size_mb << (20 - SECTOR_SHIFT)), ++ display_size(vg->cmd, (uint64_t)vg->extent_size * vdo_pool_max_extents)); ++ return 0; ++ } ++ extents = vdo_pool_max_extents; ++ log_verbose("Using maximal supported VDO POOL volume size %s (with slab size %s).", ++ display_size(vg->cmd, (uint64_t)vg->extent_size * extents), ++ display_size(vg->cmd, (uint64_t)lp->vdo_params.slab_size_mb << (20 - SECTOR_SHIFT))); ++ } ++ } ++ + if (lcp->percent != PERCENT_NONE) { + /* FIXME Don't do the adjustment for parallel allocation with PERCENT_ORIGIN! */ + lp->approx_alloc = 1; +@@ -699,15 +717,23 @@ static int _read_cache_params(struct cmd_context *cmd, + } + + static int _read_vdo_params(struct cmd_context *cmd, +- struct lvcreate_params *lp) ++ struct lvcreate_params *lp, ++ struct lvcreate_cmdline_params *lcp) + { + if (!seg_is_vdo(lp)) + return 1; + + // prefiling settings here +- if (!fill_vdo_target_params(cmd, &lp->vdo_params, &lp->vdo_pool_header_size, NULL)) ++ if (!fill_vdo_target_params(cmd, &lp->vdo_params, &lp->vdo_pool_header_size, NULL)) + return_0; + ++ if ((lcp->virtual_size <= DM_VDO_LOGICAL_SIZE_MAXIMUM) && ++ ((lcp->virtual_size + lp->vdo_pool_header_size) > DM_VDO_LOGICAL_SIZE_MAXIMUM)) { ++ log_verbose("Dropping VDO pool header size to 0 to support maximal size %s.", ++ display_size(cmd, DM_VDO_LOGICAL_SIZE_MAXIMUM)); ++ lp->vdo_pool_header_size = 0; ++ } ++ + // override with optional vdo settings + if (!get_vdo_settings(cmd, &lp->vdo_params, NULL)) + return_0; +@@ -1203,7 +1229,7 @@ static int _lvcreate_params(struct cmd_context *cmd, + &lp->pool_metadata_size, &lp->pool_metadata_spare, + &lp->chunk_size, &lp->discards, &lp->zero_new_blocks)) || + !_read_cache_params(cmd, lp) || +- !_read_vdo_params(cmd, lp) || ++ !_read_vdo_params(cmd, lp, lcp) || + !_read_mirror_and_raid_params(cmd, lp)) + return_0; + +@@ -1589,13 +1615,19 @@ static int _check_pool_parameters(struct cmd_context *cmd, + } + + static int _check_vdo_parameters(struct volume_group *vg, struct lvcreate_params *lp, +- struct lvcreate_cmdline_params *lcp) ++ struct lvcreate_cmdline_params *lcp) + { +- if (seg_is_vdo(lp) && lp->snapshot) { ++ if (lp->snapshot) { + log_error("Please either create VDO or snapshot."); + return 0; + } + ++ if (lcp->virtual_size > DM_VDO_LOGICAL_SIZE_MAXIMUM) { ++ log_error("Maximal supported VDO virtual size is %s.", ++ display_size(vg->cmd, DM_VDO_LOGICAL_SIZE_MAXIMUM)); ++ return 0; ++ } ++ + return 1; + } + +@@ -1716,12 +1748,12 @@ static int _lvcreate_single(struct cmd_context *cmd, const char *vg_name, + if (seg_is_thin(lp) && !_check_thin_parameters(vg, lp, lcp)) + goto_out; + +- if (!_check_pool_parameters(cmd, vg, lp, lcp)) +- goto_out; +- + if (seg_is_vdo(lp) && !_check_vdo_parameters(vg, lp, lcp)) + return_0; + ++ if (!_check_pool_parameters(cmd, vg, lp, lcp)) ++ goto_out; ++ + /* All types are checked */ + if (!_check_zero_parameters(cmd, lp)) + return_0; +-- +2.41.0 + diff --git a/SOURCES/0080-vdo-suffle-code-for-better-error-path-handling.patch b/SOURCES/0080-vdo-suffle-code-for-better-error-path-handling.patch new file mode 100644 index 0000000..128b317 --- /dev/null +++ b/SOURCES/0080-vdo-suffle-code-for-better-error-path-handling.patch @@ -0,0 +1,58 @@ +From f5c194afc5d904e3a936dc167ac14b204c049969 Mon Sep 17 00:00:00 2001 +From: Zdenek Kabelac +Date: Sat, 9 Jul 2022 21:33:57 +0200 +Subject: [PATCH 080/115] vdo: suffle code for better error path handling + +For failing dm_ no need to report 2nd. error, +but we missed to report error with 'updated==NULL'. + +(cherry picked from commit 493acb9195cef185b38ae4e4ffb84b984e5cc08c) +--- + tools/toollib.c | 28 ++++++++++++++-------------- + 1 file changed, 14 insertions(+), 14 deletions(-) + +diff --git a/tools/toollib.c b/tools/toollib.c +index d9a1a92ec..71bf26d39 100644 +--- a/tools/toollib.c ++++ b/tools/toollib.c +@@ -1343,23 +1343,23 @@ int get_vdo_settings(struct cmd_context *cmd, + u |= VDO_CHANGE_ONLINE; + } + +- if (updated) { +- // validation of updated VDO option +- if (!dm_vdo_validate_target_params(vtp, 0 /* vdo_size */)) { +-err: +- if (is_lvchange) +- log_error("Cannot change VDO setting \"vdo_%s\" in existing VDO pool.", +- option); +- else +- log_error("Invalid argument for VDO setting \"vdo_%s\".", +- option); +- goto out; +- } ++ // validation of updated VDO option ++ if (!dm_vdo_validate_target_params(vtp, 0 /* vdo_size */)) ++ goto_out; + ++ if (updated) + *updated = u; +- } + +- r = 1; ++ r = 1; // success ++ goto out; ++err: ++ if (is_lvchange) ++ log_error("Cannot change VDO setting \"vdo_%s\" in existing VDO pool.", ++ option); ++ else ++ log_error("Invalid argument for VDO setting \"vdo_%s\".", ++ option); ++ + out: + if (result) + dm_config_destroy(result); +-- +2.41.0 + diff --git a/SOURCES/0081-vdo-use-only-verbose-log-level-for-reformating.patch b/SOURCES/0081-vdo-use-only-verbose-log-level-for-reformating.patch new file mode 100644 index 0000000..05f7ca2 --- /dev/null +++ b/SOURCES/0081-vdo-use-only-verbose-log-level-for-reformating.patch @@ -0,0 +1,55 @@ +From fee817aff4bbd4a8dad7ceae7da179997dc7d359 Mon Sep 17 00:00:00 2001 +From: Zdenek Kabelac +Date: Mon, 15 Aug 2022 13:08:59 +0200 +Subject: [PATCH 081/115] vdo: use only verbose log level for reformating + +When lvcreate is makeing VDO pool and user has not specified -V size, +ATM we actually run 'vdoformat' twice to get properly 'extent' aligned +size matching lvm2 properties - so the 2nd. run of vdoformat actually +can stay with 'log_verbose()' so the standard printed result +is not showing confusing info (which is now also correctly using +print_unless_silent) + +(cherry picked from commit fc5bc5985d03aef5846cb98882d17815fc00ca15) +--- + lib/metadata/vdo_manip.c | 10 ++++++++-- + 1 file changed, 8 insertions(+), 2 deletions(-) + +diff --git a/lib/metadata/vdo_manip.c b/lib/metadata/vdo_manip.c +index 4ccde40b1..637fd1e5d 100644 +--- a/lib/metadata/vdo_manip.c ++++ b/lib/metadata/vdo_manip.c +@@ -249,6 +249,7 @@ static int _format_vdo_pool_data_lv(struct logical_volume *data_lv, + FILE *f; + uint64_t lb; + unsigned slabbits; ++ unsigned reformating = 0; + int args = 1; + char buf_args[5][128]; + char buf[256]; /* buffer for short disk header (64B) */ +@@ -351,8 +352,12 @@ reformat: + } + if ((c = strchr(buf, '\n'))) + *c = 0; /* cut last '\n' away */ +- if (buf[0]) +- log_print(" %s", buf); /* Print vdo_format messages */ ++ if (buf[0]) { ++ if (reformating) ++ log_verbose(" %s", buf); /* Print vdo_format messages */ ++ else ++ log_print_unless_silent(" %s", buf); /* Print vdo_format messages */ ++ } + } + + if (!pipe_close(&pdata)) { +@@ -372,6 +377,7 @@ reformat: + *logical_size = logical_size_aligned; + argv[1] = (char*) "--force"; + args = 2; ++ reformating = 1; + log_verbose("Reformating VDO to align virtual size %s by extent size.", + display_size(data_lv->vg->cmd, *logical_size)); + goto reformat; +-- +2.41.0 + diff --git a/SOURCES/0082-vdo-reset-errno-before-strtoull.patch b/SOURCES/0082-vdo-reset-errno-before-strtoull.patch new file mode 100644 index 0000000..308e7fd --- /dev/null +++ b/SOURCES/0082-vdo-reset-errno-before-strtoull.patch @@ -0,0 +1,27 @@ +From a4f39c5bbff286cc1323f799d269e35bf8e615b6 Mon Sep 17 00:00:00 2001 +From: Zdenek Kabelac +Date: Thu, 18 Aug 2022 13:55:29 +0200 +Subject: [PATCH 082/115] vdo: reset errno before strtoull + +Missed errno reset in commit ebad057579aeff0980a1b8af7eaacd56e62ed0c9. + +(cherry picked from commit 309df239e3ee474f3a5337f8fa3c0a1f7b0a89e5) +--- + lib/metadata/vdo_manip.c | 1 + + 1 file changed, 1 insertion(+) + +diff --git a/lib/metadata/vdo_manip.c b/lib/metadata/vdo_manip.c +index 637fd1e5d..0db401537 100644 +--- a/lib/metadata/vdo_manip.c ++++ b/lib/metadata/vdo_manip.c +@@ -618,6 +618,7 @@ static int _get_memory_info(uint64_t *total_mb, uint64_t *available_mb) + if (!found) + continue; // not interesting + ++ errno = 0; + *(found->value) = (uint64_t) strtoull(e, &tail, 10); + + if ((e == tail) || errno) +-- +2.41.0 + diff --git a/SOURCES/0083-vdo-extend-volume-and-pool-without-flush.patch b/SOURCES/0083-vdo-extend-volume-and-pool-without-flush.patch new file mode 100644 index 0000000..08f4363 --- /dev/null +++ b/SOURCES/0083-vdo-extend-volume-and-pool-without-flush.patch @@ -0,0 +1,66 @@ +From c3d533a0ea20aa26868c55959365a52f3eaf34d1 Mon Sep 17 00:00:00 2001 +From: Zdenek Kabelac +Date: Fri, 19 Aug 2022 14:48:01 +0200 +Subject: [PATCH 083/115] vdo: extend volume and pool without flush + +When the volume size is extended, there is no need to flush +IO operations (nothing can be targeting new space yet). +VDO target is supported as target that can safely work with +this condition. + +Such support is also needed, when extending VDOPOOL size +while the pool is reaching its capacity - since this allows +to continue working without reaching 'out-of-space' condition +due to flushing of all in flight IO. + +(cherry picked from commit e26c21cb8d5841141dcfdfc77f46da1108a81255) +--- + WHATS_NEW | 1 + + lib/activate/activate.c | 6 +++++- + lib/activate/dev_manager.c | 2 ++ + 3 files changed, 8 insertions(+), 1 deletion(-) + +diff --git a/WHATS_NEW b/WHATS_NEW +index 705fbde74..48b1d7d86 100644 +--- a/WHATS_NEW ++++ b/WHATS_NEW +@@ -1,5 +1,6 @@ + Version 2.03.17 - + =============================== ++ Extend VDO and VDOPOOL without flushing and locking fs. + Fix lvconvert --test --type vdo-pool execution. + Fix vdo_slab_size_mb value for converted VDO volume. + +diff --git a/lib/activate/activate.c b/lib/activate/activate.c +index 727bd2386..76740bb2b 100644 +--- a/lib/activate/activate.c ++++ b/lib/activate/activate.c +@@ -2124,7 +2124,11 @@ static int _lv_suspend(struct cmd_context *cmd, const char *lvid_s, + * TODO: Relax this limiting condition further */ + if (!flush_required && + (lv_is_pvmove(lv) || pvmove_lv || +- (!lv_is_mirror(lv) && !lv_is_thin_pool(lv) && !lv_is_thin_volume(lv)))) { ++ (!lv_is_mirror(lv) && ++ !lv_is_thin_volume(lv) && ++ !lv_is_thin_pool(lv) && ++ !lv_is_vdo(lv) && ++ !lv_is_vdo_pool(lv)))) { + log_debug("Requiring flush for LV %s.", display_lvname(lv)); + flush_required = 1; + } +diff --git a/lib/activate/dev_manager.c b/lib/activate/dev_manager.c +index 284254d68..9058510e4 100644 +--- a/lib/activate/dev_manager.c ++++ b/lib/activate/dev_manager.c +@@ -3826,6 +3826,8 @@ static int _tree_action(struct dev_manager *dm, const struct logical_volume *lv, + * non 'thin pool/volume' and size increase */ + else if (!lv_is_thin_volume(lv) && + !lv_is_thin_pool(lv) && ++ !lv_is_vdo(lv) && ++ !lv_is_vdo_pool(lv) && + dm_tree_node_size_changed(root)) + dm->flush_required = 1; + +-- +2.41.0 + diff --git a/SOURCES/0084-tests-vdo-emulation-without-vdo-binary.patch b/SOURCES/0084-tests-vdo-emulation-without-vdo-binary.patch new file mode 100644 index 0000000..1f4036a --- /dev/null +++ b/SOURCES/0084-tests-vdo-emulation-without-vdo-binary.patch @@ -0,0 +1,52 @@ +From fc1c105c75b81b4cecf24e7924605abcceb91d67 Mon Sep 17 00:00:00 2001 +From: Zdenek Kabelac +Date: Tue, 6 Sep 2022 16:44:24 +0200 +Subject: [PATCH 084/115] tests: vdo emulation without vdo binary + +Avoid inserting 'vdo' binary into path - and use +alias and VDO_BINARY shell vars for emulation. + +(cherry picked from commit 15ad2b8e5585b89bc3f09a53567f17eb70f45714) +--- + test/lib/lvm_vdo_wrapper.sh | 2 +- + test/shell/vdo-convert.sh | 11 ++++++++++- + 2 files changed, 11 insertions(+), 2 deletions(-) + +diff --git a/test/lib/lvm_vdo_wrapper.sh b/test/lib/lvm_vdo_wrapper.sh +index d622d6456..90d0b2ce8 100755 +--- a/test/lib/lvm_vdo_wrapper.sh ++++ b/test/lib/lvm_vdo_wrapper.sh +@@ -344,7 +344,7 @@ vdo_dry_ vdo_remove_ -f "$vdo_confFile" -n "$vdo_name" || true + # + # MAIN + # +-case "$1" in ++case "${1-}" in + "create") shift; vdo_create_ "$@" ;; + "remove") shift; vdo_remove_ "$@" ;; + "stop") shift; vdo_stop_ "$@" ;; +diff --git a/test/shell/vdo-convert.sh b/test/shell/vdo-convert.sh +index f1d04d596..5bf53d081 100644 +--- a/test/shell/vdo-convert.sh ++++ b/test/shell/vdo-convert.sh +@@ -29,7 +29,16 @@ aux prepare_dmeventd + # + # Main + # +-which vdo || skip ++if not which vdo ; then ++ which lvm_vdo_wrapper || skip "Missing 'lvm_vdo_wrapper'." ++ which oldvdoformat || skip "Emulation of vdo manager 'oldvdoformat' missing." ++ which oldvdoprepareforlvm || skip "Emulation of vdo manager 'oldvdoprepareforlvm' missing." ++ # enable expansion of aliasis within script itself ++ shopt -s expand_aliases ++ alias vdo='lvm_vdo_wrapper' ++ export VDO_BINARY=lvm_vdo_wrapper ++ echo "Using 'lvm_vdo_wrapper' emulation of 'vdo' manager." ++fi + which mkfs.ext4 || skip + export MKE2FS_CONFIG="$TESTDIR/lib/mke2fs.conf" + +-- +2.41.0 + diff --git a/SOURCES/0085-device_mapper-add-parser-for-vdo-metadata.patch b/SOURCES/0085-device_mapper-add-parser-for-vdo-metadata.patch new file mode 100644 index 0000000..2642d5e --- /dev/null +++ b/SOURCES/0085-device_mapper-add-parser-for-vdo-metadata.patch @@ -0,0 +1,373 @@ +From fd16a667a985ef116f3deed1910b99aa0859f244 Mon Sep 17 00:00:00 2001 +From: Zdenek Kabelac +Date: Wed, 26 Oct 2022 14:33:09 +0200 +Subject: [PATCH 085/115] device_mapper: add parser for vdo metadata + +Add very simplistic parser of vdo metadata to be able to obtain +logical_blocks stored within vdo metadata - as lvm2 may +submit smaller value due to internal aligment rules. + +To avoid creation of mismatching table line - use this number +instead the one provided by lvm2. + +(cherry picked from commit 829ab017082eaad253ebd28ad7d7ae7f3936dbcb) +--- + device_mapper/Makefile | 3 +- + device_mapper/libdm-deptree.c | 15 ++ + device_mapper/vdo/target.h | 2 + + device_mapper/vdo/vdo_reader.c | 279 +++++++++++++++++++++++++++++++++ + 4 files changed, 298 insertions(+), 1 deletion(-) + create mode 100644 device_mapper/vdo/vdo_reader.c + +diff --git a/device_mapper/Makefile b/device_mapper/Makefile +index d3b791eb5..a322235cb 100644 +--- a/device_mapper/Makefile ++++ b/device_mapper/Makefile +@@ -1,4 +1,4 @@ +-# Copyright (C) 2018 Red Hat, Inc. All rights reserved. ++# Copyright (C) 2018 - 2022 Red Hat, Inc. All rights reserved. + # + # This file is part of the device-mapper userspace tools. + # +@@ -29,6 +29,7 @@ DEVICE_MAPPER_SOURCE=\ + device_mapper/regex/parse_rx.c \ + device_mapper/regex/ttree.c \ + device_mapper/vdo/status.c \ ++ device_mapper/vdo/vdo_reader.c \ + device_mapper/vdo/vdo_target.c + + DEVICE_MAPPER_TARGET = device_mapper/libdevice-mapper.a +diff --git a/device_mapper/libdm-deptree.c b/device_mapper/libdm-deptree.c +index 2d382037c..0445e1b4b 100644 +--- a/device_mapper/libdm-deptree.c ++++ b/device_mapper/libdm-deptree.c +@@ -2857,6 +2857,7 @@ static int _vdo_emit_segment_line(struct dm_task *dmt, + int pos = 0; + char data[DM_FORMAT_DEV_BUFSIZE]; + char data_dev[128]; // for /dev/dm-XXXX ++ uint64_t logical_blocks; + + if (!_build_dev_string(data, sizeof(data), seg->vdo_data)) + return_0; +@@ -2866,6 +2867,20 @@ static int _vdo_emit_segment_line(struct dm_task *dmt, + return 0; + } + ++ if (dm_vdo_parse_logical_size(data_dev, &logical_blocks)) { ++ logical_blocks *= 8; ++ if (seg->size != logical_blocks) { ++ if (seg->size > logical_blocks) { ++ log_error("Virtual size of VDO volume is smaller then expected (" FMTu64 " > " FMTu64 ").", ++ seg->size, logical_blocks); ++ return 1; ++ } ++ log_debug_activation("Increasing VDO virtual volume size from " FMTu64 " to " FMTu64 ".", ++ seg->size, logical_blocks); ++ seg->size = logical_blocks; ++ } ++ } ++ + if (seg->vdo_version < 4) { + EMIT_PARAMS(pos, "V2 %s " FMTu64 " %u " FMTu64 " %u %s %s %s ", + data_dev, +diff --git a/device_mapper/vdo/target.h b/device_mapper/vdo/target.h +index 60c5bff56..bd21bb5d7 100644 +--- a/device_mapper/vdo/target.h ++++ b/device_mapper/vdo/target.h +@@ -108,6 +108,8 @@ struct dm_vdo_target_params { + bool dm_vdo_validate_target_params(const struct dm_vdo_target_params *vtp, + uint64_t vdo_size); + ++bool dm_vdo_parse_logical_size(const char *vdo_path, uint64_t *logical_blocks); ++ + //---------------------------------------------------------------- + + #endif +diff --git a/device_mapper/vdo/vdo_reader.c b/device_mapper/vdo/vdo_reader.c +new file mode 100644 +index 000000000..b765af042 +--- /dev/null ++++ b/device_mapper/vdo/vdo_reader.c +@@ -0,0 +1,279 @@ ++/* ++ * Copyright (C) 2022 Red Hat, Inc. All rights reserved. ++ * ++ * This file is part of the device-mapper userspace tools. ++ * ++ * This copyrighted material is made available to anyone wishing to use, ++ * modify, copy, or redistribute it subject to the terms and conditions ++ * of the GNU Lesser General Public License v.2.1. ++ * ++ * You should have received a copy of the GNU Lesser General Public License ++ * along with this program; if not, write to the Free Software Foundation, ++ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA ++ */ ++ ++/* ++ * Based on VDO sources: https://github.com/dm-vdo/vdo ++ * ++ * Simplified parser of VDO superblock to obtain basic VDO parameteers ++ * ++ * TODO: maybe switch to some library in the future ++ */ ++ ++//#define _GNU_SOURCE 1 ++//#define _LARGEFILE64_SOURCE 1 ++ ++#include "device_mapper/misc/dmlib.h" ++ ++#include "target.h" ++ ++#include "lib/mm/xlate.h" ++//#include "linux/byteorder/big_endian.h" ++//#include "linux/byteorder/little_endian.h" ++//#define le32_to_cpu __le32_to_cpu ++//#define le64_to_cpu __le64_to_cpu ++ ++ ++#include ++#include ++#include /* For block ioctl definitions */ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++typedef unsigned char uuid_t[16]; ++ ++#define __packed __attribute__((packed)) ++ ++static const char _MAGIC_NUMBER[] = "dmvdo001"; ++#define MAGIC_NUMBER_SIZE (sizeof(_MAGIC_NUMBER) - 1) ++ ++struct vdo_version_number { ++ uint32_t major_version; ++ uint32_t minor_version; ++} __packed; ++ ++/* ++ * The registry of component ids for use in headers ++ */ ++enum { ++ SUPER_BLOCK = 0, ++ FIXED_LAYOUT = 1, ++ RECOVERY_JOURNAL = 2, ++ SLAB_DEPOT = 3, ++ BLOCK_MAP = 4, ++ GEOMETRY_BLOCK = 5, ++}; /* ComponentID */ ++ ++struct vdo_header { ++ uint32_t id; /* The component this is a header for */ ++ struct vdo_version_number version; /* The version of the data format */ ++ size_t size; /* The size of the data following this header */ ++} __packed; ++ ++struct vdo_geometry_block { ++ char magic_number[MAGIC_NUMBER_SIZE]; ++ struct vdo_header header; ++ uint32_t checksum; ++} __packed; ++ ++struct vdo_config { ++ uint64_t logical_blocks; /* number of logical blocks */ ++ uint64_t physical_blocks; /* number of physical blocks */ ++ uint64_t slab_size; /* number of blocks in a slab */ ++ uint64_t recovery_journal_size; /* number of recovery journal blocks */ ++ uint64_t slab_journal_blocks; /* number of slab journal blocks */ ++} __packed; ++ ++struct vdo_component_41_0 { ++ uint32_t state; ++ uint64_t complete_recoveries; ++ uint64_t read_only_recoveries; ++ struct vdo_config config; /* packed */ ++ uint64_t nonce; ++} __packed; ++ ++enum vdo_volume_region_id { ++ VDO_INDEX_REGION = 0, ++ VDO_DATA_REGION = 1, ++ VDO_VOLUME_REGION_COUNT, ++}; ++ ++struct vdo_volume_region { ++ /* The ID of the region */ ++ enum vdo_volume_region_id id; ++ /* ++ * The absolute starting offset on the device. The region continues ++ * until the next region begins. ++ */ ++ uint64_t start_block; ++} __packed; ++ ++struct vdo_index_config { ++ uint32_t mem; ++ uint32_t unused; ++ uint8_t sparse; ++} __packed; ++ ++struct vdo_volume_geometry { ++ uint32_t release_version; ++ uint64_t nonce; ++ uuid_t uuid; ++ uint64_t bio_offset; ++ struct vdo_volume_region regions[VDO_VOLUME_REGION_COUNT]; ++ struct vdo_index_config index_config; ++} __packed; ++ ++/* Decoding mostly only some used stucture members */ ++ ++static void _vdo_decode_version(struct vdo_version_number *v) ++{ ++ v->major_version = le32_to_cpu(v->major_version); ++ v->minor_version = le32_to_cpu(v->minor_version); ++} ++ ++static void _vdo_decode_header(struct vdo_header *h) ++{ ++ h->id = le32_to_cpu(h->id); ++ _vdo_decode_version(&h->version); ++ h->size = le64_to_cpu(h->size); ++} ++ ++static void _vdo_decode_geometry_region(struct vdo_volume_region *vr) ++{ ++ vr->id = le32_to_cpu(vr->id); ++ vr->start_block = le32_to_cpu(vr->start_block); ++} ++ ++static void _vdo_decode_volume_geometry(struct vdo_volume_geometry *vg) ++{ ++ vg->release_version = le64_to_cpu(vg->release_version); ++ vg->nonce = le64_to_cpu(vg->nonce); ++ _vdo_decode_geometry_region(&vg->regions[VDO_DATA_REGION]); ++} ++ ++static void _vdo_decode_config(struct vdo_config *vc) ++{ ++ vc->logical_blocks = le64_to_cpu(vc->logical_blocks); ++ vc->physical_blocks = le64_to_cpu(vc->physical_blocks); ++ vc->slab_size = le64_to_cpu(vc->slab_size); ++ vc->recovery_journal_size = le64_to_cpu(vc->recovery_journal_size); ++ vc->slab_journal_blocks = le64_to_cpu(vc->slab_journal_blocks); ++} ++ ++static void _vdo_decode_pvc(struct vdo_component_41_0 *pvc) ++{ ++ _vdo_decode_config(&pvc->config); ++ pvc->nonce = le64_to_cpu(pvc->nonce); ++} ++ ++bool dm_vdo_parse_logical_size(const char *vdo_path, uint64_t *logical_blocks) ++{ ++ char buffer[4096]; ++ int fh, n; ++ bool r = false; ++ off_t l; ++ struct stat st; ++ uint64_t size; ++ uint64_t regpos; ++ ++ struct vdo_header h; ++ struct vdo_version_number vn; ++ struct vdo_volume_geometry vg; ++ struct vdo_component_41_0 pvc; ++ ++ *logical_blocks = 0; ++ if ((fh = open(vdo_path, O_RDONLY)) == -1) { ++ log_sys_error("Failed to open VDO backend %s", vdo_path); ++ goto err; ++ } ++ ++ if (ioctl(fh, BLKGETSIZE64, &size) == -1) { ++ if (errno != ENOTTY) { ++ log_sys_error("ioctl", vdo_path); ++ goto err; ++ } ++ ++ /* lets retry for file sizes */ ++ if (fstat(fh, &st) < 0) { ++ log_sys_error("fstat", vdo_path); ++ goto err; ++ } ++ ++ size = st.st_size; ++ } ++ ++ if ((n = read(fh, buffer, sizeof(buffer))) < 0) { ++ log_sys_error("read", vdo_path); ++ goto err; ++ } ++ ++ if (strncmp(buffer, _MAGIC_NUMBER, MAGIC_NUMBER_SIZE)) { ++ log_sys_error("mismatch header", vdo_path); ++ goto err; ++ } ++ ++ memcpy(&h, buffer + MAGIC_NUMBER_SIZE, sizeof(h)); ++ _vdo_decode_header(&h); ++ ++ if (h.version.major_version != 5) { ++ log_error("Unsupported VDO version %u.%u.", h.version.major_version, h.version.minor_version); ++ goto err; ++ } ++ ++ memcpy(&vg, buffer + MAGIC_NUMBER_SIZE + sizeof(h), sizeof(vg)); ++ _vdo_decode_volume_geometry(&vg); ++ ++ regpos = vg.regions[VDO_DATA_REGION].start_block * 4096; ++ ++ if ((regpos + sizeof(buffer)) > size) { ++ log_error("File/Device is shorter and can't provide requested VDO volume region at " FMTu64 " > " FMTu64 ".", regpos, size); ++ goto err; ++ } ++ ++ if ((l = lseek(fh, regpos, SEEK_SET)) < 0) { ++ log_sys_error("lseek", vdo_path); ++ goto err; ++ } ++ ++ if ((n = read(fh, buffer, sizeof(buffer))) < 0) { ++ log_sys_error("read error", vdo_path); ++ goto err; ++ } ++ ++ ++ memcpy(&vn, buffer + sizeof(struct vdo_geometry_block), sizeof(vn)); ++ _vdo_decode_version(&vn); ++ ++ if (vn.major_version > 41) { ++ log_error("Unknown VDO component version %u.", vn.major_version); // should be 41! ++ goto err; ++ } ++ ++ memcpy(&pvc, buffer + sizeof(struct vdo_geometry_block) + sizeof(vn), sizeof(pvc)); ++ _vdo_decode_pvc(&pvc); ++ ++ if (pvc.nonce != vg.nonce) { ++ log_error("Mismatching VDO nonce " FMTu64 " != " FMTu64 ".", pvc.nonce, vg.nonce); ++ goto err; ++ } ++ ++#if 0 ++ log_debug("LogBlocks " FMTu64 ".", pvc.config.logical_blocks); ++ log_debug("PhyBlocks " FMTu64 ".", pvc.config.physical_blocks); ++ log_debug("SlabSize " FMTu64 ".", pvc.config.slab_size); ++ log_debug("RecJourSize " FMTu64 ".", pvc.config.recovery_journal_size); ++ log_debug("SlabJouSize " FMTu64 ".", pvc.config.slab_journal_blocks); ++#endif ++ ++ *logical_blocks = pvc.config.logical_blocks; ++ r = true; ++err: ++ (void) close(fh); ++ ++ return r; ++} +-- +2.41.0 + diff --git a/SOURCES/0086-dev_manager-accept-misalined-vdo-pools.patch b/SOURCES/0086-dev_manager-accept-misalined-vdo-pools.patch new file mode 100644 index 0000000..229cd80 --- /dev/null +++ b/SOURCES/0086-dev_manager-accept-misalined-vdo-pools.patch @@ -0,0 +1,78 @@ +From 13ce88a2b204e9b05206b624bbcca00446312ebb Mon Sep 17 00:00:00 2001 +From: Zdenek Kabelac +Date: Wed, 26 Oct 2022 14:38:29 +0200 +Subject: [PATCH 086/115] dev_manager: accept misalined vdo pools. + +Since lvm2 may create VDO pool virtual size aligned only on extent size +while VDO itself is just 4K aligned - we need to support such misalign. + +(cherry picked from commit 2e79b005c2013fb03d8a48a3cfd8e70a982dd65b) +--- + lib/activate/dev_manager.c | 8 +++++--- + lib/metadata/vdo_manip.c | 10 ++-------- + 2 files changed, 7 insertions(+), 11 deletions(-) + +diff --git a/lib/activate/dev_manager.c b/lib/activate/dev_manager.c +index 9058510e4..5b72bf772 100644 +--- a/lib/activate/dev_manager.c ++++ b/lib/activate/dev_manager.c +@@ -263,7 +263,7 @@ static int _info_run(const char *dlid, struct dm_info *dminfo, + int dmtask; + int with_flush; /* TODO: arg for _info_run */ + void *target = NULL; +- uint64_t target_start, target_length, start, length, length_crop = 0; ++ uint64_t target_start, target_length, start, extent_size, length, length_crop = 0; + char *target_name, *target_params; + const char *devname; + +@@ -292,8 +292,8 @@ static int _info_run(const char *dlid, struct dm_info *dminfo, + + /* Query status only for active device */ + if (seg_status && dminfo->exists) { +- start = length = seg_status->seg->lv->vg->extent_size; +- start *= seg_status->seg->le; ++ extent_size = length = seg_status->seg->lv->vg->extent_size; ++ start = extent_size * seg_status->seg->le; + length *= _seg_len(seg_status->seg); + + /* Uses max DM_THIN_MAX_METADATA_SIZE sectors for metadata device */ +@@ -314,6 +314,8 @@ static int _info_run(const char *dlid, struct dm_info *dminfo, + + if ((start == target_start) && + ((length == target_length) || ++ ((lv_is_vdo_pool(seg_status->seg->lv)) && /* should fit within extent size */ ++ (length < target_length) && ((length + extent_size) > target_length)) || + (length_crop && (length_crop == target_length)))) + break; /* Keep target_params when matching segment is found */ + +diff --git a/lib/metadata/vdo_manip.c b/lib/metadata/vdo_manip.c +index 0db401537..1cd2130a7 100644 +--- a/lib/metadata/vdo_manip.c ++++ b/lib/metadata/vdo_manip.c +@@ -263,7 +263,6 @@ static int _format_vdo_pool_data_lv(struct logical_volume *data_lv, + return 0; + } + +-reformat: + if (*logical_size) { + logical_size_aligned = 0; + if (dm_snprintf(buf_args[args], sizeof(buf_args[0]), "--logical-size=" FMTu64 "K", +@@ -374,13 +373,8 @@ reformat: + // align obtained size to extent size + logical_size_aligned = *logical_size / data_lv->vg->extent_size * data_lv->vg->extent_size; + if (*logical_size != logical_size_aligned) { +- *logical_size = logical_size_aligned; +- argv[1] = (char*) "--force"; +- args = 2; +- reformating = 1; +- log_verbose("Reformating VDO to align virtual size %s by extent size.", +- display_size(data_lv->vg->cmd, *logical_size)); +- goto reformat; ++ log_debug("Using bigger VDO virtual size unaligned on extent size by %s.", ++ display_size(data_lv->vg->cmd, *logical_size - logical_size_aligned)); + } + } + +-- +2.41.0 + diff --git a/SOURCES/0087-device_mapper-vdo-V4-avoid-messaging.patch b/SOURCES/0087-device_mapper-vdo-V4-avoid-messaging.patch new file mode 100644 index 0000000..772dba9 --- /dev/null +++ b/SOURCES/0087-device_mapper-vdo-V4-avoid-messaging.patch @@ -0,0 +1,49 @@ +From 9399376608f5a81164c7bf9c8f3d724991a152b9 Mon Sep 17 00:00:00 2001 +From: Zdenek Kabelac +Date: Thu, 27 Oct 2022 23:58:42 +0200 +Subject: [PATCH 087/115] device_mapper: vdo V4 avoid messaging + +With V4 format build table line with compression and +deduplication and skip sending any messages to set up +these parameters. + +(cherry picked from commit 36a923926c2c27c1a8a5ac262387d2a4d3e620f8) +--- + device_mapper/libdm-deptree.c | 10 +++++++--- + 1 file changed, 7 insertions(+), 3 deletions(-) + +diff --git a/device_mapper/libdm-deptree.c b/device_mapper/libdm-deptree.c +index 0445e1b4b..02a56c8e3 100644 +--- a/device_mapper/libdm-deptree.c ++++ b/device_mapper/libdm-deptree.c +@@ -2894,12 +2894,15 @@ static int _vdo_emit_segment_line(struct dm_task *dmt, + (seg->vdo_params.write_policy == DM_VDO_WRITE_POLICY_ASYNC_UNSAFE) ? "async-unsafe" : "auto", // policy + seg->vdo_name); + } else { +- EMIT_PARAMS(pos, "V4 %s " FMTu64 " %u " FMTu64 " %u ", ++ EMIT_PARAMS(pos, "V4 %s " FMTu64 " %u " FMTu64 " %u " ++ "deduplication %s compression %s ", + data_dev, + seg->vdo_data_size / 8, // this parameter is in 4K units + seg->vdo_params.minimum_io_size * UINT32_C(512), // sector to byte units + seg->vdo_params.block_map_cache_size_mb * UINT64_C(256), // 1MiB -> 4KiB units +- seg->vdo_params.block_map_era_length); ++ seg->vdo_params.block_map_era_length, ++ seg->vdo_params.use_deduplication ? "on" : "off", ++ seg->vdo_params.use_compression ? "on" : "off"); + } + + EMIT_PARAMS(pos, "maxDiscard %u ack %u bio %u bioRotationInterval %u cpu %u hash %u logical %u physical %u", +@@ -4376,7 +4379,8 @@ int dm_tree_node_add_vdo_target(struct dm_tree_node *node, + seg->vdo_name = vdo_pool_name; + seg->vdo_data_size = data_size; + +- node->props.send_messages = 2; ++ if (seg->vdo_version < 4) ++ node->props.send_messages = 2; + + return 1; + } +-- +2.41.0 + diff --git a/SOURCES/0088-vdo-replace-errors-with-debug.patch b/SOURCES/0088-vdo-replace-errors-with-debug.patch new file mode 100644 index 0000000..776704c --- /dev/null +++ b/SOURCES/0088-vdo-replace-errors-with-debug.patch @@ -0,0 +1,121 @@ +From f67de990a60777410e2029deb7322243716e9b5d Mon Sep 17 00:00:00 2001 +From: Zdenek Kabelac +Date: Fri, 4 Nov 2022 16:27:56 +0100 +Subject: [PATCH 088/115] vdo: replace errors with debug + +As we actully use reading of VDO metadata only as extra 'information' source, +and not error command - switch to 'log_debug()' severity with messages +out of parser code. + +(cherry picked from commit 218c7d44b5ac64b6c38dfd40885a22008b5cec81) +--- + device_mapper/vdo/vdo_reader.c | 32 ++++++++++++++++---------------- + 1 file changed, 16 insertions(+), 16 deletions(-) + +diff --git a/device_mapper/vdo/vdo_reader.c b/device_mapper/vdo/vdo_reader.c +index b765af042..4f91f8011 100644 +--- a/device_mapper/vdo/vdo_reader.c ++++ b/device_mapper/vdo/vdo_reader.c +@@ -188,19 +188,19 @@ bool dm_vdo_parse_logical_size(const char *vdo_path, uint64_t *logical_blocks) + + *logical_blocks = 0; + if ((fh = open(vdo_path, O_RDONLY)) == -1) { +- log_sys_error("Failed to open VDO backend %s", vdo_path); ++ log_sys_debug("Failed to open VDO backend %s.", vdo_path); + goto err; + } + + if (ioctl(fh, BLKGETSIZE64, &size) == -1) { + if (errno != ENOTTY) { +- log_sys_error("ioctl", vdo_path); ++ log_sys_debug("ioctl", vdo_path); + goto err; + } + + /* lets retry for file sizes */ + if (fstat(fh, &st) < 0) { +- log_sys_error("fstat", vdo_path); ++ log_sys_debug("fstat", vdo_path); + goto err; + } + +@@ -208,12 +208,12 @@ bool dm_vdo_parse_logical_size(const char *vdo_path, uint64_t *logical_blocks) + } + + if ((n = read(fh, buffer, sizeof(buffer))) < 0) { +- log_sys_error("read", vdo_path); ++ log_sys_debug("read", vdo_path); + goto err; + } + + if (strncmp(buffer, _MAGIC_NUMBER, MAGIC_NUMBER_SIZE)) { +- log_sys_error("mismatch header", vdo_path); ++ log_debug_activation("Found mismatching VDO magic header in %s.", vdo_path); + goto err; + } + +@@ -221,7 +221,7 @@ bool dm_vdo_parse_logical_size(const char *vdo_path, uint64_t *logical_blocks) + _vdo_decode_header(&h); + + if (h.version.major_version != 5) { +- log_error("Unsupported VDO version %u.%u.", h.version.major_version, h.version.minor_version); ++ log_debug_activation("Unsupported VDO version %u.%u.", h.version.major_version, h.version.minor_version); + goto err; + } + +@@ -231,17 +231,17 @@ bool dm_vdo_parse_logical_size(const char *vdo_path, uint64_t *logical_blocks) + regpos = vg.regions[VDO_DATA_REGION].start_block * 4096; + + if ((regpos + sizeof(buffer)) > size) { +- log_error("File/Device is shorter and can't provide requested VDO volume region at " FMTu64 " > " FMTu64 ".", regpos, size); ++ log_debug_activation("File/Device is shorter and can't provide requested VDO volume region at " FMTu64 " > " FMTu64 ".", regpos, size); + goto err; + } + + if ((l = lseek(fh, regpos, SEEK_SET)) < 0) { +- log_sys_error("lseek", vdo_path); ++ log_sys_debug("lseek", vdo_path); + goto err; + } + + if ((n = read(fh, buffer, sizeof(buffer))) < 0) { +- log_sys_error("read error", vdo_path); ++ log_sys_debug("read", vdo_path); + goto err; + } + +@@ -250,7 +250,7 @@ bool dm_vdo_parse_logical_size(const char *vdo_path, uint64_t *logical_blocks) + _vdo_decode_version(&vn); + + if (vn.major_version > 41) { +- log_error("Unknown VDO component version %u.", vn.major_version); // should be 41! ++ log_debug_activation("Unknown VDO component version %u.", vn.major_version); // should be 41! + goto err; + } + +@@ -258,16 +258,16 @@ bool dm_vdo_parse_logical_size(const char *vdo_path, uint64_t *logical_blocks) + _vdo_decode_pvc(&pvc); + + if (pvc.nonce != vg.nonce) { +- log_error("Mismatching VDO nonce " FMTu64 " != " FMTu64 ".", pvc.nonce, vg.nonce); ++ log_debug_activation("VDO metadata has mismatching VDO nonces " FMTu64 " != " FMTu64 ".", pvc.nonce, vg.nonce); + goto err; + } + + #if 0 +- log_debug("LogBlocks " FMTu64 ".", pvc.config.logical_blocks); +- log_debug("PhyBlocks " FMTu64 ".", pvc.config.physical_blocks); +- log_debug("SlabSize " FMTu64 ".", pvc.config.slab_size); +- log_debug("RecJourSize " FMTu64 ".", pvc.config.recovery_journal_size); +- log_debug("SlabJouSize " FMTu64 ".", pvc.config.slab_journal_blocks); ++ log_debug_activation("LogBlocks " FMTu64 ".", pvc.config.logical_blocks); ++ log_debug_activation("PhyBlocks " FMTu64 ".", pvc.config.physical_blocks); ++ log_debug_activation("SlabSize " FMTu64 ".", pvc.config.slab_size); ++ log_debug_activation("RecJourSize " FMTu64 ".", pvc.config.recovery_journal_size); ++ log_debug_activation("SlabJouSize " FMTu64 ".", pvc.config.slab_journal_blocks); + #endif + + *logical_blocks = pvc.config.logical_blocks; +-- +2.41.0 + diff --git a/SOURCES/0089-vdo-enhance-detection-of-virtual-size.patch b/SOURCES/0089-vdo-enhance-detection-of-virtual-size.patch new file mode 100644 index 0000000..fcae02a --- /dev/null +++ b/SOURCES/0089-vdo-enhance-detection-of-virtual-size.patch @@ -0,0 +1,98 @@ +From c66a3010e0b90a5eb2a17717ecc73f62859d1eea Mon Sep 17 00:00:00 2001 +From: Zdenek Kabelac +Date: Sat, 5 Nov 2022 23:10:36 +0100 +Subject: [PATCH 089/115] vdo: enhance detection of virtual size + +Improve detection of VDO virtual size - so it's not reading VDO +metadata when VDO device is already active and instead we reuse +existing table line for knowing existing metadata size. + +NOTE: if there is ever going to be added support for reduction +of VDO virtual size - this method will need to be reworked to +allow size difference only within 'extent_size' alignment. + +(cherry picked from commit 8e9410594b3113386a667df400b2c229c745cb6a) +--- + device_mapper/libdm-deptree.c | 44 ++++++++++++++++++++++++----------- + 1 file changed, 31 insertions(+), 13 deletions(-) + +diff --git a/device_mapper/libdm-deptree.c b/device_mapper/libdm-deptree.c +index 02a56c8e3..39af7b1d4 100644 +--- a/device_mapper/libdm-deptree.c ++++ b/device_mapper/libdm-deptree.c +@@ -2850,7 +2850,7 @@ static int _thin_pool_emit_segment_line(struct dm_task *dmt, + return 1; + } + +-static int _vdo_emit_segment_line(struct dm_task *dmt, ++static int _vdo_emit_segment_line(struct dm_task *dmt, uint32_t major, uint32_t minor, + struct load_segment *seg, + char *params, size_t paramsize) + { +@@ -2858,6 +2858,10 @@ static int _vdo_emit_segment_line(struct dm_task *dmt, + char data[DM_FORMAT_DEV_BUFSIZE]; + char data_dev[128]; // for /dev/dm-XXXX + uint64_t logical_blocks; ++ struct dm_task *vdo_dmt; ++ uint64_t start, length = 0; ++ char *type = NULL; ++ char *vdo_params = NULL; + + if (!_build_dev_string(data, sizeof(data), seg->vdo_data)) + return_0; +@@ -2867,18 +2871,32 @@ static int _vdo_emit_segment_line(struct dm_task *dmt, + return 0; + } + +- if (dm_vdo_parse_logical_size(data_dev, &logical_blocks)) { +- logical_blocks *= 8; +- if (seg->size != logical_blocks) { +- if (seg->size > logical_blocks) { +- log_error("Virtual size of VDO volume is smaller then expected (" FMTu64 " > " FMTu64 ").", +- seg->size, logical_blocks); +- return 1; +- } +- log_debug_activation("Increasing VDO virtual volume size from " FMTu64 " to " FMTu64 ".", +- seg->size, logical_blocks); +- seg->size = logical_blocks; ++ /* ++ * If there is already running VDO target, read 'existing' virtual size out of table line ++ * and avoid reading it them from VDO metadata device ++ * ++ * NOTE: ATM VDO virtual size can be ONLY extended thus it's simple to recongnize 'right' size. ++ * However if there would be supported also reduction, this check would need to check range. ++ */ ++ if ((vdo_dmt = dm_task_create(DM_DEVICE_TABLE))) { ++ if (dm_task_set_major(vdo_dmt, major) && ++ dm_task_set_minor(vdo_dmt, minor) && ++ dm_task_run(vdo_dmt)) { ++ (void) dm_get_next_target(vdo_dmt, NULL, &start, &length, &type, &vdo_params); ++ if (!type || strcmp(type, "vdo")) ++ length = 0; + } ++ ++ dm_task_destroy(vdo_dmt); ++ } ++ ++ if (!length && dm_vdo_parse_logical_size(data_dev, &logical_blocks)) ++ length = logical_blocks * 8; ++ ++ if (seg->size < length) { ++ log_debug_activation("Correcting VDO virtual volume size from " FMTu64 " to " FMTu64 ".", ++ seg->size, length); ++ seg->size = length; + } + + if (seg->vdo_version < 4) { +@@ -2980,7 +2998,7 @@ static int _emit_segment_line(struct dm_task *dmt, uint32_t major, + EMIT_PARAMS(pos, "%u %u ", seg->area_count, seg->stripe_size); + break; + case SEG_VDO: +- if (!_vdo_emit_segment_line(dmt, seg, params, paramsize)) ++ if (!_vdo_emit_segment_line(dmt, major, minor, seg, params, paramsize)) + return_0; + break; + case SEG_CRYPT: +-- +2.41.0 + diff --git a/SOURCES/0090-vdo-improve-validation-message.patch b/SOURCES/0090-vdo-improve-validation-message.patch new file mode 100644 index 0000000..0eb45d7 --- /dev/null +++ b/SOURCES/0090-vdo-improve-validation-message.patch @@ -0,0 +1,32 @@ +From 57eb6c1b9c3122a33da38c561e5177183577a958 Mon Sep 17 00:00:00 2001 +From: Zdenek Kabelac +Date: Tue, 8 Nov 2022 12:40:21 +0100 +Subject: [PATCH 090/115] vdo: improve validation message + +Rephrase. + +(cherry picked from commit 403779333be419352d6062da592ea78e6096e12b) +--- + device_mapper/vdo/vdo_target.c | 6 +++--- + 1 file changed, 3 insertions(+), 3 deletions(-) + +diff --git a/device_mapper/vdo/vdo_target.c b/device_mapper/vdo/vdo_target.c +index ab3fff26a..a8a753e39 100644 +--- a/device_mapper/vdo/vdo_target.c ++++ b/device_mapper/vdo/vdo_target.c +@@ -156,9 +156,9 @@ bool dm_vdo_validate_target_params(const struct dm_vdo_target_params *vtp, + } + + if (vdo_size > DM_VDO_LOGICAL_SIZE_MAXIMUM) { +- log_error("VDO logical size is by " FMTu64 "KiB bigger then limit " FMTu64 "TiB.", +- (vdo_size - DM_VDO_LOGICAL_SIZE_MAXIMUM) / 2, +- DM_VDO_LOGICAL_SIZE_MAXIMUM / (UINT64_C(1024) * 1024 * 1024 * 1024 >> SECTOR_SHIFT)); ++ log_error("VDO logical size is larger than limit " FMTu64 " TiB by " FMTu64 " KiB.", ++ DM_VDO_LOGICAL_SIZE_MAXIMUM / (UINT64_C(1024) * 1024 * 1024 * 1024 >> SECTOR_SHIFT), ++ (vdo_size - DM_VDO_LOGICAL_SIZE_MAXIMUM) / 2); + valid = false; + } + +-- +2.41.0 + diff --git a/SOURCES/0091-vdo-fix-reader-error-path.patch b/SOURCES/0091-vdo-fix-reader-error-path.patch new file mode 100644 index 0000000..e653746 --- /dev/null +++ b/SOURCES/0091-vdo-fix-reader-error-path.patch @@ -0,0 +1,28 @@ +From 05b4d98a51f09e8fdb555cdf1678de8645d5e9c4 Mon Sep 17 00:00:00 2001 +From: Zdenek Kabelac +Date: Tue, 20 Dec 2022 13:52:31 +0100 +Subject: [PATCH 091/115] vdo: fix reader error path + +Nothing to be closed on this error path. + +(cherry picked from commit b6b1c19365d20d926d8aa39bf591731a7f9bb75a) +--- + device_mapper/vdo/vdo_reader.c | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/device_mapper/vdo/vdo_reader.c b/device_mapper/vdo/vdo_reader.c +index 4f91f8011..3596afbd3 100644 +--- a/device_mapper/vdo/vdo_reader.c ++++ b/device_mapper/vdo/vdo_reader.c +@@ -189,7 +189,7 @@ bool dm_vdo_parse_logical_size(const char *vdo_path, uint64_t *logical_blocks) + *logical_blocks = 0; + if ((fh = open(vdo_path, O_RDONLY)) == -1) { + log_sys_debug("Failed to open VDO backend %s.", vdo_path); +- goto err; ++ return false; + } + + if (ioctl(fh, BLKGETSIZE64, &size) == -1) { +-- +2.41.0 + diff --git a/SOURCES/0092-vdo-check-memory-only-in-non-critical-section.patch b/SOURCES/0092-vdo-check-memory-only-in-non-critical-section.patch new file mode 100644 index 0000000..e475c90 --- /dev/null +++ b/SOURCES/0092-vdo-check-memory-only-in-non-critical-section.patch @@ -0,0 +1,38 @@ +From 71c8df45c7d7efddad66e379d492b6f92cdfb4b9 Mon Sep 17 00:00:00 2001 +From: Zdenek Kabelac +Date: Tue, 10 Jan 2023 21:12:22 +0100 +Subject: [PATCH 092/115] vdo: check memory only in non critical section + +When we are actually resizing VDO device - we need to check size only in +non-critical section - otherwise we are checking + +(cherry picked from commit 773b88e028ab2965a8c185f5f2147334f8f2bbfd) +--- + lib/vdo/vdo.c | 4 +++- + 1 file changed, 3 insertions(+), 1 deletion(-) + +diff --git a/lib/vdo/vdo.c b/lib/vdo/vdo.c +index 9efb424f0..d2d14d146 100644 +--- a/lib/vdo/vdo.c ++++ b/lib/vdo/vdo.c +@@ -23,6 +23,7 @@ + #include "lib/metadata/metadata.h" + #include "lib/metadata/lv_alloc.h" + #include "lib/metadata/segtype.h" ++#include "lib/mm/memlock.h" + #include "base/memory/zalloc.h" + + static const char _vdo_module[] = MODULE_NAME_VDO; +@@ -374,7 +375,8 @@ static int _vdo_pool_add_target_line(struct dev_manager *dm, + return 0; + } + +- if (!check_vdo_constrains(cmd, seg->lv->size, seg_lv(seg, 0)->size, &seg->vdo_params)) ++ if (!critical_section() && ++ !check_vdo_constrains(cmd, seg->lv->size, get_vdo_pool_virtual_size(seg), &seg->vdo_params)) + return_0; + + if (!(vdo_pool_name = dm_build_dm_name(mem, seg->lv->vg->name, seg->lv->name, lv_layer(seg->lv)))) +-- +2.41.0 + diff --git a/SOURCES/0093-vdo-read-live-vdo-size-configuration.patch b/SOURCES/0093-vdo-read-live-vdo-size-configuration.patch new file mode 100644 index 0000000..3704200 --- /dev/null +++ b/SOURCES/0093-vdo-read-live-vdo-size-configuration.patch @@ -0,0 +1,192 @@ +From 696036c20f8b452f0eb8482a9aa13878ce398452 Mon Sep 17 00:00:00 2001 +From: Zdenek Kabelac +Date: Sun, 15 Jan 2023 21:24:28 +0100 +Subject: [PATCH 093/115] vdo: read live vdo size configuration + +Introduce struct vdo_pool_size_config usable to calculate necessary +memory size for active VDO volume. + +Function lv_vdo_pool_size_config() is able to read out this +configuration out of runtime DM table line. + +(cherry picked from commit 1bed2cafe83096cf01cfff5cd2cd64ecb32b7d78) +--- + lib/activate/activate.c | 31 +++++++++++++++ + lib/activate/activate.h | 2 + + lib/activate/dev_manager.c | 65 ++++++++++++++++++++++++++++++++ + lib/activate/dev_manager.h | 3 ++ + lib/metadata/metadata-exported.h | 6 +++ + 5 files changed, 107 insertions(+) + +diff --git a/lib/activate/activate.c b/lib/activate/activate.c +index 76740bb2b..c951523c5 100644 +--- a/lib/activate/activate.c ++++ b/lib/activate/activate.c +@@ -322,6 +322,11 @@ int lv_vdo_pool_percent(const struct logical_volume *lv, dm_percent_t *percent) + { + return 0; + } ++int lv_vdo_pool_size_config(const struct logical_volume *lv, ++ struct vdo_pool_size_config *cfg) ++{ ++ return 0; ++} + int lvs_in_vg_activated(const struct volume_group *vg) + { + return 0; +@@ -1346,6 +1351,32 @@ int lv_vdo_pool_percent(const struct logical_volume *lv, dm_percent_t *percent) + return 1; + } + ++/* ++ * lv_vdo_pool_size_config obtains size configuration from active VDO table line ++ * ++ * If the 'params' string has been already retrieved, use it. ++ * If the mempool already exists, use it. ++ * ++ */ ++int lv_vdo_pool_size_config(const struct logical_volume *lv, ++ struct vdo_pool_size_config *cfg) ++{ ++ struct dev_manager *dm; ++ int r; ++ ++ if (!lv_info(lv->vg->cmd, lv, 1, NULL, 0, 0)) ++ return 1; /* Inactive VDO pool -> no runtime config */ ++ ++ if (!(dm = dev_manager_create(lv->vg->cmd, lv->vg->name, !lv_is_pvmove(lv)))) ++ return_0; ++ ++ r = dev_manager_vdo_pool_size_config(dm, lv, cfg); ++ ++ dev_manager_destroy(dm); ++ ++ return r; ++} ++ + static int _lv_active(struct cmd_context *cmd, const struct logical_volume *lv) + { + struct lvinfo info; +diff --git a/lib/activate/activate.h b/lib/activate/activate.h +index edd385aef..ca2b7f559 100644 +--- a/lib/activate/activate.h ++++ b/lib/activate/activate.h +@@ -200,6 +200,8 @@ int lv_thin_pool_status(const struct logical_volume *lv, int flush, + int lv_vdo_pool_status(const struct logical_volume *lv, int flush, + struct lv_status_vdo **status); + int lv_vdo_pool_percent(const struct logical_volume *lv, dm_percent_t *percent); ++int lv_vdo_pool_size_config(const struct logical_volume *lv, ++ struct vdo_pool_size_config *cfg); + + /* + * Return number of LVs in the VG that are active. +diff --git a/lib/activate/dev_manager.c b/lib/activate/dev_manager.c +index 5b72bf772..4924d8c56 100644 +--- a/lib/activate/dev_manager.c ++++ b/lib/activate/dev_manager.c +@@ -1901,6 +1901,71 @@ out: + return r; + } + ++int dev_manager_vdo_pool_size_config(struct dev_manager *dm, ++ const struct logical_volume *lv, ++ struct vdo_pool_size_config *cfg) ++{ ++ const char *dlid; ++ struct dm_info info; ++ uint64_t start, length; ++ struct dm_task *dmt = NULL; ++ char *type = NULL; ++ char *params = NULL; ++ int r = 0; ++ unsigned version = 0; ++ ++ memset(cfg, 0, sizeof(*cfg)); ++ ++ if (!(dlid = build_dm_uuid(dm->mem, lv, lv_layer(lv)))) ++ return_0; ++ ++ if (!(dmt = _setup_task_run(DM_DEVICE_TABLE, &info, NULL, dlid, 0, 0, 0, 0, 0, 0))) ++ return_0; ++ ++ if (!info.exists) ++ goto inactive; /* VDO device is not active, should not happen here... */ ++ ++ log_debug_activation("Checking VDO pool table line for LV %s.", ++ display_lvname(lv)); ++ ++ if (dm_get_next_target(dmt, NULL, &start, &length, &type, ¶ms)) { ++ log_error("More then one table line found for %s.", ++ display_lvname(lv)); ++ goto out; ++ } ++ ++ if (!type || strcmp(type, TARGET_NAME_VDO)) { ++ log_error("Expected %s segment type but got %s instead.", ++ TARGET_NAME_VDO, type ? type : "NULL"); ++ goto out; ++ } ++ ++ if (sscanf(params, "V%u %*s " FMTu64 " %*u " FMTu32, ++ &version, &cfg->physical_size, &cfg->block_map_cache_size_mb) != 3) { ++ log_error("Failed to parse VDO parameters %s for LV %s.", ++ params, display_lvname(lv)); ++ goto out; ++ } ++ ++ switch (version) { ++ case 2: break; ++ case 4: break; ++ default: log_warn("WARNING: Unknown VDO table line version %u.", version); ++ } ++ ++ cfg->virtual_size = length; ++ cfg->physical_size *= 8; // From 4K unit to 512B ++ cfg->block_map_cache_size_mb /= 256; // From 4K unit to MiB ++ cfg->index_memory_size_mb = first_seg(lv)->vdo_params.index_memory_size_mb; // Preserved ++ ++inactive: ++ r = 1; ++out: ++ dm_task_destroy(dmt); ++ ++ return r; ++} ++ + + /*************************/ + /* NEW CODE STARTS HERE */ +diff --git a/lib/activate/dev_manager.h b/lib/activate/dev_manager.h +index 27092f2b9..2f7ce6099 100644 +--- a/lib/activate/dev_manager.h ++++ b/lib/activate/dev_manager.h +@@ -81,6 +81,9 @@ int dev_manager_thin_pool_status(struct dev_manager *dm, + int dev_manager_vdo_pool_status(struct dev_manager *dm, + const struct logical_volume *lv, int flush, + struct lv_status_vdo **status, int *exists); ++int dev_manager_vdo_pool_size_config(struct dev_manager *dm, ++ const struct logical_volume *lv, ++ struct vdo_pool_size_config *cfg); + int dev_manager_suspend(struct dev_manager *dm, const struct logical_volume *lv, + struct lv_activate_opts *laopts, int lockfs, int flush_required); + int dev_manager_activate(struct dev_manager *dm, const struct logical_volume *lv, +diff --git a/lib/metadata/metadata-exported.h b/lib/metadata/metadata-exported.h +index f735baa55..de21f48f2 100644 +--- a/lib/metadata/metadata-exported.h ++++ b/lib/metadata/metadata-exported.h +@@ -1379,6 +1379,12 @@ int fill_vdo_target_params(struct cmd_context *cmd, + struct dm_vdo_target_params *vtp, + uint64_t *vdo_pool_header_size, + struct profile *profile); ++struct vdo_pool_size_config { ++ uint64_t physical_size; ++ uint64_t virtual_size; ++ uint32_t block_map_cache_size_mb; ++ uint32_t index_memory_size_mb; ++}; + int check_vdo_constrains(struct cmd_context *cmd, uint64_t physical_size, + uint64_t virtual_size, struct dm_vdo_target_params *vtp); + /* -- metadata/vdo_manip.c */ +-- +2.41.0 + diff --git a/SOURCES/0094-lvcreate-fix-error-path-return-values.patch b/SOURCES/0094-lvcreate-fix-error-path-return-values.patch new file mode 100644 index 0000000..86df0e8 --- /dev/null +++ b/SOURCES/0094-lvcreate-fix-error-path-return-values.patch @@ -0,0 +1,45 @@ +From 52e019c6133fbd68676bc6fecb40da9cb7e1986e Mon Sep 17 00:00:00 2001 +From: Zdenek Kabelac +Date: Tue, 8 Nov 2022 12:39:25 +0100 +Subject: [PATCH 094/115] lvcreate: fix error path return values + +Return failing error code for return path, as 'return 0' in this +case was returning success. + +(cherry picked from commit b9f35e07db41f3dd6bea2c91a8bc1bff93a4d406) +--- + tools/lvcreate.c | 6 +++--- + 1 file changed, 3 insertions(+), 3 deletions(-) + +diff --git a/tools/lvcreate.c b/tools/lvcreate.c +index 3eee5de90..d3fa5281f 100644 +--- a/tools/lvcreate.c ++++ b/tools/lvcreate.c +@@ -1749,21 +1749,21 @@ static int _lvcreate_single(struct cmd_context *cmd, const char *vg_name, + goto_out; + + if (seg_is_vdo(lp) && !_check_vdo_parameters(vg, lp, lcp)) +- return_0; ++ goto_out; + + if (!_check_pool_parameters(cmd, vg, lp, lcp)) + goto_out; + + /* All types are checked */ + if (!_check_zero_parameters(cmd, lp)) +- return_0; ++ goto_out; + + if (!_update_extents_params(vg, lp, lcp)) + goto_out; + + if (seg_is_vdo(lp) && !check_vdo_constrains(cmd, (uint64_t)lp->extents * vg->extent_size, + lcp->virtual_size, &lp->vdo_params)) +- return_0; ++ goto_out; + + if (seg_is_thin(lp) && !_validate_internal_thin_processing(lp)) + goto_out; +-- +2.41.0 + diff --git a/SOURCES/0095-vdo-fix-and-enhance-vdo-constain-checking.patch b/SOURCES/0095-vdo-fix-and-enhance-vdo-constain-checking.patch new file mode 100644 index 0000000..10d37f8 --- /dev/null +++ b/SOURCES/0095-vdo-fix-and-enhance-vdo-constain-checking.patch @@ -0,0 +1,228 @@ +From a22a9aa39b7521cc04b5ac3ae4629cd022a4bd6d Mon Sep 17 00:00:00 2001 +From: Zdenek Kabelac +Date: Sun, 15 Jan 2023 21:27:37 +0100 +Subject: [PATCH 095/115] vdo: fix and enhance vdo constain checking + +Enhance checking vdo constains so it also handles changes of active VDO LVs +where only added difference is considered now. + +For this also the reported informational message about used memory +was improved to only list consuming RAM blocks. + +(cherry picked from commit 2451bc568feb9b3d0cb4d7c15e3c723f4d0c8cc9) +--- + WHATS_NEW | 4 ++ + lib/metadata/metadata-exported.h | 4 +- + lib/metadata/vdo_manip.c | 77 ++++++++++++++++++++++++-------- + lib/vdo/vdo.c | 24 +++++++++- + tools/lvcreate.c | 8 +++- + 5 files changed, 92 insertions(+), 25 deletions(-) + +diff --git a/WHATS_NEW b/WHATS_NEW +index 48b1d7d86..7f8a16e24 100644 +--- a/WHATS_NEW ++++ b/WHATS_NEW +@@ -1,3 +1,7 @@ ++version 2.03.19 - ++==================================== ++ Fix and improve runtime memory size detection for VDO volumes. ++ + Version 2.03.17 - + =============================== + Extend VDO and VDOPOOL without flushing and locking fs. +diff --git a/lib/metadata/metadata-exported.h b/lib/metadata/metadata-exported.h +index de21f48f2..6ad89a9a4 100644 +--- a/lib/metadata/metadata-exported.h ++++ b/lib/metadata/metadata-exported.h +@@ -1385,8 +1385,8 @@ struct vdo_pool_size_config { + uint32_t block_map_cache_size_mb; + uint32_t index_memory_size_mb; + }; +-int check_vdo_constrains(struct cmd_context *cmd, uint64_t physical_size, +- uint64_t virtual_size, struct dm_vdo_target_params *vtp); ++ ++int check_vdo_constrains(struct cmd_context *cmd, const struct vdo_pool_size_config *cfg); + /* -- metadata/vdo_manip.c */ + + struct logical_volume *find_pvmove_lv(struct volume_group *vg, +diff --git a/lib/metadata/vdo_manip.c b/lib/metadata/vdo_manip.c +index 1cd2130a7..787101094 100644 +--- a/lib/metadata/vdo_manip.c ++++ b/lib/metadata/vdo_manip.c +@@ -24,6 +24,7 @@ + #include "lib/misc/lvm-exec.h" + + #include // sysinfo ++#include + + const char *get_vdo_compression_state_name(enum dm_vdo_compression_state state) + { +@@ -647,39 +648,76 @@ static uint64_t _round_sectors_to_tib(uint64_t s) + return (s + ((UINT64_C(1) << (40 - SECTOR_SHIFT)) - 1)) >> (40 - SECTOR_SHIFT); + } + +-int check_vdo_constrains(struct cmd_context *cmd, uint64_t physical_size, +- uint64_t virtual_size, struct dm_vdo_target_params *vtp) ++__attribute__ ((format(printf, 3, 4))) ++static int _vdo_snprintf(char **buf, size_t *bufsize, const char *format, ...) + { ++ int n; ++ va_list ap; ++ ++ va_start(ap, format); ++ n = vsnprintf(*buf, *bufsize, format, ap); ++ va_end(ap); ++ ++ if (n < 0 || ((unsigned) n >= *bufsize)) ++ return -1; ++ ++ *buf += n; ++ *bufsize -= n; ++ ++ return n; ++} ++ ++int check_vdo_constrains(struct cmd_context *cmd, const struct vdo_pool_size_config *cfg) ++{ ++ static const char *_split[] = { "", " and", ",", "," }; + uint64_t req_mb, total_mb, available_mb; +- uint64_t phy_mb = _round_sectors_to_tib(UINT64_C(268) * physical_size); // 268 MiB per 1 TiB of physical size +- uint64_t virt_mb = _round_1024(UINT64_C(1638) * _round_sectors_to_tib(virtual_size)); // 1.6 MiB per 1 TiB +- uint64_t cache_mb = _round_1024(UINT64_C(1177) * vtp->block_map_cache_size_mb); // 1.15 MiB per 1 MiB cache size ++ uint64_t phy_mb = _round_sectors_to_tib(UINT64_C(268) * cfg->physical_size); // 268 MiB per 1 TiB of physical size ++ uint64_t virt_mb = _round_1024(UINT64_C(1638) * _round_sectors_to_tib(cfg->virtual_size)); // 1.6 MiB per 1 TiB ++ uint64_t cache_mb = _round_1024(UINT64_C(1177) * cfg->block_map_cache_size_mb); // 1.15 MiB per 1 MiB cache size + char msg[512]; ++ size_t mlen = sizeof(msg); ++ char *pmsg = msg; ++ int cnt, has_cnt; + +- if (cache_mb < 150) ++ if (cfg->block_map_cache_size_mb && (cache_mb < 150)) + cache_mb = 150; // always at least 150 MiB for block map + + // total required memory for VDO target +- req_mb = 38 + vtp->index_memory_size_mb + virt_mb + phy_mb + cache_mb; ++ req_mb = 38 + cfg->index_memory_size_mb + virt_mb + phy_mb + cache_mb; + + _get_memory_info(&total_mb, &available_mb); + +- (void)snprintf(msg, sizeof(msg), "VDO configuration needs %s RAM for physical volume size %s, " +- "%s RAM for virtual volume size %s, %s RAM for block map cache size %s and " +- "%s RAM for index memory.", +- display_size(cmd, phy_mb << (20 - SECTOR_SHIFT)), +- display_size(cmd, physical_size), +- display_size(cmd, virt_mb << (20 - SECTOR_SHIFT)), +- display_size(cmd, virtual_size), +- display_size(cmd, cache_mb << (20 - SECTOR_SHIFT)), +- display_size(cmd, ((uint64_t)vtp->block_map_cache_size_mb) << (20 - SECTOR_SHIFT)), +- display_size(cmd, ((uint64_t)vtp->index_memory_size_mb) << (20 - SECTOR_SHIFT))); ++ has_cnt = cnt = (phy_mb ? 1 : 0) + ++ (virt_mb ? 1 : 0) + ++ (cfg->block_map_cache_size_mb ? 1 : 0) + ++ (cfg->index_memory_size_mb ? 1 : 0); ++ ++ if (phy_mb) ++ (void)_vdo_snprintf(&pmsg, &mlen, " %s RAM for physical volume size %s%s", ++ display_size(cmd, phy_mb << (20 - SECTOR_SHIFT)), ++ display_size(cmd, cfg->physical_size), _split[--cnt]); ++ ++ if (virt_mb) ++ (void)_vdo_snprintf(&pmsg, &mlen, " %s RAM for virtual volume size %s%s", ++ display_size(cmd, virt_mb << (20 - SECTOR_SHIFT)), ++ display_size(cmd, cfg->virtual_size), _split[--cnt]); ++ ++ if (cfg->block_map_cache_size_mb) ++ (void)_vdo_snprintf(&pmsg, &mlen, " %s RAM for block map cache size %s%s", ++ display_size(cmd, cache_mb << (20 - SECTOR_SHIFT)), ++ display_size(cmd, ((uint64_t)cfg->block_map_cache_size_mb) << (20 - SECTOR_SHIFT)), ++ _split[--cnt]); ++ ++ if (cfg->index_memory_size_mb) ++ (void)_vdo_snprintf(&pmsg, &mlen, " %s RAM for index memory", ++ display_size(cmd, ((uint64_t)cfg->index_memory_size_mb) << (20 - SECTOR_SHIFT))); + + if (req_mb > available_mb) { + log_error("Not enough free memory for VDO target. %s RAM is required, but only %s RAM is available.", + display_size(cmd, req_mb << (20 - SECTOR_SHIFT)), + display_size(cmd, available_mb << (20 - SECTOR_SHIFT))); +- log_print_unless_silent("%s", msg); ++ if (has_cnt) ++ log_print_unless_silent("VDO configuration needs%s.", msg); + return 0; + } + +@@ -687,7 +725,8 @@ int check_vdo_constrains(struct cmd_context *cmd, uint64_t physical_size, + display_size(cmd, req_mb << (20 - SECTOR_SHIFT)), + display_size(cmd, available_mb << (20 - SECTOR_SHIFT))); + +- log_verbose("%s", msg); ++ if (has_cnt) ++ log_verbose("VDO configuration needs%s.", msg); + + return 1; + } +diff --git a/lib/vdo/vdo.c b/lib/vdo/vdo.c +index d2d14d146..133678ae7 100644 +--- a/lib/vdo/vdo.c ++++ b/lib/vdo/vdo.c +@@ -355,6 +355,27 @@ static int _vdo_pool_target_status_compatible(const char *type) + return (strcmp(type, TARGET_NAME_VDO) == 0); + } + ++static int _vdo_check(struct cmd_context *cmd, const struct lv_segment *seg) ++{ ++ ++ struct vdo_pool_size_config cfg = { 0 }; ++ ++ if (!lv_vdo_pool_size_config(seg->lv, &cfg)) ++ return_0; ++ ++ /* Check if we are just adding more size to the already running vdo pool */ ++ if (seg->lv->size >= cfg.physical_size) ++ cfg.physical_size = seg->lv->size - cfg.physical_size; ++ if (get_vdo_pool_virtual_size(seg) >= cfg.virtual_size) ++ cfg.virtual_size = get_vdo_pool_virtual_size(seg) - cfg.virtual_size; ++ if (seg->vdo_params.block_map_cache_size_mb >= cfg.block_map_cache_size_mb) ++ cfg.block_map_cache_size_mb = seg->vdo_params.block_map_cache_size_mb - cfg.block_map_cache_size_mb; ++ if (seg->vdo_params.index_memory_size_mb >= cfg.index_memory_size_mb) ++ cfg.index_memory_size_mb = seg->vdo_params.index_memory_size_mb - cfg.index_memory_size_mb; ++ ++ return check_vdo_constrains(cmd, &cfg); ++} ++ + static int _vdo_pool_add_target_line(struct dev_manager *dm, + struct dm_pool *mem, + struct cmd_context *cmd, +@@ -375,8 +396,7 @@ static int _vdo_pool_add_target_line(struct dev_manager *dm, + return 0; + } + +- if (!critical_section() && +- !check_vdo_constrains(cmd, seg->lv->size, get_vdo_pool_virtual_size(seg), &seg->vdo_params)) ++ if (!critical_section() && !_vdo_check(cmd, seg)) + return_0; + + if (!(vdo_pool_name = dm_build_dm_name(mem, seg->lv->vg->name, seg->lv->name, lv_layer(seg->lv)))) +diff --git a/tools/lvcreate.c b/tools/lvcreate.c +index d3fa5281f..a2e260c5d 100644 +--- a/tools/lvcreate.c ++++ b/tools/lvcreate.c +@@ -1761,8 +1761,12 @@ static int _lvcreate_single(struct cmd_context *cmd, const char *vg_name, + if (!_update_extents_params(vg, lp, lcp)) + goto_out; + +- if (seg_is_vdo(lp) && !check_vdo_constrains(cmd, (uint64_t)lp->extents * vg->extent_size, +- lcp->virtual_size, &lp->vdo_params)) ++ if (seg_is_vdo(lp) && ++ !check_vdo_constrains(cmd, &(struct vdo_pool_size_config) { ++ .physical_size = (uint64_t)lp->extents * vg->extent_size, ++ .virtual_size = lcp->virtual_size, ++ .block_map_cache_size_mb = lp->vdo_params.block_map_cache_size_mb, ++ .index_memory_size_mb = lp->vdo_params.index_memory_size_mb })) + goto_out; + + if (seg_is_thin(lp) && !_validate_internal_thin_processing(lp)) +-- +2.41.0 + diff --git a/SOURCES/0096-vdo-resize-requires-active-vdopool-volume.patch b/SOURCES/0096-vdo-resize-requires-active-vdopool-volume.patch new file mode 100644 index 0000000..20f2bce --- /dev/null +++ b/SOURCES/0096-vdo-resize-requires-active-vdopool-volume.patch @@ -0,0 +1,37 @@ +From 72749491665de8f090292af86a62fc4405e7e8fe Mon Sep 17 00:00:00 2001 +From: Marian Csontos +Date: Thu, 22 Jun 2023 16:55:47 +0200 +Subject: [PATCH 096/115] vdo: resize requires active vdopool volume + +ATM kernel VDO target does not handle resize of inactive VDO LVs +so prevent users corrupting such LVs and require active such LVs. + +(manually backported from commit c20f01a0cbb24e8276734a00f26a510cf6daaf21) +--- + lib/metadata/lv_manip.c | 10 ++++++++++ + 1 file changed, 10 insertions(+) + +diff --git a/lib/metadata/lv_manip.c b/lib/metadata/lv_manip.c +index 9523e3e66..0a51e0315 100644 +--- a/lib/metadata/lv_manip.c ++++ b/lib/metadata/lv_manip.c +@@ -5190,6 +5190,16 @@ static int _lvresize_check(struct logical_volume *lv, + struct volume_group *vg = lv->vg; + struct lv_segment *seg = first_seg(lv); + ++ if (lv_is_vdo(lv) && !lv_is_active(lv)) { ++ log_error("Cannot resize inactive VDO logical volume %s.", display_lvname(lv)); ++ return 0; ++ } ++ ++ if (lv_is_vdo_pool(lv) && !lv_is_active(lv_lock_holder(lv))) { ++ log_error("Cannot resize inactive VDO POOL volume %s.", display_lvname(lv)); ++ return 0; ++ } ++ + if (lv_is_external_origin(lv)) { + /* + * Since external-origin can be activated read-only, +-- +2.41.0 + diff --git a/SOURCES/0097-tests-vdo-resizing.patch b/SOURCES/0097-tests-vdo-resizing.patch new file mode 100644 index 0000000..625c7df --- /dev/null +++ b/SOURCES/0097-tests-vdo-resizing.patch @@ -0,0 +1,37 @@ +From 686d6ecdccf0a72f149c42c6cfe2d630bfe03ded Mon Sep 17 00:00:00 2001 +From: Zdenek Kabelac +Date: Mon, 16 Jan 2023 01:11:37 +0100 +Subject: [PATCH 097/115] tests: vdo resizing + +(cherry picked from commit aa09232dc4fb825f4e643bbe87a95e2318dd5a93) +--- + test/shell/lvresize-vdo.sh | 15 +++++++++++++++ + 1 file changed, 15 insertions(+) + +diff --git a/test/shell/lvresize-vdo.sh b/test/shell/lvresize-vdo.sh +index 6fe0e9c26..cbe8ce967 100644 +--- a/test/shell/lvresize-vdo.sh ++++ b/test/shell/lvresize-vdo.sh +@@ -32,4 +32,19 @@ check lv_field $vg/${lv2}_vdata size "6.00g" + lvresize -L6G $vg/$lv1 + check lv_field $vg/$lv1 size "6.00g" + ++# Check too large size ++not lvresize -L4P $vg/$lv1 2>err ++grep "Volume too large" err ++ ++# Can't resize inactive VDO ++lvchange -an $vg ++not lvresize -L10G $vg/$lv1 2>err ++grep "Cannot resize inactive" err ++ ++not lvresize -L10G $vg/$lv2 2>err ++grep "Cannot resize inactive" err ++ ++not lvresize -L10G $vg/${lv2}_vdata 2>err ++grep "Cannot resize inactive" err ++ + vgremove -ff $vg +-- +2.41.0 + diff --git a/SOURCES/0098-tests-check-failing-vdo-conversion.patch b/SOURCES/0098-tests-check-failing-vdo-conversion.patch new file mode 100644 index 0000000..302afe4 --- /dev/null +++ b/SOURCES/0098-tests-check-failing-vdo-conversion.patch @@ -0,0 +1,40 @@ +From 31e71842d4be795a8f675d0cd50f8a7b8d737cd9 Mon Sep 17 00:00:00 2001 +From: Zdenek Kabelac +Date: Fri, 17 Feb 2023 00:06:56 +0100 +Subject: [PATCH 098/115] tests: check failing vdo conversion + +When we detect not usable vdo on a machine, check lvconvert fails. + +(cherry picked from commit 06c5c29443fdd48730a282a5db16e46c793db27b) +--- + test/shell/lvconvert-vdo.sh | 10 ++++++++-- + 1 file changed, 8 insertions(+), 2 deletions(-) + +diff --git a/test/shell/lvconvert-vdo.sh b/test/shell/lvconvert-vdo.sh +index c42d8f25a..7c0bce56b 100644 +--- a/test/shell/lvconvert-vdo.sh ++++ b/test/shell/lvconvert-vdo.sh +@@ -18,12 +18,18 @@ SKIP_WITH_LVMPOLLD=1 + # + # Main + # +-aux have_vdo 6 2 0 || skip +- + aux prepare_vg 2 6400 + + # Conversion to vdo-pool + lvcreate -L5G -n $lv1 $vg ++ ++if not aux have_vdo 6 2 0 ; then ++# For unsupported VDO let's check lvconvert fails ++ not lvconvert --yes --type vdo-pool $vg/$lv1 |& tee out ++ vgremove -ff $vg ++ exit ++fi ++ + # Check there is big prompting warning + not lvconvert --type vdo-pool $vg/$lv1 |& tee out + grep "WARNING" out +-- +2.41.0 + diff --git a/SOURCES/0099-vdo-use-fixed-size-vdopool-wrapper.patch b/SOURCES/0099-vdo-use-fixed-size-vdopool-wrapper.patch new file mode 100644 index 0000000..3eb7f18 --- /dev/null +++ b/SOURCES/0099-vdo-use-fixed-size-vdopool-wrapper.patch @@ -0,0 +1,38 @@ +From 51e90d49bae0a1d92687876f161c8463529fb702 Mon Sep 17 00:00:00 2001 +From: Zdenek Kabelac +Date: Mon, 6 Mar 2023 14:52:59 +0100 +Subject: [PATCH 099/115] vdo: use fixed size vdopool wrapper + +Instead of using size of 'empty header' in vdopool use fixed size 4K +for a 'wrappeing' vdo-pool device. + +This fixes the issue when user tried to activate vdo-pool after +a conversion from vdo managed device with 'vgchange -ay' - where +this command activated all LVs with 'vdo-pool' wrapping device as well, +but this converted pool uses 0-length header. + +This 4k size should usually prevent other tools like 'blkid' recognize +such device as anything - so it shouldn't cause any problems with +duplicate indentification of devices. + +(cherry picked from commit e8e6347ba3f59cbd2f7e92aa707543b90fa607a3) +--- + lib/activate/dev_manager.c | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/lib/activate/dev_manager.c b/lib/activate/dev_manager.c +index 4924d8c56..0c029917f 100644 +--- a/lib/activate/dev_manager.c ++++ b/lib/activate/dev_manager.c +@@ -3043,7 +3043,7 @@ static int _add_layer_target_to_dtree(struct dev_manager *dm, + + /* Add linear mapping over layered LV */ + /* From VDO layer expose ONLY vdo pool header, we would need to use virtual size otherwise */ +- if (!add_linear_area_to_dtree(dnode, lv_is_vdo_pool(lv) ? first_seg(lv)->vdo_pool_header_size : lv->size, ++ if (!add_linear_area_to_dtree(dnode, lv_is_vdo_pool(lv) ? 8 : lv->size, + lv->vg->extent_size, + lv->vg->cmd->use_linear_target, + lv->vg->name, lv->name) || +-- +2.41.0 + diff --git a/SOURCES/0100-vdo-document-feature-option.patch b/SOURCES/0100-vdo-document-feature-option.patch new file mode 100644 index 0000000..0ca67fd --- /dev/null +++ b/SOURCES/0100-vdo-document-feature-option.patch @@ -0,0 +1,48 @@ +From 5e6aa69ec03848942153c7b65b270a0503363e58 Mon Sep 17 00:00:00 2001 +From: Zdenek Kabelac +Date: Thu, 27 Apr 2023 23:17:40 +0200 +Subject: [PATCH 100/115] vdo: document feature option + +(cherry picked from commit 534269d0fb834fb9a3b115bf859319fc1acb2275) +--- + conf/example.conf.in | 4 ++-- + lib/config/config_settings.h | 4 ++-- + 2 files changed, 4 insertions(+), 4 deletions(-) + +diff --git a/conf/example.conf.in b/conf/example.conf.in +index 897622b9d..c74800676 100644 +--- a/conf/example.conf.in ++++ b/conf/example.conf.in +@@ -1318,10 +1318,10 @@ global { + # Configuration option global/vdo_disabled_features. + # Features to not use in the vdo driver. + # This can be helpful for testing, or to avoid using a feature that is +- # causing problems. Features include: online_rename ++ # causing problems. Features include: online_rename, version4 + # + # Example +- # vdo_disabled_features = [ "online_rename" ] ++ # vdo_disabled_features = [ "online_rename", "version4" ] + # + # This configuration option does not have a default value defined. + +diff --git a/lib/config/config_settings.h b/lib/config/config_settings.h +index 2c91e8bb6..a4beda654 100644 +--- a/lib/config/config_settings.h ++++ b/lib/config/config_settings.h +@@ -1265,10 +1265,10 @@ cfg_array(global_vdo_format_options_CFG, "vdo_format_options", global_CFG_SECTIO + cfg_array(global_vdo_disabled_features_CFG, "vdo_disabled_features", global_CFG_SECTION, CFG_ALLOW_EMPTY | CFG_DEFAULT_UNDEFINED, CFG_TYPE_STRING, NULL, vsn(2, 3, 11), NULL, 0, NULL, + "Features to not use in the vdo driver.\n" + "This can be helpful for testing, or to avoid using a feature that is\n" +- "causing problems. Features include: online_rename\n" ++ "causing problems. Features include: online_rename, version4\n" + "#\n" + "Example\n" +- "vdo_disabled_features = [ \"online_rename\" ]\n" ++ "vdo_disabled_features = [ \"online_rename\", \"version4\" ]\n" + "#\n") + + cfg(global_fsadm_executable_CFG, "fsadm_executable", global_CFG_SECTION, CFG_DEFAULT_COMMENTED, CFG_TYPE_STRING, DEFAULT_FSADM_PATH, vsn(2, 2, 170), "@FSADM_PATH@", 0, NULL, +-- +2.41.0 + diff --git a/SOURCES/0101-vdo-fix-and-enhance-vdo-metadata-reader.patch b/SOURCES/0101-vdo-fix-and-enhance-vdo-metadata-reader.patch new file mode 100644 index 0000000..74feb72 --- /dev/null +++ b/SOURCES/0101-vdo-fix-and-enhance-vdo-metadata-reader.patch @@ -0,0 +1,65 @@ +From 3b7c0786cce47b0f8b260605d25365145aefbda6 Mon Sep 17 00:00:00 2001 +From: Zdenek Kabelac +Date: Fri, 9 Jun 2023 13:21:35 +0200 +Subject: [PATCH 101/115] vdo: fix and enhance vdo metadata reader + +Improve metadata parser to handle volume_geometry bio_offset, +which needs to be substracted from 'region' start_block when present. + +This bio_offset block is non-zero i.e. with converted VDO volumes. + +Also fix some converted structure value (but they are not in use). + +(cherry picked from commit 00633f8b668d94a58756d96f5927fd690b8f41d3) +--- + device_mapper/vdo/vdo_reader.c | 12 ++++++++---- + 1 file changed, 8 insertions(+), 4 deletions(-) + +diff --git a/device_mapper/vdo/vdo_reader.c b/device_mapper/vdo/vdo_reader.c +index 3596afbd3..2451947cd 100644 +--- a/device_mapper/vdo/vdo_reader.c ++++ b/device_mapper/vdo/vdo_reader.c +@@ -146,13 +146,14 @@ static void _vdo_decode_header(struct vdo_header *h) + static void _vdo_decode_geometry_region(struct vdo_volume_region *vr) + { + vr->id = le32_to_cpu(vr->id); +- vr->start_block = le32_to_cpu(vr->start_block); ++ vr->start_block = le64_to_cpu(vr->start_block); + } + + static void _vdo_decode_volume_geometry(struct vdo_volume_geometry *vg) + { +- vg->release_version = le64_to_cpu(vg->release_version); ++ vg->release_version = le32_to_cpu(vg->release_version); + vg->nonce = le64_to_cpu(vg->nonce); ++ vg->bio_offset = le64_to_cpu(vg->bio_offset); + _vdo_decode_geometry_region(&vg->regions[VDO_DATA_REGION]); + } + +@@ -225,10 +226,14 @@ bool dm_vdo_parse_logical_size(const char *vdo_path, uint64_t *logical_blocks) + goto err; + } + ++ if (h.id != 5) { ++ log_debug_activation("Expected geometry VDO block instead of block %u.", h.id); ++ goto err; ++ } + memcpy(&vg, buffer + MAGIC_NUMBER_SIZE + sizeof(h), sizeof(vg)); + _vdo_decode_volume_geometry(&vg); + +- regpos = vg.regions[VDO_DATA_REGION].start_block * 4096; ++ regpos = (vg.regions[VDO_DATA_REGION].start_block - vg.bio_offset) * 4096; + + if ((regpos + sizeof(buffer)) > size) { + log_debug_activation("File/Device is shorter and can't provide requested VDO volume region at " FMTu64 " > " FMTu64 ".", regpos, size); +@@ -245,7 +250,6 @@ bool dm_vdo_parse_logical_size(const char *vdo_path, uint64_t *logical_blocks) + goto err; + } + +- + memcpy(&vn, buffer + sizeof(struct vdo_geometry_block), sizeof(vn)); + _vdo_decode_version(&vn); + +-- +2.41.0 + diff --git a/SOURCES/0102-tests-cleanup-some-shellcheck-warns.patch b/SOURCES/0102-tests-cleanup-some-shellcheck-warns.patch new file mode 100644 index 0000000..f3252c9 --- /dev/null +++ b/SOURCES/0102-tests-cleanup-some-shellcheck-warns.patch @@ -0,0 +1,31 @@ +From 062b014c04d15ad5a5c0b8aaeb30166c33e0d763 Mon Sep 17 00:00:00 2001 +From: Marian Csontos +Date: Thu, 22 Jun 2023 18:01:27 +0200 +Subject: [PATCH 102/115] tests: cleanup some shellcheck warns + +(manually backported some changes from commit 773bc013778df631f9ee14e7a79d5f02211b1e67) +--- + scripts/lvm_import_vdo.sh | 4 ++-- + 1 file changed, 2 insertions(+), 2 deletions(-) + +diff --git a/scripts/lvm_import_vdo.sh b/scripts/lvm_import_vdo.sh +index c4c1d152e..134c96513 100755 +--- a/scripts/lvm_import_vdo.sh ++++ b/scripts/lvm_import_vdo.sh +@@ -255,11 +255,11 @@ convert2lvm_() { + # Find largest matching VG name to our 'default' vgname + LASTVGNAME=$(LC_ALL=C "$LVM" vgs -oname -O-name --noheadings -S name=~${VGNAME} | grep -E "${VGNAME}[0-9]? ?" | head -1 || true) + if test -n "$LASTVGNAME" ; then +- LASTVGNAME=${LASTVGNAME#*${VGNAME}} ++ LASTVGNAME=${LASTVGNAME#*"${VGNAME}"} + # If the number is becoming too high, try some random number + test "$LASTVGNAME" -gt 99999999 2>/dev/null && LASTVGNAME=$RANDOM + # Generate new unused VG name +- VGNAME="${VGNAME}$(( ${LASTVGNAME} + 1 ))" ++ VGNAME="${VGNAME}$(( LASTVGNAME + 1 ))" + verbose "Selected unused volume group name $VGNAME." + fi + fi +-- +2.41.0 + diff --git a/SOURCES/0103-vdo-enhance-lvm_vdo_import.patch b/SOURCES/0103-vdo-enhance-lvm_vdo_import.patch new file mode 100644 index 0000000..8917468 --- /dev/null +++ b/SOURCES/0103-vdo-enhance-lvm_vdo_import.patch @@ -0,0 +1,564 @@ +From 80a59e88c995b9935042e2d3f803b411353bf6d5 Mon Sep 17 00:00:00 2001 +From: Zdenek Kabelac +Date: Thu, 11 May 2023 20:07:37 +0200 +Subject: [PATCH 103/115] vdo: enhance lvm_vdo_import + +Add support for usage of 'dm-snapshot' for doing whole device conversion +in a snapshot which could be merged once the whole conversion has been +made. +This helps with cases where there would be any unexpected failure in the +middle of conversion process and user can continue using original +device until problem in conversion is fixed. + +Import tool now uses 'truncate' tool to create a small backend file (20M) for loop device +to hold COW exception store. + +Option --vdo-config has been added to allow specifing location of vdo +configuration file. + +Option --no-snapshot allows to use import tool without creation of +snapshot, however recovery after finished VDO conversion and unfinished +lvm2 conversion is then very difficult. + +Option --uuid-prefix allow to specify DM UUID prefix for snapshot +device. + +Use read with -r. + +(cherry picked from commit be6c34212b05d8c59c0ba68e26c56cddf61c9f04) +--- + scripts/lvm_import_vdo.sh | 375 ++++++++++++++++++++++++++++---------- + 1 file changed, 282 insertions(+), 93 deletions(-) + +diff --git a/scripts/lvm_import_vdo.sh b/scripts/lvm_import_vdo.sh +index 134c96513..bae4b52d1 100755 +--- a/scripts/lvm_import_vdo.sh ++++ b/scripts/lvm_import_vdo.sh +@@ -1,6 +1,6 @@ + #!/bin/bash + # +-# Copyright (C) 2021 Red Hat, Inc. All rights reserved. ++# Copyright (C) 2021-2023 Red Hat, Inc. All rights reserved. + # + # This file is part of LVM2. + # +@@ -19,7 +19,7 @@ + # Needed utilities: + # lvm, dmsetup, + # vdo, +-# grep, awk, sed, blockdev, readlink, stat, mkdir ++# grep, awk, sed, blockdev, readlink, stat, mkdir, truncate + # + # Conversion is using 'vdo convert' support from VDO manager to move + # existing VDO header by 2M which makes space to place in PV header +@@ -29,6 +29,9 @@ + set -euE -o pipefail + + TOOL=lvm_import_vdo ++IMPORT_NAME="VDO_${TOOL}_${RANDOM}$$" ++test ${#IMPORT_NAME} -lt 100 || error "Random name \"$IMPORT_NAME\" is too long!" ++TEMPDIR="${TMPDIR:-/tmp}/$IMPORT_NAME" + + _SAVEPATH=$PATH + PATH="/sbin:/usr/sbin:/bin:/usr/sbin:$PATH" +@@ -36,26 +39,40 @@ PATH="/sbin:/usr/sbin:/bin:/usr/sbin:$PATH" + # user may override lvm location by setting LVM_BINARY + LVM=${LVM_BINARY:-lvm} + VDO=${VDO_BINARY:-vdo} +-VDOCONF=${VDOCONF:-} + BLOCKDEV="blockdev" ++LOSETUP="losetup" + READLINK="readlink" + READLINK_E="-e" + STAT="stat" + MKDIR="mkdir" ++TRUNCATE="truncate" + DMSETUP="dmsetup" + +-TEMPDIR="${TMPDIR:-/tmp}/${TOOL}_${RANDOM}$$" + DM_DEV_DIR="${DM_DEV_DIR:-/dev}" +- +-DEVICENAME="" ++DM_UUID_PREFIX="${DM_UUID_PREFIX:-}" ++DM_VG_NAME= ++DM_LV_NAME= ++VDO_CONFIG=${VDO_CONFIG:-} # can be overriden with --vdo-config ++VDOCONF= ++test -n "$VDO_CONFIG" && VDOCONF="-f $VDO_CONFIG" ++ ++DEVICE= ++VGNAME= ++LVNAME= + DEVMAJOR=0 + DEVMINOR=0 + PROMPTING="" ++USE_VDO_DM_SNAPSHOT=1 ++VDO_DM_SNAPSHOT_NAME= ++VDO_DM_SNAPSHOT_DEVICE= ++VDO_SNAPSHOT_LOOP= + + DRY=0 + VERB="" + FORCE="" + YES="" ++ABORT_AFTER_VDO_CONVERT=0 ++VDO_ALLOCATION_PARAMS= + + # default name for converted VG and its VDO LV + DEFAULT_NAME="vdovg/vdolvol" +@@ -74,6 +91,9 @@ tool_usage() { + echo " -v | --verbose Be verbose" + echo " -y | --yes Answer \"yes\" at any prompts" + echo " --dry-run Print verbosely commands without running them" ++ echo " --no-snapshot Do not use snapshot for converted VDO device" ++ echo " --uuid-prefix Prefix for DM snapshot uuid" ++ echo " --vdo-config Configuration file for VDO manager" + + exit + } +@@ -102,12 +122,79 @@ dry() { + cleanup() { + trap '' 2 + ++ test -n "$VDO_DM_SNAPSHOT_NAME" && { "$DMSETUP" remove "$VDO_DM_SNAPSHOT_NAME" || true ; } ++ test -n "$VDO_SNAPSHOT_LOOP" && { "$LOSETUP" -d "$VDO_SNAPSHOT_LOOP" || true ; } ++ + test -z "$PROMPTING" || echo "No" +- rm -rf "$TEMPDIR" ++ rm -rf "$TEMPDIR" || true + # error exit status for break + exit "${1:-1}" + } + ++# Create snapshot target like for persistent snapshot with 16KiB chunksize ++snapshot_target_line_() { ++ echo "0 $("$BLOCKDEV" --getsize "$1") snapshot${3:-} $1 $2 P 32" ++} ++ ++snapshot_create_() { ++ VDO_DM_SNAPSHOT_NAME="${IMPORT_NAME}_snap" ++ local file="$TEMPDIR/$VDO_DM_SNAPSHOT_NAME" ++ ++ # TODO: maybe use ramdisk via 'brd' device ?) ++ "$TRUNCATE" -s 20M "$file" ++ VDO_SNAPSHOT_LOOP=$("$LOSETUP" -f --show "$file") ++ "$DMSETUP" create "$VDO_DM_SNAPSHOT_NAME" -u "${DM_UUID_PREFIX}-${VDO_DM_SNAPSHOT_NAME}-priv" --table "$(snapshot_target_line_ "$1" "$VDO_SNAPSHOT_LOOP")" ++ VDO_DM_SNAPSHOT_DEVICE="$DM_DEV_DIR/mapper/$VDO_DM_SNAPSHOT_NAME" ++ verbose "Snapshot of VDO device $1 created: $VDO_DM_SNAPSHOT_DEVICE." ++} ++ ++snapshot_merge_() { ++ local status ++ local inital_status ++ ++ initial_status=( $("$DMSETUP" status "$VDO_DM_SNAPSHOT_NAME") ) ++ "$DMSETUP" reload "$VDO_DM_SNAPSHOT_NAME" --table "$(snapshot_target_line_ "$1" "$VDO_SNAPSHOT_LOOP" -merge)" ++ "$DMSETUP" suspend "$VDO_DM_SNAPSHOT_NAME" || { ++ error "ABORTING: Failed to initialize snapshot merge! Origin volume is unchanged." ++ } ++ ++ verbose "Merging converted VDO volume..." ++ # Running merging ++ "$DMSETUP" resume "$VDO_DM_SNAPSHOT_NAME" ++ ++ #du -h "$TEMPDIR/$VDO_DM_SNAPSHOT_NAME" ++ ++ # Loop for a while, till the snapshot is merged. ++ # Should be nearly instantenious. ++ # FIXME: Recovery when something prevents merging is hard ++ for i in $(seq 1 20) ; do ++ status=( $("$DMSETUP" status "$VDO_DM_SNAPSHOT_NAME") ) ++ # Check if merging is finished ++ test "${status[3]%/*}" = "${status[4]}" && break ++ # Wait a bit and retry ++ sleep .2 ++ done ++ test "${status[3]%/*}" = "${status[4]}" || { ++ # FIXME: Now what shall we do ??? Help.... ++ # Keep snapshot in table for possible analysis... ++ VDO_DM_SNAPSHOT_NAME= ++ VDO_SNAPSHOT_LOOP= ++ echo "Initial snapshot status ${initial_status[*]}" ++ echo "Failing merge snapshot status ${status[*]}" ++ error "ABORTING: Snapshot failed to merge! (Administrator required...)" ++ } ++ sync ++ "$DMSETUP" remove "$VDO_DM_SNAPSHOT_NAME" || { ++ sleep 1 # sleep and retry once more ++ "$DMSETUP" remove "$VDO_DM_SNAPSHOT_NAME" || { ++ error "ABORTING: Cannot remove snapshot $VDO_DM_SNAPSHOT_NAME! (check volume autoactivation...)" ++ } ++ } ++ VDO_DM_SNAPSHOT_NAME= ++ "$LOSETUP" -d "$VDO_SNAPSHOT_LOOP" ++ VDO_SNAPSHOT_LOOP= ++} ++ + get_enabled_value_() { + case "$1" in + enabled) echo "1" ;; +@@ -221,46 +308,191 @@ parse_yaml_() { + ) < "$yaml_file" + } + +-# convert existing VDO volume into lvm2 volume ++# ++# Convert VDO volume on LV to VDOPool within this VG ++# ++# This conversion requires the size of VDO virtual volume has to be expressed in the VG's extent size. ++# Currently this enforces a user to reduce the VG extent size to the smaller size (up to 4KiB). ++# ++# TODO: We may eventually relax this condition just like we are doing rounding for convert_non_lv_() ++# Let's if there would be any singly user requiring this feauture. ++# It may allow to better use larger VDO volume size (in TiB ranges). ++# ++convert_lv_() { ++ local vdo_logicalSize=$1 ++ local extent_size ++ local pvfree ++ ++ pvfree=$("$LVM" lvs -o size --units b --nosuffix --noheadings "$DM_VG_NAME/$DM_LV_NAME") ++ pvfree=$(( pvfree / 1024 )) # to KiB ++ # select largest possible extent size that can exactly express both sizes ++ extent_size=$(get_largest_extent_size_ "$pvfree" "$vdo_logicalSize") ++ ++ # validate existing VG extent_size can express virtual VDO size ++ vg_extent_size=$("$LVM" vgs -o vg_extent_size --units b --nosuffix --noheadings "$VGNAME") ++ vg_extent_size=$(( vg_extent_size / 1024 )) ++ ++ test "$vg_extent_size" -le "$extent_size" || { ++ error "Please vgchange extent_size to at most $extent_size KiB or extend and align virtual size of VDO device on $vg_extent_size KiB before retrying conversion." ++ } ++ verbose "Renaming existing LV to be used as _vdata volume for VDO pool LV." ++ dry "$LVM" lvrename $YES $VERB "$VGNAME/$DM_LV_NAME" "$VGNAME/${LVNAME}_vpool" || { ++ error "Rename of LV \"$VGNAME/$DM_LV_NAME\" failed, while VDO header has been already moved!" ++ } ++ ++ verbose "Converting to VDO pool." ++ dry "$LVM" lvconvert $YES $VERB $FORCE --config "$VDO_ALLOCATION_PARAMS" -Zn -V "${vdo_logicalSize}k" -n "$LVNAME" --type vdo-pool "$VGNAME/${LVNAME}_vpool" ++ ++ verbose "Removing now unused VDO entry from VDO configuration." ++ dry "$VDO" remove $VDOCONF $VERB --force --name "$VDONAME" ++} ++ ++# ++# Convert VDO volume on a device to VG with VDOPool LV ++# ++# Convert device with the use of snapshot on top of original VDO volume (can be optionally disabled) ++# Once the whole conversion is finished, snapshot is merged (During the short periof time of merging ++# user must ensure there will be no power-off!) ++# ++# For best use the latest version of vdoprepareforlvm tool is required. ++convert_non_lv_() { ++ local vdo_logicalSize=$1 ++ local extent_size ++ local output ++ local pvfree ++ ++ if [ "$USE_VDO_DM_SNAPSHOT" = "1" ]; then ++ dry snapshot_create_ "$DEVICE" ++ sed "s:$DEVICE:$VDO_DM_SNAPSHOT_DEVICE:" "$TEMPDIR/vdoconf.yml" > "$TEMPDIR/vdo_snap.yml" ++ # Let VDO manager operate on snapshot volume ++ VDOCONF="-f $TEMPDIR/vdo_snap.yml" ++ fi ++ ++ verbose "Moving VDO header." ++ output=$(dry "$VDO" convert $VDOCONF --force --name "$VDONAME") ++ ++ if [ "$ABORT_AFTER_VDO_CONVERT" != "0" ] ; then ++ verbose "Aborting VDO coversion after moving VDO, volume is useless!" ++ cleanup 0 ++ fi ++ ++ # Parse result from VDO preparation/conversion tool ++ # New version of the tool provides output with alingment and offset ++ local vdo_length=0 ++ local vdo_aligned=0 ++ local vdo_offset=0 ++ local vdo_non_converted=0 ++ while IFS= read -r line ; do ++ case "$line" in ++ "Non converted"*) vdo_non_converted=1 ;; ++ "Length"*) vdo_length=${line##* = } ;; ++ "Conversion completed"*) ++ vdo_aligned=${line##*aligned on } ++ vdo_aligned=${vdo_aligned%%[!0-9]*} ++ vdo_offset=${line##*offset } ++ # backward compatibility with report from older version ++ vdo_offset=${vdo_offset##*by } ++ vdo_offset=${vdo_offset%%[!0-9]*} ++ ;; ++ esac ++ done <<< "$output" ++ ++ # In case we operation with snapshot, all lvm2 operation will also run on top of snapshot ++ local devices=${VDO_DM_SNAPSHOT_DEVICE:-$DEVICE} ++ ++ dry "$LVM" pvcreate $YES --devices "$devices" --dataalignment "$vdo_offset"b "$devices" || { ++ error "Creation of PV on \"$DEVICE\" failed, while VDO header has been already moved!" ++ } ++ ++ # Obtain free space in this new PV ++ # after 'vdo convert' call there is ~(1-2)M free space at the front of the device ++ pvfree=$("$BLOCKDEV" --getsize64 "$DEVICE") ++ pvfree=$(( ( pvfree - vdo_offset ) / 1024 )) # to KiB ++ if [ -n "$vdo_aligned" ] && [ "$vdo_aligned" != "0" ]; then ++ extent_size=$(( vdo_aligned / 1024 )) ++ else ++ extent_size=$(get_largest_extent_size_ "$pvfree" "$vdo_logicalSize") ++ fi ++ ++ # Round virtual size to the LOWER size expressed in extent units. ++ # lvm is parsing VDO metadata and can read real full size and use it instead of this smaller value. ++ # To precisely byte-synchronize the size of VDO LV, user can lvresize such VDO LV later. ++ vdo_logicalSize=$(( ( vdo_logicalSize / extent_size ) * extent_size )) ++ ++ verbose "Creating VG \"${NAME%/*}\" with extent size $extent_size KiB." ++ dry "$LVM" vgcreate $YES $VERB --devices "$devices" -s "${extent_size}k" "$VGNAME" "$devices" || { ++ error "Creation of VG \"$VGNAME\" failed, while VDO header has been already moved!" ++ } ++ ++ verbose "Creating VDO pool data LV from all extents in volume group $VGNAME." ++ dry "$LVM" lvcreate -Zn -Wn -an $YES $VERB --devices "$devices" -l100%VG -n "${LVNAME}_vpool" "$VGNAME" "$devices" ++ ++ verbose "Converting to VDO pool." ++ dry "$LVM" lvconvert $YES $VERB $FORCE --devices "$devices" --config "$VDO_ALLOCATION_PARAMS" -Zn -V "${vdo_logicalSize}k" -n "$LVNAME" --type vdo-pool "$VGNAME/${LVNAME}_vpool" ++ ++ dry "$LVM" vgchange -an $VERB $FORCE --devices "$devices" "$VGNAME" ++ ++ if [ "$USE_VDO_DM_SNAPSHOT" = "1" ]; then ++ if [ -z "$YES" ]; then ++ PROMPTING=yes ++ echo "Warning: Do not interrupt merging process once it starts (VDO data may become irrecoverable)!" ++ echo -n "Do you want to merge converted VDO device \"$DEVICE\" to VDO LV \"$VGNAME/$LVNAME\"? [y|N]: " ++ read -r -n 1 -s ANSWER ++ case "${ANSWER:0:1}" in ++ y|Y ) echo "Yes" ;; ++ * ) echo "No" ; PROMPTING=""; cleanup 1 ;; ++ esac ++ PROMPTING="" ++ YES="-y" # From now, now prompting ++ fi ++ ++ dry snapshot_merge_ "$DEVICE" ++ if [ -e "$TEMPDIR/vdo_snap.yml" ]; then ++ dry cp "$TEMPDIR/vdo_snap.yml" "$VDO_CONFIG" ++ else ++ dry rm -f "$VDO_CONFIG" ++ fi ++ verbose "Merging of VDO device finished." ++ fi ++ ++ dry "$LVM" lvchange -ay $VERB $FORCE "$VGNAME/$LVNAME" ++} ++ ++# Convert existing VDO volume into lvm2 volume + convert2lvm_() { +- local DEVICE=$1 +- local VGNAME=${NAME%/*} +- local LVNAME=${NAME#*/} + local VDONAME + local TRVDONAME +- local EXTENTSZ +- local IS_LV=1 + local FOUND="" + local MAJOR=0 + local MINOR=0 +- local DM_VG_NAME +- local DM_LV_NAME + ++ VGNAME=${NAME%/*} ++ LVNAME=${NAME#*/} + DM_UUID="" + detect_lv_ "$DEVICE" + case "$DM_UUID" in + LVM-*) eval "$("$DMSETUP" splitname --nameprefixes --noheadings --separator ' ' "$DM_NAME")" + if [ -z "$VGNAME" ] || [ "$VGNAME" = "$LVNAME" ] ; then + VGNAME=$DM_VG_NAME +- verbose "Using existing volume group name $VGNAME." ++ verbose "Using existing volume group name \"$VGNAME\"." + test -n "$LVNAME" || LVNAME=$DM_LV_NAME +- elif test "$VGNAME" != "$DM_VG_NAME" ; then ++ elif [ "$VGNAME" != "$DM_VG_NAME" ]; then + error "Volume group name \"$VGNAME\" does not match name \"$DM_VG_NAME\" for VDO device \"$DEVICE\"." + fi + ;; +- *) IS_LV=0 ++ *) + # Check if we need to generate unused $VGNANE + if [ -z "$VGNAME" ] || [ "$VGNAME" = "$LVNAME" ] ; then + VGNAME=${DEFAULT_NAME%/*} + # Find largest matching VG name to our 'default' vgname +- LASTVGNAME=$(LC_ALL=C "$LVM" vgs -oname -O-name --noheadings -S name=~${VGNAME} | grep -E "${VGNAME}[0-9]? ?" | head -1 || true) +- if test -n "$LASTVGNAME" ; then ++ LASTVGNAME=$(LC_ALL=C "$LVM" vgs -oname -O-name --noheadings -S name=~"${VGNAME}" | grep -E "${VGNAME}[0-9]? ?" | head -1 || true) ++ if [ -n "$LASTVGNAME" ]; then + LASTVGNAME=${LASTVGNAME#*"${VGNAME}"} + # If the number is becoming too high, try some random number + test "$LASTVGNAME" -gt 99999999 2>/dev/null && LASTVGNAME=$RANDOM + # Generate new unused VG name + VGNAME="${VGNAME}$(( LASTVGNAME + 1 ))" +- verbose "Selected unused volume group name $VGNAME." ++ verbose "Selected unused volume group name \"$VGNAME\"." + fi + fi + # New VG is created, LV name should be always unused. +@@ -269,13 +501,14 @@ convert2lvm_() { + ;; + esac + +- verbose "Checked whether device $1 is already LV ($IS_LV)." ++ verbose "Checked whether device \"$DEVICE\" is already logical volume." + + "$MKDIR" -p -m 0000 "$TEMPDIR" || error "Failed to create $TEMPDIR." + + # TODO: might use directly /etc/vdoconf.yml (avoding need of 'vdo' manager) + verbose "Getting YAML VDO configuration." + "$VDO" printConfigFile $VDOCONF >"$TEMPDIR/vdoconf.yml" ++ test -s "$TEMPDIR/vdoconf.yml" || error "Cannot work without VDO configuration" + + # Check list of devices in VDO configure file for their major:minor + # and match with given $DEVICE devmajor:devminor +@@ -290,8 +523,8 @@ convert2lvm_() { + } + done + +- test -n "$FOUND" || error "Can't find matching device in vdo configuration file." +- verbose "Found matching device $FOUND $MAJOR:$MINOR" ++ test -n "$FOUND" || error "Can't find matching device in VDO configuration file." ++ verbose "Found matching device $FOUND $MAJOR:$MINOR." + + VDONAME=$(awk -v DNAME="$FOUND" '/.*VDOService$/ {VNAME=substr($1, 0, length($1) - 1)} /[[:space:]]*device:/ { if ($2 ~ DNAME) {print VNAME}}' "$TEMPDIR/vdoconf.yml") + TRVDONAME=$(echo "$VDONAME" | tr '-' '_') +@@ -313,7 +546,7 @@ convert2lvm_() { + + verbose "Converted VDO device has logical/physical size $vdo_logicalSize/$vdo_physicalSize KiB." + +- PARAMS=$(cat < +Date: Fri, 9 Jun 2023 13:24:19 +0200 +Subject: [PATCH 104/115] vdo: man updates for lvm_import_vdo + +(cherry picked from commit 17ee5df8576acf628d88fd6840ce55aa1388d6d1) +--- + man/lvm_import_vdo.8_main | 23 +++++++++++++++++++++++ + 1 file changed, 23 insertions(+) + +diff --git a/man/lvm_import_vdo.8_main b/man/lvm_import_vdo.8_main +index 23755a3aa..20be6847d 100644 +--- a/man/lvm_import_vdo.8_main ++++ b/man/lvm_import_vdo.8_main +@@ -61,6 +61,20 @@ Answer "yes" at any prompts. + .BR --dry-run + Print verbosely commands without running them. + . ++.TP ++.BR --no-snapshot ++With this option conversion tool will not use snapshot (COW storage) for conversion ++in your $TMPDIR filesystem and it will directly manipulate with VDO device in-place. ++Warning: the snapshot usage makes the whole conversion transactional and ++the snapshot can be just simply merged once the whole conversion ++has successfully proceeded. In the case of error the snapshot is just removed. ++Without the use of snapshot there is higher risk of problems with recoverability in ++case some unexpected error occurs. ++. ++.TP ++.BR --vdo-config ++Configuration file for VDO manager. Can be used to specify configuration for vdo manager. ++. + . + .SH DIAGNOSTICS + . +@@ -83,6 +97,15 @@ The temporary directory name for mount points. Defaults to "\fI/tmp\fP". + .B DM_DEV_DIR + The device directory name. + Defaults to "\fI/dev\fP" and must be an absolute path. ++.TP ++.B DM_UUID_PREFIX ++Specify uuid prefix for snapshot volume used during vdo conversion. ++.TP ++.B LVM_BINARY ++Allow to overide command called from lvm. Defaults to "\fIlvm\fP". ++.TP ++.B VDO_BINARY ++Allow to overide command called from vdo. Defaults to "\fIvdo\fP". + . + .SH SEE ALSO + . +-- +2.41.0 + diff --git a/SOURCES/0105-tests-vdo-manager-wrapper-tool-updates.patch b/SOURCES/0105-tests-vdo-manager-wrapper-tool-updates.patch new file mode 100644 index 0000000..1bbbb47 --- /dev/null +++ b/SOURCES/0105-tests-vdo-manager-wrapper-tool-updates.patch @@ -0,0 +1,68 @@ +From f38f89313938906cee1ef945847c26f37d7bb7fe Mon Sep 17 00:00:00 2001 +From: Zdenek Kabelac +Date: Fri, 9 Jun 2023 13:24:56 +0200 +Subject: [PATCH 105/115] tests: vdo manager wrapper tool updates + +Support size specification in KiB units. +Add some theoretical support for some new options from vdoprepareforlvm. + +(cherry picked from commit 7f661a24c4f7fa8f161f17da093e456f0d60e106) +--- + test/lib/lvm_vdo_wrapper.sh | 13 +++++++++++-- + 1 file changed, 11 insertions(+), 2 deletions(-) + +diff --git a/test/lib/lvm_vdo_wrapper.sh b/test/lib/lvm_vdo_wrapper.sh +index 90d0b2ce8..131f3e872 100755 +--- a/test/lib/lvm_vdo_wrapper.sh ++++ b/test/lib/lvm_vdo_wrapper.sh +@@ -11,7 +11,7 @@ LVM_VDO_FORMAT=${LVM_VDO_FORMAT-"oldvdoformat"} + # tool for shifting VDO metadata header by 2MiB + LVM_VDO_PREPARE=${LVM_VDO_PREPARE-"oldvdoprepareforlvm"} + # default vdo conf file +-LVM_VDO_DEFAULT_CONF=${LVM_VDO_DEFAULT_CONF-"/tmp/vdoconf.yml"} ++LVM_VDO_DEFAULT_CONF=${LVM_VDO_DEFAULT_CONF-"${TMPDIR:-/tmp}/vdoconf.yml"} + + vdo_die_() { + echo -e "$@" >&2 +@@ -39,6 +39,7 @@ vdo_get_kb_size_with_unit_() { + esac + + case "$1" in ++ *[kK]) sz=1 ;; + *[mM]) sz=1024 ;; + *[gG]) sz=$(( 1024 * 1024 )) ;; + *[tT]) sz=$(( 1024 * 1024 * 1024 )) ;; +@@ -318,6 +319,10 @@ local vdo_force= + local vdo_name= + local vdo_verbose= + local vdo_device= ++local vdo_dry_run= ++local vdo_check= ++local vdo_version= ++local vdo_help= + + while [ "$#" -ne 0 ] + do +@@ -326,6 +331,10 @@ do + "--name"|"-n") shift; vdo_name=$1 ;; + "--verbose"|"-d"|"--debug") vdo_verbose="-v" ;; + "--force") vdo_force="--force" ;; ++ "--dry-run") vdo_dry_run="--dry-run" ;; ++ "--check") vdo_check="--check" ;; ++ "--version") vdo_version="--version" ;; ++ "--help") vdo_help="--help" ;; + esac + shift + done +@@ -337,7 +346,7 @@ vdo_device=$(awk -v vdovolname="$vdo_name" 'BEGIN { have=0 } + "$vdo_confFile") + + #dmsetup status --target vdo "$vdo_name" || true +-vdo_dry_ "$LVM_VDO_PREPARE" "$vdo_device" ++vdo_dry_ "$LVM_VDO_PREPARE" $vdo_dry_run $vdo_check $vdo_version $vdo_help "$vdo_device" + vdo_dry_ vdo_remove_ -f "$vdo_confFile" -n "$vdo_name" || true + } + +-- +2.41.0 + diff --git a/SOURCES/0106-tests-update-test-to-handle-different-status.patch b/SOURCES/0106-tests-update-test-to-handle-different-status.patch new file mode 100644 index 0000000..91fee23 --- /dev/null +++ b/SOURCES/0106-tests-update-test-to-handle-different-status.patch @@ -0,0 +1,40 @@ +From 0fe0f35cda550aa3876126171c119904902ad2b4 Mon Sep 17 00:00:00 2001 +From: Zdenek Kabelac +Date: Fri, 4 Nov 2022 17:00:48 +0100 +Subject: [PATCH 106/115] tests: update test to handle different status + +Since now we change deduplication with V4 table line change, +the modification tends to be faster and we can capture for a few ms +also 'status' about opening or closing deduplication index. +Use 'grep -E' to handle both words. + +(cherry picked from commit 0fed9b097120648301faa586970a47b8b4d629ff) +--- + test/shell/lvchange-vdo.sh | 7 ++++--- + 1 file changed, 4 insertions(+), 3 deletions(-) + +diff --git a/test/shell/lvchange-vdo.sh b/test/shell/lvchange-vdo.sh +index 7cc44d6bc..b11edf35c 100644 +--- a/test/shell/lvchange-vdo.sh ++++ b/test/shell/lvchange-vdo.sh +@@ -40,13 +40,14 @@ check grep_dmsetup status $vg-vdopool-vpool " online online " + + # dedulication_ARG + lvchange --deduplication n $vg/vdopool +-check grep_dmsetup status $vg-vdopool-vpool " offline online " ++check grep_dmsetup status $vg-vdopool-vpool -E " offline|closed online " ++ + lvchange --deduplication y $vg/vdopool +-check grep_dmsetup status $vg-vdopool-vpool " online online " ++check grep_dmsetup status $vg-vdopool-vpool -E " online|opening online " + + + lvchange --compression n --deduplication n $vg/vdopool +-check grep_dmsetup status $vg-vdopool-vpool " offline offline " ++check grep_dmsetup status $vg-vdopool-vpool -E " offline|closed offline " + + # --vdosettings needs inactive LV + not lvchange --vdosettings 'ack_threads=8' $vg/vdopool +-- +2.41.0 + diff --git a/SOURCES/0107-tests-updates.patch b/SOURCES/0107-tests-updates.patch new file mode 100644 index 0000000..ba39993 --- /dev/null +++ b/SOURCES/0107-tests-updates.patch @@ -0,0 +1,158 @@ +From a621c3a6dcd8e08315579f725b6565a583a2be45 Mon Sep 17 00:00:00 2001 +From: Zdenek Kabelac +Date: Fri, 9 Jun 2023 13:28:23 +0200 +Subject: [PATCH 107/115] tests: updates + +Add some more complicated conversion tests. + +(cherry picked from commit 9105890d43bd1b6fe514114f671af2e946e13083) +--- + test/shell/vdo-convert.sh | 67 +++++++++++++++++++++++++++------------ + 1 file changed, 46 insertions(+), 21 deletions(-) + +diff --git a/test/shell/vdo-convert.sh b/test/shell/vdo-convert.sh +index 5bf53d081..cb3dd7100 100644 +--- a/test/shell/vdo-convert.sh ++++ b/test/shell/vdo-convert.sh +@@ -18,10 +18,12 @@ SKIP_WITH_LVMPOLLD=1 + . lib/inittest + + # Use local for this test vdo configuratoin +-VDOCONF="-f vdotestconf.yml" ++VDO_CONFIG="vdotestconf.yml" ++VDOCONF="-f $VDO_CONFIG" + #VDOCONF="" +-export VDOCONF ++export VDOCONF VDO_CONFIG + VDONAME="${PREFIX}-TESTVDO" ++export DM_UUID_PREFIX=$PREFIX + + # VDO automatically starts dmeventd + aux prepare_dmeventd +@@ -44,11 +46,11 @@ export MKE2FS_CONFIG="$TESTDIR/lib/mke2fs.conf" + + aux have_vdo 6 2 0 || skip + +-aux prepare_devs 2 10000 ++aux prepare_devs 2 20000 + + aux extend_filter_LVMTEST + +- ++export TMPDIR=$PWD + + + # Conversion can be made with this version of vdo driver +@@ -59,14 +61,16 @@ aux have_vdo 6 2 5 || skip + # + # In this case we do not need to move any VDO headers. + # ++if [ 1 -eq 0 ]; then + vgcreate $vg "$dev1" + + lvcreate -L5G -n $lv1 $vg + ++# use some not so 'well' aligned virtual|logical size + vdo create $VDOCONF --name "$VDONAME" --device "$DM_DEV_DIR/$vg/$lv1" --vdoSlabSize 128M --vdoLogicalSize 10G + + mkfs -E nodiscard "$DM_DEV_DIR/mapper/$VDONAME" +- ++##XXXXX + # Different VG name fails + not lvm_import_vdo -y -v --name $vg1/$lv1 "$DM_DEV_DIR/$vg/$lv1" + +@@ -130,26 +134,25 @@ vdo create $VDOCONF --name "$VDONAME" --device "$dev1" --vdoSlabSize 128M --vdoL + + mkfs -E nodiscard "$DM_DEV_DIR/mapper/$VDONAME" + +-lvm_import_vdo -y -v --name $vg1/$lv2 "$dev1" ++lvm_import_vdo --vdo-config "$VDO_CONFIG" -y -v --name $vg1/$lv2 "$dev1" + + fsck -n "$DM_DEV_DIR/$vg1/$lv2" + + vgremove -f $vg1 ++fi + +-aux teardown_devs +- +- +-# Check with some real non-DM device from system +-# this needs to dropping DM_DEV_DIR ++aux wipefs_a "$dev1" + +-aux prepare_loop 60000 || skip ++# let's assume users with VDO target have 'new' enough version of stat too ++# otherwise use more universal code from lvm_vdo_import ++read major minor < <(stat -c '%Hr %Lr' $(readlink -e "$dev1")) ++dmsetup create "$PREFIX-vdotest" --table "0 30280004 linear $major:$minor 32" + +-test -f LOOP +-LOOP=$(< LOOP) +-LOOP="${DM_DEV_DIR}/${LOOP##/dev/}" ++TEST="$DM_DEV_DIR/mapper/$PREFIX-vdotest" + +-aux extend_filter "a|$LOOP|" +-aux extend_devices "$LOOP" ++aux wipefs_a "$TEST" ++aux extend_filter "a|$TEST|" ++aux extend_devices "$TEST" + + # + # Unfortunatelly generates this in syslog: +@@ -165,7 +168,9 @@ aux extend_devices "$LOOP" + # + # automate... + # +-vdo create $VDOCONF --name "$VDONAME" --device "$LOOP" --vdoSlabSize 128M --vdoLogicalSize 23G\ ++ ++# use slightly smaller size then 'rounded' 23G - to enforce vdo_logicalSize rounding ++vdo create $VDOCONF --name "$VDONAME" --device "$TEST" --vdoSlabSize 128M --vdoLogicalSize 24117240K\ + --blockMapCacheSize 192 \ + --blockMapPeriod 2048 \ + --emulate512 disabled \ +@@ -179,19 +184,39 @@ vdo create $VDOCONF --name "$VDONAME" --device "$LOOP" --vdoSlabSize 128M --vdoL + --vdoHashZoneThreads 3 \ + --vdoLogicalThreads 3 \ + --writePolicy async-unsafe ++dmsetup table + + # Get VDO table line + dmsetup table "$VDONAME" | tr " " "\n" | sed -e '5,6d' -e '12d' | tee vdo-orig + ++# For conversion we + aux lvmconf 'global/vdo_disabled_features = [ "version4" ]' + +-lvm_import_vdo -y --name $vg/$lv "$LOOP" +-lvs -a $vg ++# ++# Try to prepare 'broken' case where header was moved by older tool to 2M position ++# ++export LVM_VDO_PREPARE=oldvdoprepareforlvm2M ++if which "$LVM_VDO_PREPARE" ; then ++# Use old vdoprepareforlvm tool, that always moves header to 2M offset ++cp "$VDO_CONFIG" "$VDO_CONFIG.backup" ++lvm_import_vdo --abort-after-vdo-convert --vdo-config "$VDO_CONFIG" -v -y --name $vg/$lv "$TEST" ++# Restore VDO configuration (as it's been removed with succeful vdo conversion ++cp "$VDO_CONFIG.backup" "$VDO_CONFIG" ++# Check VDO header is seen at 2M offset ++blkid -c /dev/null --probe --offset 2M "$TEST" ++fi ++unset LVM_VDO_PREPARE ++ ++#lvm_import_vdo --no-snapshot --vdo-config "$VDO_CONFIG" -v -y --name $vg/$lv "$TEST" ++lvm_import_vdo --vdo-config "$VDO_CONFIG" --uuid-prefix "$PREFIX" -v -y --name $vg/$lv "$TEST" + ++# Compare converted LV uses same VDO table line + dmsetup table "$vg-${lv}_vpool-vpool" | tr " " "\n" | sed -e '5,6d' -e '12d' | tee new-vdo-lv + + # Check there is a match between VDO and LV managed volume + # (when differentiating parameters are deleted first) + diff -u vdo-orig new-vdo-lv || die "Found mismatching VDO table lines!" + +-check lv_field $vg/$lv size "23.00g" ++check lv_field $vg/$lv size "<23.00g" ++unset LVM_VDO_PREPARE ++ +-- +2.41.0 + diff --git a/SOURCES/0108-WHATS_NEW-update.patch b/SOURCES/0108-WHATS_NEW-update.patch new file mode 100644 index 0000000..627bf6c --- /dev/null +++ b/SOURCES/0108-WHATS_NEW-update.patch @@ -0,0 +1,28 @@ +From d698509aa858b9da78e9c3719ba80dfcb15fd4cb Mon Sep 17 00:00:00 2001 +From: Zdenek Kabelac +Date: Fri, 9 Jun 2023 17:52:00 +0200 +Subject: [PATCH 108/115] WHATS_NEW: update + +(cherry picked from commit 7ac5dbfd09e771f087a9467aca8a0d5905084459) +--- + WHATS_NEW | 6 +++++- + 1 file changed, 5 insertions(+), 1 deletion(-) + +diff --git a/WHATS_NEW b/WHATS_NEW +index 7f8a16e24..a2bec7e11 100644 +--- a/WHATS_NEW ++++ b/WHATS_NEW +@@ -1,5 +1,9 @@ ++version 2.03.22 - ++================================= ++ Enhance lvm_import_vdo and use snapshot when converting VDO volume. ++ Fix parsing of VDO metadata. ++ + version 2.03.19 - +-==================================== + Fix and improve runtime memory size detection for VDO volumes. + + Version 2.03.17 - +-- +2.41.0 + diff --git a/SOURCES/0109-lvm_import_vdo-correct-the-converted-virtual-size.patch b/SOURCES/0109-lvm_import_vdo-correct-the-converted-virtual-size.patch new file mode 100644 index 0000000..b6fb637 --- /dev/null +++ b/SOURCES/0109-lvm_import_vdo-correct-the-converted-virtual-size.patch @@ -0,0 +1,55 @@ +From 3603348e5035d58aa6c458cdf273760be9af7462 Mon Sep 17 00:00:00 2001 +From: Zdenek Kabelac +Date: Fri, 23 Jun 2023 18:01:32 +0200 +Subject: [PATCH 109/115] lvm_import_vdo: correct the converted virtual size + +Ensure the volume after conversion has the properly aligned size to the +volume group extent size. This would be visible when using virtual size, +that cannot be divided by extent size. + +Before the user had to manually adjust the size after conversion to get +access to all data stored on VDO volume. + +(cherry picked from commit 657dde6208ddc8172b7d0e1466751046c8ddaeaa) +--- + scripts/lvm_import_vdo.sh | 10 ++++++++-- + 1 file changed, 8 insertions(+), 2 deletions(-) + +diff --git a/scripts/lvm_import_vdo.sh b/scripts/lvm_import_vdo.sh +index bae4b52d1..13197c6fb 100755 +--- a/scripts/lvm_import_vdo.sh ++++ b/scripts/lvm_import_vdo.sh +@@ -357,6 +357,7 @@ convert_lv_() { + # For best use the latest version of vdoprepareforlvm tool is required. + convert_non_lv_() { + local vdo_logicalSize=$1 ++ local vdo_logicalSizeRounded + local extent_size + local output + local pvfree +@@ -417,7 +418,7 @@ convert_non_lv_() { + # Round virtual size to the LOWER size expressed in extent units. + # lvm is parsing VDO metadata and can read real full size and use it instead of this smaller value. + # To precisely byte-synchronize the size of VDO LV, user can lvresize such VDO LV later. +- vdo_logicalSize=$(( ( vdo_logicalSize / extent_size ) * extent_size )) ++ vdo_logicalSizeRounded=$(( ( vdo_logicalSize / extent_size ) * extent_size )) + + verbose "Creating VG \"${NAME%/*}\" with extent size $extent_size KiB." + dry "$LVM" vgcreate $YES $VERB --devices "$devices" -s "${extent_size}k" "$VGNAME" "$devices" || { +@@ -428,7 +429,12 @@ convert_non_lv_() { + dry "$LVM" lvcreate -Zn -Wn -an $YES $VERB --devices "$devices" -l100%VG -n "${LVNAME}_vpool" "$VGNAME" "$devices" + + verbose "Converting to VDO pool." +- dry "$LVM" lvconvert $YES $VERB $FORCE --devices "$devices" --config "$VDO_ALLOCATION_PARAMS" -Zn -V "${vdo_logicalSize}k" -n "$LVNAME" --type vdo-pool "$VGNAME/${LVNAME}_vpool" ++ dry "$LVM" lvconvert $YES $VERB $FORCE --devices "$devices" --config "$VDO_ALLOCATION_PARAMS" -Zn -V "${vdo_logicalSizeRounded}k" -n "$LVNAME" --type vdo-pool "$VGNAME/${LVNAME}_vpool" ++ if [ "$vdo_logicalSizeRounded" -lt "$vdo_logicalSize" ] ; then ++ # need to extend virtal size to be covering all the converted area ++ # let lvm2 to round to the proper virtual size of VDO LV ++ dry "$LVM" lvextend $YES $VERB --fs ignore --devices "$devices" -L "$vdo_logicalSize"k "$VGNAME/$LVNAME" ++ fi + + dry "$LVM" vgchange -an $VERB $FORCE --devices "$devices" "$VGNAME" + +-- +2.41.0 + diff --git a/SOURCES/0110-tests-update-test-for-automatic-virtual-size-correct.patch b/SOURCES/0110-tests-update-test-for-automatic-virtual-size-correct.patch new file mode 100644 index 0000000..e623219 --- /dev/null +++ b/SOURCES/0110-tests-update-test-for-automatic-virtual-size-correct.patch @@ -0,0 +1,71 @@ +From 36ef0612d9bc8866dc72c6c9896db0014fe68cd1 Mon Sep 17 00:00:00 2001 +From: Zdenek Kabelac +Date: Fri, 23 Jun 2023 18:03:25 +0200 +Subject: [PATCH 110/115] tests: update test for automatic virtual size + correction + +lvm_import_vdo will adjust the virtual size within conversion +so the volume can be use without any futher tweaking. + +(cherry picked from commit 8d09124c6204d49f6bda833229c96e8a76665127) +--- + test/shell/vdo-convert.sh | 16 ++++++++++++---- + 1 file changed, 12 insertions(+), 4 deletions(-) + +diff --git a/test/shell/vdo-convert.sh b/test/shell/vdo-convert.sh +index cb3dd7100..231939cf4 100644 +--- a/test/shell/vdo-convert.sh ++++ b/test/shell/vdo-convert.sh +@@ -61,7 +61,6 @@ aux have_vdo 6 2 5 || skip + # + # In this case we do not need to move any VDO headers. + # +-if [ 1 -eq 0 ]; then + vgcreate $vg "$dev1" + + lvcreate -L5G -n $lv1 $vg +@@ -139,7 +138,6 @@ lvm_import_vdo --vdo-config "$VDO_CONFIG" -y -v --name $vg1/$lv2 "$dev1" + fsck -n "$DM_DEV_DIR/$vg1/$lv2" + + vgremove -f $vg1 +-fi + + aux wipefs_a "$dev1" + +@@ -189,6 +187,8 @@ dmsetup table + # Get VDO table line + dmsetup table "$VDONAME" | tr " " "\n" | sed -e '5,6d' -e '12d' | tee vdo-orig + ++mkfs.ext4 -E nodiscard "$DM_DEV_DIR/mapper/$VDONAME" ++ + # For conversion we + aux lvmconf 'global/vdo_disabled_features = [ "version4" ]' + +@@ -209,14 +209,22 @@ unset LVM_VDO_PREPARE + + #lvm_import_vdo --no-snapshot --vdo-config "$VDO_CONFIG" -v -y --name $vg/$lv "$TEST" + lvm_import_vdo --vdo-config "$VDO_CONFIG" --uuid-prefix "$PREFIX" -v -y --name $vg/$lv "$TEST" ++dmsetup table ++ ++# check our filesystem is OK ++fsck -n "$DM_DEV_DIR/$vg/$lv" + + # Compare converted LV uses same VDO table line + dmsetup table "$vg-${lv}_vpool-vpool" | tr " " "\n" | sed -e '5,6d' -e '12d' | tee new-vdo-lv + ++tail -n+3 vdo-orig >vdo-orig-3 ++tail -n+3 new-vdo-lv >new-vdo-lv-3 ++ + # Check there is a match between VDO and LV managed volume + # (when differentiating parameters are deleted first) +-diff -u vdo-orig new-vdo-lv || die "Found mismatching VDO table lines!" ++# we need to skip first 2 lines as the device size gets rounded to match VG extent size ++diff -u vdo-orig-3 new-vdo-lv-3 || die "Found mismatching VDO table lines!" + +-check lv_field $vg/$lv size "<23.00g" ++check lv_field $vg/$lv size "23.00g" + unset LVM_VDO_PREPARE + +-- +2.41.0 + diff --git a/SOURCES/0111-vdo-check_point_frequence-is-unused-option.patch b/SOURCES/0111-vdo-check_point_frequence-is-unused-option.patch new file mode 100644 index 0000000..29d3136 --- /dev/null +++ b/SOURCES/0111-vdo-check_point_frequence-is-unused-option.patch @@ -0,0 +1,174 @@ +From e001f919e8cc8f4f0cf9cad304fdf4ff4556e326 Mon Sep 17 00:00:00 2001 +From: Zdenek Kabelac +Date: Wed, 28 Jun 2023 11:55:20 +0200 +Subject: [PATCH 111/115] vdo: check_point_frequence is unused option + +This vdo parameter existed in the early stage of integration of vdo into lvm2, +but later it's been removed from vdoformat tool - so actually if +there would be any non-zero value it would cause error on lvcreate. +Option was not stored on disk in lvm2 metadata. + +Remove this vdo parameter from lvm2 sources. + +(Although this vdo parameter will be still accepted on cmdline through + --vdosettings option, but it will be ignored.) + +(cherry picked from commit 6ff65e675582265a673136f1edd5358b26c9e481) +--- + conf/example.conf.in | 5 ----- + conf/vdo-small.profile | 1 - + device_mapper/vdo/target.h | 1 - + lib/config/config_settings.h | 4 ++-- + lib/metadata/merge.c | 1 - + lib/metadata/vdo_manip.c | 10 ---------- + man/lvmvdo.7_main | 1 - + scripts/lvm_import_vdo.sh | 1 - + tools/toollib.c | 6 +++++- + 9 files changed, 7 insertions(+), 23 deletions(-) + +diff --git a/conf/example.conf.in b/conf/example.conf.in +index c74800676..d5af25061 100644 +--- a/conf/example.conf.in ++++ b/conf/example.conf.in +@@ -650,11 +650,6 @@ allocation { + # This configuration option has an automatic default value. + # vdo_block_map_period = 16380 + +- # Configuration option allocation/vdo_check_point_frequency. +- # The default check point frequency for VDO volume. +- # This configuration option has an automatic default value. +- # vdo_check_point_frequency = 0 +- + # Configuration option allocation/vdo_use_sparse_index. + # Enables sparse indexing for VDO volume. + # This configuration option has an automatic default value. +diff --git a/conf/vdo-small.profile b/conf/vdo-small.profile +index 2044fc27f..97b5b37dd 100644 +--- a/conf/vdo-small.profile ++++ b/conf/vdo-small.profile +@@ -8,7 +8,6 @@ allocation { + vdo_minimum_io_size=4096 + vdo_block_map_cache_size_mb=128 + vdo_block_map_period=16380 +- vdo_check_point_frequency=0 + vdo_use_sparse_index=0 + vdo_index_memory_size_mb=256 + vdo_slab_size_mb=2048 +diff --git a/device_mapper/vdo/target.h b/device_mapper/vdo/target.h +index bd21bb5d7..353320f9e 100644 +--- a/device_mapper/vdo/target.h ++++ b/device_mapper/vdo/target.h +@@ -81,7 +81,6 @@ struct dm_vdo_target_params { + uint32_t block_map_era_length; // format period + uint32_t block_map_period; // supported alias + }; +- uint32_t check_point_frequency; + uint32_t index_memory_size_mb; // format + + uint32_t slab_size_mb; // format +diff --git a/lib/config/config_settings.h b/lib/config/config_settings.h +index a4beda654..3a99ea582 100644 +--- a/lib/config/config_settings.h ++++ b/lib/config/config_settings.h +@@ -729,8 +729,8 @@ cfg(allocation_vdo_block_map_era_length_CFG, "vdo_block_map_period", allocation_ + "The maximum and recommended value is " DM_TO_STRING(DM_VDO_BLOCK_MAP_ERA_LENGTH_MAXIMUM) + "; the minimum value is " DM_TO_STRING(DM_VDO_BLOCK_MAP_ERA_LENGTH_MINIMUM) ".\n") + +-cfg(allocation_vdo_check_point_frequency_CFG, "vdo_check_point_frequency", allocation_CFG_SECTION, CFG_PROFILABLE | CFG_PROFILABLE_METADATA | CFG_DEFAULT_COMMENTED, CFG_TYPE_INT, DEFAULT_VDO_CHECK_POINT_FREQUENCY, VDO_1ST_VSN, NULL, 0, NULL, +- "The default check point frequency for VDO volume.\n") ++cfg(allocation_vdo_check_point_frequency_CFG, "vdo_check_point_frequency", allocation_CFG_SECTION, CFG_PROFILABLE | CFG_PROFILABLE_METADATA | CFG_DEFAULT_COMMENTED, CFG_TYPE_INT, DEFAULT_VDO_CHECK_POINT_FREQUENCY, VDO_1ST_VSN, NULL, vsn(2, 3, 22), NULL, ++ "Deprecated option to set default check point frequency for VDO volume.\n") + + // vdo format + cfg(allocation_vdo_use_sparse_index_CFG, "vdo_use_sparse_index", allocation_CFG_SECTION, CFG_PROFILABLE | CFG_PROFILABLE_METADATA | CFG_DEFAULT_COMMENTED, CFG_TYPE_INT, DEFAULT_VDO_USE_SPARSE_INDEX, VDO_1ST_VSN, NULL, 0, NULL, +diff --git a/lib/metadata/merge.c b/lib/metadata/merge.c +index 5209f51b5..25b2bc33a 100644 +--- a/lib/metadata/merge.c ++++ b/lib/metadata/merge.c +@@ -555,7 +555,6 @@ static void _check_lv_segment(struct logical_volume *lv, struct lv_segment *seg, + if (seg->vdo_params.minimum_io_size | + seg->vdo_params.block_map_cache_size_mb | + seg->vdo_params.block_map_era_length | +- seg->vdo_params.check_point_frequency | + seg->vdo_params.index_memory_size_mb | + seg->vdo_params.slab_size_mb | + seg->vdo_params.max_discard | +diff --git a/lib/metadata/vdo_manip.c b/lib/metadata/vdo_manip.c +index 787101094..952b1e71d 100644 +--- a/lib/metadata/vdo_manip.c ++++ b/lib/metadata/vdo_manip.c +@@ -283,14 +283,6 @@ static int _format_vdo_pool_data_lv(struct logical_volume *data_lv, + argv[args] = buf_args[args]; + args++; + +- if (vtp->check_point_frequency) { +- if (dm_snprintf(buf_args[args], sizeof(buf_args[0]), "--uds-checkpoint-frequency=%u", +- vtp->check_point_frequency) < 0) +- return_0; +- argv[args] = buf_args[args]; +- args++; +- } +- + /* Convert size to GiB units or one of these strings: 0.25, 0.50, 0.75 */ + if (vtp->index_memory_size_mb >= 1024) { + if (dm_snprintf(buf_args[args], sizeof(buf_args[0]), "--uds-memory-size=%u", +@@ -516,8 +508,6 @@ int fill_vdo_target_params(struct cmd_context *cmd, + find_config_tree_int64(cmd, allocation_vdo_block_map_cache_size_mb_CFG, profile); + vtp->block_map_era_length = + find_config_tree_int(cmd, allocation_vdo_block_map_era_length_CFG, profile); +- vtp->check_point_frequency = +- find_config_tree_int(cmd, allocation_vdo_check_point_frequency_CFG, profile); + vtp->use_sparse_index = + find_config_tree_int(cmd, allocation_vdo_use_sparse_index_CFG, profile); + vtp->index_memory_size_mb = +diff --git a/man/lvmvdo.7_main b/man/lvmvdo.7_main +index 8c3e3eeaa..a31e5dd4d 100644 +--- a/man/lvmvdo.7_main ++++ b/man/lvmvdo.7_main +@@ -165,7 +165,6 @@ vdo_use_metadata_hints=1 + vdo_minimum_io_size=4096 + vdo_block_map_cache_size_mb=128 + vdo_block_map_period=16380 +-vdo_check_point_frequency=0 + vdo_use_sparse_index=0 + vdo_index_memory_size_mb=256 + vdo_slab_size_mb=2048 +diff --git a/scripts/lvm_import_vdo.sh b/scripts/lvm_import_vdo.sh +index 13197c6fb..307233429 100755 +--- a/scripts/lvm_import_vdo.sh ++++ b/scripts/lvm_import_vdo.sh +@@ -560,7 +560,6 @@ allocation { + vdo_minimum_io_size = $vdo_logicalBlockSize + vdo_block_map_cache_size_mb = $(( $(get_kb_size_with_unit_ "$vdo_blockMapCacheSize") / 1024 )) + vdo_block_map_period = $vdo_blockMapPeriod +- vdo_check_point_frequency = $vdo_indexCfreq + vdo_use_sparse_index = $(get_enabled_value_ "$vdo_indexSparse") + vdo_index_memory_size_mb = $(awk "BEGIN {print $vdo_indexMemory * 1024}") + vdo_slab_size_mb = $(( $(get_kb_size_with_unit_ "$vdo_slabSize") / 1024 )) +diff --git a/tools/toollib.c b/tools/toollib.c +index 71bf26d39..87207f86f 100644 +--- a/tools/toollib.c ++++ b/tools/toollib.c +@@ -1309,7 +1309,6 @@ int get_vdo_settings(struct cmd_context *cmd, + // Settings bellow cannot be changed with lvchange command + is_lvchange = checked_lvchange; + +- DO_OFFLINE(check_point_frequency); + DO_OFFLINE(index_memory_size_mb); + DO_OFFLINE(minimum_io_size); + DO_OFFLINE(slab_size_mb); +@@ -1326,6 +1325,11 @@ int get_vdo_settings(struct cmd_context *cmd, + continue; + } + ++ if (_compare_vdo_option(cn->key, "check_point_frequency")) { ++ log_verbose("Ignoring deprecated --vdosettings option \"%s\" and its value.", cn->key); ++ continue; /* Accept & ignore deprecated option */ ++ } ++ + log_error("Unknown VDO setting \"%s\".", cn->key); + goto out; + } +-- +2.41.0 + diff --git a/SOURCES/0112-vdo-indent-for-lvdisplay.patch b/SOURCES/0112-vdo-indent-for-lvdisplay.patch new file mode 100644 index 0000000..3548108 --- /dev/null +++ b/SOURCES/0112-vdo-indent-for-lvdisplay.patch @@ -0,0 +1,35 @@ +From eb32a7ca3eac92a1975863a641e40bf30df86df1 Mon Sep 17 00:00:00 2001 +From: Zdenek Kabelac +Date: Wed, 28 Jun 2023 11:53:29 +0200 +Subject: [PATCH 112/115] vdo: indent for lvdisplay + +(cherry picked from commit e48c9826e3069b06a5aa31e1e59200dce4915983) +--- + lib/vdo/vdo.c | 4 ++-- + 1 file changed, 2 insertions(+), 2 deletions(-) + +diff --git a/lib/vdo/vdo.c b/lib/vdo/vdo.c +index 133678ae7..6d3b67421 100644 +--- a/lib/vdo/vdo.c ++++ b/lib/vdo/vdo.c +@@ -173,7 +173,7 @@ static void _vdo_pool_display(const struct lv_segment *seg) + display_size(cmd, vtp->minimum_io_size)); + log_print(" Block map cache sz\t%s", + display_size(cmd, vtp->block_map_cache_size_mb * UINT64_C(2 * 1024))); +- log_print(" Block map era length\t%u", vtp->block_map_era_length); ++ log_print(" Block map era length %u", vtp->block_map_era_length); + + _print_yes_no("Sparse index", vtp->use_sparse_index); + +@@ -190,7 +190,7 @@ static void _vdo_pool_display(const struct lv_segment *seg) + log_print(" # Hash zone threads\t%u", (unsigned) vtp->hash_zone_threads); + log_print(" # Logical threads\t%u", (unsigned) vtp->logical_threads); + log_print(" # Physical threads\t%u", (unsigned) vtp->physical_threads); +- log_print(" Max discard\t%u", (unsigned) vtp->max_discard); ++ log_print(" Max discard\t\t%u", (unsigned) vtp->max_discard); + log_print(" Write policy\t%s", get_vdo_write_policy_name(vtp->write_policy)); + } + +-- +2.41.0 + diff --git a/SOURCES/0113-vdo-lvm_import_vdo-correct-parsing-output.patch b/SOURCES/0113-vdo-lvm_import_vdo-correct-parsing-output.patch new file mode 100644 index 0000000..edabcf1 --- /dev/null +++ b/SOURCES/0113-vdo-lvm_import_vdo-correct-parsing-output.patch @@ -0,0 +1,50 @@ +From d50c0cdcda52494823ee28cc6f88911bf3153ef7 Mon Sep 17 00:00:00 2001 +From: Zdenek Kabelac +Date: Thu, 29 Jun 2023 13:06:18 +0200 +Subject: [PATCH 113/115] vdo: lvm_import_vdo correct parsing output + +Output from vdo manager may actually indent output with spaces, +so trim leading and ending space. + +Also add support for verbosity flag for vdo conversion tool. + +(cherry picked from commit 8b75bbe47d2a2961b0fb029ad7a01dc37ee6a8e5) +--- + scripts/lvm_import_vdo.sh | 7 ++++--- + 1 file changed, 4 insertions(+), 3 deletions(-) + +diff --git a/scripts/lvm_import_vdo.sh b/scripts/lvm_import_vdo.sh +index 307233429..319bcb33d 100755 +--- a/scripts/lvm_import_vdo.sh ++++ b/scripts/lvm_import_vdo.sh +@@ -370,7 +370,7 @@ convert_non_lv_() { + fi + + verbose "Moving VDO header." +- output=$(dry "$VDO" convert $VDOCONF --force --name "$VDONAME") ++ output=$(dry "$VDO" convert $VDOCONF $VERB --force --name "$VDONAME") + + if [ "$ABORT_AFTER_VDO_CONVERT" != "0" ] ; then + verbose "Aborting VDO coversion after moving VDO, volume is useless!" +@@ -384,7 +384,8 @@ convert_non_lv_() { + local vdo_offset=0 + local vdo_non_converted=0 + while IFS= read -r line ; do +- case "$line" in ++ # trim leading spaces ++ case "$(echo $line)" in + "Non converted"*) vdo_non_converted=1 ;; + "Length"*) vdo_length=${line##* = } ;; + "Conversion completed"*) +@@ -579,7 +580,7 @@ EOF + verbose "VDO conversion parameters: $VDO_ALLOCATION_PARAMS" + + verbose "Stopping VDO volume." +- dry "$VDO" stop $VDOCONF --name "$VDONAME" ++ dry "$VDO" stop $VDOCONF --name "$VDONAME" $VERB + + # If user has not provided '--yes', prompt before conversion + if [ -z "$YES" ] && [ "$USE_VDO_DM_SNAPSHOT" != "1" ]; then +-- +2.41.0 + diff --git a/SOURCES/0114-lvm_import_vdo-spelling-fixes.patch b/SOURCES/0114-lvm_import_vdo-spelling-fixes.patch new file mode 100644 index 0000000..704d487 --- /dev/null +++ b/SOURCES/0114-lvm_import_vdo-spelling-fixes.patch @@ -0,0 +1,104 @@ +From e6fc4b2d369d41d004b930f5c8642f63d64b8a22 Mon Sep 17 00:00:00 2001 +From: Zdenek Kabelac +Date: Thu, 29 Jun 2023 13:05:55 +0200 +Subject: [PATCH 114/115] lvm_import_vdo: spelling fixes + +(cherry picked from commit 8a07f57ba478c16ca5d5eacf778ee2708109aa65) +--- + scripts/lvm_import_vdo.sh | 20 ++++++++++---------- + 1 file changed, 10 insertions(+), 10 deletions(-) + +diff --git a/scripts/lvm_import_vdo.sh b/scripts/lvm_import_vdo.sh +index 319bcb33d..b5e1a9e16 100755 +--- a/scripts/lvm_import_vdo.sh ++++ b/scripts/lvm_import_vdo.sh +@@ -52,7 +52,7 @@ DM_DEV_DIR="${DM_DEV_DIR:-/dev}" + DM_UUID_PREFIX="${DM_UUID_PREFIX:-}" + DM_VG_NAME= + DM_LV_NAME= +-VDO_CONFIG=${VDO_CONFIG:-} # can be overriden with --vdo-config ++VDO_CONFIG=${VDO_CONFIG:-} # can be overridden with --vdo-config + VDOCONF= + test -n "$VDO_CONFIG" && VDOCONF="-f $VDO_CONFIG" + +@@ -150,7 +150,7 @@ snapshot_create_() { + + snapshot_merge_() { + local status +- local inital_status ++ local initial_status + + initial_status=( $("$DMSETUP" status "$VDO_DM_SNAPSHOT_NAME") ) + "$DMSETUP" reload "$VDO_DM_SNAPSHOT_NAME" --table "$(snapshot_target_line_ "$1" "$VDO_SNAPSHOT_LOOP" -merge)" +@@ -165,7 +165,7 @@ snapshot_merge_() { + #du -h "$TEMPDIR/$VDO_DM_SNAPSHOT_NAME" + + # Loop for a while, till the snapshot is merged. +- # Should be nearly instantenious. ++ # Should be nearly instantaneous. + # FIXME: Recovery when something prevents merging is hard + for i in $(seq 1 20) ; do + status=( $("$DMSETUP" status "$VDO_DM_SNAPSHOT_NAME") ) +@@ -231,7 +231,7 @@ get_largest_extent_size_() { + } + + # detect LV on the given device +-# dereference device name if it is symbolic link ++# deference device name if it is symbolic link + detect_lv_() { + local DEVICE=$1 + local SYSVOLUME +@@ -315,7 +315,7 @@ parse_yaml_() { + # Currently this enforces a user to reduce the VG extent size to the smaller size (up to 4KiB). + # + # TODO: We may eventually relax this condition just like we are doing rounding for convert_non_lv_() +-# Let's if there would be any singly user requiring this feauture. ++# Let's if there would be any singly user requiring this feature. + # It may allow to better use larger VDO volume size (in TiB ranges). + # + convert_lv_() { +@@ -351,7 +351,7 @@ convert_lv_() { + # Convert VDO volume on a device to VG with VDOPool LV + # + # Convert device with the use of snapshot on top of original VDO volume (can be optionally disabled) +-# Once the whole conversion is finished, snapshot is merged (During the short periof time of merging ++# Once the whole conversion is finished, snapshot is merged (During the short period time of merging + # user must ensure there will be no power-off!) + # + # For best use the latest version of vdoprepareforlvm tool is required. +@@ -373,12 +373,12 @@ convert_non_lv_() { + output=$(dry "$VDO" convert $VDOCONF $VERB --force --name "$VDONAME") + + if [ "$ABORT_AFTER_VDO_CONVERT" != "0" ] ; then +- verbose "Aborting VDO coversion after moving VDO, volume is useless!" ++ verbose "Aborting VDO conversion after moving VDO header, volume is useless!" + cleanup 0 + fi + + # Parse result from VDO preparation/conversion tool +- # New version of the tool provides output with alingment and offset ++ # New version of the tool provides output with alignment and offset + local vdo_length=0 + local vdo_aligned=0 + local vdo_offset=0 +@@ -432,7 +432,7 @@ convert_non_lv_() { + verbose "Converting to VDO pool." + dry "$LVM" lvconvert $YES $VERB $FORCE --devices "$devices" --config "$VDO_ALLOCATION_PARAMS" -Zn -V "${vdo_logicalSizeRounded}k" -n "$LVNAME" --type vdo-pool "$VGNAME/${LVNAME}_vpool" + if [ "$vdo_logicalSizeRounded" -lt "$vdo_logicalSize" ] ; then +- # need to extend virtal size to be covering all the converted area ++ # need to extend virtual size to be covering all the converted area + # let lvm2 to round to the proper virtual size of VDO LV + dry "$LVM" lvextend $YES $VERB --fs ignore --devices "$devices" -L "$vdo_logicalSize"k "$VGNAME/$LVNAME" + fi +@@ -512,7 +512,7 @@ convert2lvm_() { + + "$MKDIR" -p -m 0000 "$TEMPDIR" || error "Failed to create $TEMPDIR." + +- # TODO: might use directly /etc/vdoconf.yml (avoding need of 'vdo' manager) ++ # TODO: might use directly /etc/vdoconf.yml (avoiding need of 'vdo' manager) + verbose "Getting YAML VDO configuration." + "$VDO" printConfigFile $VDOCONF >"$TEMPDIR/vdoconf.yml" + test -s "$TEMPDIR/vdoconf.yml" || error "Cannot work without VDO configuration" +-- +2.41.0 + diff --git a/SOURCES/0115-vdo-support-version-4.patch b/SOURCES/0115-vdo-support-version-4.patch new file mode 100644 index 0000000..b30060d --- /dev/null +++ b/SOURCES/0115-vdo-support-version-4.patch @@ -0,0 +1,90 @@ +From 50b89ccac3d145d0173eed1a72ad15b21e7a26c2 Mon Sep 17 00:00:00 2001 +From: Zdenek Kabelac +Date: Sun, 2 Jul 2023 22:02:18 +0200 +Subject: [PATCH 115/115] vdo: support version 4 + +Properly parse VDO volumes formatted with geometry block version 4.0. + +(cherry picked from commit b90c5d60156c59c33ac440dd5ea7681457684f64) +--- + device_mapper/vdo/vdo_reader.c | 38 ++++++++++++++++++++++++++++------ + 1 file changed, 32 insertions(+), 6 deletions(-) + +diff --git a/device_mapper/vdo/vdo_reader.c b/device_mapper/vdo/vdo_reader.c +index 2451947cd..15d8b7abb 100644 +--- a/device_mapper/vdo/vdo_reader.c ++++ b/device_mapper/vdo/vdo_reader.c +@@ -128,6 +128,14 @@ struct vdo_volume_geometry { + struct vdo_index_config index_config; + } __packed; + ++struct vdo_volume_geometry_4 { ++ uint32_t release_version; ++ uint64_t nonce; ++ uuid_t uuid; ++ struct vdo_volume_region regions[VDO_VOLUME_REGION_COUNT]; ++ struct vdo_index_config index_config; ++} __packed; ++ + /* Decoding mostly only some used stucture members */ + + static void _vdo_decode_version(struct vdo_version_number *v) +@@ -157,6 +165,16 @@ static void _vdo_decode_volume_geometry(struct vdo_volume_geometry *vg) + _vdo_decode_geometry_region(&vg->regions[VDO_DATA_REGION]); + } + ++static void _vdo_decode_volume_geometry_4(struct vdo_volume_geometry *vg, ++ struct vdo_volume_geometry_4 *vg_4) ++{ ++ vg->release_version = le32_to_cpu(vg_4->release_version); ++ vg->nonce = le64_to_cpu(vg_4->nonce); ++ vg->bio_offset = 0; ++ vg->regions[VDO_DATA_REGION] = vg_4->regions[VDO_DATA_REGION]; ++ _vdo_decode_geometry_region(&vg->regions[VDO_DATA_REGION]); ++} ++ + static void _vdo_decode_config(struct vdo_config *vc) + { + vc->logical_blocks = le64_to_cpu(vc->logical_blocks); +@@ -185,6 +203,7 @@ bool dm_vdo_parse_logical_size(const char *vdo_path, uint64_t *logical_blocks) + struct vdo_header h; + struct vdo_version_number vn; + struct vdo_volume_geometry vg; ++ struct vdo_volume_geometry_4 vg_4; + struct vdo_component_41_0 pvc; + + *logical_blocks = 0; +@@ -221,17 +240,24 @@ bool dm_vdo_parse_logical_size(const char *vdo_path, uint64_t *logical_blocks) + memcpy(&h, buffer + MAGIC_NUMBER_SIZE, sizeof(h)); + _vdo_decode_header(&h); + +- if (h.version.major_version != 5) { +- log_debug_activation("Unsupported VDO version %u.%u.", h.version.major_version, h.version.minor_version); ++ if (h.id != 5) { ++ log_debug_activation("Expected geometry VDO block instead of block %u.", h.id); + goto err; + } + +- if (h.id != 5) { +- log_debug_activation("Expected geometry VDO block instead of block %u.", h.id); ++ switch (h.version.major_version) { ++ case 4: ++ memcpy(&vg_4, buffer + MAGIC_NUMBER_SIZE + sizeof(h), sizeof(vg_4)); ++ _vdo_decode_volume_geometry_4(&vg, &vg_4); ++ break; ++ case 5: ++ memcpy(&vg, buffer + MAGIC_NUMBER_SIZE + sizeof(h), sizeof(vg)); ++ _vdo_decode_volume_geometry(&vg); ++ break; ++ default: ++ log_debug_activation("Unsupported VDO version %u.%u.", h.version.major_version, h.version.minor_version); + goto err; + } +- memcpy(&vg, buffer + MAGIC_NUMBER_SIZE + sizeof(h), sizeof(vg)); +- _vdo_decode_volume_geometry(&vg); + + regpos = (vg.regions[VDO_DATA_REGION].start_block - vg.bio_offset) * 4096; + +-- +2.41.0 + diff --git a/SOURCES/0116-vdo-use-long-verbose.patch b/SOURCES/0116-vdo-use-long-verbose.patch new file mode 100644 index 0000000..f916b5c --- /dev/null +++ b/SOURCES/0116-vdo-use-long-verbose.patch @@ -0,0 +1,28 @@ +From adaa7034883c7f237fa35a80e93563ea56ea9362 Mon Sep 17 00:00:00 2001 +From: Zdenek Kabelac +Date: Mon, 17 Jul 2023 16:22:29 +0200 +Subject: [PATCH 1/2] vdo: use long --verbose + +vdo tools needs long option. + +(cherry picked from commit 1c2782354c9ad49f1e2c9af58d7f5085f0632cb5) +--- + scripts/lvm_import_vdo.sh | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/scripts/lvm_import_vdo.sh b/scripts/lvm_import_vdo.sh +index b5e1a9e16..f0e93075a 100755 +--- a/scripts/lvm_import_vdo.sh ++++ b/scripts/lvm_import_vdo.sh +@@ -619,7 +619,7 @@ do + "-f"|"--force" ) FORCE="-f" ;; + "-h"|"--help" ) tool_usage ;; + "-n"|"--name" ) shift; NAME=$1 ;; +- "-v"|"--verbose") VERB="-v" ;; ++ "-v"|"--verbose") VERB="--verbose" ;; + "-y"|"--yes" ) YES="-y" ;; + "--abort-after-vdo-convert" ) ABORT_AFTER_VDO_CONVERT=1; USE_VDO_DM_SNAPSHOT=0 ;; # For testing only + "--dry-run" ) DRY="1" ; VERB="-v" ;; +-- +2.41.0 + diff --git a/SOURCES/0117-vdo-Sync-lvm_import_vdo.sh-with-main-as-of-970e4d295.patch b/SOURCES/0117-vdo-Sync-lvm_import_vdo.sh-with-main-as-of-970e4d295.patch new file mode 100644 index 0000000..ba5e072 --- /dev/null +++ b/SOURCES/0117-vdo-Sync-lvm_import_vdo.sh-with-main-as-of-970e4d295.patch @@ -0,0 +1,542 @@ +From 34255cb85ad476fd5b4ba3f30a53ea3dc084fcbb Mon Sep 17 00:00:00 2001 +From: Marian Csontos +Date: Thu, 7 Sep 2023 15:06:33 +0200 +Subject: [PATCH] vdo: Sync lvm_import_vdo.sh with main as of 970e4d295edd. + +--- + scripts/lvm_import_vdo.sh | 277 ++++++++++++++++++++++++-------------- + 1 file changed, 176 insertions(+), 101 deletions(-) + +diff --git a/scripts/lvm_import_vdo.sh b/scripts/lvm_import_vdo.sh +index f0e93075a..7f3f413e9 100755 +--- a/scripts/lvm_import_vdo.sh ++++ b/scripts/lvm_import_vdo.sh +@@ -36,6 +36,9 @@ TEMPDIR="${TMPDIR:-/tmp}/$IMPORT_NAME" + _SAVEPATH=$PATH + PATH="/sbin:/usr/sbin:/bin:/usr/sbin:$PATH" + ++# Set of trapped signals ++declare -a SIGNALS=("HUP" "INT" "QUIT" "ABRT" "TERM" "EXIT") ++ + # user may override lvm location by setting LVM_BINARY + LVM=${LVM_BINARY:-lvm} + VDO=${VDO_BINARY:-vdo} +@@ -52,7 +55,9 @@ DM_DEV_DIR="${DM_DEV_DIR:-/dev}" + DM_UUID_PREFIX="${DM_UUID_PREFIX:-}" + DM_VG_NAME= + DM_LV_NAME= ++DEFAULT_VDO_CONFIG="/etc/vdoconf.yml" # Default location of vdo's manager config file + VDO_CONFIG=${VDO_CONFIG:-} # can be overridden with --vdo-config ++VDO_CONFIG_RESTORE= + VDOCONF= + test -n "$VDO_CONFIG" && VDOCONF="-f $VDO_CONFIG" + +@@ -61,16 +66,17 @@ VGNAME= + LVNAME= + DEVMAJOR=0 + DEVMINOR=0 +-PROMPTING="" +-USE_VDO_DM_SNAPSHOT=1 ++PROMPTING= ++USE_VDO_DM_SNAPSHOT="--yes" + VDO_DM_SNAPSHOT_NAME= + VDO_DM_SNAPSHOT_DEVICE= + VDO_SNAPSHOT_LOOP= ++VDO_INCONSISTENT= + + DRY=0 +-VERB="" +-FORCE="" +-YES="" ++VERB= ++FORCE= ++YES= + ABORT_AFTER_VDO_CONVERT=0 + VDO_ALLOCATION_PARAMS= + +@@ -78,6 +84,25 @@ VDO_ALLOCATION_PARAMS= + DEFAULT_NAME="vdovg/vdolvol" + NAME="" + ++# predefine empty ++vdo_ackThreads= ++vdo_bioRotationInterval= ++vdo_bioThreads= ++vdo_blockMapCacheSize= ++vdo_blockMapPeriod= ++vdo_compression= ++vdo_cpuThreads= ++vdo_deduplication= ++vdo_hashZoneThreads= ++vdo_indexMemory= ++vdo_indexSparse= ++vdo_logicalBlockSize= ++vdo_logicalThreads= ++vdo_maxDiscardSize= ++vdo_physicalThreads= ++vdo_slabSize= ++vdo_writePolicy= ++ + # help message + tool_usage() { + echo "${TOOL}: Utility to convert VDO volume to VDO LV." +@@ -107,7 +132,11 @@ error() { + for i in "$@" ; do + echo "$TOOL: $i" >&2 + done +- cleanup 1 ++ return 1 ++} ++ ++warn() { ++ echo "$TOOL: WARNING: $i" >&2 + } + + dry() { +@@ -120,15 +149,31 @@ dry() { + } + + cleanup() { +- trap '' 2 ++ RC=$? # Return code + 128 of the last command eg INT=2 + 128 -> 130 + +- test -n "$VDO_DM_SNAPSHOT_NAME" && { "$DMSETUP" remove "$VDO_DM_SNAPSHOT_NAME" || true ; } +- test -n "$VDO_SNAPSHOT_LOOP" && { "$LOSETUP" -d "$VDO_SNAPSHOT_LOOP" || true ; } ++ trap '' "${SIGNALS[@]}" # mute trap for all signals to not interrupt cleanup() on any next signal ++ ++ [ -z "$PROMPTING" ] || echo "No" ++ ++ [ -e "$VDO_CONFIG_RESTORE" ] && { dry cp -a "$VDO_CONFIG_RESTORE" "${VDO_CONFIG:-"$DEFAULT_VDO_CONFIG"}" || true ; } ++ ++ if [ -n "$VDO_DM_SNAPSHOT_NAME" ]; then ++ dry "$LVM" vgchange -an --devices "$VDO_DM_SNAPSHOT_DEVICE" "$VGNAME" &>/dev/null || true ++ for i in {1..20} ; do ++ [ "$(dry "$DMSETUP" info --noheading -co open "$VDO_DM_SNAPSHOT_NAME")" = "0" ] && break ++ sleep .1 ++ done ++ dry "$DMSETUP" remove "$VDO_DM_SNAPSHOT_NAME" &>/dev/null || true ++ fi ++ ++ ++ [ -n "$VDO_SNAPSHOT_LOOP" ] && { dry "$LOSETUP" -d "$VDO_SNAPSHOT_LOOP" || true ; } ++ ++ [ -z "$VDO_INCONSISTENT" ] || echo "$TOOL: VDO volume import process exited unexpectedly!" >&2 + +- test -z "$PROMPTING" || echo "No" + rm -rf "$TEMPDIR" || true +- # error exit status for break +- exit "${1:-1}" ++ ++ exit "$RC" + } + + # Create snapshot target like for persistent snapshot with 16KiB chunksize +@@ -143,7 +188,7 @@ snapshot_create_() { + # TODO: maybe use ramdisk via 'brd' device ?) + "$TRUNCATE" -s 20M "$file" + VDO_SNAPSHOT_LOOP=$("$LOSETUP" -f --show "$file") +- "$DMSETUP" create "$VDO_DM_SNAPSHOT_NAME" -u "${DM_UUID_PREFIX}-${VDO_DM_SNAPSHOT_NAME}-priv" --table "$(snapshot_target_line_ "$1" "$VDO_SNAPSHOT_LOOP")" ++ "$DMSETUP" create "$VDO_DM_SNAPSHOT_NAME" -u "${DM_UUID_PREFIX}${VDO_DM_SNAPSHOT_NAME}-priv" --table "$(snapshot_target_line_ "$1" "$VDO_SNAPSHOT_LOOP")" + VDO_DM_SNAPSHOT_DEVICE="$DM_DEV_DIR/mapper/$VDO_DM_SNAPSHOT_NAME" + verbose "Snapshot of VDO device $1 created: $VDO_DM_SNAPSHOT_DEVICE." + } +@@ -158,7 +203,9 @@ snapshot_merge_() { + error "ABORTING: Failed to initialize snapshot merge! Origin volume is unchanged." + } + +- verbose "Merging converted VDO volume..." ++ verbose "Merging converted VDO volume \"$VDO_DM_SNAPSHOT_NAME\"." ++ VDO_INCONSISTENT=1 ++ + # Running merging + "$DMSETUP" resume "$VDO_DM_SNAPSHOT_NAME" + +@@ -166,30 +213,37 @@ snapshot_merge_() { + + # Loop for a while, till the snapshot is merged. + # Should be nearly instantaneous. +- # FIXME: Recovery when something prevents merging is hard ++ # FIXME: Recovery when something prevents merging is hard + for i in $(seq 1 20) ; do + status=( $("$DMSETUP" status "$VDO_DM_SNAPSHOT_NAME") ) + # Check if merging is finished +- test "${status[3]%/*}" = "${status[4]}" && break ++ [ "${status[3]%/*}" = "${status[4]}" ] && break + # Wait a bit and retry + sleep .2 + done +- test "${status[3]%/*}" = "${status[4]}" || { ++ ++ if [ "${status[3]%/*}" != "${status[4]}" ]; then + # FIXME: Now what shall we do ??? Help.... +- # Keep snapshot in table for possible analysis... ++ # Keep snapshot in DM table for possible analysis... + VDO_DM_SNAPSHOT_NAME= + VDO_SNAPSHOT_LOOP= +- echo "Initial snapshot status ${initial_status[*]}" +- echo "Failing merge snapshot status ${status[*]}" ++ echo "$TOOL: Initial snapshot status ${initial_status[*]}" ++ echo "$TOOL: Failing merge snapshot status ${status[*]}" + error "ABORTING: Snapshot failed to merge! (Administrator required...)" +- } +- sync ++ fi ++ ++ VDO_INCONSISTENT= ++ VDO_CONFIG_RESTORE= ++ ++ verbose "Converted VDO volume is merged to \"$1\"." ++ + "$DMSETUP" remove "$VDO_DM_SNAPSHOT_NAME" || { + sleep 1 # sleep and retry once more + "$DMSETUP" remove "$VDO_DM_SNAPSHOT_NAME" || { + error "ABORTING: Cannot remove snapshot $VDO_DM_SNAPSHOT_NAME! (check volume autoactivation...)" + } + } ++ + VDO_DM_SNAPSHOT_NAME= + "$LOSETUP" -d "$VDO_SNAPSHOT_LOOP" + VDO_SNAPSHOT_LOOP= +@@ -222,9 +276,9 @@ get_largest_extent_size_() { + + for i in 8 16 32 64 128 256 512 1024 2048 4096 ; do + d=$(( $1 / i )) +- test $(( d * i )) -eq "$1" || break ++ [ $(( d * i )) -eq "$1" ] || break + d=$(( $2 / i )) +- test $(( d * i )) -eq "$2" || break ++ [ $(( d * i )) -eq "$2" ] || break + max=$i + done + echo "$max" +@@ -239,7 +293,7 @@ detect_lv_() { + + DEVICE=${1/#"${DM_DEV_DIR}/"/} + DEVICE=$("$READLINK" $READLINK_E "$DM_DEV_DIR/$DEVICE" || true) +- test -n "$DEVICE" || error "Readlink cannot access device \"$1\"." ++ [ -n "$DEVICE" ] || error "Readlink cannot access device \"$1\"." + RDEVICE=$DEVICE + case "$RDEVICE" in + # hardcoded /dev since udev does not create these entries elsewhere +@@ -251,12 +305,12 @@ detect_lv_() { + ;; + *) + RSTAT=$("$STAT" --format "DEVMAJOR=\$((0x%t)) DEVMINOR=\$((0x%T))" "$RDEVICE" || true) +- test -n "$RSTAT" || error "Cannot get major:minor for \"$DEVICE\"." ++ [ -n "$RSTAT" ] || error "Cannot get major:minor for \"$DEVICE\"." + eval "$RSTAT" + ;; + esac + +- test "$DEVMAJOR" != "$(grep device-mapper /proc/devices | cut -f1 -d' ')" && return ++ [ "$DEVMAJOR" != "$(grep device-mapper /proc/devices | cut -f1 -d' ')" ] && return + + DEV="$("$DMSETUP" info -c -j "$DEVMAJOR" -m "$DEVMINOR" -o uuid,name --noheadings --nameprefixes --separator ' ')" + case "$DEV" in +@@ -332,9 +386,10 @@ convert_lv_() { + vg_extent_size=$("$LVM" vgs -o vg_extent_size --units b --nosuffix --noheadings "$VGNAME") + vg_extent_size=$(( vg_extent_size / 1024 )) + +- test "$vg_extent_size" -le "$extent_size" || { ++ [ "$vg_extent_size" -le "$extent_size" ] || { + error "Please vgchange extent_size to at most $extent_size KiB or extend and align virtual size of VDO device on $vg_extent_size KiB before retrying conversion." + } ++ + verbose "Renaming existing LV to be used as _vdata volume for VDO pool LV." + dry "$LVM" lvrename $YES $VERB "$VGNAME/$DM_LV_NAME" "$VGNAME/${LVNAME}_vpool" || { + error "Rename of LV \"$VGNAME/$DM_LV_NAME\" failed, while VDO header has been already moved!" +@@ -362,19 +417,39 @@ convert_non_lv_() { + local output + local pvfree + +- if [ "$USE_VDO_DM_SNAPSHOT" = "1" ]; then ++ if [ -n "$USE_VDO_DM_SNAPSHOT" ]; then + dry snapshot_create_ "$DEVICE" +- sed "s:$DEVICE:$VDO_DM_SNAPSHOT_DEVICE:" "$TEMPDIR/vdoconf.yml" > "$TEMPDIR/vdo_snap.yml" ++ sed "s|$DEVICE|$VDO_DM_SNAPSHOT_DEVICE|" "$TEMPDIR/vdoconf.yml" > "$TEMPDIR/vdo_snap.yml" ++ # In case of error in the middle of conversion restore original config file ++ VDO_CONFIG_RESTORE="$TEMPDIR/vdoconf.yml" + # Let VDO manager operate on snapshot volume +- VDOCONF="-f $TEMPDIR/vdo_snap.yml" ++ dry cp -a "$TEMPDIR/vdo_snap.yml" "${VDO_CONFIG:-"$DEFAULT_VDO_CONFIG"}" ++ else ++ # If error in the following section, report possible problems ahead ++ VDO_INCONSISTENT=1 + fi + +- verbose "Moving VDO header." +- output=$(dry "$VDO" convert $VDOCONF $VERB --force --name "$VDONAME") ++ # In case we operate with snapshot, all lvm2 operation will also run on top of snapshot ++ local device=${VDO_DM_SNAPSHOT_DEVICE:-$DEVICE} ++ ++ # Check if there is not already an existing PV header, this would have fail on pvcreate after conversion ++ "$LVM" pvs --devices "$device" "$device" 2>/dev/null && { ++ error "Cannot convert volume \"$DEVICE\" with existing PV header." ++ } ++ ++ verbose "Moving VDO header on \"$device\"." + +- if [ "$ABORT_AFTER_VDO_CONVERT" != "0" ] ; then +- verbose "Aborting VDO conversion after moving VDO header, volume is useless!" +- cleanup 0 ++ output=$(dry "$VDO" convert $VDOCONF $VERB --force --name "$VDONAME" 2>&1) || { ++ local rc=$? ++ echo "$output" ++ error "Failed to convert VDO volume \"$DEVICE\" (exit code $rc)." ++ } ++ ++ echo "$output" ++ ++ if [ "$ABORT_AFTER_VDO_CONVERT" != "0" ]; then ++ warn "Aborting VDO conversion after moving VDO header, volume is useless!" ++ return 0 + fi + + # Parse result from VDO preparation/conversion tool +@@ -399,12 +474,7 @@ convert_non_lv_() { + esac + done <<< "$output" + +- # In case we operation with snapshot, all lvm2 operation will also run on top of snapshot +- local devices=${VDO_DM_SNAPSHOT_DEVICE:-$DEVICE} +- +- dry "$LVM" pvcreate $YES --devices "$devices" --dataalignment "$vdo_offset"b "$devices" || { +- error "Creation of PV on \"$DEVICE\" failed, while VDO header has been already moved!" +- } ++ dry "$LVM" pvcreate $YES $VERB $FORCE --devices "$device" --dataalignment "$vdo_offset"b "$device" + + # Obtain free space in this new PV + # after 'vdo convert' call there is ~(1-2)M free space at the front of the device +@@ -421,47 +491,53 @@ convert_non_lv_() { + # To precisely byte-synchronize the size of VDO LV, user can lvresize such VDO LV later. + vdo_logicalSizeRounded=$(( ( vdo_logicalSize / extent_size ) * extent_size )) + +- verbose "Creating VG \"${NAME%/*}\" with extent size $extent_size KiB." +- dry "$LVM" vgcreate $YES $VERB --devices "$devices" -s "${extent_size}k" "$VGNAME" "$devices" || { +- error "Creation of VG \"$VGNAME\" failed, while VDO header has been already moved!" +- } ++ verbose "Creating volume group \"$VGNAME\" with the extent size $extent_size KiB." ++ dry "$LVM" vgcreate $YES $VERB --devices "$device" -s "${extent_size}k" "$VGNAME" "$device" + +- verbose "Creating VDO pool data LV from all extents in volume group $VGNAME." +- dry "$LVM" lvcreate -Zn -Wn -an $YES $VERB --devices "$devices" -l100%VG -n "${LVNAME}_vpool" "$VGNAME" "$devices" ++ verbose "Creating VDO pool data LV from all extents in the volume group \"$VGNAME\"." ++ dry "$LVM" lvcreate -Zn -Wn -an $YES $VERB --devices "$device" -l100%VG -n "${LVNAME}_vpool" "$VGNAME" "$device" + + verbose "Converting to VDO pool." +- dry "$LVM" lvconvert $YES $VERB $FORCE --devices "$devices" --config "$VDO_ALLOCATION_PARAMS" -Zn -V "${vdo_logicalSizeRounded}k" -n "$LVNAME" --type vdo-pool "$VGNAME/${LVNAME}_vpool" +- if [ "$vdo_logicalSizeRounded" -lt "$vdo_logicalSize" ] ; then ++ dry "$LVM" lvconvert ${USE_VDO_DM_SNAPSHOT:-"$YES"} $VERB $FORCE --devices "$device" --config "$VDO_ALLOCATION_PARAMS" -Zn -V "${vdo_logicalSizeRounded}k" -n "$LVNAME" --type vdo-pool "$VGNAME/${LVNAME}_vpool" ++ ++ if [ "$vdo_logicalSizeRounded" -lt "$vdo_logicalSize" ]; then + # need to extend virtual size to be covering all the converted area + # let lvm2 to round to the proper virtual size of VDO LV +- dry "$LVM" lvextend $YES $VERB --fs ignore --devices "$devices" -L "$vdo_logicalSize"k "$VGNAME/$LVNAME" ++ dry "$LVM" lvextend $YES $VERB --devices "$device" -L "$vdo_logicalSize"k "$VGNAME/$LVNAME" + fi + +- dry "$LVM" vgchange -an $VERB $FORCE --devices "$devices" "$VGNAME" +- +- if [ "$USE_VDO_DM_SNAPSHOT" = "1" ]; then +- if [ -z "$YES" ]; then +- PROMPTING=yes +- echo "Warning: Do not interrupt merging process once it starts (VDO data may become irrecoverable)!" +- echo -n "Do you want to merge converted VDO device \"$DEVICE\" to VDO LV \"$VGNAME/$LVNAME\"? [y|N]: " +- read -r -n 1 -s ANSWER +- case "${ANSWER:0:1}" in +- y|Y ) echo "Yes" ;; +- * ) echo "No" ; PROMPTING=""; cleanup 1 ;; +- esac +- PROMPTING="" +- YES="-y" # From now, now prompting +- fi ++ VDO_INCONSISTENT= + +- dry snapshot_merge_ "$DEVICE" +- if [ -e "$TEMPDIR/vdo_snap.yml" ]; then +- dry cp "$TEMPDIR/vdo_snap.yml" "$VDO_CONFIG" +- else +- dry rm -f "$VDO_CONFIG" +- fi +- verbose "Merging of VDO device finished." ++ [ -z "$USE_VDO_DM_SNAPSHOT" ] && return # no-snapshot case finished ++ ++ dry "$LVM" vgchange -an $VERB $FORCE --devices "$device" "$VGNAME" ++ ++ # Prevent unwanted auto activation when VG is merged ++ dry "$LVM" vgchange --setautoactivation n $VERB $FORCE --devices "$device" "$VGNAME" ++ ++ if [ -z "$YES" ]; then ++ PROMPTING=yes ++ warn "Do not interrupt merging process once it starts (VDO data may become irrecoverable)!" ++ echo -n "$TOOL: Do you want to merge converted VDO device \"$DEVICE\" to VDO LV \"$VGNAME/$LVNAME\"? [y|N]: " ++ read -r -n 1 -s ANSWER ++ case "${ANSWER:0:1}" in ++ y|Y ) echo "Yes" ;; ++ * ) echo "No" ; PROMPTING=""; return 1 ;; ++ esac ++ PROMPTING="" ++ YES="-y" # From now, now prompting + fi + ++ dry snapshot_merge_ "$DEVICE" ++ ++ # For systems using devicesfile add 'merged' PV into system.devices. ++ # Bypassing use of --valuesonly to keep compatibility with older lvm. ++ local usedev=$("$LVM" lvmconfig --typeconfig full devices/use_devicesfile || true) ++ [ "${usedev#*=}" = "1" ] && dry "$LVM" lvmdevices --adddev "$DEVICE" ++ ++ # Restore auto activation for a VG ++ dry "$LVM" vgchange --setautoactivation y $VERB $FORCE "$VGNAME" ++ + dry "$LVM" lvchange -ay $VERB $FORCE "$VGNAME/$LVNAME" + } + +@@ -479,43 +555,43 @@ convert2lvm_() { + detect_lv_ "$DEVICE" + case "$DM_UUID" in + LVM-*) eval "$("$DMSETUP" splitname --nameprefixes --noheadings --separator ' ' "$DM_NAME")" +- if [ -z "$VGNAME" ] || [ "$VGNAME" = "$LVNAME" ] ; then ++ if [ -z "$VGNAME" ] || [ "$VGNAME" = "$LVNAME" ]; then + VGNAME=$DM_VG_NAME + verbose "Using existing volume group name \"$VGNAME\"." +- test -n "$LVNAME" || LVNAME=$DM_LV_NAME ++ [ -n "$LVNAME" ] || LVNAME=$DM_LV_NAME + elif [ "$VGNAME" != "$DM_VG_NAME" ]; then + error "Volume group name \"$VGNAME\" does not match name \"$DM_VG_NAME\" for VDO device \"$DEVICE\"." + fi + ;; + *) + # Check if we need to generate unused $VGNANE +- if [ -z "$VGNAME" ] || [ "$VGNAME" = "$LVNAME" ] ; then ++ if [ -z "$VGNAME" ] || [ "$VGNAME" = "$LVNAME" ]; then + VGNAME=${DEFAULT_NAME%/*} + # Find largest matching VG name to our 'default' vgname +- LASTVGNAME=$(LC_ALL=C "$LVM" vgs -oname -O-name --noheadings -S name=~"${VGNAME}" | grep -E "${VGNAME}[0-9]? ?" | head -1 || true) ++ LASTVGNAME=$(LC_ALL=C "$LVM" vgs -oname -O-name --noheadings -S name=~"${VGNAME}" | grep -m 1 -E "${VGNAME}[0-9]? ?" || true) + if [ -n "$LASTVGNAME" ]; then + LASTVGNAME=${LASTVGNAME#*"${VGNAME}"} + # If the number is becoming too high, try some random number +- test "$LASTVGNAME" -gt 99999999 2>/dev/null && LASTVGNAME=$RANDOM ++ [ -n "$LASTVGNAME" ] && [ "$LASTVGNAME" -gt 99999999 ] && LASTVGNAME=$RANDOM + # Generate new unused VG name + VGNAME="${VGNAME}$(( LASTVGNAME + 1 ))" + verbose "Selected unused volume group name \"$VGNAME\"." + fi + fi + # New VG is created, LV name should be always unused. +- test -n "$LVNAME" || LVNAME=${DEFAULT_NAME#*/} ++ [ -n "$LVNAME" ] || LVNAME=${DEFAULT_NAME#*/} + "$LVM" vgs "$VGNAME" >/dev/null 2>&1 && error "Cannot use already existing volume group name \"$VGNAME\"." + ;; + esac + + verbose "Checked whether device \"$DEVICE\" is already logical volume." + +- "$MKDIR" -p -m 0000 "$TEMPDIR" || error "Failed to create $TEMPDIR." ++ "$MKDIR" -p -m 0000 "$TEMPDIR" || error "Failed to create \"$TEMPDIR\"." + + # TODO: might use directly /etc/vdoconf.yml (avoiding need of 'vdo' manager) + verbose "Getting YAML VDO configuration." + "$VDO" printConfigFile $VDOCONF >"$TEMPDIR/vdoconf.yml" +- test -s "$TEMPDIR/vdoconf.yml" || error "Cannot work without VDO configuration" ++ [ -s "$TEMPDIR/vdoconf.yml" ] || error "Cannot work without VDO configuration." + + # Check list of devices in VDO configure file for their major:minor + # and match with given $DEVICE devmajor:devminor +@@ -524,13 +600,13 @@ convert2lvm_() { + DEV=$("$READLINK" $READLINK_E "$i") || continue + RSTAT=$("$STAT" --format "MAJOR=\$((0x%t)) MINOR=\$((0x%T))" "$DEV" 2>/dev/null) || continue + eval "$RSTAT" +- test "$MAJOR" = "$DEVMAJOR" && test "$MINOR" = "$DEVMINOR" && { +- test -z "$FOUND" || error "VDO configuration contains duplicate entries $FOUND and $i" ++ if [ "$MAJOR" = "$DEVMAJOR" ] && [ "$MINOR" = "$DEVMINOR" ]; then ++ [ -z "$FOUND" ] || error "VDO configuration contains duplicate entries $FOUND and $i." + FOUND=$i +- } ++ fi + done + +- test -n "$FOUND" || error "Can't find matching device in VDO configuration file." ++ [ -n "$FOUND" ] || error "Can't find matching device in VDO configuration file." + verbose "Found matching device $FOUND $MAJOR:$MINOR." + + VDONAME=$(awk -v DNAME="$FOUND" '/.*VDOService$/ {VNAME=substr($1, 0, length($1) - 1)} /[[:space:]]*device:/ { if ($2 ~ DNAME) {print VNAME}}' "$TEMPDIR/vdoconf.yml") +@@ -541,7 +617,7 @@ convert2lvm_() { + case "$DM_OPEN" in + Device*) ;; # no devices + *) eval "$DM_OPEN" +- test "${DM_OPEN:-0}" -eq 0 || error "Cannot convert in use VDO volume \"$VDONAME\"!" ++ [ "${DM_OPEN:-0}" -eq 0 ] || error "Cannot convert in use VDO volume \"$VDONAME\"!" + ;; + esac + +@@ -583,21 +659,22 @@ EOF + dry "$VDO" stop $VDOCONF --name "$VDONAME" $VERB + + # If user has not provided '--yes', prompt before conversion +- if [ -z "$YES" ] && [ "$USE_VDO_DM_SNAPSHOT" != "1" ]; then ++ if [ -z "$YES" ] && [ -z "$USE_VDO_DM_SNAPSHOT" ]; then + PROMPTING=yes +- echo -n "Convert VDO device \"$DEVICE\" to VDO LV \"$VGNAME/$LVNAME\"? [y|N]: " ++ echo -n "$TOOL: Convert VDO device \"$DEVICE\" to VDO LV \"$VGNAME/$LVNAME\"? [y|N]: " + read -r -n 1 -s ANSWER + case "${ANSWER:0:1}" in + y|Y ) echo "Yes" ;; +- * ) echo "No" ; PROMPTING=""; cleanup 1 ;; ++ * ) echo "No" ; PROMPTING=""; return 1 ;; + esac + PROMPTING="" + YES="-y" # From now, no prompting + fi + + # Make a backup of the existing VDO yaml configuration file +- test -e "$VDO_CONFIG" && dry cp -a "$VDO_CONFIG" "${VDO_CONFIG}.backup" ++ [ -e "$VDO_CONFIG" ] && dry cp -a "$VDO_CONFIG" "${VDO_CONFIG}.backup" + ++ DEVICE=$FOUND + case "$DM_UUID" in + LVM-*) convert_lv_ "$vdo_logicalSize" ;; + *) convert_non_lv_ "$vdo_logicalSize" ;; +@@ -608,9 +685,9 @@ EOF + # start point of this script + # - parsing parameters + ############################# +-trap "cleanup 2" 2 ++trap "cleanup" "${SIGNALS[@]}" + +-test "$#" -eq 0 && tool_usage ++[ "$#" -eq 0 ] && tool_usage + + while [ "$#" -ne 0 ] + do +@@ -621,19 +698,17 @@ do + "-n"|"--name" ) shift; NAME=$1 ;; + "-v"|"--verbose") VERB="--verbose" ;; + "-y"|"--yes" ) YES="-y" ;; +- "--abort-after-vdo-convert" ) ABORT_AFTER_VDO_CONVERT=1; USE_VDO_DM_SNAPSHOT=0 ;; # For testing only +- "--dry-run" ) DRY="1" ; VERB="-v" ;; +- "--no-snapshot" ) USE_VDO_DM_SNAPSHOT=0 ;; +- "--uuid-prefix" ) shift; DM_UUID_PREFIX=$1 ;; # For testing only +- "--vdo-config" ) shift; VDO_CONFIG=$1 ; VDOCONF="-f $VDO_CONFIG" ;; +- "-*") error "Wrong argument \"$1\". (see: $TOOL --help)" ;; ++ "--abort-after-vdo-convert"|"--abortaftervdoconvert" ) ABORT_AFTER_VDO_CONVERT=1; USE_VDO_DM_SNAPSHOT= ;; # For testing only ++ "--dry-run"|"--dryrun" ) DRY="1" ; VERB="-v" ;; ++ "--no-snapshot"|"--nosnapshot" ) USE_VDO_DM_SNAPSHOT= ;; ++ "--uuid-prefix"|"--uuidprefix" ) shift; DM_UUID_PREFIX=$1 ;; # For testing only ++ "--vdo-config"|"--vdoconfig" ) shift; VDO_CONFIG=$1 ; VDOCONF="-f $VDO_CONFIG" ;; ++ -* ) error "Wrong argument \"$1\". (see: $TOOL --help)" ;; + *) DEVICE=$1 ;; # device name does not start with '-' + esac + shift + done + +-test -n "$DEVICE" || error "Device name is not specified. (see: $TOOL --help)" ++[ -n "$DEVICE" ] || error "Device name is not specified. (see: $TOOL --help)" + + convert2lvm_ +- +-cleanup 0 +-- +2.41.0 + diff --git a/SPECS/lvm2.spec b/SPECS/lvm2.spec index 67d6339..4991af7 100644 --- a/SPECS/lvm2.spec +++ b/SPECS/lvm2.spec @@ -55,7 +55,7 @@ %global commit 4dc5d4ac7e7a9457ccc46ff04796b347e58bf4da %global shortcommit %(c=%{commit}; echo ${c:0:7}) %endif -#%%global rel_suffix .test +#%%global rel_suffix .bz2233901_1 # Do not reset Release to 1 unless both lvm2 and device-mapper # versions are increased together. @@ -68,7 +68,7 @@ Version: 2.03.14 %if 0%{?from_snapshot} Release: 0.1.20210426git%{shortcommit}%{?dist}%{?rel_suffix} %else -Release: 9%{?dist}%{?rel_suffix} +Release: 13%{?dist}%{?rel_suffix} %endif License: GPLv2 URL: http://sourceware.org/lvm2 @@ -155,6 +155,57 @@ Patch66: 0065-make-generate.patch Patch67: 0066-vdo-use-single-validator.patch Patch68: 0067-vdo-use-defines-also-for-configuration-defines.patch Patch69: 0068-vdo-report-supported-range-in-error-path.patch +# BZ 2173312: +Patch70: 0069-vdo-ensure-VDO-config-is-removed.patch +Patch71: 0070-vdo-fix-conversion-of-vdo_slab_size_mb.patch +Patch72: 0071-tests-add-lvm_vdo_wrapper.patch +Patch73: 0072-tests-update-for-wrapper.patch +Patch74: 0073-lvmdbusd-Change-unit-test-vdo-minimum-size.patch +Patch75: 0074-vdo-fix-conversion-of-vdo_slab_size_mb-2nd.patch +Patch76: 0075-vdo-update-info-about-memory.patch +Patch77: 0076-vdo-check-vdo-memory-constrains.patch +Patch78: 0077-vdo-add-reformating-to-extent-size-aligned-virtual-s.patch +Patch79: 0078-vdo-support-v4-kernel-target-line.patch +Patch80: 0079-vdo-enhance-lvcreate-validation.patch +Patch81: 0080-vdo-suffle-code-for-better-error-path-handling.patch +Patch82: 0081-vdo-use-only-verbose-log-level-for-reformating.patch +Patch83: 0082-vdo-reset-errno-before-strtoull.patch +Patch84: 0083-vdo-extend-volume-and-pool-without-flush.patch +Patch85: 0084-tests-vdo-emulation-without-vdo-binary.patch +Patch86: 0085-device_mapper-add-parser-for-vdo-metadata.patch +Patch87: 0086-dev_manager-accept-misalined-vdo-pools.patch +Patch88: 0087-device_mapper-vdo-V4-avoid-messaging.patch +Patch89: 0088-vdo-replace-errors-with-debug.patch +Patch90: 0089-vdo-enhance-detection-of-virtual-size.patch +Patch91: 0090-vdo-improve-validation-message.patch +Patch92: 0091-vdo-fix-reader-error-path.patch +Patch93: 0092-vdo-check-memory-only-in-non-critical-section.patch +Patch94: 0093-vdo-read-live-vdo-size-configuration.patch +Patch95: 0094-lvcreate-fix-error-path-return-values.patch +Patch96: 0095-vdo-fix-and-enhance-vdo-constain-checking.patch +Patch97: 0096-vdo-resize-requires-active-vdopool-volume.patch +Patch98: 0097-tests-vdo-resizing.patch +Patch99: 0098-tests-check-failing-vdo-conversion.patch +Patch100: 0099-vdo-use-fixed-size-vdopool-wrapper.patch +Patch101: 0100-vdo-document-feature-option.patch +Patch102: 0101-vdo-fix-and-enhance-vdo-metadata-reader.patch +Patch103: 0102-tests-cleanup-some-shellcheck-warns.patch +Patch104: 0103-vdo-enhance-lvm_vdo_import.patch +Patch105: 0104-vdo-man-updates-for-lvm_import_vdo.patch +Patch106: 0105-tests-vdo-manager-wrapper-tool-updates.patch +Patch107: 0106-tests-update-test-to-handle-different-status.patch +Patch108: 0107-tests-updates.patch +Patch109: 0108-WHATS_NEW-update.patch +Patch110: 0109-lvm_import_vdo-correct-the-converted-virtual-size.patch +Patch111: 0110-tests-update-test-for-automatic-virtual-size-correct.patch +Patch112: 0111-vdo-check_point_frequence-is-unused-option.patch +Patch113: 0112-vdo-indent-for-lvdisplay.patch +Patch114: 0113-vdo-lvm_import_vdo-correct-parsing-output.patch +Patch115: 0114-lvm_import_vdo-spelling-fixes.patch +Patch116: 0115-vdo-support-version-4.patch +Patch117: 0116-vdo-use-long-verbose.patch +# BZ 2233901: +Patch118: 0117-vdo-Sync-lvm_import_vdo.sh-with-main-as-of-970e4d295.patch BuildRequires: gcc %if %{enable_testsuite} @@ -210,79 +261,10 @@ or more physical volumes and creating one or more logical volumes %prep %if 0%{?from_snapshot} -%setup -q -n lvm2-%{commit} +%autosetup -n lvm2-%{commit} %else -%setup -q -n LVM2.%{version} -%patch0 -p1 -b .backup0 +%autosetup -p1 -n LVM2.%{version} %endif -%patch1 -p1 -b .backup1 -%patch2 -p1 -b .backup2 -%patch3 -p1 -b .backup3 -%patch4 -p1 -b .backup4 -%patch5 -p1 -b .backup5 -%patch6 -p1 -b .backup6 -%patch7 -p1 -b .backup7 -%patch8 -p1 -b .backup8 -%patch9 -p1 -b .backup9 -%patch10 -p1 -b .backup10 -%patch11 -p1 -b .backup11 -%patch12 -p1 -b .backup12 -%patch13 -p1 -b .backup13 -%patch14 -p1 -b .backup14 -%patch15 -p1 -b .backup15 -%patch16 -p1 -b .backup16 -#%%patch17 -p1 -b .backup17 -%patch18 -p1 -b .backup18 -%patch19 -p1 -b .backup19 -%patch20 -p1 -b .backup20 -%patch21 -p1 -b .backup21 -%patch22 -p1 -b .backup22 -%patch23 -p1 -b .backup23 -%patch24 -p1 -b .backup24 -%patch25 -p1 -b .backup25 -%patch26 -p1 -b .backup26 -%patch27 -p1 -b .backup27 -%patch28 -p1 -b .backup28 -%patch29 -p1 -b .backup29 -%patch30 -p1 -b .backup30 -%patch31 -p1 -b .backup31 -%patch32 -p1 -b .backup32 -%patch33 -p1 -b .backup33 -%patch34 -p1 -b .backup34 -%patch35 -p1 -b .backup35 -%patch36 -p1 -b .backup36 -%patch37 -p1 -b .backup37 -%patch38 -p1 -b .backup38 -%patch39 -p1 -b .backup39 -%patch40 -p1 -b .backup40 -%patch41 -p1 -b .backup41 -%patch42 -p1 -b .backup42 -%patch43 -p1 -b .backup43 -%patch44 -p1 -b .backup44 -%patch45 -p1 -b .backup45 -%patch46 -p1 -b .backup46 -%patch47 -p1 -b .backup47 -%patch48 -p1 -b .backup48 -%patch49 -p1 -b .backup49 -%patch50 -p1 -b .backup50 -%patch51 -p1 -b .backup51 -%patch52 -p1 -b .backup52 -%patch53 -p1 -b .backup53 -%patch54 -p1 -b .backup54 -%patch55 -p1 -b .backup55 -%patch57 -p1 -b .backup57 -%patch58 -p1 -b .backup58 -%patch59 -p1 -b .backup59 -%patch60 -p1 -b .backup60 -%patch61 -p1 -b .backup61 -%patch62 -p1 -b .backup62 -%patch63 -p1 -b .backup63 -%patch64 -p1 -b .backup64 -%patch65 -p1 -b .backup65 -%patch66 -p1 -b .backup66 -%patch67 -p1 -b .backup67 -%patch68 -p1 -b .backup68 -%patch69 -p1 -b .backup69 %build %global _default_pid_dir /run @@ -901,6 +883,18 @@ An extensive functional testsuite for LVM2. %endif %changelog +* Mon Sep 18 2023 Marian Csontos - 2.03.14-13 +- Fix error paths in lvm_import_vdo.sh. + +* Thu Sep 07 2023 Marian Csontos - 2.03.14-12 +- Fix handling of devices with symlinks in lvm_import_vdo.sh. + +* Thu Jul 27 2023 Marian Csontos - 2.03.14-11 +- Fix verbose option in lvm_import_vdo.sh. + +* Thu Jul 13 2023 Marian Csontos - 2.03.14-10 +- Improve VDO support. + * Thu Dec 08 2022 Marian Csontos - 2.03.14-9 - Fix handling of invalid values passed to vdo_block_map_era_length.