rpms/ostree
7
0
Fork 0
Code Issues Pull Requests Releases Activity
60c277d027
ostree/0001-Support-overlayfs-whiteouts-on-checkout.patch

516 lines
22 KiB
Diff
Raw Normal View History

Auto sync2gitlab import of ostree-2022.2-6.el8.src.rpm
2022-10-20 20:12:12 +00:00
From 0085494e350c72599fc5c0e00422885d80b3c660 Mon Sep 17 00:00:00 2001
From: Miguel Angel Ajo <majopela@redhat.com>
Date: Mon, 19 Sep 2022 17:15:24 +0200
Subject: [PATCH] Support overlayfs whiteouts on checkout
Introduces an intermediate format for overlayfs storage, where
.wh-ostree. prefixed files will be converted into char 0:0
whiteout devices used by overlayfs to mark deletions across layers.
The CI scripts now uses a volume for the scratch directories
previously in /var/tmp otherwise we cannot create whiteout
devices into an overlayfs mounted filesystem.
Related-Issue: #2712
(cherry picked from commit e234b630f85b97e48ecf45d5aaba9b1aa64e6b54)
---
.github/workflows/tests.yml | 8 +-
Makefile-tests.am | 1 +
bash/ostree | 1 +
man/ostree-checkout.xml | 11 ++
src/libostree/ostree-repo-checkout.c | 129 ++++++++++++++++++++-
src/libostree/ostree-repo.h | 5 +-
src/libostree/ostree-sysroot-deploy.c | 2 +-
src/ostree/ot-builtin-checkout.c | 7 +-
tests/archive-test.sh | 7 +-
tests/basic-test.sh | 29 ++++-
tests/kolainst/data-shared/libtest-core.sh | 7 ++
tests/libtest.sh | 52 ++++++++-
tests/test-admin-deploy-whiteouts.sh | 42 +++++++
13 files changed, 292 insertions(+), 9 deletions(-)
create mode 100755 tests/test-admin-deploy-whiteouts.sh
--- a/Makefile-tests.am
+++ b/Makefile-tests.am
@@ -107,6 +107,7 @@ _installed_or_uninstalled_test_scripts = \
tests/test-admin-deploy-nomerge.sh \
tests/test-admin-deploy-none.sh \
tests/test-admin-deploy-bootid-gc.sh \
+ tests/test-admin-deploy-whiteouts.sh \
tests/test-osupdate-dtb.sh \
tests/test-admin-instutil-set-kargs.sh \
tests/test-admin-upgrade-not-backwards.sh \
diff --git a/bash/ostree b/bash/ostree
index 46363315..6f3b86ea 100644
--- a/bash/ostree
+++ b/bash/ostree
@@ -249,6 +249,7 @@ _ostree_checkout() {
--union-identical
--user-mode -U
--whiteouts
+ --process-passthrough-whiteouts
"
local options_with_args="
diff --git a/man/ostree-checkout.xml b/man/ostree-checkout.xml
index 4ed53a91..8f7d4f9b 100644
--- a/man/ostree-checkout.xml
+++ b/man/ostree-checkout.xml
@@ -114,6 +114,17 @@ License along with this library. If not, see <https://www.gnu.org/licenses/>.
</para></listitem>
</varlistentry>
+ <varlistentry>
+ <term><option>--process-passthrough-whiteouts</option></term>
+
+ <listitem><para>
+ Enable overlayfs whiteout extraction into 0:0 character devices.
+ Overlayfs whiteouts are encoded inside ostree as <literal>.ostree-wh.filename</literal>
+ and extracted as 0:0 character devices. This is useful to carry
+ container storage embedded into ostree.
+ </para></listitem>
+ </varlistentry>
+
<varlistentry>
<term><option>--allow-noent</option></term>
diff --git a/src/libostree/ostree-repo-checkout.c b/src/libostree/ostree-repo-checkout.c
index 663292a9..7c7d0cc7 100644
--- a/src/libostree/ostree-repo-checkout.c
+++ b/src/libostree/ostree-repo-checkout.c
@@ -35,6 +35,8 @@
#define WHITEOUT_PREFIX ".wh."
#define OPAQUE_WHITEOUT_NAME ".wh..wh..opq"
+#define OVERLAYFS_WHITEOUT_PREFIX ".ostree-wh."
+
/* Per-checkout call state/caching */
typedef struct {
GString *path_buf; /* buffer for real path if filtering enabled */
@@ -582,6 +584,117 @@ checkout_file_hardlink (OstreeRepo *self,
return TRUE;
}
+static gboolean
+_checkout_overlayfs_whiteout_at_no_overwrite (OstreeRepoCheckoutAtOptions *options,
+ int destination_dfd,
+ const char *destination_name,
+ GFileInfo *file_info,
+ GVariant *xattrs,
+ gboolean *found_exant_file,
+ GCancellable *cancellable,
+ GError **error)
+{
+ if (found_exant_file != NULL)
+ *found_exant_file = FALSE;
+ guint32 file_mode = g_file_info_get_attribute_uint32 (file_info, "unix::mode");
+ if (mknodat(destination_dfd, destination_name, (file_mode & ~S_IFMT) | S_IFCHR, (dev_t)0) < 0)
+ {
+ if (errno == EEXIST && found_exant_file != NULL)
+ {
+ *found_exant_file = TRUE;
+ return TRUE;
+ }
+ return glnx_throw_errno_prefix (error, "Creating whiteout char device");
+ }
+ if (options->mode != OSTREE_REPO_CHECKOUT_MODE_USER)
+ {
+ if (xattrs != NULL &&
+ !glnx_dfd_name_set_all_xattrs(destination_dfd, destination_name, xattrs,
+ cancellable, error))
+ return glnx_throw_errno_prefix (error, "Setting xattrs for whiteout char device");
+
+ if (TEMP_FAILURE_RETRY(fchownat(destination_dfd, destination_name,
+ g_file_info_get_attribute_uint32 (file_info, "unix::uid"),
+ g_file_info_get_attribute_uint32 (file_info, "unix::gid"),
+ AT_SYMLINK_NOFOLLOW) < 0))
+ return glnx_throw_errno_prefix (error, "fchownat");
+ if (TEMP_FAILURE_RETRY (fchmodat (destination_dfd, destination_name, file_mode & ~S_IFMT, 0)) < 0)
+ return glnx_throw_errno_prefix (error, "fchmodat %s to 0%o", destination_name, file_mode & ~S_IFMT);
+ }
+
+ return TRUE;
+}
+
+static gboolean
+_checkout_overlayfs_whiteout_at (OstreeRepo *repo,
+ OstreeRepoCheckoutAtOptions *options,
+ int destination_dfd,
+ const char *destination_name,
+ GFileInfo *file_info,
+ GVariant *xattrs,
+ GCancellable *cancellable,
+ GError **error)
+{
+ gboolean found_exant_file = FALSE;
+ if (!_checkout_overlayfs_whiteout_at_no_overwrite(options, destination_dfd, destination_name,
+ file_info, xattrs,&found_exant_file,
+ cancellable, error))
+ return FALSE;
+
+ if (!found_exant_file)
+ return TRUE;
+
+ guint32 uid = g_file_info_get_attribute_uint32 (file_info, "unix::uid");
+ guint32 gid = g_file_info_get_attribute_uint32 (file_info, "unix::gid");
+ guint32 file_mode = g_file_info_get_attribute_uint32 (file_info, "unix::mode");
+
+ struct stat dest_stbuf;
+
+ switch(options->overwrite_mode)
+ {
+ case OSTREE_REPO_CHECKOUT_OVERWRITE_NONE:
+ return FALSE;
+ case OSTREE_REPO_CHECKOUT_OVERWRITE_UNION_FILES:
+ if (!ot_ensure_unlinked_at (destination_dfd, destination_name, error))
+ return FALSE;
+ return _checkout_overlayfs_whiteout_at_no_overwrite(options, destination_dfd, destination_name,
+ file_info, xattrs, NULL, cancellable, error);
+ case OSTREE_REPO_CHECKOUT_OVERWRITE_ADD_FILES:
+ return TRUE;
+
+ case OSTREE_REPO_CHECKOUT_OVERWRITE_UNION_IDENTICAL:
+ if (!glnx_fstatat(destination_dfd, destination_name, &dest_stbuf, AT_SYMLINK_NOFOLLOW,
+ error))
+ return FALSE;
+ if (!(repo->disable_xattrs || repo->mode == OSTREE_REPO_MODE_BARE_USER_ONLY))
+ {
+ g_autoptr(GVariant) fs_xattrs;
+ if (!glnx_dfd_name_get_all_xattrs (destination_dfd, destination_name,
+ &fs_xattrs, cancellable, error))
+ return FALSE;
+ if (!g_variant_equal(fs_xattrs, xattrs))
+ return glnx_throw(error, "existing destination file %s xattrs don't match",
+ destination_name);
+ }
+ if (options->mode != OSTREE_REPO_CHECKOUT_MODE_USER)
+ {
+ if (gid != dest_stbuf.st_gid)
+ return glnx_throw(error, "existing destination file %s does not match gid %d",
+ destination_name, gid);
+
+ if (uid != dest_stbuf.st_uid)
+ return glnx_throw(error, "existing destination file %s does not match uid %d",
+ destination_name, gid);
+
+ if ((file_mode & ALLPERMS) != (dest_stbuf.st_mode & ALLPERMS))
+ return glnx_throw(error, "existing destination file %s does not match mode %o",
+ destination_name, file_mode);
+ }
+ break;
+ }
+ return TRUE;
+}
+
static gboolean
checkout_one_file_at (OstreeRepo *repo,
OstreeRepoCheckoutAtOptions *options,
@@ -603,7 +716,8 @@ checkout_one_file_at (OstreeRepo *repo,
/* FIXME - avoid the GFileInfo here */
g_autoptr(GFileInfo) source_info = NULL;
- if (!ostree_repo_load_file (repo, checksum, NULL, &source_info, NULL,
+ g_autoptr(GVariant) source_xattrs = NULL;
+ if (!ostree_repo_load_file (repo, checksum, NULL, &source_info, &source_xattrs,
cancellable, error))
return FALSE;
@@ -623,6 +737,7 @@ checkout_one_file_at (OstreeRepo *repo,
const gboolean is_unreadable = (!is_symlink && (source_mode & S_IRUSR) == 0);
const gboolean is_whiteout = (!is_symlink && options->process_whiteouts &&
g_str_has_prefix (destination_name, WHITEOUT_PREFIX));
+ const gboolean is_overlayfs_whiteout = (!is_symlink && g_str_has_prefix (destination_name, OVERLAYFS_WHITEOUT_PREFIX));
const gboolean is_reg_zerosized = (!is_symlink && g_file_info_get_size (source_info) == 0);
const gboolean override_user_unreadable = (options->mode == OSTREE_REPO_CHECKOUT_MODE_USER && is_unreadable);
@@ -643,6 +758,18 @@ checkout_one_file_at (OstreeRepo *repo,
need_copy = FALSE;
}
+ else if (is_overlayfs_whiteout && options->process_passthrough_whiteouts)
+ {
+ const char *name = destination_name + (sizeof (OVERLAYFS_WHITEOUT_PREFIX) - 1);
+
+ if (!name[0])
+ return glnx_throw (error, "Invalid empty overlayfs whiteout '%s'", name);
+
+ g_assert (name[0] != '/'); /* Sanity */
+
+ return _checkout_overlayfs_whiteout_at(repo, options, destination_dfd, name,
+ source_info, source_xattrs, cancellable, error);
+ }
else if (is_reg_zerosized || override_user_unreadable)
{
/* In https://github.com/ostreedev/ostree/commit/673cacd633f9d6b653cdea530657d3e780a41bbd we
diff --git a/src/libostree/ostree-repo.h b/src/libostree/ostree-repo.h
index 98571170..b3d7f986 100644
--- a/src/libostree/ostree-repo.h
+++ b/src/libostree/ostree-repo.h
@@ -989,8 +989,9 @@ typedef struct {
gboolean force_copy; /* Since: 2017.6 */
gboolean bareuseronly_dirs; /* Since: 2017.7 */
gboolean force_copy_zerosized; /* Since: 2018.9 */
- gboolean unused_bools[4];
- /* 4 byte hole on 64 bit */
+ gboolean process_passthrough_whiteouts;
+ gboolean unused_bools[3];
+ /* 3 byte hole on 64 bit */
const char *subpath;
diff --git a/src/libostree/ostree-sysroot-deploy.c b/src/libostree/ostree-sysroot-deploy.c
index 404f336f..5c98103b 100644
--- a/src/libostree/ostree-sysroot-deploy.c
+++ b/src/libostree/ostree-sysroot-deploy.c
@@ -641,7 +641,7 @@ checkout_deployment_tree (OstreeSysroot *sysroot,
return FALSE;
/* Generate hardlink farm, then opendir it */
- OstreeRepoCheckoutAtOptions checkout_opts = { 0, };
+ OstreeRepoCheckoutAtOptions checkout_opts = { .process_passthrough_whiteouts = TRUE };
if (!ostree_repo_checkout_at (repo, &checkout_opts, osdeploy_dfd,
checkout_target_name, csum,
cancellable, error))
diff --git a/src/ostree/ot-builtin-checkout.c b/src/ostree/ot-builtin-checkout.c
index d69c8b0b..bfa43885 100644
--- a/src/ostree/ot-builtin-checkout.c
+++ b/src/ostree/ot-builtin-checkout.c
@@ -37,6 +37,7 @@ static gboolean opt_union;
static gboolean opt_union_add;
static gboolean opt_union_identical;
static gboolean opt_whiteouts;
+static gboolean opt_process_passthrough_whiteouts;
static gboolean opt_from_stdin;
static char *opt_from_file;
static gboolean opt_disable_fsync;
@@ -77,6 +78,7 @@ static GOptionEntry options[] = {
{ "union-add", 0, 0, G_OPTION_ARG_NONE, &opt_union_add, "Keep existing files/directories, only add new", NULL },
{ "union-identical", 0, 0, G_OPTION_ARG_NONE, &opt_union_identical, "When layering checkouts, error out if a file would be replaced with a different version, but add new files and directories", NULL },
{ "whiteouts", 0, 0, G_OPTION_ARG_NONE, &opt_whiteouts, "Process 'whiteout' (Docker style) entries", NULL },
+ { "process-passthrough-whiteouts", 0, 0, G_OPTION_ARG_NONE, &opt_process_passthrough_whiteouts, "Enable overlayfs whiteout extraction into char 0:0 devices", NULL },
{ "allow-noent", 0, 0, G_OPTION_ARG_NONE, &opt_allow_noent, "Do nothing if specified path does not exist", NULL },
{ "from-stdin", 0, 0, G_OPTION_ARG_NONE, &opt_from_stdin, "Process many checkouts from standard input", NULL },
{ "from-file", 0, 0, G_OPTION_ARG_STRING, &opt_from_file, "Process many checkouts from input file", "FILE" },
@@ -129,7 +131,8 @@ process_one_checkout (OstreeRepo *repo,
if (opt_disable_cache || opt_whiteouts || opt_require_hardlinks ||
opt_union_add || opt_force_copy || opt_force_copy_zerosized ||
opt_bareuseronly_dirs || opt_union_identical ||
- opt_skiplist_file || opt_selinux_policy || opt_selinux_prefix)
+ opt_skiplist_file || opt_selinux_policy || opt_selinux_prefix ||
+ opt_process_passthrough_whiteouts)
{
OstreeRepoCheckoutAtOptions checkout_options = { 0, };
@@ -162,6 +165,8 @@ process_one_checkout (OstreeRepo *repo,
}
if (opt_whiteouts)
checkout_options.process_whiteouts = TRUE;
+ if (opt_process_passthrough_whiteouts)
+ checkout_options.process_passthrough_whiteouts = TRUE;
if (subpath)
checkout_options.subpath = subpath;
diff --git a/tests/archive-test.sh b/tests/archive-test.sh
index b6d84979..6b45790e 100644
--- a/tests/archive-test.sh
+++ b/tests/archive-test.sh
@@ -71,6 +71,11 @@ mkdir -p test-overlays
date > test-overlays/overlaid-file
$OSTREE commit ${COMMIT_ARGS} -b test-base --base test2 --owner-uid 42 --owner-gid 42 test-overlays/
$OSTREE ls -R test-base > ls.txt
-assert_streq "$(wc -l < ls.txt)" 14
+if can_create_whiteout_devices; then
+ assert_streq "$(wc -l < ls.txt)" 17
+else
+ assert_streq "$(wc -l < ls.txt)" 14
+fi
+
assert_streq "$(grep '42.*42' ls.txt | wc -l)" 2
echo "ok commit overlay base"
diff --git a/tests/basic-test.sh b/tests/basic-test.sh
index 04506c3d..0878e6f6 100644
--- a/tests/basic-test.sh
+++ b/tests/basic-test.sh
@@ -19,7 +19,7 @@
set -euo pipefail
-echo "1..$((87 + ${extra_basic_tests:-0}))"
+echo "1..$((89 + ${extra_basic_tests:-0}))"
CHECKOUT_U_ARG=""
CHECKOUT_H_ARGS="-H"
@@ -1187,3 +1187,30 @@ if test "$(id -u)" != "0"; then
else
echo "ok # SKIP not run when root"
fi
+
+if ! skip_one_without_whiteouts_devices; then
+ cd ${test_tmpdir}
+ rm checkout-test2 -rf
+ $OSTREE checkout test2 checkout-test2
+
+ assert_not_has_file checkout-test2/whiteouts/whiteout
+ assert_not_has_file checkout-test2/whiteouts/whiteout2
+ assert_has_file checkout-test2/whiteouts/.ostree-wh.whiteout
+ assert_has_file checkout-test2/whiteouts/.ostree-wh.whiteout2
+
+ echo "ok checkout: no whiteout passthrough by default"
+fi
+
+if ! skip_one_without_whiteouts_devices; then
+ cd ${test_tmpdir}
+ rm checkout-test2 -rf
+ $OSTREE checkout --process-passthrough-whiteouts test2 checkout-test2
+
+ assert_not_has_file checkout-test2/whiteouts/.ostree-wh.whiteout
+ assert_not_has_file checkout-test2/whiteouts/.ostree-wh.whiteout2
+
+ assert_is_whiteout_device checkout-test2/whiteouts/whiteout
+ assert_is_whiteout_device checkout-test2/whiteouts/whiteout2
+
+ echo "ok checkout: whiteout with overlayfs passthrough processing"
+fi
diff --git a/tests/libtest.sh b/tests/libtest.sh
index 686f08dc..5830f210 100755
--- a/tests/libtest.sh
+++ b/tests/libtest.sh
@@ -148,6 +148,20 @@ if ! have_selinux_relabel; then
fi
echo done
+# whiteout char 0:0 devices can be created as regular users, but
+# cannot be created inside containers mounted via overlayfs
+can_create_whiteout_devices() {
+ mknod -m 000 ${test_tmpdir}/.test-whiteout c 0 0 || return 1
+ rm -f ${test_tmpdir}/.test-whiteout
+ return 0
+}
+
+echo -n checking for overlayfs whiteouts...
+if ! can_create_whiteout_devices; then
+ export OSTREE_NO_WHITEOUTS=1
+fi
+echo done
+
if test -n "${OT_TESTS_DEBUG:-}"; then
set -x
fi
@@ -245,6 +259,15 @@ setup_test_repository () {
ln -s nonexistent baz/alink
mkdir baz/another/
echo x > baz/another/y
+
+ # if we are running inside a container we cannot test
+ # the overlayfs whiteout marker passthrough
+ if ! test -n "${OSTREE_NO_WHITEOUTS:-}"; then
+ mkdir whiteouts
+ touch whiteouts/.ostree-wh.whiteout
+ touch whiteouts/.ostree-wh.whiteout2
+ chmod 755 whiteouts/.ostree-wh.whiteout2
+ fi
umask "${oldumask}"
cd ${test_tmpdir}/files
@@ -406,7 +429,7 @@ setup_os_repository () {
mkdir osdata
cd osdata
kver=3.6.0
- mkdir -p usr/bin ${bootdir} usr/lib/modules/${kver} usr/share usr/etc
+ mkdir -p usr/bin ${bootdir} usr/lib/modules/${kver} usr/share usr/etc usr/container/layers/abcd
kernel_path=${bootdir}/vmlinuz
initramfs_path=${bootdir}/initramfs.img
# the HMAC file is only in /usr/lib/modules
@@ -449,6 +472,17 @@ EOF
mkdir -p usr/etc/testdirectory
echo "a default daemon file" > usr/etc/testdirectory/test
+ # if we are running inside a container we cannot test
+ # the overlayfs whiteout marker passthrough
+ if ! test -n "${OSTREE_NO_WHITEOUTS:-}"; then
+ # overlayfs whiteout passhthrough marker files
+ touch usr/container/layers/abcd/.ostree-wh.whiteout
+ chmod 400 usr/container/layers/abcd/.ostree-wh.whiteout
+
+ touch usr/container/layers/abcd/.ostree-wh.whiteout2
+ chmod 777 usr/container/layers/abcd/.ostree-wh.whiteout2
+ fi
+
${CMD_PREFIX} ostree --repo=${test_tmpdir}/testos-repo commit ${bootable_flag} --add-metadata-string version=1.0.9 -b testos/buildmain/x86_64-runtime -s "Build"
# Ensure these commits have distinct second timestamps
@@ -588,6 +622,22 @@ skip_without_user_xattrs () {
fi
}
+# Usage: if ! skip_one_without_whiteouts_devices; then ... more tests ...; fi
+skip_one_without_whiteouts_devices() {
+ if ! can_create_whiteout_devices; then
+ echo "ok # SKIP - this test requires whiteout device support (test outside containers)"
+ return 0
+ else
+ return 1
+ fi
+}
+
+skip_without_whiteouts_devices () {
+ if ! can_create_whiteout_devices; then
+ skip "this test requires whiteout device support (test outside containers)"
+ fi
+}
+
_have_systemd_and_libmount=''
have_systemd_and_libmount() {
if test "${_have_systemd_and_libmount}" = ''; then
diff --git a/tests/test-admin-deploy-whiteouts.sh b/tests/test-admin-deploy-whiteouts.sh
new file mode 100755
index 00000000..66421949
--- /dev/null
+++ b/tests/test-admin-deploy-whiteouts.sh
@@ -0,0 +1,42 @@
+#!/bin/bash
+#
+# Copyright (C) 2022 Red Hat, Inc.
+#
+# SPDX-License-Identifier: LGPL-2.0+
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library. If not, see <https://www.gnu.org/licenses/>.
+
+set -euox pipefail
+
+. $(dirname $0)/libtest.sh
+
+skip_without_whiteouts_devices
+
+# Exports OSTREE_SYSROOT so --sysroot not needed.
+setup_os_repository "archive" "syslinux"
+${CMD_PREFIX} ostree --repo=sysroot/ostree/repo pull-local --remote=testos testos-repo testos/buildmain/x86_64-runtime
+
+echo "1..3"
+${CMD_PREFIX} ostree admin deploy --os=testos --karg=root=LABEL=foo --karg=testkarg=1 testos:testos/buildmain/x86_64-runtime
+origdeployment=$(${CMD_PREFIX} ostree admin --sysroot=sysroot --print-current-dir)
+
+assert_is_whiteout_device "${origdeployment}"/usr/container/layers/abcd/whiteout
+echo "ok whiteout deployment"
+
+assert_not_has_file "${origdeployment}"/usr/container/layers/abcd/.ostree-wh.whiteout
+echo "ok .ostree-wh.whiteout not created"
+
+assert_file_has_mode "${origdeployment}"/usr/container/layers/abcd/whiteout 400
+assert_file_has_mode "${origdeployment}"/usr/container/layers/abcd/whiteout2 777
+echo "ok whiteout permissions are preserved"
--
2.37.3
Reference in New Issue Copy Permalink