leapp-repository/0037-overlay-lib-Deprecate-old-ovl-internal-functions-ref.patch

From e4fa8671351a73ddd6b56c70a7834a2c304df9cc Mon Sep 17 00:00:00 2001
From: Petr Stodulka <pstodulk@redhat.com>
Date: Mon, 10 Jul 2023 15:20:24 +0200
Subject: [PATCH 37/42] overlay lib: Deprecate old ovl internal functions
 (refactoring)

We are going to redesign the use of overlay images during the upgrade
to resolve number of issues we have with the old solution. However,
we need to keep the old solution as a fallback (read below). This
is small preparation to keep the new and old code separated safely.

Reasoning for the fallback:
* There is a chance the new solution could raise also some problems
mainly for systems with many partitions/volumes in fstab, or when
they are using many loop devices already - as the new solution will
require to create loop device for each partition/volume noted in
the fstab.
* Also RHEL 7 is going to switch to ELS on Jun 2024 after which the
project will be fixing just critical bugfixes for in-place upgrades.
This problem blocking the upgrade is not considered to be critical.
---
 .../common/libraries/overlaygen.py            | 223 +++++++++---------
 1 file changed, 117 insertions(+), 106 deletions(-)

diff --git a/repos/system_upgrade/common/libraries/overlaygen.py b/repos/system_upgrade/common/libraries/overlaygen.py
index b544f88c..e0d88fe5 100644
--- a/repos/system_upgrade/common/libraries/overlaygen.py
+++ b/repos/system_upgrade/common/libraries/overlaygen.py
@@ -13,15 +13,6 @@ OVERLAY_DO_NOT_MOUNT = ('tmpfs', 'devpts', 'sysfs', 'proc', 'cramfs', 'sysv', 'v
 MountPoints = namedtuple('MountPoints', ['fs_file', 'fs_vfstype'])
 
 
-def _ensure_enough_diskimage_space(space_needed, directory):
-    stat = os.statvfs(directory)
-    if (stat.f_frsize * stat.f_bavail) < (space_needed * 1024 * 1024):
-        message = ('Not enough space available for creating required disk images in {directory}. ' +
-                   'Needed: {space_needed} MiB').format(space_needed=space_needed, directory=directory)
-        api.current_logger().error(message)
-        raise StopActorExecutionError(message)
-
-
 def _get_mountpoints(storage_info):
     mount_points = set()
     for entry in storage_info.fstab:
@@ -43,41 +34,6 @@ def _mount_dir(mounts_dir, mountpoint):
     return os.path.join(mounts_dir, _mount_name(mountpoint))
 
 
-def _prepare_required_mounts(scratch_dir, mounts_dir, mount_points, xfs_info):
-    result = {
-        mount_point.fs_file: mounting.NullMount(
-            _mount_dir(mounts_dir, mount_point.fs_file)) for mount_point in mount_points
-    }
-
-    if not xfs_info.mountpoints_without_ftype:
-        return result
-
-    space_needed = _overlay_disk_size() * len(xfs_info.mountpoints_without_ftype)
-    disk_images_directory = os.path.join(scratch_dir, 'diskimages')
-
-    # Ensure we cleanup old disk images before we check for space constraints.
-    run(['rm', '-rf', disk_images_directory])
-    _create_diskimages_dir(scratch_dir, disk_images_directory)
-    _ensure_enough_diskimage_space(space_needed, scratch_dir)
-
-    mount_names = [mount_point.fs_file for mount_point in mount_points]
-
-    # TODO(pstodulk): this (adding rootfs into the set always) is hotfix for
-    # bz #1911802 (not ideal one..). The problem occurs one rootfs is ext4 fs,
-    # but /var/lib/leapp/... is under XFS without ftype; In such a case we can
-    # see still the very same problems as before. But letting you know that
-    # probably this is not the final solution, as we could possibly see the
-    # same problems on another partitions too (needs to be tested...). However,
-    # it could fit for now until we provide the complete solution around XFS
-    # workarounds (including management of required spaces for virtual FSs per
-    # mountpoints - without that, we cannot fix this properly)
-    for mountpoint in set(xfs_info.mountpoints_without_ftype + ['/']):
-        if mountpoint in mount_names:
-            image = _create_mount_disk_image(disk_images_directory, mountpoint)
-            result[mountpoint] = mounting.LoopMount(source=image, target=_mount_dir(mounts_dir, mountpoint))
-    return result
-
-
 @contextlib.contextmanager
 def _build_overlay_mount(root_mount, mounts):
     if not root_mount:
@@ -96,21 +52,6 @@ def _build_overlay_mount(root_mount, mounts):
                         yield mount
 
 
-def _overlay_disk_size():
-    """
-    Convenient function to retrieve the overlay disk size
-    """
-    try:
-        env_size = os.getenv('LEAPP_OVL_SIZE', default='2048')
-        disk_size = int(env_size)
-    except ValueError:
-        disk_size = 2048
-        api.current_logger().warning(
-            'Invalid "LEAPP_OVL_SIZE" environment variable "%s". Setting default "%d" value', env_size, disk_size
-        )
-    return disk_size
-
-
 def cleanup_scratch(scratch_dir, mounts_dir):
     """
     Function to cleanup the scratch directory
@@ -128,52 +69,6 @@ def cleanup_scratch(scratch_dir, mounts_dir):
     api.current_logger().debug('Recursively removed scratch directory %s.', scratch_dir)
 
 
-def _create_mount_disk_image(disk_images_directory, path):
-    """
-    Creates the mount disk image, for cases when we hit XFS with ftype=0
-    """
-    diskimage_path = os.path.join(disk_images_directory, _mount_name(path))
-    disk_size = _overlay_disk_size()
-
-    api.current_logger().debug('Attempting to create disk image with size %d MiB at %s', disk_size, diskimage_path)
-    utils.call_with_failure_hint(
-        cmd=['/bin/dd', 'if=/dev/zero', 'of={}'.format(diskimage_path), 'bs=1M', 'count={}'.format(disk_size)],
-        hint='Please ensure that there is enough diskspace in {} at least {} MiB are needed'.format(
-            diskimage_path, disk_size)
-    )
-
-    api.current_logger().debug('Creating ext4 filesystem in disk image at %s', diskimage_path)
-    try:
-        utils.call_with_oserror_handled(cmd=['/sbin/mkfs.ext4', '-F', diskimage_path])
-    except CalledProcessError as e:
-        api.current_logger().error('Failed to create ext4 filesystem %s', exc_info=True)
-        raise StopActorExecutionError(
-            message=str(e)
-        )
-
-    return diskimage_path
-
-
-def _create_diskimages_dir(scratch_dir, diskimages_dir):
-    """
-    Prepares directories for disk images
-    """
-    api.current_logger().debug('Creating disk images directory.')
-    try:
-        utils.makedirs(diskimages_dir)
-        api.current_logger().debug('Done creating disk images directory.')
-    except OSError:
-        api.current_logger().error('Failed to create disk images directory %s', diskimages_dir, exc_info=True)
-
-        # This is an attempt for giving the user a chance to resolve it on their own
-        raise StopActorExecutionError(
-            message='Failed to prepare environment for package download while creating directories.',
-            details={
-                'hint': 'Please ensure that {scratch_dir} is empty and modifiable.'.format(scratch_dir=scratch_dir)
-            }
-        )
-
-
 def _create_mounts_dir(scratch_dir, mounts_dir):
     """
     Prepares directories for mounts
@@ -214,7 +109,7 @@ def create_source_overlay(mounts_dir, scratch_dir, xfs_info, storage_info, mount
         scratch_dir=scratch_dir, mounts_dir=mounts_dir))
     try:
         _create_mounts_dir(scratch_dir, mounts_dir)
-        mounts = _prepare_required_mounts(scratch_dir, mounts_dir, _get_mountpoints(storage_info), xfs_info)
+        mounts = _prepare_required_mounts_old(scratch_dir, mounts_dir, _get_mountpoints(storage_info), xfs_info)
         with mounts.pop('/') as root_mount:
             with mounting.OverlayMount(name='system_overlay', source='/', workdir=root_mount.target) as root_overlay:
                 if mount_target:
@@ -228,3 +123,119 @@ def create_source_overlay(mounts_dir, scratch_dir, xfs_info, storage_info, mount
     except Exception:
         cleanup_scratch(scratch_dir, mounts_dir)
         raise
+
+
+# #############################################################################
+# Deprecated OVL solution ...
+# This is going to be removed in future as the whole functionality is going to
+# be replaced by new one. The problem is that the new solution can potentially
+# negatively affect systems with many loop mountpoints, so let's keep this
+# as a workaround for now. I am separating the old and new code in this way
+# to make the future removal easy.
+# IMPORTANT: Before an update of functions above, ensure the functionality of
+# the code below is not affected, otherwise copy the function below with the
+# "_old" suffix.
+# #############################################################################
+def _ensure_enough_diskimage_space_old(space_needed, directory):
+    stat = os.statvfs(directory)
+    if (stat.f_frsize * stat.f_bavail) < (space_needed * 1024 * 1024):
+        message = ('Not enough space available for creating required disk images in {directory}. ' +
+                   'Needed: {space_needed} MiB').format(space_needed=space_needed, directory=directory)
+        api.current_logger().error(message)
+        raise StopActorExecutionError(message)
+
+
+def _overlay_disk_size_old():
+    """
+    Convenient function to retrieve the overlay disk size
+    """
+    try:
+        env_size = os.getenv('LEAPP_OVL_SIZE', default='2048')
+        disk_size = int(env_size)
+    except ValueError:
+        disk_size = 2048
+        api.current_logger().warning(
+            'Invalid "LEAPP_OVL_SIZE" environment variable "%s". Setting default "%d" value', env_size, disk_size
+        )
+    return disk_size
+
+
+def _create_diskimages_dir_old(scratch_dir, diskimages_dir):
+    """
+    Prepares directories for disk images
+    """
+    api.current_logger().debug('Creating disk images directory.')
+    try:
+        utils.makedirs(diskimages_dir)
+        api.current_logger().debug('Done creating disk images directory.')
+    except OSError:
+        api.current_logger().error('Failed to create disk images directory %s', diskimages_dir, exc_info=True)
+
+        # This is an attempt for giving the user a chance to resolve it on their own
+        raise StopActorExecutionError(
+            message='Failed to prepare environment for package download while creating directories.',
+            details={
+                'hint': 'Please ensure that {scratch_dir} is empty and modifiable.'.format(scratch_dir=scratch_dir)
+            }
+        )
+
+
+def _create_mount_disk_image_old(disk_images_directory, path):
+    """
+    Creates the mount disk image, for cases when we hit XFS with ftype=0
+    """
+    diskimage_path = os.path.join(disk_images_directory, _mount_name(path))
+    disk_size = _overlay_disk_size_old()
+
+    api.current_logger().debug('Attempting to create disk image with size %d MiB at %s', disk_size, diskimage_path)
+    utils.call_with_failure_hint(
+        cmd=['/bin/dd', 'if=/dev/zero', 'of={}'.format(diskimage_path), 'bs=1M', 'count={}'.format(disk_size)],
+        hint='Please ensure that there is enough diskspace in {} at least {} MiB are needed'.format(
+            diskimage_path, disk_size)
+    )
+
+    api.current_logger().debug('Creating ext4 filesystem in disk image at %s', diskimage_path)
+    try:
+        utils.call_with_oserror_handled(cmd=['/sbin/mkfs.ext4', '-F', diskimage_path])
+    except CalledProcessError as e:
+        api.current_logger().error('Failed to create ext4 filesystem %s', exc_info=True)
+        raise StopActorExecutionError(
+            message=str(e)
+        )
+
+    return diskimage_path
+
+
+def _prepare_required_mounts_old(scratch_dir, mounts_dir, mount_points, xfs_info):
+    result = {
+        mount_point.fs_file: mounting.NullMount(
+            _mount_dir(mounts_dir, mount_point.fs_file)) for mount_point in mount_points
+    }
+
+    if not xfs_info.mountpoints_without_ftype:
+        return result
+
+    space_needed = _overlay_disk_size_old() * len(xfs_info.mountpoints_without_ftype)
+    disk_images_directory = os.path.join(scratch_dir, 'diskimages')
+
+    # Ensure we cleanup old disk images before we check for space constraints.
+    run(['rm', '-rf', disk_images_directory])
+    _create_diskimages_dir_old(scratch_dir, disk_images_directory)
+    _ensure_enough_diskimage_space_old(space_needed, scratch_dir)
+
+    mount_names = [mount_point.fs_file for mount_point in mount_points]
+
+    # TODO(pstodulk): this (adding rootfs into the set always) is hotfix for
+    # bz #1911802 (not ideal one..). The problem occurs one rootfs is ext4 fs,
+    # but /var/lib/leapp/... is under XFS without ftype; In such a case we can
+    # see still the very same problems as before. But letting you know that
+    # probably this is not the final solution, as we could possibly see the
+    # same problems on another partitions too (needs to be tested...). However,
+    # it could fit for now until we provide the complete solution around XFS
+    # workarounds (including management of required spaces for virtual FSs per
+    # mountpoints - without that, we cannot fix this properly)
+    for mountpoint in set(xfs_info.mountpoints_without_ftype + ['/']):
+        if mountpoint in mount_names:
+            image = _create_mount_disk_image_old(disk_images_directory, mountpoint)
+            result[mountpoint] = mounting.LoopMount(source=image, target=_mount_dir(mounts_dir, mountpoint))
+    return result
-- 
2.41.0