From 47fce173e75408d9a7a26225d389161caf72e244 Mon Sep 17 00:00:00 2001 From: Michal Hecko Date: Sun, 31 Aug 2025 23:49:57 +0200 Subject: [PATCH 04/55] feat(mount_unit_gen): generate mount units for the upgrade initramfs Run systemd-fstab-generator to produce mount units that correspond to the content of source system's fstab. The generated mount units are then modified to mount /target into /sysroot/target, to reflect that the root of the source system is mounted as /sysroot. These mount units are made dependencies of local-fs.target, and, therefore, will be triggered by systemd before the upgrade. Assisted-by: Cursor (Claude Sonnet 4) Jira-ref: RHEL-35446 @pstodulk: Updated the code to cover also other systemd targets that can be covered by systemd-fstab-generator. Also cover the situation when a directory with systemd target (requires, wants) already exists. Tests have been updated. Note that there are still possible issues hidden in the generate mount unit files as we update at this moment just the `Where` clause however we are not touching anything else. (Before, After, RequiresMountsFor, ...). But keeping that for future development and testing. The call for `mount -a` is still present, we expect followup PRs at this point. Co-authored-by: Petr Stodulka --- .../initramfs/mount_units_generator/actor.py | 22 ++ .../libraries/mount_unit_generator.py | 307 ++++++++++++++++++ .../tests/test_mount_unit_generation.py | 269 +++++++++++++++ 3 files changed, 598 insertions(+) create mode 100644 repos/system_upgrade/common/actors/initramfs/mount_units_generator/actor.py create mode 100644 repos/system_upgrade/common/actors/initramfs/mount_units_generator/libraries/mount_unit_generator.py create mode 100644 repos/system_upgrade/common/actors/initramfs/mount_units_generator/tests/test_mount_unit_generation.py diff --git a/repos/system_upgrade/common/actors/initramfs/mount_units_generator/actor.py b/repos/system_upgrade/common/actors/initramfs/mount_units_generator/actor.py new file mode 100644 index 00000000..5fe25515 --- /dev/null +++ b/repos/system_upgrade/common/actors/initramfs/mount_units_generator/actor.py @@ -0,0 +1,22 @@ +from leapp.actors import Actor +from leapp.libraries.actor import mount_unit_generator as mount_unit_generator_lib +from leapp.models import TargetUserSpaceInfo, UpgradeInitramfsTasks +from leapp.tags import InterimPreparationPhaseTag, IPUWorkflowTag + + +class MountUnitGenerator(Actor): + """ + Sets up storage initialization using systemd's mount units in the upgrade container. + """ + + name = 'mount_unit_generator' + consumes = ( + TargetUserSpaceInfo, + ) + produces = ( + UpgradeInitramfsTasks, + ) + tags = (IPUWorkflowTag, InterimPreparationPhaseTag) + + def process(self): + mount_unit_generator_lib.setup_storage_initialization() diff --git a/repos/system_upgrade/common/actors/initramfs/mount_units_generator/libraries/mount_unit_generator.py b/repos/system_upgrade/common/actors/initramfs/mount_units_generator/libraries/mount_unit_generator.py new file mode 100644 index 00000000..e1060559 --- /dev/null +++ b/repos/system_upgrade/common/actors/initramfs/mount_units_generator/libraries/mount_unit_generator.py @@ -0,0 +1,307 @@ +import os +import shutil +import tempfile + +from leapp.exceptions import StopActorExecutionError +from leapp.libraries.common import mounting +from leapp.libraries.stdlib import api, CalledProcessError, run +from leapp.models import TargetUserSpaceInfo, UpgradeInitramfsTasks + + +def run_systemd_fstab_generator(output_directory): + api.current_logger().debug( + 'Generating mount units for the source system into {}'.format(output_directory) + ) + + try: + generator_cmd = [ + '/usr/lib/systemd/system-generators/systemd-fstab-generator', + output_directory, + output_directory, + output_directory + ] + run(generator_cmd) + except CalledProcessError as error: + api.current_logger().error( + 'Failed to generate mount units using systemd-fstab-generator. Error: {}'.format(error) + ) + details = {'details': str(error)} + raise StopActorExecutionError( + 'Failed to generate mount units using systemd-fstab-generator', + details + ) + + api.current_logger().debug( + 'Mount units successfully generated into {}'.format(output_directory) + ) + + +def _read_unit_file_lines(unit_file_path): # Encapsulate IO for tests + with open(unit_file_path) as unit_file: + return unit_file.readlines() + + +def _write_unit_file_lines(unit_file_path, lines): # Encapsulate IO for tests + with open(unit_file_path, 'w') as unit_file: + unit_file.write('\n'.join(lines) + '\n') + + +def _delete_file(file_path): + os.unlink(file_path) + + +def _prefix_mount_unit_with_sysroot(mount_unit_path, new_unit_destination): + """ + Prefix the mount target with /sysroot as expected in the upgrade initramfs. + + A new mount unit file is written to new_unit_destination. + """ + # NOTE(pstodulk): Note that right now we update just the 'Where' key, however + # what about RequiresMountsFor, .. there could be some hidden dragons. + # In case of issues, investigate these values in generated unit files. + api.current_logger().debug( + 'Prefixing {}\'s mount target with /sysroot. Output will be written to {}'.format( + mount_unit_path, + new_unit_destination + ) + ) + unit_lines = _read_unit_file_lines(mount_unit_path) + + output_lines = [] + for line in unit_lines: + line = line.strip() + if not line.startswith('Where='): + output_lines.append(line) + continue + + _, destination = line.split('=', 1) + new_destination = os.path.join('/sysroot', destination.lstrip('/')) + + output_lines.append('Where={}'.format(new_destination)) + + _write_unit_file_lines(new_unit_destination, output_lines) + + api.current_logger().debug( + 'Done. Modified mount unit successfully written to {}'.format(new_unit_destination) + ) + + +def prefix_all_mount_units_with_sysroot(dir_containing_units): + for unit_file_path in os.listdir(dir_containing_units): + # systemd requires mount path to be in the unit name + modified_unit_destination = 'sysroot-{}'.format(unit_file_path) + modified_unit_destination = os.path.join(dir_containing_units, modified_unit_destination) + + unit_file_path = os.path.join(dir_containing_units, unit_file_path) + + if not unit_file_path.endswith('.mount'): + api.current_logger().debug( + 'Skipping {} when prefixing mount units with /sysroot - not a mount unit.'.format( + unit_file_path + ) + ) + continue + + _prefix_mount_unit_with_sysroot(unit_file_path, modified_unit_destination) + + _delete_file(unit_file_path) + api.current_logger().debug('Original mount unit {} removed.'.format(unit_file_path)) + + +def _fix_symlinks_in_dir(dir_containing_mount_units, target_dir): + """ + Fix broken symlinks in given target_dir due to us modifying (renaming) the mount units. + + The target_dir contains symlinks to the (mount) units that are required + in order for the local-fs.target to be reached. However, we renamed these units to reflect + that we have changed their mount destinations by prefixing the mount destination with /sysroot. + Hence, we regenerate the symlinks. + """ + + target_dir_path = os.path.join(dir_containing_mount_units, target_dir) + if not os.path.exists(target_dir_path): + api.current_logger().debug( + 'The {} directory does not exist. Skipping' + .format(target_dir) + ) + return + + api.current_logger().debug( + 'Removing the old {} directory from {}.' + .format(target_dir, dir_containing_mount_units) + ) + + shutil.rmtree(target_dir_path) + os.mkdir(target_dir_path) + + api.current_logger().debug('Populating {} with new symlinks.'.format(target_dir)) + + for unit_file in os.listdir(dir_containing_mount_units): + if not unit_file.endswith('.mount'): + continue + + place_fastlink_at = os.path.join(target_dir_path, unit_file) + fastlink_points_to = os.path.join('../', unit_file) + try: + run(['ln', '-s', fastlink_points_to, place_fastlink_at]) + + api.current_logger().debug( + 'Dependency on {} created.'.format(unit_file) + ) + except CalledProcessError as err: + err_descr = ( + 'Failed to create required unit dependencies under {} for the upgrade initramfs.' + .format(target_dir) + ) + details = {'details': str(err)} + raise StopActorExecutionError(err_descr, details=details) + + +def fix_symlinks_in_targets(dir_containing_mount_units): + """ + Fix broken symlinks in *.target.* directories caused by earlier modified mount units. + + Generated mount unit files are part of one of systemd targets (list below), + which means that a symlink from a systemd target to exists for each of + them. Based on this, systemd knows when (local or remote file systems?) + they must (".requires" suffix") or could (".wants" suffix) be mounted. + See the man 5 systemd.mount for more details how mount units are split into + these targets. + + The list of possible target directories where these mount units could end: + * local-fs.target.requires + * local-fs.target.wants + * local-fs-pre.target.requires + * local-fs-pre.target.wants + * remote-fs.target.requires + * remote-fs.target.wants + * remote-fs-pre.target.requires + * remote-fs-pre.target.wants + Most likely, unit files are not generated for "*pre*" targets, but to be + sure really. Longer list does not cause any issues in this code. + + In most cases, "local-fs.target.requires" is the only important directory + for us during the upgrade. But in some (sometimes common) cases we will + need some of the others as well. + + These directories do not have to necessarily exists if there are no mount + unit files that could be put there. But most likely "local-fs.target.requires" + will always exists. + """ + dir_list = [ + 'local-fs.target.requires', + 'local-fs.target.wants', + 'local-fs-pre.target.requires', + 'local-fs-pre.target.wants', + 'remote-fs.target.requires', + 'remote-fs.target.wants', + 'remote-fs-pre.target.requires', + 'remote-fs-pre.target.wants', + ] + for tdir in dir_list: + _fix_symlinks_in_dir(dir_containing_mount_units, tdir) + + +def copy_units_into_system_location(upgrade_container_ctx, dir_with_our_mount_units): + """ + Copy units and their .wants/.requires directories into the target userspace container. + + :return: A list of files in the target userspace that were created by copying. + :rtype: list[str] + """ + dest_inside_container = '/usr/lib/systemd/system' + + api.current_logger().debug( + 'Copying generated mount units for upgrade from {} to {}'.format( + dir_with_our_mount_units, + upgrade_container_ctx.full_path(dest_inside_container) + ) + ) + + copied_files = [] + prefix_len_to_drop = len(upgrade_container_ctx.base_dir) + + # We cannot rely on mounting library when copying into container + # as we want to control what happens to symlinks and + # shutil.copytree in Python3.6 fails if dst directory exists already + # - which happens in some cases when copying these files. + for root, dummy_dirs, files in os.walk(dir_with_our_mount_units): + rel_path = os.path.relpath(root, dir_with_our_mount_units) + if rel_path == '.': + rel_path = '' + dst_dir = os.path.join(upgrade_container_ctx.full_path(dest_inside_container), rel_path) + os.makedirs(dst_dir, mode=0o755, exist_ok=True) + + for file in files: + src_file = os.path.join(root, file) + dst_file = os.path.join(dst_dir, file) + api.current_logger().debug( + 'Copying mount unit file {} to {}'.format(src_file, dst_file) + ) + if os.path.islink(dst_file): + # If the target file already exists and it is a symlink, it will + # fail and we want to overwrite this. + # NOTE(pstodulk): You could think that it cannot happen, but + # in future possibly it could happen, so let's rather be careful + # and handle it. If the dst file exists, we want to overwrite it + # for sure + _delete_file(dst_file) + shutil.copy2(src_file, dst_file, follow_symlinks=False) + copied_files.append(dst_file[prefix_len_to_drop:]) + + return copied_files + + +def remove_units_for_targets_that_are_already_mounted_by_dracut(dir_with_our_mount_units): + """ + Remove mount units for mount targets that are already mounted by dracut. + + Namely, remove mount units: + '-.mount' (mounts /) + 'usr.mount' (mounts /usr) + """ + + # NOTE: remount-fs.service creates dependency cycles that are nondeterministically broken + # by systemd, causing unpredictable failures. The service is supposed to remount root + # and /usr, reapplying mount options from /etc/fstab. However, the fstab file present in + # the initramfs is not the fstab from the source system, and, therefore, it is pointless + # to require the service. It would make sense after we switched root during normal boot + # process. + already_mounted_units = [ + '-.mount', + 'usr.mount', + 'local-fs.target.wants/systemd-remount-fs.service' + ] + + for unit in already_mounted_units: + unit_location = os.path.join(dir_with_our_mount_units, unit) + + if not os.path.exists(unit_location): + api.current_logger().debug('The {} unit does not exists, no need to remove it.'.format(unit)) + continue + + _delete_file(unit_location) + + +def request_units_inclusion_in_initramfs(files_to_include): + api.current_logger().debug('Including the following files into initramfs: {}'.format(files_to_include)) + + additional_files = [ + '/usr/sbin/swapon' # If the system has swap, we have also generated a swap unit to activate it + ] + + tasks = UpgradeInitramfsTasks(include_files=files_to_include + additional_files) + api.produce(tasks) + + +def setup_storage_initialization(): + userspace_info = next(api.consume(TargetUserSpaceInfo), None) + + with mounting.NspawnActions(base_dir=userspace_info.path) as upgrade_container_ctx: + with tempfile.TemporaryDirectory(dir='/var/lib/leapp/', prefix='tmp_systemd_fstab_') as workspace_path: + run_systemd_fstab_generator(workspace_path) + remove_units_for_targets_that_are_already_mounted_by_dracut(workspace_path) + prefix_all_mount_units_with_sysroot(workspace_path) + fix_symlinks_in_targets(workspace_path) + mount_unit_files = copy_units_into_system_location(upgrade_container_ctx, workspace_path) + request_units_inclusion_in_initramfs(mount_unit_files) diff --git a/repos/system_upgrade/common/actors/initramfs/mount_units_generator/tests/test_mount_unit_generation.py b/repos/system_upgrade/common/actors/initramfs/mount_units_generator/tests/test_mount_unit_generation.py new file mode 100644 index 00000000..b814f6ce --- /dev/null +++ b/repos/system_upgrade/common/actors/initramfs/mount_units_generator/tests/test_mount_unit_generation.py @@ -0,0 +1,269 @@ +import os +import shutil + +import pytest + +from leapp.exceptions import StopActorExecutionError +from leapp.libraries.actor import mount_unit_generator +from leapp.libraries.common.testutils import logger_mocked +from leapp.libraries.stdlib import api, CalledProcessError +from leapp.models import TargetUserSpaceInfo, UpgradeInitramfsTasks + + +def test_run_systemd_fstab_generator_successful_generation(monkeypatch): + """Test successful mount unit generation.""" + + output_dir = '/tmp/test_output' + expected_cmd = [ + '/usr/lib/systemd/system-generators/systemd-fstab-generator', + output_dir, + output_dir, + output_dir + ] + + def mock_run(command): + assert command == expected_cmd + + return { + "stdout": "", + "stderr": "", + "exit_code": 0, + } + + monkeypatch.setattr(mount_unit_generator, 'run', mock_run) + mount_unit_generator.run_systemd_fstab_generator(output_dir) + + +def test_run_systemd_fstab_generator_failure(monkeypatch): + """Test handling of systemd-fstab-generator failure.""" + output_dir = '/tmp/test_output' + expected_cmd = [ + '/usr/lib/systemd/system-generators/systemd-fstab-generator', + output_dir, + output_dir, + output_dir + ] + + def mock_run(command): + assert command == expected_cmd + raise CalledProcessError(message='Generator failed', command=['test'], result={'exit_code': 1}) + + monkeypatch.setattr(mount_unit_generator, 'run', mock_run) + monkeypatch.setattr(api, 'current_logger', logger_mocked()) + + with pytest.raises(StopActorExecutionError): + mount_unit_generator.run_systemd_fstab_generator(output_dir) + + +def test_prefix_mount_unit_with_sysroot(monkeypatch): + """Test prefixing a single mount unit with /sysroot.""" + monkeypatch.setattr(api, 'current_logger', logger_mocked()) + + input_content = [ + "[Unit]\n", + "Description=Test Mount\n", + "[Mount]\n", + "Where=/home\n", + "What=/dev/sda1\n" + ] + + expected_output_lines = [ + "[Unit]", + "Description=Test Mount", + "[Mount]", + "Where=/sysroot/home", + "What=/dev/sda1" + ] + + def mock_read_unit_file_lines(unit_file_path): + return input_content + + def mock_write_unit_file_lines(unit_file_path, lines): + assert unit_file_path == '/test/output.mount' + assert lines == expected_output_lines + + monkeypatch.setattr(mount_unit_generator, '_read_unit_file_lines', mock_read_unit_file_lines) + monkeypatch.setattr(mount_unit_generator, '_write_unit_file_lines', mock_write_unit_file_lines) + + mount_unit_generator._prefix_mount_unit_with_sysroot( + '/test/input.mount', + '/test/output.mount' + ) + + +def test_prefix_all_mount_units_with_sysroot(monkeypatch): + """Test prefixing all mount units in a directory.""" + + expected_changes = { + '/test/dir/home.mount': { + 'new_unit_destination': '/test/dir/sysroot-home.mount', + 'should_be_deleted': True, + 'deleted': False, + }, + '/test/dir/var.mount': { + 'new_unit_destination': '/test/dir/sysroot-var.mount', + 'should_be_deleted': True, + 'deleted': False, + }, + '/test/dir/not-a-mount.service': { + 'new_unit_destination': None, + 'should_be_deleted': False, + 'deleted': False, + } + } + + def mock_listdir(dir_path): + return ['home.mount', 'var.mount', 'not-a-mount.service'] + + def mock_delete_file(file_path): + assert file_path in expected_changes + expected_changes[file_path]['deleted'] = True + + def mock_prefix(unit_file_path, new_unit_destination): + assert expected_changes[unit_file_path]['new_unit_destination'] == new_unit_destination + + monkeypatch.setattr('os.listdir', mock_listdir) + monkeypatch.setattr(mount_unit_generator, '_delete_file', mock_delete_file) + monkeypatch.setattr(mount_unit_generator, '_prefix_mount_unit_with_sysroot', mock_prefix) + + mount_unit_generator.prefix_all_mount_units_with_sysroot('/test/dir') + + for original_mount_unit_location in expected_changes: + should_be_deleted = expected_changes[original_mount_unit_location]['should_be_deleted'] + was_deleted = expected_changes[original_mount_unit_location]['deleted'] + assert should_be_deleted == was_deleted + + +@pytest.mark.parametrize('dirname', ( + 'local-fs.target.requires', + 'local-fs.target.wants', + 'local-fs-pre.target.requires', + 'local-fs-pre.target.wants', + 'remote-fs.target.requires', + 'remote-fs.target.wants', + 'remote-fs-pre.target.requires', + 'remote-fs-pre.target.wants', +)) +def test_fix_symlinks_in_dir(monkeypatch, dirname): + """Test fixing local-fs.target.requires symlinks.""" + + DIR_PATH = os.path.join('/test/dir/', dirname) + + def mock_rmtree(dir_path): + assert dir_path == DIR_PATH + + def mock_mkdir(dir_path): + assert dir_path == DIR_PATH + + def mock_listdir(dir_path): + return ['sysroot-home.mount', 'sysroot-var.mount', 'not-a-mount.service'] + + def mock_os_path_exist(dir_path): + assert dir_path == DIR_PATH + return dir_path == DIR_PATH + + expected_calls = [ + ['ln', '-s', '../sysroot-home.mount', os.path.join(DIR_PATH, 'sysroot-home.mount')], + ['ln', '-s', '../sysroot-var.mount', os.path.join(DIR_PATH, 'sysroot-var.mount')] + ] + call_count = 0 + + def mock_run(command): + nonlocal call_count + assert command in expected_calls + call_count += 1 + return { + "stdout": "", + "stderr": "", + "exit_code": 0, + } + + monkeypatch.setattr('shutil.rmtree', mock_rmtree) + monkeypatch.setattr('os.mkdir', mock_mkdir) + monkeypatch.setattr('os.listdir', mock_listdir) + monkeypatch.setattr('os.path.exists', mock_os_path_exist) + monkeypatch.setattr(mount_unit_generator, 'run', mock_run) + + mount_unit_generator._fix_symlinks_in_dir('/test/dir', dirname) + + +# Test the copy_units_into_system_location function +def test_copy_units_mixed_content(monkeypatch): + """Test copying units with mixed files and directories.""" + + def mock_walk(dir_path): + tuples_to_yield = [ + ('/source/dir', ['local-fs.target.requires'], ['unit1.mount', 'unit2.mount']), + ('/source/dir/local-fs.target.requires', [], ['unit1.mount', 'unit2.mount']), + ] + for i in tuples_to_yield: + yield i + + def mock_isdir(path): + return 'local-fs.target.requires' in path + + def _make_couple(sub_path): + return ( + os.path.join('/source/dir/', sub_path), + os.path.join('/container/usr/lib/systemd/system/', sub_path) + ) + + def mock_copy2(src, dst, follow_symlinks=True): + valid_combinations = [ + _make_couple('unit1.mount'), + _make_couple('unit2.mount'), + _make_couple('local-fs.target.requires/unit1.mount'), + _make_couple('local-fs.target.requires/unit2.mount'), + ] + assert not follow_symlinks + assert (src, dst) in valid_combinations + + def mock_islink(file_path): + return file_path == '/container/usr/lib/systemd/system/local-fs.target.requires/unit2.mount' + + class MockedDeleteFile: + def __init__(self): + self.removal_called = False + + def __call__(self, file_path): + assert file_path == '/container/usr/lib/systemd/system/local-fs.target.requires/unit2.mount' + self.removal_called = True + + def mock_makedirs(dst_dir, mode=0o777, exist_ok=False): + assert exist_ok + assert mode == 0o755 + + allowed_paths = [ + '/container/usr/lib/systemd/system', + '/container/usr/lib/systemd/system/local-fs.target.requires' + ] + assert dst_dir.rstrip('/') in allowed_paths + + monkeypatch.setattr(os, 'walk', mock_walk) + monkeypatch.setattr(os, 'makedirs', mock_makedirs) + monkeypatch.setattr(os.path, 'isdir', mock_isdir) + monkeypatch.setattr(os.path, 'islink', mock_islink) + monkeypatch.setattr(mount_unit_generator, '_delete_file', MockedDeleteFile()) + monkeypatch.setattr(shutil, 'copy2', mock_copy2) + + class MockedContainerContext: + def __init__(self): + self.base_dir = '/container' + + def full_path(self, path): + return os.path.join('/container', path.lstrip('/')) + + mock_container = MockedContainerContext() + + files = mount_unit_generator.copy_units_into_system_location( + mock_container, '/source/dir' + ) + + expected_files = [ + '/usr/lib/systemd/system/unit1.mount', + '/usr/lib/systemd/system/unit2.mount', + '/usr/lib/systemd/system/local-fs.target.requires/unit1.mount', + '/usr/lib/systemd/system/local-fs.target.requires/unit2.mount', + ] + assert sorted(files) == sorted(expected_files) + assert mount_unit_generator._delete_file.removal_called -- 2.51.1