diff --git a/0031-Add-possibility-to-add-kernel-drivers-to-initrd.patch b/0031-Add-possibility-to-add-kernel-drivers-to-initrd.patch new file mode 100644 index 0000000..4d4ea26 --- /dev/null +++ b/0031-Add-possibility-to-add-kernel-drivers-to-initrd.patch @@ -0,0 +1,1192 @@ +From 030e1fccac6f15d4d5179a42ba43ba373cd727cf Mon Sep 17 00:00:00 2001 +From: David Kubek +Date: Wed, 24 May 2023 10:40:23 +0200 +Subject: [PATCH 31/42] Add possibility to add kernel drivers to initrd + +Before this change there was no possibility for developers to specify what +kernel drivers should be included in the upgrade/target initramfs. This +includes third-party drivers, which are necessary for system upgrades in some +spefic use cases. + +Changes include: + - A new model `KernelModule` (analogous to the `DracutModule` model) has been + created, to handle kernel drivers. + - Added an `include_kernel_drivers` field in the `UpgradeInitramfsTasks` model + to handle a list of these drivers. + - Data in the `include_kernel_drivers` field is processed correctly to detect + conflicting paths. + - Modified the `generate-iniram.sh` script to accept and process the new data + - Added checks for kernel drivers, in the `CheckInitramfsTasks` actor. + - Updated the unit-tests accordingly. +--- + .../libraries/modscan.py | 1 + + .../libraries/checkinitramfstasks.py | 91 +++++---- + .../tests/unit_test_checkinitramfstasks.py | 57 +++++- + .../libraries/targetinitramfsgenerator.py | 97 ++++++++-- + .../tests/test_targetinitramfsgenerator.py | 174 +++++++++++------- + .../files/generate-initram.sh | 13 ++ + .../libraries/upgradeinitramfsgenerator.py | 141 +++++++++++--- + .../unit_test_upgradeinitramfsgenerator.py | 125 +++++++++---- + .../system_upgrade/common/models/initramfs.py | 47 +++++ + 9 files changed, 563 insertions(+), 183 deletions(-) + +diff --git a/repos/system_upgrade/common/actors/commonleappdracutmodules/libraries/modscan.py b/repos/system_upgrade/common/actors/commonleappdracutmodules/libraries/modscan.py +index 2b8d78a4..15150a50 100644 +--- a/repos/system_upgrade/common/actors/commonleappdracutmodules/libraries/modscan.py ++++ b/repos/system_upgrade/common/actors/commonleappdracutmodules/libraries/modscan.py +@@ -31,6 +31,7 @@ _REQUIRED_PACKAGES = [ + 'kernel-core', + 'kernel-modules', + 'keyutils', ++ 'kmod', + 'lldpad', + 'lvm2', + 'mdadm', +diff --git a/repos/system_upgrade/common/actors/initramfs/checkinitramfstasks/libraries/checkinitramfstasks.py b/repos/system_upgrade/common/actors/initramfs/checkinitramfstasks/libraries/checkinitramfstasks.py +index cd87f74d..0d7d8317 100644 +--- a/repos/system_upgrade/common/actors/initramfs/checkinitramfstasks/libraries/checkinitramfstasks.py ++++ b/repos/system_upgrade/common/actors/initramfs/checkinitramfstasks/libraries/checkinitramfstasks.py +@@ -6,11 +6,11 @@ from leapp.libraries.stdlib import api + from leapp.models import TargetInitramfsTasks, UpgradeInitramfsTasks + + DRACUT_MOD_DIR = '/usr/lib/dracut/modules.d/' +-SUMMARY_DRACUT_FMT = ( +- 'The requested dracut modules for the initramfs are in conflict.' +- ' At least one dracut module is specified to be installed from' +- ' multiple paths. The list of conflicting dracut module names' +- ' with paths is listed below: {}' ++SUMMARY_FMT = ( ++ 'The requested {kind} modules for the initramfs are in conflict.' ++ ' At least one {kind} module is specified to be installed from' ++ ' multiple paths. The list of conflicting {kind} module names' ++ ' with paths is listed below: {conflicts}' + ) + + +@@ -22,51 +22,72 @@ def _printable_modules(conflicts): + return ''.join(output) + + +-def _treat_path(dmodule): ++def _treat_path_dracut(dmodule): + """ + In case the path is not set, set the expected path of the dracut module. + """ ++ + if not dmodule.module_path: + return os.path.join(DRACUT_MOD_DIR, dmodule.name) + return dmodule.module_path + + +-def _detect_dracut_modules_conflicts(msgtype): ++def _treat_path_kernel(kmodule): ++ """ ++ In case the path of a kernel module is not set, indicate that the module is ++ taken from the current system. ++ """ ++ ++ if not kmodule.module_path: ++ return kmodule.name + ' (system)' ++ return kmodule.module_path ++ ++ ++def _detect_modules_conflicts(msgtype, kind): + """ + Return dict of modules with conflicting tasks + +- In this case when a dracut module should be applied but different +- sources are specified. E.g.: +- include dracut modules X where, ++ In this case when a module should be applied but different sources are ++ specified. E.g.: ++ include modules X where, + msg A) X + msg B) X from custom path + """ +- dracut_modules = defaultdict(set) ++ ++ modules_map = { ++ 'dracut': { ++ 'msgattr': 'include_dracut_modules', ++ 'treat_path_fn': _treat_path_dracut, ++ }, ++ 'kernel': { ++ 'msgattr': 'include_kernel_modules', ++ 'treat_path_fn': _treat_path_kernel ++ }, ++ } ++ ++ modules = defaultdict(set) + for msg in api.consume(msgtype): +- for dmodule in msg.include_dracut_modules: +- dracut_modules[dmodule.name].add(_treat_path(dmodule)) +- return {key: val for key, val in dracut_modules.items() if len(val) > 1} ++ for module in getattr(msg, modules_map[kind]['msgattr']): ++ treat_path_fn = modules_map[kind]['treat_path_fn'] ++ modules[module.name].add(treat_path_fn(module)) ++ return {key: val for key, val in modules.items() if len(val) > 1} ++ ++ ++def report_conflicts(msgname, kind, msgtype): ++ conflicts = _detect_modules_conflicts(msgtype, kind) ++ if not conflicts: ++ return ++ report = [ ++ reporting.Title('Conflicting requirements of {kind} modules for the {msgname} initramfs'.format( ++ kind=kind, msgname=msgname)), ++ reporting.Summary(SUMMARY_FMT.format(kind=kind, conflicts=_printable_modules(conflicts))), ++ reporting.Severity(reporting.Severity.HIGH), ++ reporting.Groups([reporting.Groups.SANITY, reporting.Groups.INHIBITOR]), ++ ] ++ reporting.create_report(report) + + + def process(): +- conflicts = _detect_dracut_modules_conflicts(UpgradeInitramfsTasks) +- if conflicts: +- report = [ +- reporting.Title('Conflicting requirements of dracut modules for the upgrade initramfs'), +- reporting.Summary(SUMMARY_DRACUT_FMT.format(_printable_modules(conflicts))), +- reporting.Severity(reporting.Severity.HIGH), +- reporting.Groups([reporting.Groups.SANITY]), +- reporting.Groups([reporting.Groups.INHIBITOR]), +- ] +- reporting.create_report(report) +- +- conflicts = _detect_dracut_modules_conflicts(TargetInitramfsTasks) +- if conflicts: +- report = [ +- reporting.Title('Conflicting requirements of dracut modules for the target initramfs'), +- reporting.Summary(SUMMARY_DRACUT_FMT.format(_printable_modules(conflicts))), +- reporting.Severity(reporting.Severity.HIGH), +- reporting.Groups([reporting.Groups.SANITY]), +- reporting.Groups([reporting.Groups.INHIBITOR]), +- ] +- reporting.create_report(report) ++ report_conflicts('upgrade', 'kernel', UpgradeInitramfsTasks) ++ report_conflicts('upgrade', 'dracut', UpgradeInitramfsTasks) ++ report_conflicts('target', 'dracut', TargetInitramfsTasks) +diff --git a/repos/system_upgrade/common/actors/initramfs/checkinitramfstasks/tests/unit_test_checkinitramfstasks.py b/repos/system_upgrade/common/actors/initramfs/checkinitramfstasks/tests/unit_test_checkinitramfstasks.py +index aad79c73..fca15f73 100644 +--- a/repos/system_upgrade/common/actors/initramfs/checkinitramfstasks/tests/unit_test_checkinitramfstasks.py ++++ b/repos/system_upgrade/common/actors/initramfs/checkinitramfstasks/tests/unit_test_checkinitramfstasks.py +@@ -6,7 +6,7 @@ from leapp import reporting + from leapp.libraries.actor import checkinitramfstasks + from leapp.libraries.common.testutils import create_report_mocked, CurrentActorMocked + from leapp.libraries.stdlib import api +-from leapp.models import DracutModule, Report, TargetInitramfsTasks, UpgradeInitramfsTasks ++from leapp.models import DracutModule, KernelModule, TargetInitramfsTasks, UpgradeInitramfsTasks + from leapp.utils.report import is_inhibitor + + +@@ -14,7 +14,8 @@ def gen_UIT(modules): + if not isinstance(modules, list): + modules = [modules] + dracut_modules = [DracutModule(name=i[0], module_path=i[1]) for i in modules] +- return UpgradeInitramfsTasks(include_dracut_modules=dracut_modules) ++ kernel_modules = [KernelModule(name=i[0], module_path=i[1]) for i in modules] ++ return UpgradeInitramfsTasks(include_dracut_modules=dracut_modules, include_kernel_modules=kernel_modules) + + + def gen_TIT(modules): +@@ -71,9 +72,57 @@ def gen_TIT(modules): + TargetInitramfsTasks, + ), + ]) +-def test_conflict_detection(monkeypatch, expected_res, input_msgs, test_msg_type): ++def test_dracut_conflict_detection(monkeypatch, expected_res, input_msgs, test_msg_type): + monkeypatch.setattr(api, 'current_actor', CurrentActorMocked(msgs=input_msgs)) +- res = checkinitramfstasks._detect_dracut_modules_conflicts(test_msg_type) ++ res = checkinitramfstasks._detect_modules_conflicts(test_msg_type, 'dracut') ++ assert res == expected_res ++ ++ ++@pytest.mark.parametrize('expected_res,input_msgs,test_msg_type', [ ++ ( ++ {}, ++ [], ++ UpgradeInitramfsTasks, ++ ), ++ ( ++ {}, ++ [gen_UIT([('modA', 'pathA'), ('modB', 'pathB')])], ++ UpgradeInitramfsTasks, ++ ), ++ ( ++ {}, ++ [gen_UIT([('modA', 'pathA'), ('modA', 'pathA')])], ++ UpgradeInitramfsTasks, ++ ), ++ ( ++ {'modA': {'pathA', 'pathB'}}, ++ [gen_UIT([('modA', 'pathA'), ('modA', 'pathB')])], ++ UpgradeInitramfsTasks, ++ ), ++ ( ++ {'modA': {'pathA', 'pathB'}}, ++ [gen_UIT(('modA', 'pathA')), gen_UIT(('modA', 'pathB'))], ++ UpgradeInitramfsTasks, ++ ), ++ ( ++ {'modA': {'pathA', 'pathB'}}, ++ [gen_UIT([('modA', 'pathA'), ('modA', 'pathB'), ('modB', 'pathC')])], ++ UpgradeInitramfsTasks, ++ ), ++ ( ++ {'modA': {'modA (system)', 'pathB'}}, ++ [gen_UIT([('modA', None), ('modA', 'pathB')])], ++ UpgradeInitramfsTasks, ++ ), ++ ( ++ {}, ++ [gen_UIT([('modA', 'pathA'), ('modA', 'pathB')])], ++ TargetInitramfsTasks, ++ ), ++]) ++def test_kernel_conflict_detection(monkeypatch, expected_res, input_msgs, test_msg_type): ++ monkeypatch.setattr(api, 'current_actor', CurrentActorMocked(msgs=input_msgs)) ++ res = checkinitramfstasks._detect_modules_conflicts(test_msg_type, 'kernel') + assert res == expected_res + + +diff --git a/repos/system_upgrade/common/actors/initramfs/targetinitramfsgenerator/libraries/targetinitramfsgenerator.py b/repos/system_upgrade/common/actors/initramfs/targetinitramfsgenerator/libraries/targetinitramfsgenerator.py +index 1a7a3e19..39666017 100644 +--- a/repos/system_upgrade/common/actors/initramfs/targetinitramfsgenerator/libraries/targetinitramfsgenerator.py ++++ b/repos/system_upgrade/common/actors/initramfs/targetinitramfsgenerator/libraries/targetinitramfsgenerator.py +@@ -1,3 +1,7 @@ ++import errno ++import os ++import shutil ++ + from leapp.exceptions import StopActorExecutionError + from leapp.libraries.stdlib import api, CalledProcessError, run + from leapp.models import InitrdIncludes # deprecated +@@ -7,25 +11,66 @@ from leapp.utils.deprecation import suppress_deprecation + DRACUT_DIR = '/usr/lib/dracut/modules.d/' + + +-def copy_dracut_modules(modules): ++def _get_target_kernel_modules_dir(kernel_version): ++ """ ++ Return the path where the custom kernel modules should be copied. ++ """ ++ ++ modules_dir = os.path.join('/', 'lib', 'modules', kernel_version, 'extra', 'leapp') ++ ++ return modules_dir ++ ++ ++def _copy_modules(modules, dst_dir, kind): + """ +- Copy every dracut module with specified path into the expected directory. ++ Copy modules of given kind to the specified destination directory. ++ ++ Attempts to remove an cleanup by removing the existing destination ++ directory. If the directory does not exist, it is created anew. Then, for ++ each module message, it checks if the module has a module path specified. If ++ the module already exists in the destination directory, a debug message is ++ logged, and the operation is skipped. Otherwise, the module is copied to the ++ destination directory. + +- original content is overwritten if exists + """ +- # FIXME: use just python functions instead of shell cmds ++ ++ try: ++ os.makedirs(dst_dir) ++ except OSError as exc: ++ if exc.errno == errno.EEXIST and os.path.isdir(dst_dir): ++ pass ++ else: ++ raise ++ + for module in modules: + if not module.module_path: + continue ++ ++ dst_path = os.path.join(dst_dir, os.path.basename(module.module_path)) ++ if os.path.exists(dst_path): ++ api.current_logger().debug( ++ 'The {name} {kind} module has been already installed. Skipping.' ++ .format(name=module.name, kind=kind)) ++ continue ++ ++ copy_fn = shutil.copytree ++ if os.path.isfile(module.module_path): ++ copy_fn = shutil.copy2 ++ + try: +- # context.copytree_to(module.module_path, os.path.join(DRACUT_DIR, os.path.basename(module.module_path))) +- run(['cp', '-f', '-a', module.module_path, DRACUT_DIR]) +- except CalledProcessError as e: +- api.current_logger().error('Failed to copy dracut module "{name}" from "{source}" to "{target}"'.format( +- name=module.name, source=module.module_path, target=DRACUT_DIR), exc_info=True) +- # FIXME: really do we want to raise the error and stop execution completely??.... ++ api.current_logger().debug( ++ 'Copying {kind} module "{name}" to "{path}".' ++ .format(kind=kind, name=module.name, path=dst_path)) ++ ++ copy_fn(module.module_path, dst_path) ++ except shutil.Error as e: ++ api.current_logger().error( ++ 'Failed to copy {kind} module "{name}" from "{source}" to "{target}"'.format( ++ kind=kind, name=module.name, source=module.module_path, target=dst_dir), ++ exc_info=True) + raise StopActorExecutionError( +- message='Failed to install dracut modules required in the target initramfs. Error: {}'.format(str(e)) ++ message='Failed to install {kind} modules required in the initram. Error: {error}'.format( ++ kind=kind, error=str(e)) + ) + + +@@ -43,9 +88,11 @@ def _get_modules(): + # supposed to create any such tasks before the reporting phase, so we + # are able to check it. + # +- modules = [] ++ modules = {'dracut': [], 'kernel': []} + for task in api.consume(TargetInitramfsTasks): +- modules.extend(task.include_dracut_modules) ++ modules['dracut'].extend(task.include_dracut_modules) ++ modules['kernel'].extend(task.include_kernel_modules) ++ + return modules + + +@@ -53,7 +100,7 @@ def process(): + files = _get_files() + modules = _get_modules() + +- if not files and not modules: ++ if not files and not modules['kernel'] and not modules['dracut']: + api.current_logger().debug( + 'No additional files or modules required to add into the target initramfs.') + return +@@ -65,15 +112,29 @@ def process(): + details={'Problem': 'Did not receive a message with installed RHEL-8 kernel version' + ' (InstalledTargetKernelVersion)'}) + +- copy_dracut_modules(modules) ++ _copy_modules(modules['dracut'], DRACUT_DIR, 'dracut') ++ _copy_modules(modules['kernel'], _get_target_kernel_modules_dir(target_kernel.version), 'kernel') ++ ++ # Discover any new modules and regenerate modules.dep ++ should_regenerate = any(module.module_path is not None for module in modules['kernel']) ++ if should_regenerate: ++ try: ++ run(['depmod', target_kernel.version, '-a']) ++ except CalledProcessError as e: ++ raise StopActorExecutionError('Failed to generate modules.dep and map files.', details={'details': str(e)}) ++ + try: + # multiple files|modules need to be quoted, see --install | --add in dracut(8) +- module_names = list({module.name for module in modules}) ++ dracut_module_names = list({module.name for module in modules['dracut']}) ++ kernel_module_names = list({module.name for module in modules['kernel']}) + cmd = ['dracut', '-f', '--kver', target_kernel.version] + if files: + cmd += ['--install', '{}'.format(' '.join(files))] +- if modules: +- cmd += ['--add', '{}'.format(' '.join(module_names))] ++ if modules['dracut']: ++ cmd += ['--add', '{}'.format(' '.join(dracut_module_names))] ++ if modules['kernel']: ++ cmd += ['--add-drivers', '{}'.format(' '.join(kernel_module_names))] ++ + run(cmd) + except CalledProcessError as e: + # just hypothetic check, it should not die +diff --git a/repos/system_upgrade/common/actors/initramfs/targetinitramfsgenerator/tests/test_targetinitramfsgenerator.py b/repos/system_upgrade/common/actors/initramfs/targetinitramfsgenerator/tests/test_targetinitramfsgenerator.py +index 8403a431..57daca75 100644 +--- a/repos/system_upgrade/common/actors/initramfs/targetinitramfsgenerator/tests/test_targetinitramfsgenerator.py ++++ b/repos/system_upgrade/common/actors/initramfs/targetinitramfsgenerator/tests/test_targetinitramfsgenerator.py +@@ -6,12 +6,9 @@ from leapp.libraries.common.testutils import CurrentActorMocked, logger_mocked + from leapp.libraries.stdlib import api, CalledProcessError + from leapp.utils.deprecation import suppress_deprecation + +-from leapp.models import ( # isort:skip +- InitrdIncludes, # deprecated +- DracutModule, +- InstalledTargetKernelVersion, +- TargetInitramfsTasks +-) ++from leapp.models import ( # isort:skip ++ InitrdIncludes, # deprecated ++ DracutModule, KernelModule, InstalledTargetKernelVersion, TargetInitramfsTasks) + + FILES = ['/file1', '/file2', '/dir/subdir/subsubdir/file3', '/file4', '/file5'] + MODULES = [ +@@ -25,13 +22,19 @@ NO_INCLUDE_MSG = 'No additional files or modules required to add into the target + + + def raise_call_error(args=None): +- raise CalledProcessError( +- message='A Leapp Command Error occurred.', +- command=args, +- result={'signal': None, 'exit_code': 1, 'pid': 0, 'stdout': 'fake', 'stderr': 'fake'}) ++ raise CalledProcessError(message='A Leapp Command Error occurred.', ++ command=args, ++ result={ ++ 'signal': None, ++ 'exit_code': 1, ++ 'pid': 0, ++ 'stdout': 'fake', ++ 'stderr': 'fake' ++ }) + + + class RunMocked(object): ++ + def __init__(self, raise_err=False): + self.called = 0 + self.args = [] +@@ -44,20 +47,26 @@ class RunMocked(object): + raise_call_error(args) + + +-def gen_TIT(modules, files): +- if not isinstance(modules, list): +- modules = [modules] +- if not isinstance(files, list): +- files = [files] +- dracut_modules = [DracutModule(name=i[0], module_path=i[1]) for i in modules] +- return TargetInitramfsTasks(include_files=files, include_dracut_modules=dracut_modules) ++def _ensure_list(data): ++ return data if isinstance(data, list) else [data] ++ ++ ++def gen_TIT(dracut_modules, kernel_modules, files): ++ files = _ensure_list(files) ++ ++ dracut_modules = [DracutModule(name=i[0], module_path=i[1]) for i in _ensure_list(dracut_modules)] ++ kernel_modules = [KernelModule(name=i[0], module_path=i[1]) for i in _ensure_list(kernel_modules)] ++ ++ return TargetInitramfsTasks( ++ include_files=files, ++ include_dracut_modules=dracut_modules, ++ include_kernel_modules=kernel_modules, ++ ) + + + @suppress_deprecation(InitrdIncludes) + def gen_InitrdIncludes(files): +- if not isinstance(files, list): +- files = [files] +- return InitrdIncludes(files=files) ++ return InitrdIncludes(files=_ensure_list(files)) + + + def test_no_includes(monkeypatch): +@@ -77,12 +86,12 @@ TEST_CASES = [ + gen_InitrdIncludes(FILES[3:]), + ], + [ +- gen_TIT([], FILES[0:3]), +- gen_TIT([], FILES[3:]), ++ gen_TIT([], [], FILES[0:3]), ++ gen_TIT([], [], FILES[3:]), + ], + [ + gen_InitrdIncludes(FILES[0:3]), +- gen_TIT([], FILES[3:]), ++ gen_TIT([], [], FILES[3:]), + ], + ] + +@@ -93,7 +102,7 @@ def test_no_kernel_version(monkeypatch, msgs): + monkeypatch.setattr(api, 'current_actor', CurrentActorMocked(msgs=msgs)) + monkeypatch.setattr(targetinitramfsgenerator, 'run', run_mocked) + # FIXME +- monkeypatch.setattr(targetinitramfsgenerator, 'copy_dracut_modules', lambda dummy: None) ++ monkeypatch.setattr(targetinitramfsgenerator, '_copy_modules', lambda *_: None) + + with pytest.raises(StopActorExecutionError) as e: + targetinitramfsgenerator.process() +@@ -105,11 +114,11 @@ def test_no_kernel_version(monkeypatch, msgs): + def test_dracut_fail(monkeypatch, msgs): + run_mocked = RunMocked(raise_err=True) + monkeypatch.setattr(api, 'current_actor', CurrentActorMocked(msgs=msgs)) +- monkeypatch.setattr(api, 'current_actor', CurrentActorMocked( +- msgs=msgs+[InstalledTargetKernelVersion(version=KERNEL_VERSION)])) ++ monkeypatch.setattr(api, 'current_actor', ++ CurrentActorMocked(msgs=msgs + [InstalledTargetKernelVersion(version=KERNEL_VERSION)])) + monkeypatch.setattr(targetinitramfsgenerator, 'run', run_mocked) + # FIXME +- monkeypatch.setattr(targetinitramfsgenerator, 'copy_dracut_modules', lambda dummy: None) ++ monkeypatch.setattr(targetinitramfsgenerator, '_copy_modules', lambda *_: None) + + with pytest.raises(StopActorExecutionError) as e: + targetinitramfsgenerator.process() +@@ -117,47 +126,71 @@ def test_dracut_fail(monkeypatch, msgs): + assert run_mocked.called + + +-@pytest.mark.parametrize('msgs,files,modules', [ +- # deprecated set +- ([gen_InitrdIncludes(FILES[0])], FILES[0:1], []), +- ([gen_InitrdIncludes(FILES)], FILES, []), +- ([gen_InitrdIncludes(FILES[0:3]), gen_InitrdIncludes(FILES[3:])], FILES, []), +- ([gen_InitrdIncludes(FILES[0:3]), gen_InitrdIncludes(FILES)], FILES, []), +- +- # new set for files only +- ([gen_TIT([], FILES[0])], FILES[0:1], []), +- ([gen_TIT([], FILES)], FILES, []), +- ([gen_TIT([], FILES[0:3]), gen_TIT([], FILES[3:])], FILES, []), +- ([gen_TIT([], FILES[0:3]), gen_TIT([], FILES)], FILES, []), +- +- # deprecated and new msgs for files only +- ([gen_InitrdIncludes(FILES[0:3]), gen_TIT([], FILES[3:])], FILES, []), +- +- # modules only +- ([gen_TIT(MODULES[0], [])], [], MODULES[0:1]), +- ([gen_TIT(MODULES, [])], [], MODULES), +- ([gen_TIT(MODULES[0:3], []), gen_TIT(MODULES[3], [])], [], MODULES), +- +- # modules only - duplicates; see notes in the library +- ([gen_TIT(MODULES[0:3], []), gen_TIT(MODULES, [])], [], MODULES), +- +- # modules + files (new only) +- ([gen_TIT(MODULES, FILES)], FILES, MODULES), +- ([gen_TIT(MODULES[0:3], FILES[0:3]), gen_TIT(MODULES[3:], FILES[3:])], FILES, MODULES), +- ([gen_TIT(MODULES, []), gen_TIT([], FILES)], FILES, MODULES), +- +- # modules + files with deprecated msgs +- ([gen_TIT(MODULES, []), gen_InitrdIncludes(FILES)], FILES, MODULES), +- ([gen_TIT(MODULES, FILES[0:3]), gen_InitrdIncludes(FILES[3:])], FILES, MODULES), +- +-]) +-def test_flawless(monkeypatch, msgs, files, modules): ++@pytest.mark.parametrize( ++ 'msgs,files,dracut_modules,kernel_modules', ++ [ ++ # deprecated set ++ ([gen_InitrdIncludes(FILES[0])], FILES[0:1], [], []), ++ ([gen_InitrdIncludes(FILES)], FILES, [], []), ++ ([gen_InitrdIncludes(FILES[0:3]), gen_InitrdIncludes(FILES[3:])], FILES, [], []), ++ ([gen_InitrdIncludes(FILES[0:3]), gen_InitrdIncludes(FILES)], FILES, [], []), ++ ++ # new set for files only ++ ([gen_TIT([], [], FILES[0])], FILES[0:1], [], []), ++ ([gen_TIT([], [], FILES)], FILES, [], []), ++ ([gen_TIT([], [], FILES[0:3]), gen_TIT([], [], FILES[3:])], FILES, [], []), ++ ([gen_TIT([], [], FILES[0:3]), gen_TIT([], [], FILES)], FILES, [], []), ++ ++ # deprecated and new msgs for files only ++ ([gen_InitrdIncludes(FILES[0:3]), gen_TIT([], [], FILES[3:])], FILES, [], []), ++ ++ # dracut modules only ++ ([gen_TIT(MODULES[0], [], [])], [], MODULES[0:1], []), ++ ([gen_TIT(MODULES, [], [])], [], MODULES, []), ++ ([gen_TIT(MODULES[0:3], [], []), gen_TIT(MODULES[3], [], [])], [], MODULES, []), ++ ++ # kernel modules only ++ ([gen_TIT([], MODULES[0], [])], [], [], MODULES[0:1]), ++ ([gen_TIT([], MODULES, [])], [], [], MODULES), ++ ([gen_TIT([], MODULES[0:3], []), gen_TIT([], MODULES[3], [])], [], [], MODULES), ++ ++ # modules only - duplicates; see notes in the library ++ ([gen_TIT(MODULES[0:3], [], []), gen_TIT(MODULES, [], [])], [], MODULES, []), ++ ([gen_TIT([], MODULES[0:3], []), gen_TIT([], MODULES, [])], [], [], MODULES), ++ ++ # modules + files (new only) ++ ([gen_TIT(MODULES, [], FILES)], FILES, MODULES, []), ++ ([gen_TIT([], MODULES, FILES)], FILES, [], MODULES), ++ ++ ([gen_TIT(MODULES[0:3], [], FILES[0:3]), gen_TIT(MODULES[3:], [], FILES[3:])], FILES, MODULES, []), ++ ([gen_TIT([], MODULES[0:3], FILES[0:3]), gen_TIT([], MODULES[3:], FILES[3:])], FILES, [], MODULES), ++ ++ ([gen_TIT(MODULES, [], []), gen_TIT([], [], FILES)], FILES, MODULES, []), ++ ([gen_TIT([], MODULES, []), gen_TIT([], [], FILES)], FILES, [], MODULES), ++ ++ # kernel + dracut modules ++ ( ++ [ ++ gen_TIT(MODULES[0:3], MODULES[0:3], FILES[0:3]), ++ gen_TIT(MODULES[3:], MODULES[3:], FILES[3:]) ++ ], ++ FILES, MODULES, MODULES ++ ), ++ ++ # modules + files with deprecated msgs ++ ([gen_TIT(MODULES, [], []), gen_InitrdIncludes(FILES)], FILES, MODULES, []), ++ ([gen_TIT([], MODULES, []), gen_InitrdIncludes(FILES)], FILES, [], MODULES), ++ ++ ([gen_TIT(MODULES, [], FILES[0:3]), gen_InitrdIncludes(FILES[3:])], FILES, MODULES, []), ++ ([gen_TIT([], MODULES, FILES[0:3]), gen_InitrdIncludes(FILES[3:])], FILES, [], MODULES), ++ ]) ++def test_flawless(monkeypatch, msgs, files, dracut_modules, kernel_modules): + _msgs = msgs + [InstalledTargetKernelVersion(version=KERNEL_VERSION)] + run_mocked = RunMocked() + monkeypatch.setattr(api, 'current_actor', CurrentActorMocked(msgs=_msgs)) + monkeypatch.setattr(targetinitramfsgenerator, 'run', run_mocked) + # FIXME +- monkeypatch.setattr(targetinitramfsgenerator, 'copy_dracut_modules', lambda dummy: None) ++ monkeypatch.setattr(targetinitramfsgenerator, '_copy_modules', lambda *_: None) + + targetinitramfsgenerator.process() + assert run_mocked.called +@@ -171,11 +204,20 @@ def test_flawless(monkeypatch, msgs, files, modules): + else: + assert '--install' not in run_mocked.args + +- # check modules +- if modules: ++ # check dracut modules ++ if dracut_modules: + assert '--add' in run_mocked.args + arg = run_mocked.args[run_mocked.args.index('--add') + 1] +- for m in modules: ++ for m in dracut_modules: + assert m[0] in arg + else: + assert '--add' not in run_mocked.args ++ ++ # check kernel modules ++ if kernel_modules: ++ assert '--add-drivers' in run_mocked.args ++ arg = run_mocked.args[run_mocked.args.index('--add-drivers') + 1] ++ for m in kernel_modules: ++ assert m[0] in arg ++ else: ++ assert '--add-drivers' not in run_mocked.args +diff --git a/repos/system_upgrade/common/actors/initramfs/upgradeinitramfsgenerator/files/generate-initram.sh b/repos/system_upgrade/common/actors/initramfs/upgradeinitramfsgenerator/files/generate-initram.sh +index d6934147..9648234c 100755 +--- a/repos/system_upgrade/common/actors/initramfs/upgradeinitramfsgenerator/files/generate-initram.sh ++++ b/repos/system_upgrade/common/actors/initramfs/upgradeinitramfsgenerator/files/generate-initram.sh +@@ -29,6 +29,9 @@ dracut_install_modules() + } + + ++# KERNEL_MODULES_ADD and DRACUT_MODULES_ADD are expected to be expanded and ++# we do not want to prevent word splitting in that case. ++# shellcheck disable=SC2086 + build() { + dracut_install_modules + +@@ -67,6 +70,15 @@ build() { + DRACUT_MODULES_ADD=$(echo "--add $LEAPP_ADD_DRACUT_MODULES" | sed 's/,/ --add /g') + fi + ++ KERNEL_MODULES_ADD="" ++ if [[ -n "$LEAPP_ADD_KERNEL_MODULES" ]]; then ++ depmod "${KERNEL_VERSION}" -a ++ KERNEL_MODULES_ADD=$( ++ echo "--add-drivers $LEAPP_ADD_KERNEL_MODULES" | ++ sed 's/,/ --add-drivers /g' ++ ) ++ fi ++ + DRACUT_INSTALL="systemd-nspawn" + if [[ -n "$LEAPP_DRACUT_INSTALL_FILES" ]]; then + DRACUT_INSTALL="$DRACUT_INSTALL $LEAPP_DRACUT_INSTALL_FILES" +@@ -89,6 +101,7 @@ build() { + --confdir "$DRACUT_CONF_DIR" \ + --install "$DRACUT_INSTALL" \ + $DRACUT_MODULES_ADD \ ++ $KERNEL_MODULES_ADD \ + "$DRACUT_MDADMCONF_ARG" \ + "$DRACUT_LVMCONF_ARG" \ + --no-hostonly \ +diff --git a/repos/system_upgrade/common/actors/initramfs/upgradeinitramfsgenerator/libraries/upgradeinitramfsgenerator.py b/repos/system_upgrade/common/actors/initramfs/upgradeinitramfsgenerator/libraries/upgradeinitramfsgenerator.py +index 2f145217..f141d9e3 100644 +--- a/repos/system_upgrade/common/actors/initramfs/upgradeinitramfsgenerator/libraries/upgradeinitramfsgenerator.py ++++ b/repos/system_upgrade/common/actors/initramfs/upgradeinitramfsgenerator/libraries/upgradeinitramfsgenerator.py +@@ -1,10 +1,11 @@ + import os + import shutil ++from distutils.version import LooseVersion + + from leapp.exceptions import StopActorExecutionError + from leapp.libraries.common import dnfplugin, mounting + from leapp.libraries.common.config.version import get_target_major_version +-from leapp.libraries.stdlib import api ++from leapp.libraries.stdlib import api, CalledProcessError + from leapp.models import RequiredUpgradeInitramPackages # deprecated + from leapp.models import UpgradeDracutModule # deprecated + from leapp.models import ( +@@ -21,6 +22,45 @@ INITRAM_GEN_SCRIPT_NAME = 'generate-initram.sh' + DRACUT_DIR = '/dracut' + + ++def _get_target_kernel_version(context): ++ """ ++ Get the version of the most recent kernel version within the container. ++ """ ++ ++ kernel_version = None ++ try: ++ results = context.call(['rpm', '-qa', 'kernel-core'], split=True) ++ ++ versions = [ver.replace('kernel-core-', '') for ver in results['stdout']] ++ api.current_logger().debug( ++ 'Versions detected {versions}.' ++ .format(versions=versions)) ++ sorted_versions = sorted(versions, key=LooseVersion, reverse=True) ++ kernel_version = next(iter(sorted_versions), None) ++ except CalledProcessError: ++ raise StopActorExecutionError( ++ 'Cannot get version of the installed kernel.', ++ details={'Problem': 'Could not query the currently installed kernel through rmp.'}) ++ ++ if not kernel_version: ++ raise StopActorExecutionError( ++ 'Cannot get version of the installed kernel.', ++ details={'Problem': 'A rpm query for the available kernels did not produce any results.'}) ++ ++ return kernel_version ++ ++ ++def _get_target_kernel_modules_dir(context): ++ """ ++ Return the path where the custom kernel modules should be copied. ++ """ ++ ++ kernel_version = _get_target_kernel_version(context) ++ modules_dir = os.path.join('/', 'lib', 'modules', kernel_version, 'extra', 'leapp') ++ ++ return modules_dir ++ ++ + def _reinstall_leapp_repository_hint(): + """ + Convenience function for creating a detail for StopActorExecutionError with a hint to reinstall the +@@ -31,39 +71,81 @@ def _reinstall_leapp_repository_hint(): + } + + +-def copy_dracut_modules(context, modules): ++def _copy_modules(context, modules, dst_dir, kind): + """ +- Copy dracut modules into the target userspace. ++ Copy modules of given kind to the specified destination directory. ++ ++ Attempts to remove an cleanup by removing the existing destination ++ directory. If the directory does not exist, it is created anew. Then, for ++ each module message, it checks if the module has a module path specified. If ++ the module already exists in the destination directory, a debug message is ++ logged, and the operation is skipped. Otherwise, the module is copied to the ++ destination directory. + +- If duplicated requirements to copy a dracut module are detected, +- log the debug msg and skip any try to copy a dracut module into the +- target userspace that already exists inside DRACTUR_DIR. + """ ++ + try: +- context.remove_tree(DRACUT_DIR) ++ context.remove_tree(dst_dir) + except EnvironmentError: + pass ++ ++ context.makedirs(dst_dir) ++ + for module in modules: + if not module.module_path: + continue +- dst_path = os.path.join(DRACUT_DIR, os.path.basename(module.module_path)) ++ ++ dst_path = os.path.join(dst_dir, os.path.basename(module.module_path)) + if os.path.exists(context.full_path(dst_path)): +- # we are safe to skip it as we now the module is from the same path +- # regarding the actor checking all initramfs tasks + api.current_logger().debug( +- 'The {name} dracut module has been already installed. Skipping.' +- .format(name=module.name)) ++ 'The {name} {kind} module has been already installed. Skipping.' ++ .format(name=module.name, kind=kind)) + continue ++ ++ copy_fn = context.copytree_to ++ if os.path.isfile(module.module_path): ++ copy_fn = context.copy_to ++ + try: +- context.copytree_to(module.module_path, dst_path) ++ api.current_logger().debug( ++ 'Copying {kind} module "{name}" to "{path}".' ++ .format(kind=kind, name=module.name, path=dst_path)) ++ ++ copy_fn(module.module_path, dst_path) + except shutil.Error as e: +- api.current_logger().error('Failed to copy dracut module "{name}" from "{source}" to "{target}"'.format( +- name=module.name, source=module.module_path, target=context.full_path(DRACUT_DIR)), exc_info=True) ++ api.current_logger().error( ++ 'Failed to copy {kind} module "{name}" from "{source}" to "{target}"'.format( ++ kind=kind, name=module.name, source=module.module_path, target=context.full_path(dst_dir)), ++ exc_info=True) + raise StopActorExecutionError( +- message='Failed to install dracut modules required in the initram. Error: {}'.format(str(e)) ++ message='Failed to install {kind} modules required in the initram. Error: {error}'.format( ++ kind=kind, error=str(e)) + ) + + ++def copy_dracut_modules(context, modules): ++ """ ++ Copy dracut modules into the target userspace. ++ ++ If a module cannot be copied, an error message is logged, and a ++ StopActorExecutionError exception is raised. ++ """ ++ ++ _copy_modules(context, modules, DRACUT_DIR, 'dracut') ++ ++ ++def copy_kernel_modules(context, modules): ++ """ ++ Copy kernel modules into the target userspace. ++ ++ If a module cannot be copied, an error message is logged, and a ++ StopActorExecutionError exception is raised. ++ """ ++ ++ dst_dir = _get_target_kernel_modules_dir(context) ++ _copy_modules(context, modules, dst_dir, 'kernel') ++ ++ + @suppress_deprecation(UpgradeDracutModule) + def _get_dracut_modules(): + return list(api.consume(UpgradeDracutModule)) +@@ -153,30 +235,43 @@ def generate_initram_disk(context): + """ + Function to actually execute the init ramdisk creation. + +- Includes handling of specified dracut modules from the host when needed. +- The check for the 'conflicting' dracut modules is in a separate actor. ++ Includes handling of specified dracut and kernel modules from the host when ++ needed. The check for the 'conflicting' modules is in a separate actor. + """ + env = {} + if get_target_major_version() == '9': + env = {'SYSTEMD_SECCOMP': '0'} ++ + # TODO(pstodulk): Add possibility to add particular drivers + # Issue #645 +- modules = _get_dracut_modules() # deprecated ++ modules = { ++ 'dracut': _get_dracut_modules(), # deprecated ++ 'kernel': [], ++ } + files = set() + for task in api.consume(UpgradeInitramfsTasks): +- modules.extend(task.include_dracut_modules) ++ modules['dracut'].extend(task.include_dracut_modules) ++ modules['kernel'].extend(task.include_kernel_modules) + files.update(task.include_files) +- copy_dracut_modules(context, modules) ++ ++ copy_dracut_modules(context, modules['dracut']) ++ copy_kernel_modules(context, modules['kernel']) ++ + # FIXME: issue #376 + context.call([ + '/bin/sh', '-c', +- 'LEAPP_ADD_DRACUT_MODULES="{modules}" LEAPP_KERNEL_ARCH={arch} ' ++ 'LEAPP_KERNEL_VERSION={kernel_version} ' ++ 'LEAPP_ADD_DRACUT_MODULES="{dracut_modules}" LEAPP_KERNEL_ARCH={arch} ' ++ 'LEAPP_ADD_KERNEL_MODULES="{kernel_modules}" ' + 'LEAPP_DRACUT_INSTALL_FILES="{files}" {cmd}'.format( +- modules=','.join([mod.name for mod in modules]), ++ kernel_version=_get_target_kernel_version(context), ++ dracut_modules=','.join([mod.name for mod in modules['dracut']]), ++ kernel_modules=','.join([mod.name for mod in modules['kernel']]), + arch=api.current_actor().configuration.architecture, + files=' '.join(files), + cmd=os.path.join('/', INITRAM_GEN_SCRIPT_NAME)) + ], env=env) ++ + copy_boot_files(context) + + +diff --git a/repos/system_upgrade/common/actors/initramfs/upgradeinitramfsgenerator/tests/unit_test_upgradeinitramfsgenerator.py b/repos/system_upgrade/common/actors/initramfs/upgradeinitramfsgenerator/tests/unit_test_upgradeinitramfsgenerator.py +index cd9d0546..a2f1c837 100644 +--- a/repos/system_upgrade/common/actors/initramfs/upgradeinitramfsgenerator/tests/unit_test_upgradeinitramfsgenerator.py ++++ b/repos/system_upgrade/common/actors/initramfs/upgradeinitramfsgenerator/tests/unit_test_upgradeinitramfsgenerator.py +@@ -10,12 +10,12 @@ from leapp.libraries.common.testutils import CurrentActorMocked, logger_mocked, + from leapp.utils.deprecation import suppress_deprecation + + from leapp.models import ( # isort:skip +- FIPSInfo, + RequiredUpgradeInitramPackages, # deprecated + UpgradeDracutModule, # deprecated + BootContent, + CopyFile, + DracutModule, ++ KernelModule, + TargetUserSpaceUpgradeTasks, + UpgradeInitramfsTasks, + ) +@@ -42,30 +42,36 @@ def adjust_cwd(): + os.chdir(previous_cwd) + + ++def _ensure_list(data): ++ return data if isinstance(data, list) else [data] ++ ++ + def gen_TUSU(packages, copy_files=None): +- if not isinstance(packages, list): +- packages = [packages] ++ packages = _ensure_list(packages) ++ + if not copy_files: + copy_files = [] +- elif not isinstance(copy_files, list): +- copy_files = [copy_files] ++ copy_files = _ensure_list(copy_files) ++ + return TargetUserSpaceUpgradeTasks(install_rpms=packages, copy_files=copy_files) + + + @suppress_deprecation(RequiredUpgradeInitramPackages) + def gen_RUIP(packages): +- if not isinstance(packages, list): +- packages = [packages] ++ packages = _ensure_list(packages) + return RequiredUpgradeInitramPackages(packages=packages) + + +-def gen_UIT(modules, files): +- if not isinstance(modules, list): +- modules = [modules] +- if not isinstance(files, list): +- files = [files] +- dracut_modules = [DracutModule(name=i[0], module_path=i[1]) for i in modules] +- return UpgradeInitramfsTasks(include_files=files, include_dracut_modules=dracut_modules) ++def gen_UIT(dracut_modules, kernel_modules, files): ++ files = _ensure_list(files) ++ ++ dracut_modules = [DracutModule(name=i[0], module_path=i[1]) for i in _ensure_list(dracut_modules)] ++ kernel_modules = [KernelModule(name=i[0], module_path=i[1]) for i in _ensure_list(kernel_modules)] ++ ++ return UpgradeInitramfsTasks(include_files=files, ++ include_dracut_modules=dracut_modules, ++ include_kernel_modules=kernel_modules, ++ ) + + + @suppress_deprecation(UpgradeDracutModule) +@@ -81,6 +87,7 @@ class MockedContext(object): + self.called_copytree_from = [] + self.called_copy_to = [] + self.called_call = [] ++ self.called_makedirs = [] + self.content = set() + self.base_dir = "/base/dir" + """ +@@ -108,6 +115,9 @@ class MockedContext(object): + self.called_copy_to.append((src, dst)) + self.content.add(dst) + ++ def makedirs(self, path): ++ self.called_makedirs.append(path) ++ + def remove_tree(self, path): + # make list for iteration as change of the set is expected during the + # iteration, which could lead to runtime error +@@ -240,38 +250,50 @@ def test_prepare_userspace_for_initram(monkeypatch, adjust_cwd, input_msgs, pkgs + assert _sort_files(upgradeinitramfsgenerator._copy_files.args[1]) == _files + + +-@pytest.mark.parametrize('input_msgs,modules', [ ++@pytest.mark.parametrize('input_msgs,dracut_modules,kernel_modules', [ + # test dracut modules with UpgradeDracutModule(s) - orig functionality +- (gen_UDM_list(MODULES[0]), MODULES[0]), +- (gen_UDM_list(MODULES), MODULES), ++ (gen_UDM_list(MODULES[0]), MODULES[0], []), ++ (gen_UDM_list(MODULES), MODULES, []), + + # test dracut modules with UpgradeInitramfsTasks - new functionality +- ([gen_UIT(MODULES[0], [])], MODULES[0]), +- ([gen_UIT(MODULES, [])], MODULES), ++ ([gen_UIT(MODULES[0], MODULES[0], [])], MODULES[0], MODULES[0]), ++ ([gen_UIT(MODULES, MODULES, [])], MODULES, MODULES), + + # test dracut modules with old and new models +- (gen_UDM_list(MODULES[1]) + [gen_UIT(MODULES[2], [])], MODULES[1:3]), +- (gen_UDM_list(MODULES[2:]) + [gen_UIT(MODULES[0:2], [])], MODULES), ++ (gen_UDM_list(MODULES[1]) + [gen_UIT(MODULES[2], [], [])], MODULES[1:3], []), ++ (gen_UDM_list(MODULES[2:]) + [gen_UIT(MODULES[0:2], [], [])], MODULES, []), ++ (gen_UDM_list(MODULES[1]) + [gen_UIT([], MODULES[2], [])], MODULES[1], MODULES[2]), ++ (gen_UDM_list(MODULES[2:]) + [gen_UIT([], MODULES[0:2], [])], MODULES[2:], MODULES[0:2]), + + # TODO(pstodulk): test include files missing (relates #376) + ]) +-def test_generate_initram_disk(monkeypatch, input_msgs, modules): ++def test_generate_initram_disk(monkeypatch, input_msgs, dracut_modules, kernel_modules): + context = MockedContext() + curr_actor = CurrentActorMocked(msgs=input_msgs, arch=architecture.ARCH_X86_64) + monkeypatch.setattr(upgradeinitramfsgenerator.api, 'current_actor', curr_actor) + monkeypatch.setattr(upgradeinitramfsgenerator, 'copy_dracut_modules', MockedCopyArgs()) ++ monkeypatch.setattr(upgradeinitramfsgenerator, '_get_target_kernel_version', lambda _: '') ++ monkeypatch.setattr(upgradeinitramfsgenerator, 'copy_kernel_modules', MockedCopyArgs()) + monkeypatch.setattr(upgradeinitramfsgenerator, 'copy_boot_files', lambda dummy: None) + upgradeinitramfsgenerator.generate_initram_disk(context) + + # test now just that all modules have been passed for copying - so we know + # all modules have been consumed +- detected_modules = set() +- _modules = set(modules) if isinstance(modules, list) else set([modules]) ++ detected_dracut_modules = set() ++ _dracut_modules = set(dracut_modules) if isinstance(dracut_modules, list) else set([dracut_modules]) + for dracut_module in upgradeinitramfsgenerator.copy_dracut_modules.args[1]: + module = (dracut_module.name, dracut_module.module_path) +- assert module in _modules +- detected_modules.add(module) +- assert detected_modules == _modules ++ assert module in _dracut_modules ++ detected_dracut_modules.add(module) ++ assert detected_dracut_modules == _dracut_modules ++ ++ detected_kernel_modules = set() ++ _kernel_modules = set(kernel_modules) if isinstance(kernel_modules, list) else set([kernel_modules]) ++ for kernel_module in upgradeinitramfsgenerator.copy_kernel_modules.args[1]: ++ module = (kernel_module.name, kernel_module.module_path) ++ assert module in _kernel_modules ++ detected_kernel_modules.add(module) ++ assert detected_kernel_modules == _kernel_modules + + # TODO(pstodulk): this test is not created properly, as context.call check + # is skipped completely. Testing will more convenient with fixed #376 +@@ -300,7 +322,8 @@ def test_copy_dracut_modules_rmtree_ignore(monkeypatch): + assert context.content + + +-def test_copy_dracut_modules_fail(monkeypatch): ++@pytest.mark.parametrize('kind', ['dracut', 'kernel']) ++def test_copy_modules_fail(monkeypatch, kind): + context = MockedContext() + + def copytree_to_error(src, dst): +@@ -313,15 +336,30 @@ def test_copy_dracut_modules_fail(monkeypatch): + context.copytree_to = copytree_to_error + monkeypatch.setattr(os.path, 'exists', mock_context_path_exists) + monkeypatch.setattr(upgradeinitramfsgenerator.api, 'current_logger', MockedLogger()) +- dmodules = [DracutModule(name='foo', module_path='/path/foo')] ++ monkeypatch.setattr(upgradeinitramfsgenerator, '_get_target_kernel_modules_dir', lambda _: '/kernel_modules') ++ ++ module_class = None ++ copy_fn = None ++ if kind == 'dracut': ++ module_class = DracutModule ++ copy_fn = upgradeinitramfsgenerator.copy_dracut_modules ++ dst_path = 'dracut' ++ elif kind == 'kernel': ++ module_class = KernelModule ++ copy_fn = upgradeinitramfsgenerator.copy_kernel_modules ++ dst_path = 'kernel_modules' ++ ++ modules = [module_class(name='foo', module_path='/path/foo')] + with pytest.raises(StopActorExecutionError) as err: +- upgradeinitramfsgenerator.copy_dracut_modules(context, dmodules) +- assert err.value.message.startswith('Failed to install dracut modules') +- expected_err_log = 'Failed to copy dracut module "foo" from "/path/foo" to "/base/dir/dracut"' ++ copy_fn(context, modules) ++ assert err.value.message.startswith('Failed to install {kind} modules'.format(kind=kind)) ++ expected_err_log = 'Failed to copy {kind} module "foo" from "/path/foo" to "/base/dir/{dst_path}"'.format( ++ kind=kind, dst_path=dst_path) + assert expected_err_log in upgradeinitramfsgenerator.api.current_logger.errmsg + + +-def test_copy_dracut_modules_duplicate_skip(monkeypatch): ++@pytest.mark.parametrize('kind', ['dracut', 'kernel']) ++def test_copy_modules_duplicate_skip(monkeypatch, kind): + context = MockedContext() + + def mock_context_path_exists(path): +@@ -330,10 +368,23 @@ def test_copy_dracut_modules_duplicate_skip(monkeypatch): + + monkeypatch.setattr(os.path, 'exists', mock_context_path_exists) + monkeypatch.setattr(upgradeinitramfsgenerator.api, 'current_logger', MockedLogger()) +- dm = DracutModule(name='foo', module_path='/path/foo') +- dmodules = [dm, dm] +- debugmsg = 'The foo dracut module has been already installed. Skipping.' +- upgradeinitramfsgenerator.copy_dracut_modules(context, dmodules) ++ monkeypatch.setattr(upgradeinitramfsgenerator, '_get_target_kernel_modules_dir', lambda _: '/kernel_modules') ++ ++ module_class = None ++ copy_fn = None ++ if kind == 'dracut': ++ module_class = DracutModule ++ copy_fn = upgradeinitramfsgenerator.copy_dracut_modules ++ elif kind == 'kernel': ++ module_class = KernelModule ++ copy_fn = upgradeinitramfsgenerator.copy_kernel_modules ++ ++ module = module_class(name='foo', module_path='/path/foo') ++ modules = [module, module] ++ ++ copy_fn(context, modules) ++ ++ debugmsg = 'The foo {kind} module has been already installed. Skipping.'.format(kind=kind) + assert context.content + assert len(context.called_copy_to) == 1 + assert debugmsg in upgradeinitramfsgenerator.api.current_logger.dbgmsg +diff --git a/repos/system_upgrade/common/models/initramfs.py b/repos/system_upgrade/common/models/initramfs.py +index a5d1416e..03b71125 100644 +--- a/repos/system_upgrade/common/models/initramfs.py ++++ b/repos/system_upgrade/common/models/initramfs.py +@@ -40,6 +40,46 @@ class DracutModule(Model): + """ + + ++class KernelModule(Model): ++ """ ++ Specify a kernel module that should be included into the initramfs ++ ++ The specified kernel module has to be compatible with the target system. ++ ++ See the description of UpgradeInitramfsTasks and TargetInitramfsTasks ++ for more information about the role of initramfs in the in-place upgrade ++ process. ++ """ ++ topic = BootPrepTopic ++ ++ name = fields.String() ++ """ ++ The kernel module that should be added (--add-drivers option of dracut) ++ when a initramfs is built. The possible options are ++ ++ 1. ``=[/...]`` like ``=drivers/hid`` ++ 2. ```` ++ """ ++ ++ module_path = fields.Nullable(fields.String(default=None)) ++ """ ++ module_path specifies kernel modules that are supposed to be copied ++ ++ If the path is not set, the given name will just be activated. IOW, ++ if the kernel module is stored outside the /usr/lib/modules/$(uname -r)/ ++ directory, set the absolute path to it, so leapp will manage it during ++ the upgrade to ensure the module will be added into the initramfs. ++ ++ The module has to be stored on the local storage mounted in a persistent ++ fashion (/etc/fstab entry). In such a case, it is recommended to store it ++ into the 'files' directory of an actor generating this object. ++ ++ Note: It's expected to set the full path from the host POV. In case ++ of actions inside containers, the module is still copied from the HOST ++ into the container workspace. ++ """ ++ ++ + class UpgradeInitramfsTasks(Model): + """ + Influence generating of the (leapp) upgrade initramfs +@@ -73,6 +113,13 @@ class UpgradeInitramfsTasks(Model): + See the DracutModule model for more information. + """ + ++ include_kernel_modules = fields.List(fields.Model(KernelModule), default=[]) ++ """ ++ List of kernel modules that should be installed in the initramfs. ++ ++ See the KernelModule model for more information. ++ """ ++ + + class TargetInitramfsTasks(UpgradeInitramfsTasks): + """ +-- +2.41.0 + diff --git a/0032-Use-correct-flag-and-ENV-var-to-disable-insights-reg.patch b/0032-Use-correct-flag-and-ENV-var-to-disable-insights-reg.patch new file mode 100644 index 0000000..12deebb --- /dev/null +++ b/0032-Use-correct-flag-and-ENV-var-to-disable-insights-reg.patch @@ -0,0 +1,33 @@ +From f9eef56f9555120117d5d9df0ed46e5517562fd3 Mon Sep 17 00:00:00 2001 +From: Christoph Dwertmann +Date: Tue, 18 Jul 2023 03:36:42 +1000 +Subject: [PATCH 32/42] Use correct flag and ENV var to disable insights + registration (#1089) + +Doc: Fix doc for disabling registration to RH Insights + +The original document speaks about `LEAPP_NO_INSIGHTS_AUTOREGISTER` and +the `--no-insights-autoregister` option. However the correct envar is +`LEAPP_NO_INSIGHTS_REGISTER` and the option is `--no-insights-register` +--- + .../libraries/checkinsightsautoregister.py | 4 ++-- + 1 file changed, 2 insertions(+), 2 deletions(-) + +diff --git a/repos/system_upgrade/common/actors/checkinsightsautoregister/libraries/checkinsightsautoregister.py b/repos/system_upgrade/common/actors/checkinsightsautoregister/libraries/checkinsightsautoregister.py +index 98cf8e2e..762f3c08 100644 +--- a/repos/system_upgrade/common/actors/checkinsightsautoregister/libraries/checkinsightsautoregister.py ++++ b/repos/system_upgrade/common/actors/checkinsightsautoregister/libraries/checkinsightsautoregister.py +@@ -28,8 +28,8 @@ def _report_registration_info(installing_client): + summary = ( + "After the upgrade, this system will be automatically registered into Red Hat Insights." + "{}" +- " To skip the automatic registration, use the '--no-insights-autoregister' command line option or" +- " set the NO_INSIGHTS_AUTOREGISTER environment variable." ++ " To skip the automatic registration, use the '--no-insights-register' command line option or" ++ " set the LEAPP_NO_INSIGHTS_REGISTER environment variable." + ).format(pkg_msg.format(INSIGHTS_CLIENT_PKG) if installing_client else "") + + reporting.create_report( +-- +2.41.0 + diff --git a/0033-CLI-Use-new-Leapp-output-APIs-reports-summary-better.patch b/0033-CLI-Use-new-Leapp-output-APIs-reports-summary-better.patch new file mode 100644 index 0000000..dbf72a3 --- /dev/null +++ b/0033-CLI-Use-new-Leapp-output-APIs-reports-summary-better.patch @@ -0,0 +1,75 @@ +From f1df66449ce3ca3062ff74a1d93d6a9e478d57f7 Mon Sep 17 00:00:00 2001 +From: Matej Matuska +Date: Thu, 16 Mar 2023 12:23:33 +0100 +Subject: [PATCH 33/42] CLI: Use new Leapp output APIs - reports summary better + +The new Leapp output APIs now display better summary about the +report. See https://github.com/oamg/leapp/pull/818 for more info. + +* Require leapp-framework versio 4.0 +* Suppress redundant-keyword-arg for pylint + pstodulk: we have one error or another and this one is not actually + so important from my POV - I would even argue that it's + not a bad habit +--- + .pylintrc | 1 + + commands/preupgrade/__init__.py | 3 ++- + commands/upgrade/__init__.py | 2 +- + packaging/leapp-repository.spec | 2 +- + 4 files changed, 5 insertions(+), 3 deletions(-) + +diff --git a/.pylintrc b/.pylintrc +index 7ddb58d6..2ef31167 100644 +--- a/.pylintrc ++++ b/.pylintrc +@@ -7,6 +7,7 @@ disable= + no-member, + no-name-in-module, + raising-bad-type, ++ redundant-keyword-arg, # it's one or the other, this one is not so bad at all + # "W" Warnings for stylistic problems or minor programming issues + no-absolute-import, + arguments-differ, +diff --git a/commands/preupgrade/__init__.py b/commands/preupgrade/__init__.py +index 614944cc..15a93110 100644 +--- a/commands/preupgrade/__init__.py ++++ b/commands/preupgrade/__init__.py +@@ -80,7 +80,8 @@ def preupgrade(args, breadcrumbs): + report_inhibitors(context) + report_files = util.get_cfg_files('report', cfg) + log_files = util.get_cfg_files('logs', cfg) +- report_info(report_files, log_files, answerfile_path, fail=workflow.failure) ++ report_info(context, report_files, log_files, answerfile_path, fail=workflow.failure) ++ + if workflow.failure: + sys.exit(1) + +diff --git a/commands/upgrade/__init__.py b/commands/upgrade/__init__.py +index b59bf79f..aa327c3b 100644 +--- a/commands/upgrade/__init__.py ++++ b/commands/upgrade/__init__.py +@@ -110,7 +110,7 @@ def upgrade(args, breadcrumbs): + util.generate_report_files(context, report_schema) + report_files = util.get_cfg_files('report', cfg) + log_files = util.get_cfg_files('logs', cfg) +- report_info(report_files, log_files, answerfile_path, fail=workflow.failure) ++ report_info(context, report_files, log_files, answerfile_path, fail=workflow.failure) + + if workflow.failure: + sys.exit(1) +diff --git a/packaging/leapp-repository.spec b/packaging/leapp-repository.spec +index 2d0d6fd8..0fce25df 100644 +--- a/packaging/leapp-repository.spec ++++ b/packaging/leapp-repository.spec +@@ -100,7 +100,7 @@ Requires: leapp-repository-dependencies = %{leapp_repo_deps} + + # IMPORTANT: this is capability provided by the leapp framework rpm. + # Check that 'version' instead of the real framework rpm version. +-Requires: leapp-framework >= 3.1, leapp-framework < 4 ++Requires: leapp-framework >= 4.0, leapp-framework < 5 + + # Since we provide sub-commands for the leapp utility, we expect the leapp + # tool to be installed as well. +-- +2.41.0 + diff --git a/0034-Update-Grub-on-component-drives-if-boot-is-on-md-dev.patch b/0034-Update-Grub-on-component-drives-if-boot-is-on-md-dev.patch new file mode 100644 index 0000000..8f287b4 --- /dev/null +++ b/0034-Update-Grub-on-component-drives-if-boot-is-on-md-dev.patch @@ -0,0 +1,663 @@ +From 2ba44076625e35aabfd2a1f9e45b2934f99f1e8d Mon Sep 17 00:00:00 2001 +From: Matej Matuska +Date: Mon, 20 Mar 2023 13:27:46 +0100 +Subject: [PATCH 34/42] Update Grub on component drives if /boot is on md + device + +On BIOS systems, previously, if /boot was on md device such as RAID +consisting of multiple partitions on different MBR/GPT partitioned +drives, the part of Grub residing in the 512 Mb after MBR was only +updated for one of the drives. Similar situation occurred on GPT +partitioned drives and the BIOS boot partition. This resulted in +outdated GRUB on the remaining drives which could cause the system to be +unbootable. + +Now, Grub is updated on all the component devices of an md array if Grub +was already installed on them before the upgrade. + +Jira: OAMG-7835 +BZ#2219544 +BZ#2140011 +--- + .../common/actors/checkgrubcore/actor.py | 7 +- + .../checkgrubcore/tests/test_checkgrubcore.py | 9 +- + .../common/actors/scangrubdevice/actor.py | 11 +-- + .../tests/test_scangrubdevice.py | 35 +++++++ + .../common/actors/updategrubcore/actor.py | 8 +- + .../libraries/updategrubcore.py | 48 ++++++---- + .../tests/test_updategrubcore.py | 39 ++++++-- + repos/system_upgrade/common/libraries/grub.py | 28 ++++++ + .../system_upgrade/common/libraries/mdraid.py | 48 ++++++++++ + .../common/libraries/tests/test_grub.py | 71 ++++++++++++-- + .../common/libraries/tests/test_mdraid.py | 94 +++++++++++++++++++ + .../system_upgrade/common/models/grubinfo.py | 12 +++ + 12 files changed, 358 insertions(+), 52 deletions(-) + create mode 100644 repos/system_upgrade/common/actors/scangrubdevice/tests/test_scangrubdevice.py + create mode 100644 repos/system_upgrade/common/libraries/mdraid.py + create mode 100644 repos/system_upgrade/common/libraries/tests/test_mdraid.py + +diff --git a/repos/system_upgrade/common/actors/checkgrubcore/actor.py b/repos/system_upgrade/common/actors/checkgrubcore/actor.py +index 6aa99797..ae9e53ef 100644 +--- a/repos/system_upgrade/common/actors/checkgrubcore/actor.py ++++ b/repos/system_upgrade/common/actors/checkgrubcore/actor.py +@@ -32,7 +32,7 @@ class CheckGrubCore(Actor): + grub_info = next(self.consume(GrubInfo), None) + if not grub_info: + raise StopActorExecutionError('Actor did not receive any GrubInfo message.') +- if grub_info.orig_device_name: ++ if grub_info.orig_devices: + create_report([ + reporting.Title( + 'GRUB2 core will be automatically updated during the upgrade' +@@ -45,8 +45,9 @@ class CheckGrubCore(Actor): + create_report([ + reporting.Title('Leapp could not identify where GRUB2 core is located'), + reporting.Summary( +- 'We assumed GRUB2 core is located on the same device as /boot, however Leapp could not ' +- 'detect GRUB2 on the device. GRUB2 core needs to be updated maually on legacy (BIOS) systems. ' ++ 'We assumed GRUB2 core is located on the same device(s) as /boot, ' ++ 'however Leapp could not detect GRUB2 on the device(s). ' ++ 'GRUB2 core needs to be updated maually on legacy (BIOS) systems. ' + ), + reporting.Severity(reporting.Severity.HIGH), + reporting.Groups([reporting.Groups.BOOT]), +diff --git a/repos/system_upgrade/common/actors/checkgrubcore/tests/test_checkgrubcore.py b/repos/system_upgrade/common/actors/checkgrubcore/tests/test_checkgrubcore.py +index fe15b65b..b834f9fe 100644 +--- a/repos/system_upgrade/common/actors/checkgrubcore/tests/test_checkgrubcore.py ++++ b/repos/system_upgrade/common/actors/checkgrubcore/tests/test_checkgrubcore.py +@@ -1,18 +1,17 @@ +-import pytest +- +-from leapp.exceptions import StopActorExecutionError + from leapp.libraries.common.config import mock_configs + from leapp.models import FirmwareFacts, GrubInfo + from leapp.reporting import Report + + NO_GRUB = 'Leapp could not identify where GRUB2 core is located' ++GRUB = 'GRUB2 core will be automatically updated during the upgrade' + + + def test_actor_update_grub(current_actor_context): + current_actor_context.feed(FirmwareFacts(firmware='bios')) +- current_actor_context.feed(GrubInfo(orig_device_name='/dev/vda')) ++ current_actor_context.feed(GrubInfo(orig_devices=['/dev/vda', '/dev/vdb'])) + current_actor_context.run(config_model=mock_configs.CONFIG) + assert current_actor_context.consume(Report) ++ assert current_actor_context.consume(Report)[0].report['title'].startswith(GRUB) + + + def test_actor_no_grub_device(current_actor_context): +@@ -31,6 +30,6 @@ def test_actor_with_efi(current_actor_context): + + def test_s390x(current_actor_context): + current_actor_context.feed(FirmwareFacts(firmware='bios')) +- current_actor_context.feed(GrubInfo(orig_device_name='/dev/vda')) ++ current_actor_context.feed(GrubInfo(orig_devices=['/dev/vda', '/dev/vdb'])) + current_actor_context.run(config_model=mock_configs.CONFIG_S390X) + assert not current_actor_context.consume(Report) +diff --git a/repos/system_upgrade/common/actors/scangrubdevice/actor.py b/repos/system_upgrade/common/actors/scangrubdevice/actor.py +index a12739e1..cb6be7ea 100644 +--- a/repos/system_upgrade/common/actors/scangrubdevice/actor.py ++++ b/repos/system_upgrade/common/actors/scangrubdevice/actor.py +@@ -7,7 +7,7 @@ from leapp.tags import FactsPhaseTag, IPUWorkflowTag + + class ScanGrubDeviceName(Actor): + """ +- Find the name of the block device where GRUB is located ++ Find the name of the block devices where GRUB is located + """ + + name = 'scan_grub_device_name' +@@ -19,8 +19,7 @@ class ScanGrubDeviceName(Actor): + if architecture.matches_architecture(architecture.ARCH_S390X): + return + +- device_name = grub.get_grub_device() +- if device_name: +- self.produce(GrubInfo(orig_device_name=device_name)) +- else: +- self.produce(GrubInfo()) ++ devices = grub.get_grub_devices() ++ grub_info = GrubInfo(orig_devices=devices) ++ grub_info.orig_device_name = devices[0] if len(devices) == 1 else None ++ self.produce(grub_info) +diff --git a/repos/system_upgrade/common/actors/scangrubdevice/tests/test_scangrubdevice.py b/repos/system_upgrade/common/actors/scangrubdevice/tests/test_scangrubdevice.py +new file mode 100644 +index 00000000..0114d717 +--- /dev/null ++++ b/repos/system_upgrade/common/actors/scangrubdevice/tests/test_scangrubdevice.py +@@ -0,0 +1,35 @@ ++from leapp.libraries.common import grub ++from leapp.libraries.common.config import mock_configs ++from leapp.models import GrubInfo ++ ++ ++def _get_grub_devices_mocked(): ++ return ['/dev/vda', '/dev/vdb'] ++ ++ ++def test_actor_scan_grub_device(current_actor_context, monkeypatch): ++ monkeypatch.setattr(grub, 'get_grub_devices', _get_grub_devices_mocked) ++ current_actor_context.run(config_model=mock_configs.CONFIG) ++ info = current_actor_context.consume(GrubInfo) ++ assert info and info[0].orig_devices == ['/dev/vda', '/dev/vdb'] ++ assert len(info) == 1, 'Expected just one GrubInfo message' ++ assert not info[0].orig_device_name ++ ++ ++def test_actor_scan_grub_device_one(current_actor_context, monkeypatch): ++ ++ def _get_grub_devices_mocked(): ++ return ['/dev/vda'] ++ ++ monkeypatch.setattr(grub, 'get_grub_devices', _get_grub_devices_mocked) ++ current_actor_context.run(config_model=mock_configs.CONFIG) ++ info = current_actor_context.consume(GrubInfo) ++ assert info and info[0].orig_devices == ['/dev/vda'] ++ assert len(info) == 1, 'Expected just one GrubInfo message' ++ assert info[0].orig_device_name == '/dev/vda' ++ ++ ++def test_actor_scan_grub_device_s390x(current_actor_context, monkeypatch): ++ monkeypatch.setattr(grub, 'get_grub_devices', _get_grub_devices_mocked) ++ current_actor_context.run(config_model=mock_configs.CONFIG_S390X) ++ assert not current_actor_context.consume(GrubInfo) +diff --git a/repos/system_upgrade/common/actors/updategrubcore/actor.py b/repos/system_upgrade/common/actors/updategrubcore/actor.py +index 4545bad6..ac9aa829 100644 +--- a/repos/system_upgrade/common/actors/updategrubcore/actor.py ++++ b/repos/system_upgrade/common/actors/updategrubcore/actor.py +@@ -21,8 +21,8 @@ class UpdateGrubCore(Actor): + def process(self): + ff = next(self.consume(FirmwareFacts), None) + if ff and ff.firmware == 'bios': +- grub_dev = grub.get_grub_device() +- if grub_dev: +- update_grub_core(grub_dev) ++ grub_devs = grub.get_grub_devices() ++ if grub_devs: ++ update_grub_core(grub_devs) + else: +- api.current_logger().warning('Leapp could not detect GRUB on {}'.format(grub_dev)) ++ api.current_logger().warning('Leapp could not detect GRUB devices') +diff --git a/repos/system_upgrade/common/actors/updategrubcore/libraries/updategrubcore.py b/repos/system_upgrade/common/actors/updategrubcore/libraries/updategrubcore.py +index 22ee3372..2bdad929 100644 +--- a/repos/system_upgrade/common/actors/updategrubcore/libraries/updategrubcore.py ++++ b/repos/system_upgrade/common/actors/updategrubcore/libraries/updategrubcore.py +@@ -1,35 +1,43 @@ + from leapp import reporting +-from leapp.exceptions import StopActorExecution + from leapp.libraries.stdlib import api, CalledProcessError, config, run + + +-def update_grub_core(grub_dev): ++def update_grub_core(grub_devs): + """ + Update GRUB core after upgrade from RHEL7 to RHEL8 + + On legacy systems, GRUB core does not get automatically updated when GRUB packages + are updated. + """ +- cmd = ['grub2-install', grub_dev] +- if config.is_debug(): +- cmd += ['-v'] +- try: +- run(cmd) +- except CalledProcessError as err: +- reporting.create_report([ +- reporting.Title('GRUB core update failed'), +- reporting.Summary(str(err)), +- reporting.Groups([reporting.Groups.BOOT]), +- reporting.Severity(reporting.Severity.HIGH), +- reporting.Remediation( +- hint='Please run "grub2-install " manually after upgrade' +- ) +- ]) +- api.current_logger().warning('GRUB core update on {} failed'.format(grub_dev)) +- raise StopActorExecution() ++ ++ successful = [] ++ failed = [] ++ for dev in grub_devs: ++ cmd = ['grub2-install', dev] ++ if config.is_debug(): ++ cmd += ['-v'] ++ try: ++ run(cmd) ++ except CalledProcessError as err: ++ api.current_logger().warning('GRUB core update on {} failed: {}'.format(dev, err)) ++ failed.append(dev) ++ continue ++ ++ successful.append(dev) ++ ++ reporting.create_report([ ++ reporting.Title('GRUB core update failed'), ++ reporting.Summary('Leapp failed to update GRUB on {}'.format(', '.join(failed))), ++ reporting.Groups([reporting.Groups.BOOT]), ++ reporting.Severity(reporting.Severity.HIGH), ++ reporting.Remediation( ++ hint='Please run "grub2-install " manually after upgrade' ++ ) ++ ]) ++ + reporting.create_report([ + reporting.Title('GRUB core successfully updated'), +- reporting.Summary('GRUB core on {} was successfully updated'.format(grub_dev)), ++ reporting.Summary('GRUB core on {} was successfully updated'.format(', '.join(successful))), + reporting.Groups([reporting.Groups.BOOT]), + reporting.Severity(reporting.Severity.INFO) + ]) +diff --git a/repos/system_upgrade/common/actors/updategrubcore/tests/test_updategrubcore.py b/repos/system_upgrade/common/actors/updategrubcore/tests/test_updategrubcore.py +index e65807a2..fe0cca50 100644 +--- a/repos/system_upgrade/common/actors/updategrubcore/tests/test_updategrubcore.py ++++ b/repos/system_upgrade/common/actors/updategrubcore/tests/test_updategrubcore.py +@@ -1,7 +1,6 @@ + import pytest + + from leapp import reporting +-from leapp.exceptions import StopActorExecution + from leapp.libraries.actor import updategrubcore + from leapp.libraries.common import testutils + from leapp.libraries.stdlib import api, CalledProcessError +@@ -32,21 +31,45 @@ class run_mocked(object): + raise_call_error(args) + + +-def test_update_grub(monkeypatch): ++@pytest.mark.parametrize('devices', [['/dev/vda'], ['/dev/vda', '/dev/vdb']]) ++def test_update_grub(monkeypatch, devices): + monkeypatch.setattr(reporting, "create_report", testutils.create_report_mocked()) + monkeypatch.setattr(updategrubcore, 'run', run_mocked()) +- updategrubcore.update_grub_core('/dev/vda') ++ updategrubcore.update_grub_core(devices) + assert reporting.create_report.called +- assert UPDATE_OK_TITLE == reporting.create_report.report_fields['title'] ++ assert UPDATE_OK_TITLE == reporting.create_report.reports[1]['title'] ++ assert all(dev in reporting.create_report.reports[1]['summary'] for dev in devices) + + +-def test_update_grub_failed(monkeypatch): ++@pytest.mark.parametrize('devices', [['/dev/vda'], ['/dev/vda', '/dev/vdb']]) ++def test_update_grub_failed(monkeypatch, devices): + monkeypatch.setattr(reporting, "create_report", testutils.create_report_mocked()) + monkeypatch.setattr(updategrubcore, 'run', run_mocked(raise_err=True)) +- with pytest.raises(StopActorExecution): +- updategrubcore.update_grub_core('/dev/vda') ++ updategrubcore.update_grub_core(devices) + assert reporting.create_report.called +- assert UPDATE_FAILED_TITLE == reporting.create_report.report_fields['title'] ++ assert UPDATE_FAILED_TITLE == reporting.create_report.reports[0]['title'] ++ assert all(dev in reporting.create_report.reports[0]['summary'] for dev in devices) ++ ++ ++def test_update_grub_success_and_fail(monkeypatch): ++ monkeypatch.setattr(reporting, "create_report", testutils.create_report_mocked()) ++ ++ def run_mocked(args): ++ if args == ['grub2-install', '/dev/vdb']: ++ raise_call_error(args) ++ else: ++ assert args == ['grub2-install', '/dev/vda'] ++ ++ monkeypatch.setattr(updategrubcore, 'run', run_mocked) ++ ++ devices = ['/dev/vda', '/dev/vdb'] ++ updategrubcore.update_grub_core(devices) ++ ++ assert reporting.create_report.called ++ assert UPDATE_FAILED_TITLE == reporting.create_report.reports[0]['title'] ++ assert '/dev/vdb' in reporting.create_report.reports[0]['summary'] ++ assert UPDATE_OK_TITLE == reporting.create_report.reports[1]['title'] ++ assert '/dev/vda' in reporting.create_report.reports[1]['summary'] + + + def test_update_grub_negative(current_actor_context): +diff --git a/repos/system_upgrade/common/libraries/grub.py b/repos/system_upgrade/common/libraries/grub.py +index f6b00f65..79b3be39 100644 +--- a/repos/system_upgrade/common/libraries/grub.py ++++ b/repos/system_upgrade/common/libraries/grub.py +@@ -1,7 +1,9 @@ + import os + + from leapp.exceptions import StopActorExecution ++from leapp.libraries.common import mdraid + from leapp.libraries.stdlib import api, CalledProcessError, run ++from leapp.utils.deprecation import deprecated + + + def has_grub(blk_dev): +@@ -59,6 +61,32 @@ def get_boot_partition(): + return boot_partition + + ++def get_grub_devices(): ++ """ ++ Get block devices where GRUB is located. We assume GRUB is on the same device ++ as /boot partition is. In case that device is an md (Multiple Device) device, all ++ of the component devices of such a device are considered. ++ ++ :return: Devices where GRUB is located ++ :rtype: list ++ """ ++ boot_device = get_boot_partition() ++ devices = [] ++ if mdraid.is_mdraid_dev(boot_device): ++ component_devs = mdraid.get_component_devices(boot_device) ++ blk_devs = [blk_dev_from_partition(dev) for dev in component_devs] ++ # remove duplicates as there might be raid on partitions on the same drive ++ # even if that's very unusual ++ devices = sorted(list(set(blk_devs))) ++ else: ++ devices.append(blk_dev_from_partition(boot_device)) ++ ++ have_grub = [dev for dev in devices if has_grub(dev)] ++ api.current_logger().info('GRUB is installed on {}'.format(",".join(have_grub))) ++ return have_grub ++ ++ ++@deprecated(since='2023-06-23', message='This function has been replaced by get_grub_devices') + def get_grub_device(): + """ + Get block device where GRUB is located. We assume GRUB is on the same device +diff --git a/repos/system_upgrade/common/libraries/mdraid.py b/repos/system_upgrade/common/libraries/mdraid.py +new file mode 100644 +index 00000000..5eb89c56 +--- /dev/null ++++ b/repos/system_upgrade/common/libraries/mdraid.py +@@ -0,0 +1,48 @@ ++from leapp.libraries.stdlib import api, CalledProcessError, run ++ ++ ++def is_mdraid_dev(dev): ++ """ ++ Check if a given device is an md (Multiple Device) device ++ ++ It is expected that the "mdadm" command is available, ++ if it's not it is assumed the device is not an md device. ++ ++ :return: True if the device is an md device, False otherwise ++ :raises CalledProcessError: If an error occurred ++ """ ++ fail_msg = 'Could not check if device "{}" is an md device: {}' ++ try: ++ result = run(['mdadm', '--query', dev]) ++ except OSError as err: ++ api.current_logger().warning(fail_msg.format(dev, err)) ++ return False ++ except CalledProcessError as err: ++ err.message = fail_msg.format(dev, err) ++ raise # let the calling actor handle the exception ++ ++ return '--detail' in result['stdout'] ++ ++ ++def get_component_devices(raid_dev): ++ """ ++ Get list of component devices in an md (Multiple Device) array ++ ++ :return: The list of component devices or None in case of error ++ :raises ValueError: If the device is not an mdraid device ++ """ ++ try: ++ # using both --verbose and --brief for medium verbosity ++ result = run(['mdadm', '--detail', '--verbose', '--brief', raid_dev]) ++ except (OSError, CalledProcessError) as err: ++ api.current_logger().warning( ++ 'Could not get md array component devices: {}'.format(err) ++ ) ++ return None ++ # example output: ++ # ARRAY /dev/md0 level=raid1 num-devices=2 metadata=1.2 name=localhost.localdomain:0 UUID=c4acea6e:d56e1598:91822e3f:fb26832c # noqa: E501; pylint: disable=line-too-long ++ # devices=/dev/vda1,/dev/vdb1 ++ if 'does not appear to be an md device' in result['stdout']: ++ raise ValueError("Expected md device, but got: {}".format(raid_dev)) ++ ++ return sorted(result['stdout'].rsplit('=', 2)[-1].strip().split(',')) +diff --git a/repos/system_upgrade/common/libraries/tests/test_grub.py b/repos/system_upgrade/common/libraries/tests/test_grub.py +index ba086854..9ced1147 100644 +--- a/repos/system_upgrade/common/libraries/tests/test_grub.py ++++ b/repos/system_upgrade/common/libraries/tests/test_grub.py +@@ -3,7 +3,7 @@ import os + import pytest + + from leapp.exceptions import StopActorExecution +-from leapp.libraries.common import grub ++from leapp.libraries.common import grub, mdraid + from leapp.libraries.common.testutils import logger_mocked + from leapp.libraries.stdlib import api, CalledProcessError + from leapp.models import DefaultGrub, DefaultGrubInfo +@@ -11,6 +11,9 @@ from leapp.models import DefaultGrub, DefaultGrubInfo + BOOT_PARTITION = '/dev/vda1' + BOOT_DEVICE = '/dev/vda' + ++MD_BOOT_DEVICE = '/dev/md0' ++MD_BOOT_DEVICES_WITH_GRUB = ['/dev/sda', '/dev/sdb'] ++ + VALID_DD = b'GRUB GeomHard DiskRead Error' + INVALID_DD = b'Nothing to see here!' + +@@ -27,10 +30,11 @@ def raise_call_error(args=None): + + class RunMocked(object): + +- def __init__(self, raise_err=False): ++ def __init__(self, raise_err=False, boot_on_raid=False): + self.called = 0 + self.args = None + self.raise_err = raise_err ++ self.boot_on_raid = boot_on_raid + + def __call__(self, args, encoding=None): + self.called += 1 +@@ -39,18 +43,22 @@ class RunMocked(object): + raise_call_error(args) + + if self.args == ['grub2-probe', '--target=device', '/boot']: +- stdout = BOOT_PARTITION ++ stdout = MD_BOOT_DEVICE if self.boot_on_raid else BOOT_PARTITION + + elif self.args == ['lsblk', '-spnlo', 'name', BOOT_PARTITION]: + stdout = BOOT_DEVICE ++ elif self.args[:-1] == ['lsblk', '-spnlo', 'name']: ++ stdout = self.args[-1][:-1] + + return {'stdout': stdout} + + + def open_mocked(fn, flags): +- return open( +- os.path.join(CUR_DIR, 'grub_valid') if fn == BOOT_DEVICE else os.path.join(CUR_DIR, 'grub_invalid'), 'r' +- ) ++ if fn == BOOT_DEVICE or fn in MD_BOOT_DEVICES_WITH_GRUB: ++ path = os.path.join(CUR_DIR, 'grub_valid') ++ else: ++ path = os.path.join(CUR_DIR, 'grub_invalid') ++ return open(path, 'r') + + + def open_invalid(fn, flags): +@@ -122,3 +130,54 @@ def test_is_blscfg_library(monkeypatch, enabled): + assert result + else: + assert not result ++ ++ ++def is_mdraid_dev_mocked(dev): ++ return dev == '/dev/md0' ++ ++ ++def test_get_grub_devices_one_device(monkeypatch): ++ run_mocked = RunMocked() ++ monkeypatch.setattr(grub, 'run', run_mocked) ++ monkeypatch.setattr(os, 'open', open_mocked) ++ monkeypatch.setattr(os, 'read', read_mocked) ++ monkeypatch.setattr(os, 'close', close_mocked) ++ monkeypatch.setattr(api, 'current_logger', logger_mocked()) ++ monkeypatch.setattr(mdraid, 'is_mdraid_dev', is_mdraid_dev_mocked) ++ ++ result = grub.get_grub_devices() ++ assert grub.run.called == 2 ++ assert [BOOT_DEVICE] == result ++ assert not api.current_logger.warnmsg ++ assert 'GRUB is installed on {}'.format(",".join(result)) in api.current_logger.infomsg ++ ++ ++@pytest.mark.parametrize( ++ ',component_devs,expected', ++ [ ++ (['/dev/sda1', '/dev/sdb1'], MD_BOOT_DEVICES_WITH_GRUB), ++ (['/dev/sda1', '/dev/sdb1', '/dev/sdc1', '/dev/sdd1'], MD_BOOT_DEVICES_WITH_GRUB), ++ (['/dev/sda2', '/dev/sdc1'], ['/dev/sda']), ++ (['/dev/sdd3', '/dev/sdb2'], ['/dev/sdb']), ++ ] ++) ++def test_get_grub_devices_raid_device(monkeypatch, component_devs, expected): ++ run_mocked = RunMocked(boot_on_raid=True) ++ monkeypatch.setattr(grub, 'run', run_mocked) ++ monkeypatch.setattr(os, 'open', open_mocked) ++ monkeypatch.setattr(os, 'read', read_mocked) ++ monkeypatch.setattr(os, 'close', close_mocked) ++ monkeypatch.setattr(api, 'current_logger', logger_mocked()) ++ monkeypatch.setattr(mdraid, 'is_mdraid_dev', is_mdraid_dev_mocked) ++ ++ def get_component_devices_mocked(raid_dev): ++ assert raid_dev == MD_BOOT_DEVICE ++ return component_devs ++ ++ monkeypatch.setattr(mdraid, 'get_component_devices', get_component_devices_mocked) ++ ++ result = grub.get_grub_devices() ++ assert grub.run.called == 1 + len(component_devs) # grub2-probe + Nx lsblk ++ assert sorted(expected) == result ++ assert not api.current_logger.warnmsg ++ assert 'GRUB is installed on {}'.format(",".join(result)) in api.current_logger.infomsg +diff --git a/repos/system_upgrade/common/libraries/tests/test_mdraid.py b/repos/system_upgrade/common/libraries/tests/test_mdraid.py +new file mode 100644 +index 00000000..6a25d736 +--- /dev/null ++++ b/repos/system_upgrade/common/libraries/tests/test_mdraid.py +@@ -0,0 +1,94 @@ ++import os ++ ++import pytest ++ ++from leapp.libraries.common import mdraid ++from leapp.libraries.common.testutils import logger_mocked ++from leapp.libraries.stdlib import api, CalledProcessError ++ ++MD_DEVICE = '/dev/md0' ++NOT_MD_DEVICE = '/dev/sda' ++ ++CUR_DIR = os.path.dirname(os.path.abspath(__file__)) ++ ++ ++def raise_call_error(args=None): ++ raise CalledProcessError( ++ message='A Leapp Command Error occurred.', ++ command=args, ++ result={'signal': None, 'exit_code': 1, 'pid': 0, 'stdout': 'fake', 'stderr': 'fake'} ++ ) ++ ++ ++class RunMocked(object): ++ ++ def __init__(self, raise_err=False): ++ self.called = 0 ++ self.args = None ++ self.raise_err = raise_err ++ ++ def __call__(self, args, encoding=None): ++ self.called += 1 ++ self.args = args ++ if self.raise_err: ++ raise_call_error(args) ++ ++ if self.args == ['mdadm', '--query', MD_DEVICE]: ++ stdout = '/dev/md0: 1022.00MiB raid1 2 devices, 0 spares. Use mdadm --detail for more detail.' ++ elif self.args == ['mdadm', '--query', NOT_MD_DEVICE]: ++ stdout = '/dev/sda: is not an md array' ++ ++ elif self.args == ['mdadm', '--detail', '--verbose', '--brief', MD_DEVICE]: ++ stdout = 'ARRAY /dev/md0 level=raid1 num-devices=2 metadata=1.2 name=localhost.localdomain:0 UUID=c4acea6e:d56e1598:91822e3f:fb26832c\n devices=/dev/sda1,/dev/sdb1' # noqa: E501; pylint: disable=line-too-long ++ elif self.args == ['mdadm', '--detail', '--verbose', '--brief', NOT_MD_DEVICE]: ++ stdout = 'mdadm: /dev/sda does not appear to be an md device' ++ ++ return {'stdout': stdout} ++ ++ ++@pytest.mark.parametrize('dev,expected', [(MD_DEVICE, True), (NOT_MD_DEVICE, False)]) ++def test_is_mdraid_dev(monkeypatch, dev, expected): ++ run_mocked = RunMocked() ++ monkeypatch.setattr(mdraid, 'run', run_mocked) ++ monkeypatch.setattr(api, 'current_logger', logger_mocked()) ++ ++ result = mdraid.is_mdraid_dev(dev) ++ assert mdraid.run.called == 1 ++ assert expected == result ++ assert not api.current_logger.warnmsg ++ ++ ++def test_is_mdraid_dev_error(monkeypatch): ++ run_mocked = RunMocked(raise_err=True) ++ monkeypatch.setattr(mdraid, 'run', run_mocked) ++ monkeypatch.setattr(api, 'current_logger', logger_mocked()) ++ ++ with pytest.raises(CalledProcessError) as err: ++ mdraid.is_mdraid_dev(MD_DEVICE) ++ ++ assert mdraid.run.called == 1 ++ expect_msg = 'Could not check if device "{}" is an md device:'.format(MD_DEVICE) ++ assert expect_msg in err.value.message ++ ++ ++def test_get_component_devices_ok(monkeypatch): ++ run_mocked = RunMocked() ++ monkeypatch.setattr(mdraid, 'run', run_mocked) ++ monkeypatch.setattr(api, 'current_logger', logger_mocked()) ++ ++ result = mdraid.get_component_devices(MD_DEVICE) ++ assert mdraid.run.called == 1 ++ assert ['/dev/sda1', '/dev/sdb1'] == result ++ assert not api.current_logger.warnmsg ++ ++ ++def test_get_component_devices_not_md_device(monkeypatch): ++ run_mocked = RunMocked() ++ monkeypatch.setattr(mdraid, 'run', run_mocked) ++ ++ with pytest.raises(ValueError) as err: ++ mdraid.get_component_devices(NOT_MD_DEVICE) ++ ++ assert mdraid.run.called == 1 ++ expect_msg = 'Expected md device, but got: {}'.format(NOT_MD_DEVICE) ++ assert expect_msg in str(err.value) +diff --git a/repos/system_upgrade/common/models/grubinfo.py b/repos/system_upgrade/common/models/grubinfo.py +index 952d01c1..f89770b4 100644 +--- a/repos/system_upgrade/common/models/grubinfo.py ++++ b/repos/system_upgrade/common/models/grubinfo.py +@@ -8,6 +8,8 @@ class GrubInfo(Model): + """ + topic = SystemFactsTopic + ++ # NOTE: @deprecated is not supported on fields ++ # @deprecated(since='2023-06-23', message='This field has been replaced by orig_devices') + orig_device_name = fields.Nullable(fields.String()) + """ + Original name of the block device where Grub is located. +@@ -17,3 +19,13 @@ class GrubInfo(Model): + it's recommended to use `leapp.libraries.common.grub.get_grub_device()` anywhere + else. + """ ++ ++ orig_devices = fields.List(fields.String(), default=[]) ++ """ ++ Original names of the block devices where Grub is located. ++ ++ The names are persistent during the boot of the system so it's safe to be used during ++ preupgrade phases. However the names could be different after the reboot, so ++ it's recommended to use `leapp.libraries.common.grub.get_grub_devices()` everywhere ++ else. ++ """ +-- +2.41.0 + diff --git a/0035-mdraid.py-lib-Check-if-usr-sbin-mdadm-exists.patch b/0035-mdraid.py-lib-Check-if-usr-sbin-mdadm-exists.patch new file mode 100644 index 0000000..eca5b40 --- /dev/null +++ b/0035-mdraid.py-lib-Check-if-usr-sbin-mdadm-exists.patch @@ -0,0 +1,86 @@ +From 2e85af59af3429e33cba91af844d50a324512bd4 Mon Sep 17 00:00:00 2001 +From: Petr Stodulka +Date: Mon, 17 Jul 2023 18:41:18 +0200 +Subject: [PATCH 35/42] mdraid.py lib: Check if /usr/sbin/mdadm exists + +Praviously the check was implemented using OSError return from `run` +function. However, in this particular case it's not safe and leads +to unexpected behaviour. Check the existence of the file explicitly +instead prior the `run` function is called. + +Update existing unit-tests and extend the test case when mdadm +is not installed. +--- + repos/system_upgrade/common/libraries/mdraid.py | 10 +++++++--- + .../common/libraries/tests/test_mdraid.py | 14 ++++++++++++++ + 2 files changed, 21 insertions(+), 3 deletions(-) + +diff --git a/repos/system_upgrade/common/libraries/mdraid.py b/repos/system_upgrade/common/libraries/mdraid.py +index 5eb89c56..5b59814f 100644 +--- a/repos/system_upgrade/common/libraries/mdraid.py ++++ b/repos/system_upgrade/common/libraries/mdraid.py +@@ -1,3 +1,5 @@ ++import os ++ + from leapp.libraries.stdlib import api, CalledProcessError, run + + +@@ -12,11 +14,13 @@ def is_mdraid_dev(dev): + :raises CalledProcessError: If an error occurred + """ + fail_msg = 'Could not check if device "{}" is an md device: {}' ++ if not os.path.exists('/usr/sbin/mdadm'): ++ api.current_logger().warning(fail_msg.format( ++ dev, '/usr/sbin/mdadm is not installed.' ++ )) ++ return False + try: + result = run(['mdadm', '--query', dev]) +- except OSError as err: +- api.current_logger().warning(fail_msg.format(dev, err)) +- return False + except CalledProcessError as err: + err.message = fail_msg.format(dev, err) + raise # let the calling actor handle the exception +diff --git a/repos/system_upgrade/common/libraries/tests/test_mdraid.py b/repos/system_upgrade/common/libraries/tests/test_mdraid.py +index 6a25d736..cb7c1059 100644 +--- a/repos/system_upgrade/common/libraries/tests/test_mdraid.py ++++ b/repos/system_upgrade/common/libraries/tests/test_mdraid.py +@@ -51,6 +51,7 @@ def test_is_mdraid_dev(monkeypatch, dev, expected): + run_mocked = RunMocked() + monkeypatch.setattr(mdraid, 'run', run_mocked) + monkeypatch.setattr(api, 'current_logger', logger_mocked()) ++ monkeypatch.setattr(os.path, 'exists', lambda dummy: True) + + result = mdraid.is_mdraid_dev(dev) + assert mdraid.run.called == 1 +@@ -62,6 +63,7 @@ def test_is_mdraid_dev_error(monkeypatch): + run_mocked = RunMocked(raise_err=True) + monkeypatch.setattr(mdraid, 'run', run_mocked) + monkeypatch.setattr(api, 'current_logger', logger_mocked()) ++ monkeypatch.setattr(os.path, 'exists', lambda dummy: True) + + with pytest.raises(CalledProcessError) as err: + mdraid.is_mdraid_dev(MD_DEVICE) +@@ -71,6 +73,18 @@ def test_is_mdraid_dev_error(monkeypatch): + assert expect_msg in err.value.message + + ++def test_is_mdraid_dev_notool(monkeypatch): ++ run_mocked = RunMocked(raise_err=True) ++ monkeypatch.setattr(mdraid, 'run', run_mocked) ++ monkeypatch.setattr(api, 'current_logger', logger_mocked()) ++ monkeypatch.setattr(os.path, 'exists', lambda dummy: False) ++ ++ result = mdraid.is_mdraid_dev(MD_DEVICE) ++ assert not result ++ assert not mdraid.run.called ++ assert api.current_logger.warnmsg ++ ++ + def test_get_component_devices_ok(monkeypatch): + run_mocked = RunMocked() + monkeypatch.setattr(mdraid, 'run', run_mocked) +-- +2.41.0 + diff --git a/0036-target_userspace_creator-Use-MOVE-instead-of-copy-fo.patch b/0036-target_userspace_creator-Use-MOVE-instead-of-copy-fo.patch new file mode 100644 index 0000000..a1c8654 --- /dev/null +++ b/0036-target_userspace_creator-Use-MOVE-instead-of-copy-fo.patch @@ -0,0 +1,66 @@ +From e76e5cebeb41125a2075fafaba94faca66df5476 Mon Sep 17 00:00:00 2001 +From: Petr Stodulka +Date: Thu, 13 Jul 2023 15:38:22 +0200 +Subject: [PATCH 36/42] target_userspace_creator: Use MOVE instead of copy for + the persistent cache + +If leapp is executed with LEAPP_DEVEL_USE_PERSISTENT_PACKAGE_CACHE=1, +the /var/dnf/cache from the target container has been copied under + /var/lib/leapp/persistent_package_cache +The negative effect was that it took too much space on the disk +(800+ MBs, depends on how much rpms have been downloaded before..) +which could lead easily to the consumed disk space on related partition, +which eventually could stop also the leapp execution as it cannot +do any meaningful operations when the disk is full (e.g. access the +database). + +This is done now without nspawn context functions as the move operation +does not make so much sense to be implemented as it's more expected +to copy to/from the container than moving files/dirs. +--- + .../libraries/userspacegen.py | 16 ++++++++++------ + 1 file changed, 10 insertions(+), 6 deletions(-) + +diff --git a/repos/system_upgrade/common/actors/targetuserspacecreator/libraries/userspacegen.py b/repos/system_upgrade/common/actors/targetuserspacecreator/libraries/userspacegen.py +index cad923fb..4cff7b30 100644 +--- a/repos/system_upgrade/common/actors/targetuserspacecreator/libraries/userspacegen.py ++++ b/repos/system_upgrade/common/actors/targetuserspacecreator/libraries/userspacegen.py +@@ -1,5 +1,6 @@ + import itertools + import os ++import shutil + + from leapp import reporting + from leapp.exceptions import StopActorExecution, StopActorExecutionError +@@ -121,9 +122,12 @@ class _InputData(object): + + def _restore_persistent_package_cache(userspace_dir): + if get_env('LEAPP_DEVEL_USE_PERSISTENT_PACKAGE_CACHE', None) == '1': +- if os.path.exists(PERSISTENT_PACKAGE_CACHE_DIR): +- with mounting.NspawnActions(base_dir=userspace_dir) as target_context: +- target_context.copytree_to(PERSISTENT_PACKAGE_CACHE_DIR, '/var/cache/dnf') ++ if not os.path.exists(PERSISTENT_PACKAGE_CACHE_DIR): ++ return ++ dst_cache = os.path.join(userspace_dir, 'var', 'cache', 'dnf') ++ if os.path.exists(dst_cache): ++ run(['rm', '-rf', dst_cache]) ++ shutil.move(PERSISTENT_PACKAGE_CACHE_DIR, dst_cache) + # We always want to remove the persistent cache here to unclutter the system + run(['rm', '-rf', PERSISTENT_PACKAGE_CACHE_DIR]) + +@@ -132,9 +136,9 @@ def _backup_to_persistent_package_cache(userspace_dir): + if get_env('LEAPP_DEVEL_USE_PERSISTENT_PACKAGE_CACHE', None) == '1': + # Clean up any dead bodies, just in case + run(['rm', '-rf', PERSISTENT_PACKAGE_CACHE_DIR]) +- if os.path.exists(os.path.join(userspace_dir, 'var', 'cache', 'dnf')): +- with mounting.NspawnActions(base_dir=userspace_dir) as target_context: +- target_context.copytree_from('/var/cache/dnf', PERSISTENT_PACKAGE_CACHE_DIR) ++ src_cache = os.path.join(userspace_dir, 'var', 'cache', 'dnf') ++ if os.path.exists(src_cache): ++ shutil.move(src_cache, PERSISTENT_PACKAGE_CACHE_DIR) + + + def _the_nogpgcheck_option_used(): +-- +2.41.0 + diff --git a/0037-overlay-lib-Deprecate-old-ovl-internal-functions-ref.patch b/0037-overlay-lib-Deprecate-old-ovl-internal-functions-ref.patch new file mode 100644 index 0000000..385ac92 --- /dev/null +++ b/0037-overlay-lib-Deprecate-old-ovl-internal-functions-ref.patch @@ -0,0 +1,293 @@ +From e4fa8671351a73ddd6b56c70a7834a2c304df9cc Mon Sep 17 00:00:00 2001 +From: Petr Stodulka +Date: Mon, 10 Jul 2023 15:20:24 +0200 +Subject: [PATCH 37/42] overlay lib: Deprecate old ovl internal functions + (refactoring) + +We are going to redesign the use of overlay images during the upgrade +to resolve number of issues we have with the old solution. However, +we need to keep the old solution as a fallback (read below). This +is small preparation to keep the new and old code separated safely. + +Reasoning for the fallback: +* There is a chance the new solution could raise also some problems +mainly for systems with many partitions/volumes in fstab, or when +they are using many loop devices already - as the new solution will +require to create loop device for each partition/volume noted in +the fstab. +* Also RHEL 7 is going to switch to ELS on Jun 2024 after which the +project will be fixing just critical bugfixes for in-place upgrades. +This problem blocking the upgrade is not considered to be critical. +--- + .../common/libraries/overlaygen.py | 223 +++++++++--------- + 1 file changed, 117 insertions(+), 106 deletions(-) + +diff --git a/repos/system_upgrade/common/libraries/overlaygen.py b/repos/system_upgrade/common/libraries/overlaygen.py +index b544f88c..e0d88fe5 100644 +--- a/repos/system_upgrade/common/libraries/overlaygen.py ++++ b/repos/system_upgrade/common/libraries/overlaygen.py +@@ -13,15 +13,6 @@ OVERLAY_DO_NOT_MOUNT = ('tmpfs', 'devpts', 'sysfs', 'proc', 'cramfs', 'sysv', 'v + MountPoints = namedtuple('MountPoints', ['fs_file', 'fs_vfstype']) + + +-def _ensure_enough_diskimage_space(space_needed, directory): +- stat = os.statvfs(directory) +- if (stat.f_frsize * stat.f_bavail) < (space_needed * 1024 * 1024): +- message = ('Not enough space available for creating required disk images in {directory}. ' + +- 'Needed: {space_needed} MiB').format(space_needed=space_needed, directory=directory) +- api.current_logger().error(message) +- raise StopActorExecutionError(message) +- +- + def _get_mountpoints(storage_info): + mount_points = set() + for entry in storage_info.fstab: +@@ -43,41 +34,6 @@ def _mount_dir(mounts_dir, mountpoint): + return os.path.join(mounts_dir, _mount_name(mountpoint)) + + +-def _prepare_required_mounts(scratch_dir, mounts_dir, mount_points, xfs_info): +- result = { +- mount_point.fs_file: mounting.NullMount( +- _mount_dir(mounts_dir, mount_point.fs_file)) for mount_point in mount_points +- } +- +- if not xfs_info.mountpoints_without_ftype: +- return result +- +- space_needed = _overlay_disk_size() * len(xfs_info.mountpoints_without_ftype) +- disk_images_directory = os.path.join(scratch_dir, 'diskimages') +- +- # Ensure we cleanup old disk images before we check for space constraints. +- run(['rm', '-rf', disk_images_directory]) +- _create_diskimages_dir(scratch_dir, disk_images_directory) +- _ensure_enough_diskimage_space(space_needed, scratch_dir) +- +- mount_names = [mount_point.fs_file for mount_point in mount_points] +- +- # TODO(pstodulk): this (adding rootfs into the set always) is hotfix for +- # bz #1911802 (not ideal one..). The problem occurs one rootfs is ext4 fs, +- # but /var/lib/leapp/... is under XFS without ftype; In such a case we can +- # see still the very same problems as before. But letting you know that +- # probably this is not the final solution, as we could possibly see the +- # same problems on another partitions too (needs to be tested...). However, +- # it could fit for now until we provide the complete solution around XFS +- # workarounds (including management of required spaces for virtual FSs per +- # mountpoints - without that, we cannot fix this properly) +- for mountpoint in set(xfs_info.mountpoints_without_ftype + ['/']): +- if mountpoint in mount_names: +- image = _create_mount_disk_image(disk_images_directory, mountpoint) +- result[mountpoint] = mounting.LoopMount(source=image, target=_mount_dir(mounts_dir, mountpoint)) +- return result +- +- + @contextlib.contextmanager + def _build_overlay_mount(root_mount, mounts): + if not root_mount: +@@ -96,21 +52,6 @@ def _build_overlay_mount(root_mount, mounts): + yield mount + + +-def _overlay_disk_size(): +- """ +- Convenient function to retrieve the overlay disk size +- """ +- try: +- env_size = os.getenv('LEAPP_OVL_SIZE', default='2048') +- disk_size = int(env_size) +- except ValueError: +- disk_size = 2048 +- api.current_logger().warning( +- 'Invalid "LEAPP_OVL_SIZE" environment variable "%s". Setting default "%d" value', env_size, disk_size +- ) +- return disk_size +- +- + def cleanup_scratch(scratch_dir, mounts_dir): + """ + Function to cleanup the scratch directory +@@ -128,52 +69,6 @@ def cleanup_scratch(scratch_dir, mounts_dir): + api.current_logger().debug('Recursively removed scratch directory %s.', scratch_dir) + + +-def _create_mount_disk_image(disk_images_directory, path): +- """ +- Creates the mount disk image, for cases when we hit XFS with ftype=0 +- """ +- diskimage_path = os.path.join(disk_images_directory, _mount_name(path)) +- disk_size = _overlay_disk_size() +- +- api.current_logger().debug('Attempting to create disk image with size %d MiB at %s', disk_size, diskimage_path) +- utils.call_with_failure_hint( +- cmd=['/bin/dd', 'if=/dev/zero', 'of={}'.format(diskimage_path), 'bs=1M', 'count={}'.format(disk_size)], +- hint='Please ensure that there is enough diskspace in {} at least {} MiB are needed'.format( +- diskimage_path, disk_size) +- ) +- +- api.current_logger().debug('Creating ext4 filesystem in disk image at %s', diskimage_path) +- try: +- utils.call_with_oserror_handled(cmd=['/sbin/mkfs.ext4', '-F', diskimage_path]) +- except CalledProcessError as e: +- api.current_logger().error('Failed to create ext4 filesystem %s', exc_info=True) +- raise StopActorExecutionError( +- message=str(e) +- ) +- +- return diskimage_path +- +- +-def _create_diskimages_dir(scratch_dir, diskimages_dir): +- """ +- Prepares directories for disk images +- """ +- api.current_logger().debug('Creating disk images directory.') +- try: +- utils.makedirs(diskimages_dir) +- api.current_logger().debug('Done creating disk images directory.') +- except OSError: +- api.current_logger().error('Failed to create disk images directory %s', diskimages_dir, exc_info=True) +- +- # This is an attempt for giving the user a chance to resolve it on their own +- raise StopActorExecutionError( +- message='Failed to prepare environment for package download while creating directories.', +- details={ +- 'hint': 'Please ensure that {scratch_dir} is empty and modifiable.'.format(scratch_dir=scratch_dir) +- } +- ) +- +- + def _create_mounts_dir(scratch_dir, mounts_dir): + """ + Prepares directories for mounts +@@ -214,7 +109,7 @@ def create_source_overlay(mounts_dir, scratch_dir, xfs_info, storage_info, mount + scratch_dir=scratch_dir, mounts_dir=mounts_dir)) + try: + _create_mounts_dir(scratch_dir, mounts_dir) +- mounts = _prepare_required_mounts(scratch_dir, mounts_dir, _get_mountpoints(storage_info), xfs_info) ++ mounts = _prepare_required_mounts_old(scratch_dir, mounts_dir, _get_mountpoints(storage_info), xfs_info) + with mounts.pop('/') as root_mount: + with mounting.OverlayMount(name='system_overlay', source='/', workdir=root_mount.target) as root_overlay: + if mount_target: +@@ -228,3 +123,119 @@ def create_source_overlay(mounts_dir, scratch_dir, xfs_info, storage_info, mount + except Exception: + cleanup_scratch(scratch_dir, mounts_dir) + raise ++ ++ ++# ############################################################################# ++# Deprecated OVL solution ... ++# This is going to be removed in future as the whole functionality is going to ++# be replaced by new one. The problem is that the new solution can potentially ++# negatively affect systems with many loop mountpoints, so let's keep this ++# as a workaround for now. I am separating the old and new code in this way ++# to make the future removal easy. ++# IMPORTANT: Before an update of functions above, ensure the functionality of ++# the code below is not affected, otherwise copy the function below with the ++# "_old" suffix. ++# ############################################################################# ++def _ensure_enough_diskimage_space_old(space_needed, directory): ++ stat = os.statvfs(directory) ++ if (stat.f_frsize * stat.f_bavail) < (space_needed * 1024 * 1024): ++ message = ('Not enough space available for creating required disk images in {directory}. ' + ++ 'Needed: {space_needed} MiB').format(space_needed=space_needed, directory=directory) ++ api.current_logger().error(message) ++ raise StopActorExecutionError(message) ++ ++ ++def _overlay_disk_size_old(): ++ """ ++ Convenient function to retrieve the overlay disk size ++ """ ++ try: ++ env_size = os.getenv('LEAPP_OVL_SIZE', default='2048') ++ disk_size = int(env_size) ++ except ValueError: ++ disk_size = 2048 ++ api.current_logger().warning( ++ 'Invalid "LEAPP_OVL_SIZE" environment variable "%s". Setting default "%d" value', env_size, disk_size ++ ) ++ return disk_size ++ ++ ++def _create_diskimages_dir_old(scratch_dir, diskimages_dir): ++ """ ++ Prepares directories for disk images ++ """ ++ api.current_logger().debug('Creating disk images directory.') ++ try: ++ utils.makedirs(diskimages_dir) ++ api.current_logger().debug('Done creating disk images directory.') ++ except OSError: ++ api.current_logger().error('Failed to create disk images directory %s', diskimages_dir, exc_info=True) ++ ++ # This is an attempt for giving the user a chance to resolve it on their own ++ raise StopActorExecutionError( ++ message='Failed to prepare environment for package download while creating directories.', ++ details={ ++ 'hint': 'Please ensure that {scratch_dir} is empty and modifiable.'.format(scratch_dir=scratch_dir) ++ } ++ ) ++ ++ ++def _create_mount_disk_image_old(disk_images_directory, path): ++ """ ++ Creates the mount disk image, for cases when we hit XFS with ftype=0 ++ """ ++ diskimage_path = os.path.join(disk_images_directory, _mount_name(path)) ++ disk_size = _overlay_disk_size_old() ++ ++ api.current_logger().debug('Attempting to create disk image with size %d MiB at %s', disk_size, diskimage_path) ++ utils.call_with_failure_hint( ++ cmd=['/bin/dd', 'if=/dev/zero', 'of={}'.format(diskimage_path), 'bs=1M', 'count={}'.format(disk_size)], ++ hint='Please ensure that there is enough diskspace in {} at least {} MiB are needed'.format( ++ diskimage_path, disk_size) ++ ) ++ ++ api.current_logger().debug('Creating ext4 filesystem in disk image at %s', diskimage_path) ++ try: ++ utils.call_with_oserror_handled(cmd=['/sbin/mkfs.ext4', '-F', diskimage_path]) ++ except CalledProcessError as e: ++ api.current_logger().error('Failed to create ext4 filesystem %s', exc_info=True) ++ raise StopActorExecutionError( ++ message=str(e) ++ ) ++ ++ return diskimage_path ++ ++ ++def _prepare_required_mounts_old(scratch_dir, mounts_dir, mount_points, xfs_info): ++ result = { ++ mount_point.fs_file: mounting.NullMount( ++ _mount_dir(mounts_dir, mount_point.fs_file)) for mount_point in mount_points ++ } ++ ++ if not xfs_info.mountpoints_without_ftype: ++ return result ++ ++ space_needed = _overlay_disk_size_old() * len(xfs_info.mountpoints_without_ftype) ++ disk_images_directory = os.path.join(scratch_dir, 'diskimages') ++ ++ # Ensure we cleanup old disk images before we check for space constraints. ++ run(['rm', '-rf', disk_images_directory]) ++ _create_diskimages_dir_old(scratch_dir, disk_images_directory) ++ _ensure_enough_diskimage_space_old(space_needed, scratch_dir) ++ ++ mount_names = [mount_point.fs_file for mount_point in mount_points] ++ ++ # TODO(pstodulk): this (adding rootfs into the set always) is hotfix for ++ # bz #1911802 (not ideal one..). The problem occurs one rootfs is ext4 fs, ++ # but /var/lib/leapp/... is under XFS without ftype; In such a case we can ++ # see still the very same problems as before. But letting you know that ++ # probably this is not the final solution, as we could possibly see the ++ # same problems on another partitions too (needs to be tested...). However, ++ # it could fit for now until we provide the complete solution around XFS ++ # workarounds (including management of required spaces for virtual FSs per ++ # mountpoints - without that, we cannot fix this properly) ++ for mountpoint in set(xfs_info.mountpoints_without_ftype + ['/']): ++ if mountpoint in mount_names: ++ image = _create_mount_disk_image_old(disk_images_directory, mountpoint) ++ result[mountpoint] = mounting.LoopMount(source=image, target=_mount_dir(mounts_dir, mountpoint)) ++ return result +-- +2.41.0 + diff --git a/0038-overlay-lib-replace-os.getenv-common.config.get_env.patch b/0038-overlay-lib-replace-os.getenv-common.config.get_env.patch new file mode 100644 index 0000000..2212761 --- /dev/null +++ b/0038-overlay-lib-replace-os.getenv-common.config.get_env.patch @@ -0,0 +1,35 @@ +From dfd1093e9bde660a33e1705143589ec79e9970b1 Mon Sep 17 00:00:00 2001 +From: Petr Stodulka +Date: Mon, 10 Jul 2023 15:47:19 +0200 +Subject: [PATCH 38/42] overlay lib: replace os.getenv common.config.get_env + +All LEAPP_* envars are supposed to be read by library function +which ensures persistent behaviour during the whole upgrade process. +--- + repos/system_upgrade/common/libraries/overlaygen.py | 3 ++- + 1 file changed, 2 insertions(+), 1 deletion(-) + +diff --git a/repos/system_upgrade/common/libraries/overlaygen.py b/repos/system_upgrade/common/libraries/overlaygen.py +index e0d88fe5..1e9c89f6 100644 +--- a/repos/system_upgrade/common/libraries/overlaygen.py ++++ b/repos/system_upgrade/common/libraries/overlaygen.py +@@ -5,6 +5,7 @@ from collections import namedtuple + + from leapp.exceptions import StopActorExecutionError + from leapp.libraries.common import mounting, utils ++from leapp.libraries.common.config import get_env + from leapp.libraries.stdlib import api, CalledProcessError, run + + OVERLAY_DO_NOT_MOUNT = ('tmpfs', 'devpts', 'sysfs', 'proc', 'cramfs', 'sysv', 'vfat') +@@ -150,7 +151,7 @@ def _overlay_disk_size_old(): + Convenient function to retrieve the overlay disk size + """ + try: +- env_size = os.getenv('LEAPP_OVL_SIZE', default='2048') ++ env_size = get_env('LEAPP_OVL_SIZE', '2048') + disk_size = int(env_size) + except ValueError: + disk_size = 2048 +-- +2.41.0 + diff --git a/0039-overlay-lib-Redesign-creation-of-the-source-overlay-.patch b/0039-overlay-lib-Redesign-creation-of-the-source-overlay-.patch new file mode 100644 index 0000000..0b7f285 --- /dev/null +++ b/0039-overlay-lib-Redesign-creation-of-the-source-overlay-.patch @@ -0,0 +1,652 @@ +From d074926c75eebc56ca640e7367638bbeaa1b61a2 Mon Sep 17 00:00:00 2001 +From: Petr Stodulka +Date: Mon, 10 Jul 2023 21:31:24 +0200 +Subject: [PATCH 39/42] overlay lib: Redesign creation of the source overlay + composition + +The in-place upgrade itself requires to do some changes on the system to be +able to perform the in-place upgrade itself - or even to be able to evaluate +if the system is possible to upgrade. However, we do not want to (and must not) +change the original system until we pass beyond the point of not return. + +For that purposes we have to create a layer above the real host file system, +where we can safely perform all operations without affecting the system +setup, rpm database, etc. Currently overlay (OVL) technology showed it is +capable to handle our requirements good enough - with some limitations. + +However, the original design we used to compose overlay layer above +the host system had number of problems: + * buggy calculation of the required free space for the upgrade RPM + transaction + * consumed too much space to handle partitions formatted with XFS + without ftype attributes (even tens GBs) + * bad UX as people had to manually adjust size of OVL disk images + * .. and couple of additional issues derivated from problems + listed above + +The new solution prepares a disk image (represented by sparse-file) +and an overlay image for each mountpoint configured in /etc/fstab, +excluding those with FS types noted in the `OVERLAY_DO_NOT_MOUNT` +set. Such prepared OVL images are then composed together to reflect +the real host filesystem. In the end everything is cleaned. + +The composition could look like this: + orig mountpoint -> disk img -> overlay img -> new mountpoint + ------------------------------------------------------------- + / -> root_ -> root_/ovl -> root_/ovl/ + /boot -> root_boot -> root_boot/ovl -> root_/ovl/boot + /var -> root_var -> root_var/ovl -> root_/ovl/var + /var/lib -> root_var_lib -> root_var_lib/ovl -> root_/ovl/var/lib + ... + +The new solution can be now problematic for system with too many partitions +and loop devices, as each disk image is loop mounted (that's same as +before, but number of disk images will be bigger in total number). +For such systems we keep for now the possibility of the fallback +to an old solution, which has number of issues mentioned above, +but it's a trade of. To fallback to the old solution, set envar: + LEAPP_OVL_LEGACY=1 + +Disk images created for OVL are formatted with XFS by default. In case of +problems, it's possible to switch to Ext4 FS using: + LEAPP_OVL_IMG_FS_EXT4=1 +XFS is better optimized for our use cases (faster initialisation +consuming less space). However we have reported several issues related +to overlay images, that happened so far only on XFS filesystems. +We are not sure about root causes, but having the possibility +to switch to Ext4 seems to be wise. In case of issues, we can simple +ask users to try the switch and see if the problem is fixed or still +present. + +Some additional technical details about other changes +* Added simple/naive checks whether the system has enough space on + the partition hosting /var/lib/leapp (usually /var). Consuming the + all space on the partition could lead to unwanted behaviour + - in the worst case if we speak about /var partition it could mean + problems also for other applications running on the system +* In case the container is larger than the expected min default or + the calculation of the required free space is lower than the + minimal protected size, return the protected size constant + (200 MiB). + +* Work just with mountpoints (paths) in the _prepare_required_mounts() +instead of with list of MountPoint named tuple. I think about the +removal of the named tuple, but let's keep it for now. + +* Make apparent size of created disk images 5% smaller to protect + failed upgrades during the transaction execution due to really + small amount of free space. + +* Cleanup the scratch directory at the end to free the consumed + space. Disks are kept after the run of leapp when + LEAPP_DEVEL_KEEP_DISK_IMGS=1 +--- + .../libraries/userspacegen.py | 4 +- + .../common/libraries/dnfplugin.py | 4 +- + .../common/libraries/overlaygen.py | 441 +++++++++++++++++- + 3 files changed, 445 insertions(+), 4 deletions(-) + +diff --git a/repos/system_upgrade/common/actors/targetuserspacecreator/libraries/userspacegen.py b/repos/system_upgrade/common/actors/targetuserspacecreator/libraries/userspacegen.py +index 4cff7b30..8400dbe7 100644 +--- a/repos/system_upgrade/common/actors/targetuserspacecreator/libraries/userspacegen.py ++++ b/repos/system_upgrade/common/actors/targetuserspacecreator/libraries/userspacegen.py +@@ -766,11 +766,13 @@ def perform(): + + indata = _InputData() + prod_cert_path = _get_product_certificate_path() ++ reserve_space = overlaygen.get_recommended_leapp_free_space(_get_target_userspace()) + with overlaygen.create_source_overlay( + mounts_dir=constants.MOUNTS_DIR, + scratch_dir=constants.SCRATCH_DIR, + storage_info=indata.storage_info, +- xfs_info=indata.xfs_info) as overlay: ++ xfs_info=indata.xfs_info, ++ scratch_reserve=reserve_space) as overlay: + with overlay.nspawn() as context: + # Mount the ISO into the scratch container + target_iso = next(api.consume(TargetOSInstallationImage), None) +diff --git a/repos/system_upgrade/common/libraries/dnfplugin.py b/repos/system_upgrade/common/libraries/dnfplugin.py +index 57b25909..fb0e8ae5 100644 +--- a/repos/system_upgrade/common/libraries/dnfplugin.py ++++ b/repos/system_upgrade/common/libraries/dnfplugin.py +@@ -381,12 +381,14 @@ def perform_transaction_install(target_userspace_info, storage_info, used_repos, + + @contextlib.contextmanager + def _prepare_perform(used_repos, target_userspace_info, xfs_info, storage_info, target_iso=None): ++ reserve_space = overlaygen.get_recommended_leapp_free_space(target_userspace_info.path) + with _prepare_transaction(used_repos=used_repos, + target_userspace_info=target_userspace_info + ) as (context, target_repoids, userspace_info): + with overlaygen.create_source_overlay(mounts_dir=userspace_info.mounts, scratch_dir=userspace_info.scratch, + xfs_info=xfs_info, storage_info=storage_info, +- mount_target=os.path.join(context.base_dir, 'installroot')) as overlay: ++ mount_target=os.path.join(context.base_dir, 'installroot'), ++ scratch_reserve=reserve_space) as overlay: + with mounting.mount_upgrade_iso_to_root_dir(target_userspace_info.path, target_iso): + yield context, overlay, target_repoids + +diff --git a/repos/system_upgrade/common/libraries/overlaygen.py b/repos/system_upgrade/common/libraries/overlaygen.py +index 1e9c89f6..3ffdd176 100644 +--- a/repos/system_upgrade/common/libraries/overlaygen.py ++++ b/repos/system_upgrade/common/libraries/overlaygen.py +@@ -6,20 +6,204 @@ from collections import namedtuple + from leapp.exceptions import StopActorExecutionError + from leapp.libraries.common import mounting, utils + from leapp.libraries.common.config import get_env ++from leapp.libraries.common.config.version import get_target_major_version + from leapp.libraries.stdlib import api, CalledProcessError, run + + OVERLAY_DO_NOT_MOUNT = ('tmpfs', 'devpts', 'sysfs', 'proc', 'cramfs', 'sysv', 'vfat') + ++# NOTE(pstodulk): what about using more closer values and than just multiply ++# the final result by magical constant?... this number is most likely going to ++# be lowered and affected by XFS vs EXT4 FSs that needs different spaces each ++# of them. ++_MAGICAL_CONSTANT_OVL_SIZE = 128 ++""" ++Average size of created disk space images. ++ ++The size can be lower or higher - usually lower. The value is higher as we want ++to rather prevent future actions in advance instead of resolving later issues ++with the missing space. ++ ++It's possible that in future we implement better heuristic that will guess ++the needed space based on size of each FS. I have been thinking to lower ++the value, as in my case most of partitions where we do not need to do ++write operations consume just ~ 33MB. However, I decided to keep it as it is ++for now to stay on the safe side. ++""" ++ ++_MAGICAL_CONSTANT_MIN_CONTAINER_SIZE_8 = 3200 ++""" ++Average space consumed to create target el8userspace container installation + pkg downloads. ++ ++Minimal container size is approx. 1GiB without download of packages for the upgrade ++(and without pkgs for the initramfs creation). The total size of the container ++ * with all pkgs downloaded ++ * final initramfs installed package set ++ * created the upgrade initramfs ++is for the minimal system ++ * ~ 2.9 GiB for IPU 7 -> 8 ++ * ~ 1.8 GiB for IPU 8 -> 9 ++when no other extra packages are installed for the needs of the upgrade. ++Keeping in mind that during the upgrade initramfs creation another 400+ MiB ++is consumed temporarily. ++ ++Using higher value to cover also the space that consumes leapp.db records. ++ ++This constant is really magical and the value can be changed in future. ++""" ++ ++_MAGICAL_CONSTANT_MIN_CONTAINER_SIZE_9 = 2200 ++""" ++Average space consumed to create target el9userspace container installation + pkg downloads. ++ ++See _MAGICAL_CONSTANT_MIN_CONTAINER_SIZE_8 for more details. ++""" ++ ++_MAGICAL_CONSTANT_MIN_PROTECTED_SIZE = 200 ++""" ++This is the minimal size (in MiB) that will be always reserved for /var/lib/leapp ++ ++In case the size of the container is larger than _MAGICAL_CONSTANT_MIN_PROTECTED_SIZE ++or close to that size, stay always with this minimal protected size defined by ++this constant. ++""" ++ + + MountPoints = namedtuple('MountPoints', ['fs_file', 'fs_vfstype']) + + ++def _get_min_container_size(): ++ if get_target_major_version() == '8': ++ return _MAGICAL_CONSTANT_MIN_CONTAINER_SIZE_8 ++ return _MAGICAL_CONSTANT_MIN_CONTAINER_SIZE_9 ++ ++ ++def get_recommended_leapp_free_space(userspace_path=None): ++ """ ++ Return recommended free space for the target container (+ pkg downloads) ++ ++ If the path to the container is set, the returned value is updated to ++ reflect already consumed space by the installed container. In case the ++ container is bigger than the minimal protected size, return at least ++ `_MAGICAL_CONSTANT_MIN_PROTECTED_SIZE`. ++ ++ It's not recommended to use this function except official actors managed ++ by OAMG group in github.com/oamg/leapp-repository. This function can be ++ changed in future, ignoring the deprecation process. ++ ++ TODO(pstodulk): this is so far the best trade off between stay safe and do ++ do not consume too much space. But need to figure out cost of the time ++ consumption. ++ ++ TODO(pstodulk): check we are not negatively affected in case of downloaded ++ rpms. We want to prevent situations when we say that customer has enough ++ space for the first run and after the download of packages we inform them ++ they do not have enough free space anymore. Note: such situation can be ++ valid in specific cases - e.g. the space is really consumed already e.g. by ++ leapp.db that has been executed manytimes. ++ ++ :param userspace_path: Path to the userspace container. ++ :type userspace_path: str ++ :rtype: int ++ """ ++ min_cont_size = _get_min_container_size() ++ if not userspace_path or not os.path.exists(userspace_path): ++ return min_cont_size ++ try: ++ # ignore symlinks and other partitions to be sure we calculate the space ++ # in reasonable time ++ cont_size = run(['du', '-sPmx', userspace_path])['stdout'].split()[0] ++ # the obtained number is in KiB. But we want to work with MiBs rather. ++ cont_size = int(cont_size) ++ except (OSError, CalledProcessError): ++ # do not care about failed cmd, in such a case, just act like userspace_path ++ # has not been set ++ api.current_logger().warning( ++ 'Cannot calculate current container size to estimate correctly required space.' ++ ' Working with the default: {} MiB' ++ .format(min_cont_size) ++ ) ++ return min_cont_size ++ if cont_size < 0: ++ api.current_logger().warning( ++ 'Cannot calculate the container size - negative size obtained: {}.' ++ ' Estimate the required size based on the default value: {} MiB' ++ .format(cont_size, min_cont_size) ++ ) ++ return min_cont_size ++ prot_size = min_cont_size - cont_size ++ if prot_size < _MAGICAL_CONSTANT_MIN_PROTECTED_SIZE: ++ api.current_logger().debug( ++ 'The size of the container is higher than the expected default.' ++ ' Use the minimal protected size instead: {} MiB.' ++ .format(_MAGICAL_CONSTANT_MIN_PROTECTED_SIZE) ++ ) ++ return _MAGICAL_CONSTANT_MIN_PROTECTED_SIZE ++ return prot_size ++ ++ ++def _get_fspace(path, convert_to_mibs=False, coefficient=1): ++ """ ++ Return the free disk space on given path. ++ ++ The default is in bytes, but if convert_to_mibs is True, return MiBs instead. ++ ++ Raises OSError if nothing exists on the given `path`. ++ ++ :param path: Path to an existing file or directory ++ :type path: str ++ :param convert_to_mibs: If True, convert the value to MiBs ++ :type convert_to_mibs: bool ++ :param coefficient: Coefficient to multiply the free space (e.g. 0.9 to have it 10% lower). Max: 1 ++ :type coefficient: float ++ :rtype: int ++ """ ++ stat = os.statvfs(path) ++ ++ # TODO(pstodulk): discuss the function params ++ coefficient = min(coefficient, 1) ++ fspace_bytes = int(stat.f_frsize * stat.f_bavail * coefficient) ++ if convert_to_mibs: ++ return int(fspace_bytes / 1024 / 1024) # noqa: W1619; pylint: disable=old-division ++ return fspace_bytes ++ ++ ++def _ensure_enough_diskimage_space(space_needed, directory): ++ # TODO(pstodulk): update the error msg/details ++ # imagine situation we inform user we need at least 800MB, ++ # so they clean /var/lib/leapp/* which can provide additional space, ++ # but the calculated required free space takes the existing content under ++ # /var/lib/leapp/ into account, so the next error msg could say: ++ # needed at least 3400 MiB - which could be confusing for users. ++ if _get_fspace(directory) < (space_needed * 1024 * 1024): ++ message = ( ++ 'Not enough space available on {directory}: Needed at least {space_needed} MiB.' ++ .format(directory=directory, space_needed=space_needed) ++ ) ++ details = {'detail': ( ++ 'The file system hosting the {directory} directory does not contain' ++ ' enough free space to proceed all parts of the in-place upgrade.' ++ ' Note the calculated required free space is the minimum derived' ++ ' from upgrades of minimal systems and the actual needed free' ++ ' space could be higher.' ++ '\nNeeded at least: {space_needed} MiB.' ++ '\nSuggested free space: {suggested} MiB (or more).' ++ .format(space_needed=space_needed, directory=directory, suggested=space_needed + 1000) ++ )} ++ if get_env('LEAPP_OVL_SIZE', None): ++ # LEAPP_OVL_SIZE has not effect as we use sparse files now. ++ details['note'] = 'The LEAPP_OVL_SIZE environment variable has no effect anymore.' ++ api.current_logger().error(message) ++ raise StopActorExecutionError(message, details=details) ++ ++ + def _get_mountpoints(storage_info): + mount_points = set() + for entry in storage_info.fstab: + if os.path.isdir(entry.fs_file) and entry.fs_vfstype not in OVERLAY_DO_NOT_MOUNT: + mount_points.add(MountPoints(entry.fs_file, entry.fs_vfstype)) + elif os.path.isdir(entry.fs_file) and entry.fs_vfstype == 'vfat': ++ # VFAT FS is not supported to be used for any system partition, ++ # so we can safely ignore it + api.current_logger().warning( + 'Ignoring vfat {} filesystem mount during upgrade process'.format(entry.fs_file) + ) +@@ -35,6 +219,81 @@ def _mount_dir(mounts_dir, mountpoint): + return os.path.join(mounts_dir, _mount_name(mountpoint)) + + ++def _get_scratch_mountpoint(mount_points, dir_path): ++ for mp in sorted(mount_points, reverse=True): ++ # we are sure that mountpoint != dir_path in this case, as the latest ++ # valid mountpoint customers could create is the parent directory ++ mod_mp = mp if mp[-1] == '/' else '{}/'.format(mp) ++ if dir_path.startswith(mod_mp): ++ # longest first, so the first one we find, is the last mp on the path ++ return mp ++ return None # making pylint happy; this is basically dead code ++ ++ ++def _prepare_required_mounts(scratch_dir, mounts_dir, storage_info, scratch_reserve): ++ """ ++ Create disk images and loop mount them. ++ ++ Ensure to create disk image for each important mountpoint configured ++ in fstab (excluding fs types noted in `OVERLAY_DO_NOT_MOUNT`). ++ Disk images reflect the free space of related partition/volume. In case ++ of partition hosting /var/lib/leapp/* calculate the free space value ++ taking `scratch_reserve` into account, as during the run of the tooling, ++ we will be consuming the space on the partition and we want to be more ++ sure that we do not consume all the space on the partition during the ++ execution - so we reduce the risk we affect run of other applications ++ due to missing space. ++ ++ Note: the partition hosting the scratch dir is expected to be the same ++ partition that is hosting the target userspace container, but it does not ++ have to be true if the code changes. Right now, let's live with that. ++ ++ See `_create_mount_disk_image` docstring for additional more details. ++ ++ :param scratch_dir: Path to the scratch directory. ++ :type scratch_dir: str ++ :param mounts_dir: Path to the directory supposed to be a mountpoint. ++ :type mounts_dir: str ++ :param storage_info: The StorageInfo message. ++ :type storage_info: leapp.models.StorageInfo ++ :param scratch_reserve: Number of MB that should be extra reserved in a partition hosting the scratch_dir. ++ :type scratch_reserve: Optional[int] ++ """ ++ mount_points = sorted([mp.fs_file for mp in _get_mountpoints(storage_info)]) ++ scratch_mp = _get_scratch_mountpoint(mount_points, scratch_dir) ++ disk_images_directory = os.path.join(scratch_dir, 'diskimages') ++ ++ # Ensure we cleanup old disk images before we check for space constraints. ++ # NOTE(pstodulk): Could we improve the process so we create imgs & calculate ++ # the required disk space just once during each leapp (pre)upgrade run? ++ run(['rm', '-rf', disk_images_directory]) ++ _create_diskimages_dir(scratch_dir, disk_images_directory) ++ ++ # TODO(pstodulk): update the calculation for bind mounted mount_points (skip) ++ # basic check whether we have enough space at all ++ space_needed = scratch_reserve + _MAGICAL_CONSTANT_OVL_SIZE * len(mount_points) ++ _ensure_enough_diskimage_space(space_needed, scratch_dir) ++ ++ # free space required on this partition should not be affected by durin the ++ # upgrade transaction execution by space consumed on creation of disk images ++ # as disk images are cleaned in the end of this functions, ++ # but we want to reserve some space in advance. ++ scratch_disk_size = _get_fspace(scratch_dir, convert_to_mibs=True) - scratch_reserve ++ ++ result = {} ++ for mountpoint in mount_points: ++ # keep the info about the free space rather 5% lower than the real value ++ disk_size = _get_fspace(mountpoint, convert_to_mibs=True, coefficient=0.95) ++ if mountpoint == scratch_mp: ++ disk_size = scratch_disk_size ++ image = _create_mount_disk_image(disk_images_directory, mountpoint, disk_size) ++ result[mountpoint] = mounting.LoopMount( ++ source=image, ++ target=_mount_dir(mounts_dir, mountpoint) ++ ) ++ return result ++ ++ + @contextlib.contextmanager + def _build_overlay_mount(root_mount, mounts): + if not root_mount: +@@ -56,20 +315,151 @@ def _build_overlay_mount(root_mount, mounts): + def cleanup_scratch(scratch_dir, mounts_dir): + """ + Function to cleanup the scratch directory ++ ++ If the mounts_dir is a mountpoint, unmount it first. ++ ++ :param scratch_dir: Path to the scratch directory. ++ :type scratch_dir: str ++ :param mounts_dir: Path to the directory supposed to be a mountpoint. ++ :type mounts_dir: str + """ + api.current_logger().debug('Cleaning up mounts') + if os.path.ismount(mounts_dir): ++ # TODO(pstodulk): this is actually obsoleted for years. mounts dir ++ # is not mountpoit anymore, it contains mountpoints. But in time of ++ # this call all MPs should be already umounted as the solution has been ++ # changed also (all MPs are handled by context managers). This code ++ # is basically dead, so keeping it as it does not hurt us now. + api.current_logger().debug('Mounts directory is a mounted disk image - Unmounting.') + try: + run(['/bin/umount', '-fl', mounts_dir]) + api.current_logger().debug('Unmounted mounted disk image.') + except (OSError, CalledProcessError) as e: + api.current_logger().warning('Failed to umount %s - message: %s', mounts_dir, str(e)) ++ if get_env('LEAPP_DEVEL_KEEP_DISK_IMGS', None) == '1': ++ # NOTE(pstodulk): From time to time, it helps me with some experiments ++ return + api.current_logger().debug('Recursively removing scratch directory %s.', scratch_dir) + shutil.rmtree(scratch_dir, onerror=utils.report_and_ignore_shutil_rmtree_error) + api.current_logger().debug('Recursively removed scratch directory %s.', scratch_dir) + + ++def _format_disk_image_ext4(diskimage_path): ++ """ ++ Format the specified disk image with Ext4 filesystem. ++ ++ The formatted file system is optimized for operations we want to do and ++ mainly for the space it needs to take for the initialisation. So use 32MiB ++ journal (that's enough for us as we do not plan to do too many operations ++ inside) for any size of the disk image. Also the lazy ++ initialisation is disabled. The formatting will be slower, but it helps ++ us to estimate better the needed amount of the space for other actions ++ done later. ++ """ ++ api.current_logger().debug('Creating ext4 filesystem in disk image at %s', diskimage_path) ++ cmd = [ ++ '/sbin/mkfs.ext4', ++ '-J', 'size=32', ++ '-E', 'lazy_itable_init=0,lazy_journal_init=0', ++ '-F', diskimage_path ++ ] ++ try: ++ utils.call_with_oserror_handled(cmd=cmd) ++ except CalledProcessError as e: ++ # FIXME(pstodulk): taken from original, but %s seems to me invalid here ++ api.current_logger().error('Failed to create ext4 filesystem %s', diskimage_path, exc_info=True) ++ raise StopActorExecutionError( ++ message=str(e) ++ ) ++ ++ ++def _format_disk_image_xfs(diskimage_path): ++ """ ++ Format the specified disk image with XFS filesystem. ++ ++ Set journal just to 32MiB always as we will not need to do too many operation ++ inside, so 32MiB should enough for us. ++ """ ++ api.current_logger().debug('Creating XFS filesystem in disk image at %s', diskimage_path) ++ cmd = ['/sbin/mkfs.xfs', '-l', 'size=32m', '-f', diskimage_path] ++ try: ++ utils.call_with_oserror_handled(cmd=cmd) ++ except CalledProcessError as e: ++ # FIXME(pstodulk): taken from original, but %s seems to me invalid here ++ api.current_logger().error('Failed to create XFS filesystem %s', diskimage_path, exc_info=True) ++ raise StopActorExecutionError( ++ message=str(e) ++ ) ++ ++ ++def _create_mount_disk_image(disk_images_directory, path, disk_size): ++ """ ++ Creates the mount disk image and return path to it. ++ ++ The disk image is represented by a sparse file which apparent size ++ corresponds usually to the free space of a particular partition/volume it ++ represents - in this function it's set by `disk_size` parameter, which should ++ be int representing the free space in MiBs. ++ ++ The created disk image is formatted with XFS (default) or Ext4 FS ++ and it's supposed to be used for write directories of an overlayfs built ++ above it. ++ ++ The disk image is formatted with Ext4 if (envar) `LEAPP_OVL_IMG_FS_EXT4=1`. ++ ++ :param disk_images_directory: Path to the directory where disk images should be stored. ++ :type disk_images_directory: str ++ :param path: Path to the mountpoint of the original (host/source) partition/volume ++ :type path: str ++ :return: Path to the created disk image ++ :rtype: str ++ """ ++ diskimage_path = os.path.join(disk_images_directory, _mount_name(path)) ++ cmd = [ ++ '/bin/dd', ++ 'if=/dev/zero', 'of={}'.format(diskimage_path), ++ 'bs=1M', 'count=0', 'seek={}'.format(disk_size) ++ ] ++ hint = ( ++ 'Please ensure that there is enough diskspace on the partition hosting' ++ 'the {} directory.' ++ .format(disk_images_directory) ++ ) ++ ++ api.current_logger().debug('Attempting to create disk image at %s', diskimage_path) ++ utils.call_with_failure_hint(cmd=cmd, hint=hint) ++ ++ if get_env('LEAPP_OVL_IMG_FS_EXT4', '0') == '1': ++ # This is alternative to XFS in case we find some issues, to be able ++ # to switch simply to Ext4, so we will be able to simple investigate ++ # possible issues between overlay <-> XFS if any happens. ++ _format_disk_image_ext4(diskimage_path) ++ else: ++ _format_disk_image_xfs(diskimage_path) ++ ++ return diskimage_path ++ ++ ++def _create_diskimages_dir(scratch_dir, diskimages_dir): ++ """ ++ Prepares directories for disk images ++ """ ++ api.current_logger().debug('Creating disk images directory.') ++ try: ++ utils.makedirs(diskimages_dir) ++ api.current_logger().debug('Done creating disk images directory.') ++ except OSError: ++ api.current_logger().error('Failed to create disk images directory %s', diskimages_dir, exc_info=True) ++ ++ # This is an attempt for giving the user a chance to resolve it on their own ++ raise StopActorExecutionError( ++ message='Failed to prepare environment for package download while creating directories.', ++ details={ ++ 'hint': 'Please ensure that {scratch_dir} is empty and modifiable.'.format(scratch_dir=scratch_dir) ++ } ++ ) ++ ++ + def _create_mounts_dir(scratch_dir, mounts_dir): + """ + Prepares directories for mounts +@@ -102,15 +492,59 @@ def _mount_dnf_cache(overlay_target): + + + @contextlib.contextmanager +-def create_source_overlay(mounts_dir, scratch_dir, xfs_info, storage_info, mount_target=None): ++def create_source_overlay(mounts_dir, scratch_dir, xfs_info, storage_info, mount_target=None, scratch_reserve=0): + """ + Context manager that prepares the source system overlay and yields the mount. ++ ++ The in-place upgrade itself requires to do some changes on the system to be ++ able to perform the in-place upgrade itself - or even to be able to evaluate ++ if the system is possible to upgrade. However, we do not want to (and must not) ++ change the original system until we pass beyond the point of not return. ++ ++ For that purposes we have to create a layer above the real host file system, ++ where we can safely perform all operations without affecting the system ++ setup, rpm database, etc. Currently overlay (OVL) technology showed it is ++ capable to handle our requirements good enough - with some limitations. ++ ++ This function prepares a disk image and an overlay layer for each ++ mountpoint configured in /etc/fstab, excluding those with FS type noted ++ in the OVERLAY_DO_NOT_MOUNT set. Such prepared OVL images are then composed ++ together to reflect the real host filesystem. In the end everything is cleaned. ++ ++ The new solution can be now problematic for system with too many partitions ++ and loop devices. For such systems we keep for now the possibility of the ++ fallback to an old solution, which has however number of issues that are ++ fixed by the new design. To fallback to the old solution, set envar: ++ LEAPP_OVL_LEGACY=1 ++ ++ Disk images created for OVL are formatted with XFS by default. In case of ++ problems, it's possible to switch to Ext4 FS using: ++ LEAPP_OVL_IMG_FS_EXT4=1 ++ ++ :param mounts_dir: Absolute path to the directory under which all mounts should happen. ++ :type mounts_dir: str ++ :param scratch_dir: Absolute path to the directory in which all disk and OVL images are stored. ++ :type scratch_dir: str ++ :param xfs_info: The XFSPresence message. ++ :type xfs_info: leapp.models.XFSPresence ++ :param storage_info: The StorageInfo message. ++ :type storage_info: leapp.models.StorageInfo ++ :param mount_target: Directory to which whole source OVL layer should be bind mounted. ++ If None (default), mounting.NullMount is created instead ++ :type mount_target: Optional[str] ++ :param scratch_reserve: Number of MB that should be extra reserved in a partition hosting the scratch_dir. ++ :type scratch_reserve: Optional[int] ++ :rtype: mounting.BindMount or mounting.NullMount + """ + api.current_logger().debug('Creating source overlay in {scratch_dir} with mounts in {mounts_dir}'.format( + scratch_dir=scratch_dir, mounts_dir=mounts_dir)) + try: + _create_mounts_dir(scratch_dir, mounts_dir) +- mounts = _prepare_required_mounts_old(scratch_dir, mounts_dir, _get_mountpoints(storage_info), xfs_info) ++ if get_env('LEAPP_OVL_LEGACY', '0') != '1': ++ mounts = _prepare_required_mounts(scratch_dir, mounts_dir, storage_info, scratch_reserve) ++ else: ++ # fallback to the deprecated OVL solution ++ mounts = _prepare_required_mounts_old(scratch_dir, mounts_dir, _get_mountpoints(storage_info), xfs_info) + with mounts.pop('/') as root_mount: + with mounting.OverlayMount(name='system_overlay', source='/', workdir=root_mount.target) as root_overlay: + if mount_target: +@@ -124,6 +558,8 @@ def create_source_overlay(mounts_dir, scratch_dir, xfs_info, storage_info, mount + except Exception: + cleanup_scratch(scratch_dir, mounts_dir) + raise ++ # cleanup always now ++ cleanup_scratch(scratch_dir, mounts_dir) + + + # ############################################################################# +@@ -133,6 +569,7 @@ def create_source_overlay(mounts_dir, scratch_dir, xfs_info, storage_info, mount + # negatively affect systems with many loop mountpoints, so let's keep this + # as a workaround for now. I am separating the old and new code in this way + # to make the future removal easy. ++# The code below is triggered when LEAPP_OVL_LEGACY=1 envar is set. + # IMPORTANT: Before an update of functions above, ensure the functionality of + # the code below is not affected, otherwise copy the function below with the + # "_old" suffix. +-- +2.41.0 + diff --git a/0040-dnfplugin.py-Update-err-msgs-and-handle-transaction-.patch b/0040-dnfplugin.py-Update-err-msgs-and-handle-transaction-.patch new file mode 100644 index 0000000..15de93a --- /dev/null +++ b/0040-dnfplugin.py-Update-err-msgs-and-handle-transaction-.patch @@ -0,0 +1,159 @@ +From 025b97088d30d5bd41a4d0b610cf2232ef150ece Mon Sep 17 00:00:00 2001 +From: Petr Stodulka +Date: Fri, 14 Jul 2023 13:42:48 +0200 +Subject: [PATCH 40/42] dnfplugin.py: Update err msgs and handle transaction + issues better + +With the redesigned overlay solution, original error messages are +misleading. Keeping original error msgs when LEAPP_OVL_LEGACY=1. + +Also handle better the error msgs generated when installing initramfs +dependencies. In case of the missing space the error has been +unhandled. Now it is handled with the correct msg also. +--- + .../common/libraries/dnfplugin.py | 101 ++++++++++++++---- + 1 file changed, 80 insertions(+), 21 deletions(-) + +diff --git a/repos/system_upgrade/common/libraries/dnfplugin.py b/repos/system_upgrade/common/libraries/dnfplugin.py +index fb0e8ae5..ffde211f 100644 +--- a/repos/system_upgrade/common/libraries/dnfplugin.py ++++ b/repos/system_upgrade/common/libraries/dnfplugin.py +@@ -2,6 +2,7 @@ import contextlib + import itertools + import json + import os ++import re + import shutil + + from leapp.exceptions import StopActorExecutionError +@@ -12,6 +13,7 @@ from leapp.libraries.stdlib import api, CalledProcessError, config + from leapp.models import DNFWorkaround + + DNF_PLUGIN_NAME = 'rhel_upgrade.py' ++_DEDICATED_URL = 'https://access.redhat.com/solutions/7011704' + + + class _DnfPluginPathStr(str): +@@ -146,6 +148,75 @@ def backup_debug_data(context): + api.current_logger().warning('Failed to copy debugdata. Message: {}'.format(str(e)), exc_info=True) + + ++def _handle_transaction_err_msg_old(stage, xfs_info, err): ++ # NOTE(pstodulk): This is going to be removed in future! ++ message = 'DNF execution failed with non zero exit code.' ++ details = {'STDOUT': err.stdout, 'STDERR': err.stderr} ++ ++ if 'more space needed on the' in err.stderr and stage != 'upgrade': ++ # Disk Requirements: ++ # At least more space needed on the filesystem. ++ # ++ article_section = 'Generic case' ++ if xfs_info.present and xfs_info.without_ftype: ++ article_section = 'XFS ftype=0 case' ++ ++ message = ('There is not enough space on the file system hosting /var/lib/leapp directory ' ++ 'to extract the packages.') ++ details = {'hint': "Please follow the instructions in the '{}' section of the article at: " ++ "link: https://access.redhat.com/solutions/5057391".format(article_section)} ++ ++ raise StopActorExecutionError(message=message, details=details) ++ ++ ++def _handle_transaction_err_msg(stage, xfs_info, err, is_container=False): ++ # ignore the fallback when the error is related to the container issue ++ # e.g. installation of packages inside the container; so it's unrelated ++ # to the upgrade transactions. ++ if get_env('LEAPP_OVL_LEGACY', '0') == '1' and not is_container: ++ _handle_transaction_err_msg_old(stage, xfs_info, err) ++ return # not needed actually as the above function raises error, but for visibility ++ NO_SPACE_STR = 'more space needed on the' ++ message = 'DNF execution failed with non zero exit code.' ++ details = {'STDOUT': err.stdout, 'STDERR': err.stderr} ++ if NO_SPACE_STR not in err.stderr: ++ raise StopActorExecutionError(message=message, details=details) ++ ++ # Disk Requirements: ++ # At least more space needed on the filesystem. ++ # ++ missing_space = [line.strip() for line in err.stderr.split('\n') if NO_SPACE_STR in line] ++ if is_container: ++ size_str = re.match(r'At least (.*) more space needed', missing_space[0]).group(1) ++ message = 'There is not enough space on the file system hosting /var/lib/leapp.' ++ hint = ( ++ 'Increase the free space on the filesystem hosting' ++ ' /var/lib/leapp by {} at minimum. It is suggested to provide' ++ ' reasonably more space to be able to perform all planned actions' ++ ' (e.g. when 200MB is missing, add 1700MB or more).\n\n' ++ 'It is also a good practice to create dedicated partition' ++ ' for /var/lib/leapp when more space is needed, which can be' ++ ' dropped after the system upgrade is fully completed' ++ ' For more info, see: {}' ++ .format(size_str, _DEDICATED_URL) ++ ) ++ # we do not want to confuse customers by the orig msg speaking about ++ # missing space on '/'. Skip the Disk Requirements section. ++ # The information is part of the hint. ++ details = {'hint': hint} ++ else: ++ message = 'There is not enough space on some file systems to perform the upgrade transaction.' ++ hint = ( ++ 'Increase the free space on listed filesystems. Presented values' ++ ' are required minimum calculated by RPM and it is suggested to' ++ ' provide reasonably more free space (e.g. when 200 MB is missing' ++ ' on /usr, add 1200MB or more).' ++ ) ++ details = {'hint': hint, 'Disk Requirements': '\n'.join(missing_space)} ++ ++ raise StopActorExecutionError(message=message, details=details) ++ ++ + def _transaction(context, stage, target_repoids, tasks, plugin_info, xfs_info, + test=False, cmd_prefix=None, on_aws=False): + """ +@@ -219,26 +290,8 @@ def _transaction(context, stage, target_repoids, tasks, plugin_info, xfs_info, + message='Failed to execute dnf. Reason: {}'.format(str(e)) + ) + except CalledProcessError as e: +- api.current_logger().error('DNF execution failed: ') +- +- message = 'DNF execution failed with non zero exit code.' +- details = {'STDOUT': e.stdout, 'STDERR': e.stderr} +- +- if 'more space needed on the' in e.stderr: +- # The stderr contains this error summary: +- # Disk Requirements: +- # At least more space needed on the filesystem. +- +- article_section = 'Generic case' +- if xfs_info.present and xfs_info.without_ftype: +- article_section = 'XFS ftype=0 case' +- +- message = ('There is not enough space on the file system hosting /var/lib/leapp directory ' +- 'to extract the packages.') +- details = {'hint': "Please follow the instructions in the '{}' section of the article at: " +- "link: https://access.redhat.com/solutions/5057391".format(article_section)} +- +- raise StopActorExecutionError(message=message, details=details) ++ api.current_logger().error('Cannot calculate, check, test, or perform the upgrade transaction.') ++ _handle_transaction_err_msg(stage, xfs_info, e, is_container=False) + finally: + if stage == 'check': + backup_debug_data(context=context) +@@ -307,7 +360,13 @@ def install_initramdisk_requirements(packages, target_userspace_info, used_repos + if get_target_major_version() == '9': + # allow handling new RHEL 9 syscalls by systemd-nspawn + env = {'SYSTEMD_SECCOMP': '0'} +- context.call(cmd, env=env) ++ try: ++ context.call(cmd, env=env) ++ except CalledProcessError as e: ++ api.current_logger().error( ++ 'Cannot install packages in the target container required to build the upgrade initramfs.' ++ ) ++ _handle_transaction_err_msg('', None, e, is_container=True) + + + def perform_transaction_install(target_userspace_info, storage_info, used_repos, tasks, plugin_info, xfs_info): +-- +2.41.0 + diff --git a/0041-upgradeinitramfsgenerator-Check-the-free-space-prior.patch b/0041-upgradeinitramfsgenerator-Check-the-free-space-prior.patch new file mode 100644 index 0000000..013c2ca --- /dev/null +++ b/0041-upgradeinitramfsgenerator-Check-the-free-space-prior.patch @@ -0,0 +1,161 @@ +From 591cdb865befff8035d53b861d9ff95b5704ed64 Mon Sep 17 00:00:00 2001 +From: Petr Stodulka +Date: Fri, 14 Jul 2023 17:32:46 +0200 +Subject: [PATCH 41/42] upgradeinitramfsgenerator: Check the free space prior + the initeramfs generation + +Under rare conditions it's possible the last piece free space +is consumed when the upgrade initramfs is generated. It's hard +to hit this problems right now without additional customisations +that consume more space than we expect. However, when it happens, +it not good situation. From this point, check the remaining free +space on the FS hosting the container. In case we have less than +500MB, do not even try. Possibly we will increase the value in future, +but consider it good enough for now. +--- + .../libraries/upgradeinitramfsgenerator.py | 73 +++++++++++++++++++ + .../unit_test_upgradeinitramfsgenerator.py | 14 ++++ + 2 files changed, 87 insertions(+) + +diff --git a/repos/system_upgrade/common/actors/initramfs/upgradeinitramfsgenerator/libraries/upgradeinitramfsgenerator.py b/repos/system_upgrade/common/actors/initramfs/upgradeinitramfsgenerator/libraries/upgradeinitramfsgenerator.py +index f141d9e3..5a686a47 100644 +--- a/repos/system_upgrade/common/actors/initramfs/upgradeinitramfsgenerator/libraries/upgradeinitramfsgenerator.py ++++ b/repos/system_upgrade/common/actors/initramfs/upgradeinitramfsgenerator/libraries/upgradeinitramfsgenerator.py +@@ -20,6 +20,7 @@ from leapp.utils.deprecation import suppress_deprecation + + INITRAM_GEN_SCRIPT_NAME = 'generate-initram.sh' + DRACUT_DIR = '/dracut' ++DEDICATED_LEAPP_PART_URL = 'https://access.redhat.com/solutions/7011704' + + + def _get_target_kernel_version(context): +@@ -231,6 +232,77 @@ def prepare_userspace_for_initram(context): + _copy_files(context, files) + + ++def _get_fspace(path, convert_to_mibs=False, coefficient=1): ++ """ ++ Return the free disk space on given path. ++ ++ The default is in bytes, but if convert_to_mibs is True, return MiBs instead. ++ ++ Raises OSError if nothing exists on the given `path`. ++ ++ :param path: Path to an existing file or directory ++ :type path: str ++ :param convert_to_mibs: If True, convert the value to MiBs ++ :type convert_to_mibs: bool ++ :param coefficient: Coefficient to multiply the free space (e.g. 0.9 to have it 10% lower). Max: 1 ++ :type coefficient: float ++ :rtype: int ++ """ ++ # TODO(pstodulk): discuss the function params ++ # NOTE(pstodulk): This func is copied from the overlaygen.py lib ++ # probably it would make sense to make it public in the utils.py lib, ++ # but for now, let's keep it private ++ stat = os.statvfs(path) ++ ++ coefficient = min(coefficient, 1) ++ fspace_bytes = int(stat.f_frsize * stat.f_bavail * coefficient) ++ if convert_to_mibs: ++ return int(fspace_bytes / 1024 / 1024) # noqa: W1619; pylint: disable=old-division ++ return fspace_bytes ++ ++ ++def _check_free_space(context): ++ """ ++ Raise StopActorExecutionError if there is less than 500MB of free space available. ++ ++ If there is not enough free space in the context, the initramfs will not be ++ generated successfully and it's hard to discover what was the issue. Also ++ the missing space is able to kill the leapp itself - trying to write to the ++ leapp.db when the FS hosting /var/lib/leapp is full, kills the framework ++ and the actor execution too - so there is no gentle way to handle such ++ exceptions when it happens. From this point, let's rather check the available ++ space in advance and stop the execution when it happens. ++ ++ It is not expected to hit this issue, but I was successful and I know ++ it's still possible even with all other changes (just it's much harder ++ now to hit it). So adding this seatbelt, that is not 100% bulletproof, ++ but I call it good enough. ++ ++ Currently protecting last 500MB. In case of problems, we can increase ++ the value. ++ """ ++ message = 'There is not enough space on the file system hosting /var/lib/leapp.' ++ hint = ( ++ 'Increase the free space on the filesystem hosting' ++ ' /var/lib/leapp by 500MB at minimum (suggested 1500MB).\n\n' ++ 'It is also a good practice to create dedicated partition' ++ ' for /var/lib/leapp when more space is needed, which can be' ++ ' dropped after the system upgrade is fully completed.' ++ ' For more info, see: {}' ++ .format(DEDICATED_LEAPP_PART_URL) ++ ) ++ detail = ( ++ 'Remaining free space is lower than 500MB which is not enough to' ++ ' be able to generate the upgrade initramfs. ' ++ ) ++ ++ if _get_fspace(context.base_dir, convert_to_mibs=True) < 500: ++ raise StopActorExecutionError( ++ message=message, ++ details={'hint': hint, 'detail': detail} ++ ) ++ ++ + def generate_initram_disk(context): + """ + Function to actually execute the init ramdisk creation. +@@ -238,6 +310,7 @@ def generate_initram_disk(context): + Includes handling of specified dracut and kernel modules from the host when + needed. The check for the 'conflicting' modules is in a separate actor. + """ ++ _check_free_space(context) + env = {} + if get_target_major_version() == '9': + env = {'SYSTEMD_SECCOMP': '0'} +diff --git a/repos/system_upgrade/common/actors/initramfs/upgradeinitramfsgenerator/tests/unit_test_upgradeinitramfsgenerator.py b/repos/system_upgrade/common/actors/initramfs/upgradeinitramfsgenerator/tests/unit_test_upgradeinitramfsgenerator.py +index a2f1c837..8068e177 100644 +--- a/repos/system_upgrade/common/actors/initramfs/upgradeinitramfsgenerator/tests/unit_test_upgradeinitramfsgenerator.py ++++ b/repos/system_upgrade/common/actors/initramfs/upgradeinitramfsgenerator/tests/unit_test_upgradeinitramfsgenerator.py +@@ -10,6 +10,7 @@ from leapp.libraries.common.testutils import CurrentActorMocked, logger_mocked, + from leapp.utils.deprecation import suppress_deprecation + + from leapp.models import ( # isort:skip ++ FIPSInfo, + RequiredUpgradeInitramPackages, # deprecated + UpgradeDracutModule, # deprecated + BootContent, +@@ -250,6 +251,16 @@ def test_prepare_userspace_for_initram(monkeypatch, adjust_cwd, input_msgs, pkgs + assert _sort_files(upgradeinitramfsgenerator._copy_files.args[1]) == _files + + ++class MockedGetFspace(object): ++ def __init__(self, space): ++ self.space = space ++ ++ def __call__(self, dummy_path, convert_to_mibs=False): ++ if not convert_to_mibs: ++ return self.space ++ return int(self.space / 1024 / 1024) # noqa: W1619; pylint: disable=old-division ++ ++ + @pytest.mark.parametrize('input_msgs,dracut_modules,kernel_modules', [ + # test dracut modules with UpgradeDracutModule(s) - orig functionality + (gen_UDM_list(MODULES[0]), MODULES[0], []), +@@ -275,8 +286,11 @@ def test_generate_initram_disk(monkeypatch, input_msgs, dracut_modules, kernel_m + monkeypatch.setattr(upgradeinitramfsgenerator, '_get_target_kernel_version', lambda _: '') + monkeypatch.setattr(upgradeinitramfsgenerator, 'copy_kernel_modules', MockedCopyArgs()) + monkeypatch.setattr(upgradeinitramfsgenerator, 'copy_boot_files', lambda dummy: None) ++ monkeypatch.setattr(upgradeinitramfsgenerator, '_get_fspace', MockedGetFspace(2*2**30)) + upgradeinitramfsgenerator.generate_initram_disk(context) + ++ # TODO(pstodulk): add tests for the check of the free space (sep. from this func) ++ + # test now just that all modules have been passed for copying - so we know + # all modules have been consumed + detected_dracut_modules = set() +-- +2.41.0 + diff --git a/0042-targetuserspacecreator-Update-err-msg-when-installin.patch b/0042-targetuserspacecreator-Update-err-msg-when-installin.patch new file mode 100644 index 0000000..6f48d8d --- /dev/null +++ b/0042-targetuserspacecreator-Update-err-msg-when-installin.patch @@ -0,0 +1,113 @@ +From 5015311197efe5f700e6d44cab7f3d49f50925c9 Mon Sep 17 00:00:00 2001 +From: Petr Stodulka +Date: Sat, 15 Jul 2023 20:20:25 +0200 +Subject: [PATCH 42/42] targetuserspacecreator: Update err msg when installing + the container + +Regarding the changes around source OVL, we need to update the error +msg properly in case the installation of the target userspace container +fails. In this case, we want to change only the part when the upgrade +fails due to missing space. +--- + .../libraries/userspacegen.py | 61 +++++++++++++++---- + 1 file changed, 50 insertions(+), 11 deletions(-) + +diff --git a/repos/system_upgrade/common/actors/targetuserspacecreator/libraries/userspacegen.py b/repos/system_upgrade/common/actors/targetuserspacecreator/libraries/userspacegen.py +index 8400dbe7..fdf873e1 100644 +--- a/repos/system_upgrade/common/actors/targetuserspacecreator/libraries/userspacegen.py ++++ b/repos/system_upgrade/common/actors/targetuserspacecreator/libraries/userspacegen.py +@@ -1,5 +1,6 @@ + import itertools + import os ++import re + import shutil + + from leapp import reporting +@@ -55,6 +56,7 @@ from leapp.utils.deprecation import suppress_deprecation + PROD_CERTS_FOLDER = 'prod-certs' + GPG_CERTS_FOLDER = 'rpm-gpg' + PERSISTENT_PACKAGE_CACHE_DIR = '/var/lib/leapp/persistent_package_cache' ++DEDICATED_LEAPP_PART_URL = 'https://access.redhat.com/solutions/7011704' + + + def _check_deprecated_rhsm_skip(): +@@ -172,6 +174,53 @@ def _import_gpg_keys(context, install_root_dir, target_major_version): + ) + + ++def _handle_transaction_err_msg_size_old(err): ++ # NOTE(pstodulk): This is going to be removed in future! ++ ++ article_section = 'Generic case' ++ xfs_info = next(api.consume(XFSPresence), XFSPresence()) ++ if xfs_info.present and xfs_info.without_ftype: ++ article_section = 'XFS ftype=0 case' ++ ++ message = ('There is not enough space on the file system hosting /var/lib/leapp directory ' ++ 'to extract the packages.') ++ details = {'hint': "Please follow the instructions in the '{}' section of the article at: " ++ "link: https://access.redhat.com/solutions/5057391".format(article_section)} ++ ++ raise StopActorExecutionError(message=message, details=details) ++ ++ ++def _handle_transaction_err_msg_size(err): ++ if get_env('LEAPP_OVL_LEGACY', '0') == '1': ++ _handle_transaction_err_msg_size_old(err) ++ return # not needed actually as the above function raises error, but for visibility ++ NO_SPACE_STR = 'more space needed on the' ++ ++ # Disk Requirements: ++ # At least more space needed on the filesystem. ++ # ++ missing_space = [line.strip() for line in err.stderr.split('\n') if NO_SPACE_STR in line] ++ size_str = re.match(r'At least (.*) more space needed', missing_space[0]).group(1) ++ message = 'There is not enough space on the file system hosting /var/lib/leapp.' ++ hint = ( ++ 'Increase the free space on the filesystem hosting' ++ ' /var/lib/leapp by {} at minimum. It is suggested to provide' ++ ' reasonably more space to be able to perform all planned actions' ++ ' (e.g. when 200MB is missing, add 1700MB or more).\n\n' ++ 'It is also a good practice to create dedicated partition' ++ ' for /var/lib/leapp when more space is needed, which can be' ++ ' dropped after the system upgrade is fully completed' ++ ' For more info, see: {}' ++ .format(size_str, DEDICATED_LEAPP_PART_URL) ++ ) ++ # we do not want to confuse customers by the orig msg speaking about ++ # missing space on '/'. Skip the Disk Requirements section. ++ # The information is part of the hint. ++ details = {'hint': hint} ++ ++ raise StopActorExecutionError(message=message, details=details) ++ ++ + def prepare_target_userspace(context, userspace_dir, enabled_repos, packages): + """ + Implement the creation of the target userspace. +@@ -210,21 +259,11 @@ def prepare_target_userspace(context, userspace_dir, enabled_repos, packages): + message = 'Unable to install RHEL {} userspace packages.'.format(target_major_version) + details = {'details': str(exc), 'stderr': exc.stderr} + +- xfs_info = next(api.consume(XFSPresence), XFSPresence()) + if 'more space needed on the' in exc.stderr: + # The stderr contains this error summary: + # Disk Requirements: + # At least more space needed on the filesystem. +- +- article_section = 'Generic case' +- if xfs_info.present and xfs_info.without_ftype: +- article_section = 'XFS ftype=0 case' +- +- message = ('There is not enough space on the file system hosting /var/lib/leapp directory ' +- 'to extract the packages.') +- details = {'hint': "Please follow the instructions in the '{}' section of the article at: " +- "link: https://access.redhat.com/solutions/5057391".format(article_section)} +- raise StopActorExecutionError(message=message, details=details) ++ _handle_transaction_err_msg_size(exc) + + # If a proxy was set in dnf config, it should be the reason why dnf + # failed since leapp does not support updates behind proxy yet. +-- +2.41.0 + diff --git a/leapp-repository.spec b/leapp-repository.spec index 7bc34a7..597b6e8 100644 --- a/leapp-repository.spec +++ b/leapp-repository.spec @@ -42,7 +42,7 @@ py2_byte_compile "%1" "%2"} Name: leapp-repository Version: 0.18.0 -Release: 4%{?dist} +Release: 5%{?dist} Summary: Repositories for leapp License: ASL 2.0 @@ -85,6 +85,18 @@ Patch0027: 0027-Update-the-repomap.json-file-to-address-changes-in-R.patch Patch0028: 0028-Add-prod-certs-and-upgrade-paths-for-8.9-9.3.patch Patch0029: 0029-Update-leapp-data-files-1.1-2.0-and-requires-repomap.patch Patch0030: 0030-el8toel9-Warn-about-deprecated-Xorg-drivers.patch +Patch0031: 0031-Add-possibility-to-add-kernel-drivers-to-initrd.patch +Patch0032: 0032-Use-correct-flag-and-ENV-var-to-disable-insights-reg.patch +Patch0033: 0033-CLI-Use-new-Leapp-output-APIs-reports-summary-better.patch +Patch0034: 0034-Update-Grub-on-component-drives-if-boot-is-on-md-dev.patch +Patch0035: 0035-mdraid.py-lib-Check-if-usr-sbin-mdadm-exists.patch +Patch0036: 0036-target_userspace_creator-Use-MOVE-instead-of-copy-fo.patch +Patch0037: 0037-overlay-lib-Deprecate-old-ovl-internal-functions-ref.patch +Patch0038: 0038-overlay-lib-replace-os.getenv-common.config.get_env.patch +Patch0039: 0039-overlay-lib-Redesign-creation-of-the-source-overlay-.patch +Patch0040: 0040-dnfplugin.py-Update-err-msgs-and-handle-transaction-.patch +Patch0041: 0041-upgradeinitramfsgenerator-Check-the-free-space-prior.patch +Patch0042: 0042-targetuserspacecreator-Update-err-msg-when-installin.patch %description @@ -130,7 +142,7 @@ Requires: leapp-repository-dependencies = %{leapp_repo_deps} # IMPORTANT: this is capability provided by the leapp framework rpm. # Check that 'version' instead of the real framework rpm version. -Requires: leapp-framework >= 3.1 +Requires: leapp-framework >= 4.0 # Since we provide sub-commands for the leapp utility, we expect the leapp # tool to be installed as well. @@ -257,6 +269,18 @@ Requires: python3-gobject-base %patch0028 -p1 %patch0029 -p1 %patch0030 -p1 +%patch0031 -p1 +%patch0032 -p1 +%patch0033 -p1 +%patch0034 -p1 +%patch0035 -p1 +%patch0036 -p1 +%patch0037 -p1 +%patch0038 -p1 +%patch0039 -p1 +%patch0040 -p1 +%patch0041 -p1 +%patch0042 -p1 %build @@ -334,6 +358,20 @@ done; # no files here %changelog +* Mon Jul 18 2023 Petr Stodulka - 0.18.0-5 +- Fix the calculation of the required free space on each partitions/volume for the upgrade transactions +- Create source overlay images with dynamic sizes to optimize disk space consumption +- Update GRUB2 when /boot resides on multiple devices aggregated in RAID +- Use new leapp CLI API which provides better report summary output +- Introduce possibility to add (custom) kernel drivers to initramfs +- Detect and report use of deprecated Xorg drivers +- Fix the generation of the report about hybrid images +- Inhibit the upgrade when unsupported x86-64 microarchitecture is detected +- Minor improvements and fixes of various reports +- Requires leapp-framework 4.0 +- Update leapp data files +- Resolves: rhbz#2140011, rhbz#2144304, rhbz#2174095, rhbz#2219544, rhbz#2215997 + * Mon Jun 19 2023 Petr Stodulka - 0.18.0-4 - Introduce new upgrade path RHEL 8.9 -> 9.3 - Update leapp data files to reflect new changes between systems