From 515a6a7b22c0848bacde96cee66449435b3340d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michal=20He=C4=8Dko?= Date: Wed, 16 Nov 2022 18:15:00 +0100 Subject: [PATCH 30/32] Support IPU using a target RHEL installation ISO image (#979) Introduced an option to use an ISO file as a target RHEL version content source With the current enhancement, it's possible to IPU using RHEL ISO image. For that case it's introduced the --iso CLI option: leapp upgrade --iso PATH_TO_RHEL_ISO The ISO must be stored on local partition (removable and network media are not allowed). Packaging: * Requires cpio * Bump leapp-repository-dependencies to 8 New models: TargetOSInstallationImage --- Makefile | 2 +- commands/preupgrade/__init__.py | 1 + commands/upgrade/__init__.py | 1 + commands/upgrade/util.py | 7 + packaging/leapp-el7toel8-deps.spec | 6 +- packaging/leapp-repository.spec | 6 +- .../common/actors/checktargetiso/actor.py | 18 ++ .../libraries/check_target_iso.py | 182 +++++++++++++++ .../tests/test_check_target_iso.py | 168 +++++++++++++ .../common/actors/createisorepofile/actor.py | 18 ++ .../libraries/create_iso_repofile.py | 36 +++ .../common/actors/dnfdryrun/actor.py | 6 +- .../common/actors/dnfpackagedownload/actor.py | 6 +- .../actors/dnftransactioncheck/actor.py | 5 +- .../actors/initramfs/mounttargetiso/actor.py | 16 ++ .../libraries/mount_target_iso.py | 27 +++ .../upgradeinitramfsgenerator/actor.py | 2 + .../libraries/upgradeinitramfsgenerator.py | 8 +- .../common/actors/localreposinhibit/actor.py | 59 +++-- .../tests/test_unit_localreposinhibit.py | 9 + .../common/actors/scantargetiso/actor.py | 16 ++ .../libraries/scan_target_os_iso.py | 96 ++++++++ .../tests/test_scan_target_iso.py | 220 ++++++++++++++++++ .../actors/targetuserspacecreator/actor.py | 4 +- .../libraries/userspacegen.py | 30 +-- .../tests/unit_test_targetuserspacecreator.py | 1 + .../common/libraries/dnfplugin.py | 47 +++- .../common/libraries/mounting.py | 20 ++ .../common/models/upgradeiso.py | 14 ++ 29 files changed, 977 insertions(+), 54 deletions(-) create mode 100644 repos/system_upgrade/common/actors/checktargetiso/actor.py create mode 100644 repos/system_upgrade/common/actors/checktargetiso/libraries/check_target_iso.py create mode 100644 repos/system_upgrade/common/actors/checktargetiso/tests/test_check_target_iso.py create mode 100644 repos/system_upgrade/common/actors/createisorepofile/actor.py create mode 100644 repos/system_upgrade/common/actors/createisorepofile/libraries/create_iso_repofile.py create mode 100644 repos/system_upgrade/common/actors/initramfs/mounttargetiso/actor.py create mode 100644 repos/system_upgrade/common/actors/initramfs/mounttargetiso/libraries/mount_target_iso.py create mode 100644 repos/system_upgrade/common/actors/scantargetiso/actor.py create mode 100644 repos/system_upgrade/common/actors/scantargetiso/libraries/scan_target_os_iso.py create mode 100644 repos/system_upgrade/common/actors/scantargetiso/tests/test_scan_target_iso.py create mode 100644 repos/system_upgrade/common/models/upgradeiso.py diff --git a/Makefile b/Makefile index e8d9f170..7342d4bf 100644 --- a/Makefile +++ b/Makefile @@ -448,7 +448,7 @@ clean_containers: fast_lint: @. $(VENVNAME)/bin/activate; \ - FILES_TO_LINT="$$(git diff --name-only $(MASTER_BRANCH)| grep '\.py$$')"; \ + FILES_TO_LINT="$$(git diff --name-only $(MASTER_BRANCH) --diff-filter AMR | grep '\.py$$')"; \ if [[ -n "$$FILES_TO_LINT" ]]; then \ pylint -j 0 $$FILES_TO_LINT && \ flake8 $$FILES_TO_LINT; \ diff --git a/commands/preupgrade/__init__.py b/commands/preupgrade/__init__.py index be2c7be8..d612fbb1 100644 --- a/commands/preupgrade/__init__.py +++ b/commands/preupgrade/__init__.py @@ -24,6 +24,7 @@ from leapp.utils.output import beautify_actor_exception, report_errors, report_i help='Set preferred channel for the IPU target.', choices=['ga', 'tuv', 'e4s', 'eus', 'aus'], value_type=str.lower) # This allows the choices to be case insensitive +@command_opt('iso', help='Use provided target RHEL installation image to perform the in-place upgrade.') @command_opt('target', choices=command_utils.get_supported_target_versions(), help='Specify RHEL version to upgrade to for {} detected upgrade flavour'.format( command_utils.get_upgrade_flavour())) diff --git a/commands/upgrade/__init__.py b/commands/upgrade/__init__.py index 39bfd525..005538ed 100644 --- a/commands/upgrade/__init__.py +++ b/commands/upgrade/__init__.py @@ -30,6 +30,7 @@ from leapp.utils.output import beautify_actor_exception, report_errors, report_i help='Set preferred channel for the IPU target.', choices=['ga', 'tuv', 'e4s', 'eus', 'aus'], value_type=str.lower) # This allows the choices to be case insensitive +@command_opt('iso', help='Use provided target RHEL installation image to perform the in-place upgrade.') @command_opt('target', choices=command_utils.get_supported_target_versions(), help='Specify RHEL version to upgrade to for {} detected upgrade flavour'.format( command_utils.get_upgrade_flavour())) diff --git a/commands/upgrade/util.py b/commands/upgrade/util.py index ce0b5433..aa433786 100644 --- a/commands/upgrade/util.py +++ b/commands/upgrade/util.py @@ -199,6 +199,13 @@ def prepare_configuration(args): if args.channel: os.environ['LEAPP_TARGET_PRODUCT_CHANNEL'] = args.channel + if args.iso: + os.environ['LEAPP_TARGET_ISO'] = args.iso + target_iso_path = os.environ.get('LEAPP_TARGET_ISO') + if target_iso_path: + # Make sure we convert rel paths into abs ones while we know what CWD is + os.environ['LEAPP_TARGET_ISO'] = os.path.abspath(target_iso_path) + # Check upgrade path and fail early if it's unsupported target_version, flavor = command_utils.vet_upgrade_path(args) os.environ['LEAPP_UPGRADE_PATH_TARGET_RELEASE'] = target_version diff --git a/packaging/leapp-el7toel8-deps.spec b/packaging/leapp-el7toel8-deps.spec index cdfa7f98..822b6f63 100644 --- a/packaging/leapp-el7toel8-deps.spec +++ b/packaging/leapp-el7toel8-deps.spec @@ -9,7 +9,7 @@ %endif -%define leapp_repo_deps 7 +%define leapp_repo_deps 8 %define leapp_framework_deps 5 # NOTE: the Version contains the %{rhel} macro just for the convenience to @@ -61,6 +61,10 @@ Requires: dnf-command(config-manager) # sure Requires: dracut +# Used to determine RHEL version of a given target RHEL installation image - +# uncompressing redhat-release package from the ISO. +Requires: cpio + # just to be sure that /etc/modprobe.d is present Requires: kmod diff --git a/packaging/leapp-repository.spec b/packaging/leapp-repository.spec index 89750927..0ffba71c 100644 --- a/packaging/leapp-repository.spec +++ b/packaging/leapp-repository.spec @@ -2,7 +2,7 @@ %global repositorydir %{leapp_datadir}/repositories %global custom_repositorydir %{leapp_datadir}/custom-repositories -%define leapp_repo_deps 7 +%define leapp_repo_deps 8 %if 0%{?rhel} == 7 %define leapp_python_sitelib %{python2_sitelib} @@ -106,6 +106,10 @@ Requires: leapp-framework >= 3.1, leapp-framework < 4 # tool to be installed as well. Requires: leapp +# Used to determine RHEL version of a given target RHEL installation image - +# uncompressing redhat-release package from the ISO. +Requires: cpio + # The leapp-repository rpm is renamed to %%{lpr_name} Obsoletes: leapp-repository < 0.14.0-%{release} Provides: leapp-repository = %{version}-%{release} diff --git a/repos/system_upgrade/common/actors/checktargetiso/actor.py b/repos/system_upgrade/common/actors/checktargetiso/actor.py new file mode 100644 index 00000000..4d602de8 --- /dev/null +++ b/repos/system_upgrade/common/actors/checktargetiso/actor.py @@ -0,0 +1,18 @@ +from leapp.actors import Actor +from leapp.libraries.actor import check_target_iso +from leapp.models import Report, StorageInfo, TargetOSInstallationImage +from leapp.tags import ChecksPhaseTag, IPUWorkflowTag + + +class CheckTargetISO(Actor): + """ + Check that the provided target ISO is a valid ISO image and is located on a persistent partition. + """ + + name = 'check_target_iso' + consumes = (StorageInfo, TargetOSInstallationImage,) + produces = (Report,) + tags = (IPUWorkflowTag, ChecksPhaseTag) + + def process(self): + check_target_iso.perform_target_iso_checks() diff --git a/repos/system_upgrade/common/actors/checktargetiso/libraries/check_target_iso.py b/repos/system_upgrade/common/actors/checktargetiso/libraries/check_target_iso.py new file mode 100644 index 00000000..b5b66901 --- /dev/null +++ b/repos/system_upgrade/common/actors/checktargetiso/libraries/check_target_iso.py @@ -0,0 +1,182 @@ +import os + +from leapp import reporting +from leapp.exceptions import StopActorExecutionError +from leapp.libraries.common.config import version +from leapp.libraries.stdlib import api, CalledProcessError, run +from leapp.models import StorageInfo, TargetOSInstallationImage + + +def inhibit_if_not_valid_iso_file(iso): + inhibit_title = None + target_os = 'RHEL {}'.format(version.get_target_major_version()) + if not os.path.exists(iso.path): + inhibit_title = 'Provided {target_os} installation ISO does not exists.'.format(target_os=target_os) + inhibit_summary_tpl = 'The supplied {target_os} ISO path \'{iso_path}\' does not point to an existing file.' + inhibit_summary = inhibit_summary_tpl.format(target_os=target_os, iso_path=iso.path) + else: + try: + # TODO(mhecko): Figure out whether we will keep this since the scan actor is mounting the ISO anyway + file_cmd_output = run(['file', '--mime', iso.path]) + if 'application/x-iso9660-image' not in file_cmd_output['stdout']: + inhibit_title = 'Provided {target_os} installation image is not a valid ISO.'.format( + target_os=target_os) + summary_tpl = ('The provided {target_os} installation image path \'{iso_path}\'' + 'does not point to a valid ISO image.') + inhibit_summary = summary_tpl.format(target_os=target_os, iso_path=iso.path) + + except CalledProcessError as err: + raise StopActorExecutionError(message='Failed to check whether {0} is an ISO file.'.format(iso.path), + details={'details': '{}'.format(err)}) + if inhibit_title: + remediation_hint = ('Check whether the supplied target OS installation path points to a valid' + '{target_os} ISO image.'.format(target_os=target_os)) + + reporting.create_report([ + reporting.Title(inhibit_title), + reporting.Summary(inhibit_summary), + reporting.Remediation(hint=remediation_hint), + reporting.Severity(reporting.Severity.MEDIUM), + reporting.Groups([reporting.Groups.INHIBITOR]), + reporting.Groups([reporting.Groups.REPOSITORY]), + ]) + return True + return False + + +def inhibit_if_failed_to_mount_iso(iso): + if iso.was_mounted_successfully: + return False + + target_os = 'RHEL {0}'.format(version.get_target_major_version()) + title = 'Failed to mount the provided {target_os} installation image.' + summary = 'The provided {target_os} installation image {iso_path} could not be mounted.' + hint = 'Verify that the provided ISO is a valid {target_os} installation image' + reporting.create_report([ + reporting.Title(title.format(target_os=target_os)), + reporting.Summary(summary.format(target_os=target_os, iso_path=iso.path)), + reporting.Remediation(hint=hint.format(target_os=target_os)), + reporting.Severity(reporting.Severity.MEDIUM), + reporting.Groups([reporting.Groups.INHIBITOR]), + reporting.Groups([reporting.Groups.REPOSITORY]), + ]) + return True + + +def inhibit_if_wrong_iso_rhel_version(iso): + # If the major version could not be determined, the iso.rhel_version will be an empty string + if not iso.rhel_version: + reporting.create_report([ + reporting.Title( + 'Failed to determine RHEL version provided by the supplied installation image.'), + reporting.Summary( + 'Could not determine what RHEL version does the supplied installation image' + ' located at {iso_path} provide.'.format(iso_path=iso.path) + ), + reporting.Remediation(hint='Check that the supplied image is a valid RHEL installation image.'), + reporting.Severity(reporting.Severity.MEDIUM), + reporting.Groups([reporting.Groups.INHIBITOR]), + reporting.Groups([reporting.Groups.REPOSITORY]), + ]) + return + + iso_rhel_major_version = iso.rhel_version.split('.')[0] + req_major_ver = version.get_target_major_version() + if iso_rhel_major_version != req_major_ver: + summary = ('The provided RHEL installation image provides RHEL {iso_rhel_ver}, however, a RHEL ' + '{required_rhel_ver} image is required for the upgrade.') + + reporting.create_report([ + reporting.Title('The provided installation image provides invalid RHEL version.'), + reporting.Summary(summary.format(iso_rhel_ver=iso.rhel_version, required_rhel_ver=req_major_ver)), + reporting.Remediation(hint='Check that the supplied image is a valid RHEL installation image.'), + reporting.Severity(reporting.Severity.MEDIUM), + reporting.Groups([reporting.Groups.INHIBITOR]), + reporting.Groups([reporting.Groups.REPOSITORY]), + ]) + + +def inhibit_if_iso_not_located_on_persistent_partition(iso): + # Check whether the filesystem that on which the ISO resides is mounted in a persistent fashion + storage_info = next(api.consume(StorageInfo), None) + if not storage_info: + raise StopActorExecutionError('Actor did not receive any StorageInfo message.') + + # Assumes that the path has been already checked for validity, e.g., the ISO path points to a file + iso_mountpoint = iso.path + while not os.path.ismount(iso_mountpoint): # Guaranteed to terminate because we must reach / eventually + iso_mountpoint = os.path.dirname(iso_mountpoint) + + is_iso_on_persistent_partition = False + for fstab_entry in storage_info.fstab: + if fstab_entry.fs_file == iso_mountpoint: + is_iso_on_persistent_partition = True + break + + if not is_iso_on_persistent_partition: + target_ver = version.get_target_major_version() + title = 'The RHEL {target_ver} installation image is not located on a persistently mounted partition' + summary = ('The provided RHEL {target_ver} installation image {iso_path} is located' + ' on a partition without an entry in /etc/fstab, causing the partition ' + ' to be persistently mounted.') + hint = ('Move the installation image to a partition that is persistently mounted, or create an /etc/fstab' + ' entry for the partition on which the installation image is located.') + + reporting.create_report([ + reporting.Title(title.format(target_ver=target_ver)), + reporting.Summary(summary.format(target_ver=target_ver, iso_path=iso.path)), + reporting.Remediation(hint=hint), + reporting.RelatedResource('file', '/etc/fstab'), + reporting.Severity(reporting.Severity.MEDIUM), + reporting.Groups([reporting.Groups.INHIBITOR]), + reporting.Groups([reporting.Groups.REPOSITORY]), + ]) + + +def inihibit_if_iso_does_not_contain_basic_repositories(iso): + missing_basic_repoids = {'BaseOS', 'AppStream'} + + for custom_repo in iso.repositories: + missing_basic_repoids.remove(custom_repo.repoid) + if not missing_basic_repoids: + break + + if missing_basic_repoids: + target_ver = version.get_target_major_version() + + title = 'Provided RHEL {target_ver} installation ISO is missing fundamental repositories.' + summary = ('The supplied RHEL {target_ver} installation ISO {iso_path} does not contain ' + '{missing_repos} repositor{suffix}') + hint = 'Check whether the supplied ISO is a valid RHEL {target_ver} installation image.' + + reporting.create_report([ + reporting.Title(title.format(target_ver=target_ver)), + reporting.Summary(summary.format(target_ver=target_ver, + iso_path=iso.path, + missing_repos=','.join(missing_basic_repoids), + suffix=('y' if len(missing_basic_repoids) == 1 else 'ies'))), + reporting.Remediation(hint=hint.format(target_ver=target_ver)), + reporting.Severity(reporting.Severity.MEDIUM), + reporting.Groups([reporting.Groups.INHIBITOR]), + reporting.Groups([reporting.Groups.REPOSITORY]), + ]) + + +def perform_target_iso_checks(): + requested_target_iso_msg_iter = api.consume(TargetOSInstallationImage) + target_iso = next(requested_target_iso_msg_iter, None) + + if not target_iso: + return + + if next(requested_target_iso_msg_iter, None): + api.current_logger().warn('Received multiple msgs with target ISO to use.') + + # Cascade the inhibiting conditions so that we do not spam the user with inhibitors + is_iso_invalid = inhibit_if_not_valid_iso_file(target_iso) + if not is_iso_invalid: + failed_to_mount_iso = inhibit_if_failed_to_mount_iso(target_iso) + if not failed_to_mount_iso: + inhibit_if_wrong_iso_rhel_version(target_iso) + inhibit_if_iso_not_located_on_persistent_partition(target_iso) + inihibit_if_iso_does_not_contain_basic_repositories(target_iso) diff --git a/repos/system_upgrade/common/actors/checktargetiso/tests/test_check_target_iso.py b/repos/system_upgrade/common/actors/checktargetiso/tests/test_check_target_iso.py new file mode 100644 index 00000000..d819bc34 --- /dev/null +++ b/repos/system_upgrade/common/actors/checktargetiso/tests/test_check_target_iso.py @@ -0,0 +1,168 @@ +import os + +import pytest + +from leapp import reporting +from leapp.libraries.actor import check_target_iso +from leapp.libraries.common.testutils import create_report_mocked, CurrentActorMocked +from leapp.libraries.stdlib import api +from leapp.models import CustomTargetRepository, FstabEntry, StorageInfo, TargetOSInstallationImage +from leapp.utils.report import is_inhibitor + + +@pytest.mark.parametrize('mount_successful', (True, False)) +def test_inhibit_on_iso_mount_failure(monkeypatch, mount_successful): + create_report_mock = create_report_mocked() + monkeypatch.setattr(reporting, 'create_report', create_report_mock) + monkeypatch.setattr(api, 'current_actor', CurrentActorMocked()) + + target_iso_msg = TargetOSInstallationImage(path='', + mountpoint='', + repositories=[], + was_mounted_successfully=mount_successful) + + check_target_iso.inhibit_if_failed_to_mount_iso(target_iso_msg) + + expected_report_count = 0 if mount_successful else 1 + assert create_report_mock.called == expected_report_count + if not mount_successful: + assert is_inhibitor(create_report_mock.reports[0]) + + +@pytest.mark.parametrize(('detected_iso_rhel_ver', 'required_target_ver', 'should_inhibit'), + (('8.6', '8.6', False), ('7.9', '8.6', True), ('8.5', '8.6', False), ('', '8.6', True))) +def test_inhibit_on_detected_rhel_version(monkeypatch, detected_iso_rhel_ver, required_target_ver, should_inhibit): + create_report_mock = create_report_mocked() + monkeypatch.setattr(reporting, 'create_report', create_report_mock) + monkeypatch.setattr(api, 'current_actor', CurrentActorMocked(dst_ver=required_target_ver)) + + target_iso_msg = TargetOSInstallationImage(path='', + mountpoint='', + repositories=[], + rhel_version=detected_iso_rhel_ver, + was_mounted_successfully=True) + + check_target_iso.inhibit_if_wrong_iso_rhel_version(target_iso_msg) + + expected_report_count = 1 if should_inhibit else 0 + assert create_report_mock.called == expected_report_count + if should_inhibit: + assert is_inhibitor(create_report_mock.reports[0]) + + +@pytest.mark.parametrize(('iso_repoids', 'should_inhibit'), + ((('BaseOS', 'AppStream'), False), (('BaseOS',), True), (('AppStream',), True), ((), True))) +def test_inhibit_on_invalid_rhel_version(monkeypatch, iso_repoids, should_inhibit): + create_report_mock = create_report_mocked() + monkeypatch.setattr(reporting, 'create_report', create_report_mock) + monkeypatch.setattr(api, 'current_actor', CurrentActorMocked()) + + iso_repositories = [CustomTargetRepository(repoid=repoid, baseurl='', name='') for repoid in iso_repoids] + + target_iso_msg = TargetOSInstallationImage(path='', + mountpoint='', + repositories=iso_repositories, + was_mounted_successfully=True) + + check_target_iso.inihibit_if_iso_does_not_contain_basic_repositories(target_iso_msg) + + expected_report_count = 1 if should_inhibit else 0 + assert create_report_mock.called == expected_report_count + if should_inhibit: + assert is_inhibitor(create_report_mock.reports[0]) + + +def test_inhibit_on_nonexistent_iso(monkeypatch): + iso_path = '/nonexistent/iso' + create_report_mock = create_report_mocked() + monkeypatch.setattr(reporting, 'create_report', create_report_mock) + monkeypatch.setattr(api, 'current_actor', CurrentActorMocked()) + + def mocked_os_path_exists(path): + assert path == iso_path, 'The actor should check only the path to ISO for existence.' + return False + + monkeypatch.setattr(os.path, 'exists', mocked_os_path_exists) + + target_iso_msg = TargetOSInstallationImage(path=iso_path, + mountpoint='', + repositories=[], + was_mounted_successfully=True) + + check_target_iso.inhibit_if_not_valid_iso_file(target_iso_msg) + + assert create_report_mock.called == 1 + assert is_inhibitor(create_report_mock.reports[0]) + + +@pytest.mark.parametrize(('filetype', 'should_inhibit'), + (('{path}: text/plain; charset=us-ascii', True), + ('{path}: application/x-iso9660-image; charset=binary', False))) +def test_inhibit_on_path_not_pointing_to_iso(monkeypatch, filetype, should_inhibit): + iso_path = '/path/not-an-iso' + create_report_mock = create_report_mocked() + monkeypatch.setattr(reporting, 'create_report', create_report_mock) + monkeypatch.setattr(api, 'current_actor', CurrentActorMocked()) + + def mocked_os_path_exists(path): + assert path == iso_path, 'The actor should check only the path to ISO for existence.' + return True + + def mocked_run(cmd, *args, **kwargs): + assert cmd[0] == 'file', 'The actor should only use `file` cmd when checking for file type.' + return {'stdout': filetype.format(path=iso_path)} + + monkeypatch.setattr(os.path, 'exists', mocked_os_path_exists) + monkeypatch.setattr(check_target_iso, 'run', mocked_run) + + target_iso_msg = TargetOSInstallationImage(path=iso_path, mountpoint='', repositories=[]) + + check_target_iso.inhibit_if_not_valid_iso_file(target_iso_msg) + + if should_inhibit: + assert create_report_mock.called == 1 + assert is_inhibitor(create_report_mock.reports[0]) + else: + assert create_report_mock.called == 0 + + +@pytest.mark.parametrize('is_persistently_mounted', (False, True)) +def test_inhibition_when_iso_not_on_persistent_partition(monkeypatch, is_persistently_mounted): + path_mountpoint = '/d0/d1' + iso_path = '/d0/d1/d2/d3/iso' + create_report_mock = create_report_mocked() + monkeypatch.setattr(reporting, 'create_report', create_report_mock) + + def os_path_ismount_mocked(path): + if path == path_mountpoint: + return True + if path == '/': # / Should be a mountpoint on every system + return True + return False + + monkeypatch.setattr(os.path, 'ismount', os_path_ismount_mocked) + + fstab_mountpoint = path_mountpoint if is_persistently_mounted else '/some/other/mountpoint' + fstab_entry = FstabEntry(fs_spec='/dev/sta2', fs_file=fstab_mountpoint, + fs_vfstype='', fs_mntops='', fs_freq='', fs_passno='') + storage_info_msg = StorageInfo(fstab=[fstab_entry]) + + monkeypatch.setattr(api, 'current_actor', CurrentActorMocked(msgs=[storage_info_msg])) + + target_iso_msg = TargetOSInstallationImage(path=iso_path, mountpoint='', repositories=[]) + check_target_iso.inhibit_if_iso_not_located_on_persistent_partition(target_iso_msg) + + if is_persistently_mounted: + assert not create_report_mock.called + else: + assert create_report_mock.called == 1 + assert is_inhibitor(create_report_mock.reports[0]) + + +def test_actor_does_not_perform_when_iso_not_used(monkeypatch): + monkeypatch.setattr(reporting, 'create_report', create_report_mocked()) + monkeypatch.setattr(api, 'current_actor', CurrentActorMocked()) + + check_target_iso.perform_target_iso_checks() + + assert not reporting.create_report.called diff --git a/repos/system_upgrade/common/actors/createisorepofile/actor.py b/repos/system_upgrade/common/actors/createisorepofile/actor.py new file mode 100644 index 00000000..5c4fa760 --- /dev/null +++ b/repos/system_upgrade/common/actors/createisorepofile/actor.py @@ -0,0 +1,18 @@ +from leapp.actors import Actor +from leapp.libraries.actor import create_iso_repofile +from leapp.models import CustomTargetRepositoryFile, TargetOSInstallationImage +from leapp.tags import IPUWorkflowTag, TargetTransactionFactsPhaseTag + + +class CreateISORepofile(Actor): + """ + Create custom repofile containing information about repositories found in target OS installation ISO, if used. + """ + + name = 'create_iso_repofile' + consumes = (TargetOSInstallationImage,) + produces = (CustomTargetRepositoryFile,) + tags = (IPUWorkflowTag, TargetTransactionFactsPhaseTag) + + def process(self): + create_iso_repofile.produce_repofile_if_iso_used() diff --git a/repos/system_upgrade/common/actors/createisorepofile/libraries/create_iso_repofile.py b/repos/system_upgrade/common/actors/createisorepofile/libraries/create_iso_repofile.py new file mode 100644 index 00000000..b4470b68 --- /dev/null +++ b/repos/system_upgrade/common/actors/createisorepofile/libraries/create_iso_repofile.py @@ -0,0 +1,36 @@ +import os + +from leapp.libraries.common.config.version import get_target_major_version +from leapp.libraries.stdlib import api +from leapp.models import CustomTargetRepositoryFile, TargetOSInstallationImage + + +def produce_repofile_if_iso_used(): + target_iso_msgs_iter = api.consume(TargetOSInstallationImage) + target_iso = next(target_iso_msgs_iter, None) + + if not target_iso: + return + + if next(target_iso_msgs_iter, None): + api.current_logger().warn('Received multiple TargetISInstallationImage messages, using the first one') + + # Mounting was successful, create a repofile to copy into target userspace + repofile_entry_template = ('[{repoid}]\n' + 'name={reponame}\n' + 'baseurl={baseurl}\n' + 'enabled=0\n' + 'gpgcheck=0\n') + + repofile_content = '' + for repo in target_iso.repositories: + repofile_content += repofile_entry_template.format(repoid=repo.repoid, + reponame=repo.repoid, + baseurl=repo.baseurl) + + target_os_path_prefix = 'el{target_major_ver}'.format(target_major_ver=get_target_major_version()) + iso_repofile_path = os.path.join('/var/lib/leapp/', '{}_iso.repo'.format(target_os_path_prefix)) + with open(iso_repofile_path, 'w') as iso_repofile: + iso_repofile.write(repofile_content) + + api.produce(CustomTargetRepositoryFile(file=iso_repofile_path)) diff --git a/repos/system_upgrade/common/actors/dnfdryrun/actor.py b/repos/system_upgrade/common/actors/dnfdryrun/actor.py index 7cfce25f..bc3267b4 100644 --- a/repos/system_upgrade/common/actors/dnfdryrun/actor.py +++ b/repos/system_upgrade/common/actors/dnfdryrun/actor.py @@ -7,6 +7,7 @@ from leapp.models import ( FilteredRpmTransactionTasks, RHUIInfo, StorageInfo, + TargetOSInstallationImage, TargetUserSpaceInfo, TransactionDryRun, UsedTargetRepositories, @@ -31,6 +32,7 @@ class DnfDryRun(Actor): FilteredRpmTransactionTasks, RHUIInfo, StorageInfo, + TargetOSInstallationImage, TargetUserSpaceInfo, UsedTargetRepositories, XFSPresence, @@ -46,10 +48,12 @@ class DnfDryRun(Actor): tasks = next(self.consume(FilteredRpmTransactionTasks), FilteredRpmTransactionTasks()) target_userspace_info = next(self.consume(TargetUserSpaceInfo), None) rhui_info = next(self.consume(RHUIInfo), None) + target_iso = next(self.consume(TargetOSInstallationImage), None) on_aws = bool(rhui_info and rhui_info.provider == 'aws') dnfplugin.perform_dry_run( tasks=tasks, used_repos=used_repos, target_userspace_info=target_userspace_info, - xfs_info=xfs_info, storage_info=storage_info, plugin_info=plugin_info, on_aws=on_aws + xfs_info=xfs_info, storage_info=storage_info, plugin_info=plugin_info, on_aws=on_aws, + target_iso=target_iso, ) self.produce(TransactionDryRun()) diff --git a/repos/system_upgrade/common/actors/dnfpackagedownload/actor.py b/repos/system_upgrade/common/actors/dnfpackagedownload/actor.py index f27045c3..b54f5627 100644 --- a/repos/system_upgrade/common/actors/dnfpackagedownload/actor.py +++ b/repos/system_upgrade/common/actors/dnfpackagedownload/actor.py @@ -6,6 +6,7 @@ from leapp.models import ( FilteredRpmTransactionTasks, RHUIInfo, StorageInfo, + TargetOSInstallationImage, TargetUserSpaceInfo, UsedTargetRepositories, XFSPresence @@ -28,6 +29,7 @@ class DnfPackageDownload(Actor): FilteredRpmTransactionTasks, RHUIInfo, StorageInfo, + TargetOSInstallationImage, TargetUserSpaceInfo, UsedTargetRepositories, XFSPresence, @@ -45,8 +47,10 @@ class DnfPackageDownload(Actor): rhui_info = next(self.consume(RHUIInfo), None) # there are several "variants" related to the *AWS* provider (aws, aws-sap) on_aws = bool(rhui_info and rhui_info.provider.startswith('aws')) + target_iso = next(self.consume(TargetOSInstallationImage), None) dnfplugin.perform_rpm_download( tasks=tasks, used_repos=used_repos, target_userspace_info=target_userspace_info, - xfs_info=xfs_info, storage_info=storage_info, plugin_info=plugin_info, on_aws=on_aws + xfs_info=xfs_info, storage_info=storage_info, plugin_info=plugin_info, on_aws=on_aws, + target_iso=target_iso ) diff --git a/repos/system_upgrade/common/actors/dnftransactioncheck/actor.py b/repos/system_upgrade/common/actors/dnftransactioncheck/actor.py index f741b77b..b545d1ce 100644 --- a/repos/system_upgrade/common/actors/dnftransactioncheck/actor.py +++ b/repos/system_upgrade/common/actors/dnftransactioncheck/actor.py @@ -5,6 +5,7 @@ from leapp.models import ( DNFWorkaround, FilteredRpmTransactionTasks, StorageInfo, + TargetOSInstallationImage, TargetUserSpaceInfo, UsedTargetRepositories, XFSPresence @@ -23,6 +24,7 @@ class DnfTransactionCheck(Actor): DNFWorkaround, FilteredRpmTransactionTasks, StorageInfo, + TargetOSInstallationImage, TargetUserSpaceInfo, UsedTargetRepositories, XFSPresence, @@ -37,9 +39,10 @@ class DnfTransactionCheck(Actor): plugin_info = list(self.consume(DNFPluginTask)) tasks = next(self.consume(FilteredRpmTransactionTasks), FilteredRpmTransactionTasks()) target_userspace_info = next(self.consume(TargetUserSpaceInfo), None) + target_iso = next(self.consume(TargetOSInstallationImage), None) if target_userspace_info: dnfplugin.perform_transaction_check( tasks=tasks, used_repos=used_repos, target_userspace_info=target_userspace_info, - xfs_info=xfs_info, storage_info=storage_info, plugin_info=plugin_info + xfs_info=xfs_info, storage_info=storage_info, plugin_info=plugin_info, target_iso=target_iso ) diff --git a/repos/system_upgrade/common/actors/initramfs/mounttargetiso/actor.py b/repos/system_upgrade/common/actors/initramfs/mounttargetiso/actor.py new file mode 100644 index 00000000..950b2694 --- /dev/null +++ b/repos/system_upgrade/common/actors/initramfs/mounttargetiso/actor.py @@ -0,0 +1,16 @@ +from leapp.actors import Actor +from leapp.libraries.actor import mount_target_iso +from leapp.models import TargetOSInstallationImage, TargetUserSpaceInfo +from leapp.tags import IPUWorkflowTag, PreparationPhaseTag + + +class MountTargetISO(Actor): + """Mounts target OS ISO in order to install upgrade packages from it.""" + + name = 'mount_target_iso' + consumes = (TargetUserSpaceInfo, TargetOSInstallationImage,) + produces = () + tags = (PreparationPhaseTag, IPUWorkflowTag) + + def process(self): + mount_target_iso.mount_target_iso() diff --git a/repos/system_upgrade/common/actors/initramfs/mounttargetiso/libraries/mount_target_iso.py b/repos/system_upgrade/common/actors/initramfs/mounttargetiso/libraries/mount_target_iso.py new file mode 100644 index 00000000..7cc45234 --- /dev/null +++ b/repos/system_upgrade/common/actors/initramfs/mounttargetiso/libraries/mount_target_iso.py @@ -0,0 +1,27 @@ +import os + +from leapp.exceptions import StopActorExecutionError +from leapp.libraries.stdlib import api, CalledProcessError, run +from leapp.models import TargetOSInstallationImage, TargetUserSpaceInfo + + +def mount_target_iso(): + target_os_iso = next(api.consume(TargetOSInstallationImage), None) + target_userspace_info = next(api.consume(TargetUserSpaceInfo), None) + + if not target_os_iso: + return + + mountpoint = os.path.join(target_userspace_info.path, target_os_iso.mountpoint[1:]) + if not os.path.exists(mountpoint): + # The target userspace container exists, however, the mountpoint has been removed during cleanup. + os.makedirs(mountpoint) + try: + run(['mount', target_os_iso.path, mountpoint]) + except CalledProcessError as err: + # Unlikely, since we are checking that the ISO is mountable and located on a persistent partition. This would + # likely mean that either the fstab entry for the partition points uses a different device that the one that + # was mounted during pre-reboot, or the fstab has been tampered with before rebooting. Either way, there is + # nothing at this point how we can recover. + msg = 'Failed to mount the target RHEL ISO file containing RPMs to install during the upgrade.' + raise StopActorExecutionError(message=msg, details={'details': '{0}'.format(err)}) diff --git a/repos/system_upgrade/common/actors/initramfs/upgradeinitramfsgenerator/actor.py b/repos/system_upgrade/common/actors/initramfs/upgradeinitramfsgenerator/actor.py index 31e3c61e..dc97172a 100644 --- a/repos/system_upgrade/common/actors/initramfs/upgradeinitramfsgenerator/actor.py +++ b/repos/system_upgrade/common/actors/initramfs/upgradeinitramfsgenerator/actor.py @@ -4,6 +4,7 @@ from leapp.models import RequiredUpgradeInitramPackages # deprecated from leapp.models import UpgradeDracutModule # deprecated from leapp.models import ( BootContent, + TargetOSInstallationImage, TargetUserSpaceInfo, TargetUserSpaceUpgradeTasks, UpgradeInitramfsTasks, @@ -27,6 +28,7 @@ class UpgradeInitramfsGenerator(Actor): name = 'upgrade_initramfs_generator' consumes = ( RequiredUpgradeInitramPackages, # deprecated + TargetOSInstallationImage, TargetUserSpaceInfo, TargetUserSpaceUpgradeTasks, UpgradeDracutModule, # deprecated diff --git a/repos/system_upgrade/common/actors/initramfs/upgradeinitramfsgenerator/libraries/upgradeinitramfsgenerator.py b/repos/system_upgrade/common/actors/initramfs/upgradeinitramfsgenerator/libraries/upgradeinitramfsgenerator.py index 991ace0e..f6539b25 100644 --- a/repos/system_upgrade/common/actors/initramfs/upgradeinitramfsgenerator/libraries/upgradeinitramfsgenerator.py +++ b/repos/system_upgrade/common/actors/initramfs/upgradeinitramfsgenerator/libraries/upgradeinitramfsgenerator.py @@ -9,6 +9,7 @@ from leapp.models import RequiredUpgradeInitramPackages # deprecated from leapp.models import UpgradeDracutModule # deprecated from leapp.models import ( BootContent, + TargetOSInstallationImage, TargetUserSpaceInfo, TargetUserSpaceUpgradeTasks, UpgradeInitramfsTasks, @@ -200,7 +201,8 @@ def copy_boot_files(context): def process(): userspace_info = next(api.consume(TargetUserSpaceInfo), None) - + target_iso = next(api.consume(TargetOSInstallationImage), None) with mounting.NspawnActions(base_dir=userspace_info.path) as context: - prepare_userspace_for_initram(context) - generate_initram_disk(context) + with mounting.mount_upgrade_iso_to_root_dir(userspace_info.path, target_iso): + prepare_userspace_for_initram(context) + generate_initram_disk(context) diff --git a/repos/system_upgrade/common/actors/localreposinhibit/actor.py b/repos/system_upgrade/common/actors/localreposinhibit/actor.py index bff65f2d..edf58792 100644 --- a/repos/system_upgrade/common/actors/localreposinhibit/actor.py +++ b/repos/system_upgrade/common/actors/localreposinhibit/actor.py @@ -1,6 +1,6 @@ from leapp import reporting from leapp.actors import Actor -from leapp.models import TMPTargetRepositoriesFacts, UsedTargetRepositories +from leapp.models import TargetOSInstallationImage, TMPTargetRepositoriesFacts, UsedTargetRepositories from leapp.reporting import Report from leapp.tags import IPUWorkflowTag, TargetTransactionChecksPhaseTag from leapp.utils.deprecation import suppress_deprecation @@ -13,41 +13,58 @@ class LocalReposInhibit(Actor): name = "local_repos_inhibit" consumes = ( UsedTargetRepositories, + TargetOSInstallationImage, TMPTargetRepositoriesFacts, ) produces = (Report,) tags = (IPUWorkflowTag, TargetTransactionChecksPhaseTag) - def file_baseurl_in_use(self): - """Check if any of target repos is local. + def collect_target_repoids_with_local_url(self, used_target_repos, target_repos_facts, target_iso): + """Collects all repoids that have a local (file://) URL. UsedTargetRepositories doesn't contain baseurl attribute. So gathering them from model TMPTargetRepositoriesFacts. """ - used_target_repos = next(self.consume(UsedTargetRepositories)).repos - target_repos = next(self.consume(TMPTargetRepositoriesFacts)).repositories - target_repo_id_to_url_map = { - repo.repoid: repo.mirrorlist or repo.metalink or repo.baseurl or "" - for repofile in target_repos - for repo in repofile.data - } - return any( - target_repo_id_to_url_map[repo.repoid].startswith("file:") - for repo in used_target_repos - ) + used_target_repoids = set(repo.repoid for repo in used_target_repos.repos) + iso_repoids = set(iso_repo.repoid for iso_repo in target_iso.repositories) if target_iso else set() + + target_repofile_data = (repofile.data for repofile in target_repos_facts.repositories) + + local_repoids = [] + for repo_data in target_repofile_data: + for target_repo in repo_data: + # Check only in repositories that are used and are not provided by the upgrade ISO, if any + if target_repo.repoid not in used_target_repoids or target_repo.repoid in iso_repoids: + continue + + # Repo fields potentially containing local URLs have different importance, check based on their prio + url_field_to_check = target_repo.mirrorlist or target_repo.metalink or target_repo.baseurl or '' + + if url_field_to_check.startswith("file://"): + local_repoids.append(target_repo.repoid) + return local_repoids def process(self): - if not all(next(self.consume(model), None) for model in self.consumes): + used_target_repos = next(self.consume(UsedTargetRepositories), None) + target_repos_facts = next(self.consume(TMPTargetRepositoriesFacts), None) + target_iso = next(self.consume(TargetOSInstallationImage), None) + + if not used_target_repos or not target_repos_facts: return - if self.file_baseurl_in_use(): - warn_msg = ( - "Local repository found (baseurl starts with file:///). " - "Currently leapp does not support this option." - ) + + local_repoids = self.collect_target_repoids_with_local_url(used_target_repos, target_repos_facts, target_iso) + if local_repoids: + suffix, verb = ("y", "has") if len(local_repoids) == 1 else ("ies", "have") + local_repoids_str = ", ".join(local_repoids) + + warn_msg = ("The following local repositor{suffix} {verb} been found: {local_repoids} " + "(their baseurl starts with file:///). Currently leapp does not support this option.") + warn_msg = warn_msg.format(suffix=suffix, verb=verb, local_repoids=local_repoids_str) self.log.warning(warn_msg) + reporting.create_report( [ - reporting.Title("Local repository detected"), + reporting.Title("Local repositor{suffix} detected".format(suffix=suffix)), reporting.Summary(warn_msg), reporting.Severity(reporting.Severity.HIGH), reporting.Groups([reporting.Groups.REPOSITORY]), diff --git a/repos/system_upgrade/common/actors/localreposinhibit/tests/test_unit_localreposinhibit.py b/repos/system_upgrade/common/actors/localreposinhibit/tests/test_unit_localreposinhibit.py index 70156751..64a79e80 100644 --- a/repos/system_upgrade/common/actors/localreposinhibit/tests/test_unit_localreposinhibit.py +++ b/repos/system_upgrade/common/actors/localreposinhibit/tests/test_unit_localreposinhibit.py @@ -3,6 +3,7 @@ import pytest from leapp.models import ( RepositoryData, RepositoryFile, + TargetOSInstallationImage, TMPTargetRepositoriesFacts, UsedTargetRepositories, UsedTargetRepository @@ -70,3 +71,11 @@ def test_unit_localreposinhibit(current_actor_context, baseurl, mirrorlist, meta ) current_actor_context.run() assert len(current_actor_context.messages()) == exp_msgs_len + + +def test_upgrade_not_inhibited_if_iso_used(current_actor_context): + repofile = RepositoryFile(file="path/to/some/file", + data=[RepositoryData(name="BASEOS", baseurl="file:///path", repoid="BASEOS")]) + current_actor_context.feed(TMPTargetRepositoriesFacts(repositories=[repofile])) + current_actor_context.feed(UsedTargetRepositories(repos=[UsedTargetRepository(repoid="BASEOS")])) + current_actor_context.feed(TargetOSInstallationImage(path='', mountpoint='', repositories=[])) diff --git a/repos/system_upgrade/common/actors/scantargetiso/actor.py b/repos/system_upgrade/common/actors/scantargetiso/actor.py new file mode 100644 index 00000000..88b1b8f5 --- /dev/null +++ b/repos/system_upgrade/common/actors/scantargetiso/actor.py @@ -0,0 +1,16 @@ +from leapp.actors import Actor +from leapp.libraries.actor import scan_target_os_iso +from leapp.models import CustomTargetRepository, TargetOSInstallationImage +from leapp.tags import FactsPhaseTag, IPUWorkflowTag + + +class ScanTargetISO(Actor): + """Scans the provided target OS ISO image to use as a content source for the IPU, if any.""" + + name = 'scan_target_os_image' + consumes = () + produces = (CustomTargetRepository, TargetOSInstallationImage,) + tags = (IPUWorkflowTag, FactsPhaseTag) + + def process(self): + scan_target_os_iso.inform_ipu_about_request_to_use_target_iso() diff --git a/repos/system_upgrade/common/actors/scantargetiso/libraries/scan_target_os_iso.py b/repos/system_upgrade/common/actors/scantargetiso/libraries/scan_target_os_iso.py new file mode 100644 index 00000000..281389cf --- /dev/null +++ b/repos/system_upgrade/common/actors/scantargetiso/libraries/scan_target_os_iso.py @@ -0,0 +1,96 @@ +import os + +import leapp.libraries.common.config as ipu_config +from leapp.libraries.common.mounting import LoopMount, MountError +from leapp.libraries.stdlib import api, CalledProcessError, run +from leapp.models import CustomTargetRepository, TargetOSInstallationImage + + +def determine_rhel_version_from_iso_mountpoint(iso_mountpoint): + baseos_packages = os.path.join(iso_mountpoint, 'BaseOS/Packages') + if os.path.isdir(baseos_packages): + def is_rh_release_pkg(pkg_name): + return pkg_name.startswith('redhat-release') and 'eula' not in pkg_name + + redhat_release_pkgs = [pkg for pkg in os.listdir(baseos_packages) if is_rh_release_pkg(pkg)] + + if not redhat_release_pkgs: + return '' # We did not determine anything + + if len(redhat_release_pkgs) > 1: + api.current_logger().warn('Multiple packages with name redhat-release* found when ' + 'determining RHEL version of the supplied installation ISO.') + + redhat_release_pkg = redhat_release_pkgs[0] + + determined_rhel_ver = '' + try: + rh_release_pkg_path = os.path.join(baseos_packages, redhat_release_pkg) + # rpm2cpio is provided by rpm; cpio is a dependency of yum (rhel7) and a dependency of dracut which is + # a dependency for leapp (rhel8+) + cpio_archive = run(['rpm2cpio', rh_release_pkg_path]) + etc_rh_release_contents = run(['cpio', '--extract', '--to-stdout', './etc/redhat-release'], + stdin=cpio_archive['stdout']) + + # 'Red Hat Enterprise Linux Server release 7.9 (Maipo)' -> ['Red Hat...', '7.9 (Maipo'] + product_release_fragments = etc_rh_release_contents['stdout'].split('release') + if len(product_release_fragments) != 2: + return '' # Unlikely. Either way we failed to parse the release + + if not product_release_fragments[0].startswith('Red Hat'): + return '' + + determined_rhel_ver = product_release_fragments[1].strip().split(' ', 1)[0] # Remove release name (Maipo) + return determined_rhel_ver + except CalledProcessError: + return '' + return '' + + +def inform_ipu_about_request_to_use_target_iso(): + target_iso_path = ipu_config.get_env('LEAPP_TARGET_ISO') + if not target_iso_path: + return + + iso_mountpoint = '/iso' + + if not os.path.exists(target_iso_path): + # If the path does not exists, do not attempt to mount it and let the upgrade be inhibited by the check actor + api.produce(TargetOSInstallationImage(path=target_iso_path, + repositories=[], + mountpoint=iso_mountpoint, + was_mounted_successfully=False)) + return + + # Mount the given ISO, extract the available repositories and determine provided RHEL version + iso_scan_mountpoint = '/var/lib/leapp/iso_scan_mountpoint' + try: + with LoopMount(source=target_iso_path, target=iso_scan_mountpoint): + required_repositories = ('BaseOS', 'AppStream') + + # Check what required repositories are present in the root of the ISO + iso_contents = os.listdir(iso_scan_mountpoint) + present_repositories = [req_repo for req_repo in required_repositories if req_repo in iso_contents] + + # Create custom repository information about the repositories found in the root of the ISO + iso_repos = [] + for repo_dir in present_repositories: + baseurl = 'file://' + os.path.join(iso_mountpoint, repo_dir) + iso_repo = CustomTargetRepository(name=repo_dir, baseurl=baseurl, repoid=repo_dir) + api.produce(iso_repo) + iso_repos.append(iso_repo) + + rhel_version = determine_rhel_version_from_iso_mountpoint(iso_scan_mountpoint) + + api.produce(TargetOSInstallationImage(path=target_iso_path, + repositories=iso_repos, + mountpoint=iso_mountpoint, + rhel_version=rhel_version, + was_mounted_successfully=True)) + except MountError: + # Do not analyze the situation any further as ISO checks will be done by another actor + iso_mountpoint = '/iso' + api.produce(TargetOSInstallationImage(path=target_iso_path, + repositories=[], + mountpoint=iso_mountpoint, + was_mounted_successfully=False)) diff --git a/repos/system_upgrade/common/actors/scantargetiso/tests/test_scan_target_iso.py b/repos/system_upgrade/common/actors/scantargetiso/tests/test_scan_target_iso.py new file mode 100644 index 00000000..4dd0a125 --- /dev/null +++ b/repos/system_upgrade/common/actors/scantargetiso/tests/test_scan_target_iso.py @@ -0,0 +1,220 @@ +import contextlib +import os +from functools import partial + +import pytest + +from leapp.libraries.actor import scan_target_os_iso +from leapp.libraries.common.mounting import MountError +from leapp.libraries.common.testutils import CurrentActorMocked, produce_mocked +from leapp.libraries.stdlib import api, CalledProcessError +from leapp.models import CustomTargetRepository, TargetOSInstallationImage + + +def fail_if_called(fail_reason, *args, **kwargs): + assert False, fail_reason + + +def test_determine_rhel_version_determination_unexpected_iso_structure_or_invalid_mountpoint(monkeypatch): + iso_mountpoint = '/some/mountpoint' + + run_mocked = partial(fail_if_called, + 'No commands should be called when mounted ISO mountpoint has unexpected structure.') + monkeypatch.setattr(scan_target_os_iso, 'run', run_mocked) + + def isdir_mocked(path): + assert path == '/some/mountpoint/BaseOS/Packages', 'Only the contents of BaseOS/Packages should be examined.' + return False + + monkeypatch.setattr(os.path, 'isdir', isdir_mocked) + + determined_version = scan_target_os_iso.determine_rhel_version_from_iso_mountpoint(iso_mountpoint) + assert not determined_version + + +def test_determine_rhel_version_valid_iso(monkeypatch): + iso_mountpoint = '/some/mountpoint' + + def isdir_mocked(path): + return True + + def listdir_mocked(path): + assert path == '/some/mountpoint/BaseOS/Packages', 'Only the contents of BaseOS/Packages should be examined.' + return ['xz-5.2.4-4.el8_6.x86_64.rpm', + 'libmodman-2.0.1-17.el8.i686.rpm', + 'redhat-release-8.7-0.3.el8.x86_64.rpm', + 'redhat-release-eula-8.7-0.3.el8.x86_64.rpm'] + + def run_mocked(cmd, *args, **kwargs): + rpm2cpio_output = 'rpm2cpio_output' + if cmd[0] == 'rpm2cpio': + assert cmd == ['rpm2cpio', '/some/mountpoint/BaseOS/Packages/redhat-release-8.7-0.3.el8.x86_64.rpm'] + return {'stdout': rpm2cpio_output} + if cmd[0] == 'cpio': + assert cmd == ['cpio', '--extract', '--to-stdout', './etc/redhat-release'] + assert kwargs['stdin'] == rpm2cpio_output + return {'stdout': 'Red Hat Enterprise Linux Server release 7.9 (Maipo)'} + raise ValueError('Unexpected command has been called.') + + monkeypatch.setattr(os.path, 'isdir', isdir_mocked) + monkeypatch.setattr(os, 'listdir', listdir_mocked) + monkeypatch.setattr(scan_target_os_iso, 'run', run_mocked) + + determined_version = scan_target_os_iso.determine_rhel_version_from_iso_mountpoint(iso_mountpoint) + assert determined_version == '7.9' + + +def test_determine_rhel_version_valid_iso_no_rh_release(monkeypatch): + iso_mountpoint = '/some/mountpoint' + + def isdir_mocked(path): + return True + + def listdir_mocked(path): + assert path == '/some/mountpoint/BaseOS/Packages', 'Only the contents of BaseOS/Packages should be examined.' + return ['xz-5.2.4-4.el8_6.x86_64.rpm', + 'libmodman-2.0.1-17.el8.i686.rpm', + 'redhat-release-eula-8.7-0.3.el8.x86_64.rpm'] + + run_mocked = partial(fail_if_called, 'No command should be called if the redhat-release package is not present.') + + monkeypatch.setattr(os.path, 'isdir', isdir_mocked) + monkeypatch.setattr(os, 'listdir', listdir_mocked) + monkeypatch.setattr(scan_target_os_iso, 'run', run_mocked) + + determined_version = scan_target_os_iso.determine_rhel_version_from_iso_mountpoint(iso_mountpoint) + assert determined_version == '' + + +def test_determine_rhel_version_rpm_extract_fails(monkeypatch): + iso_mountpoint = '/some/mountpoint' + + def isdir_mocked(path): + return True + + def listdir_mocked(path): + assert path == '/some/mountpoint/BaseOS/Packages', 'Only the contents of BaseOS/Packages should be examined.' + return ['redhat-release-8.7-0.3.el8.x86_64.rpm'] + + def run_mocked(cmd, *args, **kwargs): + raise CalledProcessError(message='Ooops.', command=cmd, result=2) + + monkeypatch.setattr(os.path, 'isdir', isdir_mocked) + monkeypatch.setattr(os, 'listdir', listdir_mocked) + monkeypatch.setattr(scan_target_os_iso, 'run', run_mocked) + + determined_version = scan_target_os_iso.determine_rhel_version_from_iso_mountpoint(iso_mountpoint) + assert determined_version == '' + + +@pytest.mark.parametrize('etc_rh_release_contents', ('', + 'Red Hat Enterprise Linux Server', + 'Fedora release 35 (Thirty Five)')) +def test_determine_rhel_version_unexpected_etc_rh_release_contents(monkeypatch, etc_rh_release_contents): + iso_mountpoint = '/some/mountpoint' + + def isdir_mocked(path): + return True + + def listdir_mocked(path): + assert path == '/some/mountpoint/BaseOS/Packages', 'Only the contents of BaseOS/Packages should be examined.' + return ['redhat-release-8.7-0.3.el8.x86_64.rpm'] + + def run_mocked(cmd, *args, **kwargs): + if cmd[0] == 'rpm2cpio': + return {'stdout': 'rpm2cpio_output'} + if cmd[0] == 'cpio': + return {'stdout': etc_rh_release_contents} + raise ValueError('Actor called an unexpected command: {0}'.format(cmd)) + + monkeypatch.setattr(os.path, 'isdir', isdir_mocked) + monkeypatch.setattr(os, 'listdir', listdir_mocked) + monkeypatch.setattr(scan_target_os_iso, 'run', run_mocked) + + determined_version = scan_target_os_iso.determine_rhel_version_from_iso_mountpoint(iso_mountpoint) + assert determined_version == '' + + +@pytest.mark.parametrize('iso_envar_set', (True, False)) +def test_iso_detection_with_no_iso(monkeypatch, iso_envar_set): + envars = {'LEAPP_TARGET_ISO': '/target_iso'} if iso_envar_set else {} + mocked_actor = CurrentActorMocked(envars=envars) + monkeypatch.setattr(api, 'current_actor', mocked_actor) + monkeypatch.setattr(api, 'produce', produce_mocked()) + + scan_target_os_iso.inform_ipu_about_request_to_use_target_iso() + assert bool(api.produce.called) == iso_envar_set + + +def test_iso_mounting_failed(monkeypatch): + envars = {'LEAPP_TARGET_ISO': '/target_iso'} + mocked_actor = CurrentActorMocked(envars=envars) + monkeypatch.setattr(api, 'current_actor', mocked_actor) + monkeypatch.setattr(api, 'produce', produce_mocked()) + + def raise_mount_error_when_called(): + raise MountError('MountError') + + monkeypatch.setattr(scan_target_os_iso, 'LoopMount', raise_mount_error_when_called) + + scan_target_os_iso.inform_ipu_about_request_to_use_target_iso() + assert api.produce.called + + assert len(api.produce.model_instances) == 1 + assert not api.produce.model_instances[0].was_mounted_successfully + + +@pytest.mark.parametrize(('repodirs_in_iso', 'expected_repoids'), + (((), ()), + (('BaseOS',), ('BaseOS',)), + (('BaseOS', 'AppStream'), ('BaseOS', 'AppStream')), + (('BaseOS', 'AppStream', 'UnknownRepo'), ('BaseOS', 'AppStream')))) +def test_iso_repository_detection(monkeypatch, repodirs_in_iso, expected_repoids): + iso_path = '/target_iso' + envars = {'LEAPP_TARGET_ISO': iso_path} + mocked_actor = CurrentActorMocked(envars=envars) + + @contextlib.contextmanager + def always_successful_loop_mount(*args, **kwargs): + yield + + def mocked_os_path_exits(path): + if path == iso_path: + return True + raise ValueError('Only the ISO path should be probed for existence.') + + def mocked_os_listdir(path): + # Add some extra files as an ISO will always have some extra files in / as the ones parametrizing this test + return list(repodirs_in_iso + ('eula.txt', 'grub', 'imgs')) + + monkeypatch.setattr(api, 'current_actor', mocked_actor) + monkeypatch.setattr(api, 'produce', produce_mocked()) + monkeypatch.setattr(scan_target_os_iso, 'LoopMount', always_successful_loop_mount) + monkeypatch.setattr(os.path, 'exists', mocked_os_path_exits) + monkeypatch.setattr(os, 'listdir', mocked_os_listdir) + monkeypatch.setattr(scan_target_os_iso, 'determine_rhel_version_from_iso_mountpoint', lambda iso_mountpoint: '7.9') + + scan_target_os_iso.inform_ipu_about_request_to_use_target_iso() + + produced_msgs = api.produce.model_instances + assert len(produced_msgs) == 1 + len(expected_repoids) + + produced_custom_repo_msgs = [] + target_iso_msg = None + for produced_msg in produced_msgs: + if isinstance(produced_msg, CustomTargetRepository): + produced_custom_repo_msgs.append(produced_msg) + else: + assert not target_iso_msg, 'Actor is expected to produce only one TargetOSInstallationImage msg' + target_iso = produced_msg + + # Do not explicitly instantiate model instances of what we expect the model instance to look like. Instead check + # for expected structural properties, leaving the actor implementation flexibility (e.g. choice of the mountpoint) + iso_mountpoint = target_iso.mountpoint + + assert target_iso.was_mounted_successfully + assert target_iso.rhel_version == '7.9' + + expected_repos = {(repoid, 'file://' + os.path.join(iso_mountpoint, repoid)) for repoid in expected_repoids} + actual_repos = {(repo.repoid, repo.baseurl) for repo in produced_custom_repo_msgs} + assert expected_repos == actual_repos diff --git a/repos/system_upgrade/common/actors/targetuserspacecreator/actor.py b/repos/system_upgrade/common/actors/targetuserspacecreator/actor.py index 04fb2e8b..b1225230 100644 --- a/repos/system_upgrade/common/actors/targetuserspacecreator/actor.py +++ b/repos/system_upgrade/common/actors/targetuserspacecreator/actor.py @@ -2,7 +2,7 @@ from leapp.actors import Actor from leapp.libraries.actor import userspacegen from leapp.libraries.common.config import get_env, version from leapp.models import RequiredTargetUserspacePackages # deprecated -from leapp.models import TMPTargetRepositoriesFacts # deprecated all the time +from leapp.models import TMPTargetRepositoriesFacts # deprecated from leapp.models import ( CustomTargetRepositoryFile, PkgManagerInfo, @@ -12,6 +12,7 @@ from leapp.models import ( RHSMInfo, RHUIInfo, StorageInfo, + TargetOSInstallationImage, TargetRepositories, TargetUserSpaceInfo, TargetUserSpacePreupgradeTasks, @@ -42,6 +43,7 @@ class TargetUserspaceCreator(Actor): RepositoriesMapping, RequiredTargetUserspacePackages, StorageInfo, + TargetOSInstallationImage, TargetRepositories, TargetUserSpacePreupgradeTasks, XFSPresence, diff --git a/repos/system_upgrade/common/actors/targetuserspacecreator/libraries/userspacegen.py b/repos/system_upgrade/common/actors/targetuserspacecreator/libraries/userspacegen.py index 00acacd9..5a6a80f2 100644 --- a/repos/system_upgrade/common/actors/targetuserspacecreator/libraries/userspacegen.py +++ b/repos/system_upgrade/common/actors/targetuserspacecreator/libraries/userspacegen.py @@ -9,7 +9,7 @@ from leapp.libraries.common.config import get_env, get_product_type from leapp.libraries.common.config.version import get_target_major_version from leapp.libraries.stdlib import api, CalledProcessError, config, run from leapp.models import RequiredTargetUserspacePackages # deprecated -from leapp.models import TMPTargetRepositoriesFacts # deprecated +from leapp.models import TMPTargetRepositoriesFacts # deprecated all the time from leapp.models import ( CustomTargetRepositoryFile, PkgManagerInfo, @@ -17,6 +17,7 @@ from leapp.models import ( RHSMInfo, RHUIInfo, StorageInfo, + TargetOSInstallationImage, TargetRepositories, TargetUserSpaceInfo, TargetUserSpacePreupgradeTasks, @@ -686,15 +687,18 @@ def perform(): storage_info=indata.storage_info, xfs_info=indata.xfs_info) as overlay: with overlay.nspawn() as context: - target_repoids = _gather_target_repositories(context, indata, prod_cert_path) - _create_target_userspace(context, indata.packages, indata.files, target_repoids) - # TODO: this is tmp solution as proper one needs significant refactoring - target_repo_facts = repofileutils.get_parsed_repofiles(context) - api.produce(TMPTargetRepositoriesFacts(repositories=target_repo_facts)) - # ## TODO ends here - api.produce(UsedTargetRepositories( - repos=[UsedTargetRepository(repoid=repo) for repo in target_repoids])) - api.produce(TargetUserSpaceInfo( - path=_get_target_userspace(), - scratch=constants.SCRATCH_DIR, - mounts=constants.MOUNTS_DIR)) + # Mount the ISO into the scratch container + target_iso = next(api.consume(TargetOSInstallationImage), None) + with mounting.mount_upgrade_iso_to_root_dir(overlay.target, target_iso): + target_repoids = _gather_target_repositories(context, indata, prod_cert_path) + _create_target_userspace(context, indata.packages, indata.files, target_repoids) + # TODO: this is tmp solution as proper one needs significant refactoring + target_repo_facts = repofileutils.get_parsed_repofiles(context) + api.produce(TMPTargetRepositoriesFacts(repositories=target_repo_facts)) + # ## TODO ends here + api.produce(UsedTargetRepositories( + repos=[UsedTargetRepository(repoid=repo) for repo in target_repoids])) + api.produce(TargetUserSpaceInfo( + path=_get_target_userspace(), + scratch=constants.SCRATCH_DIR, + mounts=constants.MOUNTS_DIR)) diff --git a/repos/system_upgrade/common/actors/targetuserspacecreator/tests/unit_test_targetuserspacecreator.py b/repos/system_upgrade/common/actors/targetuserspacecreator/tests/unit_test_targetuserspacecreator.py index 276175a1..5f544471 100644 --- a/repos/system_upgrade/common/actors/targetuserspacecreator/tests/unit_test_targetuserspacecreator.py +++ b/repos/system_upgrade/common/actors/targetuserspacecreator/tests/unit_test_targetuserspacecreator.py @@ -27,6 +27,7 @@ def adjust_cwd(): class MockedMountingBase(object): def __init__(self, **dummy_kwargs): self.called_copytree_from = [] + self.target = '' def copytree_from(self, src, dst): self.called_copytree_from.append((src, dst)) diff --git a/repos/system_upgrade/common/libraries/dnfplugin.py b/repos/system_upgrade/common/libraries/dnfplugin.py index 56b703d5..0a546637 100644 --- a/repos/system_upgrade/common/libraries/dnfplugin.py +++ b/repos/system_upgrade/common/libraries/dnfplugin.py @@ -342,22 +342,29 @@ def perform_transaction_install(target_userspace_info, storage_info, used_repos, @contextlib.contextmanager -def _prepare_perform(used_repos, target_userspace_info, xfs_info, storage_info): +def _prepare_perform(used_repos, target_userspace_info, xfs_info, storage_info, target_iso=None): with _prepare_transaction(used_repos=used_repos, target_userspace_info=target_userspace_info ) as (context, target_repoids, userspace_info): with overlaygen.create_source_overlay(mounts_dir=userspace_info.mounts, scratch_dir=userspace_info.scratch, xfs_info=xfs_info, storage_info=storage_info, mount_target=os.path.join(context.base_dir, 'installroot')) as overlay: - yield context, overlay, target_repoids + with mounting.mount_upgrade_iso_to_root_dir(target_userspace_info.path, target_iso): + yield context, overlay, target_repoids -def perform_transaction_check(target_userspace_info, used_repos, tasks, xfs_info, storage_info, plugin_info): +def perform_transaction_check(target_userspace_info, + used_repos, + tasks, + xfs_info, + storage_info, + plugin_info, + target_iso=None): """ Perform DNF transaction check using our plugin """ with _prepare_perform(used_repos=used_repos, target_userspace_info=target_userspace_info, xfs_info=xfs_info, - storage_info=storage_info) as (context, overlay, target_repoids): + storage_info=storage_info, target_iso=target_iso) as (context, overlay, target_repoids): apply_workarounds(overlay.nspawn()) dnfconfig.exclude_leapp_rpms(context) _transaction( @@ -365,12 +372,22 @@ def perform_transaction_check(target_userspace_info, used_repos, tasks, xfs_info ) -def perform_rpm_download(target_userspace_info, used_repos, tasks, xfs_info, storage_info, plugin_info, on_aws=False): +def perform_rpm_download(target_userspace_info, + used_repos, + tasks, + xfs_info, + storage_info, + plugin_info, + target_iso=None, + on_aws=False): """ Perform RPM download including the transaction test using dnf with our plugin """ - with _prepare_perform(used_repos=used_repos, target_userspace_info=target_userspace_info, xfs_info=xfs_info, - storage_info=storage_info) as (context, overlay, target_repoids): + with _prepare_perform(used_repos=used_repos, + target_userspace_info=target_userspace_info, + xfs_info=xfs_info, + storage_info=storage_info, + target_iso=target_iso) as (context, overlay, target_repoids): apply_workarounds(overlay.nspawn()) dnfconfig.exclude_leapp_rpms(context) _transaction( @@ -379,12 +396,22 @@ def perform_rpm_download(target_userspace_info, used_repos, tasks, xfs_info, sto ) -def perform_dry_run(target_userspace_info, used_repos, tasks, xfs_info, storage_info, plugin_info, on_aws=False): +def perform_dry_run(target_userspace_info, + used_repos, + tasks, + xfs_info, + storage_info, + plugin_info, + target_iso=None, + on_aws=False): """ Perform the dnf transaction test / dry-run using only cached data. """ - with _prepare_perform(used_repos=used_repos, target_userspace_info=target_userspace_info, xfs_info=xfs_info, - storage_info=storage_info) as (context, overlay, target_repoids): + with _prepare_perform(used_repos=used_repos, + target_userspace_info=target_userspace_info, + xfs_info=xfs_info, + storage_info=storage_info, + target_iso=target_iso) as (context, overlay, target_repoids): apply_workarounds(overlay.nspawn()) _transaction( context=context, stage='dry-run', target_repoids=target_repoids, plugin_info=plugin_info, tasks=tasks, diff --git a/repos/system_upgrade/common/libraries/mounting.py b/repos/system_upgrade/common/libraries/mounting.py index f272d8c7..fd079048 100644 --- a/repos/system_upgrade/common/libraries/mounting.py +++ b/repos/system_upgrade/common/libraries/mounting.py @@ -422,3 +422,23 @@ class OverlayMount(MountingBase): '-t', 'overlay', 'overlay2', '-o', 'lowerdir={},upperdir={},workdir={}'.format(self.source, self._upper_dir, self._work_dir) ] + + +def mount_upgrade_iso_to_root_dir(root_dir, target_iso): + """ + Context manager mounting the target RHEL ISO into the system root residing at `root_dir`. + + If the `target_iso` is None no action is performed. + + :param root_dir: Path to a directory containing a system root. + :type root_dir: str + :param target_iso: Description of the ISO to be mounted. + :type target_iso: Optional[TargetOSInstallationImage] + :rtype: Optional[LoopMount] + """ + if not target_iso: + return NullMount(root_dir) + + mountpoint = target_iso.mountpoint[1:] # Strip the leading / from the absolute mountpoint + mountpoint_in_root_dir = os.path.join(root_dir, mountpoint) + return LoopMount(source=target_iso.path, target=mountpoint_in_root_dir) diff --git a/repos/system_upgrade/common/models/upgradeiso.py b/repos/system_upgrade/common/models/upgradeiso.py new file mode 100644 index 00000000..da612bec --- /dev/null +++ b/repos/system_upgrade/common/models/upgradeiso.py @@ -0,0 +1,14 @@ +from leapp.models import CustomTargetRepository, fields, Model +from leapp.topics import SystemFactsTopic + + +class TargetOSInstallationImage(Model): + """ + An installation image of a target OS requested to be the source of target OS packages. + """ + topic = SystemFactsTopic + path = fields.String() + mountpoint = fields.String() + repositories = fields.List(fields.Model(CustomTargetRepository)) + rhel_version = fields.String(default='') + was_mounted_successfully = fields.Boolean(default=False) -- 2.38.1