From a03e8e5d10c1d6f3cdae216fafa0d7f0d0896494 Mon Sep 17 00:00:00 2001 From: Michal Hecko Date: Sun, 10 Nov 2024 14:36:07 +0100 Subject: [PATCH 36/40] check_rhui: read RHUI configuration Extend the check_rhui actor to read user-provided RHUI configuration. If the provided configuration values say that the user wants to overrwrite leapp's decisions, then the patch checks whether all values are provided. If so, corresponding RHUIInfo message is produced. The only implemented safe-guards are those that prevent the user from accidentaly specifying a non-existing file to be copied into the scrach container during us preparing to download target userspace content. If the user provides only some of the configuration values the upgrade is terminated early with an error, providing quick feedback about misconfiguration. The patch has been designed to allow development of upgrades on previously unknown clouds (clouds without an entry in RHUI_SETUPS). Jira ref: RHEL-56251 --- .../common/actors/cloud/checkrhui/actor.py | 4 + .../cloud/checkrhui/libraries/checkrhui.py | 102 +++++++++- .../tests/component_test_checkrhui.py | 178 ++++++++++++++++-- 3 files changed, 265 insertions(+), 19 deletions(-) diff --git a/repos/system_upgrade/common/actors/cloud/checkrhui/actor.py b/repos/system_upgrade/common/actors/cloud/checkrhui/actor.py index 593e73e5..933ffcb3 100644 --- a/repos/system_upgrade/common/actors/cloud/checkrhui/actor.py +++ b/repos/system_upgrade/common/actors/cloud/checkrhui/actor.py @@ -1,4 +1,5 @@ from leapp.actors import Actor +from leapp.configs.common.rhui import all_rhui_cfg from leapp.libraries.actor import checkrhui as checkrhui_lib from leapp.models import ( CopyFile, @@ -8,6 +9,7 @@ from leapp.models import ( RequiredTargetUserspacePackages, RHUIInfo, RpmTransactionTasks, + TargetRepositories, TargetUserSpacePreupgradeTasks ) from leapp.reporting import Report @@ -21,6 +23,7 @@ class CheckRHUI(Actor): """ name = 'checkrhui' + config_schemas = all_rhui_cfg consumes = (InstalledRPM,) produces = ( KernelCmdlineArg, @@ -28,6 +31,7 @@ class CheckRHUI(Actor): RequiredTargetUserspacePackages, Report, DNFPluginTask, RpmTransactionTasks, + TargetRepositories, TargetUserSpacePreupgradeTasks, CopyFile, ) diff --git a/repos/system_upgrade/common/actors/cloud/checkrhui/libraries/checkrhui.py b/repos/system_upgrade/common/actors/cloud/checkrhui/libraries/checkrhui.py index 3b217917..64e36e08 100644 --- a/repos/system_upgrade/common/actors/cloud/checkrhui/libraries/checkrhui.py +++ b/repos/system_upgrade/common/actors/cloud/checkrhui/libraries/checkrhui.py @@ -2,17 +2,29 @@ import itertools import os from collections import namedtuple +import leapp.configs.common.rhui as rhui_config_lib from leapp import reporting +from leapp.configs.common.rhui import ( # Import all config fields so we are not using their name attributes directly + RhuiCloudProvider, + RhuiCloudVariant, + RhuiSourcePkgs, + RhuiTargetPkgs, + RhuiTargetRepositoriesToUse, + RhuiUpgradeFiles, + RhuiUseConfig +) from leapp.exceptions import StopActorExecutionError from leapp.libraries.common import rhsm, rhui from leapp.libraries.common.config import version from leapp.libraries.stdlib import api from leapp.models import ( CopyFile, + CustomTargetRepository, DNFPluginTask, InstalledRPM, RHUIInfo, RpmTransactionTasks, + TargetRepositories, TargetRHUIPostInstallTasks, TargetRHUIPreInstallTasks, TargetRHUISetupInfo, @@ -291,11 +303,11 @@ def produce_rhui_info_to_setup_target(rhui_family, source_setup_desc, target_set api.produce(rhui_info) -def produce_rpms_to_install_into_target(source_setup, target_setup): - to_install = sorted(target_setup.clients - source_setup.clients) - to_remove = sorted(source_setup.clients - target_setup.clients) +def produce_rpms_to_install_into_target(source_clients, target_clients): + to_install = sorted(target_clients - source_clients) + to_remove = sorted(source_clients - target_clients) - api.produce(TargetUserSpacePreupgradeTasks(install_rpms=sorted(target_setup.clients))) + api.produce(TargetUserSpacePreupgradeTasks(install_rpms=sorted(target_clients))) if to_install or to_remove: api.produce(RpmTransactionTasks(to_install=to_install, to_remove=to_remove)) @@ -316,7 +328,85 @@ def inform_about_upgrade_with_rhui_without_no_rhsm(): return False +def emit_rhui_setup_tasks_based_on_config(rhui_config_dict): + config_upgrade_files = rhui_config_dict[RhuiUpgradeFiles.name] + + nonexisting_files_to_copy = [] + for source_path in config_upgrade_files: + if not os.path.exists(source_path): + nonexisting_files_to_copy.append(source_path) + + if nonexisting_files_to_copy: + details_lines = ['The following files were not found:'] + # Use .format and put backticks around paths so that weird unicode spaces will be easily seen + details_lines.extend(' - `{0}`'.format(path) for path in nonexisting_files_to_copy) + details = '\n'.join(details_lines) + + reason = 'RHUI config lists nonexisting files in its `{0}` field.'.format(RhuiUpgradeFiles.name) + raise StopActorExecutionError(reason, details={'details': details}) + + files_to_copy_into_overlay = [CopyFile(src=key, dst=value) for key, value in config_upgrade_files.items()] + preinstall_tasks = TargetRHUIPreInstallTasks(files_to_copy_into_overlay=files_to_copy_into_overlay) + + target_client_setup_info = TargetRHUISetupInfo( + preinstall_tasks=preinstall_tasks, + postinstall_tasks=TargetRHUIPostInstallTasks(), + bootstrap_target_client=False, # We don't need to install the client into overlay - user provided all files + ) + + rhui_info = RHUIInfo( + provider=rhui_config_dict[RhuiCloudProvider.name], + variant=rhui_config_dict[RhuiCloudVariant.name], + src_client_pkg_names=rhui_config_dict[RhuiSourcePkgs.name], + target_client_pkg_names=rhui_config_dict[RhuiTargetPkgs.name], + target_client_setup_info=target_client_setup_info + ) + api.produce(rhui_info) + + +def request_configured_repos_to_be_enabled(rhui_config): + config_repos_to_enable = rhui_config[RhuiTargetRepositoriesToUse.name] + custom_repos = [CustomTargetRepository(repoid=repoid) for repoid in config_repos_to_enable] + if custom_repos: + target_repos = TargetRepositories(custom_repos=custom_repos, rhel_repos=[]) + api.produce(target_repos) + + +def stop_with_err_if_config_missing_fields(config): + required_fields = [ + RhuiTargetRepositoriesToUse, + RhuiCloudProvider, + # RhuiCloudVariant, <- this is not required + RhuiSourcePkgs, + RhuiTargetPkgs, + RhuiUpgradeFiles, + ] + + missing_fields = tuple(field for field in required_fields if not config[field.name]) + if missing_fields: + field_names = (field.name for field in missing_fields) + missing_fields_str = ', '.join(field_names) + details = 'The following required RHUI config fields are missing or they are set to an empty value: {}' + details = details.format(missing_fields_str) + raise StopActorExecutionError('Provided RHUI config is missing values for required fields.', + details={'details': details}) + + def process(): + rhui_config = api.current_actor().config[rhui_config_lib.RHUI_CONFIG_SECTION] + + if rhui_config[RhuiUseConfig.name]: + api.current_logger().info('Skipping RHUI upgrade auto-configuration - using provided config instead.') + stop_with_err_if_config_missing_fields(rhui_config) + emit_rhui_setup_tasks_based_on_config(rhui_config) + + src_clients = set(rhui_config[RhuiSourcePkgs.name]) + target_clients = set(rhui_config[RhuiTargetPkgs.name]) + produce_rpms_to_install_into_target(src_clients, target_clients) + + request_configured_repos_to_be_enabled(rhui_config) + return + installed_rpm = itertools.chain(*[installed_rpm_msg.items for installed_rpm_msg in api.consume(InstalledRPM)]) installed_pkgs = {rpm.name for rpm in installed_rpm} @@ -342,7 +432,9 @@ def process(): # Instruction on how to access the target content produce_rhui_info_to_setup_target(src_rhui_setup.family, src_rhui_setup.description, target_setup_desc) - produce_rpms_to_install_into_target(src_rhui_setup.description, target_setup_desc) + source_clients = src_rhui_setup.description.clients + target_clients = target_setup_desc.clients + produce_rpms_to_install_into_target(source_clients, target_clients) if src_rhui_setup.family.provider == rhui.RHUIProvider.AWS: # We have to disable Amazon-id plugin in the initramdisk phase as there is no network diff --git a/repos/system_upgrade/common/actors/cloud/checkrhui/tests/component_test_checkrhui.py b/repos/system_upgrade/common/actors/cloud/checkrhui/tests/component_test_checkrhui.py index 27e70eea..3ac9c1b8 100644 --- a/repos/system_upgrade/common/actors/cloud/checkrhui/tests/component_test_checkrhui.py +++ b/repos/system_upgrade/common/actors/cloud/checkrhui/tests/component_test_checkrhui.py @@ -1,30 +1,43 @@ -from collections import namedtuple +import itertools +import os +from collections import defaultdict from enum import Enum import pytest from leapp import reporting +from leapp.configs.common.rhui import ( + all_rhui_cfg, + RhuiCloudProvider, + RhuiCloudVariant, + RhuiSourcePkgs, + RhuiTargetPkgs, + RhuiTargetRepositoriesToUse, + RhuiUpgradeFiles, + RhuiUseConfig +) from leapp.exceptions import StopActorExecutionError from leapp.libraries.actor import checkrhui as checkrhui_lib from leapp.libraries.common import rhsm, rhui -from leapp.libraries.common.config import mock_configs, version from leapp.libraries.common.rhui import mk_rhui_setup, RHUIFamily -from leapp.libraries.common.testutils import create_report_mocked, CurrentActorMocked, produce_mocked +from leapp.libraries.common.testutils import ( + _make_default_config, + create_report_mocked, + CurrentActorMocked, + produce_mocked +) from leapp.libraries.stdlib import api from leapp.models import ( - CopyFile, InstalledRPM, - RequiredTargetUserspacePackages, RHUIInfo, RPM, RpmTransactionTasks, + TargetRepositories, TargetRHUIPostInstallTasks, TargetRHUIPreInstallTasks, TargetRHUISetupInfo, TargetUserSpacePreupgradeTasks ) -from leapp.reporting import Report -from leapp.snactor.fixture import current_actor_context RH_PACKAGER = 'Red Hat, Inc. ' @@ -95,7 +108,8 @@ def mk_cloud_map(variants): ] ) def test_determine_rhui_src_variant(monkeypatch, extra_pkgs, rhui_setups, expected_result): - monkeypatch.setattr(api, 'current_actor', CurrentActorMocked(src_ver='7.9')) + actor = CurrentActorMocked(src_ver='7.9', config=_make_default_config(all_rhui_cfg)) + monkeypatch.setattr(api, 'current_actor', actor) installed_pkgs = {'zip', 'zsh', 'bash', 'grubby'}.union(set(extra_pkgs)) if expected_result and not isinstance(expected_result, RHUIFamily): # An exception @@ -167,7 +181,8 @@ def test_google_specific_customization(provider, should_mutate): ) def test_aws_specific_customization(monkeypatch, rhui_family, target_major, should_mutate): dst_ver = '{major}.0'.format(major=target_major) - monkeypatch.setattr(api, 'current_actor', CurrentActorMocked(dst_ver=dst_ver)) + actor = CurrentActorMocked(dst_ver=dst_ver, config=_make_default_config(all_rhui_cfg)) + monkeypatch.setattr(api, 'current_actor', actor) setup_info = mk_setup_info() checkrhui_lib.customize_rhui_setup_for_aws(rhui_family, setup_info) @@ -215,12 +230,12 @@ def produce_rhui_info_to_setup_target(monkeypatch): def test_produce_rpms_to_install_into_target(monkeypatch): - source_rhui_setup = mk_rhui_setup(clients={'src_pkg'}, leapp_pkg='leapp_pkg') - target_rhui_setup = mk_rhui_setup(clients={'target_pkg'}, leapp_pkg='leapp_pkg') + source_clients = {'src_pkg'} + target_clients = {'target_pkg'} monkeypatch.setattr(api, 'produce', produce_mocked()) - checkrhui_lib.produce_rpms_to_install_into_target(source_rhui_setup, target_rhui_setup) + checkrhui_lib.produce_rpms_to_install_into_target(source_clients, target_clients) assert len(api.produce.model_instances) == 2 userspace_tasks, target_rpm_tasks = api.produce.model_instances[0], api.produce.model_instances[1] @@ -276,7 +291,8 @@ def test_process(monkeypatch, extra_installed_pkgs, skip_rhsm, expected_action): installed_rpms = InstalledRPM(items=installed_pkgs) monkeypatch.setattr(api, 'produce', produce_mocked()) - monkeypatch.setattr(api, 'current_actor', CurrentActorMocked(src_ver='7.9', msgs=[installed_rpms])) + actor = CurrentActorMocked(src_ver='7.9', msgs=[installed_rpms], config=_make_default_config(all_rhui_cfg)) + monkeypatch.setattr(api, 'current_actor', actor) monkeypatch.setattr(reporting, 'create_report', create_report_mocked()) monkeypatch.setattr(rhsm, 'skip_rhsm', lambda: skip_rhsm) monkeypatch.setattr(rhui, 'RHUI_SETUPS', known_setups) @@ -315,7 +331,8 @@ def test_unknown_target_rhui_setup(monkeypatch, is_target_setup_known): installed_rpms = InstalledRPM(items=installed_pkgs) monkeypatch.setattr(api, 'produce', produce_mocked()) - monkeypatch.setattr(api, 'current_actor', CurrentActorMocked(src_ver='7.9', msgs=[installed_rpms])) + actor = CurrentActorMocked(src_ver='7.9', msgs=[installed_rpms], config=_make_default_config(all_rhui_cfg)) + monkeypatch.setattr(api, 'current_actor', actor) monkeypatch.setattr(reporting, 'create_report', create_report_mocked()) monkeypatch.setattr(rhsm, 'skip_rhsm', lambda: True) monkeypatch.setattr(rhui, 'RHUI_SETUPS', known_setups) @@ -374,3 +391,136 @@ def test_select_chronologically_closest(monkeypatch, setups, desired_minor, expe setup = setups[0] assert setup == expected_setup + + +def test_config_overwrites_everything(monkeypatch): + rhui_config = { + RhuiUseConfig.name: True, + RhuiSourcePkgs.name: ['client_source'], + RhuiTargetPkgs.name: ['client_target'], + RhuiCloudProvider.name: 'aws', + RhuiUpgradeFiles.name: { + '/root/file.repo': '/etc/yum.repos.d/' + }, + RhuiTargetRepositoriesToUse.name: [ + 'repoid_to_use' + ] + } + all_config = {'rhui': rhui_config} + + actor = CurrentActorMocked(config=all_config) + monkeypatch.setattr(api, 'current_actor', actor) + + function_calls = defaultdict(int) + + def mk_function_probe(fn_name): + def probe(*args, **kwargs): + function_calls[fn_name] += 1 + return probe + + monkeypatch.setattr(checkrhui_lib, + 'emit_rhui_setup_tasks_based_on_config', + mk_function_probe('emit_rhui_setup_tasks_based_on_config')) + monkeypatch.setattr(checkrhui_lib, + 'stop_with_err_if_config_missing_fields', + mk_function_probe('stop_with_err_if_config_missing_fields')) + monkeypatch.setattr(checkrhui_lib, + 'produce_rpms_to_install_into_target', + mk_function_probe('produce_rpms_to_install_into_target')) + monkeypatch.setattr(checkrhui_lib, + 'request_configured_repos_to_be_enabled', + mk_function_probe('request_configured_repos_to_be_enabled')) + + checkrhui_lib.process() + + expected_function_calls = { + 'emit_rhui_setup_tasks_based_on_config': 1, + 'stop_with_err_if_config_missing_fields': 1, + 'produce_rpms_to_install_into_target': 1, + 'request_configured_repos_to_be_enabled': 1, + } + + assert function_calls == expected_function_calls + + +def test_request_configured_repos_to_be_enabled(monkeypatch): + monkeypatch.setattr(api, 'produce', produce_mocked()) + + rhui_config = { + RhuiUseConfig.name: True, + RhuiSourcePkgs.name: ['client_source'], + RhuiTargetPkgs.name: ['client_target'], + RhuiCloudProvider.name: 'aws', + RhuiUpgradeFiles.name: { + '/root/file.repo': '/etc/yum.repos.d/' + }, + RhuiTargetRepositoriesToUse.name: [ + 'repoid1', + 'repoid2', + 'repoid3', + ] + } + + checkrhui_lib.request_configured_repos_to_be_enabled(rhui_config) + + assert api.produce.called + assert len(api.produce.model_instances) == 1 + + target_repos = api.produce.model_instances[0] + assert isinstance(target_repos, TargetRepositories) + assert not target_repos.rhel_repos + + custom_repoids = sorted(custom_repo_model.repoid for custom_repo_model in target_repos.custom_repos) + assert custom_repoids == ['repoid1', 'repoid2', 'repoid3'] + + +@pytest.mark.parametrize( + ('upgrade_files', 'existing_files'), + ( + (['/root/a', '/root/b'], ['/root/a', '/root/b']), + (['/root/a', '/root/b'], ['/root/b']), + (['/root/a', '/root/b'], []), + ) +) +def test_missing_files_in_config(monkeypatch, upgrade_files, existing_files): + upgrade_files_map = dict((source_path, '/tmp/dummy') for source_path in upgrade_files) + + rhui_config = { + RhuiUseConfig.name: True, + RhuiSourcePkgs.name: ['client_source'], + RhuiTargetPkgs.name: ['client_target'], + RhuiCloudProvider.name: 'aws', + RhuiCloudVariant.name: 'ordinary', + RhuiUpgradeFiles.name: upgrade_files_map, + RhuiTargetRepositoriesToUse.name: [ + 'repoid_to_use' + ] + } + + monkeypatch.setattr(os.path, 'exists', lambda path: path in existing_files) + monkeypatch.setattr(api, 'produce', produce_mocked()) + + should_error = (len(upgrade_files) != len(existing_files)) + if should_error: + with pytest.raises(StopActorExecutionError): + checkrhui_lib.emit_rhui_setup_tasks_based_on_config(rhui_config) + else: + checkrhui_lib.emit_rhui_setup_tasks_based_on_config(rhui_config) + assert api.produce.called + assert len(api.produce.model_instances) == 1 + + rhui_info = api.produce.model_instances[0] + assert isinstance(rhui_info, RHUIInfo) + assert rhui_info.provider == 'aws' + assert rhui_info.variant == 'ordinary' + assert rhui_info.src_client_pkg_names == ['client_source'] + assert rhui_info.target_client_pkg_names == ['client_target'] + + setup_info = rhui_info.target_client_setup_info + assert not setup_info.bootstrap_target_client + + _copies_to_perform = setup_info.preinstall_tasks.files_to_copy_into_overlay + copies_to_perform = sorted((copy.src, copy.dst) for copy in _copies_to_perform) + expected_copies = sorted(zip(upgrade_files, itertools.repeat('/tmp/dummy'))) + + assert copies_to_perform == expected_copies -- 2.47.0