leapp-repository/SOURCES/0036-check_rhui-read-RHUI-configuration.patch

From a03e8e5d10c1d6f3cdae216fafa0d7f0d0896494 Mon Sep 17 00:00:00 2001
From: Michal Hecko <mhecko@redhat.com>
Date: Sun, 10 Nov 2024 14:36:07 +0100
Subject: [PATCH 36/40] check_rhui: read RHUI configuration

Extend the check_rhui actor to read user-provided RHUI configuration.
If the provided configuration values say that the user wants to
overrwrite leapp's decisions, then the patch checks whether all values
are provided. If so, corresponding RHUIInfo message is produced. The
only implemented safe-guards are those that prevent the user from
accidentaly specifying a non-existing file to be copied into the
scrach container during us preparing to download target userspace
content. If the user provides only some of the configuration values
the upgrade is terminated early with an error, providing quick feedback
about misconfiguration. The patch has been designed to allow development
of upgrades on previously unknown clouds (clouds without an entry in
RHUI_SETUPS).

Jira ref: RHEL-56251
---
 .../common/actors/cloud/checkrhui/actor.py    |   4 +
 .../cloud/checkrhui/libraries/checkrhui.py    | 102 +++++++++-
 .../tests/component_test_checkrhui.py         | 178 ++++++++++++++++--
 3 files changed, 265 insertions(+), 19 deletions(-)

diff --git a/repos/system_upgrade/common/actors/cloud/checkrhui/actor.py b/repos/system_upgrade/common/actors/cloud/checkrhui/actor.py
index 593e73e5..933ffcb3 100644
--- a/repos/system_upgrade/common/actors/cloud/checkrhui/actor.py
+++ b/repos/system_upgrade/common/actors/cloud/checkrhui/actor.py
@@ -1,4 +1,5 @@
 from leapp.actors import Actor
+from leapp.configs.common.rhui import all_rhui_cfg
 from leapp.libraries.actor import checkrhui as checkrhui_lib
 from leapp.models import (
     CopyFile,
@@ -8,6 +9,7 @@ from leapp.models import (
     RequiredTargetUserspacePackages,
     RHUIInfo,
     RpmTransactionTasks,
+    TargetRepositories,
     TargetUserSpacePreupgradeTasks
 )
 from leapp.reporting import Report
@@ -21,6 +23,7 @@ class CheckRHUI(Actor):
     """
 
     name = 'checkrhui'
+    config_schemas = all_rhui_cfg
     consumes = (InstalledRPM,)
     produces = (
         KernelCmdlineArg,
@@ -28,6 +31,7 @@ class CheckRHUI(Actor):
         RequiredTargetUserspacePackages,
         Report, DNFPluginTask,
         RpmTransactionTasks,
+        TargetRepositories,
         TargetUserSpacePreupgradeTasks,
         CopyFile,
     )
diff --git a/repos/system_upgrade/common/actors/cloud/checkrhui/libraries/checkrhui.py b/repos/system_upgrade/common/actors/cloud/checkrhui/libraries/checkrhui.py
index 3b217917..64e36e08 100644
--- a/repos/system_upgrade/common/actors/cloud/checkrhui/libraries/checkrhui.py
+++ b/repos/system_upgrade/common/actors/cloud/checkrhui/libraries/checkrhui.py
@@ -2,17 +2,29 @@ import itertools
 import os
 from collections import namedtuple
 
+import leapp.configs.common.rhui as rhui_config_lib
 from leapp import reporting
+from leapp.configs.common.rhui import (  # Import all config fields so we are not using their name attributes directly
+    RhuiCloudProvider,
+    RhuiCloudVariant,
+    RhuiSourcePkgs,
+    RhuiTargetPkgs,
+    RhuiTargetRepositoriesToUse,
+    RhuiUpgradeFiles,
+    RhuiUseConfig
+)
 from leapp.exceptions import StopActorExecutionError
 from leapp.libraries.common import rhsm, rhui
 from leapp.libraries.common.config import version
 from leapp.libraries.stdlib import api
 from leapp.models import (
     CopyFile,
+    CustomTargetRepository,
     DNFPluginTask,
     InstalledRPM,
     RHUIInfo,
     RpmTransactionTasks,
+    TargetRepositories,
     TargetRHUIPostInstallTasks,
     TargetRHUIPreInstallTasks,
     TargetRHUISetupInfo,
@@ -291,11 +303,11 @@ def produce_rhui_info_to_setup_target(rhui_family, source_setup_desc, target_set
     api.produce(rhui_info)
 
 
-def produce_rpms_to_install_into_target(source_setup, target_setup):
-    to_install = sorted(target_setup.clients - source_setup.clients)
-    to_remove = sorted(source_setup.clients - target_setup.clients)
+def produce_rpms_to_install_into_target(source_clients, target_clients):
+    to_install = sorted(target_clients - source_clients)
+    to_remove = sorted(source_clients - target_clients)
 
-    api.produce(TargetUserSpacePreupgradeTasks(install_rpms=sorted(target_setup.clients)))
+    api.produce(TargetUserSpacePreupgradeTasks(install_rpms=sorted(target_clients)))
     if to_install or to_remove:
         api.produce(RpmTransactionTasks(to_install=to_install, to_remove=to_remove))
 
@@ -316,7 +328,85 @@ def inform_about_upgrade_with_rhui_without_no_rhsm():
     return False
 
 
+def emit_rhui_setup_tasks_based_on_config(rhui_config_dict):
+    config_upgrade_files = rhui_config_dict[RhuiUpgradeFiles.name]
+
+    nonexisting_files_to_copy = []
+    for source_path in config_upgrade_files:
+        if not os.path.exists(source_path):
+            nonexisting_files_to_copy.append(source_path)
+
+    if nonexisting_files_to_copy:
+        details_lines = ['The following files were not found:']
+        # Use .format and put backticks around paths so that weird unicode spaces will be easily seen
+        details_lines.extend('  - `{0}`'.format(path) for path in nonexisting_files_to_copy)
+        details = '\n'.join(details_lines)
+
+        reason = 'RHUI config lists nonexisting files in its `{0}` field.'.format(RhuiUpgradeFiles.name)
+        raise StopActorExecutionError(reason, details={'details': details})
+
+    files_to_copy_into_overlay = [CopyFile(src=key, dst=value) for key, value in config_upgrade_files.items()]
+    preinstall_tasks = TargetRHUIPreInstallTasks(files_to_copy_into_overlay=files_to_copy_into_overlay)
+
+    target_client_setup_info = TargetRHUISetupInfo(
+        preinstall_tasks=preinstall_tasks,
+        postinstall_tasks=TargetRHUIPostInstallTasks(),
+        bootstrap_target_client=False,  # We don't need to install the client into overlay - user provided all files
+    )
+
+    rhui_info = RHUIInfo(
+        provider=rhui_config_dict[RhuiCloudProvider.name],
+        variant=rhui_config_dict[RhuiCloudVariant.name],
+        src_client_pkg_names=rhui_config_dict[RhuiSourcePkgs.name],
+        target_client_pkg_names=rhui_config_dict[RhuiTargetPkgs.name],
+        target_client_setup_info=target_client_setup_info
+    )
+    api.produce(rhui_info)
+
+
+def request_configured_repos_to_be_enabled(rhui_config):
+    config_repos_to_enable = rhui_config[RhuiTargetRepositoriesToUse.name]
+    custom_repos = [CustomTargetRepository(repoid=repoid) for repoid in config_repos_to_enable]
+    if custom_repos:
+        target_repos = TargetRepositories(custom_repos=custom_repos, rhel_repos=[])
+        api.produce(target_repos)
+
+
+def stop_with_err_if_config_missing_fields(config):
+    required_fields = [
+        RhuiTargetRepositoriesToUse,
+        RhuiCloudProvider,
+        # RhuiCloudVariant, <- this is not required
+        RhuiSourcePkgs,
+        RhuiTargetPkgs,
+        RhuiUpgradeFiles,
+    ]
+
+    missing_fields = tuple(field for field in required_fields if not config[field.name])
+    if missing_fields:
+        field_names = (field.name for field in missing_fields)
+        missing_fields_str = ', '.join(field_names)
+        details = 'The following required RHUI config fields are missing or they are set to an empty value: {}'
+        details = details.format(missing_fields_str)
+        raise StopActorExecutionError('Provided RHUI config is missing values for required fields.',
+                                      details={'details': details})
+
+
 def process():
+    rhui_config = api.current_actor().config[rhui_config_lib.RHUI_CONFIG_SECTION]
+
+    if rhui_config[RhuiUseConfig.name]:
+        api.current_logger().info('Skipping RHUI upgrade auto-configuration - using provided config instead.')
+        stop_with_err_if_config_missing_fields(rhui_config)
+        emit_rhui_setup_tasks_based_on_config(rhui_config)
+
+        src_clients = set(rhui_config[RhuiSourcePkgs.name])
+        target_clients = set(rhui_config[RhuiTargetPkgs.name])
+        produce_rpms_to_install_into_target(src_clients, target_clients)
+
+        request_configured_repos_to_be_enabled(rhui_config)
+        return
+
     installed_rpm = itertools.chain(*[installed_rpm_msg.items for installed_rpm_msg in api.consume(InstalledRPM)])
     installed_pkgs = {rpm.name for rpm in installed_rpm}
 
@@ -342,7 +432,9 @@ def process():
     # Instruction on how to access the target content
     produce_rhui_info_to_setup_target(src_rhui_setup.family, src_rhui_setup.description, target_setup_desc)
 
-    produce_rpms_to_install_into_target(src_rhui_setup.description, target_setup_desc)
+    source_clients = src_rhui_setup.description.clients
+    target_clients = target_setup_desc.clients
+    produce_rpms_to_install_into_target(source_clients, target_clients)
 
     if src_rhui_setup.family.provider == rhui.RHUIProvider.AWS:
         # We have to disable Amazon-id plugin in the initramdisk phase as there is no network
diff --git a/repos/system_upgrade/common/actors/cloud/checkrhui/tests/component_test_checkrhui.py b/repos/system_upgrade/common/actors/cloud/checkrhui/tests/component_test_checkrhui.py
index 27e70eea..3ac9c1b8 100644
--- a/repos/system_upgrade/common/actors/cloud/checkrhui/tests/component_test_checkrhui.py
+++ b/repos/system_upgrade/common/actors/cloud/checkrhui/tests/component_test_checkrhui.py
@@ -1,30 +1,43 @@
-from collections import namedtuple
+import itertools
+import os
+from collections import defaultdict
 from enum import Enum
 
 import pytest
 
 from leapp import reporting
+from leapp.configs.common.rhui import (
+    all_rhui_cfg,
+    RhuiCloudProvider,
+    RhuiCloudVariant,
+    RhuiSourcePkgs,
+    RhuiTargetPkgs,
+    RhuiTargetRepositoriesToUse,
+    RhuiUpgradeFiles,
+    RhuiUseConfig
+)
 from leapp.exceptions import StopActorExecutionError
 from leapp.libraries.actor import checkrhui as checkrhui_lib
 from leapp.libraries.common import rhsm, rhui
-from leapp.libraries.common.config import mock_configs, version
 from leapp.libraries.common.rhui import mk_rhui_setup, RHUIFamily
-from leapp.libraries.common.testutils import create_report_mocked, CurrentActorMocked, produce_mocked
+from leapp.libraries.common.testutils import (
+    _make_default_config,
+    create_report_mocked,
+    CurrentActorMocked,
+    produce_mocked
+)
 from leapp.libraries.stdlib import api
 from leapp.models import (
-    CopyFile,
     InstalledRPM,
-    RequiredTargetUserspacePackages,
     RHUIInfo,
     RPM,
     RpmTransactionTasks,
+    TargetRepositories,
     TargetRHUIPostInstallTasks,
     TargetRHUIPreInstallTasks,
     TargetRHUISetupInfo,
     TargetUserSpacePreupgradeTasks
 )
-from leapp.reporting import Report
-from leapp.snactor.fixture import current_actor_context
 
 RH_PACKAGER = 'Red Hat, Inc. <http://bugzilla.redhat.com/bugzilla>'
 
@@ -95,7 +108,8 @@ def mk_cloud_map(variants):
     ]
 )
 def test_determine_rhui_src_variant(monkeypatch, extra_pkgs, rhui_setups, expected_result):
-    monkeypatch.setattr(api, 'current_actor', CurrentActorMocked(src_ver='7.9'))
+    actor = CurrentActorMocked(src_ver='7.9', config=_make_default_config(all_rhui_cfg))
+    monkeypatch.setattr(api, 'current_actor', actor)
     installed_pkgs = {'zip', 'zsh', 'bash', 'grubby'}.union(set(extra_pkgs))
 
     if expected_result and not isinstance(expected_result, RHUIFamily):  # An exception
@@ -167,7 +181,8 @@ def test_google_specific_customization(provider, should_mutate):
 )
 def test_aws_specific_customization(monkeypatch, rhui_family, target_major, should_mutate):
     dst_ver = '{major}.0'.format(major=target_major)
-    monkeypatch.setattr(api, 'current_actor', CurrentActorMocked(dst_ver=dst_ver))
+    actor = CurrentActorMocked(dst_ver=dst_ver, config=_make_default_config(all_rhui_cfg))
+    monkeypatch.setattr(api, 'current_actor', actor)
 
     setup_info = mk_setup_info()
     checkrhui_lib.customize_rhui_setup_for_aws(rhui_family, setup_info)
@@ -215,12 +230,12 @@ def produce_rhui_info_to_setup_target(monkeypatch):
 
 
 def test_produce_rpms_to_install_into_target(monkeypatch):
-    source_rhui_setup = mk_rhui_setup(clients={'src_pkg'}, leapp_pkg='leapp_pkg')
-    target_rhui_setup = mk_rhui_setup(clients={'target_pkg'}, leapp_pkg='leapp_pkg')
+    source_clients = {'src_pkg'}
+    target_clients = {'target_pkg'}
 
     monkeypatch.setattr(api, 'produce', produce_mocked())
 
-    checkrhui_lib.produce_rpms_to_install_into_target(source_rhui_setup, target_rhui_setup)
+    checkrhui_lib.produce_rpms_to_install_into_target(source_clients, target_clients)
 
     assert len(api.produce.model_instances) == 2
     userspace_tasks, target_rpm_tasks = api.produce.model_instances[0], api.produce.model_instances[1]
@@ -276,7 +291,8 @@ def test_process(monkeypatch, extra_installed_pkgs, skip_rhsm, expected_action):
     installed_rpms = InstalledRPM(items=installed_pkgs)
 
     monkeypatch.setattr(api, 'produce', produce_mocked())
-    monkeypatch.setattr(api, 'current_actor', CurrentActorMocked(src_ver='7.9', msgs=[installed_rpms]))
+    actor = CurrentActorMocked(src_ver='7.9', msgs=[installed_rpms], config=_make_default_config(all_rhui_cfg))
+    monkeypatch.setattr(api, 'current_actor', actor)
     monkeypatch.setattr(reporting, 'create_report', create_report_mocked())
     monkeypatch.setattr(rhsm, 'skip_rhsm', lambda: skip_rhsm)
     monkeypatch.setattr(rhui, 'RHUI_SETUPS', known_setups)
@@ -315,7 +331,8 @@ def test_unknown_target_rhui_setup(monkeypatch, is_target_setup_known):
     installed_rpms = InstalledRPM(items=installed_pkgs)
 
     monkeypatch.setattr(api, 'produce', produce_mocked())
-    monkeypatch.setattr(api, 'current_actor', CurrentActorMocked(src_ver='7.9', msgs=[installed_rpms]))
+    actor = CurrentActorMocked(src_ver='7.9', msgs=[installed_rpms], config=_make_default_config(all_rhui_cfg))
+    monkeypatch.setattr(api, 'current_actor', actor)
     monkeypatch.setattr(reporting, 'create_report', create_report_mocked())
     monkeypatch.setattr(rhsm, 'skip_rhsm', lambda: True)
     monkeypatch.setattr(rhui, 'RHUI_SETUPS', known_setups)
@@ -374,3 +391,136 @@ def test_select_chronologically_closest(monkeypatch, setups, desired_minor, expe
     setup = setups[0]
 
     assert setup == expected_setup
+
+
+def test_config_overwrites_everything(monkeypatch):
+    rhui_config = {
+        RhuiUseConfig.name: True,
+        RhuiSourcePkgs.name: ['client_source'],
+        RhuiTargetPkgs.name: ['client_target'],
+        RhuiCloudProvider.name: 'aws',
+        RhuiUpgradeFiles.name: {
+            '/root/file.repo': '/etc/yum.repos.d/'
+        },
+        RhuiTargetRepositoriesToUse.name: [
+            'repoid_to_use'
+        ]
+    }
+    all_config = {'rhui': rhui_config}
+
+    actor = CurrentActorMocked(config=all_config)
+    monkeypatch.setattr(api, 'current_actor', actor)
+
+    function_calls = defaultdict(int)
+
+    def mk_function_probe(fn_name):
+        def probe(*args, **kwargs):
+            function_calls[fn_name] += 1
+        return probe
+
+    monkeypatch.setattr(checkrhui_lib,
+                        'emit_rhui_setup_tasks_based_on_config',
+                        mk_function_probe('emit_rhui_setup_tasks_based_on_config'))
+    monkeypatch.setattr(checkrhui_lib,
+                        'stop_with_err_if_config_missing_fields',
+                        mk_function_probe('stop_with_err_if_config_missing_fields'))
+    monkeypatch.setattr(checkrhui_lib,
+                        'produce_rpms_to_install_into_target',
+                        mk_function_probe('produce_rpms_to_install_into_target'))
+    monkeypatch.setattr(checkrhui_lib,
+                        'request_configured_repos_to_be_enabled',
+                        mk_function_probe('request_configured_repos_to_be_enabled'))
+
+    checkrhui_lib.process()
+
+    expected_function_calls = {
+        'emit_rhui_setup_tasks_based_on_config': 1,
+        'stop_with_err_if_config_missing_fields': 1,
+        'produce_rpms_to_install_into_target': 1,
+        'request_configured_repos_to_be_enabled': 1,
+    }
+
+    assert function_calls == expected_function_calls
+
+
+def test_request_configured_repos_to_be_enabled(monkeypatch):
+    monkeypatch.setattr(api, 'produce', produce_mocked())
+
+    rhui_config = {
+        RhuiUseConfig.name: True,
+        RhuiSourcePkgs.name: ['client_source'],
+        RhuiTargetPkgs.name: ['client_target'],
+        RhuiCloudProvider.name: 'aws',
+        RhuiUpgradeFiles.name: {
+            '/root/file.repo': '/etc/yum.repos.d/'
+        },
+        RhuiTargetRepositoriesToUse.name: [
+            'repoid1',
+            'repoid2',
+            'repoid3',
+        ]
+    }
+
+    checkrhui_lib.request_configured_repos_to_be_enabled(rhui_config)
+
+    assert api.produce.called
+    assert len(api.produce.model_instances) == 1
+
+    target_repos = api.produce.model_instances[0]
+    assert isinstance(target_repos, TargetRepositories)
+    assert not target_repos.rhel_repos
+
+    custom_repoids = sorted(custom_repo_model.repoid for custom_repo_model in target_repos.custom_repos)
+    assert custom_repoids == ['repoid1', 'repoid2', 'repoid3']
+
+
+@pytest.mark.parametrize(
+    ('upgrade_files', 'existing_files'),
+    (
+        (['/root/a', '/root/b'], ['/root/a', '/root/b']),
+        (['/root/a', '/root/b'], ['/root/b']),
+        (['/root/a', '/root/b'], []),
+    )
+)
+def test_missing_files_in_config(monkeypatch, upgrade_files, existing_files):
+    upgrade_files_map = dict((source_path, '/tmp/dummy') for source_path in upgrade_files)
+
+    rhui_config = {
+        RhuiUseConfig.name: True,
+        RhuiSourcePkgs.name: ['client_source'],
+        RhuiTargetPkgs.name: ['client_target'],
+        RhuiCloudProvider.name: 'aws',
+        RhuiCloudVariant.name: 'ordinary',
+        RhuiUpgradeFiles.name: upgrade_files_map,
+        RhuiTargetRepositoriesToUse.name: [
+            'repoid_to_use'
+        ]
+    }
+
+    monkeypatch.setattr(os.path, 'exists', lambda path: path in existing_files)
+    monkeypatch.setattr(api, 'produce', produce_mocked())
+
+    should_error = (len(upgrade_files) != len(existing_files))
+    if should_error:
+        with pytest.raises(StopActorExecutionError):
+            checkrhui_lib.emit_rhui_setup_tasks_based_on_config(rhui_config)
+    else:
+        checkrhui_lib.emit_rhui_setup_tasks_based_on_config(rhui_config)
+        assert api.produce.called
+        assert len(api.produce.model_instances) == 1
+
+        rhui_info = api.produce.model_instances[0]
+        assert isinstance(rhui_info, RHUIInfo)
+        assert rhui_info.provider == 'aws'
+        assert rhui_info.variant == 'ordinary'
+        assert rhui_info.src_client_pkg_names == ['client_source']
+        assert rhui_info.target_client_pkg_names == ['client_target']
+
+        setup_info = rhui_info.target_client_setup_info
+        assert not setup_info.bootstrap_target_client
+
+        _copies_to_perform = setup_info.preinstall_tasks.files_to_copy_into_overlay
+        copies_to_perform = sorted((copy.src, copy.dst) for copy in _copies_to_perform)
+        expected_copies = sorted(zip(upgrade_files, itertools.repeat('/tmp/dummy')))
+
+        assert copies_to_perform == expected_copies
-- 
2.47.0