From 32d9c40ffc7ea8d08e2b85881579ede1fdaedb32 Mon Sep 17 00:00:00 2001 From: Matej Matuska Date: Thu, 7 Aug 2025 13:40:33 +0200 Subject: [PATCH 17/55] userspacegen: Add repo gathering for non-RHEL distros The _get_rh_available_repoids() function is replaced by the new get_distro_repoids() function from the distro library. This function works with all the "supported" distros. The idea is the same as for RHEL, scan well-known distro-provided repofiles for repositories. For RHEL, at least for now, the existing rhsm.get_rhsm_available_repoids() function is still used. These changes together enable the use of repomapping on distros other than RHEL, as before this change the --enablerepo option had to be used to specify target repos and they were treated as custom repos. Also, the unused _get_rhui_available_repoids() function is removed. Jira: RHEL-107212 Move get_distro_repoids() to the distro library --- .../libraries/userspacegen.py | 228 ++++++++++-------- .../tests/unit_test_targetuserspacecreator.py | 56 ++++- .../system_upgrade/common/libraries/distro.py | 192 +++++++++++++++ .../common/libraries/tests/test_distro.py | 154 ++++++++++++ 4 files changed, 524 insertions(+), 106 deletions(-) create mode 100644 repos/system_upgrade/common/libraries/tests/test_distro.py diff --git a/repos/system_upgrade/common/actors/targetuserspacecreator/libraries/userspacegen.py b/repos/system_upgrade/common/actors/targetuserspacecreator/libraries/userspacegen.py index 407cb0b7..26fec2d9 100644 --- a/repos/system_upgrade/common/actors/targetuserspacecreator/libraries/userspacegen.py +++ b/repos/system_upgrade/common/actors/targetuserspacecreator/libraries/userspacegen.py @@ -6,7 +6,7 @@ import shutil from leapp import reporting from leapp.exceptions import StopActorExecution, StopActorExecutionError from leapp.libraries.actor import constants -from leapp.libraries.common import dnfplugin, mounting, overlaygen, repofileutils, rhsm, utils +from leapp.libraries.common import distro, dnfplugin, mounting, overlaygen, repofileutils, rhsm, utils from leapp.libraries.common.config import get_distro_id, get_env, get_product_type from leapp.libraries.common.config.version import get_target_major_version from leapp.libraries.common.gpg import get_path_to_gpg_certs, is_nogpgcheck_set @@ -58,6 +58,7 @@ from leapp.utils.deprecation import suppress_deprecation PROD_CERTS_FOLDER = 'prod-certs' PERSISTENT_PACKAGE_CACHE_DIR = '/var/lib/leapp/persistent_package_cache' DEDICATED_LEAPP_PART_URL = 'https://access.redhat.com/solutions/7011704' +FMT_LIST_SEPARATOR = '\n - ' def _check_deprecated_rhsm_skip(): @@ -778,7 +779,7 @@ def _inhibit_on_duplicate_repos(repofiles): list_separator_fmt = '\n - ' api.current_logger().warning( 'The following repoids are defined multiple times:{0}{1}' - .format(list_separator_fmt, list_separator_fmt.join(duplicates)) + .format(list_separator_fmt, list_separator_fmt.join(sorted(duplicates))) ) reporting.create_report([ @@ -786,7 +787,7 @@ def _inhibit_on_duplicate_repos(repofiles): reporting.Summary( 'The following repositories are defined multiple times inside the' ' "upgrade" container:{0}{1}' - .format(list_separator_fmt, list_separator_fmt.join(duplicates)) + .format(list_separator_fmt, list_separator_fmt.join(sorted(duplicates))) ), reporting.Severity(reporting.Severity.MEDIUM), reporting.Groups([reporting.Groups.REPOSITORY]), @@ -815,21 +816,19 @@ def _get_all_available_repoids(context): return set(repoids) -def _get_rhsm_available_repoids(context): - target_major_version = get_target_major_version() +def _inhibit_if_no_base_repos(distro_repoids): # FIXME: check that required repo IDs (baseos, appstream) # + or check that all required RHEL repo IDs are available. - if rhsm.skip_rhsm(): - return set() - # Get the RHSM repos available in the target RHEL container - # TODO: very similar thing should happens for all other repofiles in container - # - repoids = rhsm.get_available_repo_ids(context) + + target_major_version = get_target_major_version() # NOTE(ivasilev) For the moment at least AppStream and BaseOS repos are required. While we are still # contemplating on what can be a generic solution to checking this, let's introduce a minimal check for # at-least-one-appstream and at-least-one-baseos among present repoids - if not repoids or all("baseos" not in ri for ri in repoids) or all("appstream" not in ri for ri in repoids): + no_baseos = all("baseos" not in ri for ri in distro_repoids) + no_appstream = all("appstream" not in ri for ri in distro_repoids) + if no_baseos or no_appstream: reporting.create_report([ + # TODO: Make the report distro agnostic reporting.Title('Cannot find required basic RHEL target repositories.'), reporting.Summary( 'This can happen when a repository ID was entered incorrectly either while using the --enablerepo' @@ -861,21 +860,6 @@ def _get_rhsm_available_repoids(context): title='Preparing for the upgrade') ]) raise StopActorExecution() - return set(repoids) - - -def _get_rhui_available_repoids(context, cloud_repo): - repofiles = repofileutils.get_parsed_repofiles(context) - - # TODO: same refactoring as Issue #486? - _inhibit_on_duplicate_repos(repofiles) - repoids = [] - for rfile in repofiles: - if rfile.file == cloud_repo and rfile.data: - repoids = [repo.repoid for repo in rfile.data] - repoids.sort() - break - return set(repoids) def get_copy_location_from_copy_in_task(context_basepath, copy_task): @@ -886,86 +870,106 @@ def get_copy_location_from_copy_in_task(context_basepath, copy_task): return copy_task.dst -def _get_rh_available_repoids(context, indata): +def _get_rhui_available_repoids(context, rhui_info): """ - RH repositories are provided either by RHSM or are stored in the expected repo file provided by - RHUI special packages (every cloud provider has itw own rpm). + Get repoids provided by the RHUI target clients + + :rtype: set[str] """ + # If we are upgrading a RHUI system, check what repositories are provided by the (already installed) target clients + setup_info = rhui_info.target_client_setup_info + target_content_access_files = set() + if setup_info.bootstrap_target_client: + target_content_access_files = _query_rpm_for_pkg_files(context, rhui_info.target_client_pkg_names) - rh_repoids = _get_rhsm_available_repoids(context) + def is_repofile(path): + return os.path.dirname(path) == '/etc/yum.repos.d' and os.path.basename(path).endswith('.repo') - # If we are upgrading a RHUI system, check what repositories are provided by the (already installed) target clients - if indata and indata.rhui_info: - setup_info = indata.rhui_info.target_client_setup_info - target_content_access_files = set() - if setup_info.bootstrap_target_client: - target_content_access_files = _query_rpm_for_pkg_files(context, indata.rhui_info.target_client_pkg_names) + def extract_repoid_from_line(line): + return line.split(':', 1)[1].strip() - def is_repofile(path): - return os.path.dirname(path) == '/etc/yum.repos.d' and os.path.basename(path).endswith('.repo') + target_ver = api.current_actor().configuration.version.target + setup_tasks = rhui_info.target_client_setup_info.preinstall_tasks.files_to_copy_into_overlay - def extract_repoid_from_line(line): - return line.split(':', 1)[1].strip() + yum_repos_d = context.full_path('/etc/yum.repos.d') + all_repofiles = {os.path.join(yum_repos_d, path) for path in os.listdir(yum_repos_d) if path.endswith('.repo')} + api.current_logger().debug('(RHUI Setup) All available repofiles: {0}'.format(' '.join(all_repofiles))) - target_ver = api.current_actor().configuration.version.target - setup_tasks = indata.rhui_info.target_client_setup_info.preinstall_tasks.files_to_copy_into_overlay + target_access_repofiles = { + context.full_path(path) for path in target_content_access_files if is_repofile(path) + } - yum_repos_d = context.full_path('/etc/yum.repos.d') - all_repofiles = {os.path.join(yum_repos_d, path) for path in os.listdir(yum_repos_d) if path.endswith('.repo')} - api.current_logger().debug('(RHUI Setup) All available repofiles: {0}'.format(' '.join(all_repofiles))) + # Exclude repofiles used to setup the target rhui access as on some platforms the repos provided by + # the client are not sufficient to install the client into target userspace (GCP) + rhui_setup_repofile_tasks = [task for task in setup_tasks if task.src.endswith('repo')] + rhui_setup_repofiles = ( + get_copy_location_from_copy_in_task(context.base_dir, copy) for copy in rhui_setup_repofile_tasks + ) + rhui_setup_repofiles = {context.full_path(repofile) for repofile in rhui_setup_repofiles} - target_access_repofiles = { - context.full_path(path) for path in target_content_access_files if is_repofile(path) - } + foreign_repofiles = all_repofiles - target_access_repofiles - rhui_setup_repofiles - # Exclude repofiles used to setup the target rhui access as on some platforms the repos provided by - # the client are not sufficient to install the client into target userspace (GCP) - rhui_setup_repofile_tasks = [task for task in setup_tasks if task.src.endswith('repo')] - rhui_setup_repofiles = ( - get_copy_location_from_copy_in_task(context.base_dir, copy) for copy in rhui_setup_repofile_tasks - ) - rhui_setup_repofiles = {context.full_path(repofile) for repofile in rhui_setup_repofiles} + api.current_logger().debug( + 'The following repofiles are considered as unknown to' + ' the target RHUI content setup and will be ignored: {0}'.format(' '.join(foreign_repofiles)) + ) - foreign_repofiles = all_repofiles - target_access_repofiles - rhui_setup_repofiles + # Rename non-client repofiles so they will not be recognized when running dnf repolist + for foreign_repofile in foreign_repofiles: + os.rename(foreign_repofile, '{0}.back'.format(foreign_repofile)) - api.current_logger().debug( - 'The following repofiles are considered as unknown to' - ' the target RHUI content setup and will be ignored: {0}'.format(' '.join(foreign_repofiles)) + rhui_repoids = set() + try: + dnf_cmd = [ + 'dnf', 'repolist', + '--releasever', target_ver, '-v', + '--enablerepo', '*', + '--disablerepo', '*-source-*', + '--disablerepo', '*-debug-*', + ] + repolist_result = context.call(dnf_cmd)['stdout'] + repoid_lines = [line for line in repolist_result.split('\n') if line.startswith('Repo-id')] + rhui_repoids.update({extract_repoid_from_line(line) for line in repoid_lines}) + + except CalledProcessError as err: + details = {'err': err.stderr, 'details': str(err)} + raise StopActorExecutionError( + message='Failed to retrieve repoids provided by target RHUI clients.', + details=details ) - # Rename non-client repofiles so they will not be recognized when running dnf repolist + finally: + # Revert the renaming of non-client repofiles for foreign_repofile in foreign_repofiles: - os.rename(foreign_repofile, '{0}.back'.format(foreign_repofile)) + os.rename('{0}.back'.format(foreign_repofile), foreign_repofile) - try: - dnf_cmd = [ - 'dnf', 'repolist', - '--releasever', target_ver, '-v', - '--enablerepo', '*', - '--disablerepo', '*-source-*', - '--disablerepo', '*-debug-*', - ] - repolist_result = context.call(dnf_cmd)['stdout'] - repoid_lines = [line for line in repolist_result.split('\n') if line.startswith('Repo-id')] - rhui_repoids = {extract_repoid_from_line(line) for line in repoid_lines} - rh_repoids.update(rhui_repoids) - - except CalledProcessError as err: - details = {'err': err.stderr, 'details': str(err)} - raise StopActorExecutionError( - message='Failed to retrieve repoids provided by target RHUI clients.', - details=details - ) + return rhui_repoids - finally: - # Revert the renaming of non-client repofiles - for foreign_repofile in foreign_repofiles: - os.rename('{0}.back'.format(foreign_repofile), foreign_repofile) - api.current_logger().debug( - 'The following repofiles are considered as provided by RedHat: {0}'.format(' '.join(rh_repoids)) - ) - return rh_repoids +def _get_distro_available_repoids(context, indata): + """ + Get repoids provided by the distribution + + On RHEL: RH repositories are provided either by RHSM or are stored in the + expected repo file provided by RHUI special packages (every cloud + provider has itw own rpm). + On other: Repositories are provided in specific repofiles (e.g. centos.repo + and centos-addons.repo on CS) + + :return: A set of repoids provided by distribution + :rtype: set[str] + """ + distro_repoids = distro.get_target_distro_repoids(context) + distro_id = get_distro_id() + rhel_and_rhsm = distro_id == 'rhel' and not rhsm.skip_rhsm() + if distro_id != 'rhel' or rhel_and_rhsm: + _inhibit_if_no_base_repos(distro_repoids) + + if indata and indata.rhui_info: + rhui_repoids = _get_rhui_available_repoids(context, indata.rhui_info) + distro_repoids.extend(rhui_repoids) + + return set(distro_repoids) @suppress_deprecation(RHELTargetRepository) # member of TargetRepositories @@ -986,17 +990,31 @@ def gather_target_repositories(context, indata): :param context: An instance of a mounting.IsolatedActions class :type context: mounting.IsolatedActions class :return: List of target system repoids - :rtype: List(string) + :rtype: set[str] """ - rh_available_repoids = _get_rh_available_repoids(context, indata) - all_available_repoids = _get_all_available_repoids(context) - target_repoids = [] - missing_custom_repoids = [] + distro_repoids = _get_distro_available_repoids(context, indata) + if distro_repoids: + api.current_logger().info( + "The following repoids are considered as provided by the '{}' distribution:{}{}".format( + get_distro_id(), + FMT_LIST_SEPARATOR, + FMT_LIST_SEPARATOR.join(sorted(distro_repoids)), + ) + ) + else: + api.current_logger().warning( + "No repoids provided by the {} distribution have been discovered".format(get_distro_id()) + ) + + all_repoids = _get_all_available_repoids(context) + + target_repoids = set() + missing_custom_repoids = set() for target_repo in api.consume(TargetRepositories): - for rhel_repo in target_repo.rhel_repos: - if rhel_repo.repoid in rh_available_repoids: - target_repoids.append(rhel_repo.repoid) + for distro_repo in target_repo.distro_repos: + if distro_repo.repoid in distro_repoids: + target_repoids.add(distro_repo.repoid) else: # TODO: We shall report that the RHEL repos that we deem necessary for # the upgrade are not available; but currently it would just print bunch of @@ -1005,12 +1023,16 @@ def gather_target_repositories(context, indata): # of the upgrade. Let's skip it for now until it's clear how we will deal # with it. pass + for custom_repo in target_repo.custom_repos: - if custom_repo.repoid in all_available_repoids: - target_repoids.append(custom_repo.repoid) + if custom_repo.repoid in all_repoids: + target_repoids.add(custom_repo.repoid) else: - missing_custom_repoids.append(custom_repo.repoid) - api.current_logger().debug("Gathered target repositories: {}".format(', '.join(target_repoids))) + missing_custom_repoids.add(custom_repo.repoid) + api.current_logger().debug( + "Gathered target repositories: {}".format(", ".join(sorted(target_repoids))) + ) + if not target_repoids: target_major_version = get_target_major_version() reporting.create_report([ @@ -1056,7 +1078,7 @@ def gather_target_repositories(context, indata): ' while using the --enablerepo option of leapp, or in a third party actor that produces a' ' CustomTargetRepositoryMessage.\n' 'The following repositories IDs could not be found in the target configuration:\n' - '- {}\n'.format('\n- '.join(missing_custom_repoids)) + '- {}\n'.format('\n- '.join(sorted(missing_custom_repoids))) ), reporting.Groups([reporting.Groups.REPOSITORY]), reporting.Groups([reporting.Groups.INHIBITOR]), @@ -1073,7 +1095,7 @@ def gather_target_repositories(context, indata): )) ]) raise StopActorExecution() - return set(target_repoids) + return target_repoids def _install_custom_repofiles(context, custom_repofiles): diff --git a/repos/system_upgrade/common/actors/targetuserspacecreator/tests/unit_test_targetuserspacecreator.py b/repos/system_upgrade/common/actors/targetuserspacecreator/tests/unit_test_targetuserspacecreator.py index f05e6bc2..2ae194d7 100644 --- a/repos/system_upgrade/common/actors/targetuserspacecreator/tests/unit_test_targetuserspacecreator.py +++ b/repos/system_upgrade/common/actors/targetuserspacecreator/tests/unit_test_targetuserspacecreator.py @@ -11,9 +11,9 @@ import pytest from leapp import models, reporting from leapp.exceptions import StopActorExecution, StopActorExecutionError from leapp.libraries.actor import userspacegen -from leapp.libraries.common import overlaygen, repofileutils, rhsm +from leapp.libraries.common import distro, overlaygen, repofileutils, rhsm from leapp.libraries.common.config import architecture -from leapp.libraries.common.testutils import CurrentActorMocked, logger_mocked, produce_mocked +from leapp.libraries.common.testutils import create_report_mocked, CurrentActorMocked, logger_mocked, produce_mocked from leapp.libraries.stdlib import api, CalledProcessError from leapp.utils.deprecation import suppress_deprecation @@ -1115,7 +1115,9 @@ def test_gather_target_repositories_rhui(monkeypatch): monkeypatch.setattr(userspacegen.api, 'current_actor', CurrentActorMocked()) monkeypatch.setattr(userspacegen, '_get_all_available_repoids', lambda x: []) monkeypatch.setattr( - userspacegen, '_get_rh_available_repoids', lambda x, y: ['rhui-1', 'rhui-2', 'rhui-3'] + userspacegen, + "_get_distro_available_repoids", + lambda dummy_context, dummy_indata: {"rhui-1", "rhui-2", "rhui-3"}, ) monkeypatch.setattr(rhsm, 'skip_rhsm', lambda: True) monkeypatch.setattr( @@ -1195,6 +1197,54 @@ def test_gather_target_repositories_baseos_appstream_not_available(monkeypatch): assert inhibitors[0].get('title', '') == 'Cannot find required basic RHEL target repositories.' +def test__get_distro_available_repoids_norhsm_norhui(monkeypatch): + """ + Empty set should be returned when on rhel and skip_rhsm == True. + """ + monkeypatch.setattr( + userspacegen.api, "current_actor", CurrentActorMocked(release_id="rhel") + ) + monkeypatch.setattr(userspacegen.api.current_actor(), 'produce', produce_mocked()) + + monkeypatch.setattr(rhsm, 'skip_rhsm', lambda: True) + monkeypatch.setattr(distro, 'get_target_distro_repoids', lambda ctx: []) + + indata = testInData(_PACKAGES_MSGS, None, None, _XFS_MSG, _STORAGEINFO_MSG, None) + # NOTE: context is not used without rhsm, for simplicity setting to None + repoids = userspacegen._get_distro_available_repoids(None, indata) + assert repoids == set() + + +@pytest.mark.parametrize( + "distro_id,skip_rhsm", [("rhel", False), ("centos", True), ("almalinux", True)] +) +def test__get_distro_available_repoids_nobaserepos_inhibit( + monkeypatch, distro_id, skip_rhsm +): + """ + Test that get_distro_available repoids reports and raises if there are no base repos. + """ + monkeypatch.setattr( + userspacegen.api, "current_actor", CurrentActorMocked(release_id=distro_id) + ) + monkeypatch.setattr(userspacegen.api.current_actor(), 'produce', produce_mocked()) + monkeypatch.setattr(reporting, "create_report", create_report_mocked()) + + monkeypatch.setattr(rhsm, 'skip_rhsm', lambda: skip_rhsm) + monkeypatch.setattr(distro, 'get_target_distro_repoids', lambda ctx: []) + + indata = testInData(_PACKAGES_MSGS, None, None, _XFS_MSG, _STORAGEINFO_MSG, None) + with pytest.raises(StopActorExecution): + # NOTE: context is not used without rhsm, for simplicity setting to None + userspacegen._get_distro_available_repoids(None, indata) + + # TODO adjust the asserts when the report is made distro agnostic + assert reporting.create_report.called == 1 + report = reporting.create_report.reports[0] + assert "Cannot find required basic RHEL target repositories" in report["title"] + assert reporting.Groups.INHIBITOR in report["groups"] + + def mocked_consume_data(): packages = {'dnf', 'dnf-command(config-manager)', 'pkgA', 'pkgB'} rhsm_info = _RHSMINFO_MSG diff --git a/repos/system_upgrade/common/libraries/distro.py b/repos/system_upgrade/common/libraries/distro.py index 2ed5eacd..d6a2381a 100644 --- a/repos/system_upgrade/common/libraries/distro.py +++ b/repos/system_upgrade/common/libraries/distro.py @@ -2,6 +2,10 @@ import json import os from leapp.exceptions import StopActorExecutionError +from leapp.libraries.common import repofileutils, rhsm +from leapp.libraries.common.config import get_distro_id +from leapp.libraries.common.config.architecture import ARCH_ACCEPTED, ARCH_X86_64 +from leapp.libraries.common.config.version import get_target_major_version from leapp.libraries.stdlib import api @@ -16,3 +20,191 @@ def get_distribution_data(distribution): raise StopActorExecutionError( 'Cannot find distribution signature configuration.', details={'Problem': 'Distribution {} was not found in {}.'.format(distribution, distributions_path)}) + + +# distro -> major_version -> repofile -> tuple of architectures where it's present +_DISTRO_REPOFILES_MAP = { + 'rhel': { + '8': {'/etc/yum.repos.d/redhat.repo': ARCH_ACCEPTED}, + '9': {'/etc/yum.repos.d/redhat.repo': ARCH_ACCEPTED}, + '10': {'/etc/yum.repos.d/redhat.repo': ARCH_ACCEPTED}, + }, + 'centos': { + '8': { + # TODO is this true on all archs? + 'CentOS-Linux-AppStream.repo': ARCH_ACCEPTED, + 'CentOS-Linux-BaseOS.repo': ARCH_ACCEPTED, + 'CentOS-Linux-ContinuousRelease.repo': ARCH_ACCEPTED, + 'CentOS-Linux-Debuginfo.repo': ARCH_ACCEPTED, + 'CentOS-Linux-Devel.repo': ARCH_ACCEPTED, + 'CentOS-Linux-Extras.repo': ARCH_ACCEPTED, + 'CentOS-Linux-FastTrack.repo': ARCH_ACCEPTED, + 'CentOS-Linux-HighAvailability.repo': ARCH_ACCEPTED, + 'CentOS-Linux-Media.repo': ARCH_ACCEPTED, + 'CentOS-Linux-Plus.repo': ARCH_ACCEPTED, + 'CentOS-Linux-PowerTools.repo': ARCH_ACCEPTED, + 'CentOS-Linux-Sources.repo': ARCH_ACCEPTED, + }, + '9': { + '/etc/yum.repos.d/centos.repo': ARCH_ACCEPTED, + '/etc/yum.repos.d/centos-addons.repo': ARCH_ACCEPTED, + }, + '10': { + '/etc/yum.repos.d/centos.repo': ARCH_ACCEPTED, + '/etc/yum.repos.d/centos-addons.repo': ARCH_ACCEPTED, + }, + }, + 'almalinux': { + '8': { + # TODO is this true on all archs? + '/etc/yum.repos.d/almalinux-ha.repo': ARCH_ACCEPTED, + '/etc/yum.repos.d/almalinux-nfv.repo': ARCH_ACCEPTED, + '/etc/yum.repos.d/almalinux-plus.repo': ARCH_ACCEPTED, + '/etc/yum.repos.d/almalinux-powertools.repo': ARCH_ACCEPTED, + '/etc/yum.repos.d/almalinux-resilientstorage.repo': ARCH_ACCEPTED, + '/etc/yum.repos.d/almalinux-rt.repo': ARCH_ACCEPTED, + '/etc/yum.repos.d/almalinux-sap.repo': ARCH_ACCEPTED, + '/etc/yum.repos.d/almalinux-saphana.repo': ARCH_ACCEPTED, + '/etc/yum.repos.d/almalinux.repo': ARCH_ACCEPTED, + }, + '9': { + '/etc/yum.repos.d/almalinux-appstream.repo': ARCH_ACCEPTED, + '/etc/yum.repos.d/almalinux-baseos.repo': ARCH_ACCEPTED, + '/etc/yum.repos.d/almalinux-crb.repo': ARCH_ACCEPTED, + '/etc/yum.repos.d/almalinux-extras.repo': ARCH_ACCEPTED, + '/etc/yum.repos.d/almalinux-highavailability.repo': ARCH_ACCEPTED, + '/etc/yum.repos.d/almalinux-plus.repo': ARCH_ACCEPTED, + '/etc/yum.repos.d/almalinux-resilientstorage.repo': ARCH_ACCEPTED, + '/etc/yum.repos.d/almalinux-sap.repo': ARCH_ACCEPTED, + '/etc/yum.repos.d/almalinux-saphana.repo': ARCH_ACCEPTED, + # RT and NFV are only on x86_64 on almalinux 9 + '/etc/yum.repos.d/almalinux-nfv.repo': (ARCH_X86_64,), + '/etc/yum.repos.d/almalinux-rt.repo': (ARCH_X86_64,), + }, + '10': { + # no resilientstorage on 10 + '/etc/yum.repos.d/almalinux-appstream.repo': ARCH_ACCEPTED, + '/etc/yum.repos.d/almalinux-baseos.repo': ARCH_ACCEPTED, + '/etc/yum.repos.d/almalinux-crb.repo': ARCH_ACCEPTED, + '/etc/yum.repos.d/almalinux-extras.repo': ARCH_ACCEPTED, + '/etc/yum.repos.d/almalinux-highavailability.repo': ARCH_ACCEPTED, + '/etc/yum.repos.d/almalinux-plus.repo': ARCH_ACCEPTED, + '/etc/yum.repos.d/almalinux-sap.repo': ARCH_ACCEPTED, + '/etc/yum.repos.d/almalinux-saphana.repo': ARCH_ACCEPTED, + # RT and NFV are only on x86_64 on almalinux 10 + '/etc/yum.repos.d/almalinux-nfv.repo': (ARCH_X86_64,), + '/etc/yum.repos.d/almalinux-rt.repo': (ARCH_X86_64,), + }, + }, +} + + +def _get_distro_repofiles(distro, major_version, arch): + """ + Get distribution provided repofiles. + + Note that this does not perform any validation, the caller must check + whether the files exist. + + :param distro: The distribution to get repofiles for. + :type distro: str + :param major_version: The major version to get repofiles for. + :type major_version: str + :param arch: The architecture to get repofiles for. + :type arch: str + :return: A list of paths to repofiles provided by distribution + :rtype: list[str] or None if no repofiles are mapped for the arguments + """ + + distro_repofiles = _DISTRO_REPOFILES_MAP.get(distro) + if not distro_repofiles: + return None + + version_repofiles = distro_repofiles.get(major_version, {}) + if not version_repofiles: + return None + + return [repofile for repofile, archs in version_repofiles.items() if arch in archs] + + +def get_target_distro_repoids(context): + """ + Get repoids defined in distro provided repofiles + + See the generic :func:`_get_distro_repoids` for more details. + + :param context: An instance of mounting.IsolatedActions class + :type context: mounting.IsolatedActions + :return: Repoids of distribution provided repositories + :type: list[str] + """ + + return get_distro_repoids( + context, + get_distro_id(), + get_target_major_version(), + api.current_actor().configuration.architecture + ) + + +def get_distro_repoids(context, distro, major_version, arch): + """ + Get repoids defined in distro provided repofiles + + On RHEL with RHSM this delegates to rhsm.get_available_repo_ids. + + Repofiles installed by RHUI client packages are not covered by this + function. + + :param context: An instance of mounting.IsolatedActions class + :type context: mounting.IsolatedActions + :param distro: The distro whose repoids to return + :type distro: str + :param major_version: The major version to get distro repoids for. + :type major_version: str + :param arch: The architecture to get distro repoids for. + :type arch: str + :return: Repoids of distribution provided repositories + :type: list[str] + """ + + if distro == 'rhel': + if rhsm.skip_rhsm(): + return [] + # Kept this todo here from the original code from + # userspacegen._get_rh_available_repoids: + # Get the RHSM repos available in the target RHEL container + # TODO: very similar thing should happens for all other repofiles in container + return rhsm.get_available_repo_ids(context) + + repofiles = repofileutils.get_parsed_repofiles(context) + distro_repofiles = _get_distro_repofiles(distro, major_version, arch) + if not distro_repofiles: + # TODO: a different way of signaling an error would be preferred (e.g. returning None), + # but since rhsm.get_available_repo_ids also raises StopActorExecutionError, + # let's make it easier for the caller for now and use it too + raise StopActorExecutionError( + "No known distro provided repofiles mapped", + details={ + "details": "distro: {}, major version: {}, architecture: {}".format( + distro, major_version, arch + ) + }, + ) + + distro_repoids = [] + for rfile in repofiles: + if rfile.file in distro_repofiles: + + if not os.path.exists(context.full_path(rfile.file)): + api.current_logger().debug( + "Expected distribution provided repofile does not exists: {}".format( + rfile + ) + ) + continue + + if rfile.data: + distro_repoids.extend([repo.repoid for repo in rfile.data]) + + return sorted(distro_repoids) diff --git a/repos/system_upgrade/common/libraries/tests/test_distro.py b/repos/system_upgrade/common/libraries/tests/test_distro.py new file mode 100644 index 00000000..3a8f174f --- /dev/null +++ b/repos/system_upgrade/common/libraries/tests/test_distro.py @@ -0,0 +1,154 @@ +import os + +import pytest + +from leapp.actors import StopActorExecutionError +from leapp.libraries.common import distro, repofileutils, rhsm +from leapp.libraries.common.config.architecture import ARCH_ACCEPTED, ARCH_ARM64, ARCH_PPC64LE, ARCH_S390X, ARCH_X86_64 +from leapp.libraries.common.distro import _get_distro_repofiles, get_distro_repoids +from leapp.libraries.common.testutils import CurrentActorMocked +from leapp.libraries.stdlib import api +from leapp.models import RepositoryData, RepositoryFile + +_RHEL_REPOFILES = ['/etc/yum.repos.d/redhat.repo'] +_CENTOS_REPOFILES = [ + "/etc/yum.repos.d/centos.repo", "/etc/yum.repos.d/centos-addons.repo" +] + + +def test_get_distro_repofiles(monkeypatch): + """ + Test the functionality, not the data. + """ + test_map = { + 'distro1': { + '8': { + 'repofile1': ARCH_ACCEPTED, + 'repofile2': [ARCH_X86_64], + }, + '9': { + 'repofile3': ARCH_ACCEPTED, + }, + }, + 'distro2': { + '8': {}, + '9': { + 'repofile2': [ARCH_X86_64], + 'repofile3': [ARCH_ARM64, ARCH_S390X, ARCH_PPC64LE], + }, + }, + } + monkeypatch.setattr(distro, '_DISTRO_REPOFILES_MAP', test_map) + + # mix of all and specific arch + repofiles = _get_distro_repofiles('distro1', '8', ARCH_X86_64) + assert repofiles == ['repofile1', 'repofile2'] + + # match all but not x86_64 + repofiles = _get_distro_repofiles('distro1', '8', ARCH_ARM64) + assert repofiles == ['repofile1'] + + repofiles = _get_distro_repofiles('distro2', '9', ARCH_X86_64) + assert repofiles == ['repofile2'] + repofiles = _get_distro_repofiles('distro2', '9', ARCH_ARM64) + assert repofiles == ['repofile3'] + repofiles = _get_distro_repofiles('distro2', '9', ARCH_S390X) + assert repofiles == ['repofile3'] + repofiles = _get_distro_repofiles('distro2', '9', ARCH_PPC64LE) + assert repofiles == ['repofile3'] + + # version not mapped + repofiles = _get_distro_repofiles('distro2', '8', ARCH_X86_64) + assert repofiles is None + + # distro not mapped + repofiles = _get_distro_repofiles('distro42', '8', ARCH_X86_64) + assert repofiles is None + + +def _make_repo(repoid): + return RepositoryData(repoid=repoid, name='name {}'.format(repoid)) + + +def _make_repofile(rfile, data=None): + if data is None: + data = [_make_repo("{}-{}".format(rfile.split("/")[-1], i)) for i in range(3)] + return RepositoryFile(file=rfile, data=data) + + +def _make_repofiles(rfiles): + return [_make_repofile(rfile) for rfile in rfiles] + + +@pytest.mark.parametrize('other_rfiles', [ + [], + [_make_repofile("foo")], + _make_repofiles(["foo", "bar"]), +]) +@pytest.mark.parametrize( + "distro_id,skip_rhsm,distro_rfiles", + [ + ("rhel", True, []), + ("rhel", True, _make_repofiles(_RHEL_REPOFILES)), + ("rhel", False, _make_repofiles(_RHEL_REPOFILES)), + ("centos", True, []), + ("centos", True, _make_repofiles(_CENTOS_REPOFILES)), + ] +) +def test_get_distro_repoids( + monkeypatch, distro_id, skip_rhsm, distro_rfiles, other_rfiles +): + """ + Tests that the correct repoids are returned + + This is a little ugly because on RHEL the get_distro_repoids function still + delegates to rhsm.get_available_repo_ids and also has different behavior + with skip_rhsm + """ + current_actor = CurrentActorMocked(release_id=distro_id if distro_id else 'rhel') + monkeypatch.setattr(api, 'current_actor', current_actor) + monkeypatch.setattr(rhsm, 'skip_rhsm', lambda: skip_rhsm) + + repofiles = other_rfiles + if distro_rfiles: + repofiles.extend(distro_rfiles) + monkeypatch.setattr(repofileutils, 'get_parsed_repofiles', lambda x: repofiles) + + distro_repoids = [] + for rfile in distro_rfiles: + distro_repoids.extend([repo.repoid for repo in rfile.data] if rfile else []) + distro_repoids.sort() + + monkeypatch.setattr(rhsm, 'get_available_repo_ids', lambda _: distro_repoids) + monkeypatch.setattr(os.path, 'exists', lambda f: f in _CENTOS_REPOFILES) + + class MockedContext: + def full_path(self, path): + return path + + repoids = get_distro_repoids(MockedContext(), distro_id, '9', 'x86_64') + + if distro_id == 'rhel' and skip_rhsm: + assert repoids == [] + else: + assert sorted(repoids) == distro_repoids + + +@pytest.mark.parametrize('other_rfiles', [ + [], + [_make_repofile("foo")], + _make_repofiles(["foo", "bar"]), +]) +def test_get_distro_repoids_no_distro_repofiles(monkeypatch, other_rfiles): + """ + Test that exception is thrown when there are no known distro provided repofiles. + """ + + def mocked_get_distro_repofiles(*args): + return [] + + monkeypatch.setattr(distro, '_get_distro_repofiles', mocked_get_distro_repofiles) + monkeypatch.setattr(repofileutils, "get_parsed_repofiles", lambda x: other_rfiles) + + with pytest.raises(StopActorExecutionError): + get_distro_repoids(None, 'somedistro', '8', 'x86_64') -- 2.51.1