diff --git a/.gitignore b/.gitignore index c31af57..d8cdd6f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,2 @@ -SOURCES/deps-pkgs-9.tar.gz -SOURCES/leapp-repository-0.19.0.tar.gz +SOURCES/deps-pkgs-10.tar.gz +SOURCES/leapp-repository-0.20.0.tar.gz diff --git a/.leapp-repository.metadata b/.leapp-repository.metadata index 1597be5..c93f55a 100644 --- a/.leapp-repository.metadata +++ b/.leapp-repository.metadata @@ -1,2 +1,2 @@ -02499ccd70d4a8e6ce9ad29bd286a317d5e0b57b SOURCES/deps-pkgs-9.tar.gz -79402ad1aa427e43bdce143f4c0641dda383eb5d SOURCES/leapp-repository-0.19.0.tar.gz +d520ada12294e4dd8837c81f92d4c184ab403d51 SOURCES/deps-pkgs-10.tar.gz +185bbb040dba48e1ea2d6c627133af594378afd4 SOURCES/leapp-repository-0.20.0.tar.gz diff --git a/SOURCES/0001-rhui-do-not-bootstrap-target-client-on-aws.patch b/SOURCES/0001-rhui-do-not-bootstrap-target-client-on-aws.patch new file mode 100644 index 0000000..dbe1f57 --- /dev/null +++ b/SOURCES/0001-rhui-do-not-bootstrap-target-client-on-aws.patch @@ -0,0 +1,251 @@ +From 921c06892f7550a3a8e2b3fe941c6272bdacf88d Mon Sep 17 00:00:00 2001 +From: mhecko +Date: Thu, 15 Feb 2024 09:56:27 +0100 +Subject: [PATCH] rhui: do not bootstrap target client on aws + +Bootstrapping target RHUI client now requires installing the entire +RHEL8 RPM stack. Threfore, do not try installing target client +and instead rely only on the files from our leapp-rhui-aws package. +--- + .../cloud/checkrhui/libraries/checkrhui.py | 6 +- + .../libraries/userspacegen.py | 104 ++++++++++++++---- + .../system_upgrade/common/models/rhuiinfo.py | 7 ++ + 3 files changed, 92 insertions(+), 25 deletions(-) + +diff --git a/repos/system_upgrade/common/actors/cloud/checkrhui/libraries/checkrhui.py b/repos/system_upgrade/common/actors/cloud/checkrhui/libraries/checkrhui.py +index 84ab40e3..e1c158c7 100644 +--- a/repos/system_upgrade/common/actors/cloud/checkrhui/libraries/checkrhui.py ++++ b/repos/system_upgrade/common/actors/cloud/checkrhui/libraries/checkrhui.py +@@ -142,7 +142,11 @@ def customize_rhui_setup_for_aws(rhui_family, setup_info): + + target_version = version.get_target_major_version() + if target_version == '8': +- return # The rhel8 plugin is packed into leapp-rhui-aws as we need python2 compatible client ++ # RHEL8 rh-amazon-rhui-client depends on amazon-libdnf-plugin that depends ++ # essentially on the entire RHEL8 RPM stack, so we cannot just swap the clients ++ # The leapp-rhui-aws will provide all necessary files to access entire RHEL8 content ++ setup_info.bootstrap_target_client = False ++ return + + amazon_plugin_copy_task = CopyFile(src='/usr/lib/python3.9/site-packages/dnf-plugins/amazon-id.py', + dst='/usr/lib/python3.6/site-packages/dnf-plugins/') +diff --git a/repos/system_upgrade/common/actors/targetuserspacecreator/libraries/userspacegen.py b/repos/system_upgrade/common/actors/targetuserspacecreator/libraries/userspacegen.py +index d917bfd5..d60bc75f 100644 +--- a/repos/system_upgrade/common/actors/targetuserspacecreator/libraries/userspacegen.py ++++ b/repos/system_upgrade/common/actors/targetuserspacecreator/libraries/userspacegen.py +@@ -853,9 +853,9 @@ def _get_rhui_available_repoids(context, cloud_repo): + return set(repoids) + + +-def get_copy_location_from_copy_in_task(context, copy_task): ++def get_copy_location_from_copy_in_task(context_basepath, copy_task): + basename = os.path.basename(copy_task.src) +- dest_in_container = context.full_path(copy_task.dst) ++ dest_in_container = os.path.join(context_basepath, copy_task.dst) + if os.path.isdir(dest_in_container): + return os.path.join(copy_task.dst, basename) + return copy_task.dst +@@ -871,7 +871,10 @@ def _get_rh_available_repoids(context, indata): + + # If we are upgrading a RHUI system, check what repositories are provided by the (already installed) target clients + if indata and indata.rhui_info: +- files_provided_by_clients = _query_rpm_for_pkg_files(context, indata.rhui_info.target_client_pkg_names) ++ setup_info = indata.rhui_info.target_client_setup_info ++ target_content_access_files = set() ++ if setup_info.bootstrap_target_client: ++ target_content_access_files = _query_rpm_for_pkg_files(context, indata.rhui_info.target_client_pkg_names) + + def is_repofile(path): + return os.path.dirname(path) == '/etc/yum.repos.d' and os.path.basename(path).endswith('.repo') +@@ -884,24 +887,33 @@ def _get_rh_available_repoids(context, indata): + + yum_repos_d = context.full_path('/etc/yum.repos.d') + all_repofiles = {os.path.join(yum_repos_d, path) for path in os.listdir(yum_repos_d) if path.endswith('.repo')} +- client_repofiles = {context.full_path(path) for path in files_provided_by_clients if is_repofile(path)} ++ api.current_logger().debug('(RHUI Setup) All available repofiles: {0}'.format(' '.join(all_repofiles))) ++ ++ target_access_repofiles = { ++ context.full_path(path) for path in target_content_access_files if is_repofile(path) ++ } + + # Exclude repofiles used to setup the target rhui access as on some platforms the repos provided by + # the client are not sufficient to install the client into target userspace (GCP) + rhui_setup_repofile_tasks = [task for task in setup_tasks if task.src.endswith('repo')] + rhui_setup_repofiles = ( +- get_copy_location_from_copy_in_task(context, copy_task) for copy_task in rhui_setup_repofile_tasks ++ get_copy_location_from_copy_in_task(context.base_dir, copy) for copy in rhui_setup_repofile_tasks + ) + rhui_setup_repofiles = {context.full_path(repofile) for repofile in rhui_setup_repofiles} + +- foreign_repofiles = all_repofiles - client_repofiles - rhui_setup_repofiles ++ foreign_repofiles = all_repofiles - target_access_repofiles - rhui_setup_repofiles ++ ++ api.current_logger().debug( ++ 'The following repofiles are considered as unknown to' ++ ' the target RHUI content setup and will be ignored: {0}'.format(' '.join(foreign_repofiles)) ++ ) + + # Rename non-client repofiles so they will not be recognized when running dnf repolist + for foreign_repofile in foreign_repofiles: + os.rename(foreign_repofile, '{0}.back'.format(foreign_repofile)) + + try: +- dnf_cmd = ['dnf', 'repolist', '--releasever', target_ver, '-v'] ++ dnf_cmd = ['dnf', 'repolist', '--releasever', target_ver, '-v', '--enablerepo', '*'] + repolist_result = context.call(dnf_cmd)['stdout'] + repoid_lines = [line for line in repolist_result.split('\n') if line.startswith('Repo-id')] + rhui_repoids = {extract_repoid_from_line(line) for line in repoid_lines} +@@ -919,6 +931,9 @@ def _get_rh_available_repoids(context, indata): + for foreign_repofile in foreign_repofiles: + os.rename('{0}.back'.format(foreign_repofile), foreign_repofile) + ++ api.current_logger().debug( ++ 'The following repofiles are considered as provided by RedHat: {0}'.format(' '.join(rh_repoids)) ++ ) + return rh_repoids + + +@@ -1086,7 +1101,7 @@ def _get_target_userspace(): + return constants.TARGET_USERSPACE.format(get_target_major_version()) + + +-def _create_target_userspace(context, packages, files, target_repoids): ++def _create_target_userspace(context, indata, packages, files, target_repoids): + """Create the target userspace.""" + target_path = _get_target_userspace() + prepare_target_userspace(context, target_path, target_repoids, list(packages)) +@@ -1096,12 +1111,57 @@ def _create_target_userspace(context, packages, files, target_repoids): + _copy_files(target_context, files) + dnfplugin.install(_get_target_userspace()) + ++ # If we used only repofiles from leapp-rhui- then remove these as they provide ++ # duplicit definitions as the target clients already installed in the target container ++ if indata.rhui_info: ++ api.current_logger().debug( ++ 'Target container should have access to content. ' ++ 'Removing repofiles from leapp-rhui- from the target..' ++ ) ++ setup_info = indata.rhui_info.target_client_setup_info ++ if not setup_info.bootstrap_target_client: ++ target_userspace_path = _get_target_userspace() ++ for copy in setup_info.preinstall_tasks.files_to_copy_into_overlay: ++ dst_in_container = get_copy_location_from_copy_in_task(target_userspace_path, copy) ++ dst_in_container = dst_in_container.strip('/') ++ dst_in_host = os.path.join(target_userspace_path, dst_in_container) ++ if os.path.isfile(dst_in_host) and dst_in_host.endswith('.repo'): ++ api.current_logger().debug('Removing repofile: {0}'.format(dst_in_host)) ++ os.remove(dst_in_host) ++ + # and do not forget to set the rhsm into the container mode again + with mounting.NspawnActions(_get_target_userspace()) as target_context: + rhsm.set_container_mode(target_context) + + +-def install_target_rhui_client_if_needed(context, indata): ++def _apply_rhui_access_preinstall_tasks(context, rhui_setup_info): ++ if rhui_setup_info.preinstall_tasks: ++ api.current_logger().debug('Applying RHUI preinstall tasks.') ++ preinstall_tasks = rhui_setup_info.preinstall_tasks ++ ++ for file_to_remove in preinstall_tasks.files_to_remove: ++ api.current_logger().debug('Removing {0} from the scratch container.'.format(file_to_remove)) ++ context.remove(file_to_remove) ++ ++ for copy_info in preinstall_tasks.files_to_copy_into_overlay: ++ api.current_logger().debug( ++ 'Copying {0} in {1} into the scratch container.'.format(copy_info.src, copy_info.dst) ++ ) ++ context.makedirs(os.path.dirname(copy_info.dst), exists_ok=True) ++ context.copy_to(copy_info.src, copy_info.dst) ++ ++ ++def _apply_rhui_access_postinstall_tasks(context, rhui_setup_info): ++ if rhui_setup_info.postinstall_tasks: ++ api.current_logger().debug('Applying RHUI postinstall tasks.') ++ for copy_info in rhui_setup_info.postinstall_tasks.files_to_copy: ++ context.makedirs(os.path.dirname(copy_info.dst), exists_ok=True) ++ debug_msg = 'Copying {0} to {1} (inside the scratch container).' ++ api.current_logger().debug(debug_msg.format(copy_info.src, copy_info.dst)) ++ context.call(['cp', copy_info.src, copy_info.dst]) ++ ++ ++def setup_target_rhui_access_if_needed(context, indata): + if not indata.rhui_info: + return + +@@ -1110,15 +1170,14 @@ def install_target_rhui_client_if_needed(context, indata): + _create_target_userspace_directories(userspace_dir) + + setup_info = indata.rhui_info.target_client_setup_info +- if setup_info.preinstall_tasks: +- preinstall_tasks = setup_info.preinstall_tasks ++ _apply_rhui_access_preinstall_tasks(context, setup_info) + +- for file_to_remove in preinstall_tasks.files_to_remove: +- context.remove(file_to_remove) +- +- for copy_info in preinstall_tasks.files_to_copy_into_overlay: +- context.makedirs(os.path.dirname(copy_info.dst), exists_ok=True) +- context.copy_to(copy_info.src, copy_info.dst) ++ if not setup_info.bootstrap_target_client: ++ # Installation of the target RHUI client is not possible and we bundle all necessary ++ # files into the leapp-rhui- packages. ++ api.current_logger().debug('Bootstrapping target RHUI client is disabled, leapp will rely ' ++ 'only on files budled in leapp-rhui- package.') ++ return + + cmd = ['dnf', '-y'] + +@@ -1149,16 +1208,13 @@ def install_target_rhui_client_if_needed(context, indata): + + context.call(cmd, callback_raw=utils.logging_handler, stdin='\n'.join(dnf_transaction_steps)) + +- if setup_info.postinstall_tasks: +- for copy_info in setup_info.postinstall_tasks.files_to_copy: +- context.makedirs(os.path.dirname(copy_info.dst), exists_ok=True) +- context.call(['cp', copy_info.src, copy_info.dst]) ++ _apply_rhui_access_postinstall_tasks(context, setup_info) + + # Do a cleanup so there are not duplicit repoids + files_owned_by_clients = _query_rpm_for_pkg_files(context, indata.rhui_info.target_client_pkg_names) + + for copy_task in setup_info.preinstall_tasks.files_to_copy_into_overlay: +- dest = get_copy_location_from_copy_in_task(context, copy_task) ++ dest = get_copy_location_from_copy_in_task(context.base_dir, copy_task) + can_be_cleaned_up = copy_task.src not in setup_info.files_supporting_client_operation + if dest not in files_owned_by_clients and can_be_cleaned_up: + context.remove(dest) +@@ -1184,10 +1240,10 @@ def perform(): + target_iso = next(api.consume(TargetOSInstallationImage), None) + with mounting.mount_upgrade_iso_to_root_dir(overlay.target, target_iso): + +- install_target_rhui_client_if_needed(context, indata) ++ setup_target_rhui_access_if_needed(context, indata) + + target_repoids = _gather_target_repositories(context, indata, prod_cert_path) +- _create_target_userspace(context, indata.packages, indata.files, target_repoids) ++ _create_target_userspace(context, indata, indata.packages, indata.files, target_repoids) + # TODO: this is tmp solution as proper one needs significant refactoring + target_repo_facts = repofileutils.get_parsed_repofiles(context) + api.produce(TMPTargetRepositoriesFacts(repositories=target_repo_facts)) +diff --git a/repos/system_upgrade/common/models/rhuiinfo.py b/repos/system_upgrade/common/models/rhuiinfo.py +index 3eaa4826..0a2e45af 100644 +--- a/repos/system_upgrade/common/models/rhuiinfo.py ++++ b/repos/system_upgrade/common/models/rhuiinfo.py +@@ -36,6 +36,13 @@ class TargetRHUISetupInfo(Model): + files_supporting_client_operation = fields.List(fields.String(), default=[]) + """A subset of files copied in preinstall tasks that should not be cleaned up.""" + ++ bootstrap_target_client = fields.Boolean(default=True) ++ """ ++ Swap the current RHUI client for the target one to facilitate access to the target content. ++ ++ When False, only files from the leapp-rhui- will be used to access target content. ++ """ ++ + + class RHUIInfo(Model): + """ +-- +2.43.0 + diff --git a/SPECS/leapp-repository.spec b/SPECS/leapp-repository.spec index 6d4d68d..434ab2b 100644 --- a/SPECS/leapp-repository.spec +++ b/SPECS/leapp-repository.spec @@ -2,7 +2,7 @@ %global repositorydir %{leapp_datadir}/repositories %global custom_repositorydir %{leapp_datadir}/custom-repositories -%define leapp_repo_deps 9 +%define leapp_repo_deps 10 %if 0%{?rhel} == 7 %define leapp_python_sitelib %{python2_sitelib} @@ -41,14 +41,14 @@ py2_byte_compile "%1" "%2"} # RHEL 8+ packages to be consistent with other leapp projects in future. Name: leapp-repository -Version: 0.19.0 -Release: 1%{?dist} +Version: 0.20.0 +Release: 2%{?dist} Summary: Repositories for leapp License: ASL 2.0 URL: https://oamg.github.io/leapp/ Source0: https://github.com/oamg/%{name}/archive/v%{version}.tar.gz#/%{name}-%{version}.tar.gz -Source1: deps-pkgs-9.tar.gz +Source1: deps-pkgs-10.tar.gz # NOTE: Our packages must be noarch. Do no drop this in any way. BuildArch: noarch @@ -56,6 +56,8 @@ BuildArch: noarch ### PATCHES HERE # Patch0001: filename.patch +Patch0001: 0001-rhui-do-not-bootstrap-target-client-on-aws.patch + %description %{summary} @@ -149,6 +151,16 @@ Provides: leapp-repository-dependencies = %{leapp_repo_deps} ################################################## Requires: dnf >= 4 Requires: pciutils + +# required to be able to format disk images with XFS file systems (default) +Requires: xfsprogs + +# required to be able to format disk images with Ext4 file systems +# NOTE: this is not happening by default, but we can expact that many customers +# will want to / need to do this - especially on RHEL 7 now. Adding this deps +# as the best trade-off to resolve this problem. +Requires: e2fsprogs + %if 0%{?rhel} && 0%{?rhel} == 7 # Required to gather system facts about SELinux Requires: libselinux-python @@ -197,6 +209,7 @@ Requires: python3-gobject-base # APPLY PATCHES HERE # %%patch0001 -p1 +%patch0001 -p1 %build @@ -274,6 +287,67 @@ done; # no files here %changelog +* Tue Feb 20 2024 Petr Stodulka - 0.20.0-2 +- Fallback to original RHUI solution on AWS to fix issues caused by changes in RHUI client +- Resolves: RHEL-16729 + +* Tue Feb 13 2024 Toshio Kuratomi - 0.20.0-1 +- Rebase to new upstream v0.20.0. +- Fix semanage import issue +- Fix handling of libvirt's systemd services +- Add a dracut breakpoint for the pre-upgrade step. +- Drop obsoleted upgrade paths (obsoleted releases: 8.6, 8.9, 9.0, 9.3) +- Resolves: RHEL-16729 + +* Tue Jan 23 2024 Toshio Kuratomi - 0.19.0-10 +- Print nice error msg when device and driver deprecation data is malformed +- Fix another cornercase when preserving symlinks to certificates in /etc/pki +- Update the leapp upgrade data files - fixing upgrades with idm-tomcatjss +- Resolves: RHEL-16729 + +* Fri Jan 19 2024 Petr Stodulka - 0.19.0-9 +- Do not try to download data files anymore when missing as the service + is obsoleted since the data is part of installed packages +- Update error messages and reports when installed upgrade data files + are malformed or missing to instruct user how to resolve it +- Update the leapp upgrade data files - bump data stream to "3.0" +- Resolves: RHEL-16729 + +* Fri Jan 12 2024 Petr Stodulka - 0.19.0-7 +- Add detection of possible usage of OpenSSL IBMCA engine on IBM Z machines +- Add detection of modified /etc/pki/tls/openssl.cnf file +- Update the leapp upgrade data files +- Fix handling of symlinks under /etc/pki with relative paths specified +- Report custom actors and modifications of the upgrade tooling +- Requires xfsprogs and e2fsprogs to ensure that Ext4 and XFS tools are installed +- Bump leapp-repository-dependencies to 10 +- Resolves: RHEL-1774, RHEL-16729 + +* Thu Nov 16 2023 Petr Stodulka - 0.19.0-5 +- Enable new upgrade path for RHEL 8.10 -> RHEL 9.4 (including RHEL with SAP HANA) +- Introduce generic transition of systemd services states during the IPU +- Introduce possibility to upgrade with local repositories +- Improve possibilities of upgrade when a proxy is configured in DNF configutation file +- Fix handling of symlinks under /etc/pki when managing certificates +- Fix the upgrade with custom https repositories +- Default to the NO_RHSM mode when subscription-manager is not installed +- Detect customized configuration of dynamic linker +- Drop the invalid `tuv` target channel for the --channel option +- Fix the issue of going out of bounds in the isccfg parser +- Fix traceback when saving the rhsm facts results and the /etc/rhsm/facts directory doesn’t exist yet +- Load all rpm repository substitutions that dnf knows about, not just "releasever" only +- Simplify handling of upgrades on systems using RHUI, reducing the maintenance burden for cloud providers +- Detect possible unexpected RPM GPG keys has been installed during RPM transaction +- Resolves: RHEL-16729 + +* Thu Nov 02 2023 Petr Stodulka - 0.19.0-4 +- Fix the upgrade for systems without subscription-manager package +- Resolves: RHEL-14901 + +* Tue Oct 31 2023 Petr Stodulka - 0.19.0-3 +- Fix the upgrade when the release is locked by new subscription-manager +- Resolves: RHEL-14901 + * Wed Aug 23 2023 Petr Stodulka - 0.19.0-1 - Rebase to v0.19.0 - Requires leapp-framework 5.0 @@ -285,7 +359,7 @@ done; - Use new leapp CLI API which provides better report summary output - Resolves: rhbz#2215997, rhbz#2222861, rhbz#2232618 -* Mon Jul 18 2023 Petr Stodulka - 0.18.0-5 +* Tue Jul 18 2023 Petr Stodulka - 0.18.0-5 - Fix the calculation of the required free space on each partitions/volume for the upgrade transactions - Create source overlay images with dynamic sizes to optimize disk space consumption - Update GRUB2 when /boot resides on multiple devices aggregated in RAID