From 0d3570809e1992331b2b545e47fe1c991c21f10c Mon Sep 17 00:00:00 2001 From: Petr Stodulka Date: Fri, 19 Jan 2024 16:28:03 +0100 Subject: [PATCH] RHEL 8.10: CTC2 candidate -1 - Do not try to download data files anymore when missing as the service is obsoleted since the data is part of installed packages - Update error messages and reports when installed upgrade data files are malformed or missing to instruct user how to resolve it - Update the leapp upgrade data files - bump data stream to "3.0" - Resolves: RHEL-18298 --- ...y_path-instead-e2e-for-public-clouds.patch | 107 ++++++ ...ata-bump-required-data-stream-to-3.0.patch | 256 ++++++++++++++ ...HEL-8-to-RHEL-9-using-RHUI-on-Alibab.patch | 38 +++ ...do-not-try-to-download-data-files-wh.patch | 49 +++ ...es-loading-update-error-msgs-and-rep.patch | 319 ++++++++++++++++++ leapp-repository.spec | 22 +- 6 files changed, 790 insertions(+), 1 deletion(-) create mode 100644 0062-Use-happy_path-instead-e2e-for-public-clouds.patch create mode 100644 0063-Update-upgrade-data-bump-required-data-stream-to-3.0.patch create mode 100644 0064-Cover-upgrades-RHEL-8-to-RHEL-9-using-RHUI-on-Alibab.patch create mode 100644 0065-load-data-files-do-not-try-to-download-data-files-wh.patch create mode 100644 0066-upgrade-data-files-loading-update-error-msgs-and-rep.patch diff --git a/0062-Use-happy_path-instead-e2e-for-public-clouds.patch b/0062-Use-happy_path-instead-e2e-for-public-clouds.patch new file mode 100644 index 0000000..f69ec23 --- /dev/null +++ b/0062-Use-happy_path-instead-e2e-for-public-clouds.patch @@ -0,0 +1,107 @@ +From 946f8c6a36962a4e7ddc5354d21fcd7d70e108f9 Mon Sep 17 00:00:00 2001 +From: Martin Kluson +Date: Fri, 12 Jan 2024 13:45:59 +0100 +Subject: [PATCH 62/66] Use `happy_path` instead `e2e` for public clouds + +`happy_path` performs similar steps as `e2e` does and is used in the rest of tiers. +--- + .packit.yaml | 41 ++++++++++++++++++++++++++--------------- + 1 file changed, 26 insertions(+), 15 deletions(-) + +diff --git a/.packit.yaml b/.packit.yaml +index 383f5314..d87f33c0 100644 +--- a/.packit.yaml ++++ b/.packit.yaml +@@ -123,13 +123,13 @@ jobs: + targets: + epel-7-x86_64: + distros: [RHEL-7.9-rhui] +- identifier: sanity-7.9to8.6-aws-e2e ++ identifier: sanity-7.9to8.6-aws + # NOTE(ivasilev) Unfortunately to use yaml templates we need to rewrite the whole tf_extra_params dict + # to use plan_filter (can't just specify one section test.tmt.plan_filter, need to specify environments.* as well) + tf_extra_params: + test: + tmt: +- plan_filter: 'tag:e2e & enabled:true' ++ plan_filter: 'tag:upgrade_happy_path & enabled:true' + environments: + - tmt: + context: +@@ -144,33 +144,42 @@ jobs: + TARGET_RELEASE: "8.6" + RHUI: "aws" + LEAPPDATA_BRANCH: "upstream" ++ LEAPP_NO_RHSM: "1" ++ USE_CUSTOM_REPOS: rhui + + - &sanity-79to88-aws + <<: *sanity-79to86-aws +- identifier: sanity-7.9to8.8-aws-e2e ++ identifier: sanity-7.9to8.8-aws + env: + SOURCE_RELEASE: "7.9" + TARGET_RELEASE: "8.8" + RHUI: "aws" + LEAPPDATA_BRANCH: "upstream" ++ LEAPP_NO_RHSM: "1" ++ USE_CUSTOM_REPOS: rhui + + - &sanity-79to89-aws + <<: *sanity-79to86-aws +- identifier: sanity-7.9to8.9-aws-e2e ++ identifier: sanity-7.9to8.9-aws + env: + SOURCE_RELEASE: "7.9" + TARGET_RELEASE: "8.9" + RHUI: "aws" + LEAPPDATA_BRANCH: "upstream" +- +-- &sanity-79to810-aws +- <<: *sanity-79to86-aws +- identifier: sanity-7.9to8.10-aws-e2e +- env: +- SOURCE_RELEASE: "7.9" +- TARGET_RELEASE: "8.10" +- RHUI: "aws" +- LEAPPDATA_BRANCH: "upstream" ++ LEAPP_NO_RHSM: "1" ++ USE_CUSTOM_REPOS: rhui ++ ++# NOTE(mkluson) RHEL 8.10 content is not publicly available (via RHUI) ++#- &sanity-79to810-aws ++# <<: *sanity-79to86-aws ++# identifier: sanity-7.9to8.10-aws ++# env: ++# SOURCE_RELEASE: "7.9" ++# TARGET_RELEASE: "8.10" ++# RHUI: "aws" ++# LEAPPDATA_BRANCH: "upstream" ++# LEAPP_NO_RHSM: "1" ++# USE_CUSTOM_REPOS: rhui + + # On-demand minimal beaker tests + - &beaker-minimal-79to86 +@@ -619,11 +628,11 @@ jobs: + targets: + epel-8-x86_64: + distros: [RHEL-8.6-rhui] +- identifier: sanity-8.6to9.0-aws-e2e ++ identifier: sanity-8.6to9.0-aws + tf_extra_params: + test: + tmt: +- plan_filter: 'tag:e2e & enabled:true' ++ plan_filter: 'tag:upgrade_happy_path & enabled:true' + environments: + - tmt: + context: +@@ -639,3 +648,5 @@ jobs: + RHSM_REPOS: "rhel-8-for-x86_64-appstream-eus-rpms,rhel-8-for-x86_64-baseos-eus-rpms" + RHUI: "aws" + LEAPPDATA_BRANCH: "upstream" ++ LEAPP_NO_RHSM: "1" ++ USE_CUSTOM_REPOS: rhui +-- +2.43.0 + diff --git a/0063-Update-upgrade-data-bump-required-data-stream-to-3.0.patch b/0063-Update-upgrade-data-bump-required-data-stream-to-3.0.patch new file mode 100644 index 0000000..51532b3 --- /dev/null +++ b/0063-Update-upgrade-data-bump-required-data-stream-to-3.0.patch @@ -0,0 +1,256 @@ +From 8552bbfd7418484b92327ded0d08a1849a693fe7 Mon Sep 17 00:00:00 2001 +From: Petr Stodulka +Date: Thu, 18 Jan 2024 14:51:34 +0100 +Subject: [PATCH 63/66] Update upgrade data + bump required data stream to 3.0 + +* Added RHEL 9 repos for Alibaba RHUI with mapping for IPU 8 -> 9 +* Actors require "3.0" in the list of provided_data_streams +* All data files updated to provide onls "3.0" data stream +* Add NL at the end of the device_driver_deprecation_data.json file + to be POSIX compatible as expected. +--- + .../files/device_driver_deprecation_data.json | 3 +- + etc/leapp/files/pes-events.json | 1 - + etc/leapp/files/repomap.json | 104 +++++++++++++++++- + .../common/libraries/config/__init__.py | 2 +- + 4 files changed, 103 insertions(+), 7 deletions(-) + +diff --git a/etc/leapp/files/device_driver_deprecation_data.json b/etc/leapp/files/device_driver_deprecation_data.json +index 7d5f5c74..c570f8d4 100644 +--- a/etc/leapp/files/device_driver_deprecation_data.json ++++ b/etc/leapp/files/device_driver_deprecation_data.json +@@ -1,6 +1,5 @@ + { + "provided_data_streams": [ +- "2.0", + "3.0" + ], + "data": [ +@@ -5058,4 +5057,4 @@ + ] + } + ] +-} +\ No newline at end of file ++} +diff --git a/etc/leapp/files/pes-events.json b/etc/leapp/files/pes-events.json +index c89a1547..5b4b4f87 100644 +--- a/etc/leapp/files/pes-events.json ++++ b/etc/leapp/files/pes-events.json +@@ -500347,7 +500347,6 @@ null + } + ], + "provided_data_streams": [ +-"2.0", + "3.0" + ], + "timestamp": "202401101404Z" +diff --git a/etc/leapp/files/repomap.json b/etc/leapp/files/repomap.json +index 9b73f2d7..1c97b7de 100644 +--- a/etc/leapp/files/repomap.json ++++ b/etc/leapp/files/repomap.json +@@ -1,5 +1,5 @@ + { +- "datetime": "202307241553Z", ++ "datetime": "202401171742Z", + "version_format": "1.2.0", + "mapping": [ + { +@@ -225,6 +225,12 @@ + "target": [ + "rhel9-rhui-google-compute-engine-leapp" + ] ++ }, ++ { ++ "source": "rhel8-rhui-custom-client-at-alibaba", ++ "target": [ ++ "rhel9-rhui-custom-client-at-alibaba" ++ ] + } + ] + } +@@ -2855,6 +2861,14 @@ + "channel": "ga", + "repo_type": "rpm" + }, ++ { ++ "major_version": "9", ++ "repoid": "rhui-rhel-9-for-aarch64-baseos-rhui-rpms", ++ "arch": "aarch64", ++ "channel": "ga", ++ "repo_type": "rpm", ++ "rhui": "alibaba" ++ }, + { + "major_version": "9", + "repoid": "rhui-rhel-9-for-x86_64-baseos-e4s-rhui-rpms", +@@ -2870,6 +2884,14 @@ + "channel": "ga", + "repo_type": "rpm", + "rhui": "google" ++ }, ++ { ++ "major_version": "9", ++ "repoid": "rhui-rhel-9-for-x86_64-baseos-rhui-rpms", ++ "arch": "x86_64", ++ "channel": "ga", ++ "repo_type": "rpm", ++ "rhui": "alibaba" + } + ] + }, +@@ -3059,6 +3081,14 @@ + "channel": "ga", + "repo_type": "rpm" + }, ++ { ++ "major_version": "9", ++ "repoid": "rhui-rhel-9-for-aarch64-appstream-rhui-rpms", ++ "arch": "aarch64", ++ "channel": "ga", ++ "repo_type": "rpm", ++ "rhui": "alibaba" ++ }, + { + "major_version": "9", + "repoid": "rhui-rhel-9-for-x86_64-appstream-e4s-rhui-rpms", +@@ -3074,6 +3104,14 @@ + "channel": "ga", + "repo_type": "rpm", + "rhui": "google" ++ }, ++ { ++ "major_version": "9", ++ "repoid": "rhui-rhel-9-for-x86_64-appstream-rhui-rpms", ++ "arch": "x86_64", ++ "channel": "ga", ++ "repo_type": "rpm", ++ "rhui": "alibaba" + } + ] + }, +@@ -3188,6 +3226,14 @@ + "channel": "ga", + "repo_type": "rpm" + }, ++ { ++ "major_version": "9", ++ "repoid": "rhui-codeready-builder-for-rhel-9-aarch64-rhui-rpms", ++ "arch": "aarch64", ++ "channel": "ga", ++ "repo_type": "rpm", ++ "rhui": "alibaba" ++ }, + { + "major_version": "9", + "repoid": "rhui-codeready-builder-for-rhel-9-x86_64-rhui-rpms", +@@ -3195,6 +3241,14 @@ + "channel": "ga", + "repo_type": "rpm", + "rhui": "google" ++ }, ++ { ++ "major_version": "9", ++ "repoid": "rhui-codeready-builder-for-rhel-9-x86_64-rhui-rpms", ++ "arch": "x86_64", ++ "channel": "ga", ++ "repo_type": "rpm", ++ "rhui": "alibaba" + } + ] + }, +@@ -3301,6 +3355,14 @@ + "repo_type": "rpm", + "rhui": "aws" + }, ++ { ++ "major_version": "9", ++ "repoid": "rhui-rhel-9-for-aarch64-supplementary-rhui-rpms", ++ "arch": "aarch64", ++ "channel": "ga", ++ "repo_type": "rpm", ++ "rhui": "alibaba" ++ }, + { + "major_version": "9", + "repoid": "rhui-rhel-9-for-x86_64-supplementary-rhui-rpms", +@@ -3308,6 +3370,14 @@ + "channel": "ga", + "repo_type": "rpm", + "rhui": "google" ++ }, ++ { ++ "major_version": "9", ++ "repoid": "rhui-rhel-9-for-x86_64-supplementary-rhui-rpms", ++ "arch": "x86_64", ++ "channel": "ga", ++ "repo_type": "rpm", ++ "rhui": "alibaba" + } + ] + }, +@@ -3687,6 +3757,14 @@ + "channel": "ga", + "repo_type": "rpm", + "rhui": "google" ++ }, ++ { ++ "major_version": "9", ++ "repoid": "rhui-rhel-9-for-x86_64-highavailability-rhui-rpms", ++ "arch": "x86_64", ++ "channel": "ga", ++ "repo_type": "rpm", ++ "rhui": "alibaba" + } + ] + }, +@@ -3788,10 +3866,30 @@ + "rhui": "google" + } + ] ++ }, ++ { ++ "pesid": "rhel9-rhui-custom-client-at-alibaba", ++ "entries": [ ++ { ++ "major_version": "9", ++ "repoid": "rhui-custom-rhui_client_at_alibaba-rhel-9", ++ "arch": "aarch64", ++ "channel": "ga", ++ "repo_type": "rpm", ++ "rhui": "alibaba" ++ }, ++ { ++ "major_version": "9", ++ "repoid": "rhui-custom-rhui_client_at_alibaba-rhel-9", ++ "arch": "x86_64", ++ "channel": "ga", ++ "repo_type": "rpm", ++ "rhui": "alibaba" ++ } ++ ] + } + ], + "provided_data_streams": [ +- "2.0", + "3.0" + ] +-} +\ No newline at end of file ++} +diff --git a/repos/system_upgrade/common/libraries/config/__init__.py b/repos/system_upgrade/common/libraries/config/__init__.py +index b3697a4d..9757948e 100644 +--- a/repos/system_upgrade/common/libraries/config/__init__.py ++++ b/repos/system_upgrade/common/libraries/config/__init__.py +@@ -3,7 +3,7 @@ from leapp.libraries.stdlib import api + + # The devel variable for target product channel can also contain 'beta' + SUPPORTED_TARGET_CHANNELS = {'ga', 'e4s', 'eus', 'aus'} +-CONSUMED_DATA_STREAM_ID = '2.0' ++CONSUMED_DATA_STREAM_ID = '3.0' + + + def get_env(name, default=None): +-- +2.43.0 + diff --git a/0064-Cover-upgrades-RHEL-8-to-RHEL-9-using-RHUI-on-Alibab.patch b/0064-Cover-upgrades-RHEL-8-to-RHEL-9-using-RHUI-on-Alibab.patch new file mode 100644 index 0000000..bfd8c47 --- /dev/null +++ b/0064-Cover-upgrades-RHEL-8-to-RHEL-9-using-RHUI-on-Alibab.patch @@ -0,0 +1,38 @@ +From f0ff88658cfa5fb4c766fb2cc44dc93fbe08958d Mon Sep 17 00:00:00 2001 +From: "jinkangkang.jkk" +Date: Wed, 29 Nov 2023 11:18:11 +0800 +Subject: [PATCH 64/66] Cover upgrades RHEL 8 to RHEL 9 using RHUI on Alibaba + cloud + +Note the repomap.json file does not cover yet mapping of repositories +between RHEL 8 and RHEL 9 for aarch64 (ARM). This will be covered +by further update of the file. +--- + repos/system_upgrade/common/libraries/rhui.py | 11 +++++++++++ + 1 file changed, 11 insertions(+) + +diff --git a/repos/system_upgrade/common/libraries/rhui.py b/repos/system_upgrade/common/libraries/rhui.py +index b31eba0b..2dfb209c 100644 +--- a/repos/system_upgrade/common/libraries/rhui.py ++++ b/repos/system_upgrade/common/libraries/rhui.py +@@ -481,6 +481,17 @@ RHUI_CLOUD_MAP = { + ('leapp-google-sap.repo', YUM_REPOS_PATH) + ], + }, ++ 'alibaba': { ++ 'src_pkg': 'aliyun_rhui_rhel8', ++ 'target_pkg': 'aliyun_rhui_rhel9', ++ 'leapp_pkg': 'leapp-rhui-alibaba', ++ 'leapp_pkg_repo': 'leapp-alibaba.repo', ++ 'files_map': [ ++ ('content.crt', RHUI_PKI_PRODUCT_DIR), ++ ('key.pem', RHUI_PKI_DIR), ++ ('leapp-alibaba.repo', YUM_REPOS_PATH) ++ ], ++ }, + }, + } + +-- +2.43.0 + diff --git a/0065-load-data-files-do-not-try-to-download-data-files-wh.patch b/0065-load-data-files-do-not-try-to-download-data-files-wh.patch new file mode 100644 index 0000000..4f0330a --- /dev/null +++ b/0065-load-data-files-do-not-try-to-download-data-files-wh.patch @@ -0,0 +1,49 @@ +From a9e48f836e3bcc4e89f26c25a94b35195db2c043 Mon Sep 17 00:00:00 2001 +From: Petr Stodulka +Date: Wed, 23 Aug 2023 12:04:05 +0200 +Subject: [PATCH 65/66] load data files: do not try to download data files when + missing + +As the leapp upgrade data files are nowadays part of the install rpm, +there is no need to download them anymore. Also, we do not have any +plans to provide the updated data files anywhere out of the rpm. + +From that point, do not try to download provided data files anymore. +--- + repos/system_upgrade/common/libraries/fetch.py | 5 ++++- + 1 file changed, 4 insertions(+), 1 deletion(-) + +diff --git a/repos/system_upgrade/common/libraries/fetch.py b/repos/system_upgrade/common/libraries/fetch.py +index 1ca26170..42fcb74c 100644 +--- a/repos/system_upgrade/common/libraries/fetch.py ++++ b/repos/system_upgrade/common/libraries/fetch.py +@@ -20,6 +20,7 @@ def _get_hint(): + rpmname = 'leapp-upgrade-el{}toel{}'.format(get_source_major_version(), get_target_major_version()) + hint = ( + 'All official data files are nowadays part of the installed rpms.' ++ ' That is the only official resource of actual official data files for in-place upgrades.' + ' This issue is usually encountered when the data files are incorrectly customized, replaced, or removed' + ' (e.g. by custom scripts).' + ' In case you want to recover the original file, remove it (if still exists)' +@@ -33,7 +34,9 @@ def _raise_error(local_path, details): + """ + If the file acquisition fails in any way, throw an informative error to stop the actor. + """ ++ rpmname = 'leapp-upgrade-el{}toel{}'.format(get_source_major_version(), get_target_major_version()) + summary = 'Data file {lp} is missing or invalid.'.format(lp=local_path) ++ + raise StopActorExecutionError(summary, details={'details': details, 'hint': _get_hint()}) + + +@@ -174,7 +177,7 @@ def load_data_asset(actor_requesting_asset, + + try: + # The asset family ID has the form (major, minor), include only `major` in the URL +- raw_asset_contents = read_or_fetch(asset_filename, data_stream=data_stream_major) ++ raw_asset_contents = read_or_fetch(asset_filename, data_stream=data_stream_major, allow_download=False) + asset_contents = json.loads(raw_asset_contents) + except ValueError: + msg = 'The {0} file (at {1}) does not contain a valid JSON object.'.format(asset_fulltext_name, asset_filename) +-- +2.43.0 + diff --git a/0066-upgrade-data-files-loading-update-error-msgs-and-rep.patch b/0066-upgrade-data-files-loading-update-error-msgs-and-rep.patch new file mode 100644 index 0000000..48ff6e0 --- /dev/null +++ b/0066-upgrade-data-files-loading-update-error-msgs-and-rep.patch @@ -0,0 +1,319 @@ +From 353cd03d5339a6f3905f8bc4f067e0758f6e1d78 Mon Sep 17 00:00:00 2001 +From: Petr Stodulka +Date: Wed, 17 Jan 2024 21:46:19 +0100 +Subject: [PATCH 66/66] upgrade data files loading: update error msgs and + repors + minor changes + +Updated number of error messages and reports to be sure that users +know what files are actually problematic. Original errors and reports +usually didn't contain the full path to an upgrade data file due to +possibility to download the file from a server. However, the posibility +to download fresh data files from a requested server is not expected +to support in the current state as the data files are part of provided +packages. + +I've been thinking quite long time whether to actually drop +or deprecate bigger part of the code to simplify the whole solution, +as currently it's not planned to have a possibility to download +some data files from servers in future. However, thinking about +upcoming challenges, I am not totally persuaded that we will not +revive that functionality in future, or that we will not want to +use it for something little bit different. From that POV (and late +phase of development prior the planned release) I think it will be +better to preserve it for now and raise a discussion about it later. + +Other changes in this PR: + * drop hardcoded name of the leapp-upgrade-elXtoelY rpm and use + the leapp.libraries.common.rpms.get_leapp_packages() function + * replace REPOSITORY group by SANITY; it was originally a mixture + of both and SANITY fits better to me from this point + * the check of consumed data sets could produce report with empty + links, as the original article(s) we referred to have been obsoleted; + so added filter for missing URLs + +Co-authored-by: Toshio Kuratomi +--- + .../libraries/check_consumed_assets.py | 31 ++++++++++++++++--- + .../deviceanddriverdeprecationdataload.py | 3 ++ + .../libraries/pes_event_parsing.py | 24 +++++++++----- + .../libraries/repositoriesmapping.py | 21 +++++++------ + .../system_upgrade/common/libraries/fetch.py | 28 ++++++++++------- + 5 files changed, 75 insertions(+), 32 deletions(-) + +diff --git a/repos/system_upgrade/common/actors/checkconsumedassets/libraries/check_consumed_assets.py b/repos/system_upgrade/common/actors/checkconsumedassets/libraries/check_consumed_assets.py +index f5998de0..1558c2fc 100644 +--- a/repos/system_upgrade/common/actors/checkconsumedassets/libraries/check_consumed_assets.py ++++ b/repos/system_upgrade/common/actors/checkconsumedassets/libraries/check_consumed_assets.py +@@ -4,10 +4,27 @@ from collections import defaultdict, namedtuple + from leapp import reporting + from leapp.libraries.common.config import get_consumed_data_stream_id + from leapp.libraries.common.fetch import ASSET_PROVIDED_DATA_STREAMS_FIELD ++from leapp.libraries.common.rpms import get_leapp_packages, LeappComponents + from leapp.libraries.stdlib import api + from leapp.models import ConsumedDataAsset + + ++def _get_hint(): ++ hint = ( ++ 'All official assets (data files) are part of the installed rpms these days.' ++ ' This issue is usually encountered when the data files are incorrectly' ++ ' customized, replaced, or removed. ' ++ ' In case you want to recover the original files, remove them (if they still exist)' ++ ' and reinstall the following rpms: {rpms}.\n' ++ 'The listed assets (data files) are usually inside the /etc/leapp/files/' ++ ' directory.' ++ .format( ++ rpms=', '.join(get_leapp_packages(component=LeappComponents.REPOSITORY)) ++ ) ++ ) ++ return hint ++ ++ + def compose_summary_for_incompatible_assets(assets, incompatibility_reason): + if not assets: + return [] +@@ -69,13 +86,16 @@ def report_incompatible_assets(assets): + summary_lines += compose_summary_for_incompatible_assets(incompatible_assets, reason) + + for asset in incompatible_assets: +- doc_url_to_title[asset.docs_url].append(asset.docs_title) ++ if asset.docs_url: ++ # Add URLs only when they are specified. docs_url could be empty string ++ doc_url_to_title[asset.docs_url].append(asset.docs_title) + + report_parts = [ + reporting.Title(title), + reporting.Summary('\n'.join(summary_lines)), + reporting.Severity(reporting.Severity.HIGH), +- reporting.Groups([reporting.Groups.INHIBITOR, reporting.Groups.REPOSITORY]), ++ reporting.Remediation(hint=_get_hint()), ++ reporting.Groups([reporting.Groups.INHIBITOR, reporting.Groups.SANITY]), + ] + + report_parts += make_report_entries_with_unique_urls(docs_url_to_title_map) +@@ -101,13 +121,16 @@ def report_malformed_assets(malformed_assets): + details = ' - The asset file {filename} contains invalid value in its "{data_streams_field}"' + details = details.format(filename=asset.filename, data_streams_field=ASSET_PROVIDED_DATA_STREAMS_FIELD) + summary_lines.append(details) +- docs_url_to_title_map[asset.docs_url].append(asset.docs_title) ++ if asset.docs_url: ++ # Add URLs only when they are specified. docs_url could be empty string ++ docs_url_to_title_map[asset.docs_url].append(asset.docs_title) + + report_parts = [ + reporting.Title(title), + reporting.Summary('\n'.join(summary_lines)), ++ reporting.Remediation(hint=_get_hint()), + reporting.Severity(reporting.Severity.HIGH), +- reporting.Groups([reporting.Groups.INHIBITOR, reporting.Groups.REPOSITORY]), ++ reporting.Groups([reporting.Groups.INHIBITOR, reporting.Groups.SANITY]), + ] + + report_parts += make_report_entries_with_unique_urls(docs_url_to_title_map) +diff --git a/repos/system_upgrade/common/actors/loaddevicedriverdeprecationdata/libraries/deviceanddriverdeprecationdataload.py b/repos/system_upgrade/common/actors/loaddevicedriverdeprecationdata/libraries/deviceanddriverdeprecationdataload.py +index 3caa4e0a..f422c2c3 100644 +--- a/repos/system_upgrade/common/actors/loaddevicedriverdeprecationdata/libraries/deviceanddriverdeprecationdataload.py ++++ b/repos/system_upgrade/common/actors/loaddevicedriverdeprecationdata/libraries/deviceanddriverdeprecationdataload.py +@@ -13,6 +13,9 @@ def process(): + supported_device_types = set(DeviceDriverDeprecationEntry.device_type.serialize()['choices']) + + data_file_name = 'device_driver_deprecation_data.json' ++ # NOTE(pstodulk): load_data_assert raises StopActorExecutionError, see ++ # the code for more info. Keeping the handling on the framework in such ++ # a case as we have no work to do in such a case here. + deprecation_data = fetch.load_data_asset(api.current_actor(), + data_file_name, + asset_fulltext_name='Device driver deprecation data', +diff --git a/repos/system_upgrade/common/actors/peseventsscanner/libraries/pes_event_parsing.py b/repos/system_upgrade/common/actors/peseventsscanner/libraries/pes_event_parsing.py +index 35bcec73..f24dda68 100644 +--- a/repos/system_upgrade/common/actors/peseventsscanner/libraries/pes_event_parsing.py ++++ b/repos/system_upgrade/common/actors/peseventsscanner/libraries/pes_event_parsing.py +@@ -8,7 +8,7 @@ from leapp import reporting + from leapp.exceptions import StopActorExecution + from leapp.libraries.common import fetch + from leapp.libraries.common.config import architecture +-from leapp.libraries.common.config.version import get_source_major_version, get_target_major_version ++from leapp.libraries.common.rpms import get_leapp_packages, LeappComponents + from leapp.libraries.stdlib import api + + # NOTE(mhecko): The modulestream field contains a set of modulestreams until the very end when we generate a Package +@@ -67,6 +67,9 @@ def get_pes_events(pes_json_directory, pes_json_filename): + :return: List of Event tuples, where each event contains event type and input/output pkgs + """ + try: ++ # NOTE(pstodulk): load_data_assert raises StopActorExecutionError, see ++ # the code for more info. Keeping the handling on the framework in such ++ # a case as we have no work to do in such a case here. + events_data = fetch.load_data_asset(api.current_actor(), + pes_json_filename, + asset_fulltext_name='PES events file', +@@ -83,22 +86,27 @@ def get_pes_events(pes_json_directory, pes_json_filename): + events_matching_arch = [e for e in all_events if not e.architectures or arch in e.architectures] + return events_matching_arch + except (ValueError, KeyError): +- rpmname = 'leapp-upgrade-el{}toel{}'.format(get_source_major_version(), get_target_major_version()) +- title = 'Missing/Invalid PES data file ({}/{})'.format(pes_json_directory, pes_json_filename) ++ local_path = os.path.join(pes_json_directory, pes_json_filename) ++ title = 'Missing/Invalid PES data file ({})'.format(local_path) + summary = ( + 'All official data files are nowadays part of the installed rpms.' + ' This issue is usually encountered when the data files are incorrectly customized, replaced, or removed' + ' (e.g. by custom scripts).' +- ' In case you want to recover the original file, remove it (if still exists)' +- ' and reinstall the {} rpm.' +- .format(rpmname) ++ ) ++ hint = ( ++ ' In case you want to recover the original {lp} file, remove it (if it still exists)' ++ ' and reinstall the following rpms: {rpms}.' ++ .format( ++ lp=local_path, ++ rpms=', '.join(get_leapp_packages(component=LeappComponents.REPOSITORY)) ++ ) + ) + reporting.create_report([ + reporting.Title(title), + reporting.Summary(summary), ++ reporting.Remediation(hint=hint), + reporting.Severity(reporting.Severity.HIGH), +- reporting.Groups([reporting.Groups.SANITY]), +- reporting.Groups([reporting.Groups.INHIBITOR]), ++ reporting.Groups([reporting.Groups.SANITY, reporting.Groups.INHIBITOR]), + reporting.RelatedResource('file', os.path.join(pes_json_directory, pes_json_filename)) + ]) + raise StopActorExecution() +diff --git a/repos/system_upgrade/common/actors/repositoriesmapping/libraries/repositoriesmapping.py b/repos/system_upgrade/common/actors/repositoriesmapping/libraries/repositoriesmapping.py +index 6f2b2e0f..8045634e 100644 +--- a/repos/system_upgrade/common/actors/repositoriesmapping/libraries/repositoriesmapping.py ++++ b/repos/system_upgrade/common/actors/repositoriesmapping/libraries/repositoriesmapping.py +@@ -4,6 +4,7 @@ from collections import defaultdict + from leapp.exceptions import StopActorExecutionError + from leapp.libraries.common.config.version import get_source_major_version, get_target_major_version + from leapp.libraries.common.fetch import load_data_asset ++from leapp.libraries.common.rpms import get_leapp_packages, LeappComponents + from leapp.libraries.stdlib import api + from leapp.models import PESIDRepositoryEntry, RepoMapEntry, RepositoriesMapping + from leapp.models.fields import ModelViolationError +@@ -130,29 +131,31 @@ class RepoMapData(object): + + + def _inhibit_upgrade(msg): +- rpmname = 'leapp-upgrade-el{}toel{}'.format(get_source_major_version(), get_target_major_version()) ++ local_path = os.path.join('/etc/leapp/file', REPOMAP_FILE) + hint = ( + 'All official data files are nowadays part of the installed rpms.' + ' This issue is usually encountered when the data files are incorrectly customized, replaced, or removed' + ' (e.g. by custom scripts).' +- ' In case you want to recover the original file, remove it (if still exists)' +- ' and reinstall the {} rpm.' +- .format(rpmname) ++ ' In case you want to recover the original {lp} file, remove the current one (if it still exists)' ++ ' and reinstall the following packages: {rpms}.' ++ .format( ++ lp=local_path, ++ rpms=', '.join(get_leapp_packages(component=LeappComponents.REPOSITORY)) ++ ) + ) + raise StopActorExecutionError(msg, details={'hint': hint}) + + + def _read_repofile(repofile): +- # NOTE: what about catch StopActorExecution error when the file cannot be +- # obtained -> then check whether old_repomap file exists and in such a case +- # inform user they have to provide the new repomap.json file (we have the +- # warning now only which could be potentially overlooked) ++ # NOTE(pstodulk): load_data_assert raises StopActorExecutionError, see ++ # the code for more info. Keeping the handling on the framework in such ++ # a case as we have no work to do in such a case here. + repofile_data = load_data_asset(api.current_actor(), + repofile, + asset_fulltext_name='Repositories mapping', + docs_url='', + docs_title='') +- return repofile_data # If the file does not contain a valid json then load_asset will do a stop actor execution ++ return repofile_data + + + def scan_repositories(read_repofile_func=_read_repofile): +diff --git a/repos/system_upgrade/common/libraries/fetch.py b/repos/system_upgrade/common/libraries/fetch.py +index 42fcb74c..82bf4ff3 100644 +--- a/repos/system_upgrade/common/libraries/fetch.py ++++ b/repos/system_upgrade/common/libraries/fetch.py +@@ -7,7 +7,7 @@ import requests + from leapp import models + from leapp.exceptions import StopActorExecutionError + from leapp.libraries.common.config import get_consumed_data_stream_id, get_env +-from leapp.libraries.common.config.version import get_source_major_version, get_target_major_version ++from leapp.libraries.common.rpms import get_leapp_packages, LeappComponents + from leapp.libraries.stdlib import api + + SERVICE_HOST_DEFAULT = "https://cert.cloud.redhat.com" +@@ -16,16 +16,18 @@ MAX_ATTEMPTS = 3 + ASSET_PROVIDED_DATA_STREAMS_FIELD = 'provided_data_streams' + + +-def _get_hint(): +- rpmname = 'leapp-upgrade-el{}toel{}'.format(get_source_major_version(), get_target_major_version()) ++def _get_hint(local_path): + hint = ( +- 'All official data files are nowadays part of the installed rpms.' +- ' That is the only official resource of actual official data files for in-place upgrades.' ++ 'All official data files are part of the installed rpms these days.' ++ ' The rpm is the only official source of the official data files for in-place upgrades.' + ' This issue is usually encountered when the data files are incorrectly customized, replaced, or removed' + ' (e.g. by custom scripts).' +- ' In case you want to recover the original file, remove it (if still exists)' +- ' and reinstall the {} rpm.' +- .format(rpmname) ++ ' In case you want to recover the original {lp} file, remove the current one (if it still exists)' ++ ' and reinstall the following packages: {rpms}.' ++ .format( ++ lp=local_path, ++ rpms=', '.join(get_leapp_packages(component=LeappComponents.REPOSITORY)) ++ ) + ) + return hint + +@@ -34,10 +36,9 @@ def _raise_error(local_path, details): + """ + If the file acquisition fails in any way, throw an informative error to stop the actor. + """ +- rpmname = 'leapp-upgrade-el{}toel{}'.format(get_source_major_version(), get_target_major_version()) + summary = 'Data file {lp} is missing or invalid.'.format(lp=local_path) + +- raise StopActorExecutionError(summary, details={'details': details, 'hint': _get_hint()}) ++ raise StopActorExecutionError(summary, details={'details': details, 'hint': _get_hint(local_path)}) + + + def _request_data(service_path, cert, proxies, timeout=REQUEST_TIMEOUT): +@@ -148,6 +149,7 @@ def load_data_asset(actor_requesting_asset, + docs_title): + """ + Load the content of the data asset with given asset_filename ++ and produce :class:`leapp.model.ConsumedDataAsset` message. + + :param Actor actor_requesting_asset: The actor instance requesting the asset file. It is necessary for the actor + to be able to produce ConsumedDataAsset message in order for leapp to be able +@@ -157,6 +159,10 @@ def load_data_asset(actor_requesting_asset, + :param str docs_url: Docs url to provide if an asset is malformed or outdated. + :param str docs_title: Title of the documentation to where `docs_url` points to. + :returns: A dict with asset contents (a parsed JSON), or None if the asset was outdated. ++ :raises StopActorExecutionError: In following cases: ++ * ConsumedDataAsset is not specified in the produces tuple of the actor_requesting_asset actor ++ * The content of the required data file is not valid JSON format ++ * The required data cannot be obtained (e.g. due to missing file) + """ + + # Check that the actor that is attempting to obtain the asset meets the contract to call this function +@@ -167,7 +173,7 @@ def load_data_asset(actor_requesting_asset, + error_hint = {'hint': ('Read documentation at the following link for more information about how to retrieve ' + 'the valid file: {0}'.format(docs_url))} + else: +- error_hint = {'hint': _get_hint()} ++ error_hint = {'hint': _get_hint(os.path.join('/etc/leapp/files', asset_filename))} + + data_stream_id = get_consumed_data_stream_id() + data_stream_major = data_stream_id.split('.', 1)[0] +-- +2.43.0 + diff --git a/leapp-repository.spec b/leapp-repository.spec index 63fafbc..133c38c 100644 --- a/leapp-repository.spec +++ b/leapp-repository.spec @@ -42,7 +42,7 @@ py2_byte_compile "%1" "%2"} Name: leapp-repository Version: 0.19.0 -Release: 7%{?dist} +Release: 8%{?dist} Summary: Repositories for leapp License: ASL 2.0 @@ -119,6 +119,13 @@ Patch0059: 0059-Introduce-custom-modifications-tracking.patch Patch0060: 0060-Rework-_copy_decouple-to-follow-relative-symlinks-an.patch Patch0061: 0061-Update-the-data-files-pes-repomap-dddd-CTC2-0.patch +# CTC2-1 +Patch0062: 0062-Use-happy_path-instead-e2e-for-public-clouds.patch +Patch0063: 0063-Update-upgrade-data-bump-required-data-stream-to-3.0.patch +Patch0064: 0064-Cover-upgrades-RHEL-8-to-RHEL-9-using-RHUI-on-Alibab.patch +Patch0065: 0065-load-data-files-do-not-try-to-download-data-files-wh.patch +Patch0066: 0066-upgrade-data-files-loading-update-error-msgs-and-rep.patch + %description %{summary} @@ -331,6 +338,11 @@ Requires: python3-gobject-base %patch0059 -p1 %patch0060 -p1 %patch0061 -p1 +%patch0062 -p1 +%patch0063 -p1 +%patch0064 -p1 +%patch0065 -p1 +%patch0066 -p1 %build @@ -408,6 +420,14 @@ done; # no files here %changelog +* Thu Jan 18 2024 Petr Stodulka - 0.19.0-8 +- Do not try to download data files anymore when missing as the service + is obsoleted since the data is part of installed packages +- Update error messages and reports when installed upgrade data files + are malformed or missing to instruct user how to resolve it +- Update the leapp upgrade data files - bump data stream to "3.0" +- Resolves: RHEL-18298 + * Fri Jan 12 2024 Petr Stodulka - 0.19.0-7 - Add detection of possible usage of OpenSSL IBMCA engine on IBM Z machines - Add detection of modified /etc/pki/tls/openssl.cnf file