From 257b6fe19ed11c765e462e0a35ecb66edfcc8199 Mon Sep 17 00:00:00 2001 From: Petr Stodulka Date: Mon, 17 Jul 2023 22:15:25 +0200 Subject: [PATCH] CTC2 builds - Bump leapp-framework to 4.0 - Improve the report summary output to make it more visible - Fix processing data in remediation instructions with non-ascii characters - Fix creation of Dialog for Component without choices - Store tracebacks from actors in leapp.db - Resolves: #2223312 --- ...-upgrade-paths-and-env-vars-in-tests.patch | 68 ++++++ 0002-Make-copr-build-functioning-again.patch | 75 ++++++ ...-not-set-COPR_CHROOTS-for-copr-build.patch | 54 +++++ 0004-Post-release-doc-update-817.patch | 215 ++++++++++++++++++ ...-of-a-remediation-with-non-ascii-dat.patch | 201 ++++++++++++++++ ...ease-mention-from-welcome-PR-message.patch | 29 +++ ...fails-for-a-Component-without-choice.patch | 28 +++ ...-differnt-types-of-Dialog-components.patch | 106 +++++++++ 0009-Generate-empty-report-correctly.patch | 51 +++++ ...wline-in-the-end-of-the-report-json-.patch | 25 ++ 0011-terminology-fixes.patch | 159 +++++++++++++ ...-build-for-rerun-42-functional-again.patch | 30 +++ 0013-fix-spelling-errors-in-docs.patch | 152 +++++++++++++ 0014-Fix-invalid-type-check.patch | 27 +++ 0015-Save-actor-tracebacks-in-leapp.db.patch | 59 +++++ 0016-squashme-Log-actor-crash.patch | 29 +++ 0017-squashme-Put-trace-in-the-details.patch | 27 +++ 0018-Add-reports-summary-to-the-output.patch | 138 +++++++++++ leapp.spec | 48 +++- 19 files changed, 1519 insertions(+), 2 deletions(-) create mode 100644 0001-Update-upgrade-paths-and-env-vars-in-tests.patch create mode 100644 0002-Make-copr-build-functioning-again.patch create mode 100644 0003-Do-not-set-COPR_CHROOTS-for-copr-build.patch create mode 100644 0004-Post-release-doc-update-817.patch create mode 100644 0005-Improve-handling-of-a-remediation-with-non-ascii-dat.patch create mode 100644 0006-Remove-review-please-mention-from-welcome-PR-message.patch create mode 100644 0007-Dialog-creation-fails-for-a-Component-without-choice.patch create mode 100644 0008-Test-differnt-types-of-Dialog-components.patch create mode 100644 0009-Generate-empty-report-correctly.patch create mode 100644 0010-Add-the-empty-newline-in-the-end-of-the-report-json-.patch create mode 100644 0011-terminology-fixes.patch create mode 100644 0012-Make-copr-build-for-rerun-42-functional-again.patch create mode 100644 0013-fix-spelling-errors-in-docs.patch create mode 100644 0014-Fix-invalid-type-check.patch create mode 100644 0015-Save-actor-tracebacks-in-leapp.db.patch create mode 100644 0016-squashme-Log-actor-crash.patch create mode 100644 0017-squashme-Put-trace-in-the-details.patch create mode 100644 0018-Add-reports-summary-to-the-output.patch diff --git a/0001-Update-upgrade-paths-and-env-vars-in-tests.patch b/0001-Update-upgrade-paths-and-env-vars-in-tests.patch new file mode 100644 index 0000000..3787ec1 --- /dev/null +++ b/0001-Update-upgrade-paths-and-env-vars-in-tests.patch @@ -0,0 +1,68 @@ +From e45ac20d1fef4a68a8b76ea00282a8f5e4c4b17b Mon Sep 17 00:00:00 2001 +From: Inessa Vasilevskaya +Date: Wed, 22 Mar 2023 10:26:21 +0100 +Subject: [PATCH 01/18] Update upgrade paths and env vars in tests + +- 8.7->9.0 substituted with 8.8->9.0 +- removed unnecessary TARGET_KERNEL env var +- set beta for 8.8->9.2 upgrade +--- + .github/workflows/tmt-tests.yml | 16 ++++++++-------- + 1 file changed, 8 insertions(+), 8 deletions(-) + +diff --git a/.github/workflows/tmt-tests.yml b/.github/workflows/tmt-tests.yml +index 634d790..d994f31 100644 +--- a/.github/workflows/tmt-tests.yml ++++ b/.github/workflows/tmt-tests.yml +@@ -76,24 +76,24 @@ jobs: + with: + copr_artifacts: ${{ needs.call_workflow_copr_build.outputs.artifacts }} + tmt_plan_regex: "^(?!.*upgrade_plugin)(?!.*tier[2-3].*)(?!.*rhsm)(?!.*c2r)(?!.*sap)(?!.*7to8)(?!.*max_sst)" +- variables: 'SOURCE_RELEASE=8.6;TARGET_RELEASE=9.0;TARGET_KERNEL=el9;RHSM_REPOS=rhel-8-for-x86_64-appstream-eus-rpms,rhel-8-for-x86_64-baseos-eus-rpms;LEAPPDATA_BRANCH=upstream' ++ variables: 'SOURCE_RELEASE=8.6;TARGET_RELEASE=9.0;RHSM_REPOS=rhel-8-for-x86_64-appstream-eus-rpms,rhel-8-for-x86_64-baseos-eus-rpms;LEAPPDATA_BRANCH=upstream' + pull_request_status_name: "8.6to9.0" + if: | + github.event.issue.pull_request + && ! startsWith(github.event.comment.body, '/rerun-sst') + && contains(fromJson('["OWNER", "MEMBER", "COLLABORATOR"]'), github.event.comment.author_association) + +- call_workflow_tests_87to90_integration: ++ call_workflow_tests_88to92_integration: + needs: call_workflow_copr_build + uses: ./.github/workflows/reuse-tests-8to9.yml + secrets: inherit + with: + copr_artifacts: ${{ needs.call_workflow_copr_build.outputs.artifacts }} + tmt_plan_regex: "^(?!.*upgrade_plugin)(?!.*tier[2-3].*)(?!.*rhsm)(?!.*c2r)(?!.*sap)(?!.*7to8)(?!.*max_sst)" +- variables: 'SOURCE_RELEASE=8.7;TARGET_RELEASE=9.0;TARGET_KERNEL=el9;RHSM_REPOS=rhel-8-for-x86_64-appstream-rpms,rhel-8-for-x86_64-baseos-rpms;LEAPPDATA_BRANCH=upstream' +- compose: 'RHEL-8.7.0-Nightly' +- pull_request_status_name: "8.7to9.0" +- tmt_context: "distro=rhel-8.7" ++ variables: 'SOURCE_RELEASE=8.8;TARGET_RELEASE=9.2;LEAPP_DEVEL_TARGET_RELEASE=9.2;RHSM_REPOS=rhel-8-for-x86_64-appstream-beta-rpms,rhel-8-for-x86_64-baseos-beta-rpms;LEAPPDATA_BRANCH=upstream' ++ compose: "RHEL-8.8.0-Nightly" ++ pull_request_status_name: "8.8to9.2" ++ tmt_context: "distro=rhel-8.8" + if: | + github.event.issue.pull_request + && ! startsWith(github.event.comment.body, '/rerun-sst') +@@ -106,7 +106,7 @@ jobs: + with: + copr_artifacts: ${{ needs.call_workflow_copr_build.outputs.artifacts }} + tmt_plan_regex: "^(?!.*tier[2-3].*)(.*max_sst.*)" +- variables: 'SOURCE_RELEASE=8.6;TARGET_RELEASE=9.0;TARGET_KERNEL=el9;RHSM_REPOS=rhel-8-for-x86_64-appstream-eus-rpms,rhel-8-for-x86_64-baseos-eus-rpms;LEAPPDATA_BRANCH=upstream' ++ variables: 'SOURCE_RELEASE=8.6;TARGET_RELEASE=9.0;RHSM_REPOS=rhel-8-for-x86_64-appstream-eus-rpms,rhel-8-for-x86_64-baseos-eus-rpms;LEAPPDATA_BRANCH=upstream' + pull_request_status_name: "8.6to9.0-sst" + update_pull_request_status: 'false' + if: | +@@ -124,7 +124,7 @@ jobs: + compose: "RHEL-8.6-rhui" + environment_settings: '{"provisioning": {"post_install_script": "#!/bin/sh\nsudo sed -i s/.*ssh-rsa/ssh-rsa/ /root/.ssh/authorized_keys"}}' + pull_request_status_name: "8to9-aws-e2e" +- variables: 'SOURCE_RELEASE=8.6;TARGET_RELEASE=9.0;TARGET_KERNEL=el9;RHSM_REPOS=rhel-8-for-x86_64-appstream-eus-rpms,rhel-8-for-x86_64-baseos-eus-rpms;RHUI=aws;LEAPPDATA_BRANCH=upstream' ++ variables: 'SOURCE_RELEASE=8.6;TARGET_RELEASE=9.0;RHSM_REPOS=rhel-8-for-x86_64-appstream-eus-rpms,rhel-8-for-x86_64-baseos-eus-rpms;RHUI=aws;LEAPPDATA_BRANCH=upstream' + if: | + github.event.issue.pull_request + && ! startsWith(github.event.comment.body, '/rerun-sst') +-- +2.41.0 + diff --git a/0002-Make-copr-build-functioning-again.patch b/0002-Make-copr-build-functioning-again.patch new file mode 100644 index 0000000..0d084eb --- /dev/null +++ b/0002-Make-copr-build-functioning-again.patch @@ -0,0 +1,75 @@ +From 5ddce5a248ec59d6f8294aae95b5ce6732464acc Mon Sep 17 00:00:00 2001 +From: Inessa Vasilevskaya +Date: Thu, 13 Apr 2023 19:52:56 +0200 +Subject: [PATCH 02/18] Make copr-build functioning again + +After some unknown changes around COPR, the building +command and the used COPR configuration file needs to be +updated. + +OAMG-8876 +--- + .github/workflows/reuse-copr-build.yml | 16 ++++++++++++---- + 1 file changed, 12 insertions(+), 4 deletions(-) + +diff --git a/.github/workflows/reuse-copr-build.yml b/.github/workflows/reuse-copr-build.yml +index 19b17ca..138c403 100644 +--- a/.github/workflows/reuse-copr-build.yml ++++ b/.github/workflows/reuse-copr-build.yml +@@ -66,7 +66,7 @@ jobs: + cat << EOF > copr_fedora.conf + [copr-cli] + login = ${{ secrets.FEDORA_COPR_LOGIN }} +- username = @oamg ++ username = oamgbot + token = ${{ secrets.FEDORA_COPR_TOKEN }} + copr_url = https://copr.fedorainfracloud.org + # expiration date: 2030-07-04 +@@ -79,18 +79,22 @@ jobs: + + - name: Trigger copr build + id: copr_build ++ env: ++ COPR_CONFIG: "copr_fedora.conf" ++ COPR_CHROOT: "epel-7-x86_64,epel-8-x86_64" ++ COPR_REPO: "@oamg/leapp" + run: | + cat << EOF > copr_fedora.conf + [copr-cli] + login = ${{ secrets.FEDORA_COPR_LOGIN }} +- username = @oamg ++ username = oamgbot + token = ${{ secrets.FEDORA_COPR_TOKEN }} + copr_url = https://copr.fedorainfracloud.org + # expiration date: 2030-07-04 + EOF + + pip install copr-cli +- PR=${{ steps.pr_nr.outputs.pr_nr }} COPR_CONFIG=copr_fedora.conf make copr_build | tee copr.log ++ PR=${{ steps.pr_nr.outputs.pr_nr }} COPR_CHROOT=$COPR_CHROOT COPR_REPO="$COPR_REPO" COPR_CONFIG=$COPR_CONFIG make copr_build | tee copr.log + + COPR_URL=$(grep -Po 'https://copr.fedorainfracloud.org/coprs/build/\d+' copr.log) + echo "::set-output name=copr_url::${COPR_URL}" +@@ -133,6 +137,10 @@ jobs: + - name: Trigger copr build for leapp-repository + id: copr_build_leapp_repository + if: ${{ steps.leapp_repository_pr_regex_match.outputs.match != '' }} ++ env: ++ COPR_CONFIG: "copr_fedora.conf" ++ COPR_CHROOT: "epel-7-x86_64,epel-8-x86_64" ++ COPR_REPO: "@oamg/leapp" + run: | + cat << EOF > copr_fedora.conf + [copr-cli] +@@ -144,7 +152,7 @@ jobs: + EOF + + pip install copr-cli +- PR=${{ steps.leapp_repository_pr.outputs.leapp_repository_pr }} COPR_CONFIG=copr_fedora.conf make copr_build | tee copr.log ++ PR=${{ steps.leapp_repository_pr.outputs.leapp_repository_pr }} COPR_CHROOT=COPR_CHROOT COPR_REPO="$COPR_REPO" COPR_CONFIG=$COPR_CONFIG make copr_build | tee copr.log + + COPR_URL=$(grep -Po 'https://copr.fedorainfracloud.org/coprs/build/\d+' copr.log) + echo "::set-output name=copr_url::${COPR_URL}" +-- +2.41.0 + diff --git a/0003-Do-not-set-COPR_CHROOTS-for-copr-build.patch b/0003-Do-not-set-COPR_CHROOTS-for-copr-build.patch new file mode 100644 index 0000000..70ff6ac --- /dev/null +++ b/0003-Do-not-set-COPR_CHROOTS-for-copr-build.patch @@ -0,0 +1,54 @@ +From f4a5a3c7f94f436b6814945539a562cbe5d25e87 Mon Sep 17 00:00:00 2001 +From: Inessa Vasilevskaya +Date: Fri, 14 Apr 2023 14:26:21 +0200 +Subject: [PATCH 03/18] Do not set COPR_CHROOTS for copr-build + +Previous patch follow up: in case of leapp this +should not be specified. + +OAMG-8876 +--- + .github/workflows/reuse-copr-build.yml | 6 ++---- + 1 file changed, 2 insertions(+), 4 deletions(-) + +diff --git a/.github/workflows/reuse-copr-build.yml b/.github/workflows/reuse-copr-build.yml +index 138c403..76c1f67 100644 +--- a/.github/workflows/reuse-copr-build.yml ++++ b/.github/workflows/reuse-copr-build.yml +@@ -81,7 +81,6 @@ jobs: + id: copr_build + env: + COPR_CONFIG: "copr_fedora.conf" +- COPR_CHROOT: "epel-7-x86_64,epel-8-x86_64" + COPR_REPO: "@oamg/leapp" + run: | + cat << EOF > copr_fedora.conf +@@ -94,7 +93,7 @@ jobs: + EOF + + pip install copr-cli +- PR=${{ steps.pr_nr.outputs.pr_nr }} COPR_CHROOT=$COPR_CHROOT COPR_REPO="$COPR_REPO" COPR_CONFIG=$COPR_CONFIG make copr_build | tee copr.log ++ PR=${{ steps.pr_nr.outputs.pr_nr }} COPR_REPO="$COPR_REPO" COPR_CONFIG=$COPR_CONFIG make copr_build | tee copr.log + + COPR_URL=$(grep -Po 'https://copr.fedorainfracloud.org/coprs/build/\d+' copr.log) + echo "::set-output name=copr_url::${COPR_URL}" +@@ -139,7 +138,6 @@ jobs: + if: ${{ steps.leapp_repository_pr_regex_match.outputs.match != '' }} + env: + COPR_CONFIG: "copr_fedora.conf" +- COPR_CHROOT: "epel-7-x86_64,epel-8-x86_64" + COPR_REPO: "@oamg/leapp" + run: | + cat << EOF > copr_fedora.conf +@@ -152,7 +150,7 @@ jobs: + EOF + + pip install copr-cli +- PR=${{ steps.leapp_repository_pr.outputs.leapp_repository_pr }} COPR_CHROOT=COPR_CHROOT COPR_REPO="$COPR_REPO" COPR_CONFIG=$COPR_CONFIG make copr_build | tee copr.log ++ PR=${{ steps.leapp_repository_pr.outputs.leapp_repository_pr }} COPR_REPO="$COPR_REPO" COPR_CONFIG=$COPR_CONFIG make copr_build | tee copr.log + + COPR_URL=$(grep -Po 'https://copr.fedorainfracloud.org/coprs/build/\d+' copr.log) + echo "::set-output name=copr_url::${COPR_URL}" +-- +2.41.0 + diff --git a/0004-Post-release-doc-update-817.patch b/0004-Post-release-doc-update-817.patch new file mode 100644 index 0000000..8262f62 --- /dev/null +++ b/0004-Post-release-doc-update-817.patch @@ -0,0 +1,215 @@ +From 5c4d39e33138fdaa5d65507581e9eb3703439cef Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Petr=20Stod=C5=AFlka?= +Date: Mon, 24 Apr 2023 12:57:04 +0200 +Subject: [PATCH 04/18] Post release doc update (#817) + +* Doc: update the list of environment variables + +* Added: + - LEAPP_DATABASE_FORCE_SYNC_ON + - LEAPP_NOGPGCHECK + - LEAPP_NO_INSIGHTS_REGISTER + - LEAPP_NO_RHSM_FACTS + - LEAPP_TARGET_ISO + - LEAPP_DEVEL_INITRAM_NETWORK + +Co-authored-by: Matej Matuska +Co-authored-by: Miriam Portman <74915975+mportman12@users.noreply.github.com> + +* Doc: replace tags & flags in the reporting guidelines + +`reporting.Tags` and `reporting.Flags` have been deprecated by +`reporting.Groups` in the past. Even when they are still accepted, +it's better to learn people they should use `reporting.Groups` +nowadays. + +Keeping reference about original divisions into Tags and Flags. + +Co-authored-by: Miriam Portman <74915975+mportman12@users.noreply.github.com> + +--------- + +Co-authored-by: Matej Matuska +Co-authored-by: Miriam Portman <74915975+mportman12@users.noreply.github.com> +--- + docs/source/el7toel8/actor-rhel7-to-rhel8.md | 19 +++++--- + docs/source/el7toel8/envars.md | 46 ++++++++++++++++++- + .../source/el7toel8/inhibit-rhel7-to-rhel8.md | 13 +++--- + 3 files changed, 62 insertions(+), 16 deletions(-) + +diff --git a/docs/source/el7toel8/actor-rhel7-to-rhel8.md b/docs/source/el7toel8/actor-rhel7-to-rhel8.md +index a6ecf37..f4e45d4 100644 +--- a/docs/source/el7toel8/actor-rhel7-to-rhel8.md ++++ b/docs/source/el7toel8/actor-rhel7-to-rhel8.md +@@ -172,8 +172,7 @@ class MyNewActor(Actor): + 'adopting-rhel-8#btrfs-has-been-removed_file-systems-and-storage' + ), + reporting.Severity(reporting.Severity.HIGH), +- reporting.Flags([reporting.Flags.INHIBITOR]), +- reporting.Tags([reporting.Tags.FILESYSTEM]), ++ reporting.Groups([reporting.Groups.INHIBITOR, reporting.Groups.FILESYSTEM]), + reporting.RelatedResource('driver', 'btrfs') + ]) + break +@@ -238,20 +237,26 @@ reporting.Remediation(hint='Please remove the dropped options from your scripts. + reporting.Remediation(playbook=) + ``` + +-**Available tags** ++**Available Groups** + ++The following groups were originally known as **Tags**: + ``` + 'accessibility', 'authentication', 'boot', 'communication', 'drivers', 'email', 'encryption', + 'filesystem', 'firewall', 'high availability', 'kernel', 'monitoring', 'network', 'OS facts', +-'python', 'repository', 'sanity', 'security', 'selinux', 'services', 'time management', ++'post', 'python', 'repository', 'sanity', 'security', 'selinux', 'services', 'time management', + 'tools', 'upgrade process' + ``` + +-In case of more report message tags then currently provided is needed, please open a GH issue or a PR. ++The following groups were originally known as **Flags**: ++``` ++'failure', 'inhibitor' ++``` + +-**Flags** ++The **failure** Group is recommended to be used when the report is related to ++a command or other action failure. + +-Besides the above mentioned **"inhibitor"** flag, there is also a **"failure"** flag which is recommended to use when we report a command or other action failure. ++If you need additional report groups, please open a GH issue or a PR, ++with the description why new required groups are needed. + + **Related resources** + +diff --git a/docs/source/el7toel8/envars.md b/docs/source/el7toel8/envars.md +index fafadda..fcc1ea3 100644 +--- a/docs/source/el7toel8/envars.md ++++ b/docs/source/el7toel8/envars.md +@@ -15,8 +15,9 @@ Overrides the automatically detected storage device with GRUB core (e.g. + + ## LEAPP_NO_RHSM + +-Do not use Red Hat Subscription Management for the upgrade. Using it has the +-same effect as using the `--no-rhsm` leapp option. ++If set to 1, Leapp does not use Red Hat Subscription Management for the upgrade. ++It's equivalent to the `--no-rhsm` leapp option. ++ + + ## LEAPP_OVL_SIZE + +@@ -79,6 +80,34 @@ case they are changed. However, in some cases it's not wanted and it leads + in malfunction network configuration (e.g. in case the bonding is configured + on the system). It's expected that NICs have to be handled manually if needed. + ++## LEAPP_DATABASE_FORCE_SYNC_ON ++ ++If set to 1, Leapp will explicitly enable synchronization on the SQLite database. ++Enabling the synchronization has negative impact on the performance ++(sometimes very negative). However, it is more reliable in case of extreme ++situations (e.g. lost power). ++Note the synchronization is nowadays switched off by default only during the phases ++executed before the reboot of the system to the upgrade environment, which we consider ++safe. As a result, we do not expect that someone would want to use this option now. ++ ++## LEAPP_NO_INSIGHTS_REGISTER ++ ++If set to 1, Leapp does not register the system into Red Hat Insights automatically. ++It's equivalent to the `--no-insights-register` leapp option. ++ ++## LEAPP_NO_RHSM_FACTS ++If set to 1, Leapp does not store migration information using Red Hat Subscription Manager. ++It's equivalent to the `--no-rhsm-facts` leapp option. ++ ++## LEAPP_NOGPGCHECK ++Set to 1 to disable RPM GPG checks (same as yum/dnf --nogpgckeck option). ++It's equivalent to the `--nogpgcheck` leapp option. ++ ++## LEAPP_TARGET_ISO ++Set the path to the target OS ISO image that should be used for the IPU. ++It's equivalent to the `--iso` leapp option. ++ ++ + ## LEAPP_UNSUPPORTED + + Necessary to use in case you use any envar with the LEAPP_DEVEL prefix +@@ -146,3 +175,16 @@ If set to 1, leapp will disable explicit synchronization on the SQLite + database. The positive effect is significant speed up of the leapp execution, + however it comes at the cost of risking a corrupted database, so it is + currently used for testing / development purposes, only. ++ ++## LEAPP_DEVEL_INITRAM_NETWORK ++You can specify one of the following values: 'network-manager', 'scripts'. ++The 'scripts' value is used for a legacy dracut module when the network is not ++handled by NetworkManager. ++Using the option allows experimental upgrades, bringing up the networking inside ++the upgrade initramfs environment (upgrade phases after the first reboot). ++It also allows the upgrade e.g. when a network based storage is used ++on the system. Currently it works only for the most simple configurations ++(e.g. when only 1 NIC is present, no rdma, no bonding, ...). Network based ++storage is not handled anyhow during the upgrade, so it's possible that the network ++based storage will not be correctly initialized and usable as expected). ++ +diff --git a/docs/source/el7toel8/inhibit-rhel7-to-rhel8.md b/docs/source/el7toel8/inhibit-rhel7-to-rhel8.md +index a20f258..5e97c78 100644 +--- a/docs/source/el7toel8/inhibit-rhel7-to-rhel8.md ++++ b/docs/source/el7toel8/inhibit-rhel7-to-rhel8.md +@@ -6,7 +6,7 @@ With latest changes on Leapp and with new actors added to the el7toel8 Leapp + repository, any actor can inhibit the upgrade process by producing a specific + message when a problem is found. The message model to use in this case is + [Report](pydoc/leapp.reporting.html#leapp.reporting.Report). +-If there is at least one Report message with `'inhibitor'` flag produced before ++If there is at least one Report message with the `'inhibitor'` group produced before + the Report phase, the upgrade will be stopped in the Reports phase, in which the + messages are being collected. It means that any Report message produced + **after** the Report phase will **not** have inhibiting effect. The details +@@ -57,7 +57,7 @@ If, instead of only adding a message to the log, the actor writer wants to make + sure that the upgrade process will be stopped in case of unsupported arch, the + actor needs to produce a [Report](/pydoc/leapp.reporting.html#leapp.reporting.Report) + message using one of the `report_*` functions from the [reporting](https://github.com/oamg/leapp-repository/blob/master/repos/system_upgrade/el7toel8/libraries/reporting.py) +-shared library with `'inhibitor'` flag. ++shared library with the `'inhibitor'` group. + + ```python + import platform +@@ -87,8 +87,7 @@ class CheckSystemArch(Actor): + reporting.Title('Unsupported architecture'), + reporting.Summary('Upgrade process is only supported on x86_64 systems.'), + reporting.Severity(reporting.Severity.HIGH), +- reporting.Tags([reporting.Tags.SANITY]), +- reporting.Flags([reporting.Flags.INHIBITOR]) ++ reporting.Groups([reporting.Groups.INHIBITOR, reporting.Groups.SANITY]), + ]) + ``` + +@@ -110,7 +109,7 @@ $ snactor run CheckSystemArch --verbose --print-output + "phase": "NON-WORKFLOW-EXECUTION", + "message": { + "hash": "dc95adcfca56eae62b7fcceeb0477a6d8257c3dddd1b05b879ebdcf05f59d504", +- "data": "{\"report\": \"{\\\"audience\\\": \\\"sysadmin\\\", \\\"flags\\\": [\\\"inhibitor\\\"], \\\"severity\\\": \\\"high\\\", \\\"summary\\\": \\\"Upgrade process is only supported on x86_64 systems.\\\", \\\"tags\\\": [\\\"sanity\\\"], \\\"title\\\": \\\"Unsupported architecture\\\"}\"}" ++ "data": "{\"report\": \"{\\\"audience\\\": \\\"sysadmin\\\", \\\"groups\\\": [\\\"inhibitor\\\", \\\"sanity\\\"], \\\"severity\\\": \\\"high\\\", \\\"summary\\\": \\\"Upgrade process is only supported on x86_64 systems.\\\", \\\"title\\\": \\\"Unsupported architecture\\\"}\"}" + }, + "type": "Report" + } +@@ -123,7 +122,7 @@ Or to inspect closely the message.data field, we could use `jq` tool: + ```sh + snactor run CheckSystemArch --verbose --print-output | jq '.[] | .message.data | fromjson' + { +- "report": "{\"audience\": \"sysadmin\", \"flags\": [\"inhibitor\"], \"severity\": \"high\", \"summary\": \"Upgrade process is only supported on x86_64 systems.\", \"tags\": [\"sanity\"], \"title\": \"Unsupported architecture\"}" ++ "report": "{\"audience\": \"sysadmin\", \"groups\": [\"inhibitor\", \"sanity\"], \"severity\": \"high\", \"summary\": \"Upgrade process is only supported on x86_64 systems.\", \"title\": \"Unsupported architecture\"}" + } + + ``` +@@ -134,7 +133,7 @@ present on the system and inhibit the upgrade process based on that check. + After all the system checks are executed by different actors, an existing actor + named [VerifyCheckResults](https://github.com/oamg/leapp-repository/tree/master/repos/system_upgrade/el7toel8/actors/verifycheckresults) + is scheduled to run in the Leapp upgrade workflow. If some [Report](/pydoc/leapp.reporting.html#leapp.reporting.Report) +-message with `'inhibitor'` flag was generated by some previous execution of ++message with the `'inhibitor'` group was generated by some previous execution of + another actor in any previous phase of the workflow, like the sample one we just + wrote, the following output will be displayed to the user: + +-- +2.41.0 + diff --git a/0005-Improve-handling-of-a-remediation-with-non-ascii-dat.patch b/0005-Improve-handling-of-a-remediation-with-non-ascii-dat.patch new file mode 100644 index 0000000..2a07309 --- /dev/null +++ b/0005-Improve-handling-of-a-remediation-with-non-ascii-dat.patch @@ -0,0 +1,201 @@ +From 2952a4ea50fd1d720410394456c98732ba6b7cc8 Mon Sep 17 00:00:00 2001 +From: Inessa Vasilevskaya +Date: Wed, 5 Apr 2023 16:53:10 +0200 +Subject: [PATCH 05/18] Improve handling of a remediation with non-ascii data + +When a remediation command is printed that manipulates +non-ascii but totally valid utf-8 strings, leapp should +not give out any tracebacks upon report generation. +The idea of a fix is to make repr implementation of Remediation +aware of utf-8 encoding. +The root of the problem is that a mixture of native text / encoded +string can appear in a report message and that caused report generating +code go crazy while creating actual log files. +Unit test has been added as well. + +Related-to: OAMG-8629 +--- + leapp/reporting/__init__.py | 43 ++++++++++++++++++++++++++++----- + leapp/utils/report.py | 35 +++++++++++++++++++-------- + tests/scripts/test_reporting.py | 18 +++++++++++++- + 3 files changed, 79 insertions(+), 17 deletions(-) + +diff --git a/leapp/reporting/__init__.py b/leapp/reporting/__init__.py +index 821f19f..7b87a38 100644 +--- a/leapp/reporting/__init__.py ++++ b/leapp/reporting/__init__.py +@@ -238,30 +238,61 @@ class BaseRemediation(BaseListPrimitive): + return ('detail', 'remediations') + + ++def _guarantee_encoded_str(text, encoding='utf-8'): ++ """Guarantees that result is a utf-8 encoded string""" ++ # Apply conversion only in py2 case ++ if six.PY3: ++ return text ++ # Unless data is already encoded -> encode it ++ try: ++ return text.encode(encoding) ++ except (UnicodeDecodeError, AttributeError): ++ return text ++ ++ ++def _guarantee_decoded_str(text, encoding='utf-8'): ++ """Guarantees that result is 'native' text""" ++ # Apply conversion only in py2 case ++ if six.PY3: ++ return text ++ # Unless data is already decoded -> decode it ++ try: ++ return text.decode(encoding) ++ except (UnicodeEncodeError, AttributeError): ++ return text ++ ++ + class RemediationCommand(BaseRemediation): + def __init__(self, value=None): + if not isinstance(value, list): + raise TypeError('Value of "RemediationCommand" must be a list') +- self._value = {'type': 'command', 'context': value} ++ self._value = {'type': 'command', 'context': [_guarantee_decoded_str(c) for c in value]} + + def __repr__(self): +- return "[{}] {}".format(self._value['type'], ' '.join(self._value['context'])) ++ # NOTE(ivasilev) As the message can contain non-ascii characters let's deal with it properly. ++ # As per python practices repr has to return an encoded string ++ return "[{}] {}".format(self._value['type'], ++ ' '.join([_guarantee_encoded_str(c) for c in self._value['context']])) + + + class RemediationHint(BaseRemediation): + def __init__(self, value=None): +- self._value = {'type': 'hint', 'context': value} ++ self._value = {'type': 'hint', 'context': _guarantee_decoded_str(value)} + + def __repr__(self): +- return "[{}] {}".format(self._value['type'], self._value['context']) ++ # NOTE(ivasilev) As the message can contain non-ascii characters let's deal with it properly. ++ # As per python practices repr has to return an encoded string ++ return "[{}] {}".format(self._value['type'], _guarantee_encoded_str(self._value['context'])) + + + class RemediationPlaybook(BaseRemediation): + def __init__(self, value=None): +- self._value = {'type': 'playbook', 'context': value} ++ self._value = {'type': 'playbook', 'context': _guarantee_decoded_str(value)} + + def __repr__(self): +- return "[{}] {}".format(self._value['type'], self._value['context']) ++ # NOTE(ivasilev) As the message can contain non-ascii characters let's deal with it properly. ++ # As per python practices repr has to return an encoded string ++ return "[{}] {}".format(self._value['type'], _guarantee_encoded_str(self._value['context'])) + + + class Remediation(object): +diff --git a/leapp/utils/report.py b/leapp/utils/report.py +index 3efb14c..71d47e3 100644 +--- a/leapp/utils/report.py ++++ b/leapp/utils/report.py +@@ -1,7 +1,10 @@ + import hashlib ++import io + import json + import os + ++import six ++ + from leapp.reporting import ( + _DEPRECATION_FLAGS, + Groups, +@@ -110,21 +113,29 @@ def generate_report_file(messages_to_report, context, path, report_schema='1.1.0 + # NOTE(ivasilev) Int conversion should not break as only specific formats of report_schema versions are allowed + report_schema_tuple = tuple(int(x) for x in report_schema.split('.')) + if path.endswith(".txt"): +- with open(path, 'w') as f: ++ with io.open(path, 'w', encoding='utf-8') as f: + for message in sorted(messages_to_report, key=importance): +- f.write('Risk Factor: {}{}\n'.format(message['severity'], +- ' (inhibitor)' if is_inhibitor(message) else '')) +- f.write('Title: {}\n'.format(message['title'])) +- f.write('Summary: {}\n'.format(message['summary'])) ++ f.write(u'Risk Factor: {}{}\n'.format(message['severity'], ++ ' (inhibitor)' if is_inhibitor(message) else '')) ++ f.write(u'Title: {}\n'.format(message['title'])) ++ f.write(u'Summary: {}\n'.format(message['summary'])) + remediation = Remediation.from_dict(message.get('detail', {})) + if remediation: +- f.write('Remediation: {}\n'.format(remediation)) ++ # NOTE(ivasilev) Decoding is necessary in case of python2 as remediation is an encoded string, ++ # while io.open expects "true text" input. For python3 repr will return proper py3 str, no ++ # decoding will be needed. ++ # This hassle and clumsiness makes me sad, so suggestions are welcome. ++ remediation_text = 'Remediation: {}\n'.format(remediation) ++ if isinstance(remediation_text, six.binary_type): ++ # This will be true for py2 where repr returns an encoded string ++ remediation_text = remediation_text.decode('utf-8') ++ f.write(remediation_text) + if report_schema_tuple > (1, 0, 0): + # report-schema 1.0.0 doesn't have a stable report key +- f.write('Key: {}\n'.format(message['key'])) +- f.write('-' * 40 + '\n') ++ f.write(u'Key: {}\n'.format(message['key'])) ++ f.write(u'-' * 40 + '\n') + elif path.endswith(".json"): +- with open(path, 'w') as f: ++ with io.open(path, 'w', encoding='utf-8') as f: + # Here all possible convertions will take place + if report_schema_tuple < (1, 1, 0): + # report-schema 1.0.0 doesn't have a stable report key +@@ -150,4 +161,8 @@ def generate_report_file(messages_to_report, context, path, report_schema='1.1.0 + # NOTE(ivasilev) As flags field is created only if there is an inhibitor + # a default value to pop has to be specified not to end up with a KeyError + msg.pop('flags', None) +- json.dump({'entries': messages_to_report, 'leapp_run_id': context}, f, indent=2) ++ # NOTE(ivasilev) Has to be done in this way (dumps + py2 conversion + write to file instead of json.dump) ++ # because of a py2 bug in the json module that can produce a mix of unicode and str objects that will be ++ # incompatible with write. https://bugs.python.org/issue13769 ++ data = json.dumps({'entries': messages_to_report, 'leapp_run_id': context}, indent=2, ensure_ascii=False) ++ f.write(data) +diff --git a/tests/scripts/test_reporting.py b/tests/scripts/test_reporting.py +index 71576fe..4a9a59b 100644 +--- a/tests/scripts/test_reporting.py ++++ b/tests/scripts/test_reporting.py +@@ -1,3 +1,4 @@ ++# -*- coding: utf-8 -*- + from collections import namedtuple + import datetime + import json +@@ -13,7 +14,7 @@ from leapp.reporting import ( + create_report_from_deprecation, + create_report_from_error, + _create_report_object, +- Audience, Flags, Groups, Key, RelatedResource, Summary, Severity, Tags, Title ++ Audience, Flags, Groups, Key, RelatedResource, Remediation, Summary, Severity, Tags, Title + ) + from leapp.utils.report import generate_report_file + +@@ -102,6 +103,21 @@ def test_report_tags_and_flags(): + assert Flags(["This is a new flag", Groups.INHIBITOR]).value == ["This is a new flag", "inhibitor"] + + ++@pytest.mark.parametrize("report_suffix", ('.json', '.txt')) ++def test_remediation_with_non_ascii_value(report_suffix): ++ report_entries = [Title('Some report title'), Summary('Some summary not used for dynamical key generation'), ++ Audience('sysadmin')] ++ # Partly related to leapp-repository PR1052 ++ rem_command = ["ln", "-snf", "root/břeťa", "/bobošík"] ++ rem_hint = "Don't forget to check /bobošík directory!" ++ rem_playbook = "bobošík.yml" ++ rem = Remediation(commands=[rem_command], hint=rem_hint, playbook=rem_playbook) ++ report_entries.append(rem) ++ report_message = _create_report_object(report_entries).report ++ with tempfile.NamedTemporaryFile(suffix=report_suffix) as reportfile: ++ generate_report_file([report_message], 'leapp-run-id', reportfile.name, '1.1.0') ++ ++ + def test_convert_from_error_to_report(): + error_dict_no_details = { + 'message': 'The system is not registered or subscribed.', +-- +2.41.0 + diff --git a/0006-Remove-review-please-mention-from-welcome-PR-message.patch b/0006-Remove-review-please-mention-from-welcome-PR-message.patch new file mode 100644 index 0000000..a84535c --- /dev/null +++ b/0006-Remove-review-please-mention-from-welcome-PR-message.patch @@ -0,0 +1,29 @@ +From 9eb9c463fc849005378a34aac22a44a9cb1a662c Mon Sep 17 00:00:00 2001 +From: Andrea Waltlova +Date: Wed, 26 Apr 2023 12:51:04 +0200 +Subject: [PATCH 06/18] Remove 'review please' mention from welcome PR message + +OAMG-8977 +--- + .github/workflows/pr-welcome-msg.yml | 4 ++-- + 1 file changed, 2 insertions(+), 2 deletions(-) + +diff --git a/.github/workflows/pr-welcome-msg.yml b/.github/workflows/pr-welcome-msg.yml +index 67393f9..143c54b 100644 +--- a/.github/workflows/pr-welcome-msg.yml ++++ b/.github/workflows/pr-welcome-msg.yml +@@ -19,9 +19,9 @@ jobs: + issue-number: ${{ github.event.pull_request.number }} + body: | + ## **Thank you for contributing to the Leapp project!** +- Please note that every PR needs to comply with the [Leapp Guidelines](https://leapp.readthedocs.io/en/latest/contributing.html#) and must pass all tests in order to be mergable. ++ Please note that every PR needs to comply with the [Leapp Guidelines](https://leapp.readthedocs.io/en/latest/contributing.html#) and must pass all tests in order to be mergeable. + If you want to request a review or rebuild a package in copr, you can use following commands as a comment: +- - **review please** to notify leapp developers of review request ++ - **review please @oamg/developers** to notify leapp developers of the review request + - **/packit copr-build** to submit a public copr build using packit + + To launch regression testing public members of oamg organization can leave the following comment: +-- +2.41.0 + diff --git a/0007-Dialog-creation-fails-for-a-Component-without-choice.patch b/0007-Dialog-creation-fails-for-a-Component-without-choice.patch new file mode 100644 index 0000000..acc4435 --- /dev/null +++ b/0007-Dialog-creation-fails-for-a-Component-without-choice.patch @@ -0,0 +1,28 @@ +From 5cda0ebbf266cbe31e2a2e78f44a4cbe0e1089c2 Mon Sep 17 00:00:00 2001 +From: Ygal Blum +Date: Mon, 1 May 2023 11:45:28 +0300 +Subject: [PATCH 07/18] Dialog creation fails for a Component without choices + +choices is not an attribute of Component only of some its subclasses + +Signed-off-by: Ygal Blum +--- + leapp/dialogs/dialog.py | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/leapp/dialogs/dialog.py b/leapp/dialogs/dialog.py +index a1d65c7..3ead810 100644 +--- a/leapp/dialogs/dialog.py ++++ b/leapp/dialogs/dialog.py +@@ -54,7 +54,7 @@ class Dialog(object): + + @property + def answerfile_sections(self): +- return {"{}.{}".format(self.scope, c.key): c.choices for c in self.components} ++ return {"{}.{}".format(self.scope, c.key): getattr(c, "choices", []) for c in self.components} + + @property + def min_label_width(self): +-- +2.41.0 + diff --git a/0008-Test-differnt-types-of-Dialog-components.patch b/0008-Test-differnt-types-of-Dialog-components.patch new file mode 100644 index 0000000..8facc2b --- /dev/null +++ b/0008-Test-differnt-types-of-Dialog-components.patch @@ -0,0 +1,106 @@ +From 07ff6eb4c383137f2b5f6730deecf8ffa9e4cd98 Mon Sep 17 00:00:00 2001 +From: Ygal Blum +Date: Thu, 4 May 2023 17:23:03 +0300 +Subject: [PATCH 08/18] Test differnt types of Dialog components + +Add a multi component dialog to one of the tester actors +Verify that the input is handled correctly + +Signed-off-by: Ygal Blum +--- + .../actor-api-tests/actors/first/actor.py | 31 +++++++++++++++++++ + tests/scripts/test_actor_api.py | 27 ++++++++++++++++ + 2 files changed, 58 insertions(+) + +diff --git a/tests/data/actor-api-tests/actors/first/actor.py b/tests/data/actor-api-tests/actors/first/actor.py +index 27650bd..2b5a6e1 100644 +--- a/tests/data/actor-api-tests/actors/first/actor.py ++++ b/tests/data/actor-api-tests/actors/first/actor.py +@@ -1,4 +1,6 @@ + from leapp.actors import Actor ++from leapp.dialogs import Dialog ++from leapp.dialogs.components import BooleanComponent, ChoiceComponent, NumberComponent, TextComponent + from leapp.tags import ActorFileApiTag + from leapp.models import ApiTestConsume, ApiTestProduce + +@@ -9,6 +11,35 @@ class First(Actor): + consumes = (ApiTestConsume,) + produces = (ApiTestProduce,) + tags = (ActorFileApiTag,) ++ dialogs = ( ++ Dialog( ++ scope='first_actor', ++ reason='need to test dialogs', ++ components=( ++ TextComponent( ++ key='text', ++ label='text', ++ description='a text value is needed', ++ ), ++ BooleanComponent( ++ key='bool', ++ label='bool', ++ description='a boolean value is needed' ++ ), ++ NumberComponent( ++ key='num', ++ label='num', ++ description='a numeric value is needed' ++ ), ++ ChoiceComponent( ++ key='choice', ++ label='choice', ++ description='need to choose one of these choices', ++ choices=('One', 'Two', 'Three', 'Four', 'Five'), ++ ), ++ ), ++ ), ++ ) + + def process(self): + pass +diff --git a/tests/scripts/test_actor_api.py b/tests/scripts/test_actor_api.py +index 0a28257..f009e68 100644 +--- a/tests/scripts/test_actor_api.py ++++ b/tests/scripts/test_actor_api.py +@@ -7,6 +7,7 @@ import py + import pytest + + from leapp.actors import Actor, get_actors ++from leapp.config import get_config + from leapp.libraries.stdlib import api + from leapp.messaging import BaseMessaging + from leapp.models import ApiTestConsume, ApiTestProduce +@@ -168,3 +169,29 @@ def test_actor_all_files_paths(leapp_forked, repository, actor_name): # noqa; p + + assert not actor.get_actor_file_path('directory/repository') + assert not api.get_actor_file_path('directory/repository') ++ ++ ++@pytest.fixture(scope="module") ++def setup_database(): ++ get_config().set('database', 'path', '/tmp/leapp-test.db') ++ ++ ++@pytest.mark.parametrize("actor_name", ('first',)) ++def test_actor_get_answers(monkeypatch, leapp_forked, setup_database, repository, actor_name): # noqa; pylint: disable=unused-argument ++ user_responses = { ++ 'text': ('expected_value', 'expected_value'), ++ 'bool': ('Yes', True), ++ 'num': (42, 42), ++ 'choice': ("3", "Four"), ++ } ++ ++ def mocked_input(title): ++ return user_responses[title.split()[0].split(':')[0].lower()][0] ++ ++ monkeypatch.setattr('leapp.dialogs.renderer.input', mocked_input) ++ ++ messaging = _TestableMessaging() ++ with _with_loaded_actor(repository, actor_name, messaging) as (_unused, actor): ++ answers = actor.get_answers(actor.dialogs[0]) ++ for component in actor.dialogs[0].components: ++ assert answers.get(component.key) == user_responses[component.key][1] +-- +2.41.0 + diff --git a/0009-Generate-empty-report-correctly.patch b/0009-Generate-empty-report-correctly.patch new file mode 100644 index 0000000..e6200c9 --- /dev/null +++ b/0009-Generate-empty-report-correctly.patch @@ -0,0 +1,51 @@ +From 48b3ec6ab6dc8657facdb5482c0edb6a4872f850 Mon Sep 17 00:00:00 2001 +From: Petr Stodulka +Date: Fri, 5 May 2023 14:41:42 +0200 +Subject: [PATCH 09/18] Generate empty report correctly + +On Python2, the json.dumps library can produce either +unicode or str output. Nowadays, the unicode is required, however +in case the input for json.dumps function does not contain any +unicode objects (e.g. when no report entry exists), it produces +str type. + +Relates: #821 +--- + leapp/utils/report.py | 5 +++++ + tests/scripts/test_reporting.py | 6 ++++++ + 2 files changed, 11 insertions(+) + +diff --git a/leapp/utils/report.py b/leapp/utils/report.py +index 71d47e3..8ffbcd3 100644 +--- a/leapp/utils/report.py ++++ b/leapp/utils/report.py +@@ -165,4 +165,9 @@ def generate_report_file(messages_to_report, context, path, report_schema='1.1.0 + # because of a py2 bug in the json module that can produce a mix of unicode and str objects that will be + # incompatible with write. https://bugs.python.org/issue13769 + data = json.dumps({'entries': messages_to_report, 'leapp_run_id': context}, indent=2, ensure_ascii=False) ++ if isinstance(data, six.binary_type): ++ # Note: if messages_to_report is empty, the produced data is str instead of unicode ++ # at least one string must be unicode for the json.dumps function to produce actually ++ # unicode object. ++ data = data.decode('utf-8') + f.write(data) +diff --git a/tests/scripts/test_reporting.py b/tests/scripts/test_reporting.py +index 4a9a59b..2b0f245 100644 +--- a/tests/scripts/test_reporting.py ++++ b/tests/scripts/test_reporting.py +@@ -118,6 +118,12 @@ def test_remediation_with_non_ascii_value(report_suffix): + generate_report_file([report_message], 'leapp-run-id', reportfile.name, '1.1.0') + + ++@pytest.mark.parametrize("report_suffix", ('.json', '.txt')) ++def test_generation_of_empty_report(report_suffix): ++ with tempfile.NamedTemporaryFile(suffix=report_suffix) as reportfile: ++ generate_report_file([], 'leapp-run-id', reportfile.name, '1.1.0') ++ ++ + def test_convert_from_error_to_report(): + error_dict_no_details = { + 'message': 'The system is not registered or subscribed.', +-- +2.41.0 + diff --git a/0010-Add-the-empty-newline-in-the-end-of-the-report-json-.patch b/0010-Add-the-empty-newline-in-the-end-of-the-report-json-.patch new file mode 100644 index 0000000..4af19bb --- /dev/null +++ b/0010-Add-the-empty-newline-in-the-end-of-the-report-json-.patch @@ -0,0 +1,25 @@ +From 032c009af6b40ee75189371e5b08420205bd7381 Mon Sep 17 00:00:00 2001 +From: Petr Stodulka +Date: Fri, 5 May 2023 15:31:39 +0200 +Subject: [PATCH 10/18] Add the empty newline in the end of the report json + file + +Text files should have empty newline in the end of the file to be +POSIX compatible. +--- + leapp/utils/report.py | 2 ++ + 1 file changed, 2 insertions(+) + +diff --git a/leapp/utils/report.py b/leapp/utils/report.py +index 8ffbcd3..d24a177 100644 +--- a/leapp/utils/report.py ++++ b/leapp/utils/report.py +@@ -171,3 +171,5 @@ def generate_report_file(messages_to_report, context, path, report_schema='1.1.0 + # unicode object. + data = data.decode('utf-8') + f.write(data) ++ # append the empty new line to be POSIX compatible ++ f.write(u'\n') +-- +2.41.0 + diff --git a/0011-terminology-fixes.patch b/0011-terminology-fixes.patch new file mode 100644 index 0000000..0c68a76 --- /dev/null +++ b/0011-terminology-fixes.patch @@ -0,0 +1,159 @@ +From b69cd77ffccd76e90116e8376437fea669356a6e Mon Sep 17 00:00:00 2001 +From: Bob Mader +Date: Mon, 15 May 2023 12:27:11 -0500 +Subject: [PATCH 11/18] terminology fixes + +--- + .../images/inplace-upgrade-workflow.svg | 55 ++++++++++--------- + docs/source/terminology.md | 4 +- + 2 files changed, 30 insertions(+), 29 deletions(-) + +diff --git a/docs/source/_static/images/inplace-upgrade-workflow.svg b/docs/source/_static/images/inplace-upgrade-workflow.svg +index 538eb17..76d8e70 100644 +--- a/docs/source/_static/images/inplace-upgrade-workflow.svg ++++ b/docs/source/_static/images/inplace-upgrade-workflow.svg +@@ -1,25 +1,24 @@ + + ++ inkscape:version="1.2.2 (b0a8486541, 2022-12-01)" ++ sodipodi:docname="inplace-upgrade-workflow.svg" ++ xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape" ++ xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd" ++ xmlns="http://www.w3.org/2000/svg" ++ xmlns:svg="http://www.w3.org/2000/svg" ++ xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#" ++ xmlns:cc="http://creativecommons.org/ns#" ++ xmlns:dc="http://purl.org/dc/elements/1.1/"> + + ++ inkscape:swatch="gradient"> + ++ height="297mm" ++ inkscape:showpageshadow="2" ++ inkscape:pagecheckerboard="0" ++ inkscape:deskcolor="#d1d1d1" /> + + +@@ -65,7 +67,6 @@ + image/svg+xml + +- + + + +@@ -86,15 +87,15 @@ + style="fill:#e6e6e6;stroke:#000000;stroke-width:0.239809;stroke-miterlimit:4;stroke-dasharray:none" /> + Main Steps ++ sodipodi:role="line">Main Phases + + + + ++ style="fill:#c6afe9;stroke:#000000;stroke-width:0.154672;fill-opacity:1" /> + +Date: Wed, 17 May 2023 11:54:55 +0200 +Subject: [PATCH 12/18] Make copr-build for /rerun 42 functional again + +copr config username for a dependent leapp-repository PR +build case wasn't changed in the 5ddce5a. This patch should +address that. + +OAMG-8876 +--- + .github/workflows/reuse-copr-build.yml | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/.github/workflows/reuse-copr-build.yml b/.github/workflows/reuse-copr-build.yml +index 76c1f67..7ccd282 100644 +--- a/.github/workflows/reuse-copr-build.yml ++++ b/.github/workflows/reuse-copr-build.yml +@@ -143,7 +143,7 @@ jobs: + cat << EOF > copr_fedora.conf + [copr-cli] + login = ${{ secrets.FEDORA_COPR_LOGIN }} +- username = @oamg ++ username = oamgbot + token = ${{ secrets.FEDORA_COPR_TOKEN }} + copr_url = https://copr.fedorainfracloud.org + # expiration date: 2030-07-04 +-- +2.41.0 + diff --git a/0013-fix-spelling-errors-in-docs.patch b/0013-fix-spelling-errors-in-docs.patch new file mode 100644 index 0000000..77874a7 --- /dev/null +++ b/0013-fix-spelling-errors-in-docs.patch @@ -0,0 +1,152 @@ +From 2c9be460399429d3aeadac80fdae8892f28e4345 Mon Sep 17 00:00:00 2001 +From: Bob Mader +Date: Thu, 18 May 2023 10:48:31 -0500 +Subject: [PATCH 13/18] fix spelling errors in docs + +--- + docs/source/_static/images/inplace-upgrade-workflow.svg | 6 +++--- + docs/source/best-practices.md | 2 +- + docs/source/compatibility-with-leapp-repository.md | 2 +- + docs/source/dependencies.md | 2 +- + docs/source/deprecation.md | 2 +- + docs/source/el7toel8/actor-rhel7-to-rhel8.md | 2 +- + docs/source/el7toel8/deprecation.md | 2 +- + docs/source/unit-testing.md | 2 +- + docs/source/workflow-apis.md | 2 +- + 9 files changed, 11 insertions(+), 11 deletions(-) + +diff --git a/docs/source/_static/images/inplace-upgrade-workflow.svg b/docs/source/_static/images/inplace-upgrade-workflow.svg +index 76d8e70..d470449 100644 +--- a/docs/source/_static/images/inplace-upgrade-workflow.svg ++++ b/docs/source/_static/images/inplace-upgrade-workflow.svg +@@ -1155,7 +1155,7 @@ + style="font-size:5.29167px;text-align:center;text-anchor:middle;stroke-width:0.264583" + y="370.38736" + x="24.260719" +- sodipodi:role="line">Pre-Thrid PartyPre-Third PartyThrid PartyThird PartyPost-Thrid PartyPost-Third Party(till April 2021) + - Models +diff --git a/docs/source/unit-testing.md b/docs/source/unit-testing.md +index 05d20df..d4feec9 100644 +--- a/docs/source/unit-testing.md ++++ b/docs/source/unit-testing.md +@@ -203,7 +203,7 @@ or + pytest {PATH_TO_ACTOR} + ``` + +-It is also possible to run only slected tests based on their name: ++It is also possible to run only selected tests based on their name: + + ```sh + pytest -k "vim" # to run all tests contains vim in name +diff --git a/docs/source/workflow-apis.md b/docs/source/workflow-apis.md +index d27ac1e..c58cd2f 100644 +--- a/docs/source/workflow-apis.md ++++ b/docs/source/workflow-apis.md +@@ -112,7 +112,7 @@ Workflow APIs support having tests defined for them. We actually encourage you t + + Tests for APIs are supposed to be defines in the `apis/tests` directory. + +-### Depedencies ++### Dependencies + + Workflow APIs can depend on another Workflow API, to allow API compositiion. Actors using APIs with dependencies on + other APIs just have to specify the API they want to use and do not need to know that those depend on other APIs. +-- +2.41.0 + diff --git a/0014-Fix-invalid-type-check.patch b/0014-Fix-invalid-type-check.patch new file mode 100644 index 0000000..2e3088f --- /dev/null +++ b/0014-Fix-invalid-type-check.patch @@ -0,0 +1,27 @@ +From 6e29780cffa8873e77c476a54dfeaed351ea18c8 Mon Sep 17 00:00:00 2001 +From: David Kubek +Date: Mon, 12 Jun 2023 11:30:55 +0200 +Subject: [PATCH 14/18] Fix invalid type check + +The `isinstance` type check in `report_info` checks the type of an +incorrect variable. This commit fixes the issue. +--- + leapp/utils/output.py | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/leapp/utils/output.py b/leapp/utils/output.py +index 686b3af..fc60442 100644 +--- a/leapp/utils/output.py ++++ b/leapp/utils/output.py +@@ -107,7 +107,7 @@ def report_errors(errors): + + def report_info(report_paths, log_paths, answerfile=None, fail=False): + report_paths = [report_paths] if not isinstance(report_paths, list) else report_paths +- log_paths = [log_paths] if not isinstance(report_paths, list) else log_paths ++ log_paths = [log_paths] if not isinstance(log_paths, list) else log_paths + + if log_paths: + sys.stdout.write("\n") +-- +2.41.0 + diff --git a/0015-Save-actor-tracebacks-in-leapp.db.patch b/0015-Save-actor-tracebacks-in-leapp.db.patch new file mode 100644 index 0000000..d87bb11 --- /dev/null +++ b/0015-Save-actor-tracebacks-in-leapp.db.patch @@ -0,0 +1,59 @@ +From 486b4aae41d3d6d8f6f1ebe31e9c57dbe5ac3690 Mon Sep 17 00:00:00 2001 +From: Inessa Vasilevskaya +Date: Wed, 7 Jun 2023 13:53:38 +0200 +Subject: [PATCH 15/18] Save actor tracebacks in leapp.db + +Now when an exception is thrown during actor +execution it won't be shown only on stdout, but +will appear in leapp.db and leapp report as high +risk report message. + +OAMG-4186 +--- + leapp/messaging/__init__.py | 7 ++++++- + leapp/workflows/__init__.py | 3 ++- + 2 files changed, 8 insertions(+), 2 deletions(-) + +diff --git a/leapp/messaging/__init__.py b/leapp/messaging/__init__.py +index 9896179..de8ac9b 100644 +--- a/leapp/messaging/__init__.py ++++ b/leapp/messaging/__init__.py +@@ -146,6 +146,11 @@ class BaseMessaging(object): + time=datetime.datetime.now()) + self._do_produce(model, actor, self._errors) + ++ def report_stacktrace(self, message, trace, actorname): ++ model = ErrorModel(message="{message}\n{trace}".format(message=message, trace=trace), ++ actor=actorname, severity="fatal", time=datetime.datetime.now()) ++ self._do_produce(model, actorname, self._errors) ++ + def request_stop_after_phase(self): + """ + If called, it will cause the workflow to stop the execution after the current phase ends. +@@ -222,7 +227,7 @@ class BaseMessaging(object): + data = json.dumps(model.dump(), sort_keys=True) + message = { + 'type': type(model).__name__, +- 'actor': type(actor).name, ++ 'actor': type(actor).name if not isinstance(actor, str) else actor, + 'topic': model.topic.name, + 'stamp': datetime.datetime.utcnow().isoformat() + 'Z', + 'phase': os.environ.get('LEAPP_CURRENT_PHASE', 'NON-WORKFLOW-EXECUTION'), +diff --git a/leapp/workflows/__init__.py b/leapp/workflows/__init__.py +index b909bb1..18cd8f2 100644 +--- a/leapp/workflows/__init__.py ++++ b/leapp/workflows/__init__.py +@@ -338,8 +338,9 @@ class Workflow(with_metaclass(WorkflowMeta)): + config_model=config_model, skip_dialogs=skip_dialogs) + try: + instance.run() +- except BaseException: ++ except BaseException as exc: + self._unhandled_exception = True ++ messaging.report_stacktrace(message=exc.message, trace=exc.exception_info, actorname=actor.name) + raise + + self._stop_after_phase_requested = messaging.stop_after_phase or self._stop_after_phase_requested +-- +2.41.0 + diff --git a/0016-squashme-Log-actor-crash.patch b/0016-squashme-Log-actor-crash.patch new file mode 100644 index 0000000..62e61a4 --- /dev/null +++ b/0016-squashme-Log-actor-crash.patch @@ -0,0 +1,29 @@ +From 1420bce48a5fb1dbe27980e783cc895852762663 Mon Sep 17 00:00:00 2001 +From: Inessa Vasilevskaya +Date: Thu, 22 Jun 2023 16:38:26 +0200 +Subject: [PATCH 16/18] [squashme] Log actor crash + +--- + leapp/workflows/__init__.py | 6 +++++- + 1 file changed, 5 insertions(+), 1 deletion(-) + +diff --git a/leapp/workflows/__init__.py b/leapp/workflows/__init__.py +index 18cd8f2..7f01e0d 100644 +--- a/leapp/workflows/__init__.py ++++ b/leapp/workflows/__init__.py +@@ -340,7 +340,11 @@ class Workflow(with_metaclass(WorkflowMeta)): + instance.run() + except BaseException as exc: + self._unhandled_exception = True +- messaging.report_stacktrace(message=exc.message, trace=exc.exception_info, actorname=actor.name) ++ messaging.report_stacktrace(message=exc.message, ++ trace=exc.exception_info, ++ actorname=actor.name) ++ current_logger.error('Actor {actor} has crashed: {trace}'.format(actor=actor.name, ++ trace=exc.exception_info)) + raise + + self._stop_after_phase_requested = messaging.stop_after_phase or self._stop_after_phase_requested +-- +2.41.0 + diff --git a/0017-squashme-Put-trace-in-the-details.patch b/0017-squashme-Put-trace-in-the-details.patch new file mode 100644 index 0000000..77fe7e5 --- /dev/null +++ b/0017-squashme-Put-trace-in-the-details.patch @@ -0,0 +1,27 @@ +From c4c180008d6446c6491e4e77bd6d23e58d166da0 Mon Sep 17 00:00:00 2001 +From: Inessa Vasilevskaya +Date: Fri, 23 Jun 2023 15:09:28 +0200 +Subject: [PATCH 17/18] [squashme] Put trace in the details + +--- + leapp/messaging/__init__.py | 4 ++-- + 1 file changed, 2 insertions(+), 2 deletions(-) + +diff --git a/leapp/messaging/__init__.py b/leapp/messaging/__init__.py +index de8ac9b..c42b5d7 100644 +--- a/leapp/messaging/__init__.py ++++ b/leapp/messaging/__init__.py +@@ -147,8 +147,8 @@ class BaseMessaging(object): + self._do_produce(model, actor, self._errors) + + def report_stacktrace(self, message, trace, actorname): +- model = ErrorModel(message="{message}\n{trace}".format(message=message, trace=trace), +- actor=actorname, severity="fatal", time=datetime.datetime.now()) ++ model = ErrorModel(message=message, details=trace, actor=actorname, severity="fatal", ++ time=datetime.datetime.now()) + self._do_produce(model, actorname, self._errors) + + def request_stop_after_phase(self): +-- +2.41.0 + diff --git a/0018-Add-reports-summary-to-the-output.patch b/0018-Add-reports-summary-to-the-output.patch new file mode 100644 index 0000000..a462757 --- /dev/null +++ b/0018-Add-reports-summary-to-the-output.patch @@ -0,0 +1,138 @@ +From 154e1c59e6a654dbd9660550e9a0c3c4d02a0ce8 Mon Sep 17 00:00:00 2001 +From: Matej Matuska +Date: Thu, 16 Mar 2023 11:49:40 +0100 +Subject: [PATCH 18/18] Add reports summary to the output + +Previously only the paths to reports were printed and as a result the +reports were easily missed. + +Leapp now prints a list of HIGH and MEDIUM priority reports along with +a summary of number of reports with individual severities to make +reports more visible. + +Also if there are any HIGH or MEDIUM severity reports, the block titles +are yellow. + +The framework-version is bumped to 4.0, because the `report_info` +funtion now takes a context_id parameter (to be able to fetch the +reports), so the new API is incompatible. +--- + leapp/utils/output.py | 66 +++++++++++++++++++++++++++++++++++++++---- + packaging/leapp.spec | 2 +- + 2 files changed, 62 insertions(+), 6 deletions(-) + +diff --git a/leapp/utils/output.py b/leapp/utils/output.py +index fc60442..ae24a1d 100644 +--- a/leapp/utils/output.py ++++ b/leapp/utils/output.py +@@ -61,6 +61,12 @@ def print_error(error): + v=details[detail].rstrip().replace('\n', '\n' + ' ' * (6 + len(detail))))) + + ++def _print_report_titles(reports): ++ width = 4 + len(str(len(reports))) ++ for position, report in enumerate(reports, start=1): ++ print('{idx:{width}}. {title}'.format(idx=position, width=width, title=report['title'])) ++ ++ + def report_inhibitors(context_id): + # The following imports are required to be here to avoid import loop problems + from leapp.utils.report import fetch_upgrade_report_messages, is_inhibitor # noqa; pylint: disable=import-outside-toplevel +@@ -70,8 +76,7 @@ def report_inhibitors(context_id): + text = 'UPGRADE INHIBITED' + with pretty_block(text=text, end_text=text, color=Color.red, target=sys.stdout): + print('Upgrade has been inhibited due to the following problems:') +- for position, report in enumerate(inhibitors, start=1): +- print('{idx:5}. Inhibitor: {title}'.format(idx=position, title=report['title'])) ++ _print_report_titles(inhibitors) + print('Consult the pre-upgrade report for details and possible remediation.') + + +@@ -105,7 +110,33 @@ def report_errors(errors): + print_error(error) + + +-def report_info(report_paths, log_paths, answerfile=None, fail=False): ++def _filter_reports(reports, severity=None, is_inhibitor=False): ++ # The following imports are required to be here to avoid import loop problems ++ from leapp.utils.report import is_inhibitor as isinhibitor # noqa; pylint: disable=import-outside-toplevel ++ if not severity: ++ return [report for report in reports if isinhibitor(report) == is_inhibitor] ++ return [report for report in reports if report['severity'] == severity and isinhibitor(report) == is_inhibitor] ++ ++ ++def _print_reports_summary(reports): ++ # The following imports are required to be here to avoid import loop problems ++ from leapp.reporting import Severity # noqa; pylint: disable=import-outside-toplevel ++ ++ inhibitors = _filter_reports(reports, is_inhibitor=True) ++ high = _filter_reports(reports, Severity.HIGH) ++ medium = _filter_reports(reports, Severity.MEDIUM) ++ low = _filter_reports(reports, Severity.LOW) ++ info = _filter_reports(reports, Severity.INFO) ++ ++ print('Reports summary:') ++ print(' Inhibitors: {:5}'.format(len(inhibitors))) ++ print(' HIGH severity reports: {:5}'.format(len(high))) ++ print(' MEDIUM severity reports: {:5}'.format(len(medium))) ++ print(' LOW severity reports: {:5}'.format(len(low))) ++ print(' INFO severity reports: {:5}'.format(len(info))) ++ ++ ++def report_info(context_id, report_paths, log_paths, answerfile=None, fail=False): + report_paths = [report_paths] if not isinstance(report_paths, list) else report_paths + log_paths = [log_paths] if not isinstance(log_paths, list) else log_paths + +@@ -115,9 +146,34 @@ def report_info(report_paths, log_paths, answerfile=None, fail=False): + sys.stdout.write("Debug output written to {path}\n".format(path=log_path)) + + if report_paths: +- with pretty_block("REPORT", target=sys.stdout, color=Color.bold if fail else Color.green): ++ # The following imports are required to be here to avoid import loop problems ++ from leapp.reporting import Severity # noqa; pylint: disable=import-outside-toplevel ++ from leapp.utils.report import fetch_upgrade_report_messages # noqa; pylint: disable=import-outside-toplevel ++ reports = fetch_upgrade_report_messages(context_id) ++ ++ high = _filter_reports(reports, Severity.HIGH) ++ medium = _filter_reports(reports, Severity.MEDIUM) ++ ++ color = Color.green ++ if medium or high: ++ color = Color.yellow ++ if fail: ++ color = Color.bold ++ ++ with pretty_block("REPORT", target=sys.stdout, color=color): ++ if high or medium: ++ print('HIGH and MEDIUM severity reports:') ++ _print_report_titles(high + medium) ++ ++ sys.stdout.write('\n') ++ _print_reports_summary(reports) ++ ++ print( ++ '\n{bold}Before continuing consult the full report:{reset}' ++ .format(bold=Color.bold, reset=Color.reset) ++ ) + for report_path in report_paths: +- sys.stdout.write("A report has been generated at {path}\n".format(path=report_path)) ++ sys.stdout.write(" A report has been generated at {path}\n".format(path=report_path)) + + if answerfile: + sys.stdout.write("Answerfile has been generated at {}\n".format(answerfile)) +diff --git a/packaging/leapp.spec b/packaging/leapp.spec +index 4f88b00..a5936e1 100644 +--- a/packaging/leapp.spec ++++ b/packaging/leapp.spec +@@ -13,7 +13,7 @@ + # This is kind of help for more flexible development of leapp repository, + # so people do not have to wait for new official release of leapp to ensure + # it is installed/used the compatible one. +-%global framework_version 3.1 ++%global framework_version 4.0 + + # IMPORTANT: everytime the requirements are changed, increment number by one + # - same for Provides in deps subpackage +-- +2.41.0 + diff --git a/leapp.spec b/leapp.spec index 0f7ccbf..7fdbedc 100644 --- a/leapp.spec +++ b/leapp.spec @@ -7,7 +7,7 @@ # it. In case of upstream, dependencies are set differently, but YUM is not # capable enough to deal with them correctly all the time; we continue to use # simplified deps in RHEL to ensure that YUM can deal with it. -%global framework_version 3.1 +%global framework_version 4.0 # IMPORTANT: everytime the requirements are changed, increment number by one # - same for Provides in deps subpackage @@ -65,7 +65,24 @@ Requires: leapp-repository # PATCHES HERE # Patch0001: filename.patch - +Patch0001: 0001-Update-upgrade-paths-and-env-vars-in-tests.patch +Patch0002: 0002-Make-copr-build-functioning-again.patch +Patch0003: 0003-Do-not-set-COPR_CHROOTS-for-copr-build.patch +Patch0004: 0004-Post-release-doc-update-817.patch +Patch0005: 0005-Improve-handling-of-a-remediation-with-non-ascii-dat.patch +Patch0006: 0006-Remove-review-please-mention-from-welcome-PR-message.patch +Patch0007: 0007-Dialog-creation-fails-for-a-Component-without-choice.patch +Patch0008: 0008-Test-differnt-types-of-Dialog-components.patch +Patch0009: 0009-Generate-empty-report-correctly.patch +Patch0010: 0010-Add-the-empty-newline-in-the-end-of-the-report-json-.patch +Patch0011: 0011-terminology-fixes.patch +Patch0012: 0012-Make-copr-build-for-rerun-42-functional-again.patch +Patch0013: 0013-fix-spelling-errors-in-docs.patch +Patch0014: 0014-Fix-invalid-type-check.patch +Patch0015: 0015-Save-actor-tracebacks-in-leapp.db.patch +Patch0016: 0016-squashme-Log-actor-crash.patch +Patch0017: 0017-squashme-Put-trace-in-the-details.patch +Patch0018: 0018-Add-reports-summary-to-the-output.patch %description Leapp utility provides the possibility to use the Leapp framework via CLI. @@ -161,6 +178,25 @@ Requires: findutils # APPLY REGISTERED PATCHES HERE # %%patch0001 -p1 +%patch0001 -p1 +%patch0002 -p1 +%patch0003 -p1 +%patch0004 -p1 +%patch0005 -p1 +%patch0006 -p1 +%patch0007 -p1 +%patch0008 -p1 +%patch0009 -p1 +%patch0010 -p1 +%patch0011 -p1 +%patch0012 -p1 +%patch0013 -p1 +%patch0014 -p1 +%patch0015 -p1 +%patch0016 -p1 +%patch0017 -p1 +%patch0018 -p1 + ################################################## # Build @@ -243,6 +279,14 @@ install -m 0644 -p man/leapp.1 %{buildroot}%{_mandir}/man1/ # no files here %changelog +* Mon Jul 17 2023 Petr Stodulka - 0.15.1-2 +- Bump leapp-framework to 4.0 +- Improve the report summary output to make it more visible +- Fix processing data in remediation instructions with non-ascii characters +- Fix creation of Dialog for Component without choices +- Store tracebacks from actors in leapp.db +- Resolves: #2223312 + * Tue Feb 21 2023 Petr Stodulka - 0.15.1-1 - Rebase to v0.15.1 - Change DAC for /var/lib/leapp to 0700 to make it accessible for root only