Apply some debugging hacks for the periodic failure bug

This is all in aid of trying to figure out what's going on in
https://pagure.io/fedora-qa/os-autoinst-distri-fedora/pull-request/312#comment-199540
 - sometimes the upgrade process just does not work, it seems
like the prepared update file gets wiped for some reason. This
is all trying to figure out exactly when that happens and
whether it's because we're running out of disk space.

Signed-off-by: Adam Williamson <awilliam@redhat.com>

lib/installedtest.pm

@@ -168,6 +168,17 @@ sub post_fail_hook {
         }
     }
 
+    # TEST TEST FOR DEBUGGING ONLY
+    if (get_var("GUI_UPGRADE")) {
+        script_run "df -h";
+        script_run "ls -l /var/lib/PackageKit";
+        script_run "ls -l /var/lib/PackageKit/prepared-upgrade";
+        $self->root_console(tty => 4);
+        send_key "ctrl-c";
+        upload_logs "/tmp/pk.log", failok => 1;
+    }
+    # END DEBUGGING
+
 }
 
 1;

lib/utils.pm

@@ -1476,6 +1476,8 @@ sub check_prerelease {
     $prerelease = 1 if ($build =~ /\.n\.\d+/ && !get_var("LIVE"));
     # if it's a respin compose we *MUST NOT* see tags
     $prerelease = 0 if ($build =~ /Respin/);
+    # bail if we've decided we don't care
+    return if ($prerelease > 1);
     # we *could* go to a console and parse fedora-release-common
     # to decide if a nightly live image should have tags or not, but
     # it seems absurd as we're almost reinventing the code that

main.pm

@@ -122,7 +122,16 @@ sub load_upgrade_tests() {
             autotest::loadtest "tests/${pu}.pm";
         }
     }
-    autotest::loadtest "tests/upgrade_run.pm";
+    # If we want the upgrade run with GUI tools, we need
+    # to load a GUI set of tests
+    if (get_var("GUI_UPGRADE")) {
+        autotest::loadtest("tests/graphical_upgrade_prerequisites.pm");
+        autotest::loadtest("tests/graphical_upgrade_run.pm");
+    }
+    # otherwise go with the CLI variant
+    else {
+        autotest::loadtest "tests/upgrade_run.pm";
+    }
     # handle additional postinstall tests
     if (get_var("POSTINSTALL")) {
         set_var('POSTINSTALL', "upgrade_postinstall " . get_var("POSTINSTALL"));

needles/gnome/desktop_package_tool_restart_upgrade.json

@@ -0,0 +1,15 @@
+{
+  "area": [
+    {
+      "xpos": 438,
+      "ypos": 277,
+      "width": 143,
+      "height": 20,
+      "type": "match"
+    }
+  ],
+  "properties": [],
+  "tags": [
+    "desktop_package_tool_restart_upgrade"
+  ]
+}

needles/gnome/desktop_package_tool_upgrade_system.json

@@ -0,0 +1,26 @@
+{
+    "area": [
+        {
+            "height": 56,
+            "type": "match",
+            "click_point": {
+                "xpos": 50,
+                "ypos": 10
+            },
+            "width": 154,
+            "xpos": 435,
+            "ypos": 277
+        },
+        {
+            "height": 30,
+            "type": "match",
+            "width": 133,
+            "xpos": 558,
+            "ypos": 162
+        }
+    ],
+    "properties": [],
+    "tags": [
+        "desktop_package_tool_upgrade_system"
+    ]
+}

needles/kde/apps/auth_required-20240214.json

@@ -0,0 +1,16 @@
+{
+  "area": [
+    {
+      "xpos": 334,
+      "ypos": 271,
+      "width": 212,
+      "height": 20,
+      "type": "match"
+    }
+  ],
+  "properties": [],
+  "tags": [
+    "auth_required",
+    "auth_required_password"
+  ]
+}

needles/kde/desktop_package_tool_restart_automatically-kde.json

@@ -0,0 +1,15 @@
+{
+  "area": [
+    {
+      "xpos": 322,
+      "ypos": 684,
+      "width": 136,
+      "height": 25,
+      "type": "match"
+    }
+  ],
+  "properties": [],
+  "tags": [
+    "desktop_package_tool_restart_automatically"
+  ]
+}

needles/kde/desktop_package_tool_upgrade_system-kde.json

@@ -0,0 +1,15 @@
+{
+    "area": [
+        {
+            "height": 20,
+            "type": "match",
+            "width": 59,
+            "xpos": 785,
+            "ypos": 90
+        }
+    ],
+    "properties": [],
+    "tags": [
+        "desktop_package_tool_upgrade_system"
+    ]
+}

templates.fif.json

@@ -3068,7 +3068,6 @@
                 "ROOT_PASSWORD": "weakpassword",
                 "TEST_TARGET": "COMPOSE",
                 "UPGRADE": "1",
-                "GUI_UPGRADE": "1",
                 "USER_LOGIN": "test",
                 "USER_PASSWORD": "weakpassword"
             }
@@ -3091,6 +3090,40 @@
                 "USER_PASSWORD": "weakpassword"
             }
         },
+        "upgrade_desktop_graphical": {
+            "profiles": {
+                "fedora-universal-x86_64-*-64bit": 30,
+                "fedora-universal-aarch64-*-aarch64": 40
+            },
+            "settings": {
+                "BOOTFROM": "c",
+                "DESKTOP": "gnome",
+                "HDD_1": "disk_f%UP1REL%_desktop_5_%ARCH%.qcow2",
+                "ROOT_PASSWORD": "weakpassword",
+                "TEST_TARGET": "COMPOSE",
+                "UPGRADE": "1",
+                "GUI_UPGRADE": "1",
+                "USER_LOGIN": "test",
+                "USER_PASSWORD": "weakpassword"
+            }
+        },
+        "upgrade_2_desktop_graphical": {
+            "profiles": {
+                "fedora-universal-x86_64-*-64bit": 30,
+                "fedora-universal-aarch64-*-aarch64": 40
+            },
+            "settings": {
+                "BOOTFROM": "c",
+                "DESKTOP": "gnome",
+                "HDD_1": "disk_f%UP2REL%_desktop_5_%ARCH%.qcow2",
+                "ROOT_PASSWORD": "weakpassword",
+                "TEST_TARGET": "COMPOSE",
+                "UPGRADE": "1",
+                "GUI_UPGRADE": "1",
+                "USER_LOGIN": "test",
+                "USER_PASSWORD": "weakpassword"
+            }
+        },
         "upgrade_kde": {
             "profiles": {
                 "fedora-universal-x86_64-*-64bit": 30
@@ -3106,6 +3139,22 @@
                 "USER_PASSWORD": "weakpassword"
             }
         },
+        "upgrade_kde_graphical": {
+            "profiles": {
+                "fedora-universal-x86_64-*-64bit": 30
+            },
+            "settings": {
+                "+HDD_1": "disk_f%UP1REL%_kde_5_x86_64.qcow2",
+                "BOOTFROM": "c",
+                "DESKTOP": "kde",
+                "ROOT_PASSWORD": "weakpassword",
+                "TEST_TARGET": "COMPOSE",
+                "UPGRADE": "1",
+                "GUI_UPGRADE": "1",
+                "USER_LOGIN": "test",
+                "USER_PASSWORD": "weakpassword"
+            }
+        },
         "upgrade_minimal": {
             "profiles": {
                 "fedora-universal-aarch64-*-aarch64": 30,

tests/_graphical_wait_login.pm

@@ -92,8 +92,11 @@ sub run {
         # root user now, for subsequent tests to work
         select_console "tty3-console";
         console_login(user => get_var("USER_LOGIN", "test"), password => get_var("USER_PASSWORD", "weakpassword"));
+        wait_still_screen 3;
         type_string "sudo su\n";
+        wait_still_screen 3;
         type_string "$password\n";
+        wait_still_screen 3;
         my $root_password = get_var("ROOT_PASSWORD") || "weakpassword";
         assert_script_run "echo 'root:$root_password' | chpasswd";
         desktop_vt;

tests/graphical_upgrade_prerequisites.pm

@@ -0,0 +1,132 @@
+use base "installedtest";
+use strict;
+use testapi;
+use utils;
+
+sub run {
+    my $self = shift;
+    my $release = get_release_number;
+    my $rawrel = get_var("RAWREL");
+    my $curr = get_var("TEST") =~ "upgrade_2" ? get_var("UP2REL") : get_var("UP1REL");
+    my $desktop = get_var("DESKTOP");
+    my $user = get_var("USER_LOGIN", "test");
+    my $pword = get_var("USER_PASSWORD", "weakpassword");
+
+    # As this is designed, the 'upgrade_preinstall' test script
+    # runs before this one and ends on the root console. Therefore,
+    # we can assume that we are still on that root console, so
+    # we can safely use script assertions.
+
+    # Make serial console writable for everyone, so that we
+    # can use script assertions for further commands as well.
+    assert_script_run("chmod 666 /dev/${serialdev}");
+
+    if ($desktop eq "gnome") {
+        # Install Gnome specific packages
+        assert_script_run("dnf install -y jq dbus-x11");
+        # FIXME on Fedora 40, update gnome-software from updates-testing
+        # to ensure we have the fix for
+        # https://gitlab.gnome.org/GNOME/gnome-software/-/issues/2514
+        # drop this when the update is stable
+        if ($curr eq "40") {
+            assert_script_run("dnf -y --enablerepo=updates-testing update gnome-software", 180);
+        }
+        # Leave the CLI and come back to the login screen.
+        desktop_vt();
+        # Log onto the graphical session
+        send_key("ret");
+        wait_still_screen(1);
+        type_very_safely("$pword\n");
+        handle_welcome_screen();
+        check_desktop();
+        # After the login, let us wait that everything settles
+        # and that we are not too quick on the system.
+        wait_still_screen(5);
+        # According to the ticket, the 'fedora.json' file
+        # which lists the available versions will be created
+        # after the Software starts. Let's start it then on Gnome.
+        menu_launch_type("software");
+        # When Software is started for the first time, it asks whether
+        # a user wants to use Third Party software. We want to Ignore
+        # this and proceed, so if we see that we click on Ignore.
+        if (check_screen("gnome_software_ignore", timeout => 60)) {
+            click_lastmatch();
+        }
+        # Wait a couple of second, just in case the file needs a little
+        # longer to be created.
+        sleep(10);
+        # Close Software
+        send_key("alt-f4");
+    }
+
+    # Switch back to the CLI for further settings.
+    $self->root_console(tty => 3);
+
+    # For Gnome desktop only.
+    if ($desktop eq "gnome") {
+        # FIXME workaround for https://bugzilla.redhat.com/show_bug.cgi?id=2268590
+        # upgrade fails if modular repo definitions are present
+        # remove when that is fixed
+        unless (script_run "ls /etc/yum.repos.d/*modul*") {
+            record_soft_failure "Modular repos present! Upgrade will fail, so removing";
+            assert_script_run "rm -f /etc/yum.repos.d/*modul*";
+        }
+        # TEST TEST FOR DEBUGGING ONLY
+        assert_script_run 'sed -i -e "s,libexec/packagekitd,libexec/packagekitd --verbose,g" /usr/lib/systemd/system/packagekit.service';
+        # END DEBUGGING
+        # Switch to a user account
+        enter_cmd("su -l $user");
+        # Navigate to the version file directory
+        assert_script_run("cd ~/.cache/gnome-software/fedora-pkgdb-collections");
+        # Replace the word 'devel' with the Rawhide version number
+        # so Rawhide behaves like other releases, if we're upgrading
+        # to Rawhide
+        if ($release eq $rawrel) {
+            assert_script_run("jq '(.collections |= map(if .version == \"devel\" then .koji_name = \"f$rawrel\" | .version = \"$rawrel\" else . end))' fedora.json > fedora-updated.json");
+            assert_script_run("mv fedora-updated.json fedora.json");
+        }
+        # Now make sure the versions we're trying to upgrade from and to
+        # are both 'active'
+        assert_script_run("jq '(.collections |= map(if .version == \"$release\" or .version == \"$curr\" then .status = \"Active\" else . end))' fedora.json > fedora-updated.json");
+        assert_script_run("mv fedora-updated.json fedora.json");
+        # upload the modified file for debugging
+        upload_logs("fedora.json", failok => 1);
+
+        # Disable blanking the screen on inactivity, because if the screen gets switched off
+        # we will have no way to make it active again.
+        assert_script_run("dbus-launch --exit-with-session gsettings set org.gnome.desktop.screensaver idle-activation-enabled false");
+        assert_script_run("dbus-launch --exit-with-session gsettings set org.gnome.desktop.screensaver lock-enabled false");
+        assert_script_run("dbus-launch --exit-with-session gsettings set org.gnome.desktop.lockdown disable-lock-screen true");
+        # Logout the regular user
+        enter_cmd("exit");
+    }
+    # For KDE desktop only.
+    elsif ($desktop eq "kde") {
+        # Replace "rawhide" with the Rawhide version number if we're
+        # upgrading to Rawhide
+        assert_script_run("sed -i 's,rawhide,$rawrel,g' /usr/share/metainfo/org.fedoraproject.fedora.metainfo.xml") if ($release eq $rawrel);
+        # Now mark the release we want to upgrade to as stable
+        assert_script_run("sed -i 's,version=\"$release\" type=\".*\" date=,version=\"$release\" type=\"stable\" date=,g' /usr/share/metainfo/org.fedoraproject.fedora.metainfo.xml");
+        # Upload the modified file for debugging
+        upload_logs("/usr/share/metainfo/org.fedoraproject.fedora.metainfo.xml", failok => 1);
+        # Switch to the regular user
+        enter_cmd("su -l $user");
+        # Wipe last update notification time
+        assert_script_run("kwriteconfig5 --file PlasmaDiscoverUpdates --group Global --key LastNotificationTime --delete");
+        # Disable the screen locker
+        assert_script_run("kwriteconfig5 --file kscreenlockerrc --group Daemon --key Autolock false qdbus org.freedesktop.ScreenSaver /ScreenSaver configure");
+        # Exit regular user
+        enter_cmd("exit");
+    }
+
+    # Reboot system to load changes
+    enter_cmd("reboot");
+}
+
+sub test_flags {
+    return {fatal => 1};
+}
+
+1;
+
+# vim: set sw=4 et:

tests/graphical_upgrade_run.pm

@@ -0,0 +1,102 @@
+use base "installedtest";
+use strict;
+use testapi;
+use utils;
+
+# This method does the basic login to a DE,
+# it can distinguish between Gnome and KDE.
+# We are using it to make the code a little
+# lighter.
+sub session_login {
+    my $desktop = shift;
+    my $pword = get_var("USER_PASSWORD") // "weakpassword";
+    if ($desktop eq "gnome") {
+        # For Gnome, we need to press Enter first
+        # to show the password field, then we type
+        # in the password.
+        send_key("ret");
+        wait_still_screen(1);
+    }
+    # For both DEs
+    type_very_safely("$pword\n");
+}
+
+sub run {
+    my $self = shift;
+    my $rawrel = get_var("RAWREL");
+    my $user = get_var("USER_LOGIN") // "test";
+    my $pword = get_var("USER_PASSWORD") // "weakpassword";
+    my $desktop = get_var("DESKTOP");
+
+    # The previous test, 'graphical_upgrade_prerequsites' reset
+    # the machine, so we will deal with booting it and
+    # login to the $desktop.
+    boot_to_login_screen();
+    session_login($desktop);
+    # If we are on Gnome, we have seen the welcome screen already
+    # in the previous step, so we do not want to repeat this.
+    # However, for KDE we will deal with it.
+    if ($desktop eq "kde") {
+        handle_welcome_screen();
+    }
+    # Let's check, that the desktop is shown.
+    check_desktop();
+    # TEST TEST FOR DEBUGGING ONLY
+    $self->root_console(tty => 4);
+    type_string "while true; do date -u >> /tmp/pk.log; df -h >> /tmp/pk.log; ls -l /var/lib/PackageKit >> /tmp/pk.log 2>&1; sleep 1; done\n";
+    $self->root_console(tty => 3);
+    desktop_vt;
+    # END DEBUGGING
+
+    # Start the package manager application depending
+    # on which DE we are on.
+    my $pkgmgr = "software";
+    $pkgmgr = "discover" if ($desktop eq "kde");
+    menu_launch_type($pkgmgr);
+
+    # On Gnome, the upgrade is safely visible when
+    # we visit the Update page by clicking on the
+    # Update icon. Let's click on that icon on
+    # both DEs, just to make sure.
+    assert_and_click("desktop_package_tool_update");
+
+    # Click the appropriate button to download the upgrade.
+    assert_and_click("desktop_package_tool_upgrade_system", timeout => 180);
+
+    if ($desktop eq "gnome") {
+        # Restart the computer to apply upgrades, when the download is complete.
+        # Downloading the upgrade packages may take a long time
+        # so let's check until we find it.
+        assert_and_click("desktop_package_tool_restart_upgrade", timeout => 1200);
+        assert_screen("auth_required");
+
+        # Type the password to confirm.
+        type_very_safely("$pword\n");
+
+        # Click on the 'restart and install' button
+        # to restart into the upgrade session.
+        assert_and_click("gnome_reboot_confirm");
+    }
+    elsif ($desktop eq "kde") {
+        # Click on Update all
+        assert_and_click("desktop_package_tool_update_apply", timeout => 1200);
+        # Once we click that button, we can check the checkbutton
+        # for restarting the computer automatically.
+        assert_and_click("desktop_package_tool_restart_automatically");
+        # When we see auth_required, it means the restart has been triggered
+        # and we need to authorize it
+        while (!check_screen("auth_required")) {
+            sleep 15;
+        }
+        type_very_safely("$pword\n");
+    }
+}
+
+
+sub test_flags {
+    return {fatal => 1};
+}
+
+1;
+
+# vim: set sw=4 et: