update testing: use a concurrent script to download packages

This uses a Python script which implements concurrent downloads
(via asyncio) to download workaround and update packages and
configure the repos. This should speed up the process for large
multi-package updates.

Signed-off-by: Adam Williamson <awilliam@redhat.com>

lib/utils.pm

@@ -7,7 +7,7 @@ use Exporter;
 
 use lockapi;
 use testapi;
-our @EXPORT = qw/run_with_error_check type_safely type_very_safely desktop_vt boot_to_login_screen console_login console_switch_layout desktop_switch_layout console_loadkeys_us do_bootloader boot_decrypt check_release menu_launch_type repo_setup setup_workaround_repo disable_updates_repos cleanup_workaround_repo console_initial_setup handle_welcome_screen gnome_initial_setup anaconda_create_user check_desktop download_modularity_tests quit_firefox advisory_get_installed_packages advisory_check_nonmatching_packages start_with_launcher quit_with_shortcut disable_firefox_studies select_rescue_mode copy_devcdrom_as_isofile get_release_number check_left_bar check_top_bar check_prerelease check_version spell_version_number _assert_and_click is_branched rec_log repos_mirrorlist register_application get_registered_applications solidify_wallpaper check_and_install_git download_testdata make_serial_writable set_update_notification_timestamp/;
+our @EXPORT = qw/run_with_error_check type_safely type_very_safely desktop_vt boot_to_login_screen console_login console_switch_layout desktop_switch_layout console_loadkeys_us do_bootloader boot_decrypt check_release menu_launch_type setup_repos repo_setup get_workarounds disable_updates_repos cleanup_workaround_repo console_initial_setup handle_welcome_screen gnome_initial_setup anaconda_create_user check_desktop download_modularity_tests quit_firefox advisory_get_installed_packages advisory_check_nonmatching_packages start_with_launcher quit_with_shortcut disable_firefox_studies select_rescue_mode copy_devcdrom_as_isofile get_release_number check_left_bar check_top_bar check_prerelease check_version spell_version_number _assert_and_click is_branched rec_log repos_mirrorlist register_application get_registered_applications solidify_wallpaper check_and_install_git download_testdata make_serial_writable set_update_notification_timestamp/;
 
 
 # We introduce this global variable to hold the list of applications that have
@@ -463,63 +463,29 @@ sub repos_mirrorlist {
     assert_script_run "sed -i -e 's,metalink,mirrorlist,g' ${files}";
 }
 
-sub cleanup_workaround_repo {
+sub get_setup_repos_script {
-    # clean up the workaround repo (see next).
+    # ensure the 'setup_repos.py' downloader script is present
-    script_run "rm -rf /mnt/workarounds_repo";
+    if (script_run "ls /usr/local/bin/setup_repos.py") {
-    script_run "rm -f /etc/yum.repos.d/workarounds.repo";
+        assert_script_run 'curl --retry-delay 10 --max-time 30 --retry 5 -o /usr/local/bin/setup_repos.py https://pagure.io/fedora-qa/os-autoinst-distri-fedora/raw/concdl/f/setup_repos.py', timeout => 180;
+        assert_script_run 'chmod ugo+x /usr/local/bin/setup_repos.py';
+    }
 }
 
-sub setup_workaround_repo {
+sub get_workarounds {
-    # we periodically need to pull an update from updates-testing in
-    # to fix some bug or other. so, here's an organized way to do it.
-    # we do this here so the workaround packages are in the repo data
-    # but *not* in the package lists generated above (those should
-    # only include packages from the update under test). we'll define
-    # a hash of releases and update IDs. if no workarounds are needed
-    # for any release, the hash can be empty and this will do nothing
     my $version = shift || get_var("VERSION");
-    cleanup_workaround_repo;
-    # write a repo config file, unless this is the support_server test
-    # and it is running on a different release than the update is for
-    # (in this case we need the repo to exist but do not want to use
-    # it on the actual support_server system)
-    unless (get_var("TEST") eq "support_server" && $version ne get_var("CURRREL")) {
-        assert_script_run 'printf "[workarounds]\nname=Workarounds repo\nbaseurl=file:///mnt/workarounds_repo\nenabled=1\nmetadata_expire=1\ngpgcheck=0" > /etc/yum.repos.d/workarounds.repo';
-    }
-    assert_script_run "mkdir -p /mnt/workarounds_repo";
-    assert_script_run "pushd /mnt/workarounds_repo";
     my %workarounds = (
         "38" => [],
         "39" => [],
         "40" => [],
     );
-    # then we'll download each update for our release:
     my $advortasks = $workarounds{$version};
-    my $pkgs = "createrepo_c";
+    return @$advortasks;
-    $pkgs .= " bodhi-client koji" if @$advortasks;
+}
-    script_run "dnf -y install $pkgs", 300;
+
-    foreach my $advortask (@$advortasks) {
+sub cleanup_workaround_repo {
-        my $cmd = "bodhi updates download --updateid=$advortask";
+    # clean up the workaround repo (see next).
-        if ($advortask =~ /^\d+$/) {
+    script_run "rm -rf /mnt/workarounds_repo";
-            my $arch = get_var("ARCH");
+    script_run "rm -f /etc/yum.repos.d/workarounds.repo";
-            $cmd = "koji download-task --arch=$arch --arch=noarch $advortask";
-        }
-        my $count = 3;
-        my $success = 0;
-        while ($count) {
-            if (script_run $cmd, 600) {
-                $count -= 1;
-            }
-            else {
-                $count = 0;
-                $success = 1;
-            }
-        }
-        die "Workaround update download failed!" unless $success;
-    }
-    # and create repo metadata
-    assert_script_run "createrepo .";
-    assert_script_run "popd";
 }
 
 sub disable_updates_repos {
@@ -563,9 +529,9 @@ sub _repo_setup_compose {
     # }
 }
 
-sub _download_packages_pre {
+sub _prepare_update_mount {
     # create and mount the filesystem where we will store update/task packages
-    # this is separate from _download_packages as it has to happen before we
+    # this is separate from setup_repos as it has to happen before we
     # enter the toolbox container on the CANNED workflow
     assert_script_run "mkdir -p /mnt/update_repo";
     # if NUMDISKS is above 1, assume we want to put the update repo on
@@ -582,13 +548,41 @@ sub _download_packages_pre {
     assert_script_run "cd /mnt/update_repo";
 }
 
-sub _download_packages {
+sub setup_repos {
-    # actually do the work of downloading packages and creating a repoistory,
+    # setup workarounds (if necessary) and updates or tag repositories,
-    # for update and task cases. a repository is needed for various reasons:
+    # using the setup_repos.py script. It's necessary to set up repos
-    # to ensure later package operations use the update packages, and for
+    # (rather than just downloading the RPMs and doing a one-time update)
-    # use when creating deliverables in the tests that do that
+    # for various reasons: to ensure later package operations use the
+    # update packages, and for use when creating deliverables in the
+    # tests that do that. Has a 'workarounds only' mode for
+    # upgrade_preinstall to use (in case we need workarounds for the
+    # pre-upgrade environment)
+    my %args = (
+        # workarounds only
+        waonly => 0,
+        # release to get workarounds for
+        version => get_var("VERSION"),
+        # whether to write repo configs
+        configs => 1,
+        @_
+    );
     my $arch = get_var("ARCH");
-    script_run "dnf -y install createrepo_c koji", 300;
+    my $tag = get_var("TAG");
+    # write the tag repo config if appropriate
+    assert_script_run 'printf "[openqa-testtag]\nname=openqa-testtag\nbaseurl=https://kojipkgs.fedoraproject.org/repos/' . "$tag/latest/$arch" . '/\ncost=2000\nenabled=1\ngpgcheck=0\n" > /etc/yum.repos.d/openqa-testtag.repo' if ($tag && !$args{waonly});
+    my @was = get_workarounds($args{version});
+    # bail if there are no workarounds:
+    # * if we're in workarounds-only mode
+    # * if we're testing a side tag (so no packages to dl)
+    if ($args{waonly} || $tag) {
+        return unless (@was);
+    }
+    # if we got this far, we're definitely downloading *something*
+    script_run "dnf -y install createrepo_c bodhi-client koji", 300;
+    get_setup_repos_script;
+    my $wastring = join(',', @was);
+    my $udstring;
+    # work out the list of update/task NVRs to test
     if (get_var("ADVISORY_NVRS") || get_var("ADVISORY_NVRS_1")) {
         # regular update case
         # old style single ADVISORY_NVRS var
@@ -606,53 +600,28 @@ sub _download_packages {
                 }
             }
         }
-        foreach my $nvr (@nvrs) {
+        $udstring = join(',', @nvrs);
-            my $kojitime = 600;
-            # texlive has a ridiculous number of subpackages
-            $kojitime = 1500 if ((rindex $nvr, "texlive", 0) == 0);
-            if (script_run "koji download-build --arch=$arch --arch=noarch $nvr 2> download.log", $kojitime) {
-                # if the error was because the build has no packages
-                # for our arch, that's okay, skip it. otherwise, die
-                if (script_run "grep 'No .*available for $nvr' download.log") {
-                    die "koji download-build failed!";
-                }
-            }
-        }
     }
     elsif (get_var("KOJITASK")) {
         # Koji task case (KOJITASK will be set). If multiple tasks,
-        # they're concatenated with underscores
+        # they're concatenated with underscores, switch to commas
-        my @tasks = split(/_/, get_var("KOJITASK"));
+        $udstring =~ s/_/,/;
-        foreach my $task (@tasks) {
-            assert_script_run "koji download-task --arch=$arch --arch=noarch $task", 600;
-        }
     }
     else {
-        die "Neither ADVISORY_NVRS nor KOJITASK nor TAG set! Don't know what to do";
+        die "Neither ADVISORY_NVRS nor KOJITASK set! Don't know what to do";
     }
-
+    my $cmd = "/usr/local/bin/setup_repos.py";
-    if (script_run 'ls *.rpm') {
+    # don't download updates if we're in workarounds-only mode or testing a tag
-        # we didn't actually download any packages (as they are all
+    $cmd .= " -u $udstring" unless ($args{waonly} || $tag);
-        # for an arch we don't test), so write dummy files
+    $cmd .= " -w $wastring" if (@was);
-        assert_script_run 'touch /mnt/updatepkgnames.txt /mnt/updatepkgs.txt';
+    # write repo config files if asked
+    $cmd .= " -c" if ($args{configs});
+    $cmd .= " $arch";
+    assert_script_run $cmd, 600;
+    unless ($args{waonly} || $tag) {
+        upload_logs "/mnt/updatepkgnames.txt";
+        upload_logs "/mnt/updatepkgs.txt";
     }
-    else {
-        # log the exact packages in the update at test time, with their
-        # source packages and epochs. we use /mnt as the path for this
-        # and similar files because, on ostree-based installs where we
-        # have to use a toolbox container for part of this, it's common
-        # to the host system and container
-        assert_script_run 'rpm -qp *.rpm --qf "%{SOURCERPM} %{NAME} %{EPOCHNUM} %{VERSION} %{RELEASE}\n" | sort -u > /mnt/updatepkgs.txt';
-        # also log just the binary package names: this is so we can check
-        # later whether any package from the update *should* have been
-        # installed, but was not
-        assert_script_run 'rpm -qp *.rpm --qf "%{NAME} " > /mnt/updatepkgnames.txt';
-    }
-    upload_logs "/mnt/updatepkgnames.txt";
-    upload_logs "/mnt/updatepkgs.txt";
-
-    # create the repo metadata
-    assert_script_run "createrepo .", timeout => 180;
 }
 
 sub _repo_setup_updates {
@@ -670,7 +639,7 @@ sub _repo_setup_updates {
 
     # prepare the directory the packages will be downloaded to, unless we're
     # testing a side tag
-    _download_packages_pre() unless ($tag);
+    _prepare_update_mount() unless ($tag);
 
     # on CANNED, we need to enter the toolbox at this point
     if (get_var("CANNED")) {
@@ -688,25 +657,22 @@ sub _repo_setup_updates {
     if (get_var("VERSION") eq get_var("RAWREL") && get_var("TEST") ne "support_server") {
         assert_script_run 'printf "[koji-rawhide]\nname=koji-rawhide\nbaseurl=https://kojipkgs.fedoraproject.org/repos/rawhide/latest/' . $arch . '/\ncost=2000\nenabled=1\ngpgcheck=0\n" > /etc/yum.repos.d/koji-rawhide.repo';
     }
-    # set up the workaround repo
-    setup_workaround_repo;
     if (get_var("CANNED")) {
         # install and use en_US.UTF-8 locale for consistent sort
         # ordering
         assert_script_run "dnf -y install glibc-langpack-en", 300;
         assert_script_run "export LC_ALL=en_US.UTF-8";
     }
-
+    # set up workarounds and updates repos (if needed)
-    # download the packages, unless we're testing a side tag
+    if (get_var("TEST") eq "support_server" && $version ne get_var("CURRREL")) {
-    _download_packages unless ($tag);
+        # don't write repo configs if this is the support_server test
-
+        # and it is running on a different release than the update is for
-    # write a repo config file, unless this is the support_server test
+        # (in this case we need the repo to exist but do not want to use
-    # and it is running on a different release than the update is for
+        # it on the actual support_server system)
-    # (in this case we need the repo to exist but do not want to use
+        setup_repos(configs => 0);
-    # it on the actual support_server system)
+    }
-    unless (get_var("TEST") eq "support_server" && $version ne get_var("CURRREL")) {
+    else {
-        assert_script_run 'printf "[advisory]\nname=Advisory repo\nbaseurl=file:///mnt/update_repo\nenabled=1\nmetadata_expire=3600\ngpgcheck=0" > /etc/yum.repos.d/advisory.repo' unless ($tag);
+        setup_repos(configs => 1);
-        assert_script_run 'printf "[openqa-testtag]\nname=openqa-testtag\nbaseurl=https://kojipkgs.fedoraproject.org/repos/' . "$tag/latest/$arch" . '/\ncost=2000\nenabled=1\ngpgcheck=0\n" > /etc/yum.repos.d/openqa-testtag.repo' if ($tag);
         # run an update now, except for upgrade or install tests,
         # where the updated packages should have been installed
         # already and we want to fail if they weren't, or CANNED
@@ -1102,9 +1068,9 @@ sub download_modularity_tests {
     my ($whitelist) = @_;
     # we need python3-yaml for the script to run
     assert_script_run 'dnf -y install python3-yaml', 180;
-    assert_script_run 'curl --verbose --retry-delay 10 --max-time 30 --retry 5 -o /root/test.py https://pagure.io/fedora-qa/modularity_testing_scripts/raw/master/f/modular_functions.py', timeout => 180;
+    assert_script_run 'curl --retry-delay 10 --max-time 30 --retry 5 -o /root/test.py https://pagure.io/fedora-qa/modularity_testing_scripts/raw/master/f/modular_functions.py', timeout => 180;
     if ($whitelist eq 'whitelist') {
-        assert_script_run 'curl --verbose --retry-delay 10 --max-time 30 --retry 5 -o /root/whitelist https://pagure.io/fedora-qa/modularity_testing_scripts/raw/master/f/whitelist', timeout => 180;
+        assert_script_run 'curl --retry-delay 10 --max-time 30 --retry 5 -o /root/whitelist https://pagure.io/fedora-qa/modularity_testing_scripts/raw/master/f/whitelist', timeout => 180;
     }
     assert_script_run 'chmod 755 /root/test.py';
 }
@@ -1260,7 +1226,7 @@ sub advisory_check_nonmatching_packages {
     upload_logs "/tmp/installedupdatepkgs.txt", failok => 1;
     upload_logs "/mnt/updatepkgs.txt", failok => 1;
     # download the check script and run it
-    assert_script_run 'curl --verbose --retry-delay 10 --max-time 30 --retry 5 -o updvercheck.py https://pagure.io/fedora-qa/os-autoinst-distri-fedora/raw/main/f/updvercheck.py', timeout => 180;
+    assert_script_run 'curl --retry-delay 10 --max-time 30 --retry 5 -o updvercheck.py https://pagure.io/fedora-qa/os-autoinst-distri-fedora/raw/main/f/updvercheck.py', timeout => 180;
     my $advisory = get_var("ADVISORY");
     my $cmd = 'python3 ./updvercheck.py /mnt/updatepkgs.txt /tmp/installedupdatepkgs.txt';
     $cmd .= " $advisory" if ($advisory);
@@ -1647,7 +1613,7 @@ sub download_testdata {
     assert_script_run("mkdir temp");
     assert_script_run("cd temp");
     # Download the compressed file with the repository content.
-    assert_script_run("curl --verbose --retry-delay 10 --max-time 120 --retry 5 -o repository.tar.gz https://pagure.io/fedora-qa/openqa_testdata/blob/thetree/f/repository.tar.gz", timeout => 600);
+    assert_script_run("curl --retry-delay 10 --max-time 120 --retry 5 -o repository.tar.gz https://pagure.io/fedora-qa/openqa_testdata/blob/thetree/f/repository.tar.gz", timeout => 600);
     # Untar it.
     assert_script_run("tar -zxvf repository.tar.gz");
     # Copy out the files into the VMs directory structure.

setup_repos.py

@@ -0,0 +1,217 @@
+#!/usr/bin/python3
+
+# Copyright Red Hat
+#
+# This file is part of os-autoinst-distri-fedora.
+#
+# This file is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# Author: Adam Williamson <awilliam@redhat.com>
+
+"""
+Package download and repository setup script for openQA update tests.
+This script uses asyncio to download the packages to be tested, and
+any 'workaround' packages, concurrently, and create repositories and
+repository configuration files for them. This is work that used to be
+done in-line in the test scripts, but doing it that way is slow for
+large multi-package updates.
+"""
+
+import argparse
+import asyncio
+import glob
+import os
+import pathlib
+import shutil
+import subprocess
+import sys
+
+# these are variables to make testing this script easier...change them
+# to /tmp for testing
+WORKAROUNDS_DIR = "/mnt/workarounds_repo"
+UPDATES_DIR = "/mnt/update_repo"
+UPDATES_FILE_PATH = "/mnt"
+
+
+class DownloadError(Exception):
+    """Exception raised when package download fails."""
+    pass
+
+
+# thanks, https://stackoverflow.com/questions/63782892
+async def run_command(*args, **kwargs):
+    """
+    Run a command with subprocess such that we can run multiple
+    concurrently.
+    """
+    # Create subprocess
+    process = await asyncio.create_subprocess_exec(
+        *args,
+        # stdout must a pipe to be accessible as process.stdout
+        stderr=asyncio.subprocess.PIPE,
+        stdout=asyncio.subprocess.PIPE,
+        **kwargs,
+    )
+    # Wait for the subprocess to finish
+    stdout, stderr = await process.communicate()
+    # Return retcode, stdout and stderr
+    return (process.returncode, stdout.decode().strip(), stderr.decode().strip())
+
+
+async def download_item(item, arch, targetdir):
+    """
+    Download something - a build or task (with koji) or an update
+    (with bodhi).
+    """
+    print(f"Downloading item {item}")
+    if item.isdigit():
+        # this will be a task ID
+        cmd = ("koji", "download-task", f"--arch={arch}", "--arch=noarch", item)
+    elif item.startswith("FEDORA-"):
+        # this is a Bodhi update ID
+        cmd = ("bodhi", "updates", "download", "--arch", arch, "--updateid", item)
+    else:
+        # assume it's an NVR
+        cmd = ("koji", "download-build", f"--arch={arch}", "--arch=noarch", item)
+    # do the download and check for failure
+    (retcode, _, stderr) = await run_command(*cmd, cwd=targetdir)
+    if retcode:
+        # "No .*available for {nvr}" indicates there are no
+        # packages for this arch in the build
+        if not f"available for {item}" in stderr:
+            return item
+    return False
+
+
+async def create_workarounds_repo(workarounds, arch, config):
+    """Set up the workarounds repository."""
+    shutil.rmtree(WORKAROUNDS_DIR, ignore_errors=True)
+    os.makedirs(WORKAROUNDS_DIR)
+    rets = []
+    if workarounds:
+        for i in range(0, len(workarounds), 20):
+            tasks = [
+                asyncio.create_task(download_item(item, arch, WORKAROUNDS_DIR))
+                for item in workarounds[i : i + 20]
+            ]
+            rets.extend(await asyncio.gather(*tasks))
+    subprocess.run(["createrepo", "."], cwd=WORKAROUNDS_DIR, check=True)
+    if config:
+        with open("/etc/yum.repos.d/workarounds.repo", "w", encoding="utf-8") as repofh:
+            repofh.write(
+                "[workarounds]\nname=Workarounds repo\n"
+                "baseurl=file:///mnt/workarounds_repo\n"
+                "enabled=1\nmetadata_expire=1\ngpgcheck=0"
+            )
+    return [ret for ret in rets if ret]
+
+
+async def create_updates_repo(items, arch, config):
+    """Set up the updates/task repository."""
+    # we do not recreate the directory as the test code has to do that
+    # since it has to mount it, before we run
+    rets = []
+    for i in range(0, len(items), 20):
+        tasks = [
+            asyncio.create_task(download_item(item, arch, UPDATES_DIR))
+            for item in items[i : i + 20]
+        ]
+        rets.extend(await asyncio.gather(*tasks))
+    subprocess.run(["createrepo", "."], cwd=UPDATES_DIR, check=True)
+    if not glob.glob(f"{UPDATES_DIR}/*.rpm"):
+        pathlib.Path(f"{UPDATES_FILE_PATH}/updatepkgnames.txt").touch()
+        pathlib.Path(f"{UPDATES_FILE_PATH}/updatepkgs.txt").touch()
+    else:
+        cmd = "rpm -qp *.rpm --qf '%{SOURCERPM} %{NAME} %{EPOCHNUM} %{VERSION} %{RELEASE}\n' | "
+        cmd += f"sort -u > {UPDATES_FILE_PATH}/updatepkgs.txt"
+        subprocess.run(cmd, shell=True, check=True, cwd=UPDATES_DIR)
+        # also log just the binary package names: this is so we can check
+        # later whether any package from the update *should* have been
+        # installed, but was not
+        subprocess.run(
+            "rpm -qp *.rpm --qf '%{NAME} ' > "
+            + f"{UPDATES_FILE_PATH}/updatepkgnames.txt",
+            shell=True,
+            check=True,
+            cwd=UPDATES_DIR,
+        )
+    if config:
+        with open("/etc/yum.repos.d/advisory.repo", "w", encoding="utf-8") as repofh:
+            repofh.write(
+                "[advisory]\nname=Advisory repo\nbaseurl=file:///mnt/update_repo\n"
+                "enabled=1\nmetadata_expire=3600\ngpgcheck=0"
+            )
+    return [ret for ret in rets if ret]
+
+
+def commalist(string):
+    """Separate a string on commas."""
+    return string.split(",")
+
+
+def parse_args():
+    """Parse CLI args with argparse."""
+    parser = argparse.ArgumentParser(
+        description="Packager downloader script for openQA tests"
+    )
+    parser.add_argument("arch", help="Architecture")
+    parser.add_argument(
+        "--workarounds",
+        "-w",
+        type=commalist,
+        help="Comma-separated list of workaround packages",
+    )
+    parser.add_argument(
+        "--updates",
+        "-u",
+        type=commalist,
+        help="Comma-separated list of update/task packages",
+    )
+    parser.add_argument(
+        "--configs", "-c", action="store_true", help="Write repo config files"
+    )
+    args = parser.parse_args()
+    if not (args.workarounds or args.updates):
+        parser.error("At least one of workarounds or updates package lists is required")
+    return args
+
+
+async def main():
+    """Do the thing!"""
+    args = parse_args()
+
+    tasks = []
+    if args.workarounds:
+        tasks.append(
+            asyncio.create_task(
+                create_workarounds_repo(args.workarounds, args.arch, args.configs)
+            )
+        )
+    if args.updates:
+        tasks.append(
+            asyncio.create_task(
+                create_updates_repo(args.updates, args.arch, args.configs)
+            )
+        )
+    failed = []
+    rets = await asyncio.gather(*tasks, return_exceptions=True)
+    for ret in rets:
+        if isinstance(ret, Exception):
+            raise ret
+        failed.extend(ret)
+    if failed:
+        sys.exit(f"Download of item(s) {', '.join(failed)} failed!")
+
+
+asyncio.run(main())

tests/_installer_build.pm

@@ -25,7 +25,7 @@ sub run {
     unless ($version > $currrel) {
         $cmd .= " --isfinal --repo=/etc/yum.repos.d/fedora-updates.repo";
     }
-    $cmd .= " --repo=/etc/yum.repos.d/workarounds.repo";
+    $cmd .= " --repo=/etc/yum.repos.d/workarounds.repo" if (get_workarounds);
     $cmd .= " --repo=/etc/yum.repos.d/koji-rawhide.repo" if ($version eq $rawrel);
     $cmd .= " --repo=/etc/yum.repos.d/advisory.repo" unless (get_var("TAG"));
     $cmd .= " --repo=/etc/yum.repos.d/openqa-testtag.repo" if (get_var("TAG"));

tests/_live_build.pm

@@ -28,6 +28,7 @@ sub run {
     my $subv = get_var("SUBVARIANT");
     my $lcsubv = lc($subv);
     my $tag = get_var("TAG");
+    my $workarounds = get_workarounds;
     if (get_var("NUMDISKS") > 2) {
         # put /var/lib/mock on the third disk, so we don't run out of
         # space on the main disk. The second disk will have already
@@ -45,14 +46,14 @@ sub run {
     # make the side and workarounds repos and the serial device available inside the mock root
     assert_script_run 'echo "config_opts[\'plugin_conf\'][\'bind_mount_enable\'] = True" >> /etc/mock/openqa.cfg';
     assert_script_run 'echo "config_opts[\'plugin_conf\'][\'bind_mount_opts\'][\'dirs\'].append((\'/mnt/update_repo\', \'/mnt/update_repo\'))" >> /etc/mock/openqa.cfg' unless ($tag);
-    assert_script_run 'echo "config_opts[\'plugin_conf\'][\'bind_mount_opts\'][\'dirs\'].append((\'/mnt/workarounds_repo\', \'/mnt/workarounds_repo\'))" >> /etc/mock/openqa.cfg';
+    assert_script_run 'echo "config_opts[\'plugin_conf\'][\'bind_mount_opts\'][\'dirs\'].append((\'/mnt/workarounds_repo\', \'/mnt/workarounds_repo\'))" >> /etc/mock/openqa.cfg' if ($workarounds);
     assert_script_run 'echo "config_opts[\'plugin_conf\'][\'bind_mount_opts\'][\'dirs\'].append((\'/dev/' . $serialdev . '\', \'/dev/' . $serialdev . '\'))" >> /etc/mock/openqa.cfg';
     my $repos = 'config_opts[\'dnf.conf\'] += \"\"\"\n';
     # add the update repo or tag repo to the config
     $repos .= '[advisory]\nname=Advisory repo\nbaseurl=file:///mnt/update_repo\nenabled=1\nmetadata_expire=3600\ngpgcheck=0\n' unless ($tag);
     $repos .= '[openqa-testtag]\nname=Tag test repo\nbaseurl=https://kojipkgs.fedoraproject.org/repos/' . "${tag}/latest/${arch}" . '\nenabled=1\nmetadata_expire=3600\ngpgcheck=0\n' if ($tag);
     # and the workaround repo
-    $repos .= '\n[workarounds]\nname=Workarounds repo\nbaseurl=file:///mnt/workarounds_repo\nenabled=1\nmetadata_expire=3600\ngpgcheck=0\n';
+    $repos .= '\n[workarounds]\nname=Workarounds repo\nbaseurl=file:///mnt/workarounds_repo\nenabled=1\nmetadata_expire=3600\ngpgcheck=0\n' if ($workarounds);
     # also the buildroot repo, for Rawhide
     if ($version eq $rawrel) {
         $repos .= '\n[koji-rawhide]\nname=Buildroot repo\nbaseurl=https://kojipkgs.fedoraproject.org/repos/f' . $version . '-build/latest/\$basearch/\nenabled=1\nmetadata_expire=3600\ngpgcheck=0\nskip_if_unavailable=1\n';
@@ -71,7 +72,7 @@ sub run {
     assert_script_run 'echo "repo --name=advisory --baseurl=file:///mnt/update_repo" >> ' . $repoks unless ($tag);
     assert_script_run 'echo "repo --name=openqa-testtag --baseurl=https://kojipkgs.fedoraproject.org/repos/' . "${tag}/latest/${arch}" . '" >> ' . $repoks if ($tag);
     # and the workarounds repo
-    assert_script_run 'echo "repo --name=workarounds --baseurl=file:///mnt/workarounds_repo" >> ' . $repoks;
+    assert_script_run 'echo "repo --name=workarounds --baseurl=file:///mnt/workarounds_repo" >> ' . $repoks if ($workarounds);
     # and the buildroot repo, for Rawhide
     if ($version eq $rawrel) {
         assert_script_run 'echo "repo --name=koji-rawhide --baseurl=https://kojipkgs.fedoraproject.org/repos/f' . $version . '-build/latest/\$basearch/" >> ' . $repoks;

tests/_ostree_build.pm

@@ -24,6 +24,7 @@ sub run {
     my $subv = get_var("SUBVARIANT");
     my $lcsubv = lc($subv);
     my $tag = get_var("TAG");
+    my $workarounds = get_workarounds;
     # mount our nice big empty scratch disk as /var/tmp
     assert_script_run "rm -rf /var/tmp/*";
     assert_script_run "echo 'type=83' | sfdisk /dev/vdc";
@@ -40,13 +41,13 @@ sub run {
     assert_script_run 'pushd workstation-ostree-config';
     assert_script_run "git checkout ${branch}";
     # now copy the advisory, workaround repo and koji-rawhide config files
-    assert_script_run 'cp /etc/yum.repos.d/workarounds.repo .';
+    assert_script_run 'cp /etc/yum.repos.d/workarounds.repo .' if ($workarounds);
     assert_script_run 'cp /etc/yum.repos.d/koji-rawhide.repo .' if ($version eq $rawrel);
     assert_script_run 'cp /etc/yum.repos.d/advisory.repo .' unless ($tag);
     assert_script_run 'cp /etc/yum.repos.d/openqa-testtag.repo .' if ($tag);
     # and add them to the config file
     my $repl = 'repos:';
-    $repl .= '\n  - workarounds';
+    $repl .= '\n  - workarounds' if ($workarounds);
     $repl .= '\n  - koji-rawhide' if ($version eq $rawrel);
     $repl .= '\n  - advisory' unless ($tag);
     $repl .= '\n  - openqa-testtag' if ($tag);
@@ -98,7 +99,7 @@ sub run {
     unless ($version > $currrel) {
         $cmd .= " --isfinal --repo=/etc/yum.repos.d/fedora-updates.repo";
     }
-    $cmd .= " --repo=/etc/yum.repos.d/workarounds.repo";
+    $cmd .= " --repo=/etc/yum.repos.d/workarounds.repo" if ($workarounds);
     $cmd .= " --repo=/etc/yum.repos.d/koji-rawhide.repo" if ($version eq $rawrel);
     $cmd .= " --repo=/etc/yum.repos.d/advisory.repo" unless ($tag);
     $cmd .= " --repo=/etc/yum.repos.d/openqa-testtag.repo" if ($tag);

tests/upgrade_preinstall.pm

@@ -13,7 +13,9 @@ sub run {
     if (index($testname, "upgrade_2") != -1) {
         $version = get_var("UP2REL");
     }
-    setup_workaround_repo $version;
+    # setup the workarounds repository for the pre-upgrade release,
+    # in case we need workarounds for that environment
+    setup_repos(waonly => 1, version => $version);
     # disable updates-testing, this is needed for the case of upgrade
     # from branched to rawhide to ensure we don't get packages from
     # updates-testing for anything we do between here and upgrade_run