From f5946e678c902eee684346f9dfb43a3c833093ac Mon Sep 17 00:00:00 2001 From: Adam Williamson Date: Tue, 2 Aug 2022 09:32:47 -0700 Subject: [PATCH] Make update testing more robust for very large updates openQA choked badly on https://bodhi.fedoraproject.org/updates/FEDORA-2022-6256981a71 because it's, well, huge - 87 builds including texlive, which has hundreds (thousands?) of subpackages. This exposed several frailties against such updates. First of all, we set NUMDISKS to at least 2 for *all* update tests, which should mean they all stash the RPMs from the update on a non-system disk and avoid problems with space exhaustion. After that, just extend a few timeouts in particularly fragile places, including one which is specific to texlive (as I don't know of any other source package with so many subpackages). Signed-off-by: Adam Williamson --- lib/utils.pm | 11 +++++++---- templates-updates.fif.json | 17 ++++++++++++++++- 2 files changed, 23 insertions(+), 5 deletions(-) diff --git a/lib/utils.pm b/lib/utils.pm index 518edcb0..73921ef3 100644 --- a/lib/utils.pm +++ b/lib/utils.pm @@ -589,7 +589,10 @@ sub _repo_setup_updates { if (get_var("ADVISORY_NVRS")) { # regular update case foreach my $nvr (split(/ /, get_var("ADVISORY_NVRS"))) { - if (script_run "koji download-build --arch=" . get_var("ARCH") . " --arch=noarch $nvr 2> download.log", 600) { + my $kojitime = 600; + # texlive has a ridiculous number of subpackages + $kojitime = 1500 if ((rindex $nvr, "texlive", 0) == 0); + if (script_run "koji download-build --arch=" . get_var("ARCH") . " --arch=noarch $nvr 2> download.log", $kojitime) { # if the error was because the build has no packages # for our arch, that's okay, skip it. otherwise, die if (script_run "grep 'No .*available for $nvr' download.log") { @@ -617,7 +620,7 @@ sub _repo_setup_updates { upload_logs "/var/log/updatepkgnames.txt"; # create the repo metadata - assert_script_run "createrepo ."; + assert_script_run "createrepo .", timeout => 180; # write a repo config file, unless this is the support_server test # and it is running on a different release than the update is for # (in this case we need the repo to exist but do not want to use @@ -1062,7 +1065,7 @@ sub advisory_get_installed_packages { # are currently installed. This is here so we can do it both in # _advisory_post and post_fail_hook. return unless (get_var("_ADVISORY_REPO_DONE")); - assert_script_run 'rpm -qa --qf "%{SOURCERPM} %{EPOCH} %{NAME}-%{VERSION}-%{RELEASE}\n" | sort -u > /tmp/allpkgs.txt'; + assert_script_run 'rpm -qa --qf "%{SOURCERPM} %{EPOCH} %{NAME}-%{VERSION}-%{RELEASE}\n" | sort -u > /tmp/allpkgs.txt', timeout => 90; # this finds lines which appear in both files # http://www.unix.com/unix-for-dummies-questions-and-answers/34549-find-matching-lines-between-2-files.html if (script_run 'comm -12 /tmp/allpkgs.txt /var/log/updatepkgs.txt > /var/log/testedpkgs.txt') { @@ -1105,7 +1108,7 @@ sub advisory_check_nonmatching_packages { # (we need four to reach bash, and half of them get eaten by perl or # something along the way). Yes, it only works with *single* quotes. Yes, # I hate escaping - script_run 'for pkg in $(cat /var/log/updatepkgnames.txt); do rpm -q $pkg && rpm -q $pkg --last | head -1 | cut -d" " -f1 | sed -e \'s,\^,\\\\\\\\^,g\' | xargs rpm -q --qf "%{SOURCERPM} %{EPOCH} %{NAME}-%{VERSION}-%{RELEASE}\n" >> /tmp/installedupdatepkgs.txt; done'; + script_run 'for pkg in $(cat /var/log/updatepkgnames.txt); do rpm -q $pkg && rpm -q $pkg --last | head -1 | cut -d" " -f1 | sed -e \'s,\^,\\\\\\\\^,g\' | xargs rpm -q --qf "%{SOURCERPM} %{EPOCH} %{NAME}-%{VERSION}-%{RELEASE}\n" >> /tmp/installedupdatepkgs.txt; done', timeout => 180; script_run 'sort -u -o /tmp/installedupdatepkgs.txt /tmp/installedupdatepkgs.txt'; # for debugging, may as well always upload these, can't hurt anything upload_logs "/tmp/installedupdatepkgs.txt", failok => 1; diff --git a/templates-updates.fif.json b/templates-updates.fif.json index 74681e81..e0c1f687 100644 --- a/templates-updates.fif.json +++ b/templates-updates.fif.json @@ -5,6 +5,7 @@ "distri": "fedora", "flavor": "updates-container", "settings": { + "NUMDISKS": "2", "RETRY": "1" }, "version": "*" @@ -14,6 +15,7 @@ "distri": "fedora", "flavor": "updates-container", "settings": { + "NUMDISKS": "2", "RETRY": "1" }, "version": "*" @@ -23,6 +25,7 @@ "distri": "fedora", "flavor": "updates-container", "settings": { + "NUMDISKS": "2", "RETRY": "1" }, "version": "*" @@ -32,6 +35,7 @@ "distri": "fedora", "flavor": "updates-everything-boot-iso", "settings": { + "NUMDISKS": "2", "RETRY": "1" }, "version": "*" @@ -42,6 +46,7 @@ "flavor": "updates-kde", "settings": { "DESKTOP": "kde", + "NUMDISKS": "2", "RETRY": "1" }, "version": "*" @@ -51,6 +56,7 @@ "distri": "fedora", "flavor": "updates-server", "settings": { + "NUMDISKS": "2", "RETRY": "1" }, "version": "*" @@ -60,6 +66,7 @@ "distri": "fedora", "flavor": "updates-server", "settings": { + "NUMDISKS": "2", "RETRY": "1" }, "version": "*" @@ -69,6 +76,7 @@ "distri": "fedora", "flavor": "updates-server-upgrade", "settings": { + "NUMDISKS": "2", "RETRY": "1" }, "version": "*" @@ -78,6 +86,7 @@ "distri": "fedora", "flavor": "updates-server-upgrade", "settings": { + "NUMDISKS": "2", "RETRY": "1" }, "version": "*" @@ -87,6 +96,7 @@ "distri": "fedora", "flavor": "updates-server-upgrade", "settings": { + "NUMDISKS": "2", "RETRY": "1" }, "version": "*" @@ -96,6 +106,7 @@ "distri": "fedora", "flavor": "updates-server", "settings": { + "NUMDISKS": "2", "RETRY": "1" }, "version": "*" @@ -108,6 +119,7 @@ "DESKTOP": "gnome", "HDDSIZEGB": "15", "LIVE": "1", + "NUMDISKS": "2", "PACKAGE_SET": "default", "RETRY": "1" }, @@ -121,6 +133,7 @@ "DESKTOP": "kde", "HDDSIZEGB": "15", "LIVE": "1", + "NUMDISKS": "2", "PACKAGE_SET": "default", "RETRY": "1" }, @@ -132,6 +145,7 @@ "flavor": "updates-workstation-upgrade", "settings": { "DESKTOP": "gnome", + "NUMDISKS": "2", "RETRY": "1" }, "version": "*" @@ -142,6 +156,7 @@ "flavor": "updates-workstation", "settings": { "DESKTOP": "gnome", + "NUMDISKS": "2", "RETRY": "1" }, "version": "*" @@ -370,7 +385,7 @@ "GRUB_POSTINSTALL": "selinux=0", "HDD_1": "disk_f%VERSION%_minimal_3_%ARCH%.qcow2", "HDDSIZEGB_3": "25", - "NUMDISKS": "3", + "+NUMDISKS": "3", "POSTINSTALL": "_live_build", "ROOT_PASSWORD": "weakpassword", "USER_LOGIN": "false"