From d5a9e4777efbef8ae80084466a702f46e0c2648c Mon Sep 17 00:00:00 2001 From: Adam Williamson Date: Wed, 19 Jul 2023 12:51:31 -0700 Subject: [PATCH] Retry all pagure.io downloads This tweaks all pagure.io downloads to be retried a few times, since we seem to be getting failures quite often. We use curl for this as it has nice options for it. Signed-off-by: Adam Williamson --- lib/utils.pm | 8 ++++---- tests/_ostree_build.pm | 4 ++-- tests/mediakit_fileconflicts.pm | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/lib/utils.pm b/lib/utils.pm index d5913a08..c321cc07 100644 --- a/lib/utils.pm +++ b/lib/utils.pm @@ -916,9 +916,9 @@ sub download_modularity_tests { my ($whitelist) = @_; # we need python3-yaml for the script to run assert_script_run 'dnf -y install python3-yaml', 180; - assert_script_run 'curl -o /root/test.py https://pagure.io/fedora-qa/modularity_testing_scripts/raw/master/f/modular_functions.py'; + assert_script_run 'curl --retry-delay 10 --max-time 30 --retry 5 -o /root/test.py https://pagure.io/fedora-qa/modularity_testing_scripts/raw/master/f/modular_functions.py', timeout => 180; if ($whitelist eq 'whitelist') { - assert_script_run 'curl -o /root/whitelist https://pagure.io/fedora-qa/modularity_testing_scripts/raw/master/f/whitelist'; + assert_script_run 'curl --retry-delay 10 --max-time 30 --retry 5 -o /root/whitelist https://pagure.io/fedora-qa/modularity_testing_scripts/raw/master/f/whitelist', timeout => 180; } assert_script_run 'chmod 755 /root/test.py'; } @@ -1080,7 +1080,7 @@ sub advisory_check_nonmatching_packages { upload_logs "/tmp/installedupdatepkgs.txt", failok => 1; upload_logs "/mnt/updateiso/updatepkgs.txt", failok => 1; # download the check script and run it - assert_script_run 'curl -o updvercheck.py https://pagure.io/fedora-qa/os-autoinst-distri-fedora/raw/main/f/updvercheck.py', timeout => 120; + assert_script_run 'curl --retry-delay 10 --max-time 30 --retry 5 -o updvercheck.py https://pagure.io/fedora-qa/os-autoinst-distri-fedora/raw/main/f/updvercheck.py', timeout => 180; my $advisory = get_var("ADVISORY"); my $cmd = 'python3 ./updvercheck.py /mnt/updateiso/updatepkgs.txt /tmp/installedupdatepkgs.txt'; $cmd .= " $advisory" if ($advisory); @@ -1462,7 +1462,7 @@ sub download_testdata { assert_script_run("mkdir temp"); assert_script_run("cd temp"); # Download the compressed file with the repository content. - assert_script_run("wget https://pagure.io/fedora-qa/openqa_testdata/blob/thetree/f/repository.tar.gz", timeout => 120); + assert_script_run("curl --retry-delay 10 --max-time 120 --retry 5 -o repository.tar.gz https://pagure.io/fedora-qa/openqa_testdata/blob/thetree/f/repository.tar.gz", timeout => 600); # Untar it. assert_script_run("tar -zxvf repository.tar.gz"); # Copy out the files into the VMs directory structure. diff --git a/tests/_ostree_build.pm b/tests/_ostree_build.pm index 8f383192..e853584e 100644 --- a/tests/_ostree_build.pm +++ b/tests/_ostree_build.pm @@ -35,7 +35,7 @@ sub run { # usually a good idea for this kinda thing assert_script_run "setenforce Permissive"; # install the tools we need - assert_script_run "dnf -y install git lorax flatpak ostree rpm-ostree wget dbus-daemon moreutils", 300; + assert_script_run "dnf -y install git lorax flatpak ostree rpm-ostree dbus-daemon moreutils", 300; # now check out workstation-ostree-config assert_script_run 'git clone https://pagure.io/workstation-ostree-config.git'; assert_script_run 'pushd workstation-ostree-config'; @@ -83,7 +83,7 @@ sub run { assert_script_run 'git clone https://pagure.io/pungi-fedora.git'; assert_script_run 'cd pungi-fedora/'; assert_script_run "git checkout ${branch}"; - assert_script_run 'wget https://pagure.io/fedora-qa/os-autoinst-distri-fedora/raw/main/f/ostree-parse-pungi.py', timeout => 120; + assert_script_run 'curl --retry-delay 10 --max-time 30 --retry 5 -o ostree-parse-pungi.py https://pagure.io/fedora-qa/os-autoinst-distri-fedora/raw/main/f/ostree-parse-pungi.py', timeout => 180; my $loraxargs = script_output "python3 ostree-parse-pungi.py $lcsubv $arch"; # this 'temporary file cleanup' thing can actually wipe bits of diff --git a/tests/mediakit_fileconflicts.pm b/tests/mediakit_fileconflicts.pm index 11679bf9..8dfe7a99 100644 --- a/tests/mediakit_fileconflicts.pm +++ b/tests/mediakit_fileconflicts.pm @@ -9,7 +9,7 @@ sub run { # mount the ISO there assert_script_run "mount /dev/cdrom /mnt/iso"; # download the check script - assert_script_run "curl -o /usr/local/bin/potential_conflict.py https://pagure.io/fedora-qa/qa-misc/raw/master/f/potential_conflict.py"; + assert_script_run "curl --retry-delay 10 --max-time 30 --retry 5 -o /usr/local/bin/potential_conflict.py https://pagure.io/fedora-qa/qa-misc/raw/master/f/potential_conflict.py", timeout => 180; # run the check assert_script_run "python3 /usr/local/bin/potential_conflict.py --repofrompath=media,/mnt/iso -r media"; }